From e8e0c0911c07d9185139eecd5a683fb31d6d9991 Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 20 Feb 2024 14:57:29 +0200 Subject: [PATCH 1/2] Add code part. --- .../deploy/arbitrum-sepolia-missing.service | 11 ++ .../deploy/arbitrum-sepolia-missing.timer | 9 ++ .../arbitrum-sepolia-moonworm-crawler.service | 17 +++ .../arbitrum-sepolia-synchronize.service | 17 +++ crawlers/deploy/deploy.bash | 34 ++++++ crawlers/deploy/monitoring-crawlers.service | 2 +- crawlers/mooncrawl/mooncrawl/blockchain.py | 21 +++- .../moonworm_crawler/continuous_crawler.py | 15 +-- .../mooncrawl/moonworm_crawler/crawler.py | 9 +- .../moonworm_crawler/function_call_crawler.py | 2 + .../moonworm_crawler/historical_crawler.py | 2 + crawlers/mooncrawl/mooncrawl/settings.py | 8 ++ .../mooncrawl/stats_worker/dashboard.py | 22 ++-- crawlers/mooncrawl/mooncrawl/version.py | 2 +- crawlers/mooncrawl/sample.env | 1 + crawlers/mooncrawl/setup.py | 2 +- moonstreamdb/alembic/env.py | 6 ++ moonstreamdb/moonstreamdb/blockchain.py | 22 +++- moonstreamdb/moonstreamdb/models.py | 100 ++++++++++++++++++ moonstreamdb/moonstreamdb/networks.py | 10 ++ moonstreamdb/moonstreamdb/version.py | 2 +- 21 files changed, 286 insertions(+), 28 deletions(-) create mode 100644 crawlers/deploy/arbitrum-sepolia-missing.service create mode 100644 crawlers/deploy/arbitrum-sepolia-missing.timer create mode 100644 crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service create mode 100644 crawlers/deploy/arbitrum-sepolia-synchronize.service diff --git a/crawlers/deploy/arbitrum-sepolia-missing.service b/crawlers/deploy/arbitrum-sepolia-missing.service new file mode 100644 index 00000000..da67801a --- /dev/null +++ b/crawlers/deploy/arbitrum-sepolia-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Arbitrum Sepolia database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_sepolia -n +CPUWeight=50 +SyslogIdentifier=arbitrum-sepolia-missing \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-sepolia-missing.timer b/crawlers/deploy/arbitrum-sepolia-missing.timer new file mode 100644 index 00000000..4f539ffa --- /dev/null +++ b/crawlers/deploy/arbitrum-sepolia-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Arbitrum Sepolia database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service b/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service new file mode 100644 index 00000000..09c3b447 --- /dev/null +++ b/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum Sepolia moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=arbitrum-sepolia-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-sepolia-synchronize.service b/crawlers/deploy/arbitrum-sepolia-synchronize.service new file mode 100644 index 00000000..512fe082 --- /dev/null +++ b/crawlers/deploy/arbitrum-sepolia-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum Sepolia block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=arbitrum-sepolia-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 28f323e4..deb23162 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -128,6 +128,12 @@ ARBITRUM_NOVA_MISSING_TIMER_FILE="arbitrum-nova-missing.timer" ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-nova-moonworm-crawler.service" ARBITRUM_NOVA_SYNCHRONIZE_SERVICE="arbitrum-nova-synchronize.service" +# Arbitrum Sepolia +ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE="arbitrum-sepolia-missing.service" +ARBITRUM_SEPOLIA_MISSING_TIMER_FILE="arbitrum-sepolia-missing.timer" +ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-sepolia-moonworm-crawler.service" +ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE="arbitrum-sepolia-synchronize.service" + set -eu echo @@ -616,3 +622,31 @@ chmod 644 "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" + + +# Arbitrum Sepolia +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia block with transactions syncronizer service definition with ${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia missing service and timer with: ${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}, ${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum Sepolia moonworm crawler service definition with ${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" + diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index d1ce9dac..92bb90a7 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index ce045613..a9e353aa 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -22,6 +22,7 @@ from .data import DateRange from .db import yield_db_session, yield_db_session_ctx from .settings import ( MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI, + MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI, MOONSTREAM_CRAWL_WORKERS, MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI, @@ -79,6 +80,8 @@ def connect( web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -120,9 +123,11 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj = block_model( block_number=block.number, difficulty=block.difficulty, - extra_data=None - if block.get("extraData", None) is None - else block.get("extraData").hex(), + extra_data=( + None + if block.get("extraData", None) is None + else block.get("extraData").hex() + ), gas_limit=block.gasLimit, gas_used=block.gasUsed, base_fee_per_gas=block.get("baseFeePerGas", None), @@ -164,6 +169,13 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.send_root = block.get("sendRoot", "") block_obj.mix_hash = block.get("mixHash", "") + if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + block_obj.sha3_uncles = block.get("sha3Uncles", "") + block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber")) + block_obj.send_count = hex_to_int(block.get("sendCount")) + block_obj.send_root = block.get("sendRoot", "") + block_obj.mix_hash = block.get("mixHash", "") + db_session.add(block_obj) @@ -209,6 +221,9 @@ def add_block_transactions( if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: tx_obj.y_parity = hex_to_int(tx.get("yParity")) + if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + tx_obj.y_parity = hex_to_int(tx.get("yParity")) + db_session.add(tx_obj) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 9df1c5fc..ee3d96a7 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -136,6 +136,9 @@ def continuous_crawler( network = Network.zksync_era elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: network = Network.arbitrum_nova + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + network = Network.arbitrum_sepolia + else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") @@ -259,9 +262,9 @@ def continuous_crawler( heartbeat_template["current_function_call_jobs_length"] = len( function_call_crawl_jobs ) - heartbeat_template[ - "function_call metrics" - ] = ethereum_state_provider.metrics + heartbeat_template["function_call metrics"] = ( + ethereum_state_provider.metrics + ) heartbeat( crawler_type=crawler_type, blockchain_type=blockchain_type, @@ -303,9 +306,9 @@ def continuous_crawler( ) ), ) - heartbeat_template[ - "die_reason" - ] = f"{e.__class__.__name__}: {e}\n error_summary: {error_summary}\n error_traceback: {error_traceback}" + heartbeat_template["die_reason"] = ( + f"{e.__class__.__name__}: {e}\n error_summary: {error_summary}\n error_traceback: {error_traceback}" + ) heartbeat_template["last_block"] = end_block heartbeat( crawler_type=crawler_type, diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index a6473751..e4b6c558 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -38,6 +38,7 @@ class SubscriptionTypes(Enum): ZKSYNC_ERA_TESTNET_BLOCKCHAIN = "zksync_era_testnet_smartcontract" ZKSYNC_ERA_BLOCKCHAIN = "zksync_era_smartcontract" ARBITRUM_NOVA_BLOCKCHAIN = "arbitrum_nova_smartcontract" + ARBITRUM_SEPOLIA_BLOCKCHAIN = "arbitrum_sepolia_smartcontract" def abi_input_signature(input_abi: Dict[str, Any]) -> str: @@ -148,6 +149,8 @@ def blockchain_type_to_subscription_type( return SubscriptionTypes.ZKSYNC_ERA_BLOCKCHAIN elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: return SubscriptionTypes.ARBITRUM_NOVA_BLOCKCHAIN + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + return SubscriptionTypes.ARBITRUM_SEPOLIA_BLOCKCHAIN else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") @@ -554,9 +557,9 @@ def update_job_tags( for contract_address, entries_ids in event.address_entries.items(): for entry_id in entries_ids.keys(): if entry_id in entry_tags_by_id: - event.address_entries[contract_address][ - entry_id - ] = entry_tags_by_id[entry_id] + event.address_entries[contract_address][entry_id] = ( + entry_tags_by_id[entry_id] + ) if isinstance(event, FunctionCallCrawlJob): for entry_id in event.entries_tags.keys(): diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py index a5c09f7f..55a1bc9a 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/function_call_crawler.py @@ -74,6 +74,8 @@ def function_call_crawler( network = Network.zksync_era elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: network = Network.arbitrum_nova + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + network = Network.arbitrum_sepolia else: raise ValueError(f"Unknown blockchain type: {blockchain_type}") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 2db76fd0..323031ff 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -67,6 +67,8 @@ def historical_crawler( network = Network.zksync_era elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: network = Network.arbitrum_nova + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + network = Network.arbitrum_sepolia else: raise Exception("Unsupported blockchain type provided") diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index d9e6d74e..d48628ce 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -130,6 +130,14 @@ if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "": "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set" ) +MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI = os.environ.get( + "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI", "" +) +if MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI == "": + raise Exception( + "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI env variable is not set" + ) + MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") try: diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 2ecdac64..161bd343 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -1,6 +1,7 @@ """ Generates dashboard. """ + import argparse import hashlib import json @@ -56,6 +57,7 @@ subscription_id_by_blockchain = { "zksync_era_testnet": "zksync_era_testnet_smartcontract", "zksync_era": "zksync_era_smartcontract", "arbitrum_nova": "arbitrum_nova_smartcontract", + "arbitrum_sepolia": "arbitrum_sepolia_smartcontract", } blockchain_by_subscription_id = { @@ -66,6 +68,7 @@ blockchain_by_subscription_id = { "wyrm_blockchain": "wyrm", "zksync_era_testnet_blockchain": "zksync_era_testnet", "arbitrum_nova_blockchain": "arbitrum_nova", + "arbitrum_sepolia_blockchain": "arbitrum_sepolia", "ethereum_smartcontract": "ethereum", "polygon_smartcontract": "polygon", "mumbai_smartcontract": "mumbai", @@ -74,6 +77,7 @@ blockchain_by_subscription_id = { "zksync_era_testnet_smartcontract": "zksync_era_testnet", "zksync_era_smartcontract": "zksync_era", "arbitrum_nova_smartcontract": "arbitrum_nova", + "arbitrum_sepolia_smartcontract": "arbitrum_sepolia", } @@ -931,9 +935,9 @@ def stats_generate_handler(args: argparse.Namespace): s3_subscription_data_object: Dict[str, Any] = {} - s3_subscription_data_object[ - "blocks_state" - ] = s3_data_object_for_contract["blocks_state"] + s3_subscription_data_object["blocks_state"] = ( + s3_data_object_for_contract["blocks_state"] + ) if dashboard_id in merged_external_calls: for ( @@ -952,9 +956,9 @@ def stats_generate_handler(args: argparse.Namespace): } ) - s3_subscription_data_object[ - "web3_metric" - ] = extention_data + s3_subscription_data_object["web3_metric"] = ( + extention_data + ) # list of user defined events @@ -966,9 +970,9 @@ def stats_generate_handler(args: argparse.Namespace): for event in events_list: if event in events_data: - s3_subscription_data_object["events"][ - event - ] = events_data[event] + s3_subscription_data_object["events"][event] = ( + events_data[event] + ) # list of user defined functions diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 612a217e..a8f1c39e 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.6" +MOONCRAWL_VERSION = "0.3.7" diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 072fd0b2..6f7498bf 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -29,6 +29,7 @@ export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.6", + "moonstreamdb>=0.3.7", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug", diff --git a/moonstreamdb/alembic/env.py b/moonstreamdb/alembic/env.py index 708727af..532f4763 100644 --- a/moonstreamdb/alembic/env.py +++ b/moonstreamdb/alembic/env.py @@ -49,6 +49,9 @@ from moonstreamdb.models import ( ArbitrumNovaBlock, ArbitrumNovaTransaction, ArbitrumNovaLabel, + ArbitrumSepoliaBlock, + ArbitrumSepoliaTransaction, + ArbitrumSepoliaLabel, ) @@ -78,6 +81,9 @@ def include_symbol(tablename, schema): ArbitrumNovaBlock.__tablename__, ArbitrumNovaTransaction.__tablename__, ArbitrumNovaLabel.__tablename__, + ArbitrumSepoliaBlock.__tablename__, + ArbitrumSepoliaTransaction.__tablename__, + ArbitrumSepoliaLabel.__tablename__, } diff --git a/moonstreamdb/moonstreamdb/blockchain.py b/moonstreamdb/moonstreamdb/blockchain.py index 66e8eb89..4c8f21c2 100644 --- a/moonstreamdb/moonstreamdb/blockchain.py +++ b/moonstreamdb/moonstreamdb/blockchain.py @@ -26,6 +26,9 @@ from .models import ( ArbitrumNovaBlock, ArbitrumNovaTransaction, ArbitrumNovaLabel, + ArbitrumSepoliaBlock, + ArbitrumSepoliaTransaction, + ArbitrumSepoliaLabel, ) @@ -38,6 +41,7 @@ class AvailableBlockchainType(Enum): ZKSYNC_ERA_TESTNET = "zksync_era_testnet" ZKSYNC_ERA = "zksync_era" ARBITRUM_NOVA = "arbitrum_nova" + ARBITRUM_SEPOLIA = "arbitrum_sepolia" def get_block_model( @@ -52,10 +56,11 @@ def get_block_model( ZkSyncEraTestnetBlock, ZkSyncEraBlock, ArbitrumNovaBlock, + ArbitrumSepoliaBlock, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaBlock, ArbitrumSepoliaBlock set proper blocks model. """ block_model: Type[ @@ -68,6 +73,7 @@ def get_block_model( ZkSyncEraTestnetBlock, ZkSyncEraBlock, ArbitrumNovaBlock, + ArbitrumSepoliaBlock, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -86,6 +92,8 @@ def get_block_model( block_model = ZkSyncEraBlock elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: block_model = ArbitrumNovaBlock + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + block_model = ArbitrumSepoliaBlock else: raise Exception("Unsupported blockchain type provided") @@ -104,10 +112,11 @@ def get_label_model( ZkSyncEraTestnetLabel, ZkSyncEraLabel, ArbitrumNovaLabel, + ArbitrumSepoliaLabel, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaLabel, ArbitrumSepoliaLabel set proper block label model. """ label_model: Type[ @@ -120,6 +129,7 @@ def get_label_model( ZkSyncEraTestnetLabel, ZkSyncEraLabel, ArbitrumNovaLabel, + ArbitrumSepoliaLabel, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -138,6 +148,8 @@ def get_label_model( label_model = ZkSyncEraLabel elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: label_model = ArbitrumNovaLabel + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + label_model = ArbitrumSepoliaLabel else: raise Exception("Unsupported blockchain type provided") @@ -156,10 +168,11 @@ def get_transaction_model( ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, ArbitrumNovaTransaction, + ArbitrumSepoliaTransaction, ] ]: """ - Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction + Depends on provided blockchain type: Ethereum, Polygon, Mumbai, XDai, Wyrm, ZkSyncEra, ZkSyncEraTestnet, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction set proper block transactions model. """ transaction_model: Type[ @@ -172,6 +185,7 @@ def get_transaction_model( ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, ArbitrumNovaTransaction, + ArbitrumSepoliaTransaction, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -190,6 +204,8 @@ def get_transaction_model( transaction_model = ZkSyncEraTransaction elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: transaction_model = ArbitrumNovaTransaction + elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + transaction_model = ArbitrumSepoliaTransaction else: raise Exception("Unsupported blockchain type provided") diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index 6200078f..602a3f8b 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -901,6 +901,106 @@ class ArbitrumNovaLabel(Base): # type: ignore ) +class ArbitrumSepoliaBlock(Base): # type: ignore + __tablename__ = "arbitrum_sepolia_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + sha3_uncles = Column(VARCHAR(256), nullable=True) + l1_block_number = Column(BigInteger, nullable=True) + send_count = Column(BigInteger, nullable=True) + send_root = Column(VARCHAR(256), nullable=True) + mix_hash = Column(VARCHAR(256), nullable=True) + + +class ArbitrumSepoliaTransaction(Base): # type: ignore + __tablename__ = "arbitrum_sepolia_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("arbitrum_sepolia_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + y_parity = Column(BigInteger, nullable=True) + + +class ArbitrumSepoliaLabel(Base): # type: ignore + __tablename__ = "arbitrum_sepolia_labels" + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ESDFunctionSignature(Base): # type: ignore """ Function signature from blockchain (Ethereum/Polygon) Signature Database. diff --git a/moonstreamdb/moonstreamdb/networks.py b/moonstreamdb/moonstreamdb/networks.py index ae4ee215..b293826c 100644 --- a/moonstreamdb/moonstreamdb/networks.py +++ b/moonstreamdb/moonstreamdb/networks.py @@ -27,6 +27,9 @@ from .models import ( ArbitrumNovaBlock, ArbitrumNovaTransaction, ArbitrumNovaLabel, + ArbitrumSepoliaBlock, + ArbitrumSepoliaTransaction, + ArbitrumSepoliaLabel, ) @@ -39,6 +42,7 @@ class Network(Enum): zksync_era_testnet = "zksync_era_testnet" zksync_era = "zksync_era" arbitrum_nova = "arbitrum_nova" + arbitrum_sepolia = "arbitrum_sepolia" tx_raw_types = Union[ @@ -50,6 +54,7 @@ tx_raw_types = Union[ ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, ArbitrumNovaTransaction, + ArbitrumSepoliaTransaction, ] MODELS: Dict[Network, Dict[str, Base]] = { @@ -93,4 +98,9 @@ MODELS: Dict[Network, Dict[str, Base]] = { "labels": ArbitrumNovaLabel, "transactions": ArbitrumNovaTransaction, }, + Network.arbitrum_sepolia: { + "blocks": ArbitrumSepoliaBlock, + "labels": ArbitrumSepoliaLabel, + "transactions": ArbitrumSepoliaTransaction, + }, } diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index 4229e2eb..6fcd0f3d 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.6" +MOONSTREAMDB_VERSION = "0.3.7" From 84c61dcf702cae3768546f7fe6441a05cc2d01eb Mon Sep 17 00:00:00 2001 From: Andrey Date: Tue, 20 Feb 2024 15:20:25 +0200 Subject: [PATCH 2/2] add migration --- ...247_arbitrum_sepolia_blockchain_support.py | 265 ++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 moonstreamdb/alembic/versions/b38b75b11247_arbitrum_sepolia_blockchain_support.py diff --git a/moonstreamdb/alembic/versions/b38b75b11247_arbitrum_sepolia_blockchain_support.py b/moonstreamdb/alembic/versions/b38b75b11247_arbitrum_sepolia_blockchain_support.py new file mode 100644 index 00000000..ca8cef2b --- /dev/null +++ b/moonstreamdb/alembic/versions/b38b75b11247_arbitrum_sepolia_blockchain_support.py @@ -0,0 +1,265 @@ +"""Arbitrum Sepolia blockchain support + +Revision ID: b38b75b11247 +Revises: 2391c3cc5050 +Create Date: 2024-02-20 15:16:40.393985 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "b38b75b11247" +down_revision = "2391c3cc5050" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "arbitrum_sepolia_blocks", + sa.Column("block_number", sa.BigInteger(), nullable=False), + sa.Column("difficulty", sa.BigInteger(), nullable=True), + sa.Column("extra_data", sa.VARCHAR(length=128), nullable=True), + sa.Column("gas_limit", sa.BigInteger(), nullable=True), + sa.Column("gas_used", sa.BigInteger(), nullable=True), + sa.Column("base_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column("hash", sa.VARCHAR(length=256), nullable=True), + sa.Column("logs_bloom", sa.VARCHAR(length=1024), nullable=True), + sa.Column("miner", sa.VARCHAR(length=256), nullable=True), + sa.Column("nonce", sa.VARCHAR(length=256), nullable=True), + sa.Column("parent_hash", sa.VARCHAR(length=256), nullable=True), + sa.Column("receipt_root", sa.VARCHAR(length=256), nullable=True), + sa.Column("uncles", sa.VARCHAR(length=256), nullable=True), + sa.Column("size", sa.Integer(), nullable=True), + sa.Column("state_root", sa.VARCHAR(length=256), nullable=True), + sa.Column("timestamp", sa.BigInteger(), nullable=True), + sa.Column("total_difficulty", sa.VARCHAR(length=256), nullable=True), + sa.Column("transactions_root", sa.VARCHAR(length=256), nullable=True), + sa.Column( + "indexed_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column("sha3_uncles", sa.VARCHAR(length=256), nullable=True), + sa.Column("l1_block_number", sa.BigInteger(), nullable=True), + sa.Column("send_count", sa.BigInteger(), nullable=True), + sa.Column("send_root", sa.VARCHAR(length=256), nullable=True), + sa.Column("mix_hash", sa.VARCHAR(length=256), nullable=True), + sa.PrimaryKeyConstraint( + "block_number", name=op.f("pk_arbitrum_sepolia_blocks") + ), + ) + op.create_index( + op.f("ix_arbitrum_sepolia_blocks_block_number"), + "arbitrum_sepolia_blocks", + ["block_number"], + unique=True, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_blocks_hash"), + "arbitrum_sepolia_blocks", + ["hash"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_blocks_timestamp"), + "arbitrum_sepolia_blocks", + ["timestamp"], + unique=False, + ) + op.create_table( + "arbitrum_sepolia_labels", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("label", sa.VARCHAR(length=256), nullable=False), + sa.Column("block_number", sa.BigInteger(), nullable=True), + sa.Column("address", sa.VARCHAR(length=256), nullable=True), + sa.Column("transaction_hash", sa.VARCHAR(length=256), nullable=True), + sa.Column("label_data", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column("block_timestamp", sa.BigInteger(), nullable=True), + sa.Column("log_index", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_arbitrum_sepolia_labels")), + sa.UniqueConstraint("id", name=op.f("uq_arbitrum_sepolia_labels_id")), + ) + op.create_index( + op.f("ix_arbitrum_sepolia_labels_address"), + "arbitrum_sepolia_labels", + ["address"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_labels_block_number"), + "arbitrum_sepolia_labels", + ["block_number"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_labels_block_timestamp"), + "arbitrum_sepolia_labels", + ["block_timestamp"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_labels_label"), + "arbitrum_sepolia_labels", + ["label"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_labels_transaction_hash"), + "arbitrum_sepolia_labels", + ["transaction_hash"], + unique=False, + ) + op.create_table( + "arbitrum_sepolia_transactions", + sa.Column("hash", sa.VARCHAR(length=256), nullable=False), + sa.Column("block_number", sa.BigInteger(), nullable=False), + sa.Column("from_address", sa.VARCHAR(length=256), nullable=True), + sa.Column("to_address", sa.VARCHAR(length=256), nullable=True), + sa.Column("gas", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column("gas_price", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column("max_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column( + "max_priority_fee_per_gas", sa.Numeric(precision=78, scale=0), nullable=True + ), + sa.Column("input", sa.Text(), nullable=True), + sa.Column("nonce", sa.VARCHAR(length=256), nullable=True), + sa.Column("transaction_index", sa.BigInteger(), nullable=True), + sa.Column("transaction_type", sa.Integer(), nullable=True), + sa.Column("value", sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column( + "indexed_at", + sa.DateTime(timezone=True), + server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), + nullable=False, + ), + sa.Column("y_parity", sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint( + ["block_number"], + ["arbitrum_sepolia_blocks.block_number"], + name=op.f( + "fk_arbitrum_sepolia_transactions_block_number_arbitrum_sepolia_blocks" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("hash", name=op.f("pk_arbitrum_sepolia_transactions")), + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_block_number"), + "arbitrum_sepolia_transactions", + ["block_number"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_from_address"), + "arbitrum_sepolia_transactions", + ["from_address"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_gas"), + "arbitrum_sepolia_transactions", + ["gas"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_gas_price"), + "arbitrum_sepolia_transactions", + ["gas_price"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_hash"), + "arbitrum_sepolia_transactions", + ["hash"], + unique=True, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_to_address"), + "arbitrum_sepolia_transactions", + ["to_address"], + unique=False, + ) + op.create_index( + op.f("ix_arbitrum_sepolia_transactions_value"), + "arbitrum_sepolia_transactions", + ["value"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ###) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_value"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_to_address"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_hash"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_gas_price"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_gas"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_from_address"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_transactions_block_number"), + table_name="arbitrum_sepolia_transactions", + ) + op.drop_table("arbitrum_sepolia_transactions") + op.drop_index( + op.f("ix_arbitrum_sepolia_labels_transaction_hash"), + table_name="arbitrum_sepolia_labels", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_labels_label"), table_name="arbitrum_sepolia_labels" + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_labels_block_timestamp"), + table_name="arbitrum_sepolia_labels", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_labels_block_number"), + table_name="arbitrum_sepolia_labels", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_labels_address"), table_name="arbitrum_sepolia_labels" + ) + op.drop_table("arbitrum_sepolia_labels") + op.drop_index( + op.f("ix_arbitrum_sepolia_blocks_timestamp"), + table_name="arbitrum_sepolia_blocks", + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_blocks_hash"), table_name="arbitrum_sepolia_blocks" + ) + op.drop_index( + op.f("ix_arbitrum_sepolia_blocks_block_number"), + table_name="arbitrum_sepolia_blocks", + ) + op.drop_table("arbitrum_sepolia_blocks") + # ### end Alembic commands ###