From 93252308f4df2af56da69399e272effe00c40ba6 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 14 Mar 2024 17:07:12 +0000 Subject: [PATCH 01/12] Extended list of moonworm crawlers to monitor --- crawlers/deploy/monitoring-crawlers.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index 05e1f379..62a1983f 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service zksync-era-testnet-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service -service xdai-moonworm-crawler.service -service wyrm-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers From b09cc626c9aa5c007e45c96c479559780ce94f3d Mon Sep 17 00:00:00 2001 From: kompotkot Date: Tue, 19 Mar 2024 13:30:42 +0000 Subject: [PATCH 02/12] Removed wyrm crawler services --- crawlers/deploy/deploy.bash | 54 ------------------- crawlers/deploy/monitoring-crawlers.service | 2 +- .../wyrm-historical-crawl-events.service | 17 ------ .../deploy/wyrm-historical-crawl-events.timer | 9 ---- ...wyrm-historical-crawl-transactions.service | 17 ------ .../wyrm-historical-crawl-transactions.timer | 9 ---- crawlers/deploy/wyrm-missing.service | 11 ---- crawlers/deploy/wyrm-missing.timer | 9 ---- crawlers/deploy/wyrm-moonworm-crawler.service | 17 ------ crawlers/deploy/wyrm-synchronize.service | 17 ------ 10 files changed, 1 insertion(+), 161 deletions(-) delete mode 100644 crawlers/deploy/wyrm-historical-crawl-events.service delete mode 100644 crawlers/deploy/wyrm-historical-crawl-events.timer delete mode 100644 crawlers/deploy/wyrm-historical-crawl-transactions.service delete mode 100644 crawlers/deploy/wyrm-historical-crawl-transactions.timer delete mode 100644 crawlers/deploy/wyrm-missing.service delete mode 100644 crawlers/deploy/wyrm-missing.timer delete mode 100644 crawlers/deploy/wyrm-moonworm-crawler.service delete mode 100644 crawlers/deploy/wyrm-synchronize.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 2ab07d37..b5f977c2 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -83,7 +83,6 @@ MUMBAI_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="mumbai-historical-crawl-transac MUMBAI_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="mumbai-historical-crawl-events.service" MUMBAI_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="mumbai-historical-crawl-events.timer" - # XDai service files XDAI_SYNCHRONIZE_SERVICE="xdai-synchronize.service" XDAI_MISSING_SERVICE_FILE="xdai-missing.service" @@ -94,16 +93,6 @@ XDai_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="xdai-historical-crawl-transaction XDai_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="xdai-historical-crawl-events.service" XDai_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="xdai-historical-crawl-events.timer" -# Wyrm service files -WYRM_SYNCHRONIZE_SERVICE="wyrm-synchronize.service" -WYRM_MISSING_SERVICE_FILE="wyrm-missing.service" -WYRM_MISSING_TIMER_FILE="wyrm-missing.timer" -WYRM_MOONWORM_CRAWLER_SERVICE_FILE="wyrm-moonworm-crawler.service" -WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="wyrm-historical-crawl-transactions.service" -WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transactions.timer" -WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service" -WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer" - # ZkSync Era ZKSYNC_ERA_SYNCHRONIZE_SERVICE="zksync-era-synchronize.service" ZKSYNC_ERA_MISSING_SERVICE_FILE="zksync-era-missing.service" @@ -477,49 +466,6 @@ cp "${SCRIPT_DIR}/${XDai_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.con XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${XDai_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrm block with transactions syncronizer service definition with ${WYRM_SYNCHRONIZE_SERVICE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_SYNCHRONIZE_SERVICE}" -cp "${SCRIPT_DIR}/${WYRM_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${WYRM_SYNCHRONIZE_SERVICE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_SYNCHRONIZE_SERVICE}" - -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrn missing service and timer with: ${WYRM_MISSING_SERVICE_FILE}, ${WYRM_MISSING_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${WYRM_MISSING_TIMER_FILE}" -cp "${SCRIPT_DIR}/${WYRM_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_MISSING_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${WYRM_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_MISSING_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_MISSING_TIMER_FILE}" - -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrm moonworm crawler service definition with ${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_MOONWORM_CRAWLER_SERVICE_FILE}" - -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrm historical transactions crawler service and timer with: ${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" - -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Wyrm historical events crawler service and timer with: ${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}, ${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" -cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" - # ZkSync Era echo echo diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index 62a1983f..3966d7cb 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service zksync-era-testnet-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service -service xdai-moonworm-crawler.service -service wyrm-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service zksync-era-testnet-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service -service xdai-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers diff --git a/crawlers/deploy/wyrm-historical-crawl-events.service b/crawlers/deploy/wyrm-historical-crawl-events.service deleted file mode 100644 index 7ecefc11..00000000 --- a/crawlers/deploy/wyrm-historical-crawl-events.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Wyrm historical crawler events -After=network.target -StartLimitIntervalSec=300 -StartLimitBurst=3 - -[Service] -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -Restart=on-failure -RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-events -CPUWeight=70 -SyslogIdentifier=wyrm-historical-crawl-events - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/wyrm-historical-crawl-events.timer b/crawlers/deploy/wyrm-historical-crawl-events.timer deleted file mode 100644 index bd402c01..00000000 --- a/crawlers/deploy/wyrm-historical-crawl-events.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Runs events historical crawler on wyrm - -[Timer] -OnBootSec=60s -OnUnitActiveSec=10m - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/wyrm-historical-crawl-transactions.service b/crawlers/deploy/wyrm-historical-crawl-transactions.service deleted file mode 100644 index 41186544..00000000 --- a/crawlers/deploy/wyrm-historical-crawl-transactions.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Wyrm historical crawler transactions -After=network.target -StartLimitIntervalSec=300 -StartLimitBurst=3 - -[Service] -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -Restart=on-failure -RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type wyrm --find-deployed-blocks --end 0 --tasks-journal --only-functions -CPUWeight=70 -SyslogIdentifier=wyrm-historical-crawl-transactions - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/wyrm-historical-crawl-transactions.timer b/crawlers/deploy/wyrm-historical-crawl-transactions.timer deleted file mode 100644 index 40ea12bd..00000000 --- a/crawlers/deploy/wyrm-historical-crawl-transactions.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Runs transactions historical crawler on wyrm - -[Timer] -OnBootSec=60s -OnUnitActiveSec=10m - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/wyrm-missing.service b/crawlers/deploy/wyrm-missing.service deleted file mode 100644 index e2d0518c..00000000 --- a/crawlers/deploy/wyrm-missing.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Fill missing blocks at Wyrm database -After=network.target - -[Service] -Type=oneshot -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain wyrm -CPUWeight=50 -SyslogIdentifier=wyrm-missing \ No newline at end of file diff --git a/crawlers/deploy/wyrm-missing.timer b/crawlers/deploy/wyrm-missing.timer deleted file mode 100644 index 8bd5e260..00000000 --- a/crawlers/deploy/wyrm-missing.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Fill missing blocks at Wyrm database - -[Timer] -OnBootSec=120s -OnUnitActiveSec=15m - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/wyrm-moonworm-crawler.service b/crawlers/deploy/wyrm-moonworm-crawler.service deleted file mode 100644 index 146e477d..00000000 --- a/crawlers/deploy/wyrm-moonworm-crawler.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Wyrm moonworm crawler -After=network.target -StartLimitIntervalSec=300 -StartLimitBurst=3 - -[Service] -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -Restart=on-failure -RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl --blockchain wyrm --poa --no-confirmations --start 0 --min-blocks-batch 1 -CPUWeight=70 -SyslogIdentifier=wyrm-moonworm-crawler - -[Install] -WantedBy=multi-user.target diff --git a/crawlers/deploy/wyrm-synchronize.service b/crawlers/deploy/wyrm-synchronize.service deleted file mode 100644 index c9f6702e..00000000 --- a/crawlers/deploy/wyrm-synchronize.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Wyrm block with transactions synchronizer -StartLimitIntervalSec=300 -StartLimitBurst=3 -After=network.target - -[Service] -Restart=on-failure -RestartSec=15s -WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl -EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain wyrm -c 0 -j 1 -CPUWeight=90 -SyslogIdentifier=wyrm-synchronize - -[Install] -WantedBy=multi-user.target \ No newline at end of file From d7281b6d1477c25599e685c97893f0484ebd1601 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Mar 2024 18:34:10 +0200 Subject: [PATCH 03/12] Icorrect response from getCode will set is_contract to zero. --- moonstreamapi/moonstreamapi/actions.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index af0f2c72..55c915e4 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -949,8 +949,15 @@ def check_if_smart_contract( web3_client = connect(blockchain_type, user_token=user_token) is_contract = False - - code = web3_client.eth.getCode(address) + breakpoint() + try: + code = web3_client.eth.getCode(address) + except Exception as e: + logger.error( + f"Error while getting code of address: {e} in blockchain: {blockchain_type}" + ) + code = b"" + if code != b"": is_contract = True From da0f089c0d605d93e46847eccb328f7ae90b09b0 Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 20 Mar 2024 18:47:00 +0200 Subject: [PATCH 04/12] Replace to warning. --- moonstreamapi/moonstreamapi/actions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/moonstreamapi/moonstreamapi/actions.py b/moonstreamapi/moonstreamapi/actions.py index 55c915e4..0dde7715 100644 --- a/moonstreamapi/moonstreamapi/actions.py +++ b/moonstreamapi/moonstreamapi/actions.py @@ -949,15 +949,14 @@ def check_if_smart_contract( web3_client = connect(blockchain_type, user_token=user_token) is_contract = False - breakpoint() try: code = web3_client.eth.getCode(address) except Exception as e: - logger.error( + logger.warning( f"Error while getting code of address: {e} in blockchain: {blockchain_type}" ) code = b"" - + if code != b"": is_contract = True From fa8756038594bf2250ef85b984cffc787ea8e57e Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Mar 2024 13:27:56 +0200 Subject: [PATCH 05/12] Add fixes. --- .../moonworm_crawler/continuous_crawler.py | 31 +++++-- .../mooncrawl/moonworm_crawler/db.py | 91 ++++++++++++------- .../moonworm_crawler/historical_crawler.py | 30 +++++- 3 files changed, 108 insertions(+), 44 deletions(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 6c3a811f..33761970 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -101,6 +101,7 @@ def continuous_crawler( heartbeat_interval: float = 60, new_jobs_refetch_interval: float = 120, access_id: Optional[UUID] = None, + max_insert_batch: int = 10000, ): crawler_type = "continuous" assert ( @@ -204,7 +205,16 @@ def continuous_crawler( f"Crawled {len(all_events)} events from {start_block} to {end_block}." ) - add_events_to_session(db_session, all_events, blockchain_type) + if len(all_events) > max_insert_batch: + + for i in range(0, len(all_events), max_insert_batch): + add_events_to_session( + db_session, + all_events[i : i + max_insert_batch], + blockchain_type, + ) + else: + add_events_to_session(db_session, all_events, blockchain_type) logger.info( f"Crawling function calls from {start_block} to {end_block}" @@ -220,9 +230,18 @@ def continuous_crawler( f"Crawled {len(all_function_calls)} function calls from {start_block} to {end_block}." ) - add_function_calls_to_session( - db_session, all_function_calls, blockchain_type - ) + if len(all_function_calls) > max_insert_batch: + + for i in range(0, len(all_function_calls), max_insert_batch): + add_function_calls_to_session( + db_session, + all_function_calls[i : i + max_insert_batch], + blockchain_type, + ) + else: + add_function_calls_to_session( + db_session, all_function_calls, blockchain_type + ) current_time = datetime.utcnow() @@ -246,11 +265,11 @@ def continuous_crawler( jobs_refetchet_time = current_time + commit_session(db_session) + if current_time - last_heartbeat_time > timedelta( seconds=heartbeat_interval ): - # Commiting to db - commit_session(db_session) # Update heartbeat heartbeat_template["last_block"] = end_block heartbeat_template["current_time"] = _date_to_str(current_time) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py index 26fd64ef..7db669c2 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py @@ -6,6 +6,7 @@ from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model from moonstreamdb.models import Base from moonworm.crawler.function_call_crawler import ContractFunctionCall # type: ignore from sqlalchemy.orm import Session +from sqlalchemy import func, text, select, values, column, Integer, String, exists from ..settings import CRAWLER_LABEL from .event_crawler import Event @@ -152,36 +153,47 @@ def add_events_to_session( blockchain_type: AvailableBlockchainType, label_name=CRAWLER_LABEL, ) -> None: + + if len(events) == 0: + return + label_model = get_label_model(blockchain_type) - events_hashes_to_save = [event.transaction_hash for event in events] + events_hashes_to_save = set([event.transaction_hash for event in events]) - existing_labels = ( - db_session.query(label_model.transaction_hash, label_model.log_index) + # Define a CTE VALUES expression to escape big IN clause + hashes_cte = select( + values(column("transaction_hash", String), name="hashes").data( + [(hash,) for hash in events_hashes_to_save] + ) + ).cte() + + # Retrieve existing transaction hashes and registered log indexes + query = ( + db_session.query( + label_model.transaction_hash.label("transaction_hash"), + func.array_agg(label_model.log_index).label("log_indexes"), + ) .filter( label_model.label == label_name, label_model.log_index != None, - label_model.transaction_hash.in_(events_hashes_to_save), + exists().where( + label_model.transaction_hash == hashes_cte.c.transaction_hash + ), ) - .all() + .group_by(label_model.transaction_hash) ) - existing_labels_transactions = [] - existing_log_index_by_tx_hash: Dict[str, List[int]] = {} - for label in existing_labels: - if label[0] not in existing_labels_transactions: - existing_labels_transactions.append(label[0]) - existing_log_index_by_tx_hash[label[0]] = [] - existing_log_index_by_tx_hash[label[0]].append(label[1]) + existing_log_index_by_tx_hash = { + row.transaction_hash: row.log_indexes for row in query + } - labels_to_save = [] - for event in events: - if event.transaction_hash not in existing_labels_transactions: - labels_to_save.append(_event_to_label(blockchain_type, event, label_name)) - elif ( - event.log_index not in existing_log_index_by_tx_hash[event.transaction_hash] - ): - labels_to_save.append(_event_to_label(blockchain_type, event, label_name)) + labels_to_save = [ + _event_to_label(blockchain_type, event, label_name) + for event in events + if event.transaction_hash not in existing_log_index_by_tx_hash + or event.log_index not in existing_log_index_by_tx_hash[event.transaction_hash] + ] logger.info(f"Saving {len(labels_to_save)} event labels to session") db_session.add_all(labels_to_save) @@ -193,27 +205,38 @@ def add_function_calls_to_session( blockchain_type: AvailableBlockchainType, label_name=CRAWLER_LABEL, ) -> None: - label_model = get_label_model(blockchain_type) - transactions_hashes_to_save = [ - function_call.transaction_hash for function_call in function_calls - ] - existing_labels = ( - db_session.query(label_model.transaction_hash) - .filter( - label_model.label == label_name, - label_model.log_index == None, - label_model.transaction_hash.in_(transactions_hashes_to_save), - ) - .all() + if len(function_calls) == 0: + return + + label_model = get_label_model(blockchain_type) + + transactions_hashes_to_save = list( + set([function_call.transaction_hash for function_call in function_calls]) ) - existing_labels_transactions = [label[0] for label in existing_labels] + # Define a CTE VALUES expression to escape big IN clause + hashes_cte = select( + values(column("transaction_hash", String), name="hashes").data( + [(hash,) for hash in transactions_hashes_to_save] + ) + ).cte() + + # Retrieve existing transaction hashes + query = db_session.query( + label_model.transaction_hash.label("transaction_hash") + ).filter( + label_model.label == label_name, + label_model.log_index.is_(None), + exists().where(label_model.transaction_hash == hashes_cte.c.transaction_hash), + ) + + existing_tx_hashes = [row.transaction_hash for row in query] labels_to_save = [ _function_call_to_label(blockchain_type, function_call) for function_call in function_calls - if function_call.transaction_hash not in existing_labels_transactions + if function_call.transaction_hash not in existing_tx_hashes ] logger.info(f"Saving {len(labels_to_save)} labels to session") diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 4091ae90..4f66a8bd 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -38,6 +38,7 @@ def historical_crawler( min_sleep_time: float = 0.1, access_id: Optional[UUID] = None, addresses_deployment_blocks: Optional[Dict[ChecksumAddress, int]] = None, + max_insert_batch: int = 10000, ): assert max_blocks_batch > 0, "max_blocks_batch must be greater than 0" assert min_sleep_time > 0, "min_sleep_time must be greater than 0" @@ -127,7 +128,18 @@ def historical_crawler( f"Crawled {len(all_events)} events from {start_block} to {batch_end_block}." ) - add_events_to_session(db_session, all_events, blockchain_type) + if len(all_events) > max_insert_batch: + + for i in range(0, len(all_events), max_insert_batch): + add_events_to_session( + db_session, + all_events[i : i + max_insert_batch], + blockchain_type, + ) + + else: + + add_events_to_session(db_session, all_events, blockchain_type) if function_call_crawl_jobs: logger.info( @@ -144,9 +156,19 @@ def historical_crawler( f"Crawled {len(all_function_calls)} function calls from {start_block} to {batch_end_block}." ) - add_function_calls_to_session( - db_session, all_function_calls, blockchain_type - ) + if len(all_function_calls) > max_insert_batch: + + for i in range(0, len(all_function_calls), max_insert_batch): + add_function_calls_to_session( + db_session, + all_function_calls[i : i + max_insert_batch], + blockchain_type, + ) + else: + + add_function_calls_to_session( + db_session, all_function_calls, blockchain_type + ) if addresses_deployment_blocks: for address, deployment_block in addresses_deployment_blocks.items(): From 4dead75827d3dc185fa4c61321a8adb233100f50 Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Mar 2024 13:29:11 +0200 Subject: [PATCH 06/12] Bump version. --- crawlers/mooncrawl/mooncrawl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index 8eb58150..e999d730 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.8" +MOONCRAWL_VERSION = "0.3.9" From e96bb59965368654090ffb017a6e9aed12318f8f Mon Sep 17 00:00:00 2001 From: Andrey Date: Mon, 25 Mar 2024 14:06:57 +0200 Subject: [PATCH 07/12] Replace != None -> isnot(None) more readable. --- crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py index 7db669c2..ad964a32 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py @@ -176,7 +176,7 @@ def add_events_to_session( ) .filter( label_model.label == label_name, - label_model.log_index != None, + label_model.log_index.isnot(None), exists().where( label_model.transaction_hash == hashes_cte.c.transaction_hash ), From 6b6b08533c899a89e2af2bba78dfa02517ba323e Mon Sep 17 00:00:00 2001 From: Andrey Date: Wed, 27 Mar 2024 22:03:27 +0200 Subject: [PATCH 08/12] Add ethreum metadata task. --- .../state_crawler/jobs/ethereum-jobs.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json index 08e85f97..ced36d16 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json @@ -31,5 +31,36 @@ } ], "address": "0xe1dC516B1486Aba548eecD2947A11273518434a4" + }, + { + "type": "function", + "stateMutability": "view", + "inputs": [ + { + "internalType": "uint256", + "name": "tokenId", + "type": "uint256", + "value": { + "type": "queryAPI", + "query_url": "template_erc721_minting", + "blockchain": "ethereum", + "params": { + "address": "0xBC4CA0EdA7647A8aB7C2061c2E118A18a936f13D" + }, + "keys": [ + "token_id" + ] + } + } + ], + "name": "tokenURI", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "address": "0xBC4CA0EdA7647A8aB7C2061c2E118A18a936f13D" } ] \ No newline at end of file From d4db66c551c5892a72b1ed53f1187371627a92e9 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Mar 2024 12:17:20 +0000 Subject: [PATCH 09/12] Models for Avalanche and zk era sepolia blockchains --- moonstreamdb/alembic/env.py | 48 ++- ...56a0b_avalanche_and_zksync_era_sepolia_.py | 331 ++++++++++++++++ moonstreamdb/moonstreamdb/blockchain.py | 81 +++- moonstreamdb/moonstreamdb/models.py | 372 ++++++++++++++++++ moonstreamdb/moonstreamdb/networks.py | 60 ++- moonstreamdb/moonstreamdb/version.py | 2 +- 6 files changed, 850 insertions(+), 44 deletions(-) create mode 100644 moonstreamdb/alembic/versions/a95cbab56a0b_avalanche_and_zksync_era_sepolia_.py diff --git a/moonstreamdb/alembic/env.py b/moonstreamdb/alembic/env.py index d2c88104..5218b212 100644 --- a/moonstreamdb/alembic/env.py +++ b/moonstreamdb/alembic/env.py @@ -25,6 +25,18 @@ target_metadata = MoonstreamBase.metadata # my_important_option = config.get_main_option("my_important_option") # ... etc. from moonstreamdb.models import ( + ArbitrumNovaBlock, + ArbitrumNovaLabel, + ArbitrumNovaTransaction, + ArbitrumSepoliaBlock, + ArbitrumSepoliaLabel, + ArbitrumSepoliaTransaction, + AvalancheBlock, + AvalancheFujiBlock, + AvalancheFujiLabel, + AvalancheFujiTransaction, + AvalancheLabel, + AvalancheTransaction, ESDEventSignature, ESDFunctionSignature, EthereumBlock, @@ -40,21 +52,21 @@ from moonstreamdb.models import ( WyrmBlock, WyrmLabel, WyrmTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, - ZkSyncEraTestnetBlock, - ZkSyncEraTestnetLabel, - ZkSyncEraTestnetTransaction, - ArbitrumNovaBlock, - ArbitrumNovaTransaction, - ArbitrumNovaLabel, - ArbitrumSepoliaBlock, - ArbitrumSepoliaTransaction, - ArbitrumSepoliaLabel, XaiBlock, XaiLabel, XaiTransaction, + XDaiBlock, + XDaiLabel, + XDaiTransaction, + ZkSyncEraBlock, + ZkSyncEraLabel, + ZkSyncEraSepoliaBlock, + ZkSyncEraSepoliaLabel, + ZkSyncEraSepoliaTransaction, + ZkSyncEraTestnetBlock, + ZkSyncEraTestnetLabel, + ZkSyncEraTestnetTransaction, + ZkSyncEraTransaction, ) @@ -81,6 +93,12 @@ def include_symbol(tablename, schema): ZkSyncEraTestnetBlock.__tablename__, ZkSyncEraTestnetLabel.__tablename__, ZkSyncEraTestnetTransaction.__tablename__, + ZkSyncEraBlock.__tablename__, + ZkSyncEraLabel.__tablename__, + ZkSyncEraTransaction.__tablename__, + ZkSyncEraSepoliaBlock.__tablename__, + ZkSyncEraSepoliaLabel.__tablename__, + ZkSyncEraSepoliaTransaction.__tablename__, ArbitrumNovaBlock.__tablename__, ArbitrumNovaTransaction.__tablename__, ArbitrumNovaLabel.__tablename__, @@ -90,6 +108,12 @@ def include_symbol(tablename, schema): XaiBlock.__tablename__, XaiLabel.__tablename__, XaiTransaction.__tablename__, + AvalancheBlock.__tablename__, + AvalancheFujiBlock.__tablename__, + AvalancheFujiLabel.__tablename__, + AvalancheFujiTransaction.__tablename__, + AvalancheLabel.__tablename__, + AvalancheTransaction.__tablename__, } diff --git a/moonstreamdb/alembic/versions/a95cbab56a0b_avalanche_and_zksync_era_sepolia_.py b/moonstreamdb/alembic/versions/a95cbab56a0b_avalanche_and_zksync_era_sepolia_.py new file mode 100644 index 00000000..961c35c9 --- /dev/null +++ b/moonstreamdb/alembic/versions/a95cbab56a0b_avalanche_and_zksync_era_sepolia_.py @@ -0,0 +1,331 @@ +"""Avalanche and Zksync Era Sepolia blockchains + +Revision ID: a95cbab56a0b +Revises: a99b97acc39e +Create Date: 2024-03-28 11:08:06.677075 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'a95cbab56a0b' +down_revision = 'a99b97acc39e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('avalanche_blocks', + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('difficulty', sa.BigInteger(), nullable=True), + sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True), + sa.Column('gas_limit', sa.BigInteger(), nullable=True), + sa.Column('gas_used', sa.BigInteger(), nullable=True), + sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True), + sa.Column('miner', sa.VARCHAR(length=256), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('state_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('timestamp', sa.BigInteger(), nullable=True), + sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), + sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True), + sa.Column('block_gas_cost', sa.VARCHAR(length=256), nullable=True), + sa.Column('ext_data_gas_used', sa.VARCHAR(length=256), nullable=True), + sa.Column('ext_data_hash', sa.VARCHAR(length=256), nullable=True), + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_avalanche_blocks')) + ) + op.create_index(op.f('ix_avalanche_blocks_block_number'), 'avalanche_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_avalanche_blocks_hash'), 'avalanche_blocks', ['hash'], unique=False) + op.create_index(op.f('ix_avalanche_blocks_timestamp'), 'avalanche_blocks', ['timestamp'], unique=False) + op.create_table('avalanche_fuji_blocks', + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('difficulty', sa.BigInteger(), nullable=True), + sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True), + sa.Column('gas_limit', sa.BigInteger(), nullable=True), + sa.Column('gas_used', sa.BigInteger(), nullable=True), + sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True), + sa.Column('miner', sa.VARCHAR(length=256), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('state_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('timestamp', sa.BigInteger(), nullable=True), + sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), + sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('block_extra_data', sa.VARCHAR(length=256), nullable=True), + sa.Column('block_gas_cost', sa.VARCHAR(length=256), nullable=True), + sa.Column('ext_data_gas_used', sa.VARCHAR(length=256), nullable=True), + sa.Column('ext_data_hash', sa.VARCHAR(length=256), nullable=True), + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_avalanche_fuji_blocks')) + ) + op.create_index(op.f('ix_avalanche_fuji_blocks_block_number'), 'avalanche_fuji_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_avalanche_fuji_blocks_hash'), 'avalanche_fuji_blocks', ['hash'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_blocks_timestamp'), 'avalanche_fuji_blocks', ['timestamp'], unique=False) + op.create_table('avalanche_fuji_labels', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('label', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=True), + sa.Column('address', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('block_timestamp', sa.BigInteger(), nullable=True), + sa.Column('log_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_avalanche_fuji_labels')), + sa.UniqueConstraint('id', name=op.f('uq_avalanche_fuji_labels_id')) + ) + op.create_index(op.f('ix_avalanche_fuji_labels_address'), 'avalanche_fuji_labels', ['address'], unique=False) + op.create_index('ix_avalanche_fuji_labels_address_block_number', 'avalanche_fuji_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_avalanche_fuji_labels_address_block_timestamp', 'avalanche_fuji_labels', ['address', 'block_timestamp'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_labels_block_number'), 'avalanche_fuji_labels', ['block_number'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_labels_block_timestamp'), 'avalanche_fuji_labels', ['block_timestamp'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_labels_label'), 'avalanche_fuji_labels', ['label'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_labels_transaction_hash'), 'avalanche_fuji_labels', ['transaction_hash'], unique=False) + op.create_table('avalanche_labels', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('label', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=True), + sa.Column('address', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('block_timestamp', sa.BigInteger(), nullable=True), + sa.Column('log_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_avalanche_labels')), + sa.UniqueConstraint('id', name=op.f('uq_avalanche_labels_id')) + ) + op.create_index(op.f('ix_avalanche_labels_address'), 'avalanche_labels', ['address'], unique=False) + op.create_index('ix_avalanche_labels_address_block_number', 'avalanche_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_avalanche_labels_address_block_timestamp', 'avalanche_labels', ['address', 'block_timestamp'], unique=False) + op.create_index(op.f('ix_avalanche_labels_block_number'), 'avalanche_labels', ['block_number'], unique=False) + op.create_index(op.f('ix_avalanche_labels_block_timestamp'), 'avalanche_labels', ['block_timestamp'], unique=False) + op.create_index(op.f('ix_avalanche_labels_label'), 'avalanche_labels', ['label'], unique=False) + op.create_index(op.f('ix_avalanche_labels_transaction_hash'), 'avalanche_labels', ['transaction_hash'], unique=False) + op.create_table('zksync_era_sepolia_blocks', + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('difficulty', sa.BigInteger(), nullable=True), + sa.Column('extra_data', sa.VARCHAR(length=128), nullable=True), + sa.Column('gas_limit', sa.BigInteger(), nullable=True), + sa.Column('gas_used', sa.BigInteger(), nullable=True), + sa.Column('base_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('logs_bloom', sa.VARCHAR(length=1024), nullable=True), + sa.Column('miner', sa.VARCHAR(length=256), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('parent_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('receipt_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('size', sa.Integer(), nullable=True), + sa.Column('state_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('timestamp', sa.BigInteger(), nullable=True), + sa.Column('total_difficulty', sa.VARCHAR(length=256), nullable=True), + sa.Column('transactions_root', sa.VARCHAR(length=256), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('mix_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('sha3_uncles', sa.VARCHAR(length=256), nullable=True), + sa.Column('l1_batch_number', sa.BigInteger(), nullable=True), + sa.Column('l1_batch_timestamp', sa.BigInteger(), nullable=True), + sa.PrimaryKeyConstraint('block_number', name=op.f('pk_zksync_era_sepolia_blocks')) + ) + op.create_index(op.f('ix_zksync_era_sepolia_blocks_block_number'), 'zksync_era_sepolia_blocks', ['block_number'], unique=True) + op.create_index(op.f('ix_zksync_era_sepolia_blocks_hash'), 'zksync_era_sepolia_blocks', ['hash'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_blocks_timestamp'), 'zksync_era_sepolia_blocks', ['timestamp'], unique=False) + op.create_table('zksync_era_sepolia_labels', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('label', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=True), + sa.Column('address', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=256), nullable=True), + sa.Column('label_data', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('block_timestamp', sa.BigInteger(), nullable=True), + sa.Column('log_index', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_zksync_era_sepolia_labels')), + sa.UniqueConstraint('id', name=op.f('uq_zksync_era_sepolia_labels_id')) + ) + op.create_index(op.f('ix_zksync_era_sepolia_labels_address'), 'zksync_era_sepolia_labels', ['address'], unique=False) + op.create_index('ix_zksync_era_sepolia_labels_address_block_number', 'zksync_era_sepolia_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_zksync_era_sepolia_labels_address_block_timestamp', 'zksync_era_sepolia_labels', ['address', 'block_timestamp'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_labels_block_number'), 'zksync_era_sepolia_labels', ['block_number'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_labels_block_timestamp'), 'zksync_era_sepolia_labels', ['block_timestamp'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_labels_label'), 'zksync_era_sepolia_labels', ['label'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_labels_transaction_hash'), 'zksync_era_sepolia_labels', ['transaction_hash'], unique=False) + op.create_table('avalanche_fuji_transactions', + sa.Column('hash', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('input', sa.Text(), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_index', sa.BigInteger(), nullable=True), + sa.Column('transaction_type', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.ForeignKeyConstraint(['block_number'], ['avalanche_fuji_blocks.block_number'], name=op.f('fk_avalanche_fuji_transactions_block_number_avalanche_fuji_blocks'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('hash', name=op.f('pk_avalanche_fuji_transactions')) + ) + op.create_index(op.f('ix_avalanche_fuji_transactions_block_number'), 'avalanche_fuji_transactions', ['block_number'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_transactions_from_address'), 'avalanche_fuji_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_transactions_gas'), 'avalanche_fuji_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_transactions_gas_price'), 'avalanche_fuji_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_transactions_hash'), 'avalanche_fuji_transactions', ['hash'], unique=True) + op.create_index(op.f('ix_avalanche_fuji_transactions_to_address'), 'avalanche_fuji_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_avalanche_fuji_transactions_value'), 'avalanche_fuji_transactions', ['value'], unique=False) + op.create_table('avalanche_transactions', + sa.Column('hash', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('input', sa.Text(), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_index', sa.BigInteger(), nullable=True), + sa.Column('transaction_type', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.ForeignKeyConstraint(['block_number'], ['avalanche_blocks.block_number'], name=op.f('fk_avalanche_transactions_block_number_avalanche_blocks'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('hash', name=op.f('pk_avalanche_transactions')) + ) + op.create_index(op.f('ix_avalanche_transactions_block_number'), 'avalanche_transactions', ['block_number'], unique=False) + op.create_index(op.f('ix_avalanche_transactions_from_address'), 'avalanche_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_avalanche_transactions_gas'), 'avalanche_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_avalanche_transactions_gas_price'), 'avalanche_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_avalanche_transactions_hash'), 'avalanche_transactions', ['hash'], unique=True) + op.create_index(op.f('ix_avalanche_transactions_to_address'), 'avalanche_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_avalanche_transactions_value'), 'avalanche_transactions', ['value'], unique=False) + op.create_table('zksync_era_sepolia_transactions', + sa.Column('hash', sa.VARCHAR(length=256), nullable=False), + sa.Column('block_number', sa.BigInteger(), nullable=False), + sa.Column('from_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('to_address', sa.VARCHAR(length=256), nullable=True), + sa.Column('gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('gas_price', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('max_priority_fee_per_gas', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('input', sa.Text(), nullable=True), + sa.Column('nonce', sa.VARCHAR(length=256), nullable=True), + sa.Column('transaction_index', sa.BigInteger(), nullable=True), + sa.Column('transaction_type', sa.Integer(), nullable=True), + sa.Column('value', sa.Numeric(precision=78, scale=0), nullable=True), + sa.Column('indexed_at', sa.DateTime(timezone=True), server_default=sa.text("TIMEZONE('utc', statement_timestamp())"), nullable=False), + sa.Column('l1_batch_number', sa.BigInteger(), nullable=True), + sa.Column('l1_batch_tx_index', sa.BigInteger(), nullable=True), + sa.ForeignKeyConstraint(['block_number'], ['zksync_era_sepolia_blocks.block_number'], name=op.f('fk_zksync_era_sepolia_transactions_block_number_zksync_era_sepolia_blocks'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('hash', name=op.f('pk_zksync_era_sepolia_transactions')) + ) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_block_number'), 'zksync_era_sepolia_transactions', ['block_number'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_from_address'), 'zksync_era_sepolia_transactions', ['from_address'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_gas'), 'zksync_era_sepolia_transactions', ['gas'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_gas_price'), 'zksync_era_sepolia_transactions', ['gas_price'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_hash'), 'zksync_era_sepolia_transactions', ['hash'], unique=True) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_to_address'), 'zksync_era_sepolia_transactions', ['to_address'], unique=False) + op.create_index(op.f('ix_zksync_era_sepolia_transactions_value'), 'zksync_era_sepolia_transactions', ['value'], unique=False) + op.create_index('ix_arbitrum_nova_labels_address_block_number', 'arbitrum_nova_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_arbitrum_nova_labels_address_block_timestamp', 'arbitrum_nova_labels', ['address', 'block_timestamp'], unique=False) + op.create_index('ix_arbitrum_sepolia_labels_address_block_number', 'arbitrum_sepolia_labels', ['address', 'block_number'], unique=False) + op.create_index('ix_arbitrum_sepolia_labels_address_block_timestamp', 'arbitrum_sepolia_labels', ['address', 'block_timestamp'], unique=False) + op.create_unique_constraint(op.f('uq_arbitrum_sepolia_labels_id'), 'arbitrum_sepolia_labels', ['id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f('uq_zksync_era_testnet_labels_id'), 'zksync_era_testnet_labels', type_='unique') + op.drop_constraint(op.f('uq_zksync_era_labels_id'), 'zksync_era_labels', type_='unique') + op.drop_constraint(op.f('uq_xdai_labels_id'), 'xdai_labels', type_='unique') + op.drop_constraint(op.f('uq_xai_labels_id'), 'xai_labels', type_='unique') + op.drop_constraint(op.f('uq_wyrm_labels_id'), 'wyrm_labels', type_='unique') + op.drop_constraint(op.f('uq_polygon_labels_id'), 'polygon_labels', type_='unique') + op.drop_constraint(op.f('uq_opensea_crawler_state_id'), 'opensea_crawler_state', type_='unique') + op.drop_constraint(op.f('uq_mumbai_labels_id'), 'mumbai_labels', type_='unique') + op.create_index('idx_ethereum_labels_opensea_nft_name', 'ethereum_labels', [sa.text("(label_data ->> 'name'::text)")], unique=False) + op.drop_constraint(op.f('uq_arbitrum_sepolia_labels_id'), 'arbitrum_sepolia_labels', type_='unique') + op.drop_index('ix_arbitrum_sepolia_labels_address_block_timestamp', table_name='arbitrum_sepolia_labels') + op.drop_index('ix_arbitrum_sepolia_labels_address_block_number', table_name='arbitrum_sepolia_labels') + op.drop_index('ix_arbitrum_nova_labels_address_block_timestamp', table_name='arbitrum_nova_labels') + op.drop_index('ix_arbitrum_nova_labels_address_block_number', table_name='arbitrum_nova_labels') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_value'), table_name='zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_to_address'), table_name='zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_hash'), table_name='zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_gas_price'), table_name='zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_gas'), table_name='zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_from_address'), table_name='zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_transactions_block_number'), table_name='zksync_era_sepolia_transactions') + op.drop_table('zksync_era_sepolia_transactions') + op.drop_index(op.f('ix_avalanche_transactions_value'), table_name='avalanche_transactions') + op.drop_index(op.f('ix_avalanche_transactions_to_address'), table_name='avalanche_transactions') + op.drop_index(op.f('ix_avalanche_transactions_hash'), table_name='avalanche_transactions') + op.drop_index(op.f('ix_avalanche_transactions_gas_price'), table_name='avalanche_transactions') + op.drop_index(op.f('ix_avalanche_transactions_gas'), table_name='avalanche_transactions') + op.drop_index(op.f('ix_avalanche_transactions_from_address'), table_name='avalanche_transactions') + op.drop_index(op.f('ix_avalanche_transactions_block_number'), table_name='avalanche_transactions') + op.drop_table('avalanche_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_value'), table_name='avalanche_fuji_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_to_address'), table_name='avalanche_fuji_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_hash'), table_name='avalanche_fuji_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_gas_price'), table_name='avalanche_fuji_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_gas'), table_name='avalanche_fuji_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_from_address'), table_name='avalanche_fuji_transactions') + op.drop_index(op.f('ix_avalanche_fuji_transactions_block_number'), table_name='avalanche_fuji_transactions') + op.drop_table('avalanche_fuji_transactions') + op.drop_index(op.f('ix_zksync_era_sepolia_labels_transaction_hash'), table_name='zksync_era_sepolia_labels') + op.drop_index(op.f('ix_zksync_era_sepolia_labels_label'), table_name='zksync_era_sepolia_labels') + op.drop_index(op.f('ix_zksync_era_sepolia_labels_block_timestamp'), table_name='zksync_era_sepolia_labels') + op.drop_index(op.f('ix_zksync_era_sepolia_labels_block_number'), table_name='zksync_era_sepolia_labels') + op.drop_index('ix_zksync_era_sepolia_labels_address_block_timestamp', table_name='zksync_era_sepolia_labels') + op.drop_index('ix_zksync_era_sepolia_labels_address_block_number', table_name='zksync_era_sepolia_labels') + op.drop_index(op.f('ix_zksync_era_sepolia_labels_address'), table_name='zksync_era_sepolia_labels') + op.drop_table('zksync_era_sepolia_labels') + op.drop_index(op.f('ix_zksync_era_sepolia_blocks_timestamp'), table_name='zksync_era_sepolia_blocks') + op.drop_index(op.f('ix_zksync_era_sepolia_blocks_hash'), table_name='zksync_era_sepolia_blocks') + op.drop_index(op.f('ix_zksync_era_sepolia_blocks_block_number'), table_name='zksync_era_sepolia_blocks') + op.drop_table('zksync_era_sepolia_blocks') + op.drop_index(op.f('ix_avalanche_labels_transaction_hash'), table_name='avalanche_labels') + op.drop_index(op.f('ix_avalanche_labels_label'), table_name='avalanche_labels') + op.drop_index(op.f('ix_avalanche_labels_block_timestamp'), table_name='avalanche_labels') + op.drop_index(op.f('ix_avalanche_labels_block_number'), table_name='avalanche_labels') + op.drop_index('ix_avalanche_labels_address_block_timestamp', table_name='avalanche_labels') + op.drop_index('ix_avalanche_labels_address_block_number', table_name='avalanche_labels') + op.drop_index(op.f('ix_avalanche_labels_address'), table_name='avalanche_labels') + op.drop_table('avalanche_labels') + op.drop_index(op.f('ix_avalanche_fuji_labels_transaction_hash'), table_name='avalanche_fuji_labels') + op.drop_index(op.f('ix_avalanche_fuji_labels_label'), table_name='avalanche_fuji_labels') + op.drop_index(op.f('ix_avalanche_fuji_labels_block_timestamp'), table_name='avalanche_fuji_labels') + op.drop_index(op.f('ix_avalanche_fuji_labels_block_number'), table_name='avalanche_fuji_labels') + op.drop_index('ix_avalanche_fuji_labels_address_block_timestamp', table_name='avalanche_fuji_labels') + op.drop_index('ix_avalanche_fuji_labels_address_block_number', table_name='avalanche_fuji_labels') + op.drop_index(op.f('ix_avalanche_fuji_labels_address'), table_name='avalanche_fuji_labels') + op.drop_table('avalanche_fuji_labels') + op.drop_index(op.f('ix_avalanche_fuji_blocks_timestamp'), table_name='avalanche_fuji_blocks') + op.drop_index(op.f('ix_avalanche_fuji_blocks_hash'), table_name='avalanche_fuji_blocks') + op.drop_index(op.f('ix_avalanche_fuji_blocks_block_number'), table_name='avalanche_fuji_blocks') + op.drop_table('avalanche_fuji_blocks') + op.drop_index(op.f('ix_avalanche_blocks_timestamp'), table_name='avalanche_blocks') + op.drop_index(op.f('ix_avalanche_blocks_hash'), table_name='avalanche_blocks') + op.drop_index(op.f('ix_avalanche_blocks_block_number'), table_name='avalanche_blocks') + op.drop_table('avalanche_blocks') + # ### end Alembic commands ### diff --git a/moonstreamdb/moonstreamdb/blockchain.py b/moonstreamdb/moonstreamdb/blockchain.py index af73e3eb..5e66dbf9 100644 --- a/moonstreamdb/moonstreamdb/blockchain.py +++ b/moonstreamdb/moonstreamdb/blockchain.py @@ -2,6 +2,18 @@ from enum import Enum from typing import Type, Union from .models import ( + ArbitrumNovaBlock, + ArbitrumNovaLabel, + ArbitrumNovaTransaction, + ArbitrumSepoliaBlock, + ArbitrumSepoliaLabel, + ArbitrumSepoliaTransaction, + AvalancheBlock, + AvalancheFujiBlock, + AvalancheFujiLabel, + AvalancheFujiTransaction, + AvalancheLabel, + AvalancheTransaction, EthereumBlock, EthereumLabel, EthereumTransaction, @@ -14,24 +26,21 @@ from .models import ( WyrmBlock, WyrmLabel, WyrmTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, - ZkSyncEraTestnetBlock, - ZkSyncEraTestnetLabel, - ZkSyncEraTestnetTransaction, - ZkSyncEraBlock, - ZkSyncEraLabel, - ZkSyncEraTransaction, - ArbitrumNovaBlock, - ArbitrumNovaTransaction, - ArbitrumNovaLabel, - ArbitrumSepoliaBlock, - ArbitrumSepoliaTransaction, - ArbitrumSepoliaLabel, XaiBlock, XaiLabel, XaiTransaction, + XDaiBlock, + XDaiLabel, + XDaiTransaction, + ZkSyncEraBlock, + ZkSyncEraLabel, + ZkSyncEraSepoliaBlock, + ZkSyncEraSepoliaLabel, + ZkSyncEraSepoliaTransaction, + ZkSyncEraTestnetBlock, + ZkSyncEraTestnetLabel, + ZkSyncEraTestnetTransaction, + ZkSyncEraTransaction, ) @@ -41,11 +50,14 @@ class AvailableBlockchainType(Enum): MUMBAI = "mumbai" XDAI = "xdai" WYRM = "wyrm" - ZKSYNC_ERA_TESTNET = "zksync_era_testnet" ZKSYNC_ERA = "zksync_era" + ZKSYNC_ERA_TESTNET = "zksync_era_testnet" + ZKSYNC_ERA_SEPOLIA = "zksync_era_sepolia" ARBITRUM_NOVA = "arbitrum_nova" ARBITRUM_SEPOLIA = "arbitrum_sepolia" XAI = "xai" + AVALANCHE = "avalanche" + AVALANCHE_FUJI = "avalanche_fuji" def get_block_model( @@ -59,9 +71,12 @@ def get_block_model( WyrmBlock, ZkSyncEraTestnetBlock, ZkSyncEraBlock, + ZkSyncEraSepoliaBlock, ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock, + AvalancheBlock, + AvalancheFujiBlock, ] ]: """ @@ -77,9 +92,12 @@ def get_block_model( WyrmBlock, ZkSyncEraTestnetBlock, ZkSyncEraBlock, + ZkSyncEraSepoliaBlock, ArbitrumNovaBlock, ArbitrumSepoliaBlock, XaiBlock, + AvalancheBlock, + AvalancheFujiBlock, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -96,12 +114,18 @@ def get_block_model( block_model = ZkSyncEraTestnetBlock elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: block_model = ZkSyncEraBlock + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: + block_model = ZkSyncEraSepoliaBlock elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: block_model = ArbitrumNovaBlock elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: block_model = ArbitrumSepoliaBlock elif blockchain_type == AvailableBlockchainType.XAI: block_model = XaiBlock + elif blockchain_type == AvailableBlockchainType.AVALANCHE: + block_model = AvalancheBlock + elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: + block_model = AvalancheFujiBlock else: raise Exception("Unsupported blockchain type provided") @@ -119,9 +143,12 @@ def get_label_model( WyrmLabel, ZkSyncEraTestnetLabel, ZkSyncEraLabel, + ZkSyncEraSepoliaLabel, ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel, + AvalancheLabel, + AvalancheFujiLabel, ] ]: """ @@ -137,9 +164,12 @@ def get_label_model( WyrmLabel, ZkSyncEraTestnetLabel, ZkSyncEraLabel, + ZkSyncEraSepoliaLabel, ArbitrumNovaLabel, ArbitrumSepoliaLabel, XaiLabel, + AvalancheLabel, + AvalancheFujiLabel, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -156,12 +186,19 @@ def get_label_model( label_model = ZkSyncEraTestnetLabel elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: label_model = ZkSyncEraLabel + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: + label_model = ZkSyncEraSepoliaLabel elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: label_model = ArbitrumNovaLabel elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: label_model = ArbitrumSepoliaLabel elif blockchain_type == AvailableBlockchainType.XAI: label_model = XaiLabel + elif blockchain_type == AvailableBlockchainType.AVALANCHE: + label_model = AvalancheLabel + elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: + label_model = AvalancheFujiLabel + else: raise Exception("Unsupported blockchain type provided") @@ -179,9 +216,12 @@ def get_transaction_model( WyrmTransaction, ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, + ZkSyncEraSepoliaTransaction, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, + AvalancheTransaction, + AvalancheFujiTransaction, ] ]: """ @@ -197,9 +237,12 @@ def get_transaction_model( WyrmTransaction, ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, + ZkSyncEraSepoliaTransaction, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, + AvalancheTransaction, + AvalancheFujiTransaction, ] ] if blockchain_type == AvailableBlockchainType.ETHEREUM: @@ -216,12 +259,18 @@ def get_transaction_model( transaction_model = ZkSyncEraTestnetTransaction elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: transaction_model = ZkSyncEraTransaction + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: + transaction_model = ZkSyncEraSepoliaTransaction elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: transaction_model = ArbitrumNovaTransaction elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: transaction_model = ArbitrumSepoliaTransaction elif blockchain_type == AvailableBlockchainType.XAI: transaction_model = XaiTransaction + elif blockchain_type == AvailableBlockchainType.AVALANCHE: + transaction_model = AvalancheTransaction + elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: + transaction_model = AvalancheFujiTransaction else: raise Exception("Unsupported blockchain type provided") diff --git a/moonstreamdb/moonstreamdb/models.py b/moonstreamdb/moonstreamdb/models.py index df223d35..e3de2306 100644 --- a/moonstreamdb/moonstreamdb/models.py +++ b/moonstreamdb/moonstreamdb/models.py @@ -801,6 +801,122 @@ class ZkSyncEraTestnetLabel(Base): # type: ignore ) +class ZkSyncEraSepoliaBlock(Base): # type: ignore + __tablename__ = "zksync_era_sepolia_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + mix_hash = Column(VARCHAR(256), nullable=True) + sha3_uncles = Column(VARCHAR(256), nullable=True) + + l1_batch_number = Column(BigInteger, nullable=True) + l1_batch_timestamp = Column(BigInteger, nullable=True) + + +class ZkSyncEraSepoliaTransaction(Base): # type: ignore + __tablename__ = "zksync_era_sepolia_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("zksync_era_sepolia_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + l1_batch_number = Column(BigInteger, nullable=True) + l1_batch_tx_index = Column(BigInteger, nullable=True) + + +class ZkSyncEraSepoliaLabel(Base): # type: ignore + __tablename__ = "zksync_era_sepolia_labels" + + __table_args__ = ( + Index( + "ix_zksync_era_sepolia_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_zksync_era_sepolia_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ArbitrumNovaBlock(Base): # type: ignore __tablename__ = "arbitrum_nova_blocks" @@ -870,6 +986,21 @@ class ArbitrumNovaTransaction(Base): # type: ignore class ArbitrumNovaLabel(Base): # type: ignore __tablename__ = "arbitrum_nova_labels" + __table_args__ = ( + Index( + "ix_arbitrum_nova_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_arbitrum_nova_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + id = Column( UUID(as_uuid=True), primary_key=True, @@ -970,6 +1101,21 @@ class ArbitrumSepoliaTransaction(Base): # type: ignore class ArbitrumSepoliaLabel(Base): # type: ignore __tablename__ = "arbitrum_sepolia_labels" + __table_args__ = ( + Index( + "ix_arbitrum_sepolia_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_arbitrum_sepolia_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + id = Column( UUID(as_uuid=True), primary_key=True, @@ -1101,6 +1247,232 @@ class XaiLabel(Base): # type: ignore ) +class AvalancheBlock(Base): # type: ignore + __tablename__ = "avalanche_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + mix_hash = Column(VARCHAR(256), nullable=True) + block_extra_data = Column(VARCHAR(256), nullable=True) + block_gas_cost = Column(VARCHAR(256), nullable=True) + ext_data_gas_used = Column(VARCHAR(256), nullable=True) + ext_data_hash = Column(VARCHAR(256), nullable=True) + + +class AvalancheTransaction(Base): # type: ignore + __tablename__ = "avalanche_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("avalanche_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + +class AvalancheLabel(Base): # type: ignore + __tablename__ = "avalanche_labels" + + __table_args__ = ( + Index( + "ix_avalanche_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_avalanche_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + +class AvalancheFujiBlock(Base): # type: ignore + __tablename__ = "avalanche_fuji_blocks" + + block_number = Column( + BigInteger, primary_key=True, unique=True, nullable=False, index=True + ) + difficulty = Column(BigInteger) + extra_data = Column(VARCHAR(128)) + gas_limit = Column(BigInteger) + gas_used = Column(BigInteger) + base_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + hash = Column(VARCHAR(256), index=True) + logs_bloom = Column(VARCHAR(1024)) + miner = Column(VARCHAR(256)) + nonce = Column(VARCHAR(256)) + parent_hash = Column(VARCHAR(256)) + receipt_root = Column(VARCHAR(256)) + uncles = Column(VARCHAR(256)) + size = Column(Integer) + state_root = Column(VARCHAR(256)) + timestamp = Column(BigInteger, index=True) + total_difficulty = Column(VARCHAR(256)) + transactions_root = Column(VARCHAR(256)) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + mix_hash = Column(VARCHAR(256), nullable=True) + block_extra_data = Column(VARCHAR(256), nullable=True) + block_gas_cost = Column(VARCHAR(256), nullable=True) + ext_data_gas_used = Column(VARCHAR(256), nullable=True) + ext_data_hash = Column(VARCHAR(256), nullable=True) + + +class AvalancheFujiTransaction(Base): # type: ignore + __tablename__ = "avalanche_fuji_transactions" + + hash = Column( + VARCHAR(256), primary_key=True, unique=True, nullable=False, index=True + ) + block_number = Column( + BigInteger, + ForeignKey("avalanche_fuji_blocks.block_number", ondelete="CASCADE"), + nullable=False, + index=True, + ) + from_address = Column(VARCHAR(256), index=True) + to_address = Column(VARCHAR(256), index=True) + gas = Column(Numeric(precision=78, scale=0), index=True) + gas_price = Column(Numeric(precision=78, scale=0), index=True) + max_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + max_priority_fee_per_gas = Column(Numeric(precision=78, scale=0), nullable=True) + input = Column(Text) + nonce = Column(VARCHAR(256)) + transaction_index = Column(BigInteger) + transaction_type = Column(Integer, nullable=True) + value = Column(Numeric(precision=78, scale=0), index=True) + + indexed_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + +class AvalancheFujiLabel(Base): # type: ignore + __tablename__ = "avalanche_fuji_labels" + + __table_args__ = ( + Index( + "ix_avalanche_fuji_labels_address_block_number", + "address", + "block_number", + unique=False, + ), + Index( + "ix_avalanche_fuji_labels_address_block_timestamp", + "address", + "block_timestamp", + unique=False, + ), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + unique=True, + nullable=False, + ) + label = Column(VARCHAR(256), nullable=False, index=True) + block_number = Column( + BigInteger, + nullable=True, + index=True, + ) + address = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + transaction_hash = Column( + VARCHAR(256), + nullable=True, + index=True, + ) + label_data = Column(JSONB, nullable=True) + block_timestamp = Column(BigInteger, index=True) + log_index = Column(Integer, nullable=True) + created_at = Column( + DateTime(timezone=True), server_default=utcnow(), nullable=False + ) + + class ESDFunctionSignature(Base): # type: ignore """ Function signature from blockchain (Ethereum/Polygon) Signature Database. diff --git a/moonstreamdb/moonstreamdb/networks.py b/moonstreamdb/moonstreamdb/networks.py index 30256ca5..361ab6ce 100644 --- a/moonstreamdb/moonstreamdb/networks.py +++ b/moonstreamdb/moonstreamdb/networks.py @@ -2,6 +2,18 @@ from enum import Enum from typing import Dict, Union from .models import ( + ArbitrumNovaBlock, + ArbitrumNovaLabel, + ArbitrumNovaTransaction, + ArbitrumSepoliaBlock, + ArbitrumSepoliaLabel, + ArbitrumSepoliaTransaction, + AvalancheBlock, + AvalancheFujiBlock, + AvalancheFujiLabel, + AvalancheFujiTransaction, + AvalancheLabel, + AvalancheTransaction, Base, EthereumBlock, EthereumLabel, @@ -15,24 +27,21 @@ from .models import ( WyrmBlock, WyrmLabel, WyrmTransaction, - XDaiBlock, - XDaiLabel, - XDaiTransaction, - ZkSyncEraTestnetBlock, - ZkSyncEraTestnetLabel, - ZkSyncEraTestnetTransaction, - ZkSyncEraBlock, - ZkSyncEraLabel, - ZkSyncEraTransaction, - ArbitrumNovaBlock, - ArbitrumNovaTransaction, - ArbitrumNovaLabel, - ArbitrumSepoliaBlock, - ArbitrumSepoliaTransaction, - ArbitrumSepoliaLabel, XaiBlock, XaiLabel, XaiTransaction, + XDaiBlock, + XDaiLabel, + XDaiTransaction, + ZkSyncEraBlock, + ZkSyncEraLabel, + ZkSyncEraSepoliaBlock, + ZkSyncEraSepoliaLabel, + ZkSyncEraSepoliaTransaction, + ZkSyncEraTestnetBlock, + ZkSyncEraTestnetLabel, + ZkSyncEraTestnetTransaction, + ZkSyncEraTransaction, ) @@ -44,9 +53,12 @@ class Network(Enum): wyrm = "wyrm" zksync_era_testnet = "zksync_era_testnet" zksync_era = "zksync_era" + zksync_era_sepolia = "zksync_era_sepolia" arbitrum_nova = "arbitrum_nova" arbitrum_sepolia = "arbitrum_sepolia" xai = "xai" + avalanche = "avalanche" + avalanche_fuji = "avalanche_fuji" tx_raw_types = Union[ @@ -57,9 +69,12 @@ tx_raw_types = Union[ XDaiTransaction, ZkSyncEraTestnetTransaction, ZkSyncEraTransaction, + ZkSyncEraSepoliaTransaction, ArbitrumNovaTransaction, ArbitrumSepoliaTransaction, XaiTransaction, + AvalancheTransaction, + AvalancheFujiTransaction, ] MODELS: Dict[Network, Dict[str, Base]] = { @@ -94,6 +109,11 @@ MODELS: Dict[Network, Dict[str, Base]] = { "transactions": ZkSyncEraTestnetTransaction, }, Network.zksync_era: { + "blocks": ZkSyncEraSepoliaBlock, + "labels": ZkSyncEraSepoliaLabel, + "transactions": ZkSyncEraSepoliaTransaction, + }, + Network.zksync_era_sepolia: { "blocks": ZkSyncEraBlock, "labels": ZkSyncEraLabel, "transactions": ZkSyncEraTransaction, @@ -113,4 +133,14 @@ MODELS: Dict[Network, Dict[str, Base]] = { "labels": XaiLabel, "transactions": XaiTransaction, }, + Network.avalanche: { + "blocks": AvalancheBlock, + "labels": AvalancheLabel, + "transactions": AvalancheTransaction, + }, + Network.avalanche_fuji: { + "blocks": AvalancheFujiBlock, + "labels": AvalancheFujiLabel, + "transactions": AvalancheFujiTransaction, + }, } diff --git a/moonstreamdb/moonstreamdb/version.py b/moonstreamdb/moonstreamdb/version.py index 73a7e3a0..0facaebf 100644 --- a/moonstreamdb/moonstreamdb/version.py +++ b/moonstreamdb/moonstreamdb/version.py @@ -2,4 +2,4 @@ Moonstream database version. """ -MOONSTREAMDB_VERSION = "0.3.8" +MOONSTREAMDB_VERSION = "0.3.9" From fe6bba86d2de2bbd8753b27a1734a47de7d9dc0a Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Mar 2024 13:37:45 +0000 Subject: [PATCH 10/12] Switch from access-id to web3-uri, zksync sepolia and avalanche --- crawlers/deploy/arbitrum-nova-missing.service | 2 +- .../arbitrum-nova-moonworm-crawler.service | 2 +- .../deploy/arbitrum-nova-synchronize.service | 2 +- .../deploy/arbitrum-sepolia-missing.service | 2 +- .../arbitrum-sepolia-moonworm-crawler.service | 2 +- .../arbitrum-sepolia-synchronize.service | 2 +- crawlers/deploy/deploy.bash | 32 +++-- .../ethereum-historical-crawl-events.service | 2 +- ...reum-historical-crawl-transactions.service | 2 +- crawlers/deploy/ethereum-missing.service | 2 +- .../deploy/ethereum-moonworm-crawler.service | 2 +- crawlers/deploy/ethereum-state-clean.service | 2 +- crawlers/deploy/ethereum-state.service | 2 +- crawlers/deploy/ethereum-synchronize.service | 2 +- crawlers/deploy/ethereum-trending.timer | 9 -- .../mumbai-historical-crawl-events.service | 2 +- ...mbai-historical-crawl-transactions.service | 2 +- crawlers/deploy/mumbai-missing.service | 2 +- .../deploy/mumbai-moonworm-crawler.service | 2 +- crawlers/deploy/mumbai-state-clean.service | 2 +- crawlers/deploy/mumbai-state.service | 2 +- crawlers/deploy/mumbai-synchronize.service | 2 +- .../polygon-historical-crawl-events.service | 2 +- ...ygon-historical-crawl-transactions.service | 2 +- crawlers/deploy/polygon-missing.service | 2 +- .../deploy/polygon-moonworm-crawler.service | 2 +- crawlers/deploy/polygon-state-clean.service | 2 +- crawlers/deploy/polygon-state.service | 2 +- crawlers/deploy/polygon-synchronize.service | 2 +- crawlers/deploy/xai-missing.service | 2 +- crawlers/deploy/xai-moonworm-crawler.service | 2 +- crawlers/deploy/xai-synchronize.service | 2 +- .../xdai-historical-crawl-events.service | 2 +- ...xdai-historical-crawl-transactions.service | 2 +- crawlers/deploy/xdai-missing.service | 2 +- crawlers/deploy/xdai-moonworm-crawler.service | 2 +- crawlers/deploy/xdai-synchronize.service | 2 +- ...zksync-era-historical-crawl-events.service | 2 +- ...-era-historical-crawl-transactions.service | 2 +- crawlers/deploy/zksync-era-missing.service | 2 +- .../zksync-era-moonworm-crawler.service | 2 +- ...ice => zksync-era-sepolia-missing.service} | 8 +- .../deploy/zksync-era-sepolia-missing.timer | 9 ++ .../zksync-era-sepolia-synchronize.service | 17 +++ .../deploy/zksync-era-state-clean.service | 2 +- crawlers/deploy/zksync-era-state.service | 2 +- .../deploy/zksync-era-synchronize.service | 2 +- ...ra-testnet-historical-crawl-events.service | 2 +- ...tnet-historical-crawl-transactions.service | 2 +- .../deploy/zksync-era-testnet-missing.service | 2 +- ...ksync-era-testnet-moonworm-crawler.service | 2 +- .../zksync-era-testnet-synchronize.service | 2 +- crawlers/mooncrawl/mooncrawl/actions.py | 28 ++--- crawlers/mooncrawl/mooncrawl/api.py | 3 +- crawlers/mooncrawl/mooncrawl/blockchain.py | 118 +++++++++--------- crawlers/mooncrawl/mooncrawl/contract/cli.py | 9 +- crawlers/mooncrawl/mooncrawl/crawler.py | 23 ++-- .../mooncrawl/generic_crawler/base.py | 11 +- .../mooncrawl/generic_crawler/cli.py | 13 +- .../mooncrawl/leaderboards_generator/cli.py | 8 +- .../mooncrawl/leaderboards_generator/utils.py | 4 +- .../mooncrawl/metadata_crawler/cli.py | 5 +- .../mooncrawl/metadata_crawler/db.py | 6 +- .../mooncrawl/moonworm_crawler/cli.py | 23 ++-- .../moonworm_crawler/continuous_crawler.py | 6 +- .../mooncrawl/moonworm_crawler/crawler.py | 4 +- .../mooncrawl/moonworm_crawler/db.py | 4 +- .../moonworm_crawler/event_crawler.py | 5 +- .../moonworm_crawler/historical_crawler.py | 6 +- .../mooncrawl/reports_crawler/cli.py | 12 +- crawlers/mooncrawl/mooncrawl/settings.py | 99 ++++++++------- .../mooncrawl/mooncrawl/state_crawler/cli.py | 43 +++---- .../mooncrawl/mooncrawl/state_crawler/db.py | 4 +- .../mooncrawl/state_crawler/web3_util.py | 26 +--- .../mooncrawl/stats_worker/dashboard.py | 25 ++-- .../mooncrawl/stats_worker/queries.py | 6 +- crawlers/mooncrawl/sample.env | 22 ++-- crawlers/mooncrawl/setup.py | 2 +- 78 files changed, 332 insertions(+), 352 deletions(-) delete mode 100644 crawlers/deploy/ethereum-trending.timer rename crawlers/deploy/{ethereum-trending.service => zksync-era-sepolia-missing.service} (57%) create mode 100644 crawlers/deploy/zksync-era-sepolia-missing.timer create mode 100644 crawlers/deploy/zksync-era-sepolia-synchronize.service diff --git a/crawlers/deploy/arbitrum-nova-missing.service b/crawlers/deploy/arbitrum-nova-missing.service index c2679270..78208ce2 100644 --- a/crawlers/deploy/arbitrum-nova-missing.service +++ b/crawlers/deploy/arbitrum-nova-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_nova -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_nova -n CPUWeight=50 SyslogIdentifier=arbitrum-nova-missing \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-nova-moonworm-crawler.service b/crawlers/deploy/arbitrum-nova-moonworm-crawler.service index 73accbd8..ef1c2bcf 100644 --- a/crawlers/deploy/arbitrum-nova-moonworm-crawler.service +++ b/crawlers/deploy/arbitrum-nova-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20 CPUWeight=70 SyslogIdentifier=arbitrum-nova-moonworm-crawler diff --git a/crawlers/deploy/arbitrum-nova-synchronize.service b/crawlers/deploy/arbitrum-nova-synchronize.service index fe569eb7..c520731a 100644 --- a/crawlers/deploy/arbitrum-nova-synchronize.service +++ b/crawlers/deploy/arbitrum-nova-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_nova -c 10 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_nova -c 10 -j 2 CPUWeight=90 SyslogIdentifier=arbitrum-nova-synchronize diff --git a/crawlers/deploy/arbitrum-sepolia-missing.service b/crawlers/deploy/arbitrum-sepolia-missing.service index da67801a..a2819a7b 100644 --- a/crawlers/deploy/arbitrum-sepolia-missing.service +++ b/crawlers/deploy/arbitrum-sepolia-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_sepolia -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_sepolia -n CPUWeight=50 SyslogIdentifier=arbitrum-sepolia-missing \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service b/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service index 09c3b447..921b8897 100644 --- a/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service +++ b/crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20 CPUWeight=70 SyslogIdentifier=arbitrum-sepolia-moonworm-crawler diff --git a/crawlers/deploy/arbitrum-sepolia-synchronize.service b/crawlers/deploy/arbitrum-sepolia-synchronize.service index 512fe082..0d5f173b 100644 --- a/crawlers/deploy/arbitrum-sepolia-synchronize.service +++ b/crawlers/deploy/arbitrum-sepolia-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2 CPUWeight=90 SyslogIdentifier=arbitrum-sepolia-synchronize diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index b5f977c2..11be0b99 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -31,8 +31,6 @@ LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer" # Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service" -ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service" -ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer" ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service" ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer" ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service" @@ -106,6 +104,9 @@ ZKSYNC_ERA_STATE_SERVICE_FILE="zksync-era-state.service" ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer" ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service" ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer" +ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE="zksync-era-sepolia-synchronize.service" +ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE="zksync-era-sepolia-missing.service" +ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE="zksync-era-sepolia-missing.timer" # ZkSync Era testnet ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service" @@ -190,15 +191,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}" "/home/ubuntu/.config/sy XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}" -echo -echo -echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE_FILE}, ${ETHEREUM_TRENDING_TIMER_FILE}" -chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" -cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TRENDING_SERVICE_FILE}" -cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TRENDING_TIMER_FILE}" -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}" @@ -582,6 +574,24 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/syste XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" +# ZkSync Era Sepolia +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era block with transactions syncronizer service definition with ${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era missing service and timer with: ${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" + # Arbitrum Nova echo echo diff --git a/crawlers/deploy/ethereum-historical-crawl-events.service b/crawlers/deploy/ethereum-historical-crawl-events.service index 881d21b2..71bf692b 100644 --- a/crawlers/deploy/ethereum-historical-crawl-events.service +++ b/crawlers/deploy/ethereum-historical-crawl-events.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 SyslogIdentifier=ethereum-historical-crawl-events diff --git a/crawlers/deploy/ethereum-historical-crawl-transactions.service b/crawlers/deploy/ethereum-historical-crawl-transactions.service index 25e11793..22d0db8e 100644 --- a/crawlers/deploy/ethereum-historical-crawl-transactions.service +++ b/crawlers/deploy/ethereum-historical-crawl-transactions.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 SyslogIdentifier=ethereum-historical-crawl-transactions diff --git a/crawlers/deploy/ethereum-missing.service b/crawlers/deploy/ethereum-missing.service index 24d097d1..dda039ae 100644 --- a/crawlers/deploy/ethereum-missing.service +++ b/crawlers/deploy/ethereum-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain ethereum -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain ethereum -n CPUWeight=50 SyslogIdentifier=ethereum-missing diff --git a/crawlers/deploy/ethereum-moonworm-crawler.service b/crawlers/deploy/ethereum-moonworm-crawler.service index 4dd189ef..e2e6a4b2 100644 --- a/crawlers/deploy/ethereum-moonworm-crawler.service +++ b/crawlers/deploy/ethereum-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b ethereum +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b ethereum CPUWeight=70 SyslogIdentifier=ethereum-moonworm-crawler diff --git a/crawlers/deploy/ethereum-state-clean.service b/crawlers/deploy/ethereum-state-clean.service index 6a7c179b..2741f05f 100644 --- a/crawlers/deploy/ethereum-state-clean.service +++ b/crawlers/deploy/ethereum-state-clean.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain ethereum -N 10000 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain ethereum -N 10000 CPUWeight=60 SyslogIdentifier=ethereum-state-clean \ No newline at end of file diff --git a/crawlers/deploy/ethereum-state.service b/crawlers/deploy/ethereum-state.service index 6a68961a..1ff7f39e 100644 --- a/crawlers/deploy/ethereum-state.service +++ b/crawlers/deploy/ethereum-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain ethereum --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain ethereum --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json CPUWeight=60 SyslogIdentifier=ethereum-state \ No newline at end of file diff --git a/crawlers/deploy/ethereum-synchronize.service b/crawlers/deploy/ethereum-synchronize.service index 8b199a63..f335d5be 100644 --- a/crawlers/deploy/ethereum-synchronize.service +++ b/crawlers/deploy/ethereum-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain ethereum -c 6 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain ethereum -c 6 -j 2 CPUWeight=90 SyslogIdentifier=ethereum-synchronize diff --git a/crawlers/deploy/ethereum-trending.timer b/crawlers/deploy/ethereum-trending.timer deleted file mode 100644 index 6dd8c93c..00000000 --- a/crawlers/deploy/ethereum-trending.timer +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Load trending Ethereum addresses to the database every 5 minutes - -[Timer] -OnBootSec=60s -OnUnitActiveSec=5m - -[Install] -WantedBy=timers.target diff --git a/crawlers/deploy/mumbai-historical-crawl-events.service b/crawlers/deploy/mumbai-historical-crawl-events.service index cc56464e..6922c827 100644 --- a/crawlers/deploy/mumbai-historical-crawl-events.service +++ b/crawlers/deploy/mumbai-historical-crawl-events.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 SyslogIdentifier=mumbai-historical-crawl-events diff --git a/crawlers/deploy/mumbai-historical-crawl-transactions.service b/crawlers/deploy/mumbai-historical-crawl-transactions.service index 9a88ac2f..7f73450e 100644 --- a/crawlers/deploy/mumbai-historical-crawl-transactions.service +++ b/crawlers/deploy/mumbai-historical-crawl-transactions.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 SyslogIdentifier=mumbai-historical-crawl-transactions diff --git a/crawlers/deploy/mumbai-missing.service b/crawlers/deploy/mumbai-missing.service index 89f8facd..4e2cc4fb 100644 --- a/crawlers/deploy/mumbai-missing.service +++ b/crawlers/deploy/mumbai-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain mumbai -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain mumbai -n CPUWeight=50 SyslogIdentifier=mumbai-missing \ No newline at end of file diff --git a/crawlers/deploy/mumbai-moonworm-crawler.service b/crawlers/deploy/mumbai-moonworm-crawler.service index 971f5ca3..3aa714ac 100644 --- a/crawlers/deploy/mumbai-moonworm-crawler.service +++ b/crawlers/deploy/mumbai-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b mumbai --confirmations 40 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b mumbai --confirmations 40 CPUWeight=70 SyslogIdentifier=mumbai-moonworm-crawler diff --git a/crawlers/deploy/mumbai-state-clean.service b/crawlers/deploy/mumbai-state-clean.service index 6e2be23e..cce345cb 100644 --- a/crawlers/deploy/mumbai-state-clean.service +++ b/crawlers/deploy/mumbai-state-clean.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain mumbai -N 10000 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain mumbai -N 10000 CPUWeight=60 SyslogIdentifier=mumbai-state-clean \ No newline at end of file diff --git a/crawlers/deploy/mumbai-state.service b/crawlers/deploy/mumbai-state.service index c27b8ca8..51e573b3 100644 --- a/crawlers/deploy/mumbai-state.service +++ b/crawlers/deploy/mumbai-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json CPUWeight=60 SyslogIdentifier=mumbai-state \ No newline at end of file diff --git a/crawlers/deploy/mumbai-synchronize.service b/crawlers/deploy/mumbai-synchronize.service index 2876316d..1486fcc5 100644 --- a/crawlers/deploy/mumbai-synchronize.service +++ b/crawlers/deploy/mumbai-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain mumbai -c 20 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain mumbai -c 20 -j 2 CPUWeight=90 SyslogIdentifier=mumbai-synchronize diff --git a/crawlers/deploy/polygon-historical-crawl-events.service b/crawlers/deploy/polygon-historical-crawl-events.service index 965e6747..49280be2 100644 --- a/crawlers/deploy/polygon-historical-crawl-events.service +++ b/crawlers/deploy/polygon-historical-crawl-events.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 SyslogIdentifier=polygon-historical-crawl-events diff --git a/crawlers/deploy/polygon-historical-crawl-transactions.service b/crawlers/deploy/polygon-historical-crawl-transactions.service index 36639e3e..1b147834 100644 --- a/crawlers/deploy/polygon-historical-crawl-transactions.service +++ b/crawlers/deploy/polygon-historical-crawl-transactions.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 SyslogIdentifier=polygon-historical-crawl-transactions diff --git a/crawlers/deploy/polygon-missing.service b/crawlers/deploy/polygon-missing.service index 3dd60cdd..93425b70 100644 --- a/crawlers/deploy/polygon-missing.service +++ b/crawlers/deploy/polygon-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain polygon -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain polygon -n CPUWeight=50 SyslogIdentifier=polygon-missing \ No newline at end of file diff --git a/crawlers/deploy/polygon-moonworm-crawler.service b/crawlers/deploy/polygon-moonworm-crawler.service index 85c9443c..6a5fe9a7 100644 --- a/crawlers/deploy/polygon-moonworm-crawler.service +++ b/crawlers/deploy/polygon-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b polygon --confirmations 50 --min-blocks-batch 20 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b polygon --confirmations 50 --min-blocks-batch 20 CPUWeight=70 SyslogIdentifier=polygon-moonworm-crawler diff --git a/crawlers/deploy/polygon-state-clean.service b/crawlers/deploy/polygon-state-clean.service index 25d581c8..ef5c0975 100644 --- a/crawlers/deploy/polygon-state-clean.service +++ b/crawlers/deploy/polygon-state-clean.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain polygon -N 10000 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain polygon -N 10000 CPUWeight=60 SyslogIdentifier=polygon-state-clean \ No newline at end of file diff --git a/crawlers/deploy/polygon-state.service b/crawlers/deploy/polygon-state.service index 9d307971..38714030 100644 --- a/crawlers/deploy/polygon-state.service +++ b/crawlers/deploy/polygon-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json CPUWeight=60 SyslogIdentifier=polygon-state \ No newline at end of file diff --git a/crawlers/deploy/polygon-synchronize.service b/crawlers/deploy/polygon-synchronize.service index 052011a7..35588b31 100644 --- a/crawlers/deploy/polygon-synchronize.service +++ b/crawlers/deploy/polygon-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain polygon --confirmations 40 --jobs 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain polygon --confirmations 40 --jobs 2 CPUWeight=90 SyslogIdentifier=polygon-synchronize diff --git a/crawlers/deploy/xai-missing.service b/crawlers/deploy/xai-missing.service index 8aee0f34..cfd5d9c3 100644 --- a/crawlers/deploy/xai-missing.service +++ b/crawlers/deploy/xai-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xai -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain xai -n CPUWeight=50 SyslogIdentifier=xai-missing \ No newline at end of file diff --git a/crawlers/deploy/xai-moonworm-crawler.service b/crawlers/deploy/xai-moonworm-crawler.service index 95d49d12..b7006828 100644 --- a/crawlers/deploy/xai-moonworm-crawler.service +++ b/crawlers/deploy/xai-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xai --confirmations 10 --min-blocks-batch 20 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b xai --confirmations 10 --min-blocks-batch 20 CPUWeight=70 SyslogIdentifier=xai-moonworm-crawler diff --git a/crawlers/deploy/xai-synchronize.service b/crawlers/deploy/xai-synchronize.service index 91c4eb70..735ea113 100644 --- a/crawlers/deploy/xai-synchronize.service +++ b/crawlers/deploy/xai-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xai -c 10 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain xai -c 10 -j 2 CPUWeight=90 SyslogIdentifier=xai-synchronize diff --git a/crawlers/deploy/xdai-historical-crawl-events.service b/crawlers/deploy/xdai-historical-crawl-events.service index 66d56405..bc2654a0 100644 --- a/crawlers/deploy/xdai-historical-crawl-events.service +++ b/crawlers/deploy/xdai-historical-crawl-events.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 SyslogIdentifier=xdai-historical-crawl-events diff --git a/crawlers/deploy/xdai-historical-crawl-transactions.service b/crawlers/deploy/xdai-historical-crawl-transactions.service index 20a3c616..f03ca8f8 100644 --- a/crawlers/deploy/xdai-historical-crawl-transactions.service +++ b/crawlers/deploy/xdai-historical-crawl-transactions.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 SyslogIdentifier=xdai-historical-crawl-transactions diff --git a/crawlers/deploy/xdai-missing.service b/crawlers/deploy/xdai-missing.service index bbe55660..2e11c122 100644 --- a/crawlers/deploy/xdai-missing.service +++ b/crawlers/deploy/xdai-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xdai -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain xdai -n CPUWeight=50 SyslogIdentifier=xdai-missing \ No newline at end of file diff --git a/crawlers/deploy/xdai-moonworm-crawler.service b/crawlers/deploy/xdai-moonworm-crawler.service index bc24cf71..00672782 100644 --- a/crawlers/deploy/xdai-moonworm-crawler.service +++ b/crawlers/deploy/xdai-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xdai +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b xdai CPUWeight=70 SyslogIdentifier=xdai-moonworm-crawler diff --git a/crawlers/deploy/xdai-synchronize.service b/crawlers/deploy/xdai-synchronize.service index 1702aada..1bd64feb 100644 --- a/crawlers/deploy/xdai-synchronize.service +++ b/crawlers/deploy/xdai-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xdai -c 60 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain xdai -c 60 -j 2 CPUWeight=90 SyslogIdentifier=xdai-synchronize diff --git a/crawlers/deploy/zksync-era-historical-crawl-events.service b/crawlers/deploy/zksync-era-historical-crawl-events.service index 992b5d5c..5f5736ae 100644 --- a/crawlers/deploy/zksync-era-historical-crawl-events.service +++ b/crawlers/deploy/zksync-era-historical-crawl-events.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 SyslogIdentifier=zksync-era-historical-crawl-events diff --git a/crawlers/deploy/zksync-era-historical-crawl-transactions.service b/crawlers/deploy/zksync-era-historical-crawl-transactions.service index 65218e90..73dfc919 100644 --- a/crawlers/deploy/zksync-era-historical-crawl-transactions.service +++ b/crawlers/deploy/zksync-era-historical-crawl-transactions.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 SyslogIdentifier=zksync-era-historical-crawl-transactions diff --git a/crawlers/deploy/zksync-era-missing.service b/crawlers/deploy/zksync-era-missing.service index 9bd08748..7a0bd88d 100644 --- a/crawlers/deploy/zksync-era-missing.service +++ b/crawlers/deploy/zksync-era-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era -n CPUWeight=50 SyslogIdentifier=zksync-era-missing \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-moonworm-crawler.service b/crawlers/deploy/zksync-era-moonworm-crawler.service index 24e4497d..c2da8976 100644 --- a/crawlers/deploy/zksync-era-moonworm-crawler.service +++ b/crawlers/deploy/zksync-era-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era --confirmations 20 --min-blocks-batch 20 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era --confirmations 20 --min-blocks-batch 20 CPUWeight=70 SyslogIdentifier=zksync-era-moonworm-crawler diff --git a/crawlers/deploy/ethereum-trending.service b/crawlers/deploy/zksync-era-sepolia-missing.service similarity index 57% rename from crawlers/deploy/ethereum-trending.service rename to crawlers/deploy/zksync-era-sepolia-missing.service index b482c509..ef09bc3a 100644 --- a/crawlers/deploy/ethereum-trending.service +++ b/crawlers/deploy/zksync-era-sepolia-missing.service @@ -1,11 +1,11 @@ [Unit] -Description=Load trending Ethereum addresses to the database +Description=Fill missing blocks at ZkSync Era Sepolia database After=network.target [Service] Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" trending -CPUWeight=30 -SyslogIdentifier=ethereum-trending \ No newline at end of file +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_sepolia -n +CPUWeight=50 +SyslogIdentifier=zksync-era-sepolia-missing \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-sepolia-missing.timer b/crawlers/deploy/zksync-era-sepolia-missing.timer new file mode 100644 index 00000000..5708282a --- /dev/null +++ b/crawlers/deploy/zksync-era-sepolia-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at ZkSync Era Sepolia database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-sepolia-synchronize.service b/crawlers/deploy/zksync-era-sepolia-synchronize.service new file mode 100644 index 00000000..67d2c3c8 --- /dev/null +++ b/crawlers/deploy/zksync-era-sepolia-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_sepolia -c 20 -j 2 +CPUWeight=90 +SyslogIdentifier=zksync-era-sepolia-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state-clean.service b/crawlers/deploy/zksync-era-state-clean.service index 4cd68f25..a8174424 100644 --- a/crawlers/deploy/zksync-era-state-clean.service +++ b/crawlers/deploy/zksync-era-state-clean.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync_era -N 10000 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain zksync_era -N 10000 CPUWeight=60 SyslogIdentifier=zksync-era-state-clean \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-state.service b/crawlers/deploy/zksync-era-state.service index e92a6ccc..f205fb33 100644 --- a/crawlers/deploy/zksync-era-state.service +++ b/crawlers/deploy/zksync-era-state.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json CPUWeight=60 SyslogIdentifier=zksync-era-state \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-synchronize.service b/crawlers/deploy/zksync-era-synchronize.service index f180b7c4..5fcd128a 100644 --- a/crawlers/deploy/zksync-era-synchronize.service +++ b/crawlers/deploy/zksync-era-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era -c 20 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era -c 20 -j 2 CPUWeight=90 SyslogIdentifier=zksync-era-synchronize diff --git a/crawlers/deploy/zksync-era-testnet-historical-crawl-events.service b/crawlers/deploy/zksync-era-testnet-historical-crawl-events.service index 97dca57b..d905bcb8 100644 --- a/crawlers/deploy/zksync-era-testnet-historical-crawl-events.service +++ b/crawlers/deploy/zksync-era-testnet-historical-crawl-events.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-events +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-events CPUWeight=70 SyslogIdentifier=zksync-era-testnet-historical-crawl-events diff --git a/crawlers/deploy/zksync-era-testnet-historical-crawl-transactions.service b/crawlers/deploy/zksync-era-testnet-historical-crawl-transactions.service index 111fccc8..4286c408 100644 --- a/crawlers/deploy/zksync-era-testnet-historical-crawl-transactions.service +++ b/crawlers/deploy/zksync-era-testnet-historical-crawl-transactions.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-functions +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-functions CPUWeight=70 SyslogIdentifier=zksync-era-testnet-historical-crawl-transactions diff --git a/crawlers/deploy/zksync-era-testnet-missing.service b/crawlers/deploy/zksync-era-testnet-missing.service index 0a1a26da..1b49ec56 100644 --- a/crawlers/deploy/zksync-era-testnet-missing.service +++ b/crawlers/deploy/zksync-era-testnet-missing.service @@ -6,6 +6,6 @@ After=network.target Type=oneshot WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_testnet -n CPUWeight=50 SyslogIdentifier=zksync-era-testnet-missing \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service b/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service index d75f9d99..a825a2ee 100644 --- a/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service +++ b/crawlers/deploy/zksync-era-testnet-moonworm-crawler.service @@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env Restart=on-failure RestartSec=15s -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era_testnet --confirmations 20 --min-blocks-batch 20 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_testnet --confirmations 20 --min-blocks-batch 20 CPUWeight=70 SyslogIdentifier=zksync-era-testnet-moonworm-crawler diff --git a/crawlers/deploy/zksync-era-testnet-synchronize.service b/crawlers/deploy/zksync-era-testnet-synchronize.service index ad9822d3..348ba1e0 100644 --- a/crawlers/deploy/zksync-era-testnet-synchronize.service +++ b/crawlers/deploy/zksync-era-testnet-synchronize.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env -ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2 +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2 CPUWeight=90 SyslogIdentifier=zksync-era-testnet-synchronize diff --git a/crawlers/mooncrawl/mooncrawl/actions.py b/crawlers/mooncrawl/mooncrawl/actions.py index aaa05693..05c265f5 100644 --- a/crawlers/mooncrawl/mooncrawl/actions.py +++ b/crawlers/mooncrawl/mooncrawl/actions.py @@ -1,24 +1,24 @@ -from collections import OrderedDict -from datetime import datetime import hashlib import json import logging import time -from typing import Any, Dict, Optional, Union import uuid - - -from bugout.data import ( - BugoutResources, -) -from bugout.exceptions import BugoutResponseException -from moonstream.client import Moonstream, ENDPOINT_QUERIES, MoonstreamQueryResultUrl # type: ignore -import requests # type: ignore -from .middleware import MoonstreamHTTPException -from .settings import bugout_client as bc - +from collections import OrderedDict +from datetime import datetime +from typing import Any, Dict, Optional, Union import boto3 # type: ignore +import requests # type: ignore +from bugout.data import BugoutResources +from bugout.exceptions import BugoutResponseException +from moonstream.client import ( # type: ignore + ENDPOINT_QUERIES, + Moonstream, + MoonstreamQueryResultUrl, +) + +from .middleware import MoonstreamHTTPException +from .settings import bugout_client as bc logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/api.py b/crawlers/mooncrawl/mooncrawl/api.py index 03f6b102..54b8715d 100644 --- a/crawlers/mooncrawl/mooncrawl/api.py +++ b/crawlers/mooncrawl/mooncrawl/api.py @@ -1,6 +1,7 @@ """ The Mooncrawl HTTP API """ + import logging import time from cgi import test @@ -37,7 +38,6 @@ from .settings import ( MOONSTREAM_S3_QUERIES_BUCKET_PREFIX, MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, - NB_CONTROLLER_ACCESS_ID, ORIGINS, ) from .settings import bugout_client as bc @@ -152,7 +152,6 @@ async def status_handler( timescales=stats_update.timescales, dashboard=dashboard_resource, subscription_by_id=subscription_by_id, - access_id=NB_CONTROLLER_ACCESS_ID, ) except Exception as e: diff --git a/crawlers/mooncrawl/mooncrawl/blockchain.py b/crawlers/mooncrawl/mooncrawl/blockchain.py index 0e1d5df3..7411b41e 100644 --- a/crawlers/mooncrawl/mooncrawl/blockchain.py +++ b/crawlers/mooncrawl/mooncrawl/blockchain.py @@ -21,19 +21,19 @@ from web3.types import BlockData from .data import DateRange from .db import yield_db_session, yield_db_session_ctx from .settings import ( - MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI, - MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI, MOONSTREAM_CRAWL_WORKERS, - MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI, - MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI, - MOONSTREAM_POLYGON_WEB3_PROVIDER_URI, - MOONSTREAM_WYRM_WEB3_PROVIDER_URI, - MOONSTREAM_XAI_WEB3_PROVIDER_URI, - MOONSTREAM_XDAI_WEB3_PROVIDER_URI, - MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI, - MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI, - NB_ACCESS_ID_HEADER, - NB_DATA_SOURCE_HEADER, + MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI, + MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI, + MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI, + MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI, + MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI, + MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI, + MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI, + MOONSTREAM_NODE_XAI_A_EXTERNAL_URI, + MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI, + MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI, + MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI, + MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI, WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS, ) @@ -48,42 +48,40 @@ class BlockCrawlError(Exception): def connect( - blockchain_type: AvailableBlockchainType, + blockchain_type: Optional[AvailableBlockchainType] = None, web3_uri: Optional[str] = None, - access_id: Optional[UUID] = None, ) -> Web3: + if blockchain_type is None and web3_uri is None: + raise Exception("Both blockchain_type and web3_uri could not be None") + web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider() - request_kwargs: Any = None - if access_id is not None: - request_kwargs = { - "headers": { - NB_ACCESS_ID_HEADER: str(access_id), - NB_DATA_SOURCE_HEADER: "blockchain", - "Content-Type": "application/json", - } - } + request_kwargs: Dict[str, Any] = {"headers": {"Content-Type": "application/json"}} if web3_uri is None: if blockchain_type == AvailableBlockchainType.ETHEREUM: - web3_uri = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.POLYGON: - web3_uri = MOONSTREAM_POLYGON_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.MUMBAI: - web3_uri = MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.XDAI: - web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI - elif blockchain_type == AvailableBlockchainType.WYRM: - web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET: - web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA: - web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA: + web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: - web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI elif blockchain_type == AvailableBlockchainType.XAI: - web3_uri = MOONSTREAM_XAI_WEB3_PROVIDER_URI + web3_uri = MOONSTREAM_NODE_XAI_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.AVALANCHE: + web3_uri = MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI + elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI: + web3_uri = MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI else: raise Exception("Wrong blockchain type provided for web3 URI") @@ -164,14 +162,10 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) if block.get("l1BatchTimestamp") is not None else None ) - if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - block_obj.sha3_uncles = block.get("sha3Uncles", "") - block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber")) - block_obj.send_count = hex_to_int(block.get("sendCount")) - block_obj.send_root = block.get("sendRoot", "") - block_obj.mix_hash = block.get("mixHash", "") - - if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: + if ( + blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA + or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA + ): block_obj.sha3_uncles = block.get("sha3Uncles", "") block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber")) block_obj.send_count = hex_to_int(block.get("sendCount")) @@ -185,6 +179,16 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType) block_obj.send_root = block.get("sendRoot", "") block_obj.mix_hash = block.get("mixHash", "") + if ( + blockchain_type == AvailableBlockchainType.AVALANCHE + or blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI + ): + block_obj.mix_hash = block.get("mixHash", "") + block_obj.block_extra_data = block.get("blockExtraData") + block_obj.block_gas_cost = block.get("blockGasCost") + block_obj.ext_data_gas_used = block.get("extDataGasUsed", "") + block_obj.ext_data_hash = block.get("extDataHash", "") + db_session.add(block_obj) @@ -227,13 +231,11 @@ def add_block_transactions( if tx.get("l1BatchTxIndex") is not None else None ) - if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA: - tx_obj.y_parity = hex_to_int(tx.get("yParity")) - - if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA: - tx_obj.y_parity = hex_to_int(tx.get("yParity")) - - if blockchain_type == AvailableBlockchainType.XAI: + if ( + blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA + or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA + or blockchain_type == AvailableBlockchainType.XAI + ): tx_obj.y_parity = hex_to_int(tx.get("yParity")) db_session.add(tx_obj) @@ -242,7 +244,7 @@ def add_block_transactions( def get_latest_blocks( blockchain_type: AvailableBlockchainType, confirmations: int = 0, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ) -> Tuple[Optional[int], int]: """ Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method). @@ -250,7 +252,7 @@ def get_latest_blocks( If confirmations > 0, and the latest block on the node has block number N, this returns the block with block_number (N - confirmations) """ - web3_client = connect(blockchain_type, access_id=access_id) + web3_client = connect(blockchain_type, web3_uri=web3_uri) latest_block_number: int = web3_client.eth.block_number if confirmations > 0: latest_block_number -= confirmations @@ -273,12 +275,12 @@ def crawl_blocks( blockchain_type: AvailableBlockchainType, blocks_numbers: List[int], with_transactions: bool = False, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ) -> None: """ Open database and geth sessions and fetch block data from blockchain. """ - web3_client = connect(blockchain_type, access_id=access_id) + web3_client = connect(blockchain_type, web3_uri=web3_uri) with yield_db_session_ctx() as db_session: pbar = tqdm(total=len(blocks_numbers)) for block_number in blocks_numbers: @@ -319,7 +321,7 @@ def check_missing_blocks( blockchain_type: AvailableBlockchainType, blocks_numbers: List[int], notransactions=False, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ) -> List[int]: """ Query block from postgres. If block does not presented in database, @@ -358,7 +360,7 @@ def check_missing_blocks( [block[0], block[1]] for block in blocks_exist_raw_query.all() ] - web3_client = connect(blockchain_type, access_id=access_id) + web3_client = connect(blockchain_type, web3_uri=web3_uri) blocks_exist_len = len(blocks_exist) pbar = tqdm(total=blocks_exist_len) @@ -400,7 +402,7 @@ def crawl_blocks_executor( block_numbers_list: List[int], with_transactions: bool = False, num_processes: int = MOONSTREAM_CRAWL_WORKERS, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ) -> None: """ Execute crawler in processes. @@ -429,7 +431,7 @@ def crawl_blocks_executor( if num_processes == 1: logger.warning("Executing block crawler in lazy mod") return crawl_blocks( - blockchain_type, block_numbers_list, with_transactions, access_id=access_id + blockchain_type, block_numbers_list, with_transactions, web3_uri=web3_uri ) else: with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor: @@ -441,7 +443,7 @@ def crawl_blocks_executor( blockchain_type, block_chunk, with_transactions, - access_id, + web3_uri, ) result.add_done_callback(record_error) results.append(result) diff --git a/crawlers/mooncrawl/mooncrawl/contract/cli.py b/crawlers/mooncrawl/mooncrawl/contract/cli.py index d7fccfbd..f1696b9c 100644 --- a/crawlers/mooncrawl/mooncrawl/contract/cli.py +++ b/crawlers/mooncrawl/mooncrawl/contract/cli.py @@ -10,7 +10,6 @@ from web3 import Web3 from ..blockchain import connect from ..db import yield_db_session_ctx -from ..settings import NB_CONTROLLER_ACCESS_ID from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore logging.basicConfig(level=logging.INFO) @@ -118,7 +117,7 @@ def run_crawler_desc( def handle_parser(args: argparse.Namespace): with yield_db_session_ctx() as session: - w3 = connect(AvailableBlockchainType.ETHEREUM, access_id=args.access_id) + w3 = connect(AvailableBlockchainType.ETHEREUM, web3_uri=args.web3_uri) if args.order == "asc": run_crawler_asc( w3=w3, @@ -156,10 +155,8 @@ def generate_parser(): parser = argparse.ArgumentParser(description="Moonstream Deployment Crawler") parser.add_argument( - "--access-id", - default=NB_CONTROLLER_ACCESS_ID, - type=UUID, - help="User access ID", + "--web3-uri", + help="Node JSON RPC uri", ) parser.add_argument( diff --git a/crawlers/mooncrawl/mooncrawl/crawler.py b/crawlers/mooncrawl/mooncrawl/crawler.py index 0dcd7f69..6d032512 100644 --- a/crawlers/mooncrawl/mooncrawl/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/crawler.py @@ -1,6 +1,7 @@ """ Moonstream crawlers CLI. """ + import argparse import json import logging @@ -12,8 +13,8 @@ from enum import Enum from typing import Iterator, List from uuid import UUID -from moonstreamdb.blockchain import AvailableBlockchainType import dateutil.parser # type: ignore +from moonstreamdb.blockchain import AvailableBlockchainType from .blockchain import ( DateRange, @@ -23,7 +24,7 @@ from .blockchain import ( trending, ) from .publish import publish_json -from .settings import MOONSTREAM_CRAWL_WORKERS, NB_CONTROLLER_ACCESS_ID +from .settings import MOONSTREAM_CRAWL_WORKERS from .version import MOONCRAWL_VERSION logging.basicConfig(level=logging.INFO) @@ -96,7 +97,7 @@ def crawler_blocks_sync_handler(args: argparse.Namespace) -> None: latest_stored_block_number, latest_block_number = get_latest_blocks( AvailableBlockchainType(args.blockchain), args.confirmations, - access_id=args.access_id, + web3_uri=args.web3_uri, ) if latest_stored_block_number is None: latest_stored_block_number = 0 @@ -140,7 +141,7 @@ def crawler_blocks_sync_handler(args: argparse.Namespace) -> None: block_numbers_list=blocks_numbers_list, with_transactions=True, num_processes=args.jobs, - access_id=args.access_id, + web3_uri=args.web3_uri, ) logger.info( f"Synchronized blocks from {latest_stored_block_number} to {latest_block_number}" @@ -159,7 +160,7 @@ def crawler_blocks_add_handler(args: argparse.Namespace) -> None: blockchain_type=AvailableBlockchainType(args.blockchain), block_numbers_list=blocks_numbers_list, with_transactions=True, - access_id=args.access_id, + web3_uri=args.web3_uri, ) logger.info( @@ -184,7 +185,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None: _, latest_block_number = get_latest_blocks( AvailableBlockchainType(args.blockchain), confirmations, - access_id=args.access_id, + web3_uri=args.web3_uri, ) start_block_number = ( latest_block_number - shift if latest_block_number - shift >= 1 else 1 @@ -200,7 +201,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None: blockchain_type=AvailableBlockchainType(args.blockchain), blocks_numbers=blocks_numbers_list, notransactions=args.notransactions, - access_id=args.access_id, + web3_uri=args.web3_uri, ) if len(missing_blocks_numbers) > 0: logger.info(f"Found {len(missing_blocks_numbers)} missing blocks") @@ -217,7 +218,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None: block_numbers_list=missing_blocks_numbers_total, with_transactions=True, num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS, - access_id=args.access_id, + web3_uri=args.web3_uri, ) logger.info( f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers " @@ -259,10 +260,8 @@ def main() -> None: time_now = datetime.now(timezone.utc) parser.add_argument( - "--access-id", - default=NB_CONTROLLER_ACCESS_ID, - type=UUID, - help="User access ID", + "--web3-uri", + help="Node JSON RPC uri", ) # Blockchain blocks parser diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py b/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py index 6fd8b41b..f81bd716 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/base.py @@ -11,11 +11,7 @@ from moonstreamdb.blockchain import ( get_label_model, get_transaction_model, ) -from moonstreamdb.models import ( - Base, - EthereumTransaction, - PolygonTransaction, -) +from moonstreamdb.models import Base, EthereumTransaction, PolygonTransaction from moonworm.crawler.function_call_crawler import ( # type: ignore ContractFunctionCall, utfy_dict, @@ -26,10 +22,7 @@ from tqdm import tqdm from web3 import Web3 from web3._utils.events import get_event_data -from ..moonworm_crawler.db import ( - add_events_to_session, - commit_session, -) +from ..moonworm_crawler.db import add_events_to_session, commit_session from ..moonworm_crawler.event_crawler import Event, get_block_timestamp logging.basicConfig(level=logging.INFO) diff --git a/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py index a11aed9d..fb465873 100644 --- a/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py @@ -10,7 +10,6 @@ from web3.middleware import geth_poa_middleware from ..blockchain import connect from ..db import yield_db_session_ctx -from ..settings import NB_CONTROLLER_ACCESS_ID from .base import crawl, get_checkpoint, populate_with_events logging.basicConfig(level=logging.INFO) @@ -39,7 +38,7 @@ def handle_nft_crawler(args: argparse.Namespace) -> None: logger.info( "No web3 provider URL provided, using default (blockchan.py: connect())" ) - web3 = connect(blockchain_type, access_id=args.access_id) + web3 = connect(blockchain_type, web3_uri=args.web3_uri) else: logger.info(f"Using web3 provider URL: {args.web3}") web3 = Web3( @@ -91,7 +90,7 @@ def populate_with_erc20_transfers(args: argparse.Namespace) -> None: logger.info( "No web3 provider URL provided, using default (blockchan.py: connect())" ) - web3 = connect(blockchain_type, access_id=args.access_id) + web3 = connect(blockchain_type, web3_uri=args.web3_uri) else: logger.info(f"Using web3 provider URL: {args.web3}") web3 = Web3( @@ -137,7 +136,7 @@ def handle_crawl(args: argparse.Namespace) -> None: logger.info( "No web3 provider URL provided, using default (blockchan.py: connect())" ) - web3 = connect(blockchain_type, access_id=args.access_id) + web3 = connect(blockchain_type, web3_uri=args.web3_uri) else: logger.info(f"Using web3 provider URL: {args.web3}") web3 = Web3( @@ -172,10 +171,8 @@ def main(): subparsers = parser.add_subparsers() parser.add_argument( - "--access-id", - default=NB_CONTROLLER_ACCESS_ID, - type=UUID, - help="User access ID", + "--web3-uri", + help="Node JSON RPC uri", ) crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi") diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py index 7e382109..8cc2def2 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py @@ -1,24 +1,22 @@ import argparse import json import logging -from typing import cast, List import uuid +from typing import List, cast import requests # type: ignore from bugout.data import BugoutSearchResult -from .utils import get_results_for_moonstream_query, leaderboard_push_batch from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, - MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, MOONSTREAM_API_URL, MOONSTREAM_ENGINE_URL, MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE, + MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID, MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, ) - from ..settings import bugout_client as bc - +from .utils import get_results_for_moonstream_query, leaderboard_push_batch logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py index 98135370..f1e9e87f 100644 --- a/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py +++ b/crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py @@ -3,8 +3,7 @@ import json import logging import os import time -from typing import Any, Dict, Optional, List - +from typing import Any, Dict, List, Optional import requests # type: ignore @@ -15,7 +14,6 @@ from ..settings import ( MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS, ) - logging.basicConfig() logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py index 94497348..81d68c55 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py @@ -9,10 +9,7 @@ from urllib.error import HTTPError from moonstreamdb.blockchain import AvailableBlockchainType -from ..db import ( - yield_db_preping_session_ctx, - yield_db_read_only_preping_session_ctx, -) +from ..db import yield_db_preping_session_ctx, yield_db_read_only_preping_session_ctx from .db import ( clean_labels_from_db, get_current_metadata_for_address, diff --git a/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py b/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py index 7cc2c3bc..c53bdbe2 100644 --- a/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py @@ -1,13 +1,13 @@ -import logging import json -from typing import Dict, Any, Optional, List +import logging +from typing import Any, Dict, List, Optional from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model from sqlalchemy.orm import Session from sqlalchemy.sql import text from ..data import TokenURIs -from ..settings import VIEW_STATE_CRAWLER_LABEL, METADATA_CRAWLER_LABEL, CRAWLER_LABEL +from ..settings import CRAWLER_LABEL, METADATA_CRAWLER_LABEL, VIEW_STATE_CRAWLER_LABEL logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py index 04fc2ed9..ce024f2f 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py @@ -9,21 +9,20 @@ from web3.middleware import geth_poa_middleware from ..db import yield_db_session_ctx from ..settings import ( - MOONSTREAM_MOONWORM_TASKS_JOURNAL, - NB_CONTROLLER_ACCESS_ID, - HISTORICAL_CRAWLER_STATUSES, HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES, + HISTORICAL_CRAWLER_STATUSES, + MOONSTREAM_MOONWORM_TASKS_JOURNAL, ) from .continuous_crawler import _retry_connect_web3, continuous_crawler from .crawler import ( SubscriptionTypes, blockchain_type_to_subscription_type, + find_all_deployed_blocks, get_crawl_job_entries, make_event_crawl_jobs, make_function_call_crawl_jobs, - find_all_deployed_blocks, - update_job_state_with_filters, moonworm_crawler_update_job_as_pickedup, + update_job_state_with_filters, ) from .db import get_first_labeled_block_number, get_last_labeled_block_number from .historical_crawler import historical_crawler @@ -71,7 +70,7 @@ def handle_crawl(args: argparse.Namespace) -> None: logger.info( "No web3 provider URL provided, using default (blockchan.py: connect())" ) - web3 = _retry_connect_web3(blockchain_type, access_id=args.access_id) + web3 = _retry_connect_web3(blockchain_type, web3_uri=args.web3_uri) else: logger.info(f"Using web3 provider URL: {args.web3}") web3 = Web3( @@ -133,7 +132,7 @@ def handle_crawl(args: argparse.Namespace) -> None: args.min_sleep_time, args.heartbeat_interval, args.new_jobs_refetch_interval, - access_id=args.access_id, + web3_uri=args.web3_uri, ) @@ -250,7 +249,7 @@ def handle_historical_crawl(args: argparse.Namespace) -> None: logger.info( "No web3 provider URL provided, using default (blockchan.py: connect())" ) - web3 = _retry_connect_web3(blockchain_type, access_id=args.access_id) + web3 = _retry_connect_web3(blockchain_type, web3_uri=args.web3_uri) else: logger.info(f"Using web3 provider URL: {args.web3}") web3 = Web3( @@ -336,7 +335,7 @@ def handle_historical_crawl(args: argparse.Namespace) -> None: end_block, args.max_blocks_batch, args.min_sleep_time, - access_id=args.access_id, + web3_uri=args.web3_uri, addresses_deployment_blocks=addresses_deployment_blocks, ) @@ -346,10 +345,8 @@ def main() -> None: parser.set_defaults(func=lambda _: parser.print_help()) parser.add_argument( - "--access-id", - default=NB_CONTROLLER_ACCESS_ID, - type=UUID, - help="User access ID", + "--web3-uri", + help="Node JSON RPC uri", ) subparsers = parser.add_subparsers() diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py index 33761970..b0feeb38 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py @@ -100,7 +100,7 @@ def continuous_crawler( min_sleep_time: float = 0.1, heartbeat_interval: float = 60, new_jobs_refetch_interval: float = 120, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, max_insert_batch: int = 10000, ): crawler_type = "continuous" @@ -119,7 +119,7 @@ def continuous_crawler( jobs_refetchet_time = crawl_start_time if web3 is None: - web3 = _retry_connect_web3(blockchain_type, access_id=access_id) + web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri) if blockchain_type == AvailableBlockchainType.ETHEREUM: network = Network.ethereum @@ -304,7 +304,7 @@ def continuous_crawler( logger.error("Too many failures, exiting") raise e try: - web3 = _retry_connect_web3(blockchain_type, access_id=access_id) + web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri) except Exception as err: logger.error(f"Failed to reconnect: {err}") logger.exception(err) diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py index 27ce102b..24d23140 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py @@ -107,7 +107,7 @@ def _retry_connect_web3( blockchain_type: AvailableBlockchainType, retry_count: int = 10, sleep_time: float = 5, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ) -> Web3: """ Retry connecting to the blockchain. @@ -115,7 +115,7 @@ def _retry_connect_web3( while retry_count > 0: retry_count -= 1 try: - web3 = connect(blockchain_type, access_id=access_id) + web3 = connect(blockchain_type, web3_uri=web3_uri) web3.eth.block_number logger.info(f"Connected to {blockchain_type}") return web3 diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py index ad964a32..8b5001ec 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py @@ -1,12 +1,12 @@ -import logging import json +import logging from typing import Dict, List, Optional from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model from moonstreamdb.models import Base from moonworm.crawler.function_call_crawler import ContractFunctionCall # type: ignore +from sqlalchemy import Integer, String, column, exists, func, select, text, values from sqlalchemy.orm import Session -from sqlalchemy import func, text, select, values, column, Integer, String, exists from ..settings import CRAWLER_LABEL from .event_crawler import Event diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py index 73f14f1b..8a0b094e 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py @@ -3,7 +3,10 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple from moonstreamdb.blockchain import AvailableBlockchainType, get_block_model -from moonworm.crawler.log_scanner import _fetch_events_chunk, _crawl_events as moonworm_autoscale_crawl_events # type: ignore +from moonworm.crawler.log_scanner import ( + _crawl_events as moonworm_autoscale_crawl_events, # type: ignore +) +from moonworm.crawler.log_scanner import _fetch_events_chunk from sqlalchemy.orm.session import Session from sqlalchemy.sql.expression import and_ from web3 import Web3 diff --git a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py index 4f66a8bd..fa51afe3 100644 --- a/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py +++ b/crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py @@ -36,7 +36,7 @@ def historical_crawler( end_block: int, max_blocks_batch: int = 100, min_sleep_time: float = 0.1, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, addresses_deployment_blocks: Optional[Dict[ChecksumAddress, int]] = None, max_insert_batch: int = 10000, ): @@ -46,7 +46,7 @@ def historical_crawler( assert end_block > 0, "end_block must be greater than 0" if web3 is None: - web3 = _retry_connect_web3(blockchain_type, access_id=access_id) + web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri) assert ( web3.eth.block_number >= start_block @@ -205,7 +205,7 @@ def historical_crawler( logger.error("Too many failures, exiting") raise e try: - web3 = _retry_connect_web3(blockchain_type, access_id=access_id) + web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri) except Exception as err: logger.error(f"Failed to reconnect: {err}") logger.exception(err) diff --git a/crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py index 374e9fcd..5c59afe1 100644 --- a/crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py @@ -3,22 +3,20 @@ import csv import datetime import json import logging -from io import StringIO -from moonstream.client import Moonstream # type: ignore -import requests # type: ignore import time +from io import StringIO +from typing import Any, Dict, Union from uuid import UUID -from typing import Any, Dict, Union +import requests # type: ignore +from moonstream.client import Moonstream # type: ignore - -from .queries import tokenomics_queries, cu_bank_queries, tokenomics_orange_dao_queries from ..actions import recive_S3_data_from_query from ..settings import ( MOONSTREAM_S3_PUBLIC_DATA_BUCKET, MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX, ) - +from .queries import cu_bank_queries, tokenomics_orange_dao_queries, tokenomics_queries logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/crawlers/mooncrawl/mooncrawl/settings.py b/crawlers/mooncrawl/mooncrawl/settings.py index 4596c325..e78b54a0 100644 --- a/crawlers/mooncrawl/mooncrawl/settings.py +++ b/crawlers/mooncrawl/mooncrawl/settings.py @@ -78,72 +78,87 @@ except: ) # Geth connection address -MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI", "" ) -if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI env variable is not set") +if MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI env variable is not set") -MOONSTREAM_POLYGON_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_POLYGON_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI", "" ) -if MOONSTREAM_POLYGON_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_POLYGON_WEB3_PROVIDER_URI env variable is not set") +if MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI env variable is not set") -MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI", "" ) -if MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI env variable is not set") +if MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI env variable is not set") -MOONSTREAM_XDAI_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_XDAI_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI", "" ) -if MOONSTREAM_XDAI_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_XDAI_WEB3_PROVIDER_URI env variable is not set") +if MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI env variable is not set") -MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_WYRM_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI", "" ) -if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set") - -MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", "" -) -if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "": +if MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI == "": raise Exception( - "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set" + "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI env variable is not set" ) -MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI", "" ) -if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set") +if MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI env variable is not set") -MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI", "" ) -if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "": +if MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI == "": raise Exception( - "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set" + "MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI env variable is not set" ) -MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI", "" ) -if MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI == "": +if MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI == "": raise Exception( - "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI env variable is not set" + "MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI env variable is not set" ) -MOONSTREAM_XAI_WEB3_PROVIDER_URI = os.environ.get( - "MOONSTREAM_XAI_WEB3_PROVIDER_URI", "" +MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI", "" ) -if MOONSTREAM_XAI_WEB3_PROVIDER_URI == "": - raise Exception("MOONSTREAM_XAI_WEB3_PROVIDER_URI env variable is not set") +if MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI == "": + raise Exception( + "MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI env variable is not set" + ) +MOONSTREAM_NODE_XAI_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_XAI_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_XAI_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_XAI_A_EXTERNAL_URI env variable is not set") + +MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI == "": + raise Exception("MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI env variable is not set") + +MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI = os.environ.get( + "MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI", "" +) +if MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI == "": + raise Exception( + "MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI env variable is not set" + ) MOONSTREAM_CRAWL_WORKERS = 4 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS") diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py b/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py index f1eae777..ab6cf9a1 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/cli.py @@ -10,23 +10,19 @@ from pprint import pprint from typing import Any, Dict, List, Optional from uuid import UUID -from moonstreamdb.blockchain import AvailableBlockchainType from moonstream.client import Moonstream # type: ignore +from moonstreamdb.blockchain import AvailableBlockchainType from web3.middleware import geth_poa_middleware from mooncrawl.moonworm_crawler.crawler import _retry_connect_web3 from ..actions import recive_S3_data_from_query +from ..blockchain import connect from ..db import PrePing_SessionLocal -from ..settings import ( - INFURA_PROJECT_ID, - NB_CONTROLLER_ACCESS_ID, - infura_networks, - multicall_contracts, -) +from ..settings import INFURA_PROJECT_ID, infura_networks, multicall_contracts from .db import clean_labels, commit_session, view_call_to_label from .Multicall2_interface import Contract as Multicall2 -from .web3_util import FunctionSignature, connect +from .web3_util import FunctionSignature logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -306,8 +302,8 @@ def parse_jobs( web3_provider_uri: Optional[str], block_number: Optional[int], batch_size: int, - access_id: UUID, moonstream_token: str, + web3_uri: Optional[str] = None, ): """ Parse jobs from list and generate web3 interfaces for each contract. @@ -321,13 +317,12 @@ def parse_jobs( if web3_provider_uri is not None: try: logger.info( - f"Connecting to blockchain : {blockchain_type} with custom provider!" + f"Connecting to blockchain: {blockchain_type} with custom provider!" ) - web3_client = connect(web3_provider_uri) - - if blockchain_type != AvailableBlockchainType.ETHEREUM: - web3_client.middleware_onion.inject(geth_poa_middleware, layer=0) + web3_client = connect( + blockchain_type=blockchain_type, web3_uri=web3_provider_uri + ) except Exception as e: logger.error( f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}" @@ -336,7 +331,7 @@ def parse_jobs( else: logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.") web3_client = _retry_connect_web3( - blockchain_type=blockchain_type, access_id=access_id + blockchain_type=blockchain_type, web3_uri=web3_uri ) logger.info(f"Crawler started connected to blockchain: {blockchain_type}") @@ -519,7 +514,7 @@ def handle_crawl(args: argparse.Namespace) -> None: blockchain_type = AvailableBlockchainType(args.blockchain) - custom_web3_provider = args.custom_web3_provider + custom_web3_provider = args.web3_uri if args.infura and INFURA_PROJECT_ID is not None: if blockchain_type not in infura_networks: @@ -535,8 +530,8 @@ def handle_crawl(args: argparse.Namespace) -> None: custom_web3_provider, args.block_number, args.batch_size, - args.access_id, args.moonstream_token, + args.web3_uri, ) @@ -563,7 +558,7 @@ def clean_labels_handler(args: argparse.Namespace) -> None: blockchain_type = AvailableBlockchainType(args.blockchain) web3_client = _retry_connect_web3( - blockchain_type=blockchain_type, access_id=args.access_id + blockchain_type=blockchain_type, web3_uri=args.web3_uri ) logger.info(f"Label cleaner connected to blockchain: {blockchain_type}") @@ -583,10 +578,8 @@ def main() -> None: parser.set_defaults(func=lambda _: parser.print_help()) parser.add_argument( - "--access-id", - default=NB_CONTROLLER_ACCESS_ID, - type=UUID, - help="User access ID", + "--web3-uri", + help="Node JSON RPC uri", ) subparsers = parser.add_subparsers() @@ -614,12 +607,6 @@ def main() -> None: action="store_true", help="Use infura as web3 provider", ) - view_state_crawler_parser.add_argument( - "--custom-web3-provider", - "-w3", - type=str, - help="Type of blovkchain wich writng in database", - ) view_state_crawler_parser.add_argument( "--block-number", "-N", type=str, help="Block number." ) diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/db.py b/crawlers/mooncrawl/mooncrawl/state_crawler/db.py index 218b62f9..b32be15f 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/db.py +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/db.py @@ -1,6 +1,6 @@ -import logging import json -from typing import Dict, Any +import logging +from typing import Any, Dict from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model from sqlalchemy.orm import Session diff --git a/crawlers/mooncrawl/mooncrawl/state_crawler/web3_util.py b/crawlers/mooncrawl/mooncrawl/state_crawler/web3_util.py index 224034a7..f80cc7f4 100644 --- a/crawlers/mooncrawl/mooncrawl/state_crawler/web3_util.py +++ b/crawlers/mooncrawl/mooncrawl/state_crawler/web3_util.py @@ -2,17 +2,17 @@ import getpass import os from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from eth_abi import decode_single, encode_single from eth_account.account import Account # type: ignore -from eth_abi import encode_single, decode_single from eth_typing.evm import ChecksumAddress from eth_utils import function_signature_to_4byte_selector from hexbytes.main import HexBytes from web3 import Web3 +from web3._utils.abi import normalize_event_input_types from web3.contract import ContractFunction from web3.providers.ipc import IPCProvider from web3.providers.rpc import HTTPProvider from web3.types import ABI, Nonce, TxParams, TxReceipt, Wei -from web3._utils.abi import normalize_event_input_types class ContractConstructor: @@ -165,28 +165,6 @@ def read_keys_from_env() -> Tuple[ChecksumAddress, str]: ) -def connect(web3_uri: str) -> Web3: - web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider() - if web3_uri.startswith("http://") or web3_uri.startswith("https://"): - web3_provider = Web3.HTTPProvider(web3_uri) - else: - web3_provider = Web3.IPCProvider(web3_uri) - web3_client = Web3(web3_provider) - return web3_client - - -def read_web3_provider_from_env() -> Web3: - provider_path = os.environ.get("MOONWORM_WEB3_PROVIDER_URI") - if provider_path is None: - raise ValueError("MOONWORM_WEB3_PROVIDER_URI env variable is not set") - return connect(provider_path) - - -def read_web3_provider_from_cli() -> Web3: - provider_path = input("Enter web3 uri path: ") - return connect(provider_path) - - def cast_to_python_type(evm_type: str) -> Callable: if evm_type.startswith(("uint", "int")): return int diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py index 10303952..e05cf215 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py @@ -40,7 +40,6 @@ from ..settings import ( MOONSTREAM_ADMIN_ACCESS_TOKEN, MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET, MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX, - NB_CONTROLLER_ACCESS_ID, ) from ..settings import bugout_client as bc @@ -367,7 +366,7 @@ def generate_list_of_names( def process_external_merged( external_calls: Dict[str, Dict[str, Any]], blockchain: AvailableBlockchainType, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ): """ Process external calls @@ -412,7 +411,7 @@ def process_external_merged( logger.error(f"Error processing external call: {e}") if external_calls_normalized: - web3_client = connect(blockchain, access_id=access_id) + web3_client = connect(blockchain, web3_uri=web3_uri) for extcall in external_calls_normalized: try: @@ -433,7 +432,7 @@ def process_external_merged( def process_external( abi_external_calls: List[Dict[str, Any]], blockchain: AvailableBlockchainType, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ): """ Request all required external data @@ -479,7 +478,7 @@ def process_external( logger.error(f"Error processing external call: {e}") if external_calls: - web3_client = connect(blockchain, access_id=access_id) + web3_client = connect(blockchain, web3_uri=web3_uri) for extcall in external_calls: try: @@ -530,7 +529,7 @@ def generate_web3_metrics( address: str, crawler_label: str, abi_json: Any, - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ) -> List[Any]: """ Generate stats for cards components @@ -543,7 +542,7 @@ def generate_web3_metrics( extention_data = process_external( abi_external_calls=abi_external_calls, blockchain=blockchain_type, - access_id=access_id, + web3_uri=web3_uri, ) extention_data.append( @@ -873,7 +872,7 @@ def stats_generate_handler(args: argparse.Namespace): external_calls_results = process_external_merged( external_calls=merged_external_calls["merged"], blockchain=blockchain_type, - access_id=args.access_id, + web3_uri=args.web3_uri, ) for address in address_dashboard_id_subscription_id_tree.keys(): @@ -1039,7 +1038,7 @@ def stats_generate_api_task( timescales: List[str], dashboard: BugoutResource, subscription_by_id: Dict[str, BugoutJournalEntity], - access_id: Optional[UUID] = None, + web3_uri: Optional[str] = None, ): """ Start crawler with generate. @@ -1115,7 +1114,7 @@ def stats_generate_api_task( address=address, # type: ignore crawler_label=crawler_label, abi_json=abi_json, - access_id=access_id, + web3_uri=web3_uri, ) # Generate blocks state information @@ -1192,10 +1191,8 @@ def main() -> None: parser.set_defaults(func=lambda _: parser.print_help()) parser.add_argument( - "--access-id", - default=NB_CONTROLLER_ACCESS_ID, - type=UUID, - help="User access ID", + "--web3-uri", + help="Node JSON RPC uri", ) subcommands = parser.add_subparsers( diff --git a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py index 4c9b9453..55052605 100644 --- a/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py +++ b/crawlers/mooncrawl/mooncrawl/stats_worker/queries.py @@ -1,21 +1,17 @@ import csv -from collections import OrderedDict import hashlib import json import logging import re +from collections import OrderedDict from io import StringIO from typing import Any, Dict - from sqlalchemy.orm import sessionmaker from sqlalchemy.sql import text from sqlalchemy.sql.expression import TextClause from ..actions import push_data_to_bucket -from ..reporter import reporter - - from ..db import RO_pre_ping_query_engine from ..reporter import reporter from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 96dd1ef3..cb8ea08d 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -21,16 +21,18 @@ export MOONSTREAM_ADMIN_ACCESS_TOKEN="" export NFT_HUMBUG_TOKEN="" # Blockchain nodes environment variables -export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI="https://" -export MOONSTREAM_XAI_WEB3_PROVIDER_URI="https://" +export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://" +export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://" export NB_CONTROLLER_ACCESS_ID="" # AWS environment variables diff --git a/crawlers/mooncrawl/setup.py b/crawlers/mooncrawl/setup.py index 085d9be3..5faebc90 100644 --- a/crawlers/mooncrawl/setup.py +++ b/crawlers/mooncrawl/setup.py @@ -37,7 +37,7 @@ setup( "bugout>=0.2.13", "chardet", "fastapi", - "moonstreamdb>=0.3.8", + "moonstreamdb>=0.3.9", "moonstream>=0.1.1", "moonworm[moonstream]>=0.6.2", "humbug", From f103d6fac89ac8947c1e120ccc0f9fb7e8e31e43 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Mar 2024 13:40:35 +0000 Subject: [PATCH 11/12] Bumped version --- crawlers/mooncrawl/mooncrawl/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawlers/mooncrawl/mooncrawl/version.py b/crawlers/mooncrawl/mooncrawl/version.py index e999d730..a6e9b65d 100644 --- a/crawlers/mooncrawl/mooncrawl/version.py +++ b/crawlers/mooncrawl/mooncrawl/version.py @@ -2,4 +2,4 @@ Moonstream crawlers version. """ -MOONCRAWL_VERSION = "0.3.9" +MOONCRAWL_VERSION = "0.4.0" From 3b0729f439cac77c1fc20ca1da6ab43474997ff5 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 28 Mar 2024 13:47:36 +0000 Subject: [PATCH 12/12] Zksync sepolia moonworm service --- crawlers/deploy/deploy.bash | 13 +++++++++++-- .../zksync-era-sepolia-moonworm-crawler.service | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 crawlers/deploy/zksync-era-sepolia-moonworm-crawler.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index 11be0b99..22c7b957 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -107,6 +107,7 @@ ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer" ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE="zksync-era-sepolia-synchronize.service" ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE="zksync-era-sepolia-missing.service" ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE="zksync-era-sepolia-missing.timer" +ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-sepolia-moonworm-crawler.service" # ZkSync Era testnet ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service" @@ -577,7 +578,7 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBO # ZkSync Era Sepolia echo echo -echo -e "${PREFIX_INFO} Replacing existing ZkSync Era block with transactions syncronizer service definition with ${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era Sepolia block with transactions syncronizer service definition with ${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload @@ -585,13 +586,21 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_E echo echo -echo -e "${PREFIX_INFO} Replacing existing ZkSync Era missing service and timer with: ${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era Sepolia missing service and timer with: ${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era Sepolia moonworm crawler service definition with ${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" + # Arbitrum Nova echo echo diff --git a/crawlers/deploy/zksync-era-sepolia-moonworm-crawler.service b/crawlers/deploy/zksync-era-sepolia-moonworm-crawler.service new file mode 100644 index 00000000..5511a95a --- /dev/null +++ b/crawlers/deploy/zksync-era-sepolia-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era Sepolia moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_sepolia --confirmations 20 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=zksync-era-sepolia-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file