Switch from access-id to web3-uri, zksync sepolia and avalanche

crawlers/deploy/arbitrum-nova-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_nova -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_nova -n
 CPUWeight=50
 SyslogIdentifier=arbitrum-nova-missing

crawlers/deploy/arbitrum-nova-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20
 CPUWeight=70
 SyslogIdentifier=arbitrum-nova-moonworm-crawler
 

crawlers/deploy/arbitrum-nova-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_nova -c 10 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_nova -c 10 -j 2
 CPUWeight=90
 SyslogIdentifier=arbitrum-nova-synchronize
 

crawlers/deploy/arbitrum-sepolia-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_sepolia -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_sepolia -n
 CPUWeight=50
 SyslogIdentifier=arbitrum-sepolia-missing

crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20
 CPUWeight=70
 SyslogIdentifier=arbitrum-sepolia-moonworm-crawler
 

crawlers/deploy/arbitrum-sepolia-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2
 CPUWeight=90
 SyslogIdentifier=arbitrum-sepolia-synchronize
 

crawlers/deploy/deploy.bash

@@ -31,8 +31,6 @@ LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
 
 # Ethereum service files
 ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
-ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
-ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
 ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
 ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
 ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service"
@@ -106,6 +104,9 @@ ZKSYNC_ERA_STATE_SERVICE_FILE="zksync-era-state.service"
 ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer"
 ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service"
 ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer"
+ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE="zksync-era-sepolia-synchronize.service"
+ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE="zksync-era-sepolia-missing.service"
+ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE="zksync-era-sepolia-missing.timer"
 
 # ZkSync Era testnet
 ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
@@ -190,15 +191,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}" "/home/ubuntu/.config/sy
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}"
 
-echo
-echo
-echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE_FILE}, ${ETHEREUM_TRENDING_TIMER_FILE}"
-chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}"
-cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TRENDING_SERVICE_FILE}"
-cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TRENDING_TIMER_FILE}"
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
-
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
@@ -582,6 +574,24 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/syste
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
 
+# ZkSync Era Sepolia
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era block with transactions syncronizer service definition with ${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing ZkSync Era missing service and timer with: ${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
+chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
+
 # Arbitrum Nova
 echo
 echo

crawlers/deploy/ethereum-historical-crawl-events.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events
 CPUWeight=70
 SyslogIdentifier=ethereum-historical-crawl-events
 

crawlers/deploy/ethereum-historical-crawl-transactions.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions
 CPUWeight=70
 SyslogIdentifier=ethereum-historical-crawl-transactions
 

crawlers/deploy/ethereum-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain ethereum -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain ethereum -n
 CPUWeight=50
 SyslogIdentifier=ethereum-missing

crawlers/deploy/ethereum-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b ethereum
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b ethereum
 CPUWeight=70
 SyslogIdentifier=ethereum-moonworm-crawler
 

crawlers/deploy/ethereum-state-clean.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain ethereum -N 10000
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain ethereum -N 10000
 CPUWeight=60
 SyslogIdentifier=ethereum-state-clean

crawlers/deploy/ethereum-state.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain ethereum --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain ethereum --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json
 CPUWeight=60
 SyslogIdentifier=ethereum-state

crawlers/deploy/ethereum-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain ethereum -c 6 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain ethereum -c 6 -j 2
 CPUWeight=90
 SyslogIdentifier=ethereum-synchronize
 

crawlers/deploy/ethereum-trending.timer

@@ -1,9 +0,0 @@
-[Unit]
-Description=Load trending Ethereum addresses to the database every 5 minutes
-
-[Timer]
-OnBootSec=60s
-OnUnitActiveSec=5m
-
-[Install]
-WantedBy=timers.target

crawlers/deploy/mumbai-historical-crawl-events.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events
 CPUWeight=70
 SyslogIdentifier=mumbai-historical-crawl-events
 

crawlers/deploy/mumbai-historical-crawl-transactions.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions
 CPUWeight=70
 SyslogIdentifier=mumbai-historical-crawl-transactions
 

crawlers/deploy/mumbai-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain mumbai -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain mumbai -n
 CPUWeight=50
 SyslogIdentifier=mumbai-missing

crawlers/deploy/mumbai-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b mumbai --confirmations 40
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b mumbai --confirmations 40
 CPUWeight=70
 SyslogIdentifier=mumbai-moonworm-crawler
 

crawlers/deploy/mumbai-state-clean.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain mumbai -N 10000
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain mumbai -N 10000
 CPUWeight=60
 SyslogIdentifier=mumbai-state-clean

crawlers/deploy/mumbai-state.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
 CPUWeight=60
 SyslogIdentifier=mumbai-state

crawlers/deploy/mumbai-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain mumbai -c 20 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain mumbai -c 20 -j 2
 CPUWeight=90
 SyslogIdentifier=mumbai-synchronize
 

crawlers/deploy/polygon-historical-crawl-events.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events
 CPUWeight=70
 SyslogIdentifier=polygon-historical-crawl-events
 

crawlers/deploy/polygon-historical-crawl-transactions.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions
 CPUWeight=70
 SyslogIdentifier=polygon-historical-crawl-transactions
 

crawlers/deploy/polygon-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain polygon -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain polygon -n
 CPUWeight=50
 SyslogIdentifier=polygon-missing

crawlers/deploy/polygon-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b polygon  --confirmations 50 --min-blocks-batch 20
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b polygon  --confirmations 50 --min-blocks-batch 20
 CPUWeight=70
 SyslogIdentifier=polygon-moonworm-crawler
 

crawlers/deploy/polygon-state-clean.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain polygon -N 10000
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain polygon -N 10000
 CPUWeight=60
 SyslogIdentifier=polygon-state-clean

crawlers/deploy/polygon-state.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
 CPUWeight=60
 SyslogIdentifier=polygon-state

crawlers/deploy/polygon-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain polygon --confirmations 40 --jobs 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain polygon --confirmations 40 --jobs 2
 CPUWeight=90
 SyslogIdentifier=polygon-synchronize
 

crawlers/deploy/xai-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xai -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain xai -n
 CPUWeight=50
 SyslogIdentifier=xai-missing

crawlers/deploy/xai-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xai --confirmations 10 --min-blocks-batch 20
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b xai --confirmations 10 --min-blocks-batch 20
 CPUWeight=70
 SyslogIdentifier=xai-moonworm-crawler
 

crawlers/deploy/xai-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xai -c 10 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain xai -c 10 -j 2
 CPUWeight=90
 SyslogIdentifier=xai-synchronize
 

crawlers/deploy/xdai-historical-crawl-events.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events
 CPUWeight=70
 SyslogIdentifier=xdai-historical-crawl-events
 

crawlers/deploy/xdai-historical-crawl-transactions.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions
 CPUWeight=70
 SyslogIdentifier=xdai-historical-crawl-transactions
 

crawlers/deploy/xdai-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xdai -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain xdai -n
 CPUWeight=50
 SyslogIdentifier=xdai-missing

crawlers/deploy/xdai-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xdai
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b xdai
 CPUWeight=70
 SyslogIdentifier=xdai-moonworm-crawler
 

crawlers/deploy/xdai-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xdai -c 60 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain xdai -c 60 -j 2
 CPUWeight=90
 SyslogIdentifier=xdai-synchronize
 

crawlers/deploy/zksync-era-historical-crawl-events.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events
 CPUWeight=70
 SyslogIdentifier=zksync-era-historical-crawl-events
 

crawlers/deploy/zksync-era-historical-crawl-transactions.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions
 CPUWeight=70
 SyslogIdentifier=zksync-era-historical-crawl-transactions
 

crawlers/deploy/zksync-era-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era -n
 CPUWeight=50
 SyslogIdentifier=zksync-era-missing

crawlers/deploy/zksync-era-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era  --confirmations 20 --min-blocks-batch 20
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era  --confirmations 20 --min-blocks-batch 20
 CPUWeight=70
 SyslogIdentifier=zksync-era-moonworm-crawler
 

crawlers/deploy/zksync-era-sepolia-missing.service

@@ -1,11 +1,11 @@
 [Unit]
-Description=Load trending Ethereum addresses to the database
+Description=Fill missing blocks at ZkSync Era Sepolia database
 After=network.target
 
 [Service]
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" trending
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_sepolia -n
-CPUWeight=30
+CPUWeight=50
-SyslogIdentifier=ethereum-trending
+SyslogIdentifier=zksync-era-sepolia-missing

crawlers/deploy/zksync-era-sepolia-missing.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Fill missing blocks at ZkSync Era Sepolia database
+
+[Timer]
+OnBootSec=120s
+OnUnitActiveSec=15m
+
+[Install]
+WantedBy=timers.target

crawlers/deploy/zksync-era-sepolia-synchronize.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=ZkSync Era block with transactions synchronizer
+StartLimitIntervalSec=300
+StartLimitBurst=3
+After=network.target
+
+[Service]
+Restart=on-failure
+RestartSec=15s
+WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
+EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_sepolia -c 20 -j 2
+CPUWeight=90
+SyslogIdentifier=zksync-era-sepolia-synchronize
+
+[Install]
+WantedBy=multi-user.target

crawlers/deploy/zksync-era-state-clean.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync_era -N 10000
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain zksync_era -N 10000
 CPUWeight=60
 SyslogIdentifier=zksync-era-state-clean

crawlers/deploy/zksync-era-state.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json
 CPUWeight=60
 SyslogIdentifier=zksync-era-state

crawlers/deploy/zksync-era-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era -c 20 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era -c 20 -j 2
 CPUWeight=90
 SyslogIdentifier=zksync-era-synchronize
 

crawlers/deploy/zksync-era-testnet-historical-crawl-events.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-events
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-events
 CPUWeight=70
 SyslogIdentifier=zksync-era-testnet-historical-crawl-events
 

crawlers/deploy/zksync-era-testnet-historical-crawl-transactions.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-functions
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-functions
 CPUWeight=70
 SyslogIdentifier=zksync-era-testnet-historical-crawl-transactions
 

crawlers/deploy/zksync-era-testnet-missing.service

@@ -6,6 +6,6 @@ After=network.target
 Type=oneshot
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_testnet -n
 CPUWeight=50
 SyslogIdentifier=zksync-era-testnet-missing

crawlers/deploy/zksync-era-testnet-moonworm-crawler.service

@@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
 Restart=on-failure
 RestartSec=15s
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era_testnet  --confirmations 20 --min-blocks-batch 20
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_testnet  --confirmations 20 --min-blocks-batch 20
 CPUWeight=70
 SyslogIdentifier=zksync-era-testnet-moonworm-crawler
 

crawlers/deploy/zksync-era-testnet-synchronize.service

@@ -9,7 +9,7 @@ Restart=on-failure
 RestartSec=15s
 WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
 EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
-ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
+ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
 CPUWeight=90
 SyslogIdentifier=zksync-era-testnet-synchronize
 

crawlers/mooncrawl/mooncrawl/actions.py

@@ -1,24 +1,24 @@
-from collections import OrderedDict
-from datetime import datetime
 import hashlib
 import json
 import logging
 import time
-from typing import Any, Dict, Optional, Union
 import uuid
-
+from collections import OrderedDict
-
+from datetime import datetime
-from bugout.data import (
+from typing import Any, Dict, Optional, Union
-    BugoutResources,
-)
-from bugout.exceptions import BugoutResponseException
-from moonstream.client import Moonstream, ENDPOINT_QUERIES, MoonstreamQueryResultUrl  # type: ignore
-import requests  # type: ignore
-from .middleware import MoonstreamHTTPException
-from .settings import bugout_client as bc
-
 
 import boto3  # type: ignore
+import requests  # type: ignore
+from bugout.data import BugoutResources
+from bugout.exceptions import BugoutResponseException
+from moonstream.client import (  # type: ignore
+    ENDPOINT_QUERIES,
+    Moonstream,
+    MoonstreamQueryResultUrl,
+)
+
+from .middleware import MoonstreamHTTPException
+from .settings import bugout_client as bc
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

crawlers/mooncrawl/mooncrawl/api.py

@@ -1,6 +1,7 @@
 """
 The Mooncrawl HTTP API
 """
+
 import logging
 import time
 from cgi import test
@@ -37,7 +38,6 @@ from .settings import (
     MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
     MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
     MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
-    NB_CONTROLLER_ACCESS_ID,
     ORIGINS,
 )
 from .settings import bugout_client as bc
@@ -152,7 +152,6 @@ async def status_handler(
             timescales=stats_update.timescales,
             dashboard=dashboard_resource,
             subscription_by_id=subscription_by_id,
-            access_id=NB_CONTROLLER_ACCESS_ID,
         )
 
     except Exception as e:

crawlers/mooncrawl/mooncrawl/blockchain.py

@@ -21,19 +21,19 @@ from web3.types import BlockData
 from .data import DateRange
 from .db import yield_db_session, yield_db_session_ctx
 from .settings import (
-    MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI,
-    MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI,
     MOONSTREAM_CRAWL_WORKERS,
-    MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI,
-    MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
-    MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI,
-    MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI,
-    MOONSTREAM_XAI_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI,
-    MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI,
-    MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI,
-    MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI,
+    MOONSTREAM_NODE_XAI_A_EXTERNAL_URI,
-    NB_ACCESS_ID_HEADER,
+    MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI,
-    NB_DATA_SOURCE_HEADER,
+    MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI,
+    MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI,
+    MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI,
     WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
 )
 
@@ -48,42 +48,40 @@ class BlockCrawlError(Exception):
 
 
 def connect(
-    blockchain_type: AvailableBlockchainType,
+    blockchain_type: Optional[AvailableBlockchainType] = None,
     web3_uri: Optional[str] = None,
-    access_id: Optional[UUID] = None,
 ) -> Web3:
+    if blockchain_type is None and web3_uri is None:
+        raise Exception("Both blockchain_type and web3_uri could not be None")
+
     web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
 
-    request_kwargs: Any = None
+    request_kwargs: Dict[str, Any] = {"headers": {"Content-Type": "application/json"}}
-    if access_id is not None:
-        request_kwargs = {
-            "headers": {
-                NB_ACCESS_ID_HEADER: str(access_id),
-                NB_DATA_SOURCE_HEADER: "blockchain",
-                "Content-Type": "application/json",
-            }
-        }
     if web3_uri is None:
         if blockchain_type == AvailableBlockchainType.ETHEREUM:
-            web3_uri = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.POLYGON:
-            web3_uri = MOONSTREAM_POLYGON_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.MUMBAI:
-            web3_uri = MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.XDAI:
-            web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI
-        elif blockchain_type == AvailableBlockchainType.WYRM:
-            web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
         elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
-            web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
-            web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI
+        elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
+            web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
-            web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
-            web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI
         elif blockchain_type == AvailableBlockchainType.XAI:
-            web3_uri = MOONSTREAM_XAI_WEB3_PROVIDER_URI
+            web3_uri = MOONSTREAM_NODE_XAI_A_EXTERNAL_URI
+        elif blockchain_type == AvailableBlockchainType.AVALANCHE:
+            web3_uri = MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI
+        elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
+            web3_uri = MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI
         else:
             raise Exception("Wrong blockchain type provided for web3 URI")
 
@@ -164,14 +162,10 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
             if block.get("l1BatchTimestamp") is not None
             else None
         )
-    if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
+    if (
-        block_obj.sha3_uncles = block.get("sha3Uncles", "")
+        blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA
-        block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
+        or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA
-        block_obj.send_count = hex_to_int(block.get("sendCount"))
+    ):
-        block_obj.send_root = block.get("sendRoot", "")
-        block_obj.mix_hash = block.get("mixHash", "")
-
-    if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
         block_obj.sha3_uncles = block.get("sha3Uncles", "")
         block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
         block_obj.send_count = hex_to_int(block.get("sendCount"))
@@ -185,6 +179,16 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
         block_obj.send_root = block.get("sendRoot", "")
         block_obj.mix_hash = block.get("mixHash", "")
 
+    if (
+        blockchain_type == AvailableBlockchainType.AVALANCHE
+        or blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI
+    ):
+        block_obj.mix_hash = block.get("mixHash", "")
+        block_obj.block_extra_data = block.get("blockExtraData")
+        block_obj.block_gas_cost = block.get("blockGasCost")
+        block_obj.ext_data_gas_used = block.get("extDataGasUsed", "")
+        block_obj.ext_data_hash = block.get("extDataHash", "")
+
     db_session.add(block_obj)
 
 
@@ -227,13 +231,11 @@ def add_block_transactions(
                 if tx.get("l1BatchTxIndex") is not None
                 else None
             )
-        if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
+        if (
-            tx_obj.y_parity = hex_to_int(tx.get("yParity"))
+            blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA
-
+            or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA
-        if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
+            or blockchain_type == AvailableBlockchainType.XAI
-            tx_obj.y_parity = hex_to_int(tx.get("yParity"))
+        ):
-
-        if blockchain_type == AvailableBlockchainType.XAI:
             tx_obj.y_parity = hex_to_int(tx.get("yParity"))
 
         db_session.add(tx_obj)
@@ -242,7 +244,7 @@ def add_block_transactions(
 def get_latest_blocks(
     blockchain_type: AvailableBlockchainType,
     confirmations: int = 0,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ) -> Tuple[Optional[int], int]:
     """
     Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method).
@@ -250,7 +252,7 @@ def get_latest_blocks(
     If confirmations > 0, and the latest block on the node has block number N, this returns the block
     with block_number (N - confirmations)
     """
-    web3_client = connect(blockchain_type, access_id=access_id)
+    web3_client = connect(blockchain_type, web3_uri=web3_uri)
     latest_block_number: int = web3_client.eth.block_number
     if confirmations > 0:
         latest_block_number -= confirmations
@@ -273,12 +275,12 @@ def crawl_blocks(
     blockchain_type: AvailableBlockchainType,
     blocks_numbers: List[int],
     with_transactions: bool = False,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ) -> None:
     """
     Open database and geth sessions and fetch block data from blockchain.
     """
-    web3_client = connect(blockchain_type, access_id=access_id)
+    web3_client = connect(blockchain_type, web3_uri=web3_uri)
     with yield_db_session_ctx() as db_session:
         pbar = tqdm(total=len(blocks_numbers))
         for block_number in blocks_numbers:
@@ -319,7 +321,7 @@ def check_missing_blocks(
     blockchain_type: AvailableBlockchainType,
     blocks_numbers: List[int],
     notransactions=False,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ) -> List[int]:
     """
     Query block from postgres. If block does not presented in database,
@@ -358,7 +360,7 @@ def check_missing_blocks(
                 [block[0], block[1]] for block in blocks_exist_raw_query.all()
             ]
 
-            web3_client = connect(blockchain_type, access_id=access_id)
+            web3_client = connect(blockchain_type, web3_uri=web3_uri)
 
             blocks_exist_len = len(blocks_exist)
             pbar = tqdm(total=blocks_exist_len)
@@ -400,7 +402,7 @@ def crawl_blocks_executor(
     block_numbers_list: List[int],
     with_transactions: bool = False,
     num_processes: int = MOONSTREAM_CRAWL_WORKERS,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ) -> None:
     """
     Execute crawler in processes.
@@ -429,7 +431,7 @@ def crawl_blocks_executor(
     if num_processes == 1:
         logger.warning("Executing block crawler in lazy mod")
         return crawl_blocks(
-            blockchain_type, block_numbers_list, with_transactions, access_id=access_id
+            blockchain_type, block_numbers_list, with_transactions, web3_uri=web3_uri
         )
     else:
         with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor:
@@ -441,7 +443,7 @@ def crawl_blocks_executor(
                     blockchain_type,
                     block_chunk,
                     with_transactions,
-                    access_id,
+                    web3_uri,
                 )
                 result.add_done_callback(record_error)
                 results.append(result)

crawlers/mooncrawl/mooncrawl/contract/cli.py

@@ -10,7 +10,6 @@ from web3 import Web3
 
 from ..blockchain import connect
 from ..db import yield_db_session_ctx
-from ..settings import NB_CONTROLLER_ACCESS_ID
 from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore
 
 logging.basicConfig(level=logging.INFO)
@@ -118,7 +117,7 @@ def run_crawler_desc(
 
 def handle_parser(args: argparse.Namespace):
     with yield_db_session_ctx() as session:
-        w3 = connect(AvailableBlockchainType.ETHEREUM, access_id=args.access_id)
+        w3 = connect(AvailableBlockchainType.ETHEREUM, web3_uri=args.web3_uri)
         if args.order == "asc":
             run_crawler_asc(
                 w3=w3,
@@ -156,10 +155,8 @@ def generate_parser():
     parser = argparse.ArgumentParser(description="Moonstream Deployment Crawler")
 
     parser.add_argument(
-        "--access-id",
+        "--web3-uri",
-        default=NB_CONTROLLER_ACCESS_ID,
+        help="Node JSON RPC uri",
-        type=UUID,
-        help="User access ID",
     )
 
     parser.add_argument(

crawlers/mooncrawl/mooncrawl/crawler.py

@@ -1,6 +1,7 @@
 """
 Moonstream crawlers CLI.
 """
+
 import argparse
 import json
 import logging
@@ -12,8 +13,8 @@ from enum import Enum
 from typing import Iterator, List
 from uuid import UUID
 
-from moonstreamdb.blockchain import AvailableBlockchainType
 import dateutil.parser  # type: ignore
+from moonstreamdb.blockchain import AvailableBlockchainType
 
 from .blockchain import (
     DateRange,
@@ -23,7 +24,7 @@ from .blockchain import (
     trending,
 )
 from .publish import publish_json
-from .settings import MOONSTREAM_CRAWL_WORKERS, NB_CONTROLLER_ACCESS_ID
+from .settings import MOONSTREAM_CRAWL_WORKERS
 from .version import MOONCRAWL_VERSION
 
 logging.basicConfig(level=logging.INFO)
@@ -96,7 +97,7 @@ def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
         latest_stored_block_number, latest_block_number = get_latest_blocks(
             AvailableBlockchainType(args.blockchain),
             args.confirmations,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
         if latest_stored_block_number is None:
             latest_stored_block_number = 0
@@ -140,7 +141,7 @@ def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
                 block_numbers_list=blocks_numbers_list,
                 with_transactions=True,
                 num_processes=args.jobs,
-                access_id=args.access_id,
+                web3_uri=args.web3_uri,
             )
         logger.info(
             f"Synchronized blocks from {latest_stored_block_number} to {latest_block_number}"
@@ -159,7 +160,7 @@ def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
             blockchain_type=AvailableBlockchainType(args.blockchain),
             block_numbers_list=blocks_numbers_list,
             with_transactions=True,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
 
     logger.info(
@@ -184,7 +185,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
         _, latest_block_number = get_latest_blocks(
             AvailableBlockchainType(args.blockchain),
             confirmations,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
         start_block_number = (
             latest_block_number - shift if latest_block_number - shift >= 1 else 1
@@ -200,7 +201,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
             blockchain_type=AvailableBlockchainType(args.blockchain),
             blocks_numbers=blocks_numbers_list,
             notransactions=args.notransactions,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
         if len(missing_blocks_numbers) > 0:
             logger.info(f"Found {len(missing_blocks_numbers)} missing blocks")
@@ -217,7 +218,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
             block_numbers_list=missing_blocks_numbers_total,
             with_transactions=True,
             num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
     logger.info(
         f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers "
@@ -259,10 +260,8 @@ def main() -> None:
     time_now = datetime.now(timezone.utc)
 
     parser.add_argument(
-        "--access-id",
+        "--web3-uri",
-        default=NB_CONTROLLER_ACCESS_ID,
+        help="Node JSON RPC uri",
-        type=UUID,
-        help="User access ID",
     )
 
     # Blockchain blocks parser

crawlers/mooncrawl/mooncrawl/generic_crawler/base.py

@@ -11,11 +11,7 @@ from moonstreamdb.blockchain import (
     get_label_model,
     get_transaction_model,
 )
-from moonstreamdb.models import (
+from moonstreamdb.models import Base, EthereumTransaction, PolygonTransaction
-    Base,
-    EthereumTransaction,
-    PolygonTransaction,
-)
 from moonworm.crawler.function_call_crawler import (  # type: ignore
     ContractFunctionCall,
     utfy_dict,
@@ -26,10 +22,7 @@ from tqdm import tqdm
 from web3 import Web3
 from web3._utils.events import get_event_data
 
-from ..moonworm_crawler.db import (
+from ..moonworm_crawler.db import add_events_to_session, commit_session
-    add_events_to_session,
-    commit_session,
-)
 from ..moonworm_crawler.event_crawler import Event, get_block_timestamp
 
 logging.basicConfig(level=logging.INFO)

crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py

@@ -10,7 +10,6 @@ from web3.middleware import geth_poa_middleware
 
 from ..blockchain import connect
 from ..db import yield_db_session_ctx
-from ..settings import NB_CONTROLLER_ACCESS_ID
 from .base import crawl, get_checkpoint, populate_with_events
 
 logging.basicConfig(level=logging.INFO)
@@ -39,7 +38,7 @@ def handle_nft_crawler(args: argparse.Namespace) -> None:
             logger.info(
                 "No web3 provider URL provided, using default (blockchan.py: connect())"
             )
-            web3 = connect(blockchain_type, access_id=args.access_id)
+            web3 = connect(blockchain_type, web3_uri=args.web3_uri)
         else:
             logger.info(f"Using web3 provider URL: {args.web3}")
             web3 = Web3(
@@ -91,7 +90,7 @@ def populate_with_erc20_transfers(args: argparse.Namespace) -> None:
             logger.info(
                 "No web3 provider URL provided, using default (blockchan.py: connect())"
             )
-            web3 = connect(blockchain_type, access_id=args.access_id)
+            web3 = connect(blockchain_type, web3_uri=args.web3_uri)
         else:
             logger.info(f"Using web3 provider URL: {args.web3}")
             web3 = Web3(
@@ -137,7 +136,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
             logger.info(
                 "No web3 provider URL provided, using default (blockchan.py: connect())"
             )
-            web3 = connect(blockchain_type, access_id=args.access_id)
+            web3 = connect(blockchain_type, web3_uri=args.web3_uri)
         else:
             logger.info(f"Using web3 provider URL: {args.web3}")
             web3 = Web3(
@@ -172,10 +171,8 @@ def main():
     subparsers = parser.add_subparsers()
 
     parser.add_argument(
-        "--access-id",
+        "--web3-uri",
-        default=NB_CONTROLLER_ACCESS_ID,
+        help="Node JSON RPC uri",
-        type=UUID,
-        help="User access ID",
     )
 
     crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi")

crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py

@@ -1,24 +1,22 @@
 import argparse
 import json
 import logging
-from typing import cast, List
 import uuid
+from typing import List, cast
 
 import requests  # type: ignore
 from bugout.data import BugoutSearchResult
 
-from .utils import get_results_for_moonstream_query, leaderboard_push_batch
 from ..settings import (
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
-    MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
     MOONSTREAM_API_URL,
     MOONSTREAM_ENGINE_URL,
     MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE,
+    MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
     MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS,
 )
-
 from ..settings import bugout_client as bc
-
+from .utils import get_results_for_moonstream_query, leaderboard_push_batch
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)

crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py

@@ -3,8 +3,7 @@ import json
 import logging
 import os
 import time
-from typing import Any, Dict, Optional, List
+from typing import Any, Dict, List, Optional
-
 
 import requests  # type: ignore
 
@@ -15,7 +14,6 @@ from ..settings import (
     MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS,
 )
 
-
 logging.basicConfig()
 logger = logging.getLogger(__name__)
 

crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py

@@ -9,10 +9,7 @@ from urllib.error import HTTPError
 
 from moonstreamdb.blockchain import AvailableBlockchainType
 
-from ..db import (
+from ..db import yield_db_preping_session_ctx, yield_db_read_only_preping_session_ctx
-    yield_db_preping_session_ctx,
-    yield_db_read_only_preping_session_ctx,
-)
 from .db import (
     clean_labels_from_db,
     get_current_metadata_for_address,

crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py

@@ -1,13 +1,13 @@
-import logging
 import json
-from typing import Dict, Any, Optional, List
+import logging
+from typing import Any, Dict, List, Optional
 
 from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import text
 
 from ..data import TokenURIs
-from ..settings import VIEW_STATE_CRAWLER_LABEL, METADATA_CRAWLER_LABEL, CRAWLER_LABEL
+from ..settings import CRAWLER_LABEL, METADATA_CRAWLER_LABEL, VIEW_STATE_CRAWLER_LABEL
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py

@@ -9,21 +9,20 @@ from web3.middleware import geth_poa_middleware
 
 from ..db import yield_db_session_ctx
 from ..settings import (
-    MOONSTREAM_MOONWORM_TASKS_JOURNAL,
-    NB_CONTROLLER_ACCESS_ID,
-    HISTORICAL_CRAWLER_STATUSES,
     HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
+    HISTORICAL_CRAWLER_STATUSES,
+    MOONSTREAM_MOONWORM_TASKS_JOURNAL,
 )
 from .continuous_crawler import _retry_connect_web3, continuous_crawler
 from .crawler import (
     SubscriptionTypes,
     blockchain_type_to_subscription_type,
+    find_all_deployed_blocks,
     get_crawl_job_entries,
     make_event_crawl_jobs,
     make_function_call_crawl_jobs,
-    find_all_deployed_blocks,
-    update_job_state_with_filters,
     moonworm_crawler_update_job_as_pickedup,
+    update_job_state_with_filters,
 )
 from .db import get_first_labeled_block_number, get_last_labeled_block_number
 from .historical_crawler import historical_crawler
@@ -71,7 +70,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
             logger.info(
                 "No web3 provider URL provided, using default (blockchan.py: connect())"
             )
-            web3 = _retry_connect_web3(blockchain_type, access_id=args.access_id)
+            web3 = _retry_connect_web3(blockchain_type, web3_uri=args.web3_uri)
         else:
             logger.info(f"Using web3 provider URL: {args.web3}")
             web3 = Web3(
@@ -133,7 +132,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
             args.min_sleep_time,
             args.heartbeat_interval,
             args.new_jobs_refetch_interval,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
 
 
@@ -250,7 +249,7 @@ def handle_historical_crawl(args: argparse.Namespace) -> None:
             logger.info(
                 "No web3 provider URL provided, using default (blockchan.py: connect())"
             )
-            web3 = _retry_connect_web3(blockchain_type, access_id=args.access_id)
+            web3 = _retry_connect_web3(blockchain_type, web3_uri=args.web3_uri)
         else:
             logger.info(f"Using web3 provider URL: {args.web3}")
             web3 = Web3(
@@ -336,7 +335,7 @@ def handle_historical_crawl(args: argparse.Namespace) -> None:
             end_block,
             args.max_blocks_batch,
             args.min_sleep_time,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
             addresses_deployment_blocks=addresses_deployment_blocks,
         )
 
@@ -346,10 +345,8 @@ def main() -> None:
     parser.set_defaults(func=lambda _: parser.print_help())
 
     parser.add_argument(
-        "--access-id",
+        "--web3-uri",
-        default=NB_CONTROLLER_ACCESS_ID,
+        help="Node JSON RPC uri",
-        type=UUID,
-        help="User access ID",
     )
 
     subparsers = parser.add_subparsers()

crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py

@@ -100,7 +100,7 @@ def continuous_crawler(
     min_sleep_time: float = 0.1,
     heartbeat_interval: float = 60,
     new_jobs_refetch_interval: float = 120,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
     max_insert_batch: int = 10000,
 ):
     crawler_type = "continuous"
@@ -119,7 +119,7 @@ def continuous_crawler(
 
     jobs_refetchet_time = crawl_start_time
     if web3 is None:
-        web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
+        web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
 
     if blockchain_type == AvailableBlockchainType.ETHEREUM:
         network = Network.ethereum
@@ -304,7 +304,7 @@ def continuous_crawler(
                     logger.error("Too many failures, exiting")
                     raise e
                 try:
-                    web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
+                    web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
                 except Exception as err:
                     logger.error(f"Failed to reconnect: {err}")
                     logger.exception(err)

crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py

@@ -107,7 +107,7 @@ def _retry_connect_web3(
     blockchain_type: AvailableBlockchainType,
     retry_count: int = 10,
     sleep_time: float = 5,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ) -> Web3:
     """
     Retry connecting to the blockchain.
@@ -115,7 +115,7 @@ def _retry_connect_web3(
     while retry_count > 0:
         retry_count -= 1
         try:
-            web3 = connect(blockchain_type, access_id=access_id)
+            web3 = connect(blockchain_type, web3_uri=web3_uri)
             web3.eth.block_number
             logger.info(f"Connected to {blockchain_type}")
             return web3

crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py

@@ -1,12 +1,12 @@
-import logging
 import json
+import logging
 from typing import Dict, List, Optional
 
 from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
 from moonstreamdb.models import Base
 from moonworm.crawler.function_call_crawler import ContractFunctionCall  # type: ignore
+from sqlalchemy import Integer, String, column, exists, func, select, text, values
 from sqlalchemy.orm import Session
-from sqlalchemy import func, text, select, values, column, Integer, String, exists
 
 from ..settings import CRAWLER_LABEL
 from .event_crawler import Event

crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py

@@ -3,7 +3,10 @@ from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Tuple
 
 from moonstreamdb.blockchain import AvailableBlockchainType, get_block_model
-from moonworm.crawler.log_scanner import _fetch_events_chunk, _crawl_events as moonworm_autoscale_crawl_events  # type: ignore
+from moonworm.crawler.log_scanner import (
+    _crawl_events as moonworm_autoscale_crawl_events,  # type: ignore
+)
+from moonworm.crawler.log_scanner import _fetch_events_chunk
 from sqlalchemy.orm.session import Session
 from sqlalchemy.sql.expression import and_
 from web3 import Web3

crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py

@@ -36,7 +36,7 @@ def historical_crawler(
     end_block: int,
     max_blocks_batch: int = 100,
     min_sleep_time: float = 0.1,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
     addresses_deployment_blocks: Optional[Dict[ChecksumAddress, int]] = None,
     max_insert_batch: int = 10000,
 ):
@@ -46,7 +46,7 @@ def historical_crawler(
     assert end_block > 0, "end_block must be greater than 0"
 
     if web3 is None:
-        web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
+        web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
 
     assert (
         web3.eth.block_number >= start_block
@@ -205,7 +205,7 @@ def historical_crawler(
                 logger.error("Too many failures, exiting")
                 raise e
             try:
-                web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
+                web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
             except Exception as err:
                 logger.error(f"Failed to reconnect: {err}")
                 logger.exception(err)

crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py

@@ -3,22 +3,20 @@ import csv
 import datetime
 import json
 import logging
-from io import StringIO
-from moonstream.client import Moonstream  # type: ignore
-import requests  # type: ignore
 import time
+from io import StringIO
+from typing import Any, Dict, Union
 from uuid import UUID
 
-from typing import Any, Dict, Union
+import requests  # type: ignore
+from moonstream.client import Moonstream  # type: ignore
 
-
-from .queries import tokenomics_queries, cu_bank_queries, tokenomics_orange_dao_queries
 from ..actions import recive_S3_data_from_query
 from ..settings import (
     MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
     MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
 )
-
+from .queries import cu_bank_queries, tokenomics_orange_dao_queries, tokenomics_queries
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

crawlers/mooncrawl/mooncrawl/settings.py

@@ -78,72 +78,87 @@ except:
     )
 
 # Geth connection address
-MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI env variable is not set")
+    raise Exception("MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI env variable is not set")
 
-MOONSTREAM_POLYGON_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_POLYGON_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_POLYGON_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_POLYGON_WEB3_PROVIDER_URI env variable is not set")
+    raise Exception("MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI env variable is not set")
 
-MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI env variable is not set")
+    raise Exception("MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI env variable is not set")
 
-MOONSTREAM_XDAI_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_XDAI_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_XDAI_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_XDAI_WEB3_PROVIDER_URI env variable is not set")
+    raise Exception("MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI env variable is not set")
 
-MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_WYRM_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
-
-MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
-    "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
-)
-if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
     raise Exception(
-        "MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
+        "MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI env variable is not set"
     )
 
-MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
+    raise Exception("MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI env variable is not set")
 
-MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI == "":
     raise Exception(
-        "MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set"
+        "MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI env variable is not set"
     )
 
-MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI == "":
     raise Exception(
-        "MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI env variable is not set"
+        "MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI env variable is not set"
     )
 
-MOONSTREAM_XAI_WEB3_PROVIDER_URI = os.environ.get(
+MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI = os.environ.get(
-    "MOONSTREAM_XAI_WEB3_PROVIDER_URI", ""
+    "MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI", ""
 )
-if MOONSTREAM_XAI_WEB3_PROVIDER_URI == "":
+if MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI == "":
-    raise Exception("MOONSTREAM_XAI_WEB3_PROVIDER_URI env variable is not set")
+    raise Exception(
+        "MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI env variable is not set"
+    )
 
+MOONSTREAM_NODE_XAI_A_EXTERNAL_URI = os.environ.get(
+    "MOONSTREAM_NODE_XAI_A_EXTERNAL_URI", ""
+)
+if MOONSTREAM_NODE_XAI_A_EXTERNAL_URI == "":
+    raise Exception("MOONSTREAM_NODE_XAI_A_EXTERNAL_URI env variable is not set")
+
+MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI = os.environ.get(
+    "MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI", ""
+)
+if MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI == "":
+    raise Exception("MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI env variable is not set")
+
+MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI = os.environ.get(
+    "MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI", ""
+)
+if MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI == "":
+    raise Exception(
+        "MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI env variable is not set"
+    )
 
 MOONSTREAM_CRAWL_WORKERS = 4
 MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")

crawlers/mooncrawl/mooncrawl/state_crawler/cli.py

@@ -10,23 +10,19 @@ from pprint import pprint
 from typing import Any, Dict, List, Optional
 from uuid import UUID
 
-from moonstreamdb.blockchain import AvailableBlockchainType
 from moonstream.client import Moonstream  # type: ignore
+from moonstreamdb.blockchain import AvailableBlockchainType
 from web3.middleware import geth_poa_middleware
 
 from mooncrawl.moonworm_crawler.crawler import _retry_connect_web3
 
 from ..actions import recive_S3_data_from_query
+from ..blockchain import connect
 from ..db import PrePing_SessionLocal
-from ..settings import (
+from ..settings import INFURA_PROJECT_ID, infura_networks, multicall_contracts
-    INFURA_PROJECT_ID,
-    NB_CONTROLLER_ACCESS_ID,
-    infura_networks,
-    multicall_contracts,
-)
 from .db import clean_labels, commit_session, view_call_to_label
 from .Multicall2_interface import Contract as Multicall2
-from .web3_util import FunctionSignature, connect
+from .web3_util import FunctionSignature
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -306,8 +302,8 @@ def parse_jobs(
     web3_provider_uri: Optional[str],
     block_number: Optional[int],
     batch_size: int,
-    access_id: UUID,
     moonstream_token: str,
+    web3_uri: Optional[str] = None,
 ):
     """
     Parse jobs from list and generate web3 interfaces for each contract.
@@ -321,13 +317,12 @@ def parse_jobs(
     if web3_provider_uri is not None:
         try:
             logger.info(
-                f"Connecting to blockchain                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    : {blockchain_type} with custom provider!"
+                f"Connecting to blockchain: {blockchain_type} with custom provider!"
             )
 
-            web3_client = connect(web3_provider_uri)
+            web3_client = connect(
-
+                blockchain_type=blockchain_type, web3_uri=web3_provider_uri
-            if blockchain_type != AvailableBlockchainType.ETHEREUM:
+            )
-                web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
         except Exception as e:
             logger.error(
                 f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}"
@@ -336,7 +331,7 @@ def parse_jobs(
     else:
         logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
         web3_client = _retry_connect_web3(
-            blockchain_type=blockchain_type, access_id=access_id
+            blockchain_type=blockchain_type, web3_uri=web3_uri
         )
 
     logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
@@ -519,7 +514,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
 
     blockchain_type = AvailableBlockchainType(args.blockchain)
 
-    custom_web3_provider = args.custom_web3_provider
+    custom_web3_provider = args.web3_uri
 
     if args.infura and INFURA_PROJECT_ID is not None:
         if blockchain_type not in infura_networks:
@@ -535,8 +530,8 @@ def handle_crawl(args: argparse.Namespace) -> None:
         custom_web3_provider,
         args.block_number,
         args.batch_size,
-        args.access_id,
         args.moonstream_token,
+        args.web3_uri,
     )
 
 
@@ -563,7 +558,7 @@ def clean_labels_handler(args: argparse.Namespace) -> None:
     blockchain_type = AvailableBlockchainType(args.blockchain)
 
     web3_client = _retry_connect_web3(
-        blockchain_type=blockchain_type, access_id=args.access_id
+        blockchain_type=blockchain_type, web3_uri=args.web3_uri
     )
 
     logger.info(f"Label cleaner connected to blockchain: {blockchain_type}")
@@ -583,10 +578,8 @@ def main() -> None:
     parser.set_defaults(func=lambda _: parser.print_help())
 
     parser.add_argument(
-        "--access-id",
+        "--web3-uri",
-        default=NB_CONTROLLER_ACCESS_ID,
+        help="Node JSON RPC uri",
-        type=UUID,
-        help="User access ID",
     )
 
     subparsers = parser.add_subparsers()
@@ -614,12 +607,6 @@ def main() -> None:
         action="store_true",
         help="Use infura as web3 provider",
     )
-    view_state_crawler_parser.add_argument(
-        "--custom-web3-provider",
-        "-w3",
-        type=str,
-        help="Type of blovkchain wich writng in database",
-    )
     view_state_crawler_parser.add_argument(
         "--block-number", "-N", type=str, help="Block number."
     )

crawlers/mooncrawl/mooncrawl/state_crawler/db.py

@@ -1,6 +1,6 @@
-import logging
 import json
-from typing import Dict, Any
+import logging
+from typing import Any, Dict
 
 from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
 from sqlalchemy.orm import Session

crawlers/mooncrawl/mooncrawl/state_crawler/web3_util.py

@@ -2,17 +2,17 @@ import getpass
 import os
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
+from eth_abi import decode_single, encode_single
 from eth_account.account import Account  # type: ignore
-from eth_abi import encode_single, decode_single
 from eth_typing.evm import ChecksumAddress
 from eth_utils import function_signature_to_4byte_selector
 from hexbytes.main import HexBytes
 from web3 import Web3
+from web3._utils.abi import normalize_event_input_types
 from web3.contract import ContractFunction
 from web3.providers.ipc import IPCProvider
 from web3.providers.rpc import HTTPProvider
 from web3.types import ABI, Nonce, TxParams, TxReceipt, Wei
-from web3._utils.abi import normalize_event_input_types
 
 
 class ContractConstructor:
@@ -165,28 +165,6 @@ def read_keys_from_env() -> Tuple[ChecksumAddress, str]:
         )
 
 
-def connect(web3_uri: str) -> Web3:
-    web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
-    if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
-        web3_provider = Web3.HTTPProvider(web3_uri)
-    else:
-        web3_provider = Web3.IPCProvider(web3_uri)
-    web3_client = Web3(web3_provider)
-    return web3_client
-
-
-def read_web3_provider_from_env() -> Web3:
-    provider_path = os.environ.get("MOONWORM_WEB3_PROVIDER_URI")
-    if provider_path is None:
-        raise ValueError("MOONWORM_WEB3_PROVIDER_URI env variable is not set")
-    return connect(provider_path)
-
-
-def read_web3_provider_from_cli() -> Web3:
-    provider_path = input("Enter web3 uri path: ")
-    return connect(provider_path)
-
-
 def cast_to_python_type(evm_type: str) -> Callable:
     if evm_type.startswith(("uint", "int")):
         return int

crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py

@@ -40,7 +40,6 @@ from ..settings import (
     MOONSTREAM_ADMIN_ACCESS_TOKEN,
     MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
     MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
-    NB_CONTROLLER_ACCESS_ID,
 )
 from ..settings import bugout_client as bc
 
@@ -367,7 +366,7 @@ def generate_list_of_names(
 def process_external_merged(
     external_calls: Dict[str, Dict[str, Any]],
     blockchain: AvailableBlockchainType,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ):
     """
     Process external calls
@@ -412,7 +411,7 @@ def process_external_merged(
             logger.error(f"Error processing external call: {e}")
 
     if external_calls_normalized:
-        web3_client = connect(blockchain, access_id=access_id)
+        web3_client = connect(blockchain, web3_uri=web3_uri)
 
     for extcall in external_calls_normalized:
         try:
@@ -433,7 +432,7 @@ def process_external_merged(
 def process_external(
     abi_external_calls: List[Dict[str, Any]],
     blockchain: AvailableBlockchainType,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ):
     """
     Request all required external data
@@ -479,7 +478,7 @@ def process_external(
             logger.error(f"Error processing external call: {e}")
 
     if external_calls:
-        web3_client = connect(blockchain, access_id=access_id)
+        web3_client = connect(blockchain, web3_uri=web3_uri)
 
     for extcall in external_calls:
         try:
@@ -530,7 +529,7 @@ def generate_web3_metrics(
     address: str,
     crawler_label: str,
     abi_json: Any,
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ) -> List[Any]:
     """
     Generate stats for cards components
@@ -543,7 +542,7 @@ def generate_web3_metrics(
     extention_data = process_external(
         abi_external_calls=abi_external_calls,
         blockchain=blockchain_type,
-        access_id=access_id,
+        web3_uri=web3_uri,
     )
 
     extention_data.append(
@@ -873,7 +872,7 @@ def stats_generate_handler(args: argparse.Namespace):
         external_calls_results = process_external_merged(
             external_calls=merged_external_calls["merged"],
             blockchain=blockchain_type,
-            access_id=args.access_id,
+            web3_uri=args.web3_uri,
         )
 
         for address in address_dashboard_id_subscription_id_tree.keys():
@@ -1039,7 +1038,7 @@ def stats_generate_api_task(
     timescales: List[str],
     dashboard: BugoutResource,
     subscription_by_id: Dict[str, BugoutJournalEntity],
-    access_id: Optional[UUID] = None,
+    web3_uri: Optional[str] = None,
 ):
     """
     Start crawler with generate.
@@ -1115,7 +1114,7 @@ def stats_generate_api_task(
                     address=address,  # type: ignore
                     crawler_label=crawler_label,
                     abi_json=abi_json,
-                    access_id=access_id,
+                    web3_uri=web3_uri,
                 )
 
                 # Generate blocks state information
@@ -1192,10 +1191,8 @@ def main() -> None:
     parser.set_defaults(func=lambda _: parser.print_help())
 
     parser.add_argument(
-        "--access-id",
+        "--web3-uri",
-        default=NB_CONTROLLER_ACCESS_ID,
+        help="Node JSON RPC uri",
-        type=UUID,
-        help="User access ID",
     )
 
     subcommands = parser.add_subparsers(

crawlers/mooncrawl/mooncrawl/stats_worker/queries.py

@@ -1,21 +1,17 @@
 import csv
-from collections import OrderedDict
 import hashlib
 import json
 import logging
 import re
+from collections import OrderedDict
 from io import StringIO
 from typing import Any, Dict
 
-
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.sql import text
 from sqlalchemy.sql.expression import TextClause
 
 from ..actions import push_data_to_bucket
-from ..reporter import reporter
-
-
 from ..db import RO_pre_ping_query_engine
 from ..reporter import reporter
 from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX

crawlers/mooncrawl/sample.env

@@ -21,16 +21,18 @@ export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout_access_token_for_moonstream>"
 export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
 
 # Blockchain nodes environment variables
-export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
+export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_polygon_node>"
+export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumbai_node>"
+export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
+export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
+export MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
+export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_node>"
+export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_nova_node>"
+export MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_sepolia_node>"
+export MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
-export MOONSTREAM_XAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xai_node>"
+export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
+export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
+export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
 export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
 
 # AWS environment variables

crawlers/mooncrawl/setup.py

@@ -37,7 +37,7 @@ setup(
         "bugout>=0.2.13",
         "chardet",
         "fastapi",
-        "moonstreamdb>=0.3.8",
+        "moonstreamdb>=0.3.9",
         "moonstream>=0.1.1",
         "moonworm[moonstream]>=0.6.2",
         "humbug",