mirror
/
moonstream
kopia lustrzana https://github.com/bugout-dev/moonstream
1
0
Forkuj 0
Kod Zgłoszenia Projekty Wydania Wiki Aktywność

Merge pull request #1038 from moonstream-to/zk-sepolia-no-access-id 

Switch from access-id to web3-uri, zksync sepolia and avalanche
pull/1039/head
Sergei Sumarokov 2024-03-28 16:48:09 +03:00 zatwierdzone przez GitHub
rodzic e8051972f0 3b0729f439
commit 0b9e4b1008
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: B5690EEEBB952194
80 zmienionych plików z 359 dodań i 353 usunięć
Pokaż statystyki Ściągnij plik aktualizacji Ściągnij plik porównania Expand all files Collapse all files

2
crawlers/deploy/arbitrum-nova-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_nova -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_nova -n
CPUWeight=50
SyslogIdentifier=arbitrum-nova-missing

2
crawlers/deploy/arbitrum-nova-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_nova --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=arbitrum-nova-moonworm-crawler

2
crawlers/deploy/arbitrum-nova-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_nova -c 10 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_nova -c 10 -j 2
CPUWeight=90
SyslogIdentifier=arbitrum-nova-synchronize

2
crawlers/deploy/arbitrum-sepolia-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain arbitrum_sepolia -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_sepolia -n
CPUWeight=50
SyslogIdentifier=arbitrum-sepolia-missing

2
crawlers/deploy/arbitrum-sepolia-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_sepolia --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=arbitrum-sepolia-moonworm-crawler

2
crawlers/deploy/arbitrum-sepolia-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_sepolia -c 10 -j 2
CPUWeight=90
SyslogIdentifier=arbitrum-sepolia-synchronize

41
crawlers/deploy/deploy.bash
Wyświetl plik

@ -31,8 +31,6 @@ LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
# Ethereum service files
ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service"
@ -106,6 +104,10 @@ ZKSYNC_ERA_STATE_SERVICE_FILE="zksync-era-state.service"
ZKSYNC_ERA_STATE_TIMER_FILE="zksync-era-state.timer"
ZKSYNC_ERA_STATE_CLEAN_SERVICE_FILE="zksync-era-state-clean.service"
ZKSYNC_ERA_STATE_CLEAN_TIMER_FILE="zksync-era-state-clean.timer"
ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE="zksync-era-sepolia-synchronize.service"
ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE="zksync-era-sepolia-missing.service"
ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE="zksync-era-sepolia-missing.timer"
ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-sepolia-moonworm-crawler.service"
# ZkSync Era testnet
ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service"
@ -190,15 +192,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}" "/home/ubuntu/.config/sy
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_SYNCHRONIZE_SERVICE_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum trending service and timer with: ${ETHEREUM_TRENDING_SERVICE_FILE}, ${ETHEREUM_TRENDING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TRENDING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TRENDING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
@ -582,6 +575,32 @@ cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/syste
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
# ZkSync Era Sepolia
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era Sepolia block with transactions syncronizer service definition with ${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era Sepolia missing service and timer with: ${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}"
echo
echo
echo -e "${PREFIX_INFO} Replacing existing ZkSync Era Sepolia moonworm crawler service definition with ${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}"
# Arbitrum Nova
echo
echo

2
crawlers/deploy/ethereum-historical-crawl-events.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=ethereum-historical-crawl-events

2
crawlers/deploy/ethereum-historical-crawl-transactions.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type ethereum --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=ethereum-historical-crawl-transactions

2
crawlers/deploy/ethereum-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain ethereum -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain ethereum -n
CPUWeight=50
SyslogIdentifier=ethereum-missing

2
crawlers/deploy/ethereum-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b ethereum
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b ethereum
CPUWeight=70
SyslogIdentifier=ethereum-moonworm-crawler

2
crawlers/deploy/ethereum-state-clean.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain ethereum -N 10000
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain ethereum -N 10000
CPUWeight=60
SyslogIdentifier=ethereum-state-clean

2
crawlers/deploy/ethereum-state.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain ethereum --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain ethereum --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/ethereum-jobs.json
CPUWeight=60
SyslogIdentifier=ethereum-state

2
crawlers/deploy/ethereum-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain ethereum -c 6 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain ethereum -c 6 -j 2
CPUWeight=90
SyslogIdentifier=ethereum-synchronize

9
crawlers/deploy/ethereum-trending.timer
Wyświetl plik

@ -1,9 +0,0 @@
[Unit]
Description=Load trending Ethereum addresses to the database every 5 minutes
[Timer]
OnBootSec=60s
OnUnitActiveSec=5m
[Install]
WantedBy=timers.target

2
crawlers/deploy/mumbai-historical-crawl-events.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=mumbai-historical-crawl-events

2
crawlers/deploy/mumbai-historical-crawl-transactions.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type mumbai --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=mumbai-historical-crawl-transactions

2
crawlers/deploy/mumbai-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain mumbai -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain mumbai -n
CPUWeight=50
SyslogIdentifier=mumbai-missing

2
crawlers/deploy/mumbai-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b mumbai --confirmations 40
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b mumbai --confirmations 40
CPUWeight=70
SyslogIdentifier=mumbai-moonworm-crawler

2
crawlers/deploy/mumbai-state-clean.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain mumbai -N 10000
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain mumbai -N 10000
CPUWeight=60
SyslogIdentifier=mumbai-state-clean

2
crawlers/deploy/mumbai-state.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain mumbai --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/mumbai-jobs.json
CPUWeight=60
SyslogIdentifier=mumbai-state

2
crawlers/deploy/mumbai-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain mumbai -c 20 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain mumbai -c 20 -j 2
CPUWeight=90
SyslogIdentifier=mumbai-synchronize

2
crawlers/deploy/polygon-historical-crawl-events.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=polygon-historical-crawl-events

2
crawlers/deploy/polygon-historical-crawl-transactions.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type polygon --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=polygon-historical-crawl-transactions

2
crawlers/deploy/polygon-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain polygon -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain polygon -n
CPUWeight=50
SyslogIdentifier=polygon-missing

2
crawlers/deploy/polygon-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b polygon --confirmations 50 --min-blocks-batch 20
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b polygon --confirmations 50 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=polygon-moonworm-crawler

2
crawlers/deploy/polygon-state-clean.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain polygon -N 10000
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain polygon -N 10000
CPUWeight=60
SyslogIdentifier=polygon-state-clean

2
crawlers/deploy/polygon-state.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain polygon --infura --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/polygon-jobs.json
CPUWeight=60
SyslogIdentifier=polygon-state

2
crawlers/deploy/polygon-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain polygon --confirmations 40 --jobs 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain polygon --confirmations 40 --jobs 2
CPUWeight=90
SyslogIdentifier=polygon-synchronize

2
crawlers/deploy/xai-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xai -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain xai -n
CPUWeight=50
SyslogIdentifier=xai-missing

2
crawlers/deploy/xai-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xai --confirmations 10 --min-blocks-batch 20
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b xai --confirmations 10 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=xai-moonworm-crawler

2
crawlers/deploy/xai-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xai -c 10 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain xai -c 10 -j 2
CPUWeight=90
SyslogIdentifier=xai-synchronize

2
crawlers/deploy/xdai-historical-crawl-events.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=xdai-historical-crawl-events

2
crawlers/deploy/xdai-historical-crawl-transactions.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type xdai --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=xdai-historical-crawl-transactions

2
crawlers/deploy/xdai-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain xdai -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain xdai -n
CPUWeight=50
SyslogIdentifier=xdai-missing

2
crawlers/deploy/xdai-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b xdai
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b xdai
CPUWeight=70
SyslogIdentifier=xdai-moonworm-crawler

2
crawlers/deploy/xdai-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain xdai -c 60 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain xdai -c 60 -j 2
CPUWeight=90
SyslogIdentifier=xdai-synchronize

2
crawlers/deploy/zksync-era-historical-crawl-events.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=zksync-era-historical-crawl-events

2
crawlers/deploy/zksync-era-historical-crawl-transactions.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=zksync-era-historical-crawl-transactions

2
crawlers/deploy/zksync-era-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era -n
CPUWeight=50
SyslogIdentifier=zksync-era-missing

2
crawlers/deploy/zksync-era-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era --confirmations 20 --min-blocks-batch 20
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era --confirmations 20 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=zksync-era-moonworm-crawler

8
crawlers/deploy/ethereum-trending.service → crawlers/deploy/zksync-era-sepolia-missing.service
Wyświetl plik

@ -1,11 +1,11 @@
[Unit]
Description=Load trending Ethereum addresses to the database
Description=Fill missing blocks at ZkSync Era Sepolia database
After=network.target
[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" trending
CPUWeight=30
SyslogIdentifier=ethereum-trending
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_sepolia -n
CPUWeight=50
SyslogIdentifier=zksync-era-sepolia-missing

9
crawlers/deploy/zksync-era-sepolia-missing.timer 100644
Wyświetl plik

@ -0,0 +1,9 @@
[Unit]
Description=Fill missing blocks at ZkSync Era Sepolia database
[Timer]
OnBootSec=120s
OnUnitActiveSec=15m
[Install]
WantedBy=timers.target

17
crawlers/deploy/zksync-era-sepolia-moonworm-crawler.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=ZkSync Era Sepolia moonworm crawler
After=network.target
StartLimitIntervalSec=300
StartLimitBurst=3
[Service]
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_sepolia --confirmations 20 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=zksync-era-sepolia-moonworm-crawler
[Install]
WantedBy=multi-user.target

17
crawlers/deploy/zksync-era-sepolia-synchronize.service 100644
Wyświetl plik

@ -0,0 +1,17 @@
[Unit]
Description=ZkSync Era block with transactions synchronizer
StartLimitIntervalSec=300
StartLimitBurst=3
After=network.target
[Service]
Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_sepolia -c 20 -j 2
CPUWeight=90
SyslogIdentifier=zksync-era-sepolia-synchronize
[Install]
WantedBy=multi-user.target

2
crawlers/deploy/zksync-era-state-clean.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" clean-state-labels --blockchain zksync_era -N 10000
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli clean-state-labels --blockchain zksync_era -N 10000
CPUWeight=60
SyslogIdentifier=zksync-era-state-clean

2
crawlers/deploy/zksync-era-state.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.state_crawler.cli crawl-jobs --moonstream-token "${MOONSTREAM_PUBLIC_QUERIES_DATA_ACCESS_TOKEN}" --blockchain zksync_era --jobs-file /home/ubuntu/moonstream/crawlers/mooncrawl/mooncrawl/state_crawler/jobs/zksync-era-jobs.json
CPUWeight=60
SyslogIdentifier=zksync-era-state

2
crawlers/deploy/zksync-era-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era -c 20 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era -c 20 -j 2
CPUWeight=90
SyslogIdentifier=zksync-era-synchronize

2
crawlers/deploy/zksync-era-testnet-historical-crawl-events.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-events
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-events
CPUWeight=70
SyslogIdentifier=zksync-era-testnet-historical-crawl-events

2
crawlers/deploy/zksync-era-testnet-historical-crawl-transactions.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-functions
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type zksync_era_testnet --find-deployed-blocks --end 0 --tasks-journal --only-functions
CPUWeight=70
SyslogIdentifier=zksync-era-testnet-historical-crawl-transactions

2
crawlers/deploy/zksync-era-testnet-missing.service
Wyświetl plik

@ -6,6 +6,6 @@ After=network.target
Type=oneshot
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era_testnet -n
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain zksync_era_testnet -n
CPUWeight=50
SyslogIdentifier=zksync-era-testnet-missing

2
crawlers/deploy/zksync-era-testnet-moonworm-crawler.service
Wyświetl plik

@ -9,7 +9,7 @@ WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
Restart=on-failure
RestartSec=15s
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era_testnet --confirmations 20 --min-blocks-batch 20
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b zksync_era_testnet --confirmations 20 --min-blocks-batch 20
CPUWeight=70
SyslogIdentifier=zksync-era-testnet-moonworm-crawler

2
crawlers/deploy/zksync-era-testnet-synchronize.service
Wyświetl plik

@ -9,7 +9,7 @@ Restart=on-failure
RestartSec=15s
WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl
EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain zksync_era_testnet -c 20 -j 2
CPUWeight=90
SyslogIdentifier=zksync-era-testnet-synchronize

28
crawlers/mooncrawl/mooncrawl/actions.py
Wyświetl plik

@ -1,24 +1,24 @@
from collections import OrderedDict
from datetime import datetime
import hashlib
import json
import logging
import time
from typing import Any, Dict, Optional, Union
import uuid
from bugout.data import (
BugoutResources,
)
from bugout.exceptions import BugoutResponseException
from moonstream.client import Moonstream, ENDPOINT_QUERIES, MoonstreamQueryResultUrl # type: ignore
import requests # type: ignore
from .middleware import MoonstreamHTTPException
from .settings import bugout_client as bc
from collections import OrderedDict
from datetime import datetime
from typing import Any, Dict, Optional, Union
import boto3 # type: ignore
import requests # type: ignore
from bugout.data import BugoutResources
from bugout.exceptions import BugoutResponseException
from moonstream.client import ( # type: ignore
ENDPOINT_QUERIES,
Moonstream,
MoonstreamQueryResultUrl,
)
from .middleware import MoonstreamHTTPException
from .settings import bugout_client as bc
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

3
crawlers/mooncrawl/mooncrawl/api.py
Wyświetl plik

@ -1,6 +1,7 @@
"""
The Mooncrawl HTTP API
"""
import logging
import time
from cgi import test
@ -37,7 +38,6 @@ from .settings import (
MOONSTREAM_S3_QUERIES_BUCKET_PREFIX,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
NB_CONTROLLER_ACCESS_ID,
ORIGINS,
)
from .settings import bugout_client as bc
@ -152,7 +152,6 @@ async def status_handler(
timescales=stats_update.timescales,
dashboard=dashboard_resource,
subscription_by_id=subscription_by_id,
access_id=NB_CONTROLLER_ACCESS_ID,
)
except Exception as e:

118
crawlers/mooncrawl/mooncrawl/blockchain.py
Wyświetl plik

@ -21,19 +21,19 @@ from web3.types import BlockData
from .data import DateRange
from .db import yield_db_session, yield_db_session_ctx
from .settings import (
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI,
MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI,
MOONSTREAM_CRAWL_WORKERS,
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI,
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI,
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI,
MOONSTREAM_WYRM_WEB3_PROVIDER_URI,
MOONSTREAM_XAI_WEB3_PROVIDER_URI,
MOONSTREAM_XDAI_WEB3_PROVIDER_URI,
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI,
MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI,
NB_ACCESS_ID_HEADER,
NB_DATA_SOURCE_HEADER,
MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI,
MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI,
MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI,
MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI,
MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI,
MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI,
MOONSTREAM_NODE_XAI_A_EXTERNAL_URI,
MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI,
MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI,
MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI,
WEB3_CLIENT_REQUEST_TIMEOUT_SECONDS,
)
@ -48,42 +48,40 @@ class BlockCrawlError(Exception):
def connect(
blockchain_type: AvailableBlockchainType,
blockchain_type: Optional[AvailableBlockchainType] = None,
web3_uri: Optional[str] = None,
access_id: Optional[UUID] = None,
) -> Web3:
if blockchain_type is None and web3_uri is None:
raise Exception("Both blockchain_type and web3_uri could not be None")
web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
request_kwargs: Any = None
if access_id is not None:
request_kwargs = {
"headers": {
NB_ACCESS_ID_HEADER: str(access_id),
NB_DATA_SOURCE_HEADER: "blockchain",
"Content-Type": "application/json",
}
}
request_kwargs: Dict[str, Any] = {"headers": {"Content-Type": "application/json"}}
if web3_uri is None:
if blockchain_type == AvailableBlockchainType.ETHEREUM:
web3_uri = MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.POLYGON:
web3_uri = MOONSTREAM_POLYGON_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.MUMBAI:
web3_uri = MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.XDAI:
web3_uri = MOONSTREAM_XDAI_WEB3_PROVIDER_URI
elif blockchain_type == AvailableBlockchainType.WYRM:
web3_uri = MOONSTREAM_WYRM_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_TESTNET:
web3_uri = MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA:
web3_uri = MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ZKSYNC_ERA_SEPOLIA:
web3_uri = MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
web3_uri = MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
web3_uri = MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.XAI:
web3_uri = MOONSTREAM_XAI_WEB3_PROVIDER_URI
web3_uri = MOONSTREAM_NODE_XAI_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.AVALANCHE:
web3_uri = MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI
elif blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI:
web3_uri = MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI
else:
raise Exception("Wrong blockchain type provided for web3 URI")
@ -164,14 +162,10 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
if block.get("l1BatchTimestamp") is not None
else None
)
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
block_obj.send_count = hex_to_int(block.get("sendCount"))
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
if (
blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA
or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA
):
block_obj.sha3_uncles = block.get("sha3Uncles", "")
block_obj.l1_block_number = hex_to_int(block.get("l1BlockNumber"))
block_obj.send_count = hex_to_int(block.get("sendCount"))
@ -185,6 +179,16 @@ def add_block(db_session, block: Any, blockchain_type: AvailableBlockchainType)
block_obj.send_root = block.get("sendRoot", "")
block_obj.mix_hash = block.get("mixHash", "")
if (
blockchain_type == AvailableBlockchainType.AVALANCHE
or blockchain_type == AvailableBlockchainType.AVALANCHE_FUJI
):
block_obj.mix_hash = block.get("mixHash", "")
block_obj.block_extra_data = block.get("blockExtraData")
block_obj.block_gas_cost = block.get("blockGasCost")
block_obj.ext_data_gas_used = block.get("extDataGasUsed", "")
block_obj.ext_data_hash = block.get("extDataHash", "")
db_session.add(block_obj)
@ -227,13 +231,11 @@ def add_block_transactions(
if tx.get("l1BatchTxIndex") is not None
else None
)
if blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA:
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
if blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA:
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
if blockchain_type == AvailableBlockchainType.XAI:
if (
blockchain_type == AvailableBlockchainType.ARBITRUM_NOVA
or blockchain_type == AvailableBlockchainType.ARBITRUM_SEPOLIA
or blockchain_type == AvailableBlockchainType.XAI
):
tx_obj.y_parity = hex_to_int(tx.get("yParity"))
db_session.add(tx_obj)
@ -242,7 +244,7 @@ def add_block_transactions(
def get_latest_blocks(
blockchain_type: AvailableBlockchainType,
confirmations: int = 0,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
) -> Tuple[Optional[int], int]:
"""
Retrieve the latest block from the connected node (connection is created by the connect(AvailableBlockchainType) method).
@ -250,7 +252,7 @@ def get_latest_blocks(
If confirmations > 0, and the latest block on the node has block number N, this returns the block
with block_number (N - confirmations)
"""
web3_client = connect(blockchain_type, access_id=access_id)
web3_client = connect(blockchain_type, web3_uri=web3_uri)
latest_block_number: int = web3_client.eth.block_number
if confirmations > 0:
latest_block_number -= confirmations
@ -273,12 +275,12 @@ def crawl_blocks(
blockchain_type: AvailableBlockchainType,
blocks_numbers: List[int],
with_transactions: bool = False,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
) -> None:
"""
Open database and geth sessions and fetch block data from blockchain.
"""
web3_client = connect(blockchain_type, access_id=access_id)
web3_client = connect(blockchain_type, web3_uri=web3_uri)
with yield_db_session_ctx() as db_session:
pbar = tqdm(total=len(blocks_numbers))
for block_number in blocks_numbers:
@ -319,7 +321,7 @@ def check_missing_blocks(
blockchain_type: AvailableBlockchainType,
blocks_numbers: List[int],
notransactions=False,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
) -> List[int]:
"""
Query block from postgres. If block does not presented in database,
@ -358,7 +360,7 @@ def check_missing_blocks(
[block[0], block[1]] for block in blocks_exist_raw_query.all()
]
web3_client = connect(blockchain_type, access_id=access_id)
web3_client = connect(blockchain_type, web3_uri=web3_uri)
blocks_exist_len = len(blocks_exist)
pbar = tqdm(total=blocks_exist_len)
@ -400,7 +402,7 @@ def crawl_blocks_executor(
block_numbers_list: List[int],
with_transactions: bool = False,
num_processes: int = MOONSTREAM_CRAWL_WORKERS,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
) -> None:
"""
Execute crawler in processes.
@ -429,7 +431,7 @@ def crawl_blocks_executor(
if num_processes == 1:
logger.warning("Executing block crawler in lazy mod")
return crawl_blocks(
blockchain_type, block_numbers_list, with_transactions, access_id=access_id
blockchain_type, block_numbers_list, with_transactions, web3_uri=web3_uri
)
else:
with ThreadPoolExecutor(max_workers=MOONSTREAM_CRAWL_WORKERS) as executor:
@ -441,7 +443,7 @@ def crawl_blocks_executor(
blockchain_type,
block_chunk,
with_transactions,
access_id,
web3_uri,
)
result.add_done_callback(record_error)
results.append(result)

9
crawlers/mooncrawl/mooncrawl/contract/cli.py
Wyświetl plik

@ -10,7 +10,6 @@ from web3 import Web3
from ..blockchain import connect
from ..db import yield_db_session_ctx
from ..settings import NB_CONTROLLER_ACCESS_ID
from .deployment_crawler import ContractDeploymentCrawler, MoonstreamDataStore
logging.basicConfig(level=logging.INFO)
@ -118,7 +117,7 @@ def run_crawler_desc(
def handle_parser(args: argparse.Namespace):
with yield_db_session_ctx() as session:
w3 = connect(AvailableBlockchainType.ETHEREUM, access_id=args.access_id)
w3 = connect(AvailableBlockchainType.ETHEREUM, web3_uri=args.web3_uri)
if args.order == "asc":
run_crawler_asc(
w3=w3,
@ -156,10 +155,8 @@ def generate_parser():
parser = argparse.ArgumentParser(description="Moonstream Deployment Crawler")
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
"--web3-uri",
help="Node JSON RPC uri",
)
parser.add_argument(

23
crawlers/mooncrawl/mooncrawl/crawler.py
Wyświetl plik

@ -1,6 +1,7 @@
"""
Moonstream crawlers CLI.
"""
import argparse
import json
import logging
@ -12,8 +13,8 @@ from enum import Enum
from typing import Iterator, List
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
import dateutil.parser # type: ignore
from moonstreamdb.blockchain import AvailableBlockchainType
from .blockchain import (
DateRange,
@ -23,7 +24,7 @@ from .blockchain import (
trending,
)
from .publish import publish_json
from .settings import MOONSTREAM_CRAWL_WORKERS, NB_CONTROLLER_ACCESS_ID
from .settings import MOONSTREAM_CRAWL_WORKERS
from .version import MOONCRAWL_VERSION
logging.basicConfig(level=logging.INFO)
@ -96,7 +97,7 @@ def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
latest_stored_block_number, latest_block_number = get_latest_blocks(
AvailableBlockchainType(args.blockchain),
args.confirmations,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
if latest_stored_block_number is None:
latest_stored_block_number = 0
@ -140,7 +141,7 @@ def crawler_blocks_sync_handler(args: argparse.Namespace) -> None:
block_numbers_list=blocks_numbers_list,
with_transactions=True,
num_processes=args.jobs,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
logger.info(
f"Synchronized blocks from {latest_stored_block_number} to {latest_block_number}"
@ -159,7 +160,7 @@ def crawler_blocks_add_handler(args: argparse.Namespace) -> None:
blockchain_type=AvailableBlockchainType(args.blockchain),
block_numbers_list=blocks_numbers_list,
with_transactions=True,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
logger.info(
@ -184,7 +185,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
_, latest_block_number = get_latest_blocks(
AvailableBlockchainType(args.blockchain),
confirmations,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
start_block_number = (
latest_block_number - shift if latest_block_number - shift >= 1 else 1
@ -200,7 +201,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
blockchain_type=AvailableBlockchainType(args.blockchain),
blocks_numbers=blocks_numbers_list,
notransactions=args.notransactions,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
if len(missing_blocks_numbers) > 0:
logger.info(f"Found {len(missing_blocks_numbers)} missing blocks")
@ -217,7 +218,7 @@ def crawler_blocks_missing_handler(args: argparse.Namespace) -> None:
block_numbers_list=missing_blocks_numbers_total,
with_transactions=True,
num_processes=1 if args.lazy else MOONSTREAM_CRAWL_WORKERS,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
logger.info(
f"Required {time.time() - startTime} with {MOONSTREAM_CRAWL_WORKERS} workers "
@ -259,10 +260,8 @@ def main() -> None:
time_now = datetime.now(timezone.utc)
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
"--web3-uri",
help="Node JSON RPC uri",
)
# Blockchain blocks parser

11
crawlers/mooncrawl/mooncrawl/generic_crawler/base.py
Wyświetl plik

@ -11,11 +11,7 @@ from moonstreamdb.blockchain import (
get_label_model,
get_transaction_model,
)
from moonstreamdb.models import (
Base,
EthereumTransaction,
PolygonTransaction,
)
from moonstreamdb.models import Base, EthereumTransaction, PolygonTransaction
from moonworm.crawler.function_call_crawler import ( # type: ignore
ContractFunctionCall,
utfy_dict,
@ -26,10 +22,7 @@ from tqdm import tqdm
from web3 import Web3
from web3._utils.events import get_event_data
from ..moonworm_crawler.db import (
add_events_to_session,
commit_session,
)
from ..moonworm_crawler.db import add_events_to_session, commit_session
from ..moonworm_crawler.event_crawler import Event, get_block_timestamp
logging.basicConfig(level=logging.INFO)

13
crawlers/mooncrawl/mooncrawl/generic_crawler/cli.py
Wyświetl plik

@ -10,7 +10,6 @@ from web3.middleware import geth_poa_middleware
from ..blockchain import connect
from ..db import yield_db_session_ctx
from ..settings import NB_CONTROLLER_ACCESS_ID
from .base import crawl, get_checkpoint, populate_with_events
logging.basicConfig(level=logging.INFO)
@ -39,7 +38,7 @@ def handle_nft_crawler(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = connect(blockchain_type, access_id=args.access_id)
web3 = connect(blockchain_type, web3_uri=args.web3_uri)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -91,7 +90,7 @@ def populate_with_erc20_transfers(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = connect(blockchain_type, access_id=args.access_id)
web3 = connect(blockchain_type, web3_uri=args.web3_uri)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -137,7 +136,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = connect(blockchain_type, access_id=args.access_id)
web3 = connect(blockchain_type, web3_uri=args.web3_uri)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -172,10 +171,8 @@ def main():
subparsers = parser.add_subparsers()
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
"--web3-uri",
help="Node JSON RPC uri",
)
crawl_parser = subparsers.add_parser("crawl", help="Crawl with abi")

8
crawlers/mooncrawl/mooncrawl/leaderboards_generator/cli.py
Wyświetl plik

@ -1,24 +1,22 @@
import argparse
import json
import logging
from typing import cast, List
import uuid
from typing import List, cast
import requests # type: ignore
from bugout.data import BugoutSearchResult
from .utils import get_results_for_moonstream_query, leaderboard_push_batch
from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
MOONSTREAM_API_URL,
MOONSTREAM_ENGINE_URL,
MOONSTREAM_LEADERBOARD_GENERATOR_BATCH_SIZE,
MOONSTREAM_LEADERBOARD_GENERATOR_JOURNAL_ID,
MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS,
)
from ..settings import bugout_client as bc
from .utils import get_results_for_moonstream_query, leaderboard_push_batch
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

4
crawlers/mooncrawl/mooncrawl/leaderboards_generator/utils.py
Wyświetl plik

@ -3,8 +3,7 @@ import json
import logging
import os
import time
from typing import Any, Dict, Optional, List
from typing import Any, Dict, List, Optional
import requests # type: ignore
@ -15,7 +14,6 @@ from ..settings import (
MOONSTREAM_LEADERBOARD_GENERATOR_PUSH_TIMEOUT_SECONDS,
)
logging.basicConfig()
logger = logging.getLogger(__name__)

5
crawlers/mooncrawl/mooncrawl/metadata_crawler/cli.py
Wyświetl plik

@ -9,10 +9,7 @@ from urllib.error import HTTPError
from moonstreamdb.blockchain import AvailableBlockchainType
from ..db import (
yield_db_preping_session_ctx,
yield_db_read_only_preping_session_ctx,
)
from ..db import yield_db_preping_session_ctx, yield_db_read_only_preping_session_ctx
from .db import (
clean_labels_from_db,
get_current_metadata_for_address,

6
crawlers/mooncrawl/mooncrawl/metadata_crawler/db.py
Wyświetl plik

@ -1,13 +1,13 @@
import logging
import json
from typing import Dict, Any, Optional, List
import logging
from typing import Any, Dict, List, Optional
from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
from sqlalchemy.orm import Session
from sqlalchemy.sql import text
from ..data import TokenURIs
from ..settings import VIEW_STATE_CRAWLER_LABEL, METADATA_CRAWLER_LABEL, CRAWLER_LABEL
from ..settings import CRAWLER_LABEL, METADATA_CRAWLER_LABEL, VIEW_STATE_CRAWLER_LABEL
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

23
crawlers/mooncrawl/mooncrawl/moonworm_crawler/cli.py
Wyświetl plik

@ -9,21 +9,20 @@ from web3.middleware import geth_poa_middleware
from ..db import yield_db_session_ctx
from ..settings import (
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
NB_CONTROLLER_ACCESS_ID,
HISTORICAL_CRAWLER_STATUSES,
HISTORICAL_CRAWLER_STATUS_TAG_PREFIXES,
HISTORICAL_CRAWLER_STATUSES,
MOONSTREAM_MOONWORM_TASKS_JOURNAL,
)
from .continuous_crawler import _retry_connect_web3, continuous_crawler
from .crawler import (
SubscriptionTypes,
blockchain_type_to_subscription_type,
find_all_deployed_blocks,
get_crawl_job_entries,
make_event_crawl_jobs,
make_function_call_crawl_jobs,
find_all_deployed_blocks,
update_job_state_with_filters,
moonworm_crawler_update_job_as_pickedup,
update_job_state_with_filters,
)
from .db import get_first_labeled_block_number, get_last_labeled_block_number
from .historical_crawler import historical_crawler
@ -71,7 +70,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = _retry_connect_web3(blockchain_type, access_id=args.access_id)
web3 = _retry_connect_web3(blockchain_type, web3_uri=args.web3_uri)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -133,7 +132,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
args.min_sleep_time,
args.heartbeat_interval,
args.new_jobs_refetch_interval,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
@ -250,7 +249,7 @@ def handle_historical_crawl(args: argparse.Namespace) -> None:
logger.info(
"No web3 provider URL provided, using default (blockchan.py: connect())"
)
web3 = _retry_connect_web3(blockchain_type, access_id=args.access_id)
web3 = _retry_connect_web3(blockchain_type, web3_uri=args.web3_uri)
else:
logger.info(f"Using web3 provider URL: {args.web3}")
web3 = Web3(
@ -336,7 +335,7 @@ def handle_historical_crawl(args: argparse.Namespace) -> None:
end_block,
args.max_blocks_batch,
args.min_sleep_time,
access_id=args.access_id,
web3_uri=args.web3_uri,
addresses_deployment_blocks=addresses_deployment_blocks,
)
@ -346,10 +345,8 @@ def main() -> None:
parser.set_defaults(func=lambda _: parser.print_help())
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
"--web3-uri",
help="Node JSON RPC uri",
)
subparsers = parser.add_subparsers()

6
crawlers/mooncrawl/mooncrawl/moonworm_crawler/continuous_crawler.py
Wyświetl plik

@ -100,7 +100,7 @@ def continuous_crawler(
min_sleep_time: float = 0.1,
heartbeat_interval: float = 60,
new_jobs_refetch_interval: float = 120,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
max_insert_batch: int = 10000,
):
crawler_type = "continuous"
@ -119,7 +119,7 @@ def continuous_crawler(
jobs_refetchet_time = crawl_start_time
if web3 is None:
web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
if blockchain_type == AvailableBlockchainType.ETHEREUM:
network = Network.ethereum
@ -304,7 +304,7 @@ def continuous_crawler(
logger.error("Too many failures, exiting")
raise e
try:
web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
except Exception as err:
logger.error(f"Failed to reconnect: {err}")
logger.exception(err)

4
crawlers/mooncrawl/mooncrawl/moonworm_crawler/crawler.py
Wyświetl plik

@ -107,7 +107,7 @@ def _retry_connect_web3(
blockchain_type: AvailableBlockchainType,
retry_count: int = 10,
sleep_time: float = 5,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
) -> Web3:
"""
Retry connecting to the blockchain.
@ -115,7 +115,7 @@ def _retry_connect_web3(
while retry_count > 0:
retry_count -= 1
try:
web3 = connect(blockchain_type, access_id=access_id)
web3 = connect(blockchain_type, web3_uri=web3_uri)
web3.eth.block_number
logger.info(f"Connected to {blockchain_type}")
return web3

4
crawlers/mooncrawl/mooncrawl/moonworm_crawler/db.py
Wyświetl plik

@ -1,12 +1,12 @@
import logging
import json
import logging
from typing import Dict, List, Optional
from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
from moonstreamdb.models import Base
from moonworm.crawler.function_call_crawler import ContractFunctionCall # type: ignore
from sqlalchemy import Integer, String, column, exists, func, select, text, values
from sqlalchemy.orm import Session
from sqlalchemy import func, text, select, values, column, Integer, String, exists
from ..settings import CRAWLER_LABEL
from .event_crawler import Event

5
crawlers/mooncrawl/mooncrawl/moonworm_crawler/event_crawler.py
Wyświetl plik

@ -3,7 +3,10 @@ from dataclasses import dataclass
from typing import Any, Dict, List, Optional, Tuple
from moonstreamdb.blockchain import AvailableBlockchainType, get_block_model
from moonworm.crawler.log_scanner import _fetch_events_chunk, _crawl_events as moonworm_autoscale_crawl_events # type: ignore
from moonworm.crawler.log_scanner import (
_crawl_events as moonworm_autoscale_crawl_events, # type: ignore
)
from moonworm.crawler.log_scanner import _fetch_events_chunk
from sqlalchemy.orm.session import Session
from sqlalchemy.sql.expression import and_
from web3 import Web3

6
crawlers/mooncrawl/mooncrawl/moonworm_crawler/historical_crawler.py
Wyświetl plik

@ -36,7 +36,7 @@ def historical_crawler(
end_block: int,
max_blocks_batch: int = 100,
min_sleep_time: float = 0.1,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
addresses_deployment_blocks: Optional[Dict[ChecksumAddress, int]] = None,
max_insert_batch: int = 10000,
):
@ -46,7 +46,7 @@ def historical_crawler(
assert end_block > 0, "end_block must be greater than 0"
if web3 is None:
web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
assert (
web3.eth.block_number >= start_block
@ -205,7 +205,7 @@ def historical_crawler(
logger.error("Too many failures, exiting")
raise e
try:
web3 = _retry_connect_web3(blockchain_type, access_id=access_id)
web3 = _retry_connect_web3(blockchain_type, web3_uri=web3_uri)
except Exception as err:
logger.error(f"Failed to reconnect: {err}")
logger.exception(err)

12
crawlers/mooncrawl/mooncrawl/reports_crawler/cli.py
Wyświetl plik

@ -3,22 +3,20 @@ import csv
import datetime
import json
import logging
from io import StringIO
from moonstream.client import Moonstream # type: ignore
import requests # type: ignore
import time
from io import StringIO
from typing import Any, Dict, Union
from uuid import UUID
from typing import Any, Dict, Union
import requests # type: ignore
from moonstream.client import Moonstream # type: ignore
from .queries import tokenomics_queries, cu_bank_queries, tokenomics_orange_dao_queries
from ..actions import recive_S3_data_from_query
from ..settings import (
MOONSTREAM_S3_PUBLIC_DATA_BUCKET,
MOONSTREAM_S3_PUBLIC_DATA_BUCKET_PREFIX,
)
from .queries import cu_bank_queries, tokenomics_orange_dao_queries, tokenomics_queries
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

99
crawlers/mooncrawl/mooncrawl/settings.py
Wyświetl plik

@ -78,72 +78,87 @@ except:
)
# Geth connection address
MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI", ""
)
if MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI env variable is not set")
if MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_POLYGON_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_POLYGON_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI", ""
)
if MOONSTREAM_POLYGON_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_POLYGON_WEB3_PROVIDER_URI env variable is not set")
if MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI", ""
)
if MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI env variable is not set")
if MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_XDAI_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_XDAI_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI", ""
)
if MOONSTREAM_XDAI_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_XDAI_WEB3_PROVIDER_URI env variable is not set")
if MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_WYRM_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_WYRM_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI", ""
)
if MOONSTREAM_WYRM_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_WYRM_WEB3_PROVIDER_URI env variable is not set")
MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI", ""
)
if MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI == "":
if MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI env variable is not set"
"MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI", ""
)
if MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI env variable is not set")
if MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI", ""
)
if MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI == "":
if MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI env variable is not set"
"MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI", ""
)
if MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI == "":
if MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI env variable is not set"
"MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_XAI_WEB3_PROVIDER_URI = os.environ.get(
"MOONSTREAM_XAI_WEB3_PROVIDER_URI", ""
MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI", ""
)
if MOONSTREAM_XAI_WEB3_PROVIDER_URI == "":
raise Exception("MOONSTREAM_XAI_WEB3_PROVIDER_URI env variable is not set")
if MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_NODE_XAI_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_XAI_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_XAI_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_XAI_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI == "":
raise Exception("MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI env variable is not set")
MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI = os.environ.get(
"MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI", ""
)
if MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI == "":
raise Exception(
"MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI env variable is not set"
)
MOONSTREAM_CRAWL_WORKERS = 4
MOONSTREAM_CRAWL_WORKERS_RAW = os.environ.get("MOONSTREAM_CRAWL_WORKERS")

43
crawlers/mooncrawl/mooncrawl/state_crawler/cli.py
Wyświetl plik

@ -10,23 +10,19 @@ from pprint import pprint
from typing import Any, Dict, List, Optional
from uuid import UUID
from moonstreamdb.blockchain import AvailableBlockchainType
from moonstream.client import Moonstream # type: ignore
from moonstreamdb.blockchain import AvailableBlockchainType
from web3.middleware import geth_poa_middleware
from mooncrawl.moonworm_crawler.crawler import _retry_connect_web3
from ..actions import recive_S3_data_from_query
from ..blockchain import connect
from ..db import PrePing_SessionLocal
from ..settings import (
INFURA_PROJECT_ID,
NB_CONTROLLER_ACCESS_ID,
infura_networks,
multicall_contracts,
)
from ..settings import INFURA_PROJECT_ID, infura_networks, multicall_contracts
from .db import clean_labels, commit_session, view_call_to_label
from .Multicall2_interface import Contract as Multicall2
from .web3_util import FunctionSignature, connect
from .web3_util import FunctionSignature
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
@ -306,8 +302,8 @@ def parse_jobs(
web3_provider_uri: Optional[str],
block_number: Optional[int],
batch_size: int,
access_id: UUID,
moonstream_token: str,
web3_uri: Optional[str] = None,
):
"""
Parse jobs from list and generate web3 interfaces for each contract.
@ -321,13 +317,12 @@ def parse_jobs(
if web3_provider_uri is not None:
try:
logger.info(
f"Connecting to blockchain : {blockchain_type} with custom provider!"
f"Connecting to blockchain: {blockchain_type} with custom provider!"
)
web3_client = connect(web3_provider_uri)
if blockchain_type != AvailableBlockchainType.ETHEREUM:
web3_client.middleware_onion.inject(geth_poa_middleware, layer=0)
web3_client = connect(
blockchain_type=blockchain_type, web3_uri=web3_provider_uri
)
except Exception as e:
logger.error(
f"Web3 connection to custom provider {web3_provider_uri} failed error: {e}"
@ -336,7 +331,7 @@ def parse_jobs(
else:
logger.info(f"Connecting to blockchain: {blockchain_type} with Node balancer.")
web3_client = _retry_connect_web3(
blockchain_type=blockchain_type, access_id=access_id
blockchain_type=blockchain_type, web3_uri=web3_uri
)
logger.info(f"Crawler started connected to blockchain: {blockchain_type}")
@ -519,7 +514,7 @@ def handle_crawl(args: argparse.Namespace) -> None:
blockchain_type = AvailableBlockchainType(args.blockchain)
custom_web3_provider = args.custom_web3_provider
custom_web3_provider = args.web3_uri
if args.infura and INFURA_PROJECT_ID is not None:
if blockchain_type not in infura_networks:
@ -535,8 +530,8 @@ def handle_crawl(args: argparse.Namespace) -> None:
custom_web3_provider,
args.block_number,
args.batch_size,
args.access_id,
args.moonstream_token,
args.web3_uri,
)
@ -563,7 +558,7 @@ def clean_labels_handler(args: argparse.Namespace) -> None:
blockchain_type = AvailableBlockchainType(args.blockchain)
web3_client = _retry_connect_web3(
blockchain_type=blockchain_type, access_id=args.access_id
blockchain_type=blockchain_type, web3_uri=args.web3_uri
)
logger.info(f"Label cleaner connected to blockchain: {blockchain_type}")
@ -583,10 +578,8 @@ def main() -> None:
parser.set_defaults(func=lambda _: parser.print_help())
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
"--web3-uri",
help="Node JSON RPC uri",
)
subparsers = parser.add_subparsers()
@ -614,12 +607,6 @@ def main() -> None:
action="store_true",
help="Use infura as web3 provider",
)
view_state_crawler_parser.add_argument(
"--custom-web3-provider",
"-w3",
type=str,
help="Type of blovkchain wich writng in database",
)
view_state_crawler_parser.add_argument(
"--block-number", "-N", type=str, help="Block number."
)

4
crawlers/mooncrawl/mooncrawl/state_crawler/db.py
Wyświetl plik

@ -1,6 +1,6 @@
import logging
import json
from typing import Dict, Any
import logging
from typing import Any, Dict
from moonstreamdb.blockchain import AvailableBlockchainType, get_label_model
from sqlalchemy.orm import Session

26
crawlers/mooncrawl/mooncrawl/state_crawler/web3_util.py
Wyświetl plik

@ -2,17 +2,17 @@ import getpass
import os
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from eth_abi import decode_single, encode_single
from eth_account.account import Account # type: ignore
from eth_abi import encode_single, decode_single
from eth_typing.evm import ChecksumAddress
from eth_utils import function_signature_to_4byte_selector
from hexbytes.main import HexBytes
from web3 import Web3
from web3._utils.abi import normalize_event_input_types
from web3.contract import ContractFunction
from web3.providers.ipc import IPCProvider
from web3.providers.rpc import HTTPProvider
from web3.types import ABI, Nonce, TxParams, TxReceipt, Wei
from web3._utils.abi import normalize_event_input_types
class ContractConstructor:
@ -165,28 +165,6 @@ def read_keys_from_env() -> Tuple[ChecksumAddress, str]:
)
def connect(web3_uri: str) -> Web3:
web3_provider: Union[IPCProvider, HTTPProvider] = Web3.IPCProvider()
if web3_uri.startswith("http://") or web3_uri.startswith("https://"):
web3_provider = Web3.HTTPProvider(web3_uri)
else:
web3_provider = Web3.IPCProvider(web3_uri)
web3_client = Web3(web3_provider)
return web3_client
def read_web3_provider_from_env() -> Web3:
provider_path = os.environ.get("MOONWORM_WEB3_PROVIDER_URI")
if provider_path is None:
raise ValueError("MOONWORM_WEB3_PROVIDER_URI env variable is not set")
return connect(provider_path)
def read_web3_provider_from_cli() -> Web3:
provider_path = input("Enter web3 uri path: ")
return connect(provider_path)
def cast_to_python_type(evm_type: str) -> Callable:
if evm_type.startswith(("uint", "int")):
return int

25
crawlers/mooncrawl/mooncrawl/stats_worker/dashboard.py
Wyświetl plik

@ -40,7 +40,6 @@ from ..settings import (
MOONSTREAM_ADMIN_ACCESS_TOKEN,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_BUCKET,
MOONSTREAM_S3_SMARTCONTRACTS_ABI_PREFIX,
NB_CONTROLLER_ACCESS_ID,
)
from ..settings import bugout_client as bc
@ -367,7 +366,7 @@ def generate_list_of_names(
def process_external_merged(
external_calls: Dict[str, Dict[str, Any]],
blockchain: AvailableBlockchainType,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
):
"""
Process external calls
@ -412,7 +411,7 @@ def process_external_merged(
logger.error(f"Error processing external call: {e}")
if external_calls_normalized:
web3_client = connect(blockchain, access_id=access_id)
web3_client = connect(blockchain, web3_uri=web3_uri)
for extcall in external_calls_normalized:
try:
@ -433,7 +432,7 @@ def process_external_merged(
def process_external(
abi_external_calls: List[Dict[str, Any]],
blockchain: AvailableBlockchainType,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
):
"""
Request all required external data
@ -479,7 +478,7 @@ def process_external(
logger.error(f"Error processing external call: {e}")
if external_calls:
web3_client = connect(blockchain, access_id=access_id)
web3_client = connect(blockchain, web3_uri=web3_uri)
for extcall in external_calls:
try:
@ -530,7 +529,7 @@ def generate_web3_metrics(
address: str,
crawler_label: str,
abi_json: Any,
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
) -> List[Any]:
"""
Generate stats for cards components
@ -543,7 +542,7 @@ def generate_web3_metrics(
extention_data = process_external(
abi_external_calls=abi_external_calls,
blockchain=blockchain_type,
access_id=access_id,
web3_uri=web3_uri,
)
extention_data.append(
@ -873,7 +872,7 @@ def stats_generate_handler(args: argparse.Namespace):
external_calls_results = process_external_merged(
external_calls=merged_external_calls["merged"],
blockchain=blockchain_type,
access_id=args.access_id,
web3_uri=args.web3_uri,
)
for address in address_dashboard_id_subscription_id_tree.keys():
@ -1039,7 +1038,7 @@ def stats_generate_api_task(
timescales: List[str],
dashboard: BugoutResource,
subscription_by_id: Dict[str, BugoutJournalEntity],
access_id: Optional[UUID] = None,
web3_uri: Optional[str] = None,
):
"""
Start crawler with generate.
@ -1115,7 +1114,7 @@ def stats_generate_api_task(
address=address, # type: ignore
crawler_label=crawler_label,
abi_json=abi_json,
access_id=access_id,
web3_uri=web3_uri,
)
# Generate blocks state information
@ -1192,10 +1191,8 @@ def main() -> None:
parser.set_defaults(func=lambda _: parser.print_help())
parser.add_argument(
"--access-id",
default=NB_CONTROLLER_ACCESS_ID,
type=UUID,
help="User access ID",
"--web3-uri",
help="Node JSON RPC uri",
)
subcommands = parser.add_subparsers(

6
crawlers/mooncrawl/mooncrawl/stats_worker/queries.py
Wyświetl plik

@ -1,21 +1,17 @@
import csv
from collections import OrderedDict
import hashlib
import json
import logging
import re
from collections import OrderedDict
from io import StringIO
from typing import Any, Dict
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import text
from sqlalchemy.sql.expression import TextClause
from ..actions import push_data_to_bucket
from ..reporter import reporter
from ..db import RO_pre_ping_query_engine
from ..reporter import reporter
from ..settings import MOONSTREAM_S3_QUERIES_BUCKET_PREFIX

2
crawlers/mooncrawl/mooncrawl/version.py
Wyświetl plik

@ -2,4 +2,4 @@
Moonstream crawlers version.
"""
MOONCRAWL_VERSION = "0.3.9"
MOONCRAWL_VERSION = "0.4.0"

22
crawlers/mooncrawl/sample.env
Wyświetl plik

@ -21,16 +21,18 @@ export MOONSTREAM_ADMIN_ACCESS_TOKEN="<Bugout_access_token_for_moonstream>"
export NFT_HUMBUG_TOKEN="<Token_for_nft_crawler>"
# Blockchain nodes environment variables
export MOONSTREAM_ETHEREUM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_ethereum_node>"
export MOONSTREAM_POLYGON_WEB3_PROVIDER_URI="https://<connection_path_uri_to_polygon_node>"
export MOONSTREAM_MUMBAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_mumbai_node>"
export MOONSTREAM_XDAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xdai_node>"
export MOONSTREAM_WYRM_WEB3_PROVIDER_URI="https://<connection_path_uri_to_wyrm_node>"
export MOONSTREAM_ZKSYNC_ERA_TESTNET_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_testnet_node>"
export MOONSTREAM_ZKSYNC_ERA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_zksync_era_node>"
export MOONSTREAM_ARBITRUM_NOVA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_nova_node>"
export MOONSTREAM_ARBITRUM_SEPOLIA_WEB3_PROVIDER_URI="https://<connection_path_uri_to_arbitrum_sepolia_node>"
export MOONSTREAM_XAI_WEB3_PROVIDER_URI="https://<connection_path_uri_to_xai_node>"
export MOONSTREAM_NODE_ETHEREUM_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_POLYGON_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_MUMBAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_XDAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_TESTNET_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ZKSYNC_ERA_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ARBITRUM_NOVA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_ARBITRUM_SEPOLIA_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_XAI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AVALANCHE_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export MOONSTREAM_NODE_AVALANCHE_FUJI_A_EXTERNAL_URI="https://<connection_path_uri_to_node>"
export NB_CONTROLLER_ACCESS_ID="<access_uuid_for_moonstream_nodebalancer>"
# AWS environment variables

2
crawlers/mooncrawl/setup.py
Wyświetl plik

@ -37,7 +37,7 @@ setup(
"bugout>=0.2.13",
"chardet",
"fastapi",
"moonstreamdb>=0.3.8",
"moonstreamdb>=0.3.9",
"moonstream>=0.1.1",
"moonworm[moonstream]>=0.6.2",
"humbug",