diff --git a/crawlers/deploy/arbitrum-one-historical-crawl-events.service b/crawlers/deploy/arbitrum-one-historical-crawl-events.service new file mode 100644 index 00000000..2a5668ab --- /dev/null +++ b/crawlers/deploy/arbitrum-one-historical-crawl-events.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum One historical crawler events +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type arbitrum_one --find-deployed-blocks --end 0 --tasks-journal --only-events +CPUWeight=70 +SyslogIdentifier=arbitrum-one-historical-crawl-events + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-one-historical-crawl-events.timer b/crawlers/deploy/arbitrum-one-historical-crawl-events.timer new file mode 100644 index 00000000..950ff327 --- /dev/null +++ b/crawlers/deploy/arbitrum-one-historical-crawl-events.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs events historical crawler on arbitrum one + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/arbitrum-one-historical-crawl-transactions.service b/crawlers/deploy/arbitrum-one-historical-crawl-transactions.service new file mode 100644 index 00000000..97c2356c --- /dev/null +++ b/crawlers/deploy/arbitrum-one-historical-crawl-transactions.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum One historical crawler transactions +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli historical-crawl --blockchain-type arbitrum_one --find-deployed-blocks --end 0 --tasks-journal --only-functions +CPUWeight=70 +SyslogIdentifier=arbitrum-one-historical-crawl-transactions + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-one-historical-crawl-transactions.timer b/crawlers/deploy/arbitrum-one-historical-crawl-transactions.timer new file mode 100644 index 00000000..025ad1ad --- /dev/null +++ b/crawlers/deploy/arbitrum-one-historical-crawl-transactions.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs transactions historical crawler on proofofplay apex + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/arbitrum-one-missing.service b/crawlers/deploy/arbitrum-one-missing.service new file mode 100644 index 00000000..b59e7530 --- /dev/null +++ b/crawlers/deploy/arbitrum-one-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at Arbitrum One database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks missing --blockchain arbitrum_one -n +CPUWeight=50 +SyslogIdentifier=arbitrum-one-missing \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-one-missing.timer b/crawlers/deploy/arbitrum-one-missing.timer new file mode 100644 index 00000000..e1962c27 --- /dev/null +++ b/crawlers/deploy/arbitrum-one-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at Arbitrum One database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/arbitrum-one-moonworm-crawler.service b/crawlers/deploy/arbitrum-one-moonworm-crawler.service new file mode 100644 index 00000000..8b8645c6 --- /dev/null +++ b/crawlers/deploy/arbitrum-one-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum One moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli crawl -b arbitrum_one --confirmations 10 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=arbitrum-one-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/arbitrum-one-synchronize.service b/crawlers/deploy/arbitrum-one-synchronize.service new file mode 100644 index 00000000..18e1da90 --- /dev/null +++ b/crawlers/deploy/arbitrum-one-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=Arbitrum One block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler blocks synchronize --blockchain arbitrum_one -c 10 -j 2 +CPUWeight=90 +SyslogIdentifier=arbitrum-one-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/deploy-basic.bash b/crawlers/deploy/deploy-basic.bash index 297b5b46..1a442dca 100755 --- a/crawlers/deploy/deploy-basic.bash +++ b/crawlers/deploy/deploy-basic.bash @@ -63,6 +63,11 @@ ZKSYNC_ERA_SEPOLIA_SYNCHRONIZE_SERVICE="zksync-era-sepolia-synchronize.service" ZKSYNC_ERA_SEPOLIA_MISSING_SERVICE_FILE="zksync-era-sepolia-missing.service" ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE="zksync-era-sepolia-missing.timer" +# Arbitrum one +ARBITRUM_ONE_SYNCHRONIZE_SERVICE="arbitrum-one-synchronize.service" +ARBITRUM_ONE_MISSING_SERVICE_FILE="arbitrum-one-missing.service" +ARBITRUM_ONE_MISSING_TIMER_FILE="arbitrum-one-missing.timer" + # Arbitrum Nova ARBITRUM_NOVA_SYNCHRONIZE_SERVICE="arbitrum-nova-synchronize.service" ARBITRUM_NOVA_MISSING_SERVICE_FILE="arbitrum-nova-missing.service" @@ -286,6 +291,25 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" "/home/ubuntu/.confi XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MISSING_TIMER_FILE}" + +# Arbitrum one +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum one block with transactions syncronizer service definition with ${ARBITRUM_ONE_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum one missing service and timer with: ${ARBITRUM_ONE_MISSING_SERVICE_FILE}, ${ARBITRUM_ONE_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_ONE_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_MISSING_TIMER_FILE}" + echo echo echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova block with transactions syncronizer service definition with ${ARBITRUM_NOVA_SYNCHRONIZE_SERVICE}" diff --git a/crawlers/deploy/deploy-historical.bash b/crawlers/deploy/deploy-historical.bash index bfd162e8..c8405974 100755 --- a/crawlers/deploy/deploy-historical.bash +++ b/crawlers/deploy/deploy-historical.bash @@ -60,6 +60,12 @@ ZKSYNC_ERA_SEPOLIA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-sepolia- ZKSYNC_ERA_SEPOLIA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-sepolia-historical-crawl-events.service" ZKSYNC_ERA_SEPOLIA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-sepolia-historical-crawl-events.timer" +# Arbitrum one +ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="arbitrum-one-historical-crawl-transactions.service" +ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="arbitrum-one-historical-crawl-transactions.timer" +ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="arbitrum-one-historical-crawl-events.service" +ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="arbitrum-one-historical-crawl-events.timer" + # ProofofPlay APEX PROOFOFPLAY_APEX_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="proofofplay-apex-historical-crawl-transactions.service" PROOFOFPLAY_APEX_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="proofofplay-apex-historical-crawl-transactions.timer" @@ -230,3 +236,21 @@ cp "${SCRIPT_DIR}/${PROOFOFPLAY_APEX_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/ho cp "${SCRIPT_DIR}/${PROOFOFPLAY_APEX_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${PROOFOFPLAY_APEX_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${PROOFOFPLAY_APEX_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum one historical transactions crawler service and timer with: ${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum one historical events crawler service and timer with: ${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}, ${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" \ No newline at end of file diff --git a/crawlers/deploy/deploy-moonworm.bash b/crawlers/deploy/deploy-moonworm.bash index 56641352..9210cfdd 100755 --- a/crawlers/deploy/deploy-moonworm.bash +++ b/crawlers/deploy/deploy-moonworm.bash @@ -31,6 +31,7 @@ AMOY_MOONWORM_CRAWLER_SERVICE_FILE="amoy-moonworm-crawler.service" XDAI_MOONWORM_CRAWLER_SERVICE_FILE="xdai-moonworm-crawler.service" ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-moonworm-crawler.service" ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-sepolia-moonworm-crawler.service" +ARBITRUM_ONE_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-one-moonworm-crawler.service" ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-nova-moonworm-crawler.service" ARBITRUM_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE="arbitrum-sepolia-moonworm-crawler.service" XAI_MOONWORM_CRAWLER_SERVICE_FILE="xai-moonworm-crawler.service" @@ -128,6 +129,15 @@ cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ub XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SEPOLIA_MOONWORM_CRAWLER_SERVICE_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing Arbitrum One moonworm crawler service definition with ${ARBITRUM_ONE_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ARBITRUM_ONE_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ARBITRUM_ONE_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ARBITRUM_ONE_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ARBITRUM_ONE_MOONWORM_CRAWLER_SERVICE_FILE}" + echo echo echo -e "${PREFIX_INFO} Replacing existing Arbitrum Nova moonworm crawler service definition with ${ARBITRUM_NOVA_MOONWORM_CRAWLER_SERVICE_FILE}" diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service index 2c248c11..3d17598a 100644 --- a/crawlers/deploy/monitoring-crawlers.service +++ b/crawlers/deploy/monitoring-crawlers.service @@ -9,7 +9,7 @@ Restart=on-failure RestartSec=15s WorkingDirectory=/home/ubuntu/ EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env -ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service amoy-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service zksync-era-sepolia-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service -service xai-sepolia-moonworm-crawler.service -service avalanche-moonworm-crawler.service -service avalanche-fuji-moonworm-crawler.service -service blast-moonworm-crawler.service -service blast-sepolia-moonworm-crawler.service -service proofofplay-apex-moonworm-crawler.service +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service amoy-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service -service zksync-era-sepolia-moonworm-crawler.service -service arbitrum-nova-moonworm-crawler.service -service arbitrum-sepolia-moonworm-crawler.service -service xai-moonworm-crawler.service -service xai-sepolia-moonworm-crawler.service -service avalanche-moonworm-crawler.service -service avalanche-fuji-moonworm-crawler.service -service blast-moonworm-crawler.service -service blast-sepolia-moonworm-crawler.service -service proofofplay-apex-moonworm-crawler.service -service arbitrum-one-moonworm-crawler.service CPUWeight=90 SyslogIdentifier=monitoring-crawlers