From 4c5e6cc3278933e645eeec44dc2593c322c9ad22 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Wed, 30 Aug 2023 13:11:17 +0000 Subject: [PATCH] Requirements update and deployment zksync mainnet scripts --- crawlers/deploy/deploy.bash | 56 ++++++++++++++++++- ...zksync-era-historical-crawl-events.service | 17 ++++++ .../zksync-era-historical-crawl-events.timer | 9 +++ ...-era-historical-crawl-transactions.service | 17 ++++++ ...nc-era-historical-crawl-transactions.timer | 9 +++ crawlers/deploy/zksync-era-missing.service | 11 ++++ crawlers/deploy/zksync-era-missing.timer | 9 +++ .../zksync-era-moonworm-crawler.service | 17 ++++++ .../deploy/zksync-era-synchronize.service | 17 ++++++ moonstreamapi/requirements.txt | 2 +- 10 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 crawlers/deploy/zksync-era-historical-crawl-events.service create mode 100644 crawlers/deploy/zksync-era-historical-crawl-events.timer create mode 100644 crawlers/deploy/zksync-era-historical-crawl-transactions.service create mode 100644 crawlers/deploy/zksync-era-historical-crawl-transactions.timer create mode 100644 crawlers/deploy/zksync-era-missing.service create mode 100644 crawlers/deploy/zksync-era-missing.timer create mode 100644 crawlers/deploy/zksync-era-moonworm-crawler.service create mode 100644 crawlers/deploy/zksync-era-synchronize.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index c0f3545f..fc32785a 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -109,6 +109,16 @@ WYRM_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="wyrm-historical-crawl-transaction WYRM_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="wyrm-historical-crawl-events.service" WYRM_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="wyrm-historical-crawl-events.timer" +# ZkSync Era +ZKSYNC_ERA_SYNCHRONIZE_SERVICE="zksync-era-synchronize.service" +ZKSYNC_ERA_MISSING_SERVICE_FILE="zksync-era-missing.service" +ZKSYNC_ERA_MISSING_TIMER_FILE="zksync-era-missing.timer" +ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE="zksync-era-moonworm-crawler.service" +ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE="zksync-era-historical-crawl-transactions.service" +ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE="zksync-era-historical-crawl-transactions.timer" +ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE="zksync-era-historical-crawl-events.service" +ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-historical-crawl-events.timer" + # ZkSync Era testnet ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE="zksync-era-testnet-synchronize.service" ZKSYNC_ERA_TESTNET_MISSING_SERVICE_FILE="zksync-era-testnet-missing.service" @@ -516,6 +526,51 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${WYRM_HIS # ZkSync Era echo echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era block with transactions syncronizer service definition with ${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_SYNCHRONIZE_SERVICE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era missing service and timer with: ${ZKSYNC_ERA_MISSING_SERVICE_FILE}, ${ZKSYNC_ERA_MISSING_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MISSING_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MISSING_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MISSING_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_MISSING_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era moonworm crawler service definition with ${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_MOONWORM_CRAWLER_SERVICE_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era historical transactions crawler service and timer with: ${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}, ${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_TRANSACTIONS_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing ZkSync Era historical events crawler service and timer with: ${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}, ${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" + + +# ZkSync Era testnet +echo +echo echo -e "${PREFIX_INFO} Replacing existing ZkSync Era testnet block with transactions syncronizer service definition with ${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" chmod 644 "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" cp "${SCRIPT_DIR}/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" "/home/ubuntu/.config/systemd/user/${ZKSYNC_ERA_TESTNET_SYNCHRONIZE_SERVICE}" @@ -558,7 +613,6 @@ XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Leaderboards worker service and timer with: ${LEADERBOARDS_WORKER_SERVICE_FILE}, ${LEADERBOARDS_WORKER_TIMER_FILE}" diff --git a/crawlers/deploy/zksync-era-historical-crawl-events.service b/crawlers/deploy/zksync-era-historical-crawl-events.service new file mode 100644 index 00000000..992b5d5c --- /dev/null +++ b/crawlers/deploy/zksync-era-historical-crawl-events.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era historical crawler events +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-events +CPUWeight=70 +SyslogIdentifier=zksync-era-historical-crawl-events + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-historical-crawl-events.timer b/crawlers/deploy/zksync-era-historical-crawl-events.timer new file mode 100644 index 00000000..aa9004ca --- /dev/null +++ b/crawlers/deploy/zksync-era-historical-crawl-events.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs events historical crawler on ZkSync Era + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/zksync-era-historical-crawl-transactions.service b/crawlers/deploy/zksync-era-historical-crawl-transactions.service new file mode 100644 index 00000000..65218e90 --- /dev/null +++ b/crawlers/deploy/zksync-era-historical-crawl-transactions.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era historical crawler transactions +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" historical-crawl --blockchain-type zksync_era --find-deployed-blocks --end 0 --tasks-journal --only-functions +CPUWeight=70 +SyslogIdentifier=zksync-era-historical-crawl-transactions + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-historical-crawl-transactions.timer b/crawlers/deploy/zksync-era-historical-crawl-transactions.timer new file mode 100644 index 00000000..495ac89c --- /dev/null +++ b/crawlers/deploy/zksync-era-historical-crawl-transactions.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs transactions historical crawler on ZkSync Era + +[Timer] +OnBootSec=60s +OnUnitActiveSec=10m + +[Install] +WantedBy=timers.target diff --git a/crawlers/deploy/zksync-era-missing.service b/crawlers/deploy/zksync-era-missing.service new file mode 100644 index 00000000..9bd08748 --- /dev/null +++ b/crawlers/deploy/zksync-era-missing.service @@ -0,0 +1,11 @@ +[Unit] +Description=Fill missing blocks at ZkSync Era database +After=network.target + +[Service] +Type=oneshot +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks missing --blockchain zksync_era -n +CPUWeight=50 +SyslogIdentifier=zksync-era-missing \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-missing.timer b/crawlers/deploy/zksync-era-missing.timer new file mode 100644 index 00000000..b5d75188 --- /dev/null +++ b/crawlers/deploy/zksync-era-missing.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Fill missing blocks at ZkSync Era database + +[Timer] +OnBootSec=120s +OnUnitActiveSec=15m + +[Install] +WantedBy=timers.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-moonworm-crawler.service b/crawlers/deploy/zksync-era-moonworm-crawler.service new file mode 100644 index 00000000..24e4497d --- /dev/null +++ b/crawlers/deploy/zksync-era-moonworm-crawler.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era moonworm crawler +After=network.target +StartLimitIntervalSec=300 +StartLimitBurst=3 + +[Service] +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +Restart=on-failure +RestartSec=15s +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.moonworm_crawler.cli --access-id "${NB_CONTROLLER_ACCESS_ID}" crawl -b zksync_era --confirmations 20 --min-blocks-batch 20 +CPUWeight=70 +SyslogIdentifier=zksync-era-moonworm-crawler + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/crawlers/deploy/zksync-era-synchronize.service b/crawlers/deploy/zksync-era-synchronize.service new file mode 100644 index 00000000..f180b7c4 --- /dev/null +++ b/crawlers/deploy/zksync-era-synchronize.service @@ -0,0 +1,17 @@ +[Unit] +Description=ZkSync Era block with transactions synchronizer +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/moonstream/crawlers/mooncrawl +EnvironmentFile=/home/ubuntu/moonstream-secrets/app.env +ExecStart=/home/ubuntu/moonstream-env/bin/python -m mooncrawl.crawler --access-id "${NB_CONTROLLER_ACCESS_ID}" blocks synchronize --blockchain zksync_era -c 20 -j 2 +CPUWeight=90 +SyslogIdentifier=zksync-era-synchronize + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/moonstreamapi/requirements.txt b/moonstreamapi/requirements.txt index 1c9c3764..62f4c140 100644 --- a/moonstreamapi/requirements.txt +++ b/moonstreamapi/requirements.txt @@ -36,7 +36,7 @@ jsonschema==4.17.0 lru-dict==1.1.8 Mako==1.2.3 MarkupSafe==2.1.1 -moonstreamdb==0.3.4 +moonstreamdb==0.3.5 multiaddr==0.0.9 multidict==6.0.2 netaddr==0.8.0