From c196c9e0d22a24b7edc3d11f3f3e604d0f18fb76 Mon Sep 17 00:00:00 2001 From: kompotkot Date: Thu, 2 Nov 2023 11:59:31 +0000 Subject: [PATCH] Crawlers systemd monitoring --- crawlers/deploy/deploy.bash | 85 ++++++++++++------- .../deploy/monitoring-crawlers-config.json | 7 ++ crawlers/deploy/monitoring-crawlers.service | 17 ++++ 3 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 crawlers/deploy/monitoring-crawlers-config.json create mode 100644 crawlers/deploy/monitoring-crawlers.service diff --git a/crawlers/deploy/deploy.bash b/crawlers/deploy/deploy.bash index fc32785a..d0b38ea0 100755 --- a/crawlers/deploy/deploy.bash +++ b/crawlers/deploy/deploy.bash @@ -22,19 +22,19 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python" PIP="${PYTHON_ENV_DIR}/bin/pip" SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}" PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env" +PARAMETERS_ENV_MONITORING_PATH="${SECRETS_DIR}/monitoring.env" SCRIPT_DIR="$(realpath $(dirname $0))" # Service files MOONCRAWL_SERVICE_FILE="mooncrawl.service" +MONITORING_CRAWLERS_SERVICE_FILE="monitoring-crawlers.service" LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service" LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer" - # Ethereum service files ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service" ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service" ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer" -ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service" ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service" ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer" ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service" @@ -51,7 +51,6 @@ POLYGON_MISSING_SERVICE_FILE="polygon-missing.service" POLYGON_MISSING_TIMER_FILE="polygon-missing.timer" POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service" POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer" -POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service" POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service" POLYGON_STATE_SERVICE_FILE="polygon-state.service" POLYGON_STATE_TIMER_FILE="polygon-state.timer" @@ -131,14 +130,6 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-histor set -eu -echo -echo -echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go" -EXEC_DIR=$(pwd) -cd "${APP_CRAWLERS_DIR}/txpool" -HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/txpool/txpool" "${APP_CRAWLERS_DIR}/txpool/main.go" -cd "${EXEC_DIR}" - echo echo echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools" @@ -156,8 +147,11 @@ HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@la echo echo -echo -e "${PREFIX_INFO} Retrieving addition deployment parameters" -mkdir -p "${SECRETS_DIR}" +echo -e "${PREFIX_INFO} Retrieving deployment parameters" +if [ ! -d "${SECRETS_DIR}" ]; then + mkdir -p "${SECRETS_DIR}" + echo -e "${PREFIX_WARN} Created new secrets directory" +fi AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+moonstream:true > "${PARAMETERS_ENV_PATH}" chmod 0640 "${PARAMETERS_ENV_PATH}" @@ -166,6 +160,45 @@ echo echo -e "${PREFIX_INFO} Add instance local IP to parameters" echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}" +echo +echo +echo -e "${PREFIX_INFO} Copy monitoring binary from AWS S3" +aws s3 cp s3://bugout-binaries/prod/monitoring/monitoring "/home/ubuntu/monitoring" +chmod +x "/home/ubuntu/monitoring" +chown ubuntu:ubuntu "/home/ubuntu/monitoring" + +echo +echo +echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters" +AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}" +chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Add instance local IP to monitoring parameters" +echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters" +echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}" + +echo +echo +echo -e "${PREFIX_INFO} Prepare monitoring configuration" +if [ ! -d "/home/ubuntu/.monitoring" ]; then + mkdir -p /home/ubuntu/.monitoring + echo -e "${PREFIX_WARN} Created monitoring configuration directory" +fi +cp "${SCRIPT_DIR}/monitoring-crawlers-config.json" /home/ubuntu/.monitoring/monitoring-crawlers-config.json + +echo +echo +if [ ! -d "/home/ubuntu/.config/systemd/user/" ]; then + mkdir -p /home/ubuntu/.config/systemd/user/ + echo -e "${PREFIX_WARN} Created user systemd directory" +fi + echo echo echo -e "${PREFIX_INFO} Replacing existing Moonstream crawlers HTTP API server service definition with ${MOONCRAWL_SERVICE_FILE}" @@ -191,14 +224,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}" -# echo -# echo -# echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}" -# chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" -# cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TXPOOL_SERVICE_FILE}" -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TXPOOL_SERVICE_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}" @@ -270,14 +295,6 @@ cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/system XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}" -# echo -# echo -# echo -e "${PREFIX_INFO} Replacing existing Polygon transaction pool crawler service definition with ${POLYGON_TXPOOL_SERVICE_FILE}" -# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" -# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_TXPOOL_SERVICE_FILE}" -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}" - echo echo echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}" @@ -620,4 +637,12 @@ chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "${SCRIPT_DIR}/${L cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_SERVICE_FILE}" cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}" XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload -XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" \ No newline at end of file +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}" + +echo +echo +echo -e "${PREFIX_INFO} Replacing existing systemd crawlers monitoring service definition with ${MONITORING_CRAWLERS_SERVICE_FILE}" +chmod 644 "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" +cp "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${MONITORING_CRAWLERS_SERVICE_FILE}" +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload +XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${MONITORING_CRAWLERS_SERVICE_FILE}" diff --git a/crawlers/deploy/monitoring-crawlers-config.json b/crawlers/deploy/monitoring-crawlers-config.json new file mode 100644 index 00000000..654139c5 --- /dev/null +++ b/crawlers/deploy/monitoring-crawlers-config.json @@ -0,0 +1,7 @@ +{ + "notification_pagerduty": true, + "notification_telegram": true, + "notification_sendgrid": true, + "notification_humbug": true, + "silent": [] +} \ No newline at end of file diff --git a/crawlers/deploy/monitoring-crawlers.service b/crawlers/deploy/monitoring-crawlers.service new file mode 100644 index 00000000..5a5aa09f --- /dev/null +++ b/crawlers/deploy/monitoring-crawlers.service @@ -0,0 +1,17 @@ +[Unit] +Description=Monitor crawlers systemd state +StartLimitIntervalSec=300 +StartLimitBurst=3 +After=network.target + +[Service] +Restart=on-failure +RestartSec=15s +WorkingDirectory=/home/ubuntu/ +EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env +ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service +CPUWeight=90 +SyslogIdentifier=monitoring-crawlers + +[Install] +WantedBy=multi-user.target \ No newline at end of file