kopia lustrzana https://github.com/bugout-dev/moonstream
Crawlers systemd monitoring
rodzic
63cd514016
commit
c196c9e0d2
|
@ -22,19 +22,19 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python"
|
||||||
PIP="${PYTHON_ENV_DIR}/bin/pip"
|
PIP="${PYTHON_ENV_DIR}/bin/pip"
|
||||||
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
|
||||||
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
|
||||||
|
PARAMETERS_ENV_MONITORING_PATH="${SECRETS_DIR}/monitoring.env"
|
||||||
SCRIPT_DIR="$(realpath $(dirname $0))"
|
SCRIPT_DIR="$(realpath $(dirname $0))"
|
||||||
|
|
||||||
# Service files
|
# Service files
|
||||||
MOONCRAWL_SERVICE_FILE="mooncrawl.service"
|
MOONCRAWL_SERVICE_FILE="mooncrawl.service"
|
||||||
|
MONITORING_CRAWLERS_SERVICE_FILE="monitoring-crawlers.service"
|
||||||
LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service"
|
LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service"
|
||||||
LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
|
LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
|
||||||
|
|
||||||
|
|
||||||
# Ethereum service files
|
# Ethereum service files
|
||||||
ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
|
ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
|
||||||
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
|
ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
|
||||||
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
|
ETHEREUM_TRENDING_TIMER_FILE="ethereum-trending.timer"
|
||||||
ETHEREUM_TXPOOL_SERVICE_FILE="ethereum-txpool.service"
|
|
||||||
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
|
ETHEREUM_MISSING_SERVICE_FILE="ethereum-missing.service"
|
||||||
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
|
ETHEREUM_MISSING_TIMER_FILE="ethereum-missing.timer"
|
||||||
ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service"
|
ETHEREUM_MOONWORM_CRAWLER_SERVICE_FILE="ethereum-moonworm-crawler.service"
|
||||||
|
@ -51,7 +51,6 @@ POLYGON_MISSING_SERVICE_FILE="polygon-missing.service"
|
||||||
POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
|
POLYGON_MISSING_TIMER_FILE="polygon-missing.timer"
|
||||||
POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
|
POLYGON_STATISTICS_SERVICE_FILE="polygon-statistics.service"
|
||||||
POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
|
POLYGON_STATISTICS_TIMER_FILE="polygon-statistics.timer"
|
||||||
POLYGON_TXPOOL_SERVICE_FILE="polygon-txpool.service"
|
|
||||||
POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
|
POLYGON_MOONWORM_CRAWLER_SERVICE_FILE="polygon-moonworm-crawler.service"
|
||||||
POLYGON_STATE_SERVICE_FILE="polygon-state.service"
|
POLYGON_STATE_SERVICE_FILE="polygon-state.service"
|
||||||
POLYGON_STATE_TIMER_FILE="polygon-state.timer"
|
POLYGON_STATE_TIMER_FILE="polygon-state.timer"
|
||||||
|
@ -131,14 +130,6 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-histor
|
||||||
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
echo
|
|
||||||
echo
|
|
||||||
echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go"
|
|
||||||
EXEC_DIR=$(pwd)
|
|
||||||
cd "${APP_CRAWLERS_DIR}/txpool"
|
|
||||||
HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/txpool/txpool" "${APP_CRAWLERS_DIR}/txpool/main.go"
|
|
||||||
cd "${EXEC_DIR}"
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
|
echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
|
||||||
|
@ -156,8 +147,11 @@ HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@la
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
|
echo -e "${PREFIX_INFO} Retrieving deployment parameters"
|
||||||
mkdir -p "${SECRETS_DIR}"
|
if [ ! -d "${SECRETS_DIR}" ]; then
|
||||||
|
mkdir -p "${SECRETS_DIR}"
|
||||||
|
echo -e "${PREFIX_WARN} Created new secrets directory"
|
||||||
|
fi
|
||||||
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+moonstream:true > "${PARAMETERS_ENV_PATH}"
|
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+moonstream:true > "${PARAMETERS_ENV_PATH}"
|
||||||
chmod 0640 "${PARAMETERS_ENV_PATH}"
|
chmod 0640 "${PARAMETERS_ENV_PATH}"
|
||||||
|
|
||||||
|
@ -166,6 +160,45 @@ echo
|
||||||
echo -e "${PREFIX_INFO} Add instance local IP to parameters"
|
echo -e "${PREFIX_INFO} Add instance local IP to parameters"
|
||||||
echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}"
|
echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Copy monitoring binary from AWS S3"
|
||||||
|
aws s3 cp s3://bugout-binaries/prod/monitoring/monitoring "/home/ubuntu/monitoring"
|
||||||
|
chmod +x "/home/ubuntu/monitoring"
|
||||||
|
chown ubuntu:ubuntu "/home/ubuntu/monitoring"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters"
|
||||||
|
AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}"
|
||||||
|
chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Add instance local IP to monitoring parameters"
|
||||||
|
echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORING_PATH}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters"
|
||||||
|
echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Prepare monitoring configuration"
|
||||||
|
if [ ! -d "/home/ubuntu/.monitoring" ]; then
|
||||||
|
mkdir -p /home/ubuntu/.monitoring
|
||||||
|
echo -e "${PREFIX_WARN} Created monitoring configuration directory"
|
||||||
|
fi
|
||||||
|
cp "${SCRIPT_DIR}/monitoring-crawlers-config.json" /home/ubuntu/.monitoring/monitoring-crawlers-config.json
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
if [ ! -d "/home/ubuntu/.config/systemd/user/" ]; then
|
||||||
|
mkdir -p /home/ubuntu/.config/systemd/user/
|
||||||
|
echo -e "${PREFIX_WARN} Created user systemd directory"
|
||||||
|
fi
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Replacing existing Moonstream crawlers HTTP API server service definition with ${MOONCRAWL_SERVICE_FILE}"
|
echo -e "${PREFIX_INFO} Replacing existing Moonstream crawlers HTTP API server service definition with ${MOONCRAWL_SERVICE_FILE}"
|
||||||
|
@ -191,14 +224,6 @@ cp "${SCRIPT_DIR}/${ETHEREUM_TRENDING_TIMER_FILE}" "/home/ubuntu/.config/systemd
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TRENDING_TIMER_FILE}"
|
||||||
|
|
||||||
# echo
|
|
||||||
# echo
|
|
||||||
# echo -e "${PREFIX_INFO} Replacing existing Ethereum transaction pool crawler service definition with ${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
|
||||||
# chmod 644 "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
|
||||||
# cp "${SCRIPT_DIR}/${ETHEREUM_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
|
||||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
|
||||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${ETHEREUM_TXPOOL_SERVICE_FILE}"
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
|
echo -e "${PREFIX_INFO} Replacing existing Ethereum missing service and timer with: ${ETHEREUM_MISSING_SERVICE_FILE}, ${ETHEREUM_MISSING_TIMER_FILE}"
|
||||||
|
@ -270,14 +295,6 @@ cp "${SCRIPT_DIR}/${POLYGON_STATISTICS_TIMER_FILE}" "/home/ubuntu/.config/system
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}"
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_STATISTICS_TIMER_FILE}"
|
||||||
|
|
||||||
# echo
|
|
||||||
# echo
|
|
||||||
# echo -e "${PREFIX_INFO} Replacing existing Polygon transaction pool crawler service definition with ${POLYGON_TXPOOL_SERVICE_FILE}"
|
|
||||||
# chmod 644 "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}"
|
|
||||||
# cp "${SCRIPT_DIR}/${POLYGON_TXPOOL_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${POLYGON_TXPOOL_SERVICE_FILE}"
|
|
||||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
|
||||||
# XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${POLYGON_TXPOOL_SERVICE_FILE}"
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
|
echo -e "${PREFIX_INFO} Replacing existing Polygon moonworm crawler service definition with ${POLYGON_MOONWORM_CRAWLER_SERVICE_FILE}"
|
||||||
|
@ -620,4 +637,12 @@ chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "${SCRIPT_DIR}/${L
|
||||||
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_SERVICE_FILE}"
|
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_SERVICE_FILE}"
|
||||||
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}"
|
cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}"
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo
|
||||||
|
echo -e "${PREFIX_INFO} Replacing existing systemd crawlers monitoring service definition with ${MONITORING_CRAWLERS_SERVICE_FILE}"
|
||||||
|
chmod 644 "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}"
|
||||||
|
cp "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${MONITORING_CRAWLERS_SERVICE_FILE}"
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
|
||||||
|
XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${MONITORING_CRAWLERS_SERVICE_FILE}"
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
{
|
||||||
|
"notification_pagerduty": true,
|
||||||
|
"notification_telegram": true,
|
||||||
|
"notification_sendgrid": true,
|
||||||
|
"notification_humbug": true,
|
||||||
|
"silent": []
|
||||||
|
}
|
|
@ -0,0 +1,17 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Monitor crawlers systemd state
|
||||||
|
StartLimitIntervalSec=300
|
||||||
|
StartLimitBurst=3
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=15s
|
||||||
|
WorkingDirectory=/home/ubuntu/
|
||||||
|
EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env
|
||||||
|
ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service
|
||||||
|
CPUWeight=90
|
||||||
|
SyslogIdentifier=monitoring-crawlers
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
Ładowanie…
Reference in New Issue