Crawlers systemd monitoring

crawlers/deploy/deploy.bash

@@ -22,14 +22,15 @@ PYTHON="${PYTHON_ENV_DIR}/bin/python"
 PIP="${PYTHON_ENV_DIR}/bin/pip"
 SECRETS_DIR="${SECRETS_DIR:-/home/ubuntu/moonstream-secrets}"
 PARAMETERS_ENV_PATH="${SECRETS_DIR}/app.env"
+PARAMETERS_ENV_MONITORING_PATH="${SECRETS_DIR}/monitoring.env"
 SCRIPT_DIR="$(realpath $(dirname $0))"
 
 # Service files
 MOONCRAWL_SERVICE_FILE="mooncrawl.service"
+MONITORING_CRAWLERS_SERVICE_FILE="monitoring-crawlers.service"
 LEADERBOARDS_WORKER_SERVICE_FILE="leaderboards-worker.service"
 LEADERBOARDS_WORKER_TIMER_FILE="leaderboards-worker.timer"
 
-
 # Ethereum service files
 ETHEREUM_SYNCHRONIZE_SERVICE_FILE="ethereum-synchronize.service"
 ETHEREUM_TRENDING_SERVICE_FILE="ethereum-trending.service"
@@ -129,14 +130,6 @@ ZKSYNC_ERA_TESTNET_HISTORICAL_CRAWL_EVENTS_TIMER_FILE="zksync-era-testnet-histor
 
 set -eu
 
-echo
-echo
-echo -e "${PREFIX_INFO} Building executable Ethereum transaction pool crawler script with Go"
-EXEC_DIR=$(pwd)
-cd "${APP_CRAWLERS_DIR}/txpool"
-HOME=/home/ubuntu /usr/local/go/bin/go build -o "${APP_CRAWLERS_DIR}/txpool/txpool" "${APP_CRAWLERS_DIR}/txpool/main.go"
-cd "${EXEC_DIR}"
-
 echo
 echo
 echo -e "${PREFIX_INFO} Upgrading Python pip and setuptools"
@@ -154,8 +147,11 @@ HOME=/home/ubuntu /usr/local/go/bin/go install github.com/bugout-dev/checkenv@la
 
 echo
 echo
-echo -e "${PREFIX_INFO} Retrieving addition deployment parameters"
-mkdir -p "${SECRETS_DIR}"
+echo -e "${PREFIX_INFO} Retrieving deployment parameters"
+if [ ! -d "${SECRETS_DIR}" ]; then
+  mkdir -p "${SECRETS_DIR}"
+  echo -e "${PREFIX_WARN} Created new secrets directory"
+fi
 AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+moonstream:true > "${PARAMETERS_ENV_PATH}"
 chmod 0640 "${PARAMETERS_ENV_PATH}"
 
@@ -164,6 +160,45 @@ echo
 echo -e "${PREFIX_INFO} Add instance local IP to parameters"
 echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_PATH}"
 
+echo
+echo
+echo -e "${PREFIX_INFO} Copy monitoring binary from AWS S3"
+aws s3 cp s3://bugout-binaries/prod/monitoring/monitoring "/home/ubuntu/monitoring"
+chmod +x "/home/ubuntu/monitoring"
+chown ubuntu:ubuntu "/home/ubuntu/monitoring"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Retrieving monitoring deployment parameters"
+AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION}" /home/ubuntu/go/bin/checkenv show aws_ssm+crawlers:true,monitoring:true > "${PARAMETERS_ENV_MONITORING_PATH}"
+chmod 0640 "${PARAMETERS_ENV_MONITORING_PATH}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Add instance local IP to monitoring parameters"
+echo "AWS_LOCAL_IPV4=$(ec2metadata --local-ipv4)" >> "${PARAMETERS_ENV_MONITORING_PATH}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Add AWS default region to monitring parameters"
+echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" >> "${PARAMETERS_ENV_MONITORING_PATH}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Prepare monitoring configuration"
+if [ ! -d "/home/ubuntu/.monitoring" ]; then
+  mkdir -p /home/ubuntu/.monitoring
+  echo -e "${PREFIX_WARN} Created monitoring configuration directory"
+fi
+cp "${SCRIPT_DIR}/monitoring-crawlers-config.json" /home/ubuntu/.monitoring/monitoring-crawlers-config.json
+
+echo
+echo
+if [ ! -d "/home/ubuntu/.config/systemd/user/" ]; then
+  mkdir -p /home/ubuntu/.config/systemd/user/
+  echo -e "${PREFIX_WARN} Created user systemd directory"
+fi
+
 echo
 echo
 echo -e "${PREFIX_INFO} Replacing existing Moonstream crawlers HTTP API server service definition with ${MOONCRAWL_SERVICE_FILE}"
@@ -602,4 +637,12 @@ chmod 644 "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "${SCRIPT_DIR}/${L
 cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_SERVICE_FILE}"
 cp "${SCRIPT_DIR}/${LEADERBOARDS_WORKER_TIMER_FILE}" "/home/ubuntu/.config/systemd/user/${LEADERBOARDS_WORKER_TIMER_FILE}"
 XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
-XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart --no-block "${LEADERBOARDS_WORKER_TIMER_FILE}"
+
+echo
+echo
+echo -e "${PREFIX_INFO} Replacing existing systemd crawlers monitoring service definition with ${MONITORING_CRAWLERS_SERVICE_FILE}"
+chmod 644 "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}"
+cp "${SCRIPT_DIR}/${MONITORING_CRAWLERS_SERVICE_FILE}" "/home/ubuntu/.config/systemd/user/${MONITORING_CRAWLERS_SERVICE_FILE}"
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user daemon-reload
+XDG_RUNTIME_DIR="/run/user/1000" systemctl --user restart "${MONITORING_CRAWLERS_SERVICE_FILE}"

crawlers/deploy/monitoring-crawlers-config.json

@@ -0,0 +1,7 @@
+{
+    "notification_pagerduty": true,
+    "notification_telegram": true,
+    "notification_sendgrid": true,
+    "notification_humbug": true,
+    "silent": []
+}

crawlers/deploy/monitoring-crawlers.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Monitor crawlers systemd state
+StartLimitIntervalSec=300
+StartLimitBurst=3
+After=network.target
+
+[Service]
+Restart=on-failure
+RestartSec=15s
+WorkingDirectory=/home/ubuntu/
+EnvironmentFile=/home/ubuntu/moonstream-secrets/monitoring.env
+ExecStart=/home/ubuntu/monitoring -plugin systemd -host "${AWS_LOCAL_IPV4}" -port 7171 -healthcheck -server -threshold 3 -config /home/ubuntu/.monitoring/monitoring-crawlers-config.json -service ethereum-moonworm-crawler.service -service mumbai-moonworm-crawler.service -service polygon-moonworm-crawler.service -service zksync-era-moonworm-crawler.service
+CPUWeight=90
+SyslogIdentifier=monitoring-crawlers
+
+[Install]
+WantedBy=multi-user.target