#!/usr/bin/env bash
# deadman-watch —— 死信守望。放在【任意一台常在线主机】(另一台节点 / 家里服务器 / 监控机)
# 上,主动 TCP 探活各节点端口;连续失败到阈值 → 告警(节点宕机/不上报时它自己报不了,
# 这层补上)。恢复也通知。
#
# 配置 /etc/pangolin-deadman.env:
#   TELEGRAM_BOT_TOKEN=...  TELEGRAM_CHAT_ID=...
#   NODES="racknerd-la=107.172.55.251:443 tokyo=1.2.3.4:443"   # name=host:port 空格分隔
#   FAIL_THRESHOLD=2        # 连续失败几次才告警(去抖)
#
# 在该常在线主机上每 3 分钟跑一次(cron,用户态即可):
#   */3 * * * * /usr/local/bin/deadman-watch >/dev/null 2>&1
set -uo pipefail

ENV_FILE=/etc/pangolin-deadman.env
[ -r "$ENV_FILE" ] || { echo "缺少 $ENV_FILE" >&2; exit 1; }
# shellcheck disable=SC1090
. "$ENV_FILE"
: "${FAIL_THRESHOLD:=2}"
STATE_DIR=/var/lib/pangolin-deadman
mkdir -p "$STATE_DIR"

send_tg() {
  local text="$1"
  [ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ] || { echo "$text"; return; }
  curl -fsS --max-time 15 \
    "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
    --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
    --data-urlencode "text=${text}" >/dev/null 2>&1 || echo "Telegram 发送失败" >&2
}

probe() { timeout 8 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null; }

for entry in ${NODES:-}; do
  name="${entry%%=*}"; hp="${entry#*=}"; host="${hp%%:*}"; port="${hp##*:}"
  statef="${STATE_DIR}/${name}.fail"
  if probe "$host" "$port"; then
    if [ -f "$statef" ]; then
      send_tg "🟢 [${name}] 恢复可达(${host}:${port})"
      rm -f "$statef"
    fi
  else
    fails=$(( $(cat "$statef" 2>/dev/null || echo 0) + 1 ))
    echo "$fails" > "$statef"
    if [ "$fails" -eq "$FAIL_THRESHOLD" ]; then
      send_tg "🔴 [${name}] 不可达/疑似宕机(${host}:${port}),连续 ${fails} 次探测失败"
    fi
  fi
done