wangjia
a879494fe1
避免瞬时尖峰/部署窗口误报(此前 sing-box 部署期 activating 触发了一次误报)。 - 采集间隔 5min → 1min(MONITOR_INTERVAL_MIN=1) - 新增 MONITOR_BREACH_RUNS=5:连续 5 次采样都异常才告警一次(状态存 /var/lib/pangolin-monitor/breach_count);恢复后发一条恢复通知并清零 - 去掉 MONITOR_SEND_OK_PULSE(被去抖逻辑取代) - 已在 racknerd 验证:连 4 次异常不报、恢复清零 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
112 lines
4.2 KiB
Bash
Executable File
112 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# pangolin-monitor —— 节点自监控:采集 CPU/内存/磁盘/进程/网络,经 Telegram 上报。
|
|
# 由 systemd timer 定期触发(见 pangolin-monitor.timer)。配置在 /etc/pangolin-monitor.env。
|
|
#
|
|
# 阈值越界 / 受监控单元掉线 / 出网不通 = 一次"异常采样";
|
|
# 连续 MONITOR_BREACH_RUNS 次异常才告警一次(去抖),恢复后发一条恢复通知。
|
|
# 建议配合 1 分钟一次的 timer:连续 5 次 = 持续 5 分钟异常才报。
|
|
# --boot:发送一条"初始化完成"启动通知。
|
|
#
|
|
# 注意:节点宕机后无法自报("不上报"类告警)需在常在线主机上跑 deadman-watch.sh(见 README)。
|
|
set -uo pipefail
|
|
|
|
ENV_FILE=/etc/pangolin-monitor.env
|
|
[ -r "$ENV_FILE" ] || { echo "缺少 $ENV_FILE" >&2; exit 1; }
|
|
# shellcheck disable=SC1090
|
|
. "$ENV_FILE"
|
|
|
|
: "${NODE_NAME:=$(hostname)}"
|
|
: "${CPU_ALERT_PCT:=90}"; : "${MEM_ALERT_PCT:=90}"; : "${DISK_ALERT_PCT:=85}"
|
|
: "${MONITOR_UNITS:=}"; : "${MONITOR_NET_PROBE:=https://www.cloudflare.com/cdn-cgi/trace}"
|
|
: "${MONITOR_BREACH_RUNS:=5}" # 连续几次采样异常才告警(去抖)
|
|
|
|
send_tg() {
|
|
local text="$1"
|
|
[ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ] || { echo "$text"; return; }
|
|
curl -fsS --max-time 15 \
|
|
"https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
|
--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
|
|
--data-urlencode "text=${text}" \
|
|
--data "disable_web_page_preview=true" >/dev/null 2>&1 \
|
|
|| echo "Telegram 发送失败" >&2
|
|
}
|
|
|
|
# CPU% —— /proc/stat 两次采样差值
|
|
cpu_pct() {
|
|
local a_idle a_tot b_idle b_tot
|
|
read -r a_idle a_tot < <(awk '/^cpu /{t=0;for(i=2;i<=NF;i++)t+=$i;print $5,t}' /proc/stat)
|
|
sleep 1
|
|
read -r b_idle b_tot < <(awk '/^cpu /{t=0;for(i=2;i<=NF;i++)t+=$i;print $5,t}' /proc/stat)
|
|
awk -v ai="$a_idle" -v at="$a_tot" -v bi="$b_idle" -v bt="$b_tot" \
|
|
'BEGIN{ d=bt-at; if(d<=0){print 0} else {printf "%.0f", 100*(1-(bi-ai)/d)} }'
|
|
}
|
|
|
|
mem_pct() { free -m | awk '/^Mem:/{printf "%.0f", ($2-$7)/$2*100}'; } # 用 available 列
|
|
disk_pct() { df -P / | awk 'NR==2{gsub("%","",$5); print $5}'; }
|
|
|
|
main() {
|
|
local boot=false
|
|
[ "${1:-}" = "--boot" ] && boot=true
|
|
|
|
local cpu mem disk net alert=0 lines=()
|
|
cpu="$(cpu_pct)"; mem="$(mem_pct)"; disk="$(disk_pct)"
|
|
|
|
[ "${cpu:-0}" -ge "$CPU_ALERT_PCT" ] && alert=1
|
|
[ "${mem:-0}" -ge "$MEM_ALERT_PCT" ] && alert=1
|
|
[ "${disk:-0}" -ge "$DISK_ALERT_PCT" ] && alert=1
|
|
|
|
lines+=("CPU ${cpu}% 内存 ${mem}% 磁盘 ${disk}%")
|
|
|
|
# 进程/单元存活
|
|
local u st ustr=""
|
|
for u in $MONITOR_UNITS; do
|
|
if ! systemctl list-unit-files "${u}.service" >/dev/null 2>&1 \
|
|
&& ! systemctl cat "${u}.service" >/dev/null 2>&1; then
|
|
continue # 单元不存在(此节点未部署该组件)→ 忽略
|
|
fi
|
|
st="$(systemctl is-active "${u}.service" 2>/dev/null || true)"
|
|
[ "$st" = "active" ] || alert=1
|
|
ustr+="${u}=${st} "
|
|
done
|
|
[ -n "$ustr" ] && lines+=("进程: ${ustr}")
|
|
|
|
# 出网连通
|
|
if curl -fsS --max-time 8 "$MONITOR_NET_PROBE" >/dev/null 2>&1; then net="ok"; else net="FAIL"; alert=1; fi
|
|
lines+=("出网: ${net}")
|
|
|
|
local ts metrics l
|
|
ts="$(date -u '+%Y-%m-%d %H:%M:%SZ')"
|
|
metrics=""
|
|
for l in "${lines[@]}"; do metrics+="${l}"$'\n'; done
|
|
|
|
# ── 去抖:每次采集,但连续 MONITOR_BREACH_RUNS 次异常才告警(避免瞬时尖峰/重启误报)──
|
|
local sdir=/var/lib/pangolin-monitor cntf alertedf cnt
|
|
mkdir -p "$sdir"
|
|
cntf="$sdir/breach_count"; alertedf="$sdir/alerted"
|
|
|
|
if $boot; then
|
|
send_tg "🟢 [${NODE_NAME}] 初始化完成 / 监控已上线"$'\n'"${ts}"$'\n'"${metrics}"
|
|
echo 0 > "$cntf"; rm -f "$alertedf"
|
|
return 0
|
|
fi
|
|
|
|
if [ "$alert" -eq 1 ]; then
|
|
cnt=$(( $(cat "$cntf" 2>/dev/null || echo 0) + 1 ))
|
|
echo "$cnt" > "$cntf"
|
|
# 达到连续阈值且尚未告警过 → 发一次(不重复刷屏)
|
|
if [ "$cnt" -ge "$MONITOR_BREACH_RUNS" ] && [ ! -f "$alertedf" ]; then
|
|
send_tg "🔴 [${NODE_NAME}] 异常告警(连续 ${cnt} 次采样异常)"$'\n'"${ts}"$'\n'"${metrics}"
|
|
touch "$alertedf"
|
|
fi
|
|
else
|
|
# 恢复:若此前已告警,发一条恢复通知
|
|
if [ -f "$alertedf" ]; then
|
|
send_tg "🟢 [${NODE_NAME}] 已恢复正常"$'\n'"${ts}"$'\n'"${metrics}"
|
|
fi
|
|
echo 0 > "$cntf"; rm -f "$alertedf"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
main "$@"
|