feat(deploy/bootstrap): 节点幂等初始化脚本(加固+pangolin用户+监控+防火墙)
全新 Debian 12 一键初始化,后续新机复用。四块: - 系统:apt 升级+基础包(含 python3-systemd)、时区、swap 补足、sysctl(BBR/fq/句柄/队列) - Pangolin:建 pangolin 非特权系统用户(/var/lib/pangolin, nologin) - 网络安全:ufw 默认拒入站、放行 SSH+443(tcp/udp);fail2ban sshd jail (精简 Debian 无 rsyslog/auth.log → backend=systemd 读 journald) - SSH 加固:root 仅密钥、可选禁密码(仅当已有 authorized_keys 时生效,防锁死) - 监控:pangolin-monitor(systemd timer,CPU/内存/磁盘/进程/出网 → Telegram, 异常+心跳);deadman-watch(EC2 侧主动探活,补"宕机/不上报"告警) 已在 racknerd(107.172.55.251)实跑验证全部步骤通过。密钥/真实配置 gitignore。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# deadman-watch —— 死信守望。放在【常在线主机】(如 EC2)上,主动 TCP 探活各节点端口;
|
||||
# 连续失败到阈值 → 告警(节点宕机/不上报时,节点自己报不了,这层补上)。恢复也通知。
|
||||
#
|
||||
# 配置 /etc/pangolin-deadman.env:
|
||||
# TELEGRAM_BOT_TOKEN=... TELEGRAM_CHAT_ID=...
|
||||
# NODES="racknerd-la=107.172.55.251:443 tokyo=1.2.3.4:443" # name=host:port 空格分隔
|
||||
# FAIL_THRESHOLD=2 # 连续失败几次才告警(去抖)
|
||||
#
|
||||
# EC2 上每 3 分钟跑一次(cron,用户态即可):
|
||||
# */3 * * * * /usr/local/bin/deadman-watch >/dev/null 2>&1
|
||||
set -uo pipefail
|
||||
|
||||
ENV_FILE=/etc/pangolin-deadman.env
|
||||
[ -r "$ENV_FILE" ] || { echo "缺少 $ENV_FILE" >&2; exit 1; }
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
: "${FAIL_THRESHOLD:=2}"
|
||||
STATE_DIR=/var/lib/pangolin-deadman
|
||||
mkdir -p "$STATE_DIR"
|
||||
|
||||
send_tg() {
|
||||
local text="$1"
|
||||
[ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ] || { echo "$text"; return; }
|
||||
curl -fsS --max-time 15 \
|
||||
"https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
|
||||
--data-urlencode "text=${text}" >/dev/null 2>&1 || echo "Telegram 发送失败" >&2
|
||||
}
|
||||
|
||||
probe() { timeout 8 bash -c "exec 3<>/dev/tcp/$1/$2" 2>/dev/null; }
|
||||
|
||||
for entry in ${NODES:-}; do
|
||||
name="${entry%%=*}"; hp="${entry#*=}"; host="${hp%%:*}"; port="${hp##*:}"
|
||||
statef="${STATE_DIR}/${name}.fail"
|
||||
if probe "$host" "$port"; then
|
||||
if [ -f "$statef" ]; then
|
||||
send_tg "🟢 [${name}] 恢复可达(${host}:${port})"
|
||||
rm -f "$statef"
|
||||
fi
|
||||
else
|
||||
fails=$(( $(cat "$statef" 2>/dev/null || echo 0) + 1 ))
|
||||
echo "$fails" > "$statef"
|
||||
if [ "$fails" -eq "$FAIL_THRESHOLD" ]; then
|
||||
send_tg "🔴 [${name}] 不可达/疑似宕机(${host}:${port}),连续 ${fails} 次探测失败"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,12 @@
|
||||
[Unit]
|
||||
Description=Pangolin 节点监控(采集并经 Telegram 上报)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
EnvironmentFile=/etc/pangolin-monitor.env
|
||||
ExecStart=/usr/local/bin/pangolin-monitor
|
||||
# 监控只读系统状态,降权运行
|
||||
User=root
|
||||
Nice=10
|
||||
Executable
+95
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env bash
|
||||
# pangolin-monitor —— 节点自监控:采集 CPU/内存/磁盘/进程/网络,经 Telegram 上报。
|
||||
# 由 systemd timer 定期触发(见 pangolin-monitor.timer)。配置在 /etc/pangolin-monitor.env。
|
||||
#
|
||||
# 阈值越界 / 受监控单元掉线 / 出网不通 → 标 🔴 并必报;
|
||||
# 一切正常 → 🟢,是否推送由 MONITOR_SEND_OK_PULSE 决定(心跳)。
|
||||
# --boot:发送一条"初始化完成"启动通知。
|
||||
#
|
||||
# 注意:节点宕机后无法自报("不上报"类告警)需在常在线主机上跑 deadman-watch.sh(见 README)。
|
||||
set -uo pipefail
|
||||
|
||||
ENV_FILE=/etc/pangolin-monitor.env
|
||||
[ -r "$ENV_FILE" ] || { echo "缺少 $ENV_FILE" >&2; exit 1; }
|
||||
# shellcheck disable=SC1090
|
||||
. "$ENV_FILE"
|
||||
|
||||
: "${NODE_NAME:=$(hostname)}"
|
||||
: "${CPU_ALERT_PCT:=90}"; : "${MEM_ALERT_PCT:=90}"; : "${DISK_ALERT_PCT:=85}"
|
||||
: "${MONITOR_UNITS:=}"; : "${MONITOR_NET_PROBE:=https://www.cloudflare.com/cdn-cgi/trace}"
|
||||
: "${MONITOR_SEND_OK_PULSE:=true}"
|
||||
|
||||
send_tg() {
|
||||
local text="$1"
|
||||
[ -n "${TELEGRAM_BOT_TOKEN:-}" ] && [ -n "${TELEGRAM_CHAT_ID:-}" ] || { echo "$text"; return; }
|
||||
curl -fsS --max-time 15 \
|
||||
"https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
|
||||
--data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
|
||||
--data-urlencode "text=${text}" \
|
||||
--data "disable_web_page_preview=true" >/dev/null 2>&1 \
|
||||
|| echo "Telegram 发送失败" >&2
|
||||
}
|
||||
|
||||
# CPU% —— /proc/stat 两次采样差值
|
||||
cpu_pct() {
|
||||
local a_idle a_tot b_idle b_tot
|
||||
read -r a_idle a_tot < <(awk '/^cpu /{t=0;for(i=2;i<=NF;i++)t+=$i;print $5,t}' /proc/stat)
|
||||
sleep 1
|
||||
read -r b_idle b_tot < <(awk '/^cpu /{t=0;for(i=2;i<=NF;i++)t+=$i;print $5,t}' /proc/stat)
|
||||
awk -v ai="$a_idle" -v at="$a_tot" -v bi="$b_idle" -v bt="$b_tot" \
|
||||
'BEGIN{ d=bt-at; if(d<=0){print 0} else {printf "%.0f", 100*(1-(bi-ai)/d)} }'
|
||||
}
|
||||
|
||||
mem_pct() { free -m | awk '/^Mem:/{printf "%.0f", ($2-$7)/$2*100}'; } # 用 available 列
|
||||
disk_pct() { df -P / | awk 'NR==2{gsub("%","",$5); print $5}'; }
|
||||
|
||||
main() {
|
||||
local boot=false
|
||||
[ "${1:-}" = "--boot" ] && boot=true
|
||||
|
||||
local cpu mem disk net alert=0 lines=()
|
||||
cpu="$(cpu_pct)"; mem="$(mem_pct)"; disk="$(disk_pct)"
|
||||
|
||||
[ "${cpu:-0}" -ge "$CPU_ALERT_PCT" ] && alert=1
|
||||
[ "${mem:-0}" -ge "$MEM_ALERT_PCT" ] && alert=1
|
||||
[ "${disk:-0}" -ge "$DISK_ALERT_PCT" ] && alert=1
|
||||
|
||||
lines+=("CPU ${cpu}% 内存 ${mem}% 磁盘 ${disk}%")
|
||||
|
||||
# 进程/单元存活
|
||||
local u st ustr=""
|
||||
for u in $MONITOR_UNITS; do
|
||||
if ! systemctl list-unit-files "${u}.service" >/dev/null 2>&1 \
|
||||
&& ! systemctl cat "${u}.service" >/dev/null 2>&1; then
|
||||
continue # 单元不存在(此节点未部署该组件)→ 忽略
|
||||
fi
|
||||
st="$(systemctl is-active "${u}.service" 2>/dev/null || true)"
|
||||
[ "$st" = "active" ] || alert=1
|
||||
ustr+="${u}=${st} "
|
||||
done
|
||||
[ -n "$ustr" ] && lines+=("进程: ${ustr}")
|
||||
|
||||
# 出网连通
|
||||
if curl -fsS --max-time 8 "$MONITOR_NET_PROBE" >/dev/null 2>&1; then net="ok"; else net="FAIL"; alert=1; fi
|
||||
lines+=("出网: ${net}")
|
||||
|
||||
local head body ts
|
||||
ts="$(date -u '+%Y-%m-%d %H:%M:%SZ')"
|
||||
if $boot; then
|
||||
head="🟢 [${NODE_NAME}] 初始化完成 / 监控已上线"
|
||||
elif [ "$alert" -eq 1 ]; then
|
||||
head="🔴 [${NODE_NAME}] 异常告警"
|
||||
else
|
||||
head="🟢 [${NODE_NAME}] 正常"
|
||||
fi
|
||||
body="${head}\n${ts}\n"
|
||||
printf -v body '%s\n%s\n' "$head" "$ts"
|
||||
local l; for l in "${lines[@]}"; do body+="${l}"$'\n'; done
|
||||
|
||||
if $boot || [ "$alert" -eq 1 ] || [ "$MONITOR_SEND_OK_PULSE" = "true" ]; then
|
||||
send_tg "$body"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=每 __INTERVAL__ 分钟运行 pangolin-monitor
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=__INTERVAL__min
|
||||
AccuracySec=20s
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user