feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]

新增 server/internal/alert 包(15G):
- 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/
  探针失联/心跳缺失/故障态)
- TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件
  10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier
- LogNotifier:slog 结构化降级实现
- 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制;
  TG 5xx 重试后 fallback 且 Notify() 返回 nil;
  runbook 文件锚点与枚举一致性

接入 scheduler(替换旧的 NotifyFault 桩):
- detect/engine.go:故障态(Rule 5)→ EventTypeFault;
  判封确认(Rule 3)→ EventTypeBlockConfirmed
- orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier
- orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed;
  熔断触发 → EventTypeBreakerTripped
- probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost
- probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s

新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件,
锚点与代码枚举对应。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-16 00:52:26 +08:00
parent cadd527680
commit 5d4b484646
11 changed files with 1387 additions and 66 deletions
+19 -26
View File
@@ -9,32 +9,14 @@ import (
"github.com/redis/go-redis/v9"
"github.com/wangjia/pangolin/server/internal/alert"
"github.com/wangjia/pangolin/server/internal/idgen"
)
// ─────────────────────────────────────────────────────────────────────────────
// Notifier — 15G interface stub
// ─────────────────────────────────────────────────────────────────────────────
// Notifier is the 15G event sink for fault notifications.
// The real implementation (15G) sends a Telegram/alerting message; the stub
// below writes to the structured logger and is used until 15G is ready.
type Notifier interface {
NotifyFault(ctx context.Context, nodeID, reason string) error
}
// LogNotifier is a Notifier stub that logs via slog.
// It is used when no real Notifier is wired up.
type LogNotifier struct{}
// NotifyFault implements Notifier.
func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
slog.Warn("node fault detected — manual review required",
"node_id", nodeID,
"reason", reason,
)
return nil
}
// Notifier is the 15G alert outlet used by the detection engine.
// It is satisfied by alert.Notifier (the real TG implementation) and by
// alert.LogNotifier (the fallback / development stub).
type Notifier = alert.Notifier
// ─────────────────────────────────────────────────────────────────────────────
// Redis key constants
@@ -91,7 +73,7 @@ func NewEngine(
cfg = &d
}
if notifier == nil {
notifier = LogNotifier{}
notifier = alert.LogNotifier{}
}
return &Engine{
probeStore: probeStore,
@@ -150,8 +132,11 @@ func (e *Engine) processNode(ctx context.Context, node NodeInfo) error {
// Streaks are left unchanged so that when the node recovers the engine
// resumes from its current position rather than re-triggering immediately.
if isFault(sig) {
reason := fmt.Sprintf("domestic_fail_isps=%d overseas_ok=false", sig.DomesticFailISPs)
if notifyErr := e.notifier.NotifyFault(ctx, node.ID, reason); notifyErr != nil {
ev := alert.NewEvent(alert.EventTypeFault, node.ID, map[string]string{
"domestic_fail_isps": fmt.Sprintf("%d", sig.DomesticFailISPs),
"overseas_ok": "false",
})
if notifyErr := e.notifier.Notify(ctx, ev); notifyErr != nil {
slog.Error("detect: notify fault", "node_id", node.ID, "error", notifyErr)
}
return nil // do not persist streak changes
@@ -254,6 +239,14 @@ func (e *Engine) processSuspect(ctx context.Context, node NodeInfo, sig NodeSign
return e.streaks.Save(ctx, node.ID, sk)
}
// Emit 判封确认 alert (15G exit channel).
confirmedEv := alert.NewEvent(alert.EventTypeBlockConfirmed, node.ID, map[string]string{
"suspect_streak": fmt.Sprintf("%d", sk.SuspectStreak),
})
if notifyErr := e.notifier.Notify(ctx, confirmedEv); notifyErr != nil {
slog.Error("detect: notify block confirmed", "node_id", node.ID, "error", notifyErr)
}
// Immediately mark down — skip draining per lifecycle policy.
downDetail := map[string]any{
"from": "blocked_confirmed",