feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]

新增 server/internal/alert 包(15G):
- 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/
  探针失联/心跳缺失/故障态)
- TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件
  10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier
- LogNotifier:slog 结构化降级实现
- 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制;
  TG 5xx 重试后 fallback 且 Notify() 返回 nil;
  runbook 文件锚点与枚举一致性

接入 scheduler(替换旧的 NotifyFault 桩):
- detect/engine.go:故障态(Rule 5)→ EventTypeFault;
  判封确认(Rule 3)→ EventTypeBlockConfirmed
- orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier
- orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed;
  熔断触发 → EventTypeBreakerTripped
- probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost
- probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s

新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件,
锚点与代码枚举对应。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-16 00:52:26 +08:00
parent cadd527680
commit 5d4b484646
11 changed files with 1387 additions and 66 deletions
+49
View File
@@ -9,6 +9,8 @@ import (
"time"
"github.com/redis/go-redis/v9"
"github.com/wangjia/pangolin/server/internal/alert"
)
// Redis TTL constants for the probe subsystem.
@@ -210,3 +212,50 @@ func (s *Store) AliveProbes(ctx context.Context) ([]string, error) {
}
return ids, nil
}
// CheckHeartbeats scans all known probe heartbeat keys and fires an
// EventTypeHeartbeatMissing alert via notifier for every probe whose last
// heartbeat timestamp is older than threshold.
//
// This is called by 15H (DetectLoop / assembly) on a periodic basis.
// Missing-key semantics apply: a key that expired (TTL elapsed) is not seen
// at all — only keys that exist but carry a stale timestamp are reported.
//
// threshold should be ≥90 s per the operational SLO.
func (s *Store) CheckHeartbeats(ctx context.Context, threshold time.Duration, notifier alert.Notifier) error {
now := time.Now().Unix()
cutoff := now - int64(threshold.Seconds())
var scanErr error
iter := s.rdb.Scan(ctx, 0, "probe:hb:*", 0).Iterator()
for iter.Next(ctx) {
k := iter.Val()
probeID := strings.TrimPrefix(k, "probe:hb:")
val, err := s.rdb.Get(ctx, k).Result()
if err != nil {
// Key may have expired between SCAN and GET; skip.
continue
}
ts, err := strconv.ParseInt(val, 10, 64)
if err != nil {
continue // corrupt value; ignore
}
if ts < cutoff {
// Heartbeat is stale: emit alert.
staleSecs := now - ts
ev := alert.NewEvent(alert.EventTypeHeartbeatMissing, probeID, map[string]string{
"stale_seconds": strconv.FormatInt(staleSecs, 10),
"threshold_s": strconv.FormatInt(int64(threshold.Seconds()), 10),
})
if notifyErr := notifier.Notify(ctx, ev); notifyErr != nil {
// Log but continue checking other probes.
scanErr = fmt.Errorf("probe: notify heartbeat missing for %s: %w", probeID, notifyErr)
}
}
}
if err := iter.Err(); err != nil {
return fmt.Errorf("probe: scan heartbeats: %w", err)
}
return scanErr
}