feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
)
|
||||
|
||||
// Redis TTL constants for the probe subsystem.
|
||||
@@ -210,3 +212,50 @@ func (s *Store) AliveProbes(ctx context.Context) ([]string, error) {
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// CheckHeartbeats scans all known probe heartbeat keys and fires an
|
||||
// EventTypeHeartbeatMissing alert via notifier for every probe whose last
|
||||
// heartbeat timestamp is older than threshold.
|
||||
//
|
||||
// This is called by 15H (DetectLoop / assembly) on a periodic basis.
|
||||
// Missing-key semantics apply: a key that expired (TTL elapsed) is not seen
|
||||
// at all — only keys that exist but carry a stale timestamp are reported.
|
||||
//
|
||||
// threshold should be ≥90 s per the operational SLO.
|
||||
func (s *Store) CheckHeartbeats(ctx context.Context, threshold time.Duration, notifier alert.Notifier) error {
|
||||
now := time.Now().Unix()
|
||||
cutoff := now - int64(threshold.Seconds())
|
||||
|
||||
var scanErr error
|
||||
iter := s.rdb.Scan(ctx, 0, "probe:hb:*", 0).Iterator()
|
||||
for iter.Next(ctx) {
|
||||
k := iter.Val()
|
||||
probeID := strings.TrimPrefix(k, "probe:hb:")
|
||||
|
||||
val, err := s.rdb.Get(ctx, k).Result()
|
||||
if err != nil {
|
||||
// Key may have expired between SCAN and GET; skip.
|
||||
continue
|
||||
}
|
||||
ts, err := strconv.ParseInt(val, 10, 64)
|
||||
if err != nil {
|
||||
continue // corrupt value; ignore
|
||||
}
|
||||
if ts < cutoff {
|
||||
// Heartbeat is stale: emit alert.
|
||||
staleSecs := now - ts
|
||||
ev := alert.NewEvent(alert.EventTypeHeartbeatMissing, probeID, map[string]string{
|
||||
"stale_seconds": strconv.FormatInt(staleSecs, 10),
|
||||
"threshold_s": strconv.FormatInt(int64(threshold.Seconds()), 10),
|
||||
})
|
||||
if notifyErr := notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
// Log but continue checking other probes.
|
||||
scanErr = fmt.Errorf("probe: notify heartbeat missing for %s: %w", probeID, notifyErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := iter.Err(); err != nil {
|
||||
return fmt.Errorf("probe: scan heartbeats: %w", err)
|
||||
}
|
||||
return scanErr
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user