feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]

新增 server/internal/alert 包(15G):
- 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/
  探针失联/心跳缺失/故障态)
- TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件
  10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier
- LogNotifier:slog 结构化降级实现
- 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制;
  TG 5xx 重试后 fallback 且 Notify() 返回 nil;
  runbook 文件锚点与枚举一致性

接入 scheduler(替换旧的 NotifyFault 桩):
- detect/engine.go:故障态(Rule 5)→ EventTypeFault;
  判封确认(Rule 3)→ EventTypeBlockConfirmed
- orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier
- orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed;
  熔断触发 → EventTypeBreakerTripped
- probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost
- probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s

新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件,
锚点与代码枚举对应。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-16 00:52:26 +08:00
parent cadd527680
commit 5d4b484646
11 changed files with 1387 additions and 66 deletions
@@ -10,6 +10,7 @@ import (
"github.com/redis/go-redis/v9"
"github.com/wangjia/pangolin/server/internal/alert"
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
)
@@ -95,7 +96,7 @@ func NewReplacer(cfg Config) *Replacer {
cfg.Breaker = StubBreaker{}
}
if cfg.Notifier == nil {
cfg.Notifier = LogNotifier{}
cfg.Notifier = alert.LogNotifier{}
}
if cfg.Clock == nil {
cfg.Clock = RealClock{}
@@ -237,11 +238,21 @@ func (r *Replacer) stepPending(ctx context.Context, uuid string, rec *ReplaceRec
if !r.breaker.Allow(nodeInfo.Tier, nodeInfo.Region) {
slog.Info("orchestrate: breaker blocked replacement",
"uuid", uuid, "tier", nodeInfo.Tier, "region", nodeInfo.Region)
// Emit 熔断触发 alert (15G exit channel).
ev := alert.NewEvent(alert.EventTypeBreakerTripped, rec.OldNode, map[string]string{
"tier": nodeInfo.Tier,
"region": nodeInfo.Region,
"replacement_uuid": uuid,
})
ev.Pool = nodeInfo.Tier + "/" + nodeInfo.Region
if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
slog.Error("orchestrate: notify breaker tripped", "uuid", uuid, "error", notifyErr)
}
return nil // stay pending; retry next Tick
}
// Watermark / quota check — stub (always passes).
// TODO(15F): implement real capacity-quota guard here.
// TODO(15F): emit EventTypeWatermarkLow when real capacity guard is wired.
rec.Phase = PhaseCreating
rec.PhaseStartedAt = r.clock.Now()
@@ -371,9 +382,14 @@ func (r *Replacer) failProbeAttempt(ctx context.Context, uuid string, rec *Repla
_ = r.rdb.SRem(ctx, replaceIndexKey, uuid).Err()
_ = r.rdb.Expire(ctx, replaceKeyPrefix+uuid, replaceTTL).Err()
alertReason := fmt.Sprintf("probing failed after %d attempts: %s", rec.Attempts, reason)
if notifyErr := r.notifier.NotifyFault(ctx, rec.OldNode, alertReason); notifyErr != nil {
slog.Error("orchestrate: notify fault", "uuid", uuid, "error", notifyErr)
// Emit 补新连续失败≥3 alert (15G exit channel).
ev := alert.NewEvent(alert.EventTypeReplenishFailed, rec.OldNode, map[string]string{
"attempts": fmt.Sprintf("%d", rec.Attempts),
"last_reason": reason,
"replacement_uuid": uuid,
})
if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
slog.Error("orchestrate: notify replenish failed", "uuid", uuid, "error", notifyErr)
}
slog.Error("orchestrate: replacement permanently failed — manual review required",
"uuid", uuid, "old_node", rec.OldNode, "attempts", rec.Attempts)