feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
|
||||
@@ -95,7 +96,7 @@ func NewReplacer(cfg Config) *Replacer {
|
||||
cfg.Breaker = StubBreaker{}
|
||||
}
|
||||
if cfg.Notifier == nil {
|
||||
cfg.Notifier = LogNotifier{}
|
||||
cfg.Notifier = alert.LogNotifier{}
|
||||
}
|
||||
if cfg.Clock == nil {
|
||||
cfg.Clock = RealClock{}
|
||||
@@ -237,11 +238,21 @@ func (r *Replacer) stepPending(ctx context.Context, uuid string, rec *ReplaceRec
|
||||
if !r.breaker.Allow(nodeInfo.Tier, nodeInfo.Region) {
|
||||
slog.Info("orchestrate: breaker blocked replacement",
|
||||
"uuid", uuid, "tier", nodeInfo.Tier, "region", nodeInfo.Region)
|
||||
// Emit 熔断触发 alert (15G exit channel).
|
||||
ev := alert.NewEvent(alert.EventTypeBreakerTripped, rec.OldNode, map[string]string{
|
||||
"tier": nodeInfo.Tier,
|
||||
"region": nodeInfo.Region,
|
||||
"replacement_uuid": uuid,
|
||||
})
|
||||
ev.Pool = nodeInfo.Tier + "/" + nodeInfo.Region
|
||||
if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
slog.Error("orchestrate: notify breaker tripped", "uuid", uuid, "error", notifyErr)
|
||||
}
|
||||
return nil // stay pending; retry next Tick
|
||||
}
|
||||
|
||||
// Watermark / quota check — stub (always passes).
|
||||
// TODO(15F): implement real capacity-quota guard here.
|
||||
// TODO(15F): emit EventTypeWatermarkLow when real capacity guard is wired.
|
||||
|
||||
rec.Phase = PhaseCreating
|
||||
rec.PhaseStartedAt = r.clock.Now()
|
||||
@@ -371,9 +382,14 @@ func (r *Replacer) failProbeAttempt(ctx context.Context, uuid string, rec *Repla
|
||||
_ = r.rdb.SRem(ctx, replaceIndexKey, uuid).Err()
|
||||
_ = r.rdb.Expire(ctx, replaceKeyPrefix+uuid, replaceTTL).Err()
|
||||
|
||||
alertReason := fmt.Sprintf("probing failed after %d attempts: %s", rec.Attempts, reason)
|
||||
if notifyErr := r.notifier.NotifyFault(ctx, rec.OldNode, alertReason); notifyErr != nil {
|
||||
slog.Error("orchestrate: notify fault", "uuid", uuid, "error", notifyErr)
|
||||
// Emit 补新连续失败≥3 alert (15G exit channel).
|
||||
ev := alert.NewEvent(alert.EventTypeReplenishFailed, rec.OldNode, map[string]string{
|
||||
"attempts": fmt.Sprintf("%d", rec.Attempts),
|
||||
"last_reason": reason,
|
||||
"replacement_uuid": uuid,
|
||||
})
|
||||
if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
slog.Error("orchestrate: notify replenish failed", "uuid", uuid, "error", notifyErr)
|
||||
}
|
||||
slog.Error("orchestrate: replacement permanently failed — manual review required",
|
||||
"uuid", uuid, "old_node", rec.OldNode, "attempts", rec.Attempts)
|
||||
|
||||
Reference in New Issue
Block a user