feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]

新增 server/internal/alert 包(15G):
- 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/
  探针失联/心跳缺失/故障态)
- TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件
  10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier
- LogNotifier:slog 结构化降级实现
- 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制;
  TG 5xx 重试后 fallback 且 Notify() 返回 nil;
  runbook 文件锚点与枚举一致性

接入 scheduler(替换旧的 NotifyFault 桩):
- detect/engine.go:故障态(Rule 5)→ EventTypeFault;
  判封确认(Rule 3)→ EventTypeBlockConfirmed
- orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier
- orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed;
  熔断触发 → EventTypeBreakerTripped
- probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost
- probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s

新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件,
锚点与代码枚举对应。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-16 00:52:26 +08:00
parent cadd527680
commit 5d4b484646
11 changed files with 1387 additions and 66 deletions
+21 -10
View File
@@ -9,6 +9,7 @@ import (
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"github.com/wangjia/pangolin/server/internal/alert"
"github.com/wangjia/pangolin/server/internal/scheduler/detect"
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
)
@@ -29,16 +30,26 @@ func (m *mockSnapshotter) SnapshotsByNode(_ context.Context, nodeID string) (map
return nil, nil
}
// recordingNotifier records NotifyFault calls for assertion.
// recordingNotifier records Notify calls for assertion.
type recordingNotifier struct {
calls []string // nodeID values
events []alert.Event
}
func (r *recordingNotifier) NotifyFault(_ context.Context, nodeID, _ string) error {
r.calls = append(r.calls, nodeID)
func (r *recordingNotifier) Notify(_ context.Context, e alert.Event) error {
r.events = append(r.events, e)
return nil
}
// hasFaultEvent returns true if any recorded event has type EventTypeFault.
func (r *recordingNotifier) hasFaultEvent() bool {
for _, e := range r.events {
if e.Type == alert.EventTypeFault {
return true
}
}
return false
}
// newTestRedis creates an in-process Redis (miniredis) and returns a connected
// client plus a cleanup function. Tests must call cleanup() at the end.
func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
@@ -345,11 +356,11 @@ func TestRules(t *testing.T) {
}
// Verify fault notification.
if tc.wantFaultNotified && len(notifier.calls) == 0 {
t.Error("expected NotifyFault to be called, but it was not")
if tc.wantFaultNotified && !notifier.hasFaultEvent() {
t.Error("expected fault Notify event to be recorded, but it was not")
}
if !tc.wantFaultNotified && len(notifier.calls) > 0 {
t.Errorf("unexpected NotifyFault calls: %v", notifier.calls)
if !tc.wantFaultNotified && notifier.hasFaultEvent() {
t.Errorf("unexpected fault Notify events: %v", notifier.events)
}
})
}
@@ -594,8 +605,8 @@ func TestFaultNoTransitionNoStreak(t *testing.T) {
if events := lc.Events(); len(events) != 0 {
t.Errorf("unexpected events: %v", events)
}
if len(notifier.calls) == 0 {
t.Error("expected NotifyFault to be called at least once")
if !notifier.hasFaultEvent() {
t.Error("expected fault Notify event to be recorded at least once")
}
}