feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,32 +9,14 @@ import (
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/idgen"
|
||||
)
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Notifier — 15G interface stub
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Notifier is the 15G event sink for fault notifications.
|
||||
// The real implementation (15G) sends a Telegram/alerting message; the stub
|
||||
// below writes to the structured logger and is used until 15G is ready.
|
||||
type Notifier interface {
|
||||
NotifyFault(ctx context.Context, nodeID, reason string) error
|
||||
}
|
||||
|
||||
// LogNotifier is a Notifier stub that logs via slog.
|
||||
// It is used when no real Notifier is wired up.
|
||||
type LogNotifier struct{}
|
||||
|
||||
// NotifyFault implements Notifier.
|
||||
func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
|
||||
slog.Warn("node fault detected — manual review required",
|
||||
"node_id", nodeID,
|
||||
"reason", reason,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
// Notifier is the 15G alert outlet used by the detection engine.
|
||||
// It is satisfied by alert.Notifier (the real TG implementation) and by
|
||||
// alert.LogNotifier (the fallback / development stub).
|
||||
type Notifier = alert.Notifier
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Redis key constants
|
||||
@@ -91,7 +73,7 @@ func NewEngine(
|
||||
cfg = &d
|
||||
}
|
||||
if notifier == nil {
|
||||
notifier = LogNotifier{}
|
||||
notifier = alert.LogNotifier{}
|
||||
}
|
||||
return &Engine{
|
||||
probeStore: probeStore,
|
||||
@@ -150,8 +132,11 @@ func (e *Engine) processNode(ctx context.Context, node NodeInfo) error {
|
||||
// Streaks are left unchanged so that when the node recovers the engine
|
||||
// resumes from its current position rather than re-triggering immediately.
|
||||
if isFault(sig) {
|
||||
reason := fmt.Sprintf("domestic_fail_isps=%d overseas_ok=false", sig.DomesticFailISPs)
|
||||
if notifyErr := e.notifier.NotifyFault(ctx, node.ID, reason); notifyErr != nil {
|
||||
ev := alert.NewEvent(alert.EventTypeFault, node.ID, map[string]string{
|
||||
"domestic_fail_isps": fmt.Sprintf("%d", sig.DomesticFailISPs),
|
||||
"overseas_ok": "false",
|
||||
})
|
||||
if notifyErr := e.notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
slog.Error("detect: notify fault", "node_id", node.ID, "error", notifyErr)
|
||||
}
|
||||
return nil // do not persist streak changes
|
||||
@@ -254,6 +239,14 @@ func (e *Engine) processSuspect(ctx context.Context, node NodeInfo, sig NodeSign
|
||||
return e.streaks.Save(ctx, node.ID, sk)
|
||||
}
|
||||
|
||||
// Emit 判封确认 alert (15G exit channel).
|
||||
confirmedEv := alert.NewEvent(alert.EventTypeBlockConfirmed, node.ID, map[string]string{
|
||||
"suspect_streak": fmt.Sprintf("%d", sk.SuspectStreak),
|
||||
})
|
||||
if notifyErr := e.notifier.Notify(ctx, confirmedEv); notifyErr != nil {
|
||||
slog.Error("detect: notify block confirmed", "node_id", node.ID, "error", notifyErr)
|
||||
}
|
||||
|
||||
// Immediately mark down — skip draining per lifecycle policy.
|
||||
downDetail := map[string]any{
|
||||
"from": "blocked_confirmed",
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/detect"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
@@ -29,16 +30,26 @@ func (m *mockSnapshotter) SnapshotsByNode(_ context.Context, nodeID string) (map
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// recordingNotifier records NotifyFault calls for assertion.
|
||||
// recordingNotifier records Notify calls for assertion.
|
||||
type recordingNotifier struct {
|
||||
calls []string // nodeID values
|
||||
events []alert.Event
|
||||
}
|
||||
|
||||
func (r *recordingNotifier) NotifyFault(_ context.Context, nodeID, _ string) error {
|
||||
r.calls = append(r.calls, nodeID)
|
||||
func (r *recordingNotifier) Notify(_ context.Context, e alert.Event) error {
|
||||
r.events = append(r.events, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasFaultEvent returns true if any recorded event has type EventTypeFault.
|
||||
func (r *recordingNotifier) hasFaultEvent() bool {
|
||||
for _, e := range r.events {
|
||||
if e.Type == alert.EventTypeFault {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// newTestRedis creates an in-process Redis (miniredis) and returns a connected
|
||||
// client plus a cleanup function. Tests must call cleanup() at the end.
|
||||
func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
|
||||
@@ -345,11 +356,11 @@ func TestRules(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify fault notification.
|
||||
if tc.wantFaultNotified && len(notifier.calls) == 0 {
|
||||
t.Error("expected NotifyFault to be called, but it was not")
|
||||
if tc.wantFaultNotified && !notifier.hasFaultEvent() {
|
||||
t.Error("expected fault Notify event to be recorded, but it was not")
|
||||
}
|
||||
if !tc.wantFaultNotified && len(notifier.calls) > 0 {
|
||||
t.Errorf("unexpected NotifyFault calls: %v", notifier.calls)
|
||||
if !tc.wantFaultNotified && notifier.hasFaultEvent() {
|
||||
t.Errorf("unexpected fault Notify events: %v", notifier.events)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -594,8 +605,8 @@ func TestFaultNoTransitionNoStreak(t *testing.T) {
|
||||
if events := lc.Events(); len(events) != 0 {
|
||||
t.Errorf("unexpected events: %v", events)
|
||||
}
|
||||
if len(notifier.calls) == 0 {
|
||||
t.Error("expected NotifyFault to be called at least once")
|
||||
if !notifier.hasFaultEvent() {
|
||||
t.Error("expected fault Notify event to be recorded at least once")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user