feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
package alert_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
)
|
||||
|
||||
// allEventTypes lists every EventType constant for completeness checks.
|
||||
var allEventTypes = []alert.EventType{
|
||||
alert.EventTypeBlockConfirmed,
|
||||
alert.EventTypeReplenishFailed,
|
||||
alert.EventTypeWatermarkLow,
|
||||
alert.EventTypeBreakerTripped,
|
||||
alert.EventTypeProbeAgentLost,
|
||||
alert.EventTypeHeartbeatMissing,
|
||||
alert.EventTypeFault,
|
||||
}
|
||||
|
||||
// expectedAnchors maps each EventType to the <a id="…"> expected in the runbook.
|
||||
var expectedAnchors = map[alert.EventType]string{
|
||||
alert.EventTypeBlockConfirmed: "block-confirmed",
|
||||
alert.EventTypeReplenishFailed: "replenish-failed",
|
||||
alert.EventTypeWatermarkLow: "watermark-low",
|
||||
alert.EventTypeBreakerTripped: "breaker-tripped",
|
||||
alert.EventTypeProbeAgentLost: "probe-agent-lost",
|
||||
alert.EventTypeHeartbeatMissing: "heartbeat-missing",
|
||||
alert.EventTypeFault: "node-fault",
|
||||
}
|
||||
|
||||
// TestRunbookAnchorsInCode verifies that every EventType produces an Event
|
||||
// whose RunbookAnchor matches the expected anchor.
|
||||
func TestRunbookAnchorsInCode(t *testing.T) {
|
||||
for _, et := range allEventTypes {
|
||||
e := alert.NewEvent(et, "node-x", nil)
|
||||
wantAnchor := "#" + expectedAnchors[et]
|
||||
if e.RunbookAnchor != wantAnchor {
|
||||
t.Errorf("EventType %q: RunbookAnchor = %q; want %q",
|
||||
et, e.RunbookAnchor, wantAnchor)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunbookFileContainsAllAnchors verifies that docs/runbook-scheduler.md
|
||||
// contains an <a id="…"> tag for every event type.
|
||||
//
|
||||
// The test is skipped when the file does not exist yet (so it never blocks CI
|
||||
// while the runbook is being drafted) and fails once the file is present but
|
||||
// an anchor is missing.
|
||||
func TestRunbookFileContainsAllAnchors(t *testing.T) {
|
||||
// Navigate up from server/internal/alert to the repo root, then to docs/.
|
||||
// Go test sets cwd to the package directory (server/internal/alert/),
|
||||
// so three levels up reaches the worktree root (where docs/ lives).
|
||||
runbookPath := "../../../docs/runbook-scheduler.md"
|
||||
|
||||
data, err := os.ReadFile(runbookPath)
|
||||
if os.IsNotExist(err) {
|
||||
t.Skip("docs/runbook-scheduler.md not found; skipping anchor check")
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("read runbook: %v", err)
|
||||
}
|
||||
content := string(data)
|
||||
|
||||
for et, anchor := range expectedAnchors {
|
||||
tag := `id="` + anchor + `"`
|
||||
if !strings.Contains(content, tag) {
|
||||
t.Errorf("runbook missing anchor for EventType %q: expected <a %s>", et, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user