5d4b484646
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
309 lines
12 KiB
Go
309 lines
12 KiB
Go
package alert_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
"github.com/redis/go-redis/v9"
|
|
|
|
"github.com/wangjia/pangolin/server/internal/alert"
|
|
)
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test helpers
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func newTestRedis(t *testing.T) *redis.Client {
|
|
t.Helper()
|
|
mr := miniredis.RunT(t)
|
|
return redis.NewClient(&redis.Options{Addr: mr.Addr()})
|
|
}
|
|
|
|
// tgServer is a tiny Telegram Bot API mock.
|
|
type tgServer struct {
|
|
statusCode atomic.Int32 // HTTP status to return; default 200
|
|
callCount atomic.Int32 // total calls received
|
|
lastBody atomic.Value // last []byte body received
|
|
srv *httptest.Server
|
|
}
|
|
|
|
func newTGServer(t *testing.T) *tgServer {
|
|
t.Helper()
|
|
ts := &tgServer{}
|
|
ts.statusCode.Store(http.StatusOK)
|
|
ts.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
ts.callCount.Add(1)
|
|
body, _ := io.ReadAll(r.Body)
|
|
ts.lastBody.Store(body)
|
|
w.WriteHeader(int(ts.statusCode.Load()))
|
|
w.Write([]byte(`{"ok":true}`)) //nolint:errcheck
|
|
}))
|
|
t.Cleanup(ts.srv.Close)
|
|
return ts
|
|
}
|
|
|
|
// parsedBody parses the last received request body as a map.
|
|
func (ts *tgServer) parsedBody(t *testing.T) map[string]string {
|
|
t.Helper()
|
|
raw, _ := ts.lastBody.Load().([]byte)
|
|
if len(raw) == 0 {
|
|
t.Fatal("tgServer: no body received yet")
|
|
}
|
|
var m map[string]string
|
|
if err := json.Unmarshal(raw, &m); err != nil {
|
|
t.Fatalf("tgServer: unmarshal body: %v", err)
|
|
}
|
|
return m
|
|
}
|
|
|
|
// newNotifier creates a TGNotifier pointing at the mock server.
|
|
func newNotifier(t *testing.T, ts *tgServer, rdb *redis.Client) *alert.TGNotifier {
|
|
t.Helper()
|
|
return alert.NewTGNotifier(alert.TGConfig{
|
|
BotToken: "test-token",
|
|
ChatID: "-1001234567",
|
|
RunbookBaseURL: "https://example.internal/runbook",
|
|
BaseURL: ts.srv.URL,
|
|
}, rdb, nil)
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: all seven event types render correct templates with runbook anchors
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestAllEventTypesRendered(t *testing.T) {
|
|
tests := []struct {
|
|
eventType alert.EventType
|
|
wantAnchor string
|
|
wantTitlePart string // substring expected in the TG message text (HTML-encoded if needed)
|
|
}{
|
|
{alert.EventTypeBlockConfirmed, "#block-confirmed", "判封确认"},
|
|
{alert.EventTypeReplenishFailed, "#replenish-failed", "补新连续失败≥3"},
|
|
// "<" is HTML-escaped to "<" in TG HTML mode — check escaped form.
|
|
{alert.EventTypeWatermarkLow, "#watermark-low", "水位<70%"},
|
|
{alert.EventTypeBreakerTripped, "#breaker-tripped", "熔断触发"},
|
|
{alert.EventTypeProbeAgentLost, "#probe-agent-lost", "探针失联"},
|
|
// ">" is HTML-escaped to ">" in TG HTML mode — check escaped form.
|
|
{alert.EventTypeHeartbeatMissing, "#heartbeat-missing", "心跳缺失>90s"},
|
|
{alert.EventTypeFault, "#node-fault", "故障态"},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(string(tc.eventType), func(t *testing.T) {
|
|
rdb := newTestRedis(t)
|
|
ts := newTGServer(t)
|
|
n := newNotifier(t, ts, rdb)
|
|
|
|
e := alert.NewEvent(tc.eventType, "node-abc", map[string]string{
|
|
"reason": "test reason",
|
|
})
|
|
|
|
ctx := context.Background()
|
|
if err := n.Notify(ctx, e); err != nil {
|
|
t.Fatalf("Notify() returned error: %v", err)
|
|
}
|
|
|
|
// Exactly one TG call must have been made.
|
|
if got := ts.callCount.Load(); got != 1 {
|
|
t.Fatalf("TG call count = %d; want 1", got)
|
|
}
|
|
|
|
body := ts.parsedBody(t)
|
|
text := body["text"]
|
|
|
|
// Message must contain the Chinese event title.
|
|
if !strings.Contains(text, tc.wantTitlePart) {
|
|
t.Errorf("message does not contain %q:\n%s", tc.wantTitlePart, text)
|
|
}
|
|
|
|
// Message must contain the runbook anchor.
|
|
if !strings.Contains(text, tc.wantAnchor) {
|
|
t.Errorf("message does not contain runbook anchor %q:\n%s", tc.wantAnchor, text)
|
|
}
|
|
|
|
// Message must contain the node ID (internal only — no domain).
|
|
if !strings.Contains(text, "node-abc") {
|
|
t.Errorf("message does not contain node ID:\n%s", text)
|
|
}
|
|
|
|
// parse_mode must be HTML.
|
|
if body["parse_mode"] != "HTML" {
|
|
t.Errorf("parse_mode = %q; want HTML", body["parse_mode"])
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: dedup — second message for same (Type, NodeID) within 10 min is suppressed
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestDedupSuppressesSecondMessage(t *testing.T) {
|
|
rdb := newTestRedis(t)
|
|
ts := newTGServer(t)
|
|
n := newNotifier(t, ts, rdb)
|
|
|
|
ctx := context.Background()
|
|
// Use a Warning-severity event (WatermarkLow) so dedup applies.
|
|
// Note: WatermarkLow has Warning severity → dedup applies.
|
|
e := alert.NewEvent(alert.EventTypeBlockConfirmed, "node-dup", nil)
|
|
|
|
// First call — must go through.
|
|
if err := n.Notify(ctx, e); err != nil {
|
|
t.Fatalf("first Notify() error: %v", err)
|
|
}
|
|
if got := ts.callCount.Load(); got != 1 {
|
|
t.Fatalf("after first call: TG count = %d; want 1", got)
|
|
}
|
|
|
|
// Second call with same (Type, NodeID) — must be suppressed (no TG call).
|
|
if err := n.Notify(ctx, e); err != nil {
|
|
t.Fatalf("second Notify() error: %v", err)
|
|
}
|
|
if got := ts.callCount.Load(); got != 1 {
|
|
t.Errorf("after second call: TG count = %d; want still 1 (dedup)", got)
|
|
}
|
|
|
|
// Third call with different NodeID — must go through (distinct dedup key).
|
|
e2 := alert.NewEvent(alert.EventTypeBlockConfirmed, "node-other", nil)
|
|
if err := n.Notify(ctx, e2); err != nil {
|
|
t.Fatalf("third Notify() error: %v", err)
|
|
}
|
|
if got := ts.callCount.Load(); got != 2 {
|
|
t.Errorf("after third call (different node): TG count = %d; want 2", got)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: critical events are never deduplicated — each call goes through
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestCriticalEventsNotDeduped(t *testing.T) {
|
|
rdb := newTestRedis(t)
|
|
ts := newTGServer(t)
|
|
n := newNotifier(t, ts, rdb)
|
|
|
|
ctx := context.Background()
|
|
// BreakerTripped is Critical severity — must bypass dedup.
|
|
e := alert.NewEvent(alert.EventTypeBreakerTripped, "node-crit", nil)
|
|
|
|
for i := 1; i <= 3; i++ {
|
|
if err := n.Notify(ctx, e); err != nil {
|
|
t.Fatalf("call %d: Notify() error: %v", i, err)
|
|
}
|
|
if got := ts.callCount.Load(); int(got) != i {
|
|
t.Errorf("after call %d: TG count = %d; want %d (no dedup for critical)", i, got, i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: TG 5xx triggers retries; after exhausting retries falls back to log;
|
|
// Notify() never returns an error.
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestTG5xxRetriesThenFallsBack(t *testing.T) {
|
|
rdb := newTestRedis(t)
|
|
ts := newTGServer(t)
|
|
|
|
// Capture fallback calls.
|
|
var fallbackCalls atomic.Int32
|
|
fallback := &capturingNotifier{
|
|
fn: func(alert.Event) { fallbackCalls.Add(1) },
|
|
}
|
|
|
|
n := alert.NewTGNotifier(alert.TGConfig{
|
|
BotToken: "test-token",
|
|
ChatID: "-1001234567",
|
|
BaseURL: ts.srv.URL,
|
|
}, rdb, fallback)
|
|
|
|
// Configure mock TG server to return 500.
|
|
ts.statusCode.Store(http.StatusInternalServerError)
|
|
|
|
ctx := context.Background()
|
|
e := alert.NewEvent(alert.EventTypeFault, "node-fail", map[string]string{"reason": "both fail"})
|
|
|
|
// Notify must return nil (not block the scheduler).
|
|
if err := n.Notify(ctx, e); err != nil {
|
|
t.Fatalf("Notify() must not return error; got %v", err)
|
|
}
|
|
|
|
// TG must have been called 3 times (1 initial + 2 retries = maxRetries+1).
|
|
if got := ts.callCount.Load(); got != 3 {
|
|
t.Errorf("TG call count = %d; want 3 (1 + 2 retries)", got)
|
|
}
|
|
|
|
// Fallback must have been called exactly once.
|
|
if got := fallbackCalls.Load(); got != 1 {
|
|
t.Errorf("fallback call count = %d; want 1", got)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: LogNotifier always succeeds (used in offline/dev mode)
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestLogNotifier(t *testing.T) {
|
|
n := alert.LogNotifier{}
|
|
ctx := context.Background()
|
|
|
|
for _, et := range []alert.EventType{
|
|
alert.EventTypeBlockConfirmed,
|
|
alert.EventTypeReplenishFailed,
|
|
alert.EventTypeWatermarkLow,
|
|
alert.EventTypeBreakerTripped,
|
|
alert.EventTypeProbeAgentLost,
|
|
alert.EventTypeHeartbeatMissing,
|
|
alert.EventTypeFault,
|
|
} {
|
|
e := alert.NewEvent(et, "node-log", nil)
|
|
if err := n.Notify(ctx, e); err != nil {
|
|
t.Errorf("LogNotifier.Notify(%s) error: %v", et, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: RunbookAnchor override takes precedence over type default
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestRunbookAnchorOverride(t *testing.T) {
|
|
rdb := newTestRedis(t)
|
|
ts := newTGServer(t)
|
|
n := newNotifier(t, ts, rdb)
|
|
|
|
e := alert.NewEvent(alert.EventTypeFault, "node-anch", nil)
|
|
e.RunbookAnchor = "#custom-anchor"
|
|
|
|
if err := n.Notify(context.Background(), e); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
body := ts.parsedBody(t)
|
|
if !strings.Contains(body["text"], "#custom-anchor") {
|
|
t.Errorf("expected #custom-anchor in message:\n%s", body["text"])
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Helpers
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
// capturingNotifier is a Notifier that calls fn on each Notify.
|
|
type capturingNotifier struct {
|
|
fn func(alert.Event)
|
|
}
|
|
|
|
func (c *capturingNotifier) Notify(_ context.Context, e alert.Event) error {
|
|
c.fn(e)
|
|
return nil
|
|
}
|