feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包（15G）： - 定义 Notifier 接口及 7 种 EventType（判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态） - TGNotifier：Bot API 发送，Critical 事件不去重，Warning/Info 事件 10min SETNX 去重窗口，失败重试 ≤2 次后降级至 LogNotifier - LogNotifier：slog 结构化降级实现 - 单测：7 种事件模板 + runbook 锚点正确性；去重窗口内第二条被抑制； TG 5xx 重试后 fallback 且 Notify() 返回 nil； runbook 文件锚点与枚举一致性 接入 scheduler（替换旧的 NotifyFault 桩）： - detect/engine.go：故障态（Rule 5）→ EventTypeFault； 判封确认（Rule 3）→ EventTypeBlockConfirmed - orchestrate/deps.go：Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go：补新失败 → EventTypeReplenishFailed； 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go：failCount ≥3 → EventTypeProbeAgentLost - probe/store.go：新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md：7 节各含含义/先查什么/处置/升级条件， 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -8,9 +8,9 @@ package orchestrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
|
||||
@@ -164,25 +164,17 @@ func (StubBreaker) Allow(_, _ string) bool { return true }
|
||||
func (StubBreaker) Record(_, _ string) {}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Notifier (15G stub)
|
||||
// Notifier (15G)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Notifier is the 15G alerting interface.
|
||||
type Notifier interface {
|
||||
NotifyFault(ctx context.Context, nodeID, reason string) error
|
||||
}
|
||||
// Notifier is the 15G unified alert outlet used by the orchestrator.
|
||||
// It is satisfied by alert.Notifier (TG implementation) and alert.LogNotifier
|
||||
// (fallback / development stub).
|
||||
type Notifier = alert.Notifier
|
||||
|
||||
// LogNotifier logs faults via slog. Used when no real notifier is wired.
|
||||
type LogNotifier struct{}
|
||||
|
||||
// NotifyFault implements Notifier.
|
||||
func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
|
||||
slog.Warn("orchestrate: replacement failed — manual review required",
|
||||
"node_id", nodeID,
|
||||
"reason", reason,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
// LogNotifier is re-exported for callers that need a no-op Notifier without
|
||||
// importing the alert package directly.
|
||||
type LogNotifier = alert.LogNotifier
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Clock (for testability)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
|
||||
@@ -95,7 +96,7 @@ func NewReplacer(cfg Config) *Replacer {
|
||||
cfg.Breaker = StubBreaker{}
|
||||
}
|
||||
if cfg.Notifier == nil {
|
||||
cfg.Notifier = LogNotifier{}
|
||||
cfg.Notifier = alert.LogNotifier{}
|
||||
}
|
||||
if cfg.Clock == nil {
|
||||
cfg.Clock = RealClock{}
|
||||
@@ -237,11 +238,21 @@ func (r *Replacer) stepPending(ctx context.Context, uuid string, rec *ReplaceRec
|
||||
if !r.breaker.Allow(nodeInfo.Tier, nodeInfo.Region) {
|
||||
slog.Info("orchestrate: breaker blocked replacement",
|
||||
"uuid", uuid, "tier", nodeInfo.Tier, "region", nodeInfo.Region)
|
||||
// Emit 熔断触发 alert (15G exit channel).
|
||||
ev := alert.NewEvent(alert.EventTypeBreakerTripped, rec.OldNode, map[string]string{
|
||||
"tier": nodeInfo.Tier,
|
||||
"region": nodeInfo.Region,
|
||||
"replacement_uuid": uuid,
|
||||
})
|
||||
ev.Pool = nodeInfo.Tier + "/" + nodeInfo.Region
|
||||
if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
slog.Error("orchestrate: notify breaker tripped", "uuid", uuid, "error", notifyErr)
|
||||
}
|
||||
return nil // stay pending; retry next Tick
|
||||
}
|
||||
|
||||
// Watermark / quota check — stub (always passes).
|
||||
// TODO(15F): implement real capacity-quota guard here.
|
||||
// TODO(15F): emit EventTypeWatermarkLow when real capacity guard is wired.
|
||||
|
||||
rec.Phase = PhaseCreating
|
||||
rec.PhaseStartedAt = r.clock.Now()
|
||||
@@ -371,9 +382,14 @@ func (r *Replacer) failProbeAttempt(ctx context.Context, uuid string, rec *Repla
|
||||
_ = r.rdb.SRem(ctx, replaceIndexKey, uuid).Err()
|
||||
_ = r.rdb.Expire(ctx, replaceKeyPrefix+uuid, replaceTTL).Err()
|
||||
|
||||
alertReason := fmt.Sprintf("probing failed after %d attempts: %s", rec.Attempts, reason)
|
||||
if notifyErr := r.notifier.NotifyFault(ctx, rec.OldNode, alertReason); notifyErr != nil {
|
||||
slog.Error("orchestrate: notify fault", "uuid", uuid, "error", notifyErr)
|
||||
// Emit 补新连续失败≥3 alert (15G exit channel).
|
||||
ev := alert.NewEvent(alert.EventTypeReplenishFailed, rec.OldNode, map[string]string{
|
||||
"attempts": fmt.Sprintf("%d", rec.Attempts),
|
||||
"last_reason": reason,
|
||||
"replacement_uuid": uuid,
|
||||
})
|
||||
if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
slog.Error("orchestrate: notify replenish failed", "uuid", uuid, "error", notifyErr)
|
||||
}
|
||||
slog.Error("orchestrate: replacement permanently failed — manual review required",
|
||||
"uuid", uuid, "old_node", rec.OldNode, "attempts", rec.Attempts)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
@@ -267,21 +268,35 @@ func passingSnapshots() map[string]probe.ProbeSnapshot {
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
type mockNotifier struct {
|
||||
mu sync.Mutex
|
||||
calls []string
|
||||
mu sync.Mutex
|
||||
events []alert.Event
|
||||
}
|
||||
|
||||
func (n *mockNotifier) NotifyFault(_ context.Context, nodeID, _ string) error {
|
||||
func (n *mockNotifier) Notify(_ context.Context, e alert.Event) error {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
n.calls = append(n.calls, nodeID)
|
||||
n.events = append(n.events, e)
|
||||
return nil
|
||||
}
|
||||
|
||||
// count returns the number of Notify calls received.
|
||||
func (n *mockNotifier) count() int {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
return len(n.calls)
|
||||
return len(n.events)
|
||||
}
|
||||
|
||||
// countByType returns the number of Notify calls with the given EventType.
|
||||
func (n *mockNotifier) countByType(t alert.EventType) int {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
c := 0
|
||||
for _, e := range n.events {
|
||||
if e.Type == t {
|
||||
c++
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -557,9 +572,9 @@ func TestProbeFailMaxAttempts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// NotifyFault must be called exactly once.
|
||||
if n := h.notifier.count(); n != 1 {
|
||||
t.Errorf("NotifyFault calls = %d; want 1", n)
|
||||
// Notify(EventTypeReplenishFailed) must be called exactly once.
|
||||
if n := h.notifier.countByType(alert.EventTypeReplenishFailed); n != 1 {
|
||||
t.Errorf("Notify(ReplenishFailed) calls = %d; want 1", n)
|
||||
}
|
||||
|
||||
// Record must be in failed phase.
|
||||
|
||||
Reference in New Issue
Block a user