feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -8,9 +8,9 @@ package orchestrate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
|
||||
@@ -164,25 +164,17 @@ func (StubBreaker) Allow(_, _ string) bool { return true }
|
||||
func (StubBreaker) Record(_, _ string) {}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Notifier (15G stub)
|
||||
// Notifier (15G)
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Notifier is the 15G alerting interface.
|
||||
type Notifier interface {
|
||||
NotifyFault(ctx context.Context, nodeID, reason string) error
|
||||
}
|
||||
// Notifier is the 15G unified alert outlet used by the orchestrator.
|
||||
// It is satisfied by alert.Notifier (TG implementation) and alert.LogNotifier
|
||||
// (fallback / development stub).
|
||||
type Notifier = alert.Notifier
|
||||
|
||||
// LogNotifier logs faults via slog. Used when no real notifier is wired.
|
||||
type LogNotifier struct{}
|
||||
|
||||
// NotifyFault implements Notifier.
|
||||
func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
|
||||
slog.Warn("orchestrate: replacement failed — manual review required",
|
||||
"node_id", nodeID,
|
||||
"reason", reason,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
// LogNotifier is re-exported for callers that need a no-op Notifier without
|
||||
// importing the alert package directly.
|
||||
type LogNotifier = alert.LogNotifier
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Clock (for testability)
|
||||
|
||||
Reference in New Issue
Block a user