feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]
新增 server/internal/alert 包(15G): - 定义 Notifier 接口及 7 种 EventType(判封确认/补新失败/水位低/熔断/ 探针失联/心跳缺失/故障态) - TGNotifier:Bot API 发送,Critical 事件不去重,Warning/Info 事件 10min SETNX 去重窗口,失败重试 ≤2 次后降级至 LogNotifier - LogNotifier:slog 结构化降级实现 - 单测:7 种事件模板 + runbook 锚点正确性;去重窗口内第二条被抑制; TG 5xx 重试后 fallback 且 Notify() 返回 nil; runbook 文件锚点与枚举一致性 接入 scheduler(替换旧的 NotifyFault 桩): - detect/engine.go:故障态(Rule 5)→ EventTypeFault; 判封确认(Rule 3)→ EventTypeBlockConfirmed - orchestrate/deps.go:Notifier 类型别名指向 alert.Notifier - orchestrate/replacer.go:补新失败 → EventTypeReplenishFailed; 熔断触发 → EventTypeBreakerTripped - probe/prober_agent.go:failCount ≥3 → EventTypeProbeAgentLost - probe/store.go:新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s 新增 docs/runbook-scheduler.md:7 节各含含义/先查什么/处置/升级条件, 锚点与代码枚举对应。 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -34,9 +34,12 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/alert"
|
||||
)
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
@@ -113,6 +116,11 @@ var aliyunISP = []struct {
|
||||
// Overridable in tests via AliyunSyntheticAgent.baseURL.
|
||||
const aliyunEndpoint = "https://cloudmonitor.cn-hangzhou.aliyuncs.com/"
|
||||
|
||||
// probeAgentLostThreshold is the number of consecutive per-ISP API failures
|
||||
// that must accumulate before an EventTypeProbeAgentLost alert is emitted.
|
||||
// This mirrors the "连续失败≥3" policy documented in the package comments.
|
||||
const probeAgentLostThreshold = 3
|
||||
|
||||
// AliyunSyntheticAgentConfig holds configuration for AliyunSyntheticAgent.
|
||||
// The AccessKeyID and AccessKeySecret must belong to a RAM sub-account with
|
||||
// minimal permissions (cloudmonitor:CreateSiteMonitor +
|
||||
@@ -158,6 +166,16 @@ type AliyunSyntheticAgent struct {
|
||||
baseURL string // overridable in tests
|
||||
failCount atomic.Int64
|
||||
logger *slog.Logger
|
||||
// notifier is the 15G exit channel for EventTypeProbeAgentLost events.
|
||||
// Nil means no alerting (development / test with no TG configured).
|
||||
notifier alert.Notifier
|
||||
}
|
||||
|
||||
// SetNotifier injects the 15G alert outlet into the agent.
|
||||
// When not set, no EventTypeProbeAgentLost alerts are emitted (dev/test mode).
|
||||
// Call before RunOnce / Probe.
|
||||
func (a *AliyunSyntheticAgent) SetNotifier(n alert.Notifier) {
|
||||
a.notifier = n
|
||||
}
|
||||
|
||||
// NewAliyunSyntheticAgent creates an AliyunSyntheticAgent.
|
||||
@@ -255,6 +273,9 @@ func (a *AliyunSyntheticAgent) RunOnce(ctx context.Context, targets []ProbeTarge
|
||||
//
|
||||
// On any API or polling error the ISP vantage is skipped (no result returned,
|
||||
// no Redis write) per the degradation contract.
|
||||
//
|
||||
// When consecutive per-ISP API failures reach probeAgentLostThreshold the
|
||||
// 探针失联 (EventTypeProbeAgentLost) alert is emitted via the injected Notifier.
|
||||
func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([]VantageResult, error) {
|
||||
var out []VantageResult
|
||||
for _, isp := range aliyunISP {
|
||||
@@ -264,6 +285,17 @@ func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([
|
||||
a.logger.Warn("prober_agent: ISP probe failed (degraded, no data written)",
|
||||
"node", target.NodeID, "isp", isp.name, "error", err,
|
||||
"consecutive_failures", cnt)
|
||||
// Emit 探针失联 alert when threshold is crossed (15G exit channel).
|
||||
if cnt >= probeAgentLostThreshold && a.notifier != nil {
|
||||
ev := alert.NewEvent(alert.EventTypeProbeAgentLost, "aliyun-synthetic", map[string]string{
|
||||
"consecutive_failures": strconv.FormatInt(cnt, 10),
|
||||
"last_isp": isp.name,
|
||||
"last_node": target.NodeID,
|
||||
})
|
||||
if notifyErr := a.notifier.Notify(ctx, ev); notifyErr != nil {
|
||||
a.logger.Warn("prober_agent: notify probe agent lost", "error", notifyErr)
|
||||
}
|
||||
}
|
||||
// Degradation: skip this vantage this cycle.
|
||||
continue
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user