feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]

新增 server/internal/alert 包（15G）：
- 定义 Notifier 接口及 7 种 EventType（判封确认/补新失败/水位低/熔断/
  探针失联/心跳缺失/故障态）
- TGNotifier：Bot API 发送，Critical 事件不去重，Warning/Info 事件
  10min SETNX 去重窗口，失败重试 ≤2 次后降级至 LogNotifier
- LogNotifier：slog 结构化降级实现
- 单测：7 种事件模板 + runbook 锚点正确性；去重窗口内第二条被抑制；
  TG 5xx 重试后 fallback 且 Notify() 返回 nil；
  runbook 文件锚点与枚举一致性

接入 scheduler（替换旧的 NotifyFault 桩）：
- detect/engine.go：故障态（Rule 5）→ EventTypeFault；
  判封确认（Rule 3）→ EventTypeBlockConfirmed
- orchestrate/deps.go：Notifier 类型别名指向 alert.Notifier
- orchestrate/replacer.go：补新失败 → EventTypeReplenishFailed；
  熔断触发 → EventTypeBreakerTripped
- probe/prober_agent.go：failCount ≥3 → EventTypeProbeAgentLost
- probe/store.go：新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s

新增 docs/runbook-scheduler.md：7 节各含含义/先查什么/处置/升级条件，
锚点与代码枚举对应。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

server/internal/scheduler/probe/prober_agent.go

@@ -34,9 +34,12 @@ import (
 	"log/slog"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
+
+	"github.com/wangjia/pangolin/server/internal/alert"
 )
 
 // --------------------------------------------------------------------------
@@ -113,6 +116,11 @@ var aliyunISP = []struct {
 // Overridable in tests via AliyunSyntheticAgent.baseURL.
 const aliyunEndpoint = "https://cloudmonitor.cn-hangzhou.aliyuncs.com/"
 
+// probeAgentLostThreshold is the number of consecutive per-ISP API failures
+// that must accumulate before an EventTypeProbeAgentLost alert is emitted.
+// This mirrors the "连续失败≥3" policy documented in the package comments.
+const probeAgentLostThreshold = 3
+
 // AliyunSyntheticAgentConfig holds configuration for AliyunSyntheticAgent.
 // The AccessKeyID and AccessKeySecret must belong to a RAM sub-account with
 // minimal permissions (cloudmonitor:CreateSiteMonitor +
@@ -158,6 +166,16 @@ type AliyunSyntheticAgent struct {
 	baseURL    string // overridable in tests
 	failCount  atomic.Int64
 	logger     *slog.Logger
+	// notifier is the 15G exit channel for EventTypeProbeAgentLost events.
+	// Nil means no alerting (development / test with no TG configured).
+	notifier alert.Notifier
+}
+
+// SetNotifier injects the 15G alert outlet into the agent.
+// When not set, no EventTypeProbeAgentLost alerts are emitted (dev/test mode).
+// Call before RunOnce / Probe.
+func (a *AliyunSyntheticAgent) SetNotifier(n alert.Notifier) {
+	a.notifier = n
 }
 
 // NewAliyunSyntheticAgent creates an AliyunSyntheticAgent.
@@ -255,6 +273,9 @@ func (a *AliyunSyntheticAgent) RunOnce(ctx context.Context, targets []ProbeTarge
 //
 // On any API or polling error the ISP vantage is skipped (no result returned,
 // no Redis write) per the degradation contract.
+//
+// When consecutive per-ISP API failures reach probeAgentLostThreshold the
+// 探针失联 (EventTypeProbeAgentLost) alert is emitted via the injected Notifier.
 func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([]VantageResult, error) {
 	var out []VantageResult
 	for _, isp := range aliyunISP {
@@ -264,6 +285,17 @@ func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([
 			a.logger.Warn("prober_agent: ISP probe failed (degraded, no data written)",
 				"node", target.NodeID, "isp", isp.name, "error", err,
 				"consecutive_failures", cnt)
+			// Emit 探针失联 alert when threshold is crossed (15G exit channel).
+			if cnt >= probeAgentLostThreshold && a.notifier != nil {
+				ev := alert.NewEvent(alert.EventTypeProbeAgentLost, "aliyun-synthetic", map[string]string{
+					"consecutive_failures": strconv.FormatInt(cnt, 10),
+					"last_isp":             isp.name,
+					"last_node":            target.NodeID,
+				})
+				if notifyErr := a.notifier.Notify(ctx, ev); notifyErr != nil {
+					a.logger.Warn("prober_agent: notify probe agent lost", "error", notifyErr)
+				}
+			}
 			// Degradation: skip this vantage this cycle.
 			continue
 		}

server/internal/scheduler/probe/store.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/redis/go-redis/v9"
+
+	"github.com/wangjia/pangolin/server/internal/alert"
 )
 
 // Redis TTL constants for the probe subsystem.
@@ -210,3 +212,50 @@ func (s *Store) AliveProbes(ctx context.Context) ([]string, error) {
 	}
 	return ids, nil
 }
+
+// CheckHeartbeats scans all known probe heartbeat keys and fires an
+// EventTypeHeartbeatMissing alert via notifier for every probe whose last
+// heartbeat timestamp is older than threshold.
+//
+// This is called by 15H (DetectLoop / assembly) on a periodic basis.
+// Missing-key semantics apply: a key that expired (TTL elapsed) is not seen
+// at all — only keys that exist but carry a stale timestamp are reported.
+//
+// threshold should be ≥90 s per the operational SLO.
+func (s *Store) CheckHeartbeats(ctx context.Context, threshold time.Duration, notifier alert.Notifier) error {
+	now := time.Now().Unix()
+	cutoff := now - int64(threshold.Seconds())
+
+	var scanErr error
+	iter := s.rdb.Scan(ctx, 0, "probe:hb:*", 0).Iterator()
+	for iter.Next(ctx) {
+		k := iter.Val()
+		probeID := strings.TrimPrefix(k, "probe:hb:")
+
+		val, err := s.rdb.Get(ctx, k).Result()
+		if err != nil {
+			// Key may have expired between SCAN and GET; skip.
+			continue
+		}
+		ts, err := strconv.ParseInt(val, 10, 64)
+		if err != nil {
+			continue // corrupt value; ignore
+		}
+		if ts < cutoff {
+			// Heartbeat is stale: emit alert.
+			staleSecs := now - ts
+			ev := alert.NewEvent(alert.EventTypeHeartbeatMissing, probeID, map[string]string{
+				"stale_seconds": strconv.FormatInt(staleSecs, 10),
+				"threshold_s":   strconv.FormatInt(int64(threshold.Seconds()), 10),
+			})
+			if notifyErr := notifier.Notify(ctx, ev); notifyErr != nil {
+				// Log but continue checking other probes.
+				scanErr = fmt.Errorf("probe: notify heartbeat missing for %s: %w", probeID, notifyErr)
+			}
+		}
+	}
+	if err := iter.Err(); err != nil {
+		return fmt.Errorf("probe: scan heartbeats: %w", err)
+	}
+	return scanErr
+}