feat(alert): 统一告警出口 TG bot + runbook [tsk_9YMHMTfWJyNB]

新增 server/internal/alert 包（15G）：
- 定义 Notifier 接口及 7 种 EventType（判封确认/补新失败/水位低/熔断/
  探针失联/心跳缺失/故障态）
- TGNotifier：Bot API 发送，Critical 事件不去重，Warning/Info 事件
  10min SETNX 去重窗口，失败重试 ≤2 次后降级至 LogNotifier
- LogNotifier：slog 结构化降级实现
- 单测：7 种事件模板 + runbook 锚点正确性；去重窗口内第二条被抑制；
  TG 5xx 重试后 fallback 且 Notify() 返回 nil；
  runbook 文件锚点与枚举一致性

接入 scheduler（替换旧的 NotifyFault 桩）：
- detect/engine.go：故障态（Rule 5）→ EventTypeFault；
  判封确认（Rule 3）→ EventTypeBlockConfirmed
- orchestrate/deps.go：Notifier 类型别名指向 alert.Notifier
- orchestrate/replacer.go：补新失败 → EventTypeReplenishFailed；
  熔断触发 → EventTypeBreakerTripped
- probe/prober_agent.go：failCount ≥3 → EventTypeProbeAgentLost
- probe/store.go：新增 CheckHeartbeats() 供 15H 检测心跳缺失>90s

新增 docs/runbook-scheduler.md：7 节各含含义/先查什么/处置/升级条件，
锚点与代码枚举对应。

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

server/internal/alert/notifier.go

@@ -0,0 +1,430 @@
+// Package alert implements the unified alert exit channel for Pangolin's
+// scheduler subsystem (task 15G).
+//
+// # Event types
+//
+// Seven fixed event types cover all failure modes detected by 15D (detect),
+// 15E (orchestrate), and 15F (probe):
+//
+//   EventTypeBlockConfirmed   — node confirmed GFW-censored after ≥6 cycles
+//   EventTypeReplenishFailed  — replacement provisioning exhausted all retries
+//   EventTypeWatermarkLow     — pool node-weight drops below 70 % capacity
+//   EventTypeBreakerTripped   — circuit breaker blocked a replacement attempt
+//   EventTypeProbeAgentLost   — third-party synthetic probe failing ≥3 times
+//   EventTypeHeartbeatMissing — first-party probe silent for >90 s
+//   EventTypeFault            — node-level outage (domestic + overseas both fail)
+//
+// # Deduplication
+//
+// Non-critical events are throttled: the same (Type, NodeID) pair fires at
+// most once per 10-minute window (Redis SETNX + TTL).  Critical events always
+// fire — they are never suppressed.
+//
+// # Alert channel safety
+//
+// Notify must never return a blocking error.  TG API failures are retried up
+// to maxRetries times, then fall back to the slog-based LogNotifier.  The
+// calling goroutine (scheduler tick) is never stalled by alert-channel faults.
+//
+// # Identity isolation
+//
+// The TG bot token and chat ID are injected via environment variables only.
+// The bot belongs to a dedicated anonymous operator account; see
+// infra/identity-isolation.md and red line 06 §2.
+//
+// # Privacy
+//
+// Event messages use internal node IDs (opaque), never public domain names.
+// This prevents domain leakage if the alert message is forwarded.
+package alert
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Event types
+// ─────────────────────────────────────────────────────────────────────────────
+
+// EventType identifies the class of alert event.
+type EventType string
+
+const (
+	// EventTypeBlockConfirmed fires when detect (15D) transitions a node from
+	// blocked_suspect to blocked_confirmed, confirming GFW censorship after
+	// ≥6 consecutive failing probe cycles.
+	// Runbook anchor: #block-confirmed
+	EventTypeBlockConfirmed EventType = "block_confirmed"
+
+	// EventTypeReplenishFailed fires when orchestrate (15E) exhausts all
+	// MaxAttempts replacement attempts for a blocked node.
+	// Runbook anchor: #replenish-failed
+	EventTypeReplenishFailed EventType = "replenish_failed"
+
+	// EventTypeWatermarkLow fires when the effective capacity of a node pool
+	// (combined routing weight of live nodes) drops below 70 % of full capacity.
+	// Runbook anchor: #watermark-low
+	EventTypeWatermarkLow EventType = "watermark_low"
+
+	// EventTypeBreakerTripped fires when the circuit breaker (15F) blocks a
+	// replacement attempt due to a burst of confirmed blocks in the same pool.
+	// Runbook anchor: #breaker-tripped
+	EventTypeBreakerTripped EventType = "breaker_tripped"
+
+	// EventTypeProbeAgentLost fires when the third-party (Aliyun) synthetic
+	// probe agent encounters ≥3 consecutive API failures.
+	// Runbook anchor: #probe-agent-lost
+	EventTypeProbeAgentLost EventType = "probe_agent_lost"
+
+	// EventTypeHeartbeatMissing fires when a first-party probe agent has not
+	// delivered any heartbeat for more than 90 seconds.
+	// Runbook anchor: #heartbeat-missing
+	EventTypeHeartbeatMissing EventType = "heartbeat_missing"
+
+	// EventTypeFault fires when both domestic AND overseas probes are failing
+	// for a node, indicating a node-level outage rather than GFW censorship.
+	// Runbook anchor: #node-fault
+	EventTypeFault EventType = "fault"
+)
+
+// Severity indicates the urgency of an event.
+type Severity string
+
+const (
+	SeverityCritical Severity = "critical" // requires immediate human action
+	SeverityWarning  Severity = "warning"  // review within the hour
+	SeverityInfo     Severity = "info"     // informational only
+)
+
+// typeMeta bundles display properties for one event type.
+type typeMeta struct {
+	Severity      Severity
+	TitleZH       string // human-readable Chinese label
+	Emoji         string // prefixed to the TG message
+	RunbookAnchor string // HTML id anchor in docs/runbook-scheduler.md
+}
+
+// typeMetaMap is indexed by EventType.
+var typeMetaMap = map[EventType]typeMeta{
+	EventTypeBlockConfirmed:   {SeverityWarning, "判封确认", "⚠️", "#block-confirmed"},
+	EventTypeReplenishFailed:  {SeverityCritical, "补新连续失败≥3", "🔴", "#replenish-failed"},
+	EventTypeWatermarkLow:     {SeverityWarning, "水位<70%", "⚠️", "#watermark-low"},
+	EventTypeBreakerTripped:   {SeverityCritical, "熔断触发", "🔴", "#breaker-tripped"},
+	EventTypeProbeAgentLost:   {SeverityWarning, "探针失联", "⚠️", "#probe-agent-lost"},
+	EventTypeHeartbeatMissing: {SeverityWarning, "心跳缺失>90s", "⚠️", "#heartbeat-missing"},
+	EventTypeFault:            {SeverityCritical, "故障态", "🔴", "#node-fault"},
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Event
+// ─────────────────────────────────────────────────────────────────────────────
+
+// Event carries all data required to render one alert notification.
+type Event struct {
+	// Type is one of the seven EventType constants.
+	Type EventType
+
+	// Severity controls dedup and display urgency.
+	// Derived from Type via NewEvent; may be overridden by the caller.
+	Severity Severity
+
+	// NodeID is the internal opaque node or probe agent identifier.
+	// MUST be an internal ID — never a public domain name (privacy).
+	NodeID string
+
+	// Pool optionally identifies the operational pool (e.g. "free/hkg").
+	Pool string
+
+	// Detail holds supplementary key-value data for the notification.
+	// Values must not contain personal user data.
+	Detail map[string]string
+
+	// RunbookAnchor is the #fragment anchor in docs/runbook-scheduler.md.
+	// Overrides the type default when non-empty.
+	RunbookAnchor string
+}
+
+// NewEvent constructs an Event with type-derived Severity and RunbookAnchor.
+// Callers may override any field after construction.
+func NewEvent(t EventType, nodeID string, detail map[string]string) Event {
+	m := typeMetaMap[t]
+	return Event{
+		Type:          t,
+		Severity:      m.Severity,
+		NodeID:        nodeID,
+		Detail:        detail,
+		RunbookAnchor: m.RunbookAnchor,
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Notifier interface
+// ─────────────────────────────────────────────────────────────────────────────
+
+// Notifier is the unified alert outlet for scheduler events.
+// Implementations must be safe for concurrent use and must never return a
+// blocking error — alert-channel failures must not halt the scheduler.
+type Notifier interface {
+	Notify(ctx context.Context, event Event) error
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// LogNotifier
+// ─────────────────────────────────────────────────────────────────────────────
+
+// LogNotifier writes every event to the structured logger.
+// It is used when TG_BOT_TOKEN is not configured, or as the fallback when the
+// TGNotifier exhausts its retries.
+type LogNotifier struct{}
+
+// Notify implements Notifier.
+func (LogNotifier) Notify(_ context.Context, e Event) error {
+	m := typeMetaMap[e.Type]
+	slog.Warn("alert: scheduler event",
+		"type", string(e.Type),
+		"title", m.TitleZH,
+		"severity", string(e.Severity),
+		"node_id", e.NodeID,
+		"pool", e.Pool,
+		"detail", e.Detail,
+		"runbook", e.RunbookAnchor,
+	)
+	return nil
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// TGNotifier
+// ─────────────────────────────────────────────────────────────────────────────
+
+const (
+	// dedupTTL is the dedup suppression window for non-critical events.
+	dedupTTL = 10 * time.Minute
+
+	// dedupKeyPrefix is the Redis key namespace for dedup tokens.
+	// Full key: alert:dedup:{type}:{nodeID}
+	dedupKeyPrefix = "alert:dedup:"
+
+	// maxRetries is the number of additional TG send attempts after the first
+	// failure.  Exhausting retries falls back to LogNotifier silently.
+	maxRetries = 2
+
+	// tgSendMessageURL is the Telegram Bot API pattern for sendMessage.
+	tgSendMessageURL = "https://api.telegram.org/bot%s/sendMessage"
+)
+
+// TGConfig holds the runtime configuration for TGNotifier.
+// All sensitive values come from environment variables; see cmd/ bootstrap.
+type TGConfig struct {
+	// BotToken is the Telegram bot token (env: TG_BOT_TOKEN).
+	// The bot must belong to a dedicated anonymous operator account (red line §2).
+	BotToken string
+
+	// ChatID is the target Telegram group or channel ID (env: TG_ALERT_CHAT_ID).
+	ChatID string
+
+	// RunbookBaseURL is prepended to the anchor to form the full runbook URL.
+	// Leave empty to embed only the #anchor fragment.
+	RunbookBaseURL string
+
+	// HTTPTimeout overrides the per-call HTTP timeout (default 10 s).
+	HTTPTimeout time.Duration
+
+	// BaseURL overrides the TG API base URL for testing.
+	// Leave empty in production.
+	BaseURL string
+}
+
+// TGNotifier sends events to a Telegram group via the Bot API.
+//
+// Deduplication:
+//   - Non-critical events: one message per (Type, NodeID) per 10-minute window
+//     (Redis SETNX + TTL).  The second identical message within the window is
+//     silently dropped.
+//   - Critical events: always delivered, no dedup.
+//
+// Retry + fallback:
+//   - On TG API failure, retries up to maxRetries (2) times before delegating
+//     to the fallback Notifier.  Notify always returns nil.
+type TGNotifier struct {
+	cfg        TGConfig
+	rdb        *redis.Client
+	httpClient *http.Client
+	fallback   Notifier
+	apiURL     string // base send URL, pre-formatted with token
+}
+
+// NewTGNotifier constructs a TGNotifier.
+// rdb is used for the dedup SETNX gate.
+// fallback is used when TG API calls are exhausted; nil defaults to LogNotifier.
+func NewTGNotifier(cfg TGConfig, rdb *redis.Client, fallback Notifier) *TGNotifier {
+	if fallback == nil {
+		fallback = LogNotifier{}
+	}
+	httpTimeout := 10 * time.Second
+	if cfg.HTTPTimeout > 0 {
+		httpTimeout = cfg.HTTPTimeout
+	}
+	baseURL := fmt.Sprintf(tgSendMessageURL, cfg.BotToken)
+	if cfg.BaseURL != "" {
+		// Test hook: override the entire API base URL (token appended separately).
+		baseURL = cfg.BaseURL
+	}
+	return &TGNotifier{
+		cfg:        cfg,
+		rdb:        rdb,
+		httpClient: &http.Client{Timeout: httpTimeout},
+		fallback:   fallback,
+		apiURL:     baseURL,
+	}
+}
+
+// Notify implements Notifier.
+func (n *TGNotifier) Notify(ctx context.Context, e Event) error {
+	// Critical events bypass the dedup gate entirely.
+	if e.Severity != SeverityCritical {
+		allowed, err := n.dedupGate(ctx, e)
+		if err != nil {
+			// Redis error → allow through (fail-open) to avoid losing alerts.
+			slog.Warn("alert: dedup Redis error; allowing through", "error", err)
+		}
+		if !allowed {
+			return nil // duplicate within the 10-min window; silently dropped
+		}
+	}
+
+	text := n.renderMessage(e)
+
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		if err := n.callTG(ctx, text); err != nil {
+			lastErr = err
+			slog.Warn("alert: TG send failed",
+				"attempt", attempt+1, "of", maxRetries+1,
+				"type", string(e.Type), "node_id", e.NodeID,
+				"error", err,
+			)
+			continue
+		}
+		return nil // delivered
+	}
+
+	// Retries exhausted: degrade to log.  Never return an error.
+	slog.Error("alert: TG delivery exhausted retries; falling back to log",
+		"type", string(e.Type), "node_id", e.NodeID, "last_error", lastErr)
+	_ = n.fallback.Notify(ctx, e)
+	return nil
+}
+
+// dedupGate returns (true, nil) if the event should be sent (key newly set),
+// or (false, nil) if it is a duplicate within the 10-min window.
+func (n *TGNotifier) dedupGate(ctx context.Context, e Event) (bool, error) {
+	key := dedupKeyPrefix + string(e.Type) + ":" + e.NodeID
+	set, err := n.rdb.SetNX(ctx, key, "1", dedupTTL).Result()
+	if err != nil {
+		return true, fmt.Errorf("alert: dedup SETNX: %w", err)
+	}
+	return set, nil
+}
+
+// renderMessage builds the Telegram HTML message body.
+// Node IDs are used as-is (internal opaque IDs, not domain names).
+func (n *TGNotifier) renderMessage(e Event) string {
+	m := typeMetaMap[e.Type]
+
+	var sb strings.Builder
+
+	// Line 1: emoji + bold event title + node id
+	sb.WriteString(m.Emoji)
+	sb.WriteString(" <b>")
+	sb.WriteString(htmlEscape(m.TitleZH))
+	sb.WriteString("</b> — 节点 <code>")
+	sb.WriteString(htmlEscape(e.NodeID))
+	sb.WriteString("</code>")
+	if e.Pool != "" {
+		sb.WriteString(" 池 <code>")
+		sb.WriteString(htmlEscape(e.Pool))
+		sb.WriteString("</code>")
+	}
+	sb.WriteString("\n")
+
+	// Line 2: severity label
+	sb.WriteString("严重度: ")
+	sb.WriteString(string(e.Severity))
+	sb.WriteString("\n")
+
+	// Detail pairs (sorted-ish; map iteration is random, determinism not required)
+	for k, v := range e.Detail {
+		sb.WriteString(htmlEscape(k))
+		sb.WriteString(": ")
+		sb.WriteString(htmlEscape(v))
+		sb.WriteString("\n")
+	}
+
+	// Runbook link
+	anchor := e.RunbookAnchor
+	if anchor == "" {
+		anchor = m.RunbookAnchor
+	}
+	link := anchor
+	if n.cfg.RunbookBaseURL != "" {
+		link = n.cfg.RunbookBaseURL + anchor
+	}
+	sb.WriteString("📖 <a href=\"")
+	sb.WriteString(link)
+	sb.WriteString("\">处置手册</a>")
+
+	return sb.String()
+}
+
+// callTG posts one sendMessage request to the Telegram Bot API.
+func (n *TGNotifier) callTG(ctx context.Context, text string) error {
+	payload, err := json.Marshal(map[string]string{
+		"chat_id":              n.cfg.ChatID,
+		"text":                 text,
+		"parse_mode":           "HTML",
+		"disable_web_page_preview": "true",
+	})
+	if err != nil {
+		return fmt.Errorf("alert: marshal TG payload: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, n.apiURL, bytes.NewReader(payload))
+	if err != nil {
+		return fmt.Errorf("alert: build TG request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := n.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("alert: TG http: %w", err)
+	}
+	defer resp.Body.Close()
+	io.Copy(io.Discard, resp.Body) //nolint:errcheck // response body drained for keep-alive
+
+	if resp.StatusCode >= 500 {
+		return fmt.Errorf("alert: TG server error HTTP %d", resp.StatusCode)
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("alert: TG unexpected HTTP %d", resp.StatusCode)
+	}
+	return nil
+}
+
+// htmlEscape escapes the five HTML special characters relevant to Telegram HTML mode.
+func htmlEscape(s string) string {
+	s = strings.ReplaceAll(s, "&", "&amp;")
+	s = strings.ReplaceAll(s, "<", "&lt;")
+	s = strings.ReplaceAll(s, ">", "&gt;")
+	s = strings.ReplaceAll(s, "\"", "&quot;")
+	return s
+}

server/internal/alert/notifier_test.go

@@ -0,0 +1,308 @@
+package alert_test
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/redis/go-redis/v9"
+
+	"github.com/wangjia/pangolin/server/internal/alert"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+func newTestRedis(t *testing.T) *redis.Client {
+	t.Helper()
+	mr := miniredis.RunT(t)
+	return redis.NewClient(&redis.Options{Addr: mr.Addr()})
+}
+
+// tgServer is a tiny Telegram Bot API mock.
+type tgServer struct {
+	statusCode atomic.Int32 // HTTP status to return; default 200
+	callCount  atomic.Int32 // total calls received
+	lastBody   atomic.Value // last []byte body received
+	srv        *httptest.Server
+}
+
+func newTGServer(t *testing.T) *tgServer {
+	t.Helper()
+	ts := &tgServer{}
+	ts.statusCode.Store(http.StatusOK)
+	ts.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ts.callCount.Add(1)
+		body, _ := io.ReadAll(r.Body)
+		ts.lastBody.Store(body)
+		w.WriteHeader(int(ts.statusCode.Load()))
+		w.Write([]byte(`{"ok":true}`)) //nolint:errcheck
+	}))
+	t.Cleanup(ts.srv.Close)
+	return ts
+}
+
+// parsedBody parses the last received request body as a map.
+func (ts *tgServer) parsedBody(t *testing.T) map[string]string {
+	t.Helper()
+	raw, _ := ts.lastBody.Load().([]byte)
+	if len(raw) == 0 {
+		t.Fatal("tgServer: no body received yet")
+	}
+	var m map[string]string
+	if err := json.Unmarshal(raw, &m); err != nil {
+		t.Fatalf("tgServer: unmarshal body: %v", err)
+	}
+	return m
+}
+
+// newNotifier creates a TGNotifier pointing at the mock server.
+func newNotifier(t *testing.T, ts *tgServer, rdb *redis.Client) *alert.TGNotifier {
+	t.Helper()
+	return alert.NewTGNotifier(alert.TGConfig{
+		BotToken:       "test-token",
+		ChatID:         "-1001234567",
+		RunbookBaseURL: "https://example.internal/runbook",
+		BaseURL:        ts.srv.URL,
+	}, rdb, nil)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: all seven event types render correct templates with runbook anchors
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestAllEventTypesRendered(t *testing.T) {
+	tests := []struct {
+		eventType     alert.EventType
+		wantAnchor    string
+		wantTitlePart string // substring expected in the TG message text (HTML-encoded if needed)
+	}{
+		{alert.EventTypeBlockConfirmed, "#block-confirmed", "判封确认"},
+		{alert.EventTypeReplenishFailed, "#replenish-failed", "补新连续失败≥3"},
+		// "<" is HTML-escaped to "&lt;" in TG HTML mode — check escaped form.
+		{alert.EventTypeWatermarkLow, "#watermark-low", "水位&lt;70%"},
+		{alert.EventTypeBreakerTripped, "#breaker-tripped", "熔断触发"},
+		{alert.EventTypeProbeAgentLost, "#probe-agent-lost", "探针失联"},
+		// ">" is HTML-escaped to "&gt;" in TG HTML mode — check escaped form.
+		{alert.EventTypeHeartbeatMissing, "#heartbeat-missing", "心跳缺失&gt;90s"},
+		{alert.EventTypeFault, "#node-fault", "故障态"},
+	}
+
+	for _, tc := range tests {
+		t.Run(string(tc.eventType), func(t *testing.T) {
+			rdb := newTestRedis(t)
+			ts := newTGServer(t)
+			n := newNotifier(t, ts, rdb)
+
+			e := alert.NewEvent(tc.eventType, "node-abc", map[string]string{
+				"reason": "test reason",
+			})
+
+			ctx := context.Background()
+			if err := n.Notify(ctx, e); err != nil {
+				t.Fatalf("Notify() returned error: %v", err)
+			}
+
+			// Exactly one TG call must have been made.
+			if got := ts.callCount.Load(); got != 1 {
+				t.Fatalf("TG call count = %d; want 1", got)
+			}
+
+			body := ts.parsedBody(t)
+			text := body["text"]
+
+			// Message must contain the Chinese event title.
+			if !strings.Contains(text, tc.wantTitlePart) {
+				t.Errorf("message does not contain %q:\n%s", tc.wantTitlePart, text)
+			}
+
+			// Message must contain the runbook anchor.
+			if !strings.Contains(text, tc.wantAnchor) {
+				t.Errorf("message does not contain runbook anchor %q:\n%s", tc.wantAnchor, text)
+			}
+
+			// Message must contain the node ID (internal only — no domain).
+			if !strings.Contains(text, "node-abc") {
+				t.Errorf("message does not contain node ID:\n%s", text)
+			}
+
+			// parse_mode must be HTML.
+			if body["parse_mode"] != "HTML" {
+				t.Errorf("parse_mode = %q; want HTML", body["parse_mode"])
+			}
+		})
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: dedup — second message for same (Type, NodeID) within 10 min is suppressed
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestDedupSuppressesSecondMessage(t *testing.T) {
+	rdb := newTestRedis(t)
+	ts := newTGServer(t)
+	n := newNotifier(t, ts, rdb)
+
+	ctx := context.Background()
+	// Use a Warning-severity event (WatermarkLow) so dedup applies.
+	// Note: WatermarkLow has Warning severity → dedup applies.
+	e := alert.NewEvent(alert.EventTypeBlockConfirmed, "node-dup", nil)
+
+	// First call — must go through.
+	if err := n.Notify(ctx, e); err != nil {
+		t.Fatalf("first Notify() error: %v", err)
+	}
+	if got := ts.callCount.Load(); got != 1 {
+		t.Fatalf("after first call: TG count = %d; want 1", got)
+	}
+
+	// Second call with same (Type, NodeID) — must be suppressed (no TG call).
+	if err := n.Notify(ctx, e); err != nil {
+		t.Fatalf("second Notify() error: %v", err)
+	}
+	if got := ts.callCount.Load(); got != 1 {
+		t.Errorf("after second call: TG count = %d; want still 1 (dedup)", got)
+	}
+
+	// Third call with different NodeID — must go through (distinct dedup key).
+	e2 := alert.NewEvent(alert.EventTypeBlockConfirmed, "node-other", nil)
+	if err := n.Notify(ctx, e2); err != nil {
+		t.Fatalf("third Notify() error: %v", err)
+	}
+	if got := ts.callCount.Load(); got != 2 {
+		t.Errorf("after third call (different node): TG count = %d; want 2", got)
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: critical events are never deduplicated — each call goes through
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestCriticalEventsNotDeduped(t *testing.T) {
+	rdb := newTestRedis(t)
+	ts := newTGServer(t)
+	n := newNotifier(t, ts, rdb)
+
+	ctx := context.Background()
+	// BreakerTripped is Critical severity — must bypass dedup.
+	e := alert.NewEvent(alert.EventTypeBreakerTripped, "node-crit", nil)
+
+	for i := 1; i <= 3; i++ {
+		if err := n.Notify(ctx, e); err != nil {
+			t.Fatalf("call %d: Notify() error: %v", i, err)
+		}
+		if got := ts.callCount.Load(); int(got) != i {
+			t.Errorf("after call %d: TG count = %d; want %d (no dedup for critical)", i, got, i)
+		}
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: TG 5xx triggers retries; after exhausting retries falls back to log;
+//       Notify() never returns an error.
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestTG5xxRetriesThenFallsBack(t *testing.T) {
+	rdb := newTestRedis(t)
+	ts := newTGServer(t)
+
+	// Capture fallback calls.
+	var fallbackCalls atomic.Int32
+	fallback := &capturingNotifier{
+		fn: func(alert.Event) { fallbackCalls.Add(1) },
+	}
+
+	n := alert.NewTGNotifier(alert.TGConfig{
+		BotToken: "test-token",
+		ChatID:   "-1001234567",
+		BaseURL:  ts.srv.URL,
+	}, rdb, fallback)
+
+	// Configure mock TG server to return 500.
+	ts.statusCode.Store(http.StatusInternalServerError)
+
+	ctx := context.Background()
+	e := alert.NewEvent(alert.EventTypeFault, "node-fail", map[string]string{"reason": "both fail"})
+
+	// Notify must return nil (not block the scheduler).
+	if err := n.Notify(ctx, e); err != nil {
+		t.Fatalf("Notify() must not return error; got %v", err)
+	}
+
+	// TG must have been called 3 times (1 initial + 2 retries = maxRetries+1).
+	if got := ts.callCount.Load(); got != 3 {
+		t.Errorf("TG call count = %d; want 3 (1 + 2 retries)", got)
+	}
+
+	// Fallback must have been called exactly once.
+	if got := fallbackCalls.Load(); got != 1 {
+		t.Errorf("fallback call count = %d; want 1", got)
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: LogNotifier always succeeds (used in offline/dev mode)
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestLogNotifier(t *testing.T) {
+	n := alert.LogNotifier{}
+	ctx := context.Background()
+
+	for _, et := range []alert.EventType{
+		alert.EventTypeBlockConfirmed,
+		alert.EventTypeReplenishFailed,
+		alert.EventTypeWatermarkLow,
+		alert.EventTypeBreakerTripped,
+		alert.EventTypeProbeAgentLost,
+		alert.EventTypeHeartbeatMissing,
+		alert.EventTypeFault,
+	} {
+		e := alert.NewEvent(et, "node-log", nil)
+		if err := n.Notify(ctx, e); err != nil {
+			t.Errorf("LogNotifier.Notify(%s) error: %v", et, err)
+		}
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: RunbookAnchor override takes precedence over type default
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestRunbookAnchorOverride(t *testing.T) {
+	rdb := newTestRedis(t)
+	ts := newTGServer(t)
+	n := newNotifier(t, ts, rdb)
+
+	e := alert.NewEvent(alert.EventTypeFault, "node-anch", nil)
+	e.RunbookAnchor = "#custom-anchor"
+
+	if err := n.Notify(context.Background(), e); err != nil {
+		t.Fatal(err)
+	}
+
+	body := ts.parsedBody(t)
+	if !strings.Contains(body["text"], "#custom-anchor") {
+		t.Errorf("expected #custom-anchor in message:\n%s", body["text"])
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+// capturingNotifier is a Notifier that calls fn on each Notify.
+type capturingNotifier struct {
+	fn func(alert.Event)
+}
+
+func (c *capturingNotifier) Notify(_ context.Context, e alert.Event) error {
+	c.fn(e)
+	return nil
+}

server/internal/alert/runbook_test.go

@@ -0,0 +1,73 @@
+package alert_test
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/wangjia/pangolin/server/internal/alert"
+)
+
+// allEventTypes lists every EventType constant for completeness checks.
+var allEventTypes = []alert.EventType{
+	alert.EventTypeBlockConfirmed,
+	alert.EventTypeReplenishFailed,
+	alert.EventTypeWatermarkLow,
+	alert.EventTypeBreakerTripped,
+	alert.EventTypeProbeAgentLost,
+	alert.EventTypeHeartbeatMissing,
+	alert.EventTypeFault,
+}
+
+// expectedAnchors maps each EventType to the <a id="…"> expected in the runbook.
+var expectedAnchors = map[alert.EventType]string{
+	alert.EventTypeBlockConfirmed:   "block-confirmed",
+	alert.EventTypeReplenishFailed:  "replenish-failed",
+	alert.EventTypeWatermarkLow:     "watermark-low",
+	alert.EventTypeBreakerTripped:   "breaker-tripped",
+	alert.EventTypeProbeAgentLost:   "probe-agent-lost",
+	alert.EventTypeHeartbeatMissing: "heartbeat-missing",
+	alert.EventTypeFault:            "node-fault",
+}
+
+// TestRunbookAnchorsInCode verifies that every EventType produces an Event
+// whose RunbookAnchor matches the expected anchor.
+func TestRunbookAnchorsInCode(t *testing.T) {
+	for _, et := range allEventTypes {
+		e := alert.NewEvent(et, "node-x", nil)
+		wantAnchor := "#" + expectedAnchors[et]
+		if e.RunbookAnchor != wantAnchor {
+			t.Errorf("EventType %q: RunbookAnchor = %q; want %q",
+				et, e.RunbookAnchor, wantAnchor)
+		}
+	}
+}
+
+// TestRunbookFileContainsAllAnchors verifies that docs/runbook-scheduler.md
+// contains an <a id="…"> tag for every event type.
+//
+// The test is skipped when the file does not exist yet (so it never blocks CI
+// while the runbook is being drafted) and fails once the file is present but
+// an anchor is missing.
+func TestRunbookFileContainsAllAnchors(t *testing.T) {
+	// Navigate up from server/internal/alert to the repo root, then to docs/.
+	// Go test sets cwd to the package directory (server/internal/alert/),
+	// so three levels up reaches the worktree root (where docs/ lives).
+	runbookPath := "../../../docs/runbook-scheduler.md"
+
+	data, err := os.ReadFile(runbookPath)
+	if os.IsNotExist(err) {
+		t.Skip("docs/runbook-scheduler.md not found; skipping anchor check")
+	}
+	if err != nil {
+		t.Fatalf("read runbook: %v", err)
+	}
+	content := string(data)
+
+	for et, anchor := range expectedAnchors {
+		tag := `id="` + anchor + `"`
+		if !strings.Contains(content, tag) {
+			t.Errorf("runbook missing anchor for EventType %q: expected <a %s>", et, tag)
+		}
+	}
+}

server/internal/scheduler/detect/engine.go

@@ -9,32 +9,14 @@ import (
 
 	"github.com/redis/go-redis/v9"
 
+	"github.com/wangjia/pangolin/server/internal/alert"
 	"github.com/wangjia/pangolin/server/internal/idgen"
 )
 
-// ─────────────────────────────────────────────────────────────────────────────
-// Notifier — 15G interface stub
-// ─────────────────────────────────────────────────────────────────────────────
-
-// Notifier is the 15G event sink for fault notifications.
-// The real implementation (15G) sends a Telegram/alerting message; the stub
-// below writes to the structured logger and is used until 15G is ready.
-type Notifier interface {
-	NotifyFault(ctx context.Context, nodeID, reason string) error
-}
-
-// LogNotifier is a Notifier stub that logs via slog.
-// It is used when no real Notifier is wired up.
-type LogNotifier struct{}
-
-// NotifyFault implements Notifier.
-func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
-	slog.Warn("node fault detected — manual review required",
-		"node_id", nodeID,
-		"reason", reason,
-	)
-	return nil
-}
+// Notifier is the 15G alert outlet used by the detection engine.
+// It is satisfied by alert.Notifier (the real TG implementation) and by
+// alert.LogNotifier (the fallback / development stub).
+type Notifier = alert.Notifier
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Redis key constants
@@ -91,7 +73,7 @@ func NewEngine(
 		cfg = &d
 	}
 	if notifier == nil {
-		notifier = LogNotifier{}
+		notifier = alert.LogNotifier{}
 	}
 	return &Engine{
 		probeStore: probeStore,
@@ -150,8 +132,11 @@ func (e *Engine) processNode(ctx context.Context, node NodeInfo) error {
 	// Streaks are left unchanged so that when the node recovers the engine
 	// resumes from its current position rather than re-triggering immediately.
 	if isFault(sig) {
-		reason := fmt.Sprintf("domestic_fail_isps=%d overseas_ok=false", sig.DomesticFailISPs)
-		if notifyErr := e.notifier.NotifyFault(ctx, node.ID, reason); notifyErr != nil {
+		ev := alert.NewEvent(alert.EventTypeFault, node.ID, map[string]string{
+			"domestic_fail_isps": fmt.Sprintf("%d", sig.DomesticFailISPs),
+			"overseas_ok":        "false",
+		})
+		if notifyErr := e.notifier.Notify(ctx, ev); notifyErr != nil {
 			slog.Error("detect: notify fault", "node_id", node.ID, "error", notifyErr)
 		}
 		return nil // do not persist streak changes
@@ -254,6 +239,14 @@ func (e *Engine) processSuspect(ctx context.Context, node NodeInfo, sig NodeSign
 				return e.streaks.Save(ctx, node.ID, sk)
 			}
 
+			// Emit 判封确认 alert (15G exit channel).
+			confirmedEv := alert.NewEvent(alert.EventTypeBlockConfirmed, node.ID, map[string]string{
+				"suspect_streak": fmt.Sprintf("%d", sk.SuspectStreak),
+			})
+			if notifyErr := e.notifier.Notify(ctx, confirmedEv); notifyErr != nil {
+				slog.Error("detect: notify block confirmed", "node_id", node.ID, "error", notifyErr)
+			}
+
 			// Immediately mark down — skip draining per lifecycle policy.
 			downDetail := map[string]any{
 				"from": "blocked_confirmed",

server/internal/scheduler/detect/engine_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/alicebob/miniredis/v2"
 	"github.com/redis/go-redis/v9"
 
+	"github.com/wangjia/pangolin/server/internal/alert"
 	"github.com/wangjia/pangolin/server/internal/scheduler/detect"
 	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
 )
@@ -29,16 +30,26 @@ func (m *mockSnapshotter) SnapshotsByNode(_ context.Context, nodeID string) (map
 	return nil, nil
 }
 
-// recordingNotifier records NotifyFault calls for assertion.
+// recordingNotifier records Notify calls for assertion.
 type recordingNotifier struct {
-	calls []string // nodeID values
+	events []alert.Event
 }
 
-func (r *recordingNotifier) NotifyFault(_ context.Context, nodeID, _ string) error {
-	r.calls = append(r.calls, nodeID)
+func (r *recordingNotifier) Notify(_ context.Context, e alert.Event) error {
+	r.events = append(r.events, e)
 	return nil
 }
 
+// hasFaultEvent returns true if any recorded event has type EventTypeFault.
+func (r *recordingNotifier) hasFaultEvent() bool {
+	for _, e := range r.events {
+		if e.Type == alert.EventTypeFault {
+			return true
+		}
+	}
+	return false
+}
+
 // newTestRedis creates an in-process Redis (miniredis) and returns a connected
 // client plus a cleanup function.  Tests must call cleanup() at the end.
 func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
@@ -345,11 +356,11 @@ func TestRules(t *testing.T) {
 			}
 
 			// Verify fault notification.
-			if tc.wantFaultNotified && len(notifier.calls) == 0 {
-				t.Error("expected NotifyFault to be called, but it was not")
+			if tc.wantFaultNotified && !notifier.hasFaultEvent() {
+				t.Error("expected fault Notify event to be recorded, but it was not")
 			}
-			if !tc.wantFaultNotified && len(notifier.calls) > 0 {
-				t.Errorf("unexpected NotifyFault calls: %v", notifier.calls)
+			if !tc.wantFaultNotified && notifier.hasFaultEvent() {
+				t.Errorf("unexpected fault Notify events: %v", notifier.events)
 			}
 		})
 	}
@@ -594,8 +605,8 @@ func TestFaultNoTransitionNoStreak(t *testing.T) {
 	if events := lc.Events(); len(events) != 0 {
 		t.Errorf("unexpected events: %v", events)
 	}
-	if len(notifier.calls) == 0 {
-		t.Error("expected NotifyFault to be called at least once")
+	if !notifier.hasFaultEvent() {
+		t.Error("expected fault Notify event to be recorded at least once")
 	}
 }
 

server/internal/scheduler/orchestrate/deps.go

@@ -8,9 +8,9 @@ package orchestrate
 
 import (
 	"context"
-	"log/slog"
 	"time"
 
+	"github.com/wangjia/pangolin/server/internal/alert"
 	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
 )
 
@@ -164,25 +164,17 @@ func (StubBreaker) Allow(_, _ string) bool { return true }
 func (StubBreaker) Record(_, _ string) {}
 
 // ─────────────────────────────────────────────────────────────────────────────
-// Notifier (15G stub)
+// Notifier (15G)
 // ─────────────────────────────────────────────────────────────────────────────
 
-// Notifier is the 15G alerting interface.
-type Notifier interface {
-	NotifyFault(ctx context.Context, nodeID, reason string) error
-}
+// Notifier is the 15G unified alert outlet used by the orchestrator.
+// It is satisfied by alert.Notifier (TG implementation) and alert.LogNotifier
+// (fallback / development stub).
+type Notifier = alert.Notifier
 
-// LogNotifier logs faults via slog. Used when no real notifier is wired.
-type LogNotifier struct{}
-
-// NotifyFault implements Notifier.
-func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
-	slog.Warn("orchestrate: replacement failed — manual review required",
-		"node_id", nodeID,
-		"reason", reason,
-	)
-	return nil
-}
+// LogNotifier is re-exported for callers that need a no-op Notifier without
+// importing the alert package directly.
+type LogNotifier = alert.LogNotifier
 
 // ─────────────────────────────────────────────────────────────────────────────
 // Clock (for testability)

server/internal/scheduler/orchestrate/replacer.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/redis/go-redis/v9"
 
+	"github.com/wangjia/pangolin/server/internal/alert"
 	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
 )
 
@@ -95,7 +96,7 @@ func NewReplacer(cfg Config) *Replacer {
 		cfg.Breaker = StubBreaker{}
 	}
 	if cfg.Notifier == nil {
-		cfg.Notifier = LogNotifier{}
+		cfg.Notifier = alert.LogNotifier{}
 	}
 	if cfg.Clock == nil {
 		cfg.Clock = RealClock{}
@@ -237,11 +238,21 @@ func (r *Replacer) stepPending(ctx context.Context, uuid string, rec *ReplaceRec
 	if !r.breaker.Allow(nodeInfo.Tier, nodeInfo.Region) {
 		slog.Info("orchestrate: breaker blocked replacement",
 			"uuid", uuid, "tier", nodeInfo.Tier, "region", nodeInfo.Region)
+		// Emit 熔断触发 alert (15G exit channel).
+		ev := alert.NewEvent(alert.EventTypeBreakerTripped, rec.OldNode, map[string]string{
+			"tier":             nodeInfo.Tier,
+			"region":           nodeInfo.Region,
+			"replacement_uuid": uuid,
+		})
+		ev.Pool = nodeInfo.Tier + "/" + nodeInfo.Region
+		if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
+			slog.Error("orchestrate: notify breaker tripped", "uuid", uuid, "error", notifyErr)
+		}
 		return nil // stay pending; retry next Tick
 	}
 
 	// Watermark / quota check — stub (always passes).
-	// TODO(15F): implement real capacity-quota guard here.
+	// TODO(15F): emit EventTypeWatermarkLow when real capacity guard is wired.
 
 	rec.Phase = PhaseCreating
 	rec.PhaseStartedAt = r.clock.Now()
@@ -371,9 +382,14 @@ func (r *Replacer) failProbeAttempt(ctx context.Context, uuid string, rec *Repla
 		_ = r.rdb.SRem(ctx, replaceIndexKey, uuid).Err()
 		_ = r.rdb.Expire(ctx, replaceKeyPrefix+uuid, replaceTTL).Err()
 
-		alertReason := fmt.Sprintf("probing failed after %d attempts: %s", rec.Attempts, reason)
-		if notifyErr := r.notifier.NotifyFault(ctx, rec.OldNode, alertReason); notifyErr != nil {
-			slog.Error("orchestrate: notify fault", "uuid", uuid, "error", notifyErr)
+		// Emit 补新连续失败≥3 alert (15G exit channel).
+		ev := alert.NewEvent(alert.EventTypeReplenishFailed, rec.OldNode, map[string]string{
+			"attempts":         fmt.Sprintf("%d", rec.Attempts),
+			"last_reason":      reason,
+			"replacement_uuid": uuid,
+		})
+		if notifyErr := r.notifier.Notify(ctx, ev); notifyErr != nil {
+			slog.Error("orchestrate: notify replenish failed", "uuid", uuid, "error", notifyErr)
 		}
 		slog.Error("orchestrate: replacement permanently failed — manual review required",
 			"uuid", uuid, "old_node", rec.OldNode, "attempts", rec.Attempts)

server/internal/scheduler/orchestrate/replacer_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/alicebob/miniredis/v2"
 	"github.com/redis/go-redis/v9"
 
+	"github.com/wangjia/pangolin/server/internal/alert"
 	"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
 	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
 )
@@ -267,21 +268,35 @@ func passingSnapshots() map[string]probe.ProbeSnapshot {
 // ─────────────────────────────────────────────────────────────────────────────
 
 type mockNotifier struct {
-	mu    sync.Mutex
-	calls []string
+	mu     sync.Mutex
+	events []alert.Event
 }
 
-func (n *mockNotifier) NotifyFault(_ context.Context, nodeID, _ string) error {
+func (n *mockNotifier) Notify(_ context.Context, e alert.Event) error {
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	n.calls = append(n.calls, nodeID)
+	n.events = append(n.events, e)
 	return nil
 }
 
+// count returns the number of Notify calls received.
 func (n *mockNotifier) count() int {
 	n.mu.Lock()
 	defer n.mu.Unlock()
-	return len(n.calls)
+	return len(n.events)
+}
+
+// countByType returns the number of Notify calls with the given EventType.
+func (n *mockNotifier) countByType(t alert.EventType) int {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	c := 0
+	for _, e := range n.events {
+		if e.Type == t {
+			c++
+		}
+	}
+	return c
 }
 
 // ─────────────────────────────────────────────────────────────────────────────
@@ -557,9 +572,9 @@ func TestProbeFailMaxAttempts(t *testing.T) {
 		}
 	}
 
-	// NotifyFault must be called exactly once.
-	if n := h.notifier.count(); n != 1 {
-		t.Errorf("NotifyFault calls = %d; want 1", n)
+	// Notify(EventTypeReplenishFailed) must be called exactly once.
+	if n := h.notifier.countByType(alert.EventTypeReplenishFailed); n != 1 {
+		t.Errorf("Notify(ReplenishFailed) calls = %d; want 1", n)
 	}
 
 	// Record must be in failed phase.

server/internal/scheduler/probe/prober_agent.go

@@ -34,9 +34,12 @@ import (
 	"log/slog"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
+
+	"github.com/wangjia/pangolin/server/internal/alert"
 )
 
 // --------------------------------------------------------------------------
@@ -113,6 +116,11 @@ var aliyunISP = []struct {
 // Overridable in tests via AliyunSyntheticAgent.baseURL.
 const aliyunEndpoint = "https://cloudmonitor.cn-hangzhou.aliyuncs.com/"
 
+// probeAgentLostThreshold is the number of consecutive per-ISP API failures
+// that must accumulate before an EventTypeProbeAgentLost alert is emitted.
+// This mirrors the "连续失败≥3" policy documented in the package comments.
+const probeAgentLostThreshold = 3
+
 // AliyunSyntheticAgentConfig holds configuration for AliyunSyntheticAgent.
 // The AccessKeyID and AccessKeySecret must belong to a RAM sub-account with
 // minimal permissions (cloudmonitor:CreateSiteMonitor +
@@ -158,6 +166,16 @@ type AliyunSyntheticAgent struct {
 	baseURL    string // overridable in tests
 	failCount  atomic.Int64
 	logger     *slog.Logger
+	// notifier is the 15G exit channel for EventTypeProbeAgentLost events.
+	// Nil means no alerting (development / test with no TG configured).
+	notifier alert.Notifier
+}
+
+// SetNotifier injects the 15G alert outlet into the agent.
+// When not set, no EventTypeProbeAgentLost alerts are emitted (dev/test mode).
+// Call before RunOnce / Probe.
+func (a *AliyunSyntheticAgent) SetNotifier(n alert.Notifier) {
+	a.notifier = n
 }
 
 // NewAliyunSyntheticAgent creates an AliyunSyntheticAgent.
@@ -255,6 +273,9 @@ func (a *AliyunSyntheticAgent) RunOnce(ctx context.Context, targets []ProbeTarge
 //
 // On any API or polling error the ISP vantage is skipped (no result returned,
 // no Redis write) per the degradation contract.
+//
+// When consecutive per-ISP API failures reach probeAgentLostThreshold the
+// 探针失联 (EventTypeProbeAgentLost) alert is emitted via the injected Notifier.
 func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([]VantageResult, error) {
 	var out []VantageResult
 	for _, isp := range aliyunISP {
@@ -264,6 +285,17 @@ func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([
 			a.logger.Warn("prober_agent: ISP probe failed (degraded, no data written)",
 				"node", target.NodeID, "isp", isp.name, "error", err,
 				"consecutive_failures", cnt)
+			// Emit 探针失联 alert when threshold is crossed (15G exit channel).
+			if cnt >= probeAgentLostThreshold && a.notifier != nil {
+				ev := alert.NewEvent(alert.EventTypeProbeAgentLost, "aliyun-synthetic", map[string]string{
+					"consecutive_failures": strconv.FormatInt(cnt, 10),
+					"last_isp":             isp.name,
+					"last_node":            target.NodeID,
+				})
+				if notifyErr := a.notifier.Notify(ctx, ev); notifyErr != nil {
+					a.logger.Warn("prober_agent: notify probe agent lost", "error", notifyErr)
+				}
+			}
 			// Degradation: skip this vantage this cycle.
 			continue
 		}

server/internal/scheduler/probe/store.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/redis/go-redis/v9"
+
+	"github.com/wangjia/pangolin/server/internal/alert"
 )
 
 // Redis TTL constants for the probe subsystem.
@@ -210,3 +212,50 @@ func (s *Store) AliveProbes(ctx context.Context) ([]string, error) {
 	}
 	return ids, nil
 }
+
+// CheckHeartbeats scans all known probe heartbeat keys and fires an
+// EventTypeHeartbeatMissing alert via notifier for every probe whose last
+// heartbeat timestamp is older than threshold.
+//
+// This is called by 15H (DetectLoop / assembly) on a periodic basis.
+// Missing-key semantics apply: a key that expired (TTL elapsed) is not seen
+// at all — only keys that exist but carry a stale timestamp are reported.
+//
+// threshold should be ≥90 s per the operational SLO.
+func (s *Store) CheckHeartbeats(ctx context.Context, threshold time.Duration, notifier alert.Notifier) error {
+	now := time.Now().Unix()
+	cutoff := now - int64(threshold.Seconds())
+
+	var scanErr error
+	iter := s.rdb.Scan(ctx, 0, "probe:hb:*", 0).Iterator()
+	for iter.Next(ctx) {
+		k := iter.Val()
+		probeID := strings.TrimPrefix(k, "probe:hb:")
+
+		val, err := s.rdb.Get(ctx, k).Result()
+		if err != nil {
+			// Key may have expired between SCAN and GET; skip.
+			continue
+		}
+		ts, err := strconv.ParseInt(val, 10, 64)
+		if err != nil {
+			continue // corrupt value; ignore
+		}
+		if ts < cutoff {
+			// Heartbeat is stale: emit alert.
+			staleSecs := now - ts
+			ev := alert.NewEvent(alert.EventTypeHeartbeatMissing, probeID, map[string]string{
+				"stale_seconds": strconv.FormatInt(staleSecs, 10),
+				"threshold_s":   strconv.FormatInt(int64(threshold.Seconds()), 10),
+			})
+			if notifyErr := notifier.Notify(ctx, ev); notifyErr != nil {
+				// Log but continue checking other probes.
+				scanErr = fmt.Errorf("probe: notify heartbeat missing for %s: %w", probeID, notifyErr)
+			}
+		}
+	}
+	if err := iter.Err(); err != nil {
+		return fmt.Errorf("probe: scan heartbeats: %w", err)
+	}
+	return scanErr
+}