feat(detect): 判定引擎 signals+rules+streak (tsk_OYEiDCzM9_0Y)

新建 server/internal/scheduler/detect/ 包，实现任务 15D： ## 核心模块 **lifecycle.go** - 定义 LifecycleService Go interface（ListNodes / TransitionStatus / SetWeight / BumpVersion / GetLoad / GetLoadHistory），供 #5 真实实现，内含乐观锁语义（0 行受影响即本周期放弃，幂等可重入） - 提供 MockLifecycle 内存 stub（SetConflict / Events 等测试辅助方法） **signals.go** - computeSignals：读 SnapshotsByNode，归一化为 NodeSignal - domesticFailISPs：按 ISP 聚合（stripPrefix "3rd-" 合并 15B/15C 同 ISP 视角）， L3 失败权重最高（L3 挂即记该 ISP 失败，即使 L1/L2 通），无数据 vantage 不计分母 - overseasOK：境外对照点全部 L1 通为真 - trafficDropPct：15min 前后半段在线数降幅百分比 - ProbeSnapshotter interface（解耦 probe.Store，便于 mock） **rules.go** - DetectConfig：全量阈值常量（DomesticFailNumerator/Denominator=2/3、 SuspectStreakMin=2、TrafficDropThreshold=80%、TrafficBaselineMin=20、 ConfirmedStreakMin=6、RecoverStreakMin=2、SuspectWeight=10） - isSuspectTriggered：整数算术避免浮点误差（fail×den >= total×num） - isFault / effectiveSuspectStreakMin / isTrafficWarning 规则助手 **streak.go** - StreakStore：Redis hash detect:streak:{node}（fail_streak / suspect_streak / recover_streak），TTL 2h（宽于 6×5min=30min），进程重启后自动恢复 **engine.go** - Engine.Tick(ctx)：列举 up/blocked_suspect 节点，按节点依序运行五条规则 - Rule 1 suspect：≥2/3 ISP 失败 + 境外正常 → 连续 2 周期 → up→blocked_suspect， weight→10，写 probe:freq:{node}=60（1min 提频标记），BumpVersion - Rule 2 流量预警：trafficDrop≥80% + 基线≥20 → suspect 门槛放宽至 1 周期 - Rule 3 confirmed：blocked_suspect 连续 6 周期 → blocked_confirmed → down（跳过 draining），入队 detect:replace:queue（JSON: nodeId + replacementUuid） - Rule 4 recover：suspect 期间境内连续 2 周期恢复 → up（weight 交 15E warmup） - Rule 5 fault：境内+境外同时失败 → 不迁移，调 Notifier.NotifyFault（log stub） - Notifier interface + LogNotifier stub（15G 就绪前输出 slog.Warn） **engine_test.go**（19 个测试，全部通过） - 表驱动覆盖：恰好 2/3 失败、1/3 失败不触发、9min vs 10min streak、流量预警 1 周期即 suspect、suspect 第 6 周期 confirmed、境内外同挂 fault 不迁移、恢复 2 周期回 up、3rd- 前缀合并、L3 覆盖 L1/L2 - 乐观锁冲突：SetConflict → 放弃本周期 → 无副作用 → 解冲突后正常重试 - 进程重启：同一 miniredis，新 Engine 读取已存在 streak 继续计数不误清零 - 产出验证：replace queue JSON 结构、probe:freq 标记、weight 变更、事件 detail 字段 **migrations/000010_node_blocked_statuses.{up,down}.sql** - 扩展 nodes.status ENUM 加入 blocked_suspect / blocked_confirmed Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-13 20:15:13 +08:00
parent 628440b3f9
commit 88757b2ac4
8 changed files with 1675 additions and 0 deletions
@@ -0,0 +1,177 @@
+package detect
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
+)
+
+// NodeSignal is the normalised set of detection signals for a single node
+// produced by one Tick.
+type NodeSignal struct {
+	NodeID string
+
+	// DomesticTotalISPs is the count of distinct domestic (CN) ISPs for which
+	// probe data was available this cycle.  Vantages with no data are excluded
+	// from both numerator and denominator per spec.
+	DomesticTotalISPs int
+
+	// DomesticFailISPs is the count of domestic ISPs judged to be failing.
+	// An ISP is failing if ANY vantage from that ISP indicates a failure at
+	// the highest available layer (L3 > L2 > L1; see isReportFailed).
+	DomesticFailISPs int
+
+	// OverseasHasData is true when at least one non-CN vantage reported this cycle.
+	OverseasHasData bool
+
+	// OverseasOK is true when every overseas vantage passed L1 this cycle.
+	// Only meaningful when OverseasHasData is true.
+	OverseasOK bool
+
+	// TrafficDropPct is the percentage drop in online-connection count computed
+	// by comparing the first half vs the second half of the 15-minute
+	// GetLoadHistory window.  Zero when insufficient history is available.
+	TrafficDropPct float64
+
+	// TrafficBaseline is the online-connection count from the most recent load
+	// sample.  Used by the traffic-warning rule's minimum-baseline guard.
+	TrafficBaseline int
+}
+
+// ProbeSnapshotter lets the Engine read probe snapshots without depending on
+// the concrete *probe.Store (facilitates mocking in unit tests).
+type ProbeSnapshotter interface {
+	SnapshotsByNode(ctx context.Context, nodeID string) (map[string]probe.ProbeSnapshot, error)
+}
+
+// computeSignals builds a NodeSignal for nodeID from the current probe
+// snapshots and the 15-minute load history.
+//
+// ISP normalisation: third-party probes (15C) use a "3rd-" prefix on their
+// ISP names (e.g. "3rd-ChinaTelecom").  This prefix is stripped before
+// grouping so that first-party (15B) and third-party vantages from the same
+// ISP are merged, per spec.
+func computeSignals(
+	ctx context.Context,
+	nodeID string,
+	snapshots map[string]probe.ProbeSnapshot,
+	lc LifecycleService,
+) NodeSignal {
+	sig := NodeSignal{NodeID: nodeID}
+
+	// domesticISPs: normalised ISP name → whether any vantage from that ISP
+	// shows a failure this cycle.
+	domesticISPs := make(map[string]bool)
+	overseasTotal := 0
+	overseasOKCount := 0
+
+	for _, snap := range snapshots {
+		v := snap.Vantage
+		r := snap.Report
+
+		switch {
+		case v.Country == "CN":
+			// Domestic vantage. Normalise ISP name and merge failures:
+			// any failing vantage from the same ISP marks that ISP as failed.
+			isp := normaliseISP(v.ISP)
+			failed := isReportFailed(r)
+			if prev, seen := domesticISPs[isp]; seen {
+				domesticISPs[isp] = prev || failed
+			} else {
+				domesticISPs[isp] = failed
+			}
+
+		case v.Country != "":
+			// Overseas vantage (any country other than CN).
+			overseasTotal++
+			if r.L1.OK {
+				overseasOKCount++
+			}
+		// Empty Country: vantage identity unknown – excluded from both pools.
+		}
+	}
+
+	sig.DomesticTotalISPs = len(domesticISPs)
+	for _, failed := range domesticISPs {
+		if failed {
+			sig.DomesticFailISPs++
+		}
+	}
+	sig.OverseasHasData = overseasTotal > 0
+	sig.OverseasOK = overseasTotal > 0 && overseasOKCount == overseasTotal
+
+	// Traffic drop: compare the older half of the 15-minute window to the
+	// newer half.  Zero is returned when there is too little history.
+	history, err := lc.GetLoadHistory(ctx, nodeID, 15*time.Minute)
+	if err == nil && len(history) > 0 {
+		sig.TrafficBaseline = history[len(history)-1].Online
+		if len(history) >= 2 {
+			sig.TrafficDropPct = trafficDropPct(history)
+		}
+	}
+
+	return sig
+}
+
+// isReportFailed determines whether a single NodeReport indicates a failure
+// at the highest available measurement layer.
+//
+// Priority (highest to lowest):
+//   - L3 present → L3.OK is the authoritative verdict.
+//     A failing L3 is recorded even when L1/L2 passed (e.g. DPI/SNI reset
+//     at the application layer after a successful TCP/TLS handshake).
+//   - L3 absent → check L1 then L2.
+func isReportFailed(r probe.NodeReport) bool {
+	if r.L3 != nil {
+		return !r.L3.OK
+	}
+	if !r.L1.OK {
+		return true
+	}
+	if r.L2 != nil && !r.L2.OK {
+		return true
+	}
+	return false
+}
+
+// normaliseISP strips the "3rd-" prefix that AliyunSyntheticAgent (15C) adds
+// to distinguish third-party vantages, yielding the canonical ISP name used
+// for grouping.
+func normaliseISP(isp string) string {
+	return strings.TrimPrefix(isp, "3rd-")
+}
+
+// trafficDropPct computes the percentage drop between the first and second
+// halves of the load-history slice (oldest → newest).
+// Returns 0 when the baseline (first-half average) is zero or when the
+// second half average is higher (traffic increased).
+func trafficDropPct(history []LoadPoint) float64 {
+	n := len(history)
+	if n < 2 {
+		return 0
+	}
+	half := n / 2
+	older := avgOnline(history[:half])
+	newer := avgOnline(history[half:])
+	if older == 0 {
+		return 0
+	}
+	drop := older - newer
+	if drop <= 0 {
+		return 0
+	}
+	return float64(drop) / float64(older) * 100
+}
+
+func avgOnline(pts []LoadPoint) int {
+	if len(pts) == 0 {
+		return 0
+	}
+	sum := 0
+	for _, p := range pts {
+		sum += p.Online
+	}
+	return sum / len(pts)
+}