88757b2ac4
新建 server/internal/scheduler/detect/ 包,实现任务 15D: ## 核心模块 **lifecycle.go** - 定义 LifecycleService Go interface(ListNodes / TransitionStatus / SetWeight / BumpVersion / GetLoad / GetLoadHistory),供 #5 真实实现,内含乐观锁语义(0 行 受影响即本周期放弃,幂等可重入) - 提供 MockLifecycle 内存 stub(SetConflict / Events 等测试辅助方法) **signals.go** - computeSignals:读 SnapshotsByNode,归一化为 NodeSignal - domesticFailISPs:按 ISP 聚合(stripPrefix "3rd-" 合并 15B/15C 同 ISP 视角), L3 失败权重最高(L3 挂即记该 ISP 失败,即使 L1/L2 通),无数据 vantage 不计分母 - overseasOK:境外对照点全部 L1 通为真 - trafficDropPct:15min 前后半段在线数降幅百分比 - ProbeSnapshotter interface(解耦 probe.Store,便于 mock) **rules.go** - DetectConfig:全量阈值常量(DomesticFailNumerator/Denominator=2/3、 SuspectStreakMin=2、TrafficDropThreshold=80%、TrafficBaselineMin=20、 ConfirmedStreakMin=6、RecoverStreakMin=2、SuspectWeight=10) - isSuspectTriggered:整数算术避免浮点误差(fail×den >= total×num) - isFault / effectiveSuspectStreakMin / isTrafficWarning 规则助手 **streak.go** - StreakStore:Redis hash detect:streak:{node}(fail_streak / suspect_streak / recover_streak),TTL 2h(宽于 6×5min=30min),进程重启后自动恢复 **engine.go** - Engine.Tick(ctx):列举 up/blocked_suspect 节点,按节点依序运行五条规则 - Rule 1 suspect:≥2/3 ISP 失败 + 境外正常 → 连续 2 周期 → up→blocked_suspect, weight→10,写 probe:freq:{node}=60(1min 提频标记),BumpVersion - Rule 2 流量预警:trafficDrop≥80% + 基线≥20 → suspect 门槛放宽至 1 周期 - Rule 3 confirmed:blocked_suspect 连续 6 周期 → blocked_confirmed → down(跳过 draining),入队 detect:replace:queue(JSON: nodeId + replacementUuid) - Rule 4 recover:suspect 期间境内连续 2 周期恢复 → up(weight 交 15E warmup) - Rule 5 fault:境内+境外同时失败 → 不迁移,调 Notifier.NotifyFault(log stub) - Notifier interface + LogNotifier stub(15G 就绪前输出 slog.Warn) **engine_test.go**(19 个测试,全部通过) - 表驱动覆盖:恰好 2/3 失败、1/3 失败不触发、9min vs 10min streak、 流量预警 1 周期即 suspect、suspect 第 6 周期 confirmed、境内外同挂 fault 不迁移、 恢复 2 周期回 up、3rd- 前缀合并、L3 覆盖 L1/L2 - 乐观锁冲突:SetConflict → 放弃本周期 → 无副作用 → 解冲突后正常重试 - 进程重启:同一 miniredis,新 Engine 读取已存在 streak 继续计数不误清零 - 产出验证:replace queue JSON 结构、probe:freq 标记、weight 变更、事件 detail 字段 **migrations/000010_node_blocked_statuses.{up,down}.sql** - 扩展 nodes.status ENUM 加入 blocked_suspect / blocked_confirmed Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
178 lines
5.1 KiB
Go
178 lines
5.1 KiB
Go
package detect
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||
)
|
||
|
||
// NodeSignal is the normalised set of detection signals for a single node
|
||
// produced by one Tick.
|
||
type NodeSignal struct {
|
||
NodeID string
|
||
|
||
// DomesticTotalISPs is the count of distinct domestic (CN) ISPs for which
|
||
// probe data was available this cycle. Vantages with no data are excluded
|
||
// from both numerator and denominator per spec.
|
||
DomesticTotalISPs int
|
||
|
||
// DomesticFailISPs is the count of domestic ISPs judged to be failing.
|
||
// An ISP is failing if ANY vantage from that ISP indicates a failure at
|
||
// the highest available layer (L3 > L2 > L1; see isReportFailed).
|
||
DomesticFailISPs int
|
||
|
||
// OverseasHasData is true when at least one non-CN vantage reported this cycle.
|
||
OverseasHasData bool
|
||
|
||
// OverseasOK is true when every overseas vantage passed L1 this cycle.
|
||
// Only meaningful when OverseasHasData is true.
|
||
OverseasOK bool
|
||
|
||
// TrafficDropPct is the percentage drop in online-connection count computed
|
||
// by comparing the first half vs the second half of the 15-minute
|
||
// GetLoadHistory window. Zero when insufficient history is available.
|
||
TrafficDropPct float64
|
||
|
||
// TrafficBaseline is the online-connection count from the most recent load
|
||
// sample. Used by the traffic-warning rule's minimum-baseline guard.
|
||
TrafficBaseline int
|
||
}
|
||
|
||
// ProbeSnapshotter lets the Engine read probe snapshots without depending on
|
||
// the concrete *probe.Store (facilitates mocking in unit tests).
|
||
type ProbeSnapshotter interface {
|
||
SnapshotsByNode(ctx context.Context, nodeID string) (map[string]probe.ProbeSnapshot, error)
|
||
}
|
||
|
||
// computeSignals builds a NodeSignal for nodeID from the current probe
|
||
// snapshots and the 15-minute load history.
|
||
//
|
||
// ISP normalisation: third-party probes (15C) use a "3rd-" prefix on their
|
||
// ISP names (e.g. "3rd-ChinaTelecom"). This prefix is stripped before
|
||
// grouping so that first-party (15B) and third-party vantages from the same
|
||
// ISP are merged, per spec.
|
||
func computeSignals(
|
||
ctx context.Context,
|
||
nodeID string,
|
||
snapshots map[string]probe.ProbeSnapshot,
|
||
lc LifecycleService,
|
||
) NodeSignal {
|
||
sig := NodeSignal{NodeID: nodeID}
|
||
|
||
// domesticISPs: normalised ISP name → whether any vantage from that ISP
|
||
// shows a failure this cycle.
|
||
domesticISPs := make(map[string]bool)
|
||
overseasTotal := 0
|
||
overseasOKCount := 0
|
||
|
||
for _, snap := range snapshots {
|
||
v := snap.Vantage
|
||
r := snap.Report
|
||
|
||
switch {
|
||
case v.Country == "CN":
|
||
// Domestic vantage. Normalise ISP name and merge failures:
|
||
// any failing vantage from the same ISP marks that ISP as failed.
|
||
isp := normaliseISP(v.ISP)
|
||
failed := isReportFailed(r)
|
||
if prev, seen := domesticISPs[isp]; seen {
|
||
domesticISPs[isp] = prev || failed
|
||
} else {
|
||
domesticISPs[isp] = failed
|
||
}
|
||
|
||
case v.Country != "":
|
||
// Overseas vantage (any country other than CN).
|
||
overseasTotal++
|
||
if r.L1.OK {
|
||
overseasOKCount++
|
||
}
|
||
// Empty Country: vantage identity unknown – excluded from both pools.
|
||
}
|
||
}
|
||
|
||
sig.DomesticTotalISPs = len(domesticISPs)
|
||
for _, failed := range domesticISPs {
|
||
if failed {
|
||
sig.DomesticFailISPs++
|
||
}
|
||
}
|
||
sig.OverseasHasData = overseasTotal > 0
|
||
sig.OverseasOK = overseasTotal > 0 && overseasOKCount == overseasTotal
|
||
|
||
// Traffic drop: compare the older half of the 15-minute window to the
|
||
// newer half. Zero is returned when there is too little history.
|
||
history, err := lc.GetLoadHistory(ctx, nodeID, 15*time.Minute)
|
||
if err == nil && len(history) > 0 {
|
||
sig.TrafficBaseline = history[len(history)-1].Online
|
||
if len(history) >= 2 {
|
||
sig.TrafficDropPct = trafficDropPct(history)
|
||
}
|
||
}
|
||
|
||
return sig
|
||
}
|
||
|
||
// isReportFailed determines whether a single NodeReport indicates a failure
|
||
// at the highest available measurement layer.
|
||
//
|
||
// Priority (highest to lowest):
|
||
// - L3 present → L3.OK is the authoritative verdict.
|
||
// A failing L3 is recorded even when L1/L2 passed (e.g. DPI/SNI reset
|
||
// at the application layer after a successful TCP/TLS handshake).
|
||
// - L3 absent → check L1 then L2.
|
||
func isReportFailed(r probe.NodeReport) bool {
|
||
if r.L3 != nil {
|
||
return !r.L3.OK
|
||
}
|
||
if !r.L1.OK {
|
||
return true
|
||
}
|
||
if r.L2 != nil && !r.L2.OK {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// normaliseISP strips the "3rd-" prefix that AliyunSyntheticAgent (15C) adds
|
||
// to distinguish third-party vantages, yielding the canonical ISP name used
|
||
// for grouping.
|
||
func normaliseISP(isp string) string {
|
||
return strings.TrimPrefix(isp, "3rd-")
|
||
}
|
||
|
||
// trafficDropPct computes the percentage drop between the first and second
|
||
// halves of the load-history slice (oldest → newest).
|
||
// Returns 0 when the baseline (first-half average) is zero or when the
|
||
// second half average is higher (traffic increased).
|
||
func trafficDropPct(history []LoadPoint) float64 {
|
||
n := len(history)
|
||
if n < 2 {
|
||
return 0
|
||
}
|
||
half := n / 2
|
||
older := avgOnline(history[:half])
|
||
newer := avgOnline(history[half:])
|
||
if older == 0 {
|
||
return 0
|
||
}
|
||
drop := older - newer
|
||
if drop <= 0 {
|
||
return 0
|
||
}
|
||
return float64(drop) / float64(older) * 100
|
||
}
|
||
|
||
func avgOnline(pts []LoadPoint) int {
|
||
if len(pts) == 0 {
|
||
return 0
|
||
}
|
||
sum := 0
|
||
for _, p := range pts {
|
||
sum += p.Online
|
||
}
|
||
return sum / len(pts)
|
||
}
|