88757b2ac4
新建 server/internal/scheduler/detect/ 包,实现任务 15D: ## 核心模块 **lifecycle.go** - 定义 LifecycleService Go interface(ListNodes / TransitionStatus / SetWeight / BumpVersion / GetLoad / GetLoadHistory),供 #5 真实实现,内含乐观锁语义(0 行 受影响即本周期放弃,幂等可重入) - 提供 MockLifecycle 内存 stub(SetConflict / Events 等测试辅助方法) **signals.go** - computeSignals:读 SnapshotsByNode,归一化为 NodeSignal - domesticFailISPs:按 ISP 聚合(stripPrefix "3rd-" 合并 15B/15C 同 ISP 视角), L3 失败权重最高(L3 挂即记该 ISP 失败,即使 L1/L2 通),无数据 vantage 不计分母 - overseasOK:境外对照点全部 L1 通为真 - trafficDropPct:15min 前后半段在线数降幅百分比 - ProbeSnapshotter interface(解耦 probe.Store,便于 mock) **rules.go** - DetectConfig:全量阈值常量(DomesticFailNumerator/Denominator=2/3、 SuspectStreakMin=2、TrafficDropThreshold=80%、TrafficBaselineMin=20、 ConfirmedStreakMin=6、RecoverStreakMin=2、SuspectWeight=10) - isSuspectTriggered:整数算术避免浮点误差(fail×den >= total×num) - isFault / effectiveSuspectStreakMin / isTrafficWarning 规则助手 **streak.go** - StreakStore:Redis hash detect:streak:{node}(fail_streak / suspect_streak / recover_streak),TTL 2h(宽于 6×5min=30min),进程重启后自动恢复 **engine.go** - Engine.Tick(ctx):列举 up/blocked_suspect 节点,按节点依序运行五条规则 - Rule 1 suspect:≥2/3 ISP 失败 + 境外正常 → 连续 2 周期 → up→blocked_suspect, weight→10,写 probe:freq:{node}=60(1min 提频标记),BumpVersion - Rule 2 流量预警:trafficDrop≥80% + 基线≥20 → suspect 门槛放宽至 1 周期 - Rule 3 confirmed:blocked_suspect 连续 6 周期 → blocked_confirmed → down(跳过 draining),入队 detect:replace:queue(JSON: nodeId + replacementUuid) - Rule 4 recover:suspect 期间境内连续 2 周期恢复 → up(weight 交 15E warmup) - Rule 5 fault:境内+境外同时失败 → 不迁移,调 Notifier.NotifyFault(log stub) - Notifier interface + LogNotifier stub(15G 就绪前输出 slog.Warn) **engine_test.go**(19 个测试,全部通过) - 表驱动覆盖:恰好 2/3 失败、1/3 失败不触发、9min vs 10min streak、 流量预警 1 周期即 suspect、suspect 第 6 周期 confirmed、境内外同挂 fault 不迁移、 恢复 2 周期回 up、3rd- 前缀合并、L3 覆盖 L1/L2 - 乐观锁冲突:SetConflict → 放弃本周期 → 无副作用 → 解冲突后正常重试 - 进程重启:同一 miniredis,新 Engine 读取已存在 streak 继续计数不误清零 - 产出验证:replace queue JSON 结构、probe:freq 标记、weight 变更、事件 detail 字段 **migrations/000010_node_blocked_statuses.{up,down}.sql** - 扩展 nodes.status ENUM 加入 blocked_suspect / blocked_confirmed Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
102 lines
3.6 KiB
Go
102 lines
3.6 KiB
Go
package detect
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/redis/go-redis/v9"
|
||
)
|
||
|
||
const (
|
||
// streakTTL is the Redis TTL for a per-node streak key.
|
||
//
|
||
// Must be comfortably wider than the longest streak window, which is the
|
||
// confirmed threshold: 6 cycles × 5 min = 30 min. Two hours provides
|
||
// enough margin to survive a process restart or a short leader-election gap
|
||
// without zeroing out a valid streak mid-observation.
|
||
streakTTL = 2 * time.Hour
|
||
)
|
||
|
||
// Streak holds the consecutive-cycle counters for a single node.
|
||
// All three counters are stored together under one Redis key to make the
|
||
// read-modify-write loop atomic with a single GET + SET round-trip.
|
||
type Streak struct {
|
||
// FailStreak is the number of consecutive detection cycles in which the
|
||
// node's domestic probes failed while the node was in "up" state.
|
||
// It is reset to zero once the suspect threshold is reached (transition
|
||
// attempted) or when domestic probes recover.
|
||
FailStreak int `json:"fail_streak"`
|
||
|
||
// SuspectStreak counts consecutive cycles spent in "blocked_suspect" where
|
||
// the GFW-block condition (domestic fail + overseas OK) still holds.
|
||
// Reaching ConfirmedStreakMin triggers the confirmed transition.
|
||
SuspectStreak int `json:"suspect_streak"`
|
||
|
||
// RecoverStreak counts consecutive cycles in "blocked_suspect" where
|
||
// domestic probes pass. Reaching RecoverStreakMin triggers recovery.
|
||
RecoverStreak int `json:"recover_streak"`
|
||
}
|
||
|
||
// StreakStore persists per-node Streak values in Redis.
|
||
//
|
||
// Key format: detect:streak:{nodeID} (JSON-encoded Streak, TTL 2 h)
|
||
//
|
||
// Missing-key semantics: a missing key means "no streak data" and is treated
|
||
// as a zero Streak. After a process restart, the first Tick loads whichever
|
||
// streaks survived in Redis and continues from that point — streaks are NOT
|
||
// re-derived from probe history on restart, which keeps the restart path
|
||
// simple at the cost of a potential single-cycle window of uncertainty.
|
||
type StreakStore struct {
|
||
rdb *redis.Client
|
||
}
|
||
|
||
// NewStreakStore creates a StreakStore backed by the given Redis client.
|
||
func NewStreakStore(rdb *redis.Client) *StreakStore {
|
||
return &StreakStore{rdb: rdb}
|
||
}
|
||
|
||
// streakKey returns the Redis key for nodeID's streak data.
|
||
func streakKey(nodeID string) string {
|
||
return "detect:streak:" + nodeID
|
||
}
|
||
|
||
// Load reads the current Streak for nodeID.
|
||
// A missing or corrupted key returns a zero Streak without error.
|
||
func (s *StreakStore) Load(ctx context.Context, nodeID string) (Streak, error) {
|
||
val, err := s.rdb.Get(ctx, streakKey(nodeID)).Result()
|
||
if err == redis.Nil {
|
||
return Streak{}, nil
|
||
}
|
||
if err != nil {
|
||
return Streak{}, fmt.Errorf("detect: streak get %s: %w", nodeID, err)
|
||
}
|
||
var sk Streak
|
||
if err := json.Unmarshal([]byte(val), &sk); err != nil {
|
||
// Corrupted entry: treat as zero (non-fatal; we overwrite on the next Save).
|
||
return Streak{}, nil
|
||
}
|
||
return sk, nil
|
||
}
|
||
|
||
// Save persists sk for nodeID with the standard streakTTL.
|
||
func (s *StreakStore) Save(ctx context.Context, nodeID string, sk Streak) error {
|
||
data, err := json.Marshal(sk)
|
||
if err != nil {
|
||
return fmt.Errorf("detect: streak marshal %s: %w", nodeID, err)
|
||
}
|
||
if err := s.rdb.Set(ctx, streakKey(nodeID), data, streakTTL).Err(); err != nil {
|
||
return fmt.Errorf("detect: streak set %s: %w", nodeID, err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// Reset deletes the streak key for nodeID (e.g. after a confirmed transition).
|
||
func (s *StreakStore) Reset(ctx context.Context, nodeID string) error {
|
||
if err := s.rdb.Del(ctx, streakKey(nodeID)).Err(); err != nil {
|
||
return fmt.Errorf("detect: streak del %s: %w", nodeID, err)
|
||
}
|
||
return nil
|
||
}
|