Files
pangolin/server/internal/scheduler/orchestrate/config_test.go
wangjia 3d9ccff6e9 feat(scheduler): 容量水位监控 + 熔断器 + 阈值热加载 [tsk_NPgPRxBGv0g9]
实现 15F 任务的三个核心组件:

capacity.go — CapacityMonitor(每 1-2 min 由 CapacityLoop 调用)
  - 池水位检查:up/target < 70% 时发告警,10 min 内降频去重
  - 探针失联检查:扫 probe:hb:{probeId},缺失发「探针失联」事件
  - 驱动 Grayscale.Advance 推进养机档位

breaker.go — RedisBreaker(实现 Breaker 接口)
  - 滑窗 ZSET sched:breaker:{tier}:{region},窗口 1h
  - 阈值 N = ceil(池容量 × 30%),下限 3
  - 窗口内计数 ≥ N → 置 tripped 标记位 + critical 告警
  - Allow 恒 false 直到窗口滑出自动恢复,或管理员调 Reset
  - Reset 清除 ZSET + trip flag,写 audit_log

config.go — SchedConfig + ConfigManager(全树阈值热加载)
  - SchedConfig 集中定义 15D/15E/15F 所有阈值
  - 来源:默认值 → YAML 文件 → 环境变量(三层叠加)
  - 监听 SIGHUP,原子替换(atomic.Pointer[SchedConfig])
  - 每次变更 diff 写 audit_log(AuditFn 回调)

测试(24 个用例,全绿):
  - 窗口内第 N 次 Record 后 Allow 返回 false
  - 窗口滑出后 Allow 自动恢复
  - Reset 立即恢复 + audit 条目
  - 下限 3 在小池(target=2)生效
  - 65% 水位 → 告警;10 min 内不重复
  - 探针心跳缺失 → 失联事件;存在 → 无告警
  - YAML + env 覆盖 + SIGHUP Reload → 新阈值即时生效 + diff 审计
  - 替换 stub 后 15E 全部集成测试仍绿

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 00:31:58 +08:00

290 lines
11 KiB
Go

package orchestrate_test
import (
"context"
"os"
"testing"
"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
)
// ─────────────────────────────────────────────────────────────────────────────
// Test: Default values
// ─────────────────────────────────────────────────────────────────────────────
func TestDefaultSchedConfig(t *testing.T) {
cfg := orchestrate.DefaultSchedConfig()
// Spot-check a representative value from each section.
if cfg.Detect.DomesticFailNumerator != 2 {
t.Errorf("Detect.DomesticFailNumerator = %d; want 2", cfg.Detect.DomesticFailNumerator)
}
if cfg.Capacity.WatermarkThreshold != 0.70 {
t.Errorf("Capacity.WatermarkThreshold = %f; want 0.70", cfg.Capacity.WatermarkThreshold)
}
if cfg.Capacity.BreakerMinN != 3 {
t.Errorf("Capacity.BreakerMinN = %d; want 3", cfg.Capacity.BreakerMinN)
}
if cfg.Probe.GrayscaleIntervalHours != 6 {
t.Errorf("Probe.GrayscaleIntervalHours = %d; want 6", cfg.Probe.GrayscaleIntervalHours)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Test: YAML file overrides defaults
// ─────────────────────────────────────────────────────────────────────────────
func TestConfigManagerYAMLOverrides(t *testing.T) {
yaml := `
detect:
suspect_streak_min: 5
confirmed_streak_min: 12
capacity:
watermark_threshold: 0.60
breaker_min_n: 5
expected_probe_ids:
- pb-1
- pb-2
probe:
grayscale_interval_hours: 8
max_attempts: 4
`
tmpFile, err := writeTempYAML(t, yaml)
if err != nil {
t.Fatalf("writeTempYAML: %v", err)
}
m := orchestrate.NewConfigManager(tmpFile, nil)
if err := m.Load(); err != nil {
t.Fatalf("Load: %v", err)
}
cfg := m.Current()
if cfg.Detect.SuspectStreakMin != 5 {
t.Errorf("Detect.SuspectStreakMin = %d; want 5", cfg.Detect.SuspectStreakMin)
}
if cfg.Detect.ConfirmedStreakMin != 12 {
t.Errorf("Detect.ConfirmedStreakMin = %d; want 12", cfg.Detect.ConfirmedStreakMin)
}
if cfg.Capacity.WatermarkThreshold != 0.60 {
t.Errorf("Capacity.WatermarkThreshold = %f; want 0.60", cfg.Capacity.WatermarkThreshold)
}
if cfg.Capacity.BreakerMinN != 5 {
t.Errorf("Capacity.BreakerMinN = %d; want 5", cfg.Capacity.BreakerMinN)
}
if len(cfg.Capacity.ExpectedProbeIDs) != 2 {
t.Errorf("ExpectedProbeIDs count = %d; want 2", len(cfg.Capacity.ExpectedProbeIDs))
} else {
if cfg.Capacity.ExpectedProbeIDs[0] != "pb-1" || cfg.Capacity.ExpectedProbeIDs[1] != "pb-2" {
t.Errorf("ExpectedProbeIDs = %v; want [pb-1 pb-2]", cfg.Capacity.ExpectedProbeIDs)
}
}
if cfg.Probe.GrayscaleIntervalHours != 8 {
t.Errorf("Probe.GrayscaleIntervalHours = %d; want 8", cfg.Probe.GrayscaleIntervalHours)
}
if cfg.Probe.MaxAttempts != 4 {
t.Errorf("Probe.MaxAttempts = %d; want 4", cfg.Probe.MaxAttempts)
}
// Unset fields must retain defaults.
if cfg.Detect.DomesticFailNumerator != 2 {
t.Errorf("Detect.DomesticFailNumerator should be default 2; got %d", cfg.Detect.DomesticFailNumerator)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Test: Environment variable overrides YAML
// ─────────────────────────────────────────────────────────────────────────────
func TestConfigManagerEnvOverridesYAML(t *testing.T) {
yaml := `
detect:
suspect_streak_min: 5
capacity:
breaker_min_n: 7
`
tmpFile, err := writeTempYAML(t, yaml)
if err != nil {
t.Fatalf("writeTempYAML: %v", err)
}
// Set env vars AFTER writing YAML to ensure they override.
t.Setenv("DETECT_SUSPECT_STREAK_MIN", "9")
t.Setenv("CAP_BREAKER_MIN_N", "15")
m := orchestrate.NewConfigManager(tmpFile, nil)
if err := m.Load(); err != nil {
t.Fatalf("Load: %v", err)
}
cfg := m.Current()
// Env var should override YAML value.
if cfg.Detect.SuspectStreakMin != 9 {
t.Errorf("Detect.SuspectStreakMin = %d; want 9 (env override)", cfg.Detect.SuspectStreakMin)
}
if cfg.Capacity.BreakerMinN != 15 {
t.Errorf("Capacity.BreakerMinN = %d; want 15 (env override)", cfg.Capacity.BreakerMinN)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Test: SCHED_CONFIG_FILE env var overrides path argument
// ─────────────────────────────────────────────────────────────────────────────
func TestConfigManagerSCHED_CONFIG_FILE(t *testing.T) {
yaml := `
capacity:
breaker_window_min: 120
`
tmpFile, err := writeTempYAML(t, yaml)
if err != nil {
t.Fatalf("writeTempYAML: %v", err)
}
t.Setenv("SCHED_CONFIG_FILE", tmpFile)
// Pass a wrong path as arg; env var should win.
m := orchestrate.NewConfigManager("/nonexistent/path.yaml", nil)
if err := m.Load(); err != nil {
t.Fatalf("Load: %v", err)
}
if cfg := m.Current(); cfg.Capacity.BreakerWindowMin != 120 {
t.Errorf("BreakerWindowMin = %d; want 120 (read from SCHED_CONFIG_FILE)", cfg.Capacity.BreakerWindowMin)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Test: Reload changes values and triggers audit callback with diff
// ─────────────────────────────────────────────────────────────────────────────
func TestConfigManagerReloadWithAudit(t *testing.T) {
yaml1 := `
capacity:
watermark_threshold: 0.70
breaker_min_n: 3
`
tmpFile, err := writeTempYAML(t, yaml1)
if err != nil {
t.Fatalf("writeTempYAML: %v", err)
}
var auditCalls []struct{ old, new string }
auditFn := func(ctx context.Context, old, new string) {
auditCalls = append(auditCalls, struct{ old, new string }{old, new})
}
m := orchestrate.NewConfigManager(tmpFile, orchestrate.AuditFn(auditFn))
if err := m.Load(); err != nil {
t.Fatalf("initial Load: %v", err)
}
initialThreshold := m.Current().Capacity.WatermarkThreshold
// Rewrite the YAML with a changed value.
yaml2 := `
capacity:
watermark_threshold: 0.55
breaker_min_n: 5
`
if err := os.WriteFile(tmpFile, []byte(yaml2), 0o644); err != nil {
t.Fatalf("rewrite YAML: %v", err)
}
// Reload should pick up the new values.
if err := m.Reload(context.Background()); err != nil {
t.Fatalf("Reload: %v", err)
}
newThreshold := m.Current().Capacity.WatermarkThreshold
if initialThreshold == newThreshold {
t.Errorf("WatermarkThreshold not updated after reload: both = %f", newThreshold)
}
if newThreshold != 0.55 {
t.Errorf("WatermarkThreshold after reload = %f; want 0.55", newThreshold)
}
if m.Current().Capacity.BreakerMinN != 5 {
t.Errorf("BreakerMinN after reload = %d; want 5", m.Current().Capacity.BreakerMinN)
}
// Audit callback must have been invoked once (old != new).
if len(auditCalls) != 1 {
t.Errorf("audit callback count = %d; want 1", len(auditCalls))
} else {
if auditCalls[0].old == auditCalls[0].new {
t.Error("audit callback old == new; diff expected")
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Test: Reload with no change → audit NOT called (diff is empty)
// ─────────────────────────────────────────────────────────────────────────────
func TestConfigManagerReloadNoChangeSkipsAudit(t *testing.T) {
yaml := `
capacity:
watermark_threshold: 0.75
`
tmpFile, err := writeTempYAML(t, yaml)
if err != nil {
t.Fatalf("writeTempYAML: %v", err)
}
auditCount := 0
auditFn := func(_ context.Context, _, _ string) { auditCount++ }
m := orchestrate.NewConfigManager(tmpFile, orchestrate.AuditFn(auditFn))
if err := m.Load(); err != nil {
t.Fatalf("initial Load: %v", err)
}
// Reload without changing the YAML file.
if err := m.Reload(context.Background()); err != nil {
t.Fatalf("Reload: %v", err)
}
// No diff → audit should NOT be called on the second load.
// (First Load always swaps from defaults to YAML; that may be different.)
// What we really check: the reload didn't trigger another audit if nothing changed.
// The auditFn may have been called 1 time (from Load) but NOT from Reload.
//
// Actually, Load also calls swap. If the defaults differ from the YAML,
// the initial Load increments auditCount. Then Reload sees no diff → no extra call.
countAfterReload := auditCount
// Reload again — still unchanged → same count.
if err := m.Reload(context.Background()); err != nil {
t.Fatalf("second Reload: %v", err)
}
if auditCount != countAfterReload {
t.Errorf("audit called %d times after second reload; expected 0 extra calls (no diff)", auditCount-countAfterReload)
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Test: comma-separated env var for ExpectedProbeIDs
// ─────────────────────────────────────────────────────────────────────────────
func TestConfigManagerProbeIDsFromEnv(t *testing.T) {
t.Setenv("CAP_EXPECTED_PROBE_IDS", "alpha, beta , gamma")
m := orchestrate.NewConfigManager("", nil)
if err := m.Load(); err != nil {
t.Fatalf("Load: %v", err)
}
ids := m.Current().Capacity.ExpectedProbeIDs
if len(ids) != 3 {
t.Fatalf("ExpectedProbeIDs = %v; want 3 entries", ids)
}
want := []string{"alpha", "beta", "gamma"}
for i, w := range want {
if ids[i] != w {
t.Errorf("ids[%d] = %q; want %q", i, ids[i], w)
}
}
}