3d9ccff6e9
实现 15F 任务的三个核心组件:
capacity.go — CapacityMonitor(每 1-2 min 由 CapacityLoop 调用)
- 池水位检查:up/target < 70% 时发告警,10 min 内降频去重
- 探针失联检查:扫 probe:hb:{probeId},缺失发「探针失联」事件
- 驱动 Grayscale.Advance 推进养机档位
breaker.go — RedisBreaker(实现 Breaker 接口)
- 滑窗 ZSET sched:breaker:{tier}:{region},窗口 1h
- 阈值 N = ceil(池容量 × 30%),下限 3
- 窗口内计数 ≥ N → 置 tripped 标记位 + critical 告警
- Allow 恒 false 直到窗口滑出自动恢复,或管理员调 Reset
- Reset 清除 ZSET + trip flag,写 audit_log
config.go — SchedConfig + ConfigManager(全树阈值热加载)
- SchedConfig 集中定义 15D/15E/15F 所有阈值
- 来源:默认值 → YAML 文件 → 环境变量(三层叠加)
- 监听 SIGHUP,原子替换(atomic.Pointer[SchedConfig])
- 每次变更 diff 写 audit_log(AuditFn 回调)
测试(24 个用例,全绿):
- 窗口内第 N 次 Record 后 Allow 返回 false
- 窗口滑出后 Allow 自动恢复
- Reset 立即恢复 + audit 条目
- 下限 3 在小池(target=2)生效
- 65% 水位 → 告警;10 min 内不重复
- 探针心跳缺失 → 失联事件;存在 → 无告警
- YAML + env 覆盖 + SIGHUP Reload → 新阈值即时生效 + diff 审计
- 替换 stub 后 15E 全部集成测试仍绿
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
290 lines
11 KiB
Go
290 lines
11 KiB
Go
package orchestrate_test
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"testing"
|
|
|
|
"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
|
|
)
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: Default values
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestDefaultSchedConfig(t *testing.T) {
|
|
cfg := orchestrate.DefaultSchedConfig()
|
|
|
|
// Spot-check a representative value from each section.
|
|
if cfg.Detect.DomesticFailNumerator != 2 {
|
|
t.Errorf("Detect.DomesticFailNumerator = %d; want 2", cfg.Detect.DomesticFailNumerator)
|
|
}
|
|
if cfg.Capacity.WatermarkThreshold != 0.70 {
|
|
t.Errorf("Capacity.WatermarkThreshold = %f; want 0.70", cfg.Capacity.WatermarkThreshold)
|
|
}
|
|
if cfg.Capacity.BreakerMinN != 3 {
|
|
t.Errorf("Capacity.BreakerMinN = %d; want 3", cfg.Capacity.BreakerMinN)
|
|
}
|
|
if cfg.Probe.GrayscaleIntervalHours != 6 {
|
|
t.Errorf("Probe.GrayscaleIntervalHours = %d; want 6", cfg.Probe.GrayscaleIntervalHours)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: YAML file overrides defaults
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestConfigManagerYAMLOverrides(t *testing.T) {
|
|
yaml := `
|
|
detect:
|
|
suspect_streak_min: 5
|
|
confirmed_streak_min: 12
|
|
capacity:
|
|
watermark_threshold: 0.60
|
|
breaker_min_n: 5
|
|
expected_probe_ids:
|
|
- pb-1
|
|
- pb-2
|
|
probe:
|
|
grayscale_interval_hours: 8
|
|
max_attempts: 4
|
|
`
|
|
tmpFile, err := writeTempYAML(t, yaml)
|
|
if err != nil {
|
|
t.Fatalf("writeTempYAML: %v", err)
|
|
}
|
|
|
|
m := orchestrate.NewConfigManager(tmpFile, nil)
|
|
if err := m.Load(); err != nil {
|
|
t.Fatalf("Load: %v", err)
|
|
}
|
|
|
|
cfg := m.Current()
|
|
|
|
if cfg.Detect.SuspectStreakMin != 5 {
|
|
t.Errorf("Detect.SuspectStreakMin = %d; want 5", cfg.Detect.SuspectStreakMin)
|
|
}
|
|
if cfg.Detect.ConfirmedStreakMin != 12 {
|
|
t.Errorf("Detect.ConfirmedStreakMin = %d; want 12", cfg.Detect.ConfirmedStreakMin)
|
|
}
|
|
if cfg.Capacity.WatermarkThreshold != 0.60 {
|
|
t.Errorf("Capacity.WatermarkThreshold = %f; want 0.60", cfg.Capacity.WatermarkThreshold)
|
|
}
|
|
if cfg.Capacity.BreakerMinN != 5 {
|
|
t.Errorf("Capacity.BreakerMinN = %d; want 5", cfg.Capacity.BreakerMinN)
|
|
}
|
|
if len(cfg.Capacity.ExpectedProbeIDs) != 2 {
|
|
t.Errorf("ExpectedProbeIDs count = %d; want 2", len(cfg.Capacity.ExpectedProbeIDs))
|
|
} else {
|
|
if cfg.Capacity.ExpectedProbeIDs[0] != "pb-1" || cfg.Capacity.ExpectedProbeIDs[1] != "pb-2" {
|
|
t.Errorf("ExpectedProbeIDs = %v; want [pb-1 pb-2]", cfg.Capacity.ExpectedProbeIDs)
|
|
}
|
|
}
|
|
if cfg.Probe.GrayscaleIntervalHours != 8 {
|
|
t.Errorf("Probe.GrayscaleIntervalHours = %d; want 8", cfg.Probe.GrayscaleIntervalHours)
|
|
}
|
|
if cfg.Probe.MaxAttempts != 4 {
|
|
t.Errorf("Probe.MaxAttempts = %d; want 4", cfg.Probe.MaxAttempts)
|
|
}
|
|
|
|
// Unset fields must retain defaults.
|
|
if cfg.Detect.DomesticFailNumerator != 2 {
|
|
t.Errorf("Detect.DomesticFailNumerator should be default 2; got %d", cfg.Detect.DomesticFailNumerator)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: Environment variable overrides YAML
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestConfigManagerEnvOverridesYAML(t *testing.T) {
|
|
yaml := `
|
|
detect:
|
|
suspect_streak_min: 5
|
|
capacity:
|
|
breaker_min_n: 7
|
|
`
|
|
tmpFile, err := writeTempYAML(t, yaml)
|
|
if err != nil {
|
|
t.Fatalf("writeTempYAML: %v", err)
|
|
}
|
|
|
|
// Set env vars AFTER writing YAML to ensure they override.
|
|
t.Setenv("DETECT_SUSPECT_STREAK_MIN", "9")
|
|
t.Setenv("CAP_BREAKER_MIN_N", "15")
|
|
|
|
m := orchestrate.NewConfigManager(tmpFile, nil)
|
|
if err := m.Load(); err != nil {
|
|
t.Fatalf("Load: %v", err)
|
|
}
|
|
|
|
cfg := m.Current()
|
|
|
|
// Env var should override YAML value.
|
|
if cfg.Detect.SuspectStreakMin != 9 {
|
|
t.Errorf("Detect.SuspectStreakMin = %d; want 9 (env override)", cfg.Detect.SuspectStreakMin)
|
|
}
|
|
if cfg.Capacity.BreakerMinN != 15 {
|
|
t.Errorf("Capacity.BreakerMinN = %d; want 15 (env override)", cfg.Capacity.BreakerMinN)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: SCHED_CONFIG_FILE env var overrides path argument
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestConfigManagerSCHED_CONFIG_FILE(t *testing.T) {
|
|
yaml := `
|
|
capacity:
|
|
breaker_window_min: 120
|
|
`
|
|
tmpFile, err := writeTempYAML(t, yaml)
|
|
if err != nil {
|
|
t.Fatalf("writeTempYAML: %v", err)
|
|
}
|
|
|
|
t.Setenv("SCHED_CONFIG_FILE", tmpFile)
|
|
|
|
// Pass a wrong path as arg; env var should win.
|
|
m := orchestrate.NewConfigManager("/nonexistent/path.yaml", nil)
|
|
if err := m.Load(); err != nil {
|
|
t.Fatalf("Load: %v", err)
|
|
}
|
|
|
|
if cfg := m.Current(); cfg.Capacity.BreakerWindowMin != 120 {
|
|
t.Errorf("BreakerWindowMin = %d; want 120 (read from SCHED_CONFIG_FILE)", cfg.Capacity.BreakerWindowMin)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: Reload changes values and triggers audit callback with diff
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestConfigManagerReloadWithAudit(t *testing.T) {
|
|
yaml1 := `
|
|
capacity:
|
|
watermark_threshold: 0.70
|
|
breaker_min_n: 3
|
|
`
|
|
tmpFile, err := writeTempYAML(t, yaml1)
|
|
if err != nil {
|
|
t.Fatalf("writeTempYAML: %v", err)
|
|
}
|
|
|
|
var auditCalls []struct{ old, new string }
|
|
auditFn := func(ctx context.Context, old, new string) {
|
|
auditCalls = append(auditCalls, struct{ old, new string }{old, new})
|
|
}
|
|
|
|
m := orchestrate.NewConfigManager(tmpFile, orchestrate.AuditFn(auditFn))
|
|
if err := m.Load(); err != nil {
|
|
t.Fatalf("initial Load: %v", err)
|
|
}
|
|
|
|
initialThreshold := m.Current().Capacity.WatermarkThreshold
|
|
|
|
// Rewrite the YAML with a changed value.
|
|
yaml2 := `
|
|
capacity:
|
|
watermark_threshold: 0.55
|
|
breaker_min_n: 5
|
|
`
|
|
if err := os.WriteFile(tmpFile, []byte(yaml2), 0o644); err != nil {
|
|
t.Fatalf("rewrite YAML: %v", err)
|
|
}
|
|
|
|
// Reload should pick up the new values.
|
|
if err := m.Reload(context.Background()); err != nil {
|
|
t.Fatalf("Reload: %v", err)
|
|
}
|
|
|
|
newThreshold := m.Current().Capacity.WatermarkThreshold
|
|
|
|
if initialThreshold == newThreshold {
|
|
t.Errorf("WatermarkThreshold not updated after reload: both = %f", newThreshold)
|
|
}
|
|
if newThreshold != 0.55 {
|
|
t.Errorf("WatermarkThreshold after reload = %f; want 0.55", newThreshold)
|
|
}
|
|
if m.Current().Capacity.BreakerMinN != 5 {
|
|
t.Errorf("BreakerMinN after reload = %d; want 5", m.Current().Capacity.BreakerMinN)
|
|
}
|
|
|
|
// Audit callback must have been invoked once (old != new).
|
|
if len(auditCalls) != 1 {
|
|
t.Errorf("audit callback count = %d; want 1", len(auditCalls))
|
|
} else {
|
|
if auditCalls[0].old == auditCalls[0].new {
|
|
t.Error("audit callback old == new; diff expected")
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: Reload with no change → audit NOT called (diff is empty)
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestConfigManagerReloadNoChangeSkipsAudit(t *testing.T) {
|
|
yaml := `
|
|
capacity:
|
|
watermark_threshold: 0.75
|
|
`
|
|
tmpFile, err := writeTempYAML(t, yaml)
|
|
if err != nil {
|
|
t.Fatalf("writeTempYAML: %v", err)
|
|
}
|
|
|
|
auditCount := 0
|
|
auditFn := func(_ context.Context, _, _ string) { auditCount++ }
|
|
|
|
m := orchestrate.NewConfigManager(tmpFile, orchestrate.AuditFn(auditFn))
|
|
if err := m.Load(); err != nil {
|
|
t.Fatalf("initial Load: %v", err)
|
|
}
|
|
|
|
// Reload without changing the YAML file.
|
|
if err := m.Reload(context.Background()); err != nil {
|
|
t.Fatalf("Reload: %v", err)
|
|
}
|
|
|
|
// No diff → audit should NOT be called on the second load.
|
|
// (First Load always swaps from defaults to YAML; that may be different.)
|
|
// What we really check: the reload didn't trigger another audit if nothing changed.
|
|
// The auditFn may have been called 1 time (from Load) but NOT from Reload.
|
|
//
|
|
// Actually, Load also calls swap. If the defaults differ from the YAML,
|
|
// the initial Load increments auditCount. Then Reload sees no diff → no extra call.
|
|
countAfterReload := auditCount
|
|
|
|
// Reload again — still unchanged → same count.
|
|
if err := m.Reload(context.Background()); err != nil {
|
|
t.Fatalf("second Reload: %v", err)
|
|
}
|
|
if auditCount != countAfterReload {
|
|
t.Errorf("audit called %d times after second reload; expected 0 extra calls (no diff)", auditCount-countAfterReload)
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
// Test: comma-separated env var for ExpectedProbeIDs
|
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
func TestConfigManagerProbeIDsFromEnv(t *testing.T) {
|
|
t.Setenv("CAP_EXPECTED_PROBE_IDS", "alpha, beta , gamma")
|
|
|
|
m := orchestrate.NewConfigManager("", nil)
|
|
if err := m.Load(); err != nil {
|
|
t.Fatalf("Load: %v", err)
|
|
}
|
|
ids := m.Current().Capacity.ExpectedProbeIDs
|
|
if len(ids) != 3 {
|
|
t.Fatalf("ExpectedProbeIDs = %v; want 3 entries", ids)
|
|
}
|
|
want := []string{"alpha", "beta", "gamma"}
|
|
for i, w := range want {
|
|
if ids[i] != w {
|
|
t.Errorf("ids[%d] = %q; want %q", i, ids[i], w)
|
|
}
|
|
}
|
|
}
|