feat(scheduler): 容量水位监控 + 熔断器 + 阈值热加载 [tsk_NPgPRxBGv0g9]

实现 15F 任务的三个核心组件： capacity.go — CapacityMonitor（每 1-2 min 由 CapacityLoop 调用） - 池水位检查：up/target < 70% 时发告警，10 min 内降频去重 - 探针失联检查：扫 probe:hb:{probeId}，缺失发「探针失联」事件 - 驱动 Grayscale.Advance 推进养机档位 breaker.go — RedisBreaker（实现 Breaker 接口） - 滑窗 ZSET sched:breaker:{tier}:{region}，窗口 1h - 阈值 N = ceil(池容量 × 30%)，下限 3 - 窗口内计数 ≥ N → 置 tripped 标记位 + critical 告警 - Allow 恒 false 直到窗口滑出自动恢复，或管理员调 Reset - Reset 清除 ZSET + trip flag，写 audit_log config.go — SchedConfig + ConfigManager（全树阈值热加载） - SchedConfig 集中定义 15D/15E/15F 所有阈值 - 来源：默认值 → YAML 文件 → 环境变量（三层叠加） - 监听 SIGHUP，原子替换（atomic.Pointer[SchedConfig]） - 每次变更 diff 写 audit_log（AuditFn 回调）测试（24 个用例，全绿）： - 窗口内第 N 次 Record 后 Allow 返回 false - 窗口滑出后 Allow 自动恢复 - Reset 立即恢复 + audit 条目 - 下限 3 在小池（target=2）生效 - 65% 水位 → 告警；10 min 内不重复 - 探针心跳缺失 → 失联事件；存在 → 无告警 - YAML + env 覆盖 + SIGHUP Reload → 新阈值即时生效 + diff 审计 - 替换 stub 后 15E 全部集成测试仍绿 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 00:31:58 +08:00
parent cadd527680
commit 3d9ccff6e9
6 changed files with 1793 additions and 0 deletions
@@ -0,0 +1,295 @@
+package orchestrate_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Mock PoolReader
+// ─────────────────────────────────────────────────────────────────────────────
+
+// mockPoolReader is a simple in-memory PoolReader for tests.
+type mockPoolReader struct {
+	mu    sync.Mutex
+	pools map[string]orchestrate.PoolStat // key = tier+":"+region
+}
+
+func newMockPoolReader() *mockPoolReader {
+	return &mockPoolReader{pools: make(map[string]orchestrate.PoolStat)}
+}
+
+func (r *mockPoolReader) setPool(tier, region string, up, target int) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.pools[tier+":"+region] = orchestrate.PoolStat{
+		Tier: tier, Region: region, UpCount: up, Target: target,
+	}
+}
+
+func (r *mockPoolReader) AllPools(_ context.Context) ([]orchestrate.PoolStat, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	out := make([]orchestrate.PoolStat, 0, len(r.pools))
+	for _, s := range r.pools {
+		out = append(out, s)
+	}
+	return out, nil
+}
+
+func (r *mockPoolReader) PoolCapacity(_ context.Context, tier, region string) (int, int, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	s, ok := r.pools[tier+":"+region]
+	if !ok {
+		return 0, 0, nil
+	}
+	return s.UpCount, s.Target, nil
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helper: build a RedisBreaker for tests
+// ─────────────────────────────────────────────────────────────────────────────
+
+func newTestBreaker(t *testing.T, pools *mockPoolReader, notifier *mockNotifier, clock *mockClock) (*orchestrate.RedisBreaker, *orchestrate.ConfigManager) {
+	t.Helper()
+	rdb, _ := newTestRedis(t)
+	cfgMgr := orchestrate.NewConfigManager("", nil)
+	_ = cfgMgr.Load()
+
+	return orchestrate.NewRedisBreaker(orchestrate.RedisBreakerConfig{
+		RDB:      rdb,
+		Pools:    pools,
+		Notifier: notifier,
+		CfgMgr:   cfgMgr,
+		Clock:    clock,
+	}), cfgMgr
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: N-th Record trips the breaker; (N+1)-th Allow returns false
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestBreakerTripsAtThreshold(t *testing.T) {
+	const (
+		tier   = "free"
+		region = "hkg"
+	)
+
+	pools := newMockPoolReader()
+	// Pool of 10 nodes → threshold = ceil(10 × 30%) = 3.
+	pools.setPool(tier, region, 10, 10)
+
+	notifier := &mockNotifier{}
+	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())
+
+	b, _ := newTestBreaker(t, pools, notifier, clock)
+
+	// Before any records: Allow should return true.
+	if !b.Allow(tier, region) {
+		t.Fatal("Allow should return true before any records")
+	}
+
+	// Record N-1 replacements (threshold-1 = 2); Allow should still be true.
+	b.Record(tier, region) // count = 1
+	b.Record(tier, region) // count = 2
+	if !b.Allow(tier, region) {
+		t.Fatalf("Allow should return true with count < threshold (count=2, threshold=3)")
+	}
+
+	// Record the N-th replacement; now count = 3 = threshold.
+	b.Record(tier, region) // count = 3
+
+	// (N+1)-th Allow: breaker is tripped → false.
+	if b.Allow(tier, region) {
+		t.Error("Allow should return false when count >= threshold (tripped)")
+	}
+
+	// Critical alert must have been emitted exactly once.
+	if n := notifier.count(); n != 1 {
+		t.Errorf("NotifyFault calls = %d; want 1 (on trip)", n)
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: Window slide-out auto-recovers the breaker
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestBreakerAutoRecoveryAfterWindowExpiry(t *testing.T) {
+	const (
+		tier   = "premium"
+		region = "sin"
+	)
+
+	pools := newMockPoolReader()
+	// Pool of 10 → threshold = 3.
+	pools.setPool(tier, region, 10, 10)
+
+	notifier := &mockNotifier{}
+	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())
+	b, _ := newTestBreaker(t, pools, notifier, clock)
+
+	// Record N entries to trip.
+	b.Record(tier, region)
+	b.Record(tier, region)
+	b.Record(tier, region)
+	if b.Allow(tier, region) {
+		t.Fatal("breaker should be tripped after 3 records (threshold=3)")
+	}
+
+	// Advance clock past the 1-h window; all ZSET entries are now stale.
+	clock.advance(61 * time.Minute)
+
+	// Allow should return true: ZREMRANGEBYSCORE removes stale entries,
+	// count drops to 0 < threshold, and the trip flag TTL has expired.
+	if !b.Allow(tier, region) {
+		t.Error("Allow should return true after the window has expired (auto-recovery)")
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: Reset manually clears the breaker with audit
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestBreakerResetRestoresAllow(t *testing.T) {
+	const (
+		tier   = "free"
+		region = "tyo"
+	)
+
+	pools := newMockPoolReader()
+	pools.setPool(tier, region, 10, 10) // threshold = 3
+
+	notifier := &mockNotifier{}
+	lc := newMockLC()
+	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())
+	rdb, _ := newTestRedis(t)
+	cfgMgr := orchestrate.NewConfigManager("", nil)
+	_ = cfgMgr.Load()
+
+	b := orchestrate.NewRedisBreaker(orchestrate.RedisBreakerConfig{
+		RDB:      rdb,
+		Pools:    pools,
+		Notifier: notifier,
+		LC:       lc,
+		CfgMgr:   cfgMgr,
+		Clock:    clock,
+	})
+
+	// Trip the breaker.
+	b.Record(tier, region)
+	b.Record(tier, region)
+	b.Record(tier, region)
+	if b.Allow(tier, region) {
+		t.Fatal("breaker should be tripped")
+	}
+
+	// Reset.
+	ctx := context.Background()
+	if err := b.Reset(ctx, tier, region, "admin-alice"); err != nil {
+		t.Fatalf("Reset error: %v", err)
+	}
+
+	// Allow should now return true.
+	if !b.Allow(tier, region) {
+		t.Error("Allow should return true after Reset")
+	}
+
+	// Audit log must mention the reset.
+	if !lc.hasAudit("admin-alice|breaker_reset|pool:" + tier + ":" + region) {
+		t.Errorf("audit log missing breaker_reset entry; got: %v", lc.auditLogs)
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: BreakerMinN = 3 enforced on tiny pool (target < 10)
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestBreakerMinNEnforcedOnSmallPool(t *testing.T) {
+	const (
+		tier   = "free"
+		region = "fra"
+	)
+
+	pools := newMockPoolReader()
+	// Pool of 2 nodes; 30% of 2 = 0.6 → ceil = 1, but min = 3.
+	pools.setPool(tier, region, 2, 2)
+
+	notifier := &mockNotifier{}
+	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())
+	b, _ := newTestBreaker(t, pools, notifier, clock)
+
+	// 2 records should NOT trip (threshold = 3, not 1).
+	b.Record(tier, region)
+	b.Record(tier, region)
+	if !b.Allow(tier, region) {
+		t.Error("Allow should return true: count=2, threshold=min(3)")
+	}
+
+	// 3rd record trips.
+	b.Record(tier, region)
+	if b.Allow(tier, region) {
+		t.Error("Allow should return false: count=3 >= minN(3)")
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: Trip alert is emitted only once per window epoch (dedup)
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestBreakerAlertSentOnlyOnce(t *testing.T) {
+	const (
+		tier   = "free"
+		region = "ams"
+	)
+
+	pools := newMockPoolReader()
+	pools.setPool(tier, region, 10, 10) // threshold = 3
+
+	notifier := &mockNotifier{}
+	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())
+	b, _ := newTestBreaker(t, pools, notifier, clock)
+
+	// Trip.
+	b.Record(tier, region)
+	b.Record(tier, region)
+	b.Record(tier, region)
+
+	// Call Allow multiple times while tripped.
+	for i := 0; i < 5; i++ {
+		b.Allow(tier, region)
+	}
+
+	// NotifyFault must have been called exactly once (from Record reaching N).
+	if n := notifier.count(); n != 1 {
+		t.Errorf("NotifyFault calls = %d; want exactly 1 (trip alert dedup)", n)
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Test: pool not found → falls back to BreakerMinN (3)
+// ─────────────────────────────────────────────────────────────────────────────
+
+func TestBreakerFallsBackToMinNForUnknownPool(t *testing.T) {
+	pools := newMockPoolReader()
+	// No pool registered for "unknown"/"xyz".
+
+	notifier := &mockNotifier{}
+	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())
+	b, _ := newTestBreaker(t, pools, notifier, clock)
+
+	b.Record("unknown", "xyz")
+	b.Record("unknown", "xyz")
+	if !b.Allow("unknown", "xyz") {
+		t.Error("Allow should return true: count=2, threshold=minN(3) for unknown pool")
+	}
+
+	b.Record("unknown", "xyz")
+	if b.Allow("unknown", "xyz") {
+		t.Error("Allow should return false: count=3 >= minN(3)")
+	}
+}