feat(scheduler): 容量水位监控 + 熔断器 + 阈值热加载 [tsk_NPgPRxBGv0g9]
实现 15F 任务的三个核心组件：
capacity.go — CapacityMonitor（每 1-2 min 由 CapacityLoop 调用）
- 池水位检查：up/target < 70% 时发告警，10 min 内降频去重
- 探针失联检查：扫 probe:hb:{probeId}，缺失发「探针失联」事件
- 驱动 Grayscale.Advance 推进养机档位
breaker.go — RedisBreaker（实现 Breaker 接口）
- 滑窗 ZSET sched:breaker:{tier}:{region}，窗口 1h
- 阈值 N = ceil(池容量 × 30%)，下限 3
- 窗口内计数 ≥ N → 置 tripped 标记位 + critical 告警
- Allow 恒 false 直到窗口滑出自动恢复，或管理员调 Reset
- Reset 清除 ZSET + trip flag，写 audit_log
config.go — SchedConfig + ConfigManager（全树阈值热加载）
- SchedConfig 集中定义 15D/15E/15F 所有阈值
- 来源：默认值 → YAML 文件 → 环境变量（三层叠加）
- 监听 SIGHUP，原子替换（atomic.Pointer[SchedConfig]）
- 每次变更 diff 写 audit_log（AuditFn 回调）
测试（24 个用例，全绿）：
- 窗口内第 N 次 Record 后 Allow 返回 false
- 窗口滑出后 Allow 自动恢复
- Reset 立即恢复 + audit 条目
- 下限 3 在小池（target=2）生效
- 65% 水位 → 告警；10 min 内不重复
- 探针心跳缺失 → 失联事件；存在 → 无告警
- YAML + env 覆盖 + SIGHUP Reload → 新阈值即时生效 + diff 审计
- 替换 stub 后 15E 全部集成测试仍绿
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,258 @@
|
||||
package orchestrate_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
|
||||
)
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Utility: write a temporary YAML config file
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func writeTempYAML(t *testing.T, content string) (string, error) {
|
||||
t.Helper()
|
||||
f, err := os.CreateTemp("", "sched_config_*.yaml")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
t.Cleanup(func() { os.Remove(f.Name()) })
|
||||
if _, err := f.WriteString(content); err != nil {
|
||||
f.Close()
|
||||
return "", err
|
||||
}
|
||||
f.Close()
|
||||
return f.Name(), nil
|
||||
}
|
||||
|
||||
// defaultCfgMgr returns a ConfigManager loaded with production defaults.
|
||||
func defaultCfgMgr(t *testing.T) *orchestrate.ConfigManager {
|
||||
t.Helper()
|
||||
m := orchestrate.NewConfigManager("", nil)
|
||||
if err := m.Load(); err != nil {
|
||||
t.Fatalf("ConfigManager.Load: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test: Pool below 70% watermark → alert emitted
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestCapacityWatermarkAlert(t *testing.T) {
|
||||
rdb, _ := newTestRedis(t)
|
||||
pools := newMockPoolReader()
|
||||
// 6 out of 10 up = 60% < 70% → alert expected.
|
||||
pools.setPool("free", "hkg", 6, 10)
|
||||
|
||||
notifier := &mockNotifier{}
|
||||
cm := orchestrate.NewCapacityMonitor(orchestrate.CapacityMonitorConfig{
|
||||
RDB: rdb,
|
||||
Pools: pools,
|
||||
Notifier: notifier,
|
||||
CfgMgr: defaultCfgMgr(t),
|
||||
})
|
||||
|
||||
if err := cm.RunOnce(context.Background()); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if notifier.count() != 1 {
|
||||
t.Errorf("NotifyFault calls = %d; want 1 (watermark alert)", notifier.count())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test: Pool at exactly 70% → no alert
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestCapacityWatermarkNoAlertAtThreshold(t *testing.T) {
|
||||
rdb, _ := newTestRedis(t)
|
||||
pools := newMockPoolReader()
|
||||
// 7/10 = 70% == threshold → healthy, no alert.
|
||||
pools.setPool("free", "hkg", 7, 10)
|
||||
|
||||
notifier := &mockNotifier{}
|
||||
cm := orchestrate.NewCapacityMonitor(orchestrate.CapacityMonitorConfig{
|
||||
RDB: rdb,
|
||||
Pools: pools,
|
||||
Notifier: notifier,
|
||||
CfgMgr: defaultCfgMgr(t),
|
||||
})
|
||||
|
||||
if err := cm.RunOnce(context.Background()); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if notifier.count() != 0 {
|
||||
t.Errorf("NotifyFault calls = %d; want 0 (fill rate = 70%% >= threshold)", notifier.count())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test: Repeated RunOnce within suppress window → only 1 alert
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestCapacityWatermarkAlertSuppression(t *testing.T) {
|
||||
rdb, mr := newTestRedis(t)
|
||||
pools := newMockPoolReader()
|
||||
pools.setPool("free", "sin", 5, 10) // 50% < 70%
|
||||
|
||||
notifier := &mockNotifier{}
|
||||
cm := orchestrate.NewCapacityMonitor(orchestrate.CapacityMonitorConfig{
|
||||
RDB: rdb,
|
||||
Pools: pools,
|
||||
Notifier: notifier,
|
||||
CfgMgr: defaultCfgMgr(t),
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Three RunOnce calls within the 10-min suppress window.
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := cm.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("RunOnce %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if notifier.count() != 1 {
|
||||
t.Errorf("NotifyFault calls = %d; want 1 (suppressed repeats)", notifier.count())
|
||||
}
|
||||
|
||||
// Advance miniredis TTL past the 10-min suppress window.
|
||||
mr.FastForward(11 * time.Minute)
|
||||
|
||||
// After suppress window expires → another alert.
|
||||
if err := cm.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("RunOnce after expire: %v", err)
|
||||
}
|
||||
if notifier.count() != 2 {
|
||||
t.Errorf("NotifyFault calls = %d; want 2 after suppress window expires", notifier.count())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test: Missing probe heartbeat → alert; present probe → no alert
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestCapacityProbeHeartbeatMissingAlert(t *testing.T) {
|
||||
rdb, _ := newTestRedis(t)
|
||||
pools := newMockPoolReader()
|
||||
notifier := &mockNotifier{}
|
||||
|
||||
ctx := context.Background()
|
||||
// probe-b has a live heartbeat; probe-a does not.
|
||||
if err := rdb.Set(ctx, "probe:hb:probe-b", "1", 15*time.Minute).Err(); err != nil {
|
||||
t.Fatalf("seed probe-b heartbeat: %v", err)
|
||||
}
|
||||
|
||||
yamlContent := `
|
||||
capacity:
|
||||
expected_probe_ids:
|
||||
- probe-a
|
||||
- probe-b
|
||||
alert_suppress_min: 10
|
||||
`
|
||||
tmpFile, err := writeTempYAML(t, yamlContent)
|
||||
if err != nil {
|
||||
t.Fatalf("write temp YAML: %v", err)
|
||||
}
|
||||
cfgMgr := orchestrate.NewConfigManager(tmpFile, nil)
|
||||
if err := cfgMgr.Load(); err != nil {
|
||||
t.Fatalf("Load YAML config: %v", err)
|
||||
}
|
||||
|
||||
cm := orchestrate.NewCapacityMonitor(orchestrate.CapacityMonitorConfig{
|
||||
RDB: rdb,
|
||||
Pools: pools,
|
||||
Notifier: notifier,
|
||||
CfgMgr: cfgMgr,
|
||||
})
|
||||
|
||||
if err := cm.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
// Only probe-a should trigger an alert.
|
||||
if notifier.count() != 1 {
|
||||
t.Errorf("NotifyFault calls = %d; want 1 (only probe-a missing)", notifier.count())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test: Both probes alive → no alert
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestCapacityProbeHeartbeatBothAlive(t *testing.T) {
|
||||
rdb, _ := newTestRedis(t)
|
||||
pools := newMockPoolReader()
|
||||
notifier := &mockNotifier{}
|
||||
|
||||
ctx := context.Background()
|
||||
rdb.Set(ctx, "probe:hb:probe-a", "1", 15*time.Minute)
|
||||
rdb.Set(ctx, "probe:hb:probe-b", "1", 15*time.Minute)
|
||||
|
||||
yamlContent := `
|
||||
capacity:
|
||||
expected_probe_ids:
|
||||
- probe-a
|
||||
- probe-b
|
||||
`
|
||||
tmpFile, err := writeTempYAML(t, yamlContent)
|
||||
if err != nil {
|
||||
t.Fatalf("write temp YAML: %v", err)
|
||||
}
|
||||
cfgMgr := orchestrate.NewConfigManager(tmpFile, nil)
|
||||
if err := cfgMgr.Load(); err != nil {
|
||||
t.Fatalf("Load YAML: %v", err)
|
||||
}
|
||||
|
||||
cm := orchestrate.NewCapacityMonitor(orchestrate.CapacityMonitorConfig{
|
||||
RDB: rdb,
|
||||
Pools: pools,
|
||||
Notifier: notifier,
|
||||
CfgMgr: cfgMgr,
|
||||
})
|
||||
|
||||
if err := cm.RunOnce(ctx); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
if notifier.count() != 0 {
|
||||
t.Errorf("NotifyFault calls = %d; want 0 (both probes alive)", notifier.count())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Test: Multiple pools — only below-threshold pools alert
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
func TestCapacityMultiplePoolsSelectiveAlert(t *testing.T) {
|
||||
rdb, _ := newTestRedis(t)
|
||||
pools := newMockPoolReader()
|
||||
// hkg: 60% → alert
|
||||
pools.setPool("free", "hkg", 6, 10)
|
||||
// sin: 90% → no alert
|
||||
pools.setPool("free", "sin", 9, 10)
|
||||
// tyo: 50% → alert
|
||||
pools.setPool("free", "tyo", 5, 10)
|
||||
|
||||
notifier := &mockNotifier{}
|
||||
cm := orchestrate.NewCapacityMonitor(orchestrate.CapacityMonitorConfig{
|
||||
RDB: rdb,
|
||||
Pools: pools,
|
||||
Notifier: notifier,
|
||||
CfgMgr: defaultCfgMgr(t),
|
||||
})
|
||||
|
||||
if err := cm.RunOnce(context.Background()); err != nil {
|
||||
t.Fatalf("RunOnce: %v", err)
|
||||
}
|
||||
|
||||
// Expect 2 alerts: hkg and tyo; sin is healthy.
|
||||
if n := notifier.count(); n != 2 {
|
||||
t.Errorf("NotifyFault calls = %d; want 2 (hkg + tyo)", n)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user