feat(scheduler): 容量水位监控 + 熔断器 + 阈值热加载 [tsk_NPgPRxBGv0g9]

实现 15F 任务的三个核心组件：

capacity.go — CapacityMonitor（每 1-2 min 由 CapacityLoop 调用）
  - 池水位检查：up/target < 70% 时发告警，10 min 内降频去重
  - 探针失联检查：扫 probe:hb:{probeId}，缺失发「探针失联」事件
  - 驱动 Grayscale.Advance 推进养机档位

breaker.go — RedisBreaker（实现 Breaker 接口）
  - 滑窗 ZSET sched:breaker:{tier}:{region}，窗口 1h
  - 阈值 N = ceil(池容量 × 30%)，下限 3
  - 窗口内计数 ≥ N → 置 tripped 标记位 + critical 告警
  - Allow 恒 false 直到窗口滑出自动恢复，或管理员调 Reset
  - Reset 清除 ZSET + trip flag，写 audit_log

config.go — SchedConfig + ConfigManager（全树阈值热加载）
  - SchedConfig 集中定义 15D/15E/15F 所有阈值
  - 来源：默认值 → YAML 文件 → 环境变量（三层叠加）
  - 监听 SIGHUP，原子替换（atomic.Pointer[SchedConfig]）
  - 每次变更 diff 写 audit_log（AuditFn 回调）

测试（24 个用例，全绿）：
  - 窗口内第 N 次 Record 后 Allow 返回 false
  - 窗口滑出后 Allow 自动恢复
  - Reset 立即恢复 + audit 条目
  - 下限 3 在小池（target=2）生效
  - 65% 水位 → 告警；10 min 内不重复
  - 探针心跳缺失 → 失联事件；存在 → 无告警
  - YAML + env 覆盖 + SIGHUP Reload → 新阈值即时生效 + diff 审计
  - 替换 stub 后 15E 全部集成测试仍绿

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

server/internal/scheduler/orchestrate/config.go

@@ -0,0 +1,402 @@
+package orchestrate
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"syscall"
+
+	"gopkg.in/yaml.v3"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// SchedConfig — master configuration tree
+// ─────────────────────────────────────────────────────────────────────────────
+
+// SchedConfig is the single source of truth for all scheduler thresholds.
+// It covers: 15D detection, 15F capacity/breaker, and 15E probe/grayscale.
+//
+// Values are layered in order:
+//  1. Defaults (DefaultSchedConfig).
+//  2. Optional YAML file (path set via SCHED_CONFIG_FILE env var, or passed
+//     to NewConfigManager).
+//  3. Environment variable overrides (applied on top of YAML).
+//
+// The live config is accessed via ConfigManager.Current() — a lock-free
+// atomic.Pointer read. Hot reload is triggered by SIGHUP; each reload
+// computes a JSON diff and calls the provided AuditFn.
+type SchedConfig struct {
+	// Detect holds 15D detection-engine thresholds.
+	Detect DetectSection `yaml:"detect"`
+	// Capacity holds 15F capacity-monitor and breaker thresholds.
+	Capacity CapacitySection `yaml:"capacity"`
+	// Probe holds 15E probe-window and grayscale-ramp settings.
+	Probe ProbeSection `yaml:"probe"`
+}
+
+// DetectSection mirrors detect.DetectConfig; task 15H bridges the two at
+// wiring time.
+type DetectSection struct {
+	// DomesticFailNumerator / DomesticFailDenominator define the ISP-failure
+	// fraction that triggers the suspect rule (default: 2/3).
+	DomesticFailNumerator   int `yaml:"domestic_fail_numerator"`
+	DomesticFailDenominator int `yaml:"domestic_fail_denominator"`
+
+	// SuspectStreakMin is the number of consecutive failing cycles before a
+	// node enters blocked_suspect (default: 2).
+	SuspectStreakMin int `yaml:"suspect_streak_min"`
+
+	// TrafficDropThreshold is the minimum percentage drop in online connections
+	// over 15 min that activates the traffic-warning rule (default: 80.0).
+	TrafficDropThreshold float64 `yaml:"traffic_drop_threshold"`
+
+	// TrafficBaselineMin is the minimum current online-connection count for
+	// the traffic-warning relaxation to apply (default: 20).
+	TrafficBaselineMin int `yaml:"traffic_baseline_min"`
+
+	// ConfirmedStreakMin is the number of consecutive cycles in blocked_suspect
+	// before promotion to blocked_confirmed (default: 6).
+	ConfirmedStreakMin int `yaml:"confirmed_streak_min"`
+
+	// RecoverStreakMin is the number of consecutive passing cycles while in
+	// blocked_suspect required to recover to up (default: 2).
+	RecoverStreakMin int `yaml:"recover_streak_min"`
+
+	// SuspectWeight is the routing weight applied when a node first enters
+	// blocked_suspect (default: 10).
+	SuspectWeight int `yaml:"suspect_weight"`
+}
+
+// CapacitySection holds 15F capacity-monitor and circuit-breaker thresholds.
+type CapacitySection struct {
+	// WatermarkThreshold is the minimum pool fill-rate (upCount/target) before
+	// an alert is emitted (default: 0.70 = 70%).
+	WatermarkThreshold float64 `yaml:"watermark_threshold"`
+
+	// AlertSuppressMin is the minimum gap in minutes between repeated alerts
+	// for the same pool or probe (default: 10 min).
+	AlertSuppressMin int `yaml:"alert_suppress_min"`
+
+	// BreakerWindowMin is the sliding-window duration in minutes for the
+	// circuit breaker (default: 60 min = 1 h).
+	BreakerWindowMin int `yaml:"breaker_window_min"`
+
+	// BreakerFractionPct is the percentage of pool target capacity that, when
+	// replaced within one window, trips the breaker (default: 30).
+	BreakerFractionPct int `yaml:"breaker_fraction_pct"`
+
+	// BreakerMinN is the absolute lower bound for the breaker trip threshold
+	// (default: 3). Overrides fraction when fraction yields a smaller number.
+	BreakerMinN int `yaml:"breaker_min_n"`
+
+	// ManualAlertThresh is the consecutive probe-failure count for a single
+	// node that triggers a 转人工 (escalate-to-human) alert. Should match
+	// 15E's MaxAttempts (default: 3).
+	ManualAlertThresh int `yaml:"manual_alert_thresh"`
+
+	// ExpectedProbeIDs is the list of probe-agent IDs that must maintain live
+	// heartbeat keys in Redis. A missing key triggers a 「探针失联」alert.
+	ExpectedProbeIDs []string `yaml:"expected_probe_ids"`
+}
+
+// ProbeSection holds 15E probe-window and grayscale-ramp settings.
+type ProbeSection struct {
+	// GrayscaleIntervalHours is the time in hours between successive grayscale
+	// weight-ramp steps (default: 6 h).
+	GrayscaleIntervalHours int `yaml:"grayscale_interval_hours"`
+
+	// ProbeTimeoutMin is the maximum time in minutes allowed for a node to
+	// pass probing per attempt (default: 15 min).
+	ProbeTimeoutMin int `yaml:"probe_timeout_min"`
+
+	// MaxAttempts is the maximum number of create+probe attempts per
+	// replacement record before marking it failed (default: 3).
+	MaxAttempts int `yaml:"max_attempts"`
+
+	// ProbeCyclesRequired is the number of consecutive passing probe ticks
+	// needed before a new node is promoted to up (default: 2).
+	ProbeCyclesRequired int `yaml:"probe_cycles_required"`
+}
+
+// DefaultSchedConfig returns a SchedConfig pre-filled with production defaults.
+func DefaultSchedConfig() SchedConfig {
+	return SchedConfig{
+		Detect: DetectSection{
+			DomesticFailNumerator:   2,
+			DomesticFailDenominator: 3,
+			SuspectStreakMin:        2,
+			TrafficDropThreshold:    80.0,
+			TrafficBaselineMin:      20,
+			ConfirmedStreakMin:       6,
+			RecoverStreakMin:        2,
+			SuspectWeight:           10,
+		},
+		Capacity: CapacitySection{
+			WatermarkThreshold: 0.70,
+			AlertSuppressMin:   10,
+			BreakerWindowMin:   60,
+			BreakerFractionPct: 30,
+			BreakerMinN:        3,
+			ManualAlertThresh:  3,
+		},
+		Probe: ProbeSection{
+			GrayscaleIntervalHours: 6,
+			ProbeTimeoutMin:        15,
+			MaxAttempts:            3,
+			ProbeCyclesRequired:    2,
+		},
+	}
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// ConfigManager
+// ─────────────────────────────────────────────────────────────────────────────
+
+// AuditFn is called after each successful config reload with the old and new
+// configurations serialised as JSON strings. The caller is responsible for
+// deciding how to persist the diff (e.g. via LifecycleService.WriteAuditLog).
+// May be nil to skip auditing.
+type AuditFn func(ctx context.Context, oldJSON, newJSON string)
+
+// ConfigManager holds the live SchedConfig and supports SIGHUP-triggered
+// hot reload. Config reads are lock-free (atomic.Pointer).
+//
+// Usage:
+//
+//	mgr := NewConfigManager("/etc/pangolin/sched.yaml", auditFn)
+//	if err := mgr.Load(); err != nil { log.Fatal(err) }
+//	mgr.WatchSIGHUP(ctx)
+//	cfg := mgr.Current()
+type ConfigManager struct {
+	path    string              // optional YAML file path (may be empty)
+	cfg     atomic.Pointer[SchedConfig]
+	auditFn AuditFn
+}
+
+// NewConfigManager creates a ConfigManager.
+//
+//   path:    optional path to the YAML config file. If SCHED_CONFIG_FILE is
+//            set in the environment it overrides this argument.
+//   auditFn: called on every reload when the config changes; may be nil.
+//
+// The manager is seeded with DefaultSchedConfig so Current() is never nil.
+func NewConfigManager(path string, fn AuditFn) *ConfigManager {
+	if p := os.Getenv("SCHED_CONFIG_FILE"); p != "" {
+		path = p
+	}
+	m := &ConfigManager{path: path, auditFn: fn}
+	dflt := DefaultSchedConfig()
+	m.cfg.Store(&dflt)
+	return m
+}
+
+// Current returns the live config. Never nil.
+func (m *ConfigManager) Current() *SchedConfig {
+	return m.cfg.Load()
+}
+
+// Load reads the config (YAML file if configured, then env overrides) and
+// atomically replaces the live config. Safe to call multiple times.
+func (m *ConfigManager) Load() error {
+	next, err := loadSchedConfig(m.path)
+	if err != nil {
+		return err
+	}
+	m.swap(context.Background(), next)
+	return nil
+}
+
+// WatchSIGHUP starts a goroutine that calls Reload whenever SIGHUP is
+// received. The goroutine stops when ctx is cancelled.
+func (m *ConfigManager) WatchSIGHUP(ctx context.Context) {
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP)
+	go func() {
+		defer signal.Stop(ch)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ch:
+				if err := m.Reload(ctx); err != nil {
+					slog.Error("sched config: SIGHUP reload failed", "error", err)
+				} else {
+					slog.Info("sched config: reloaded via SIGHUP")
+				}
+			}
+		}
+	}()
+}
+
+// Reload re-reads the config and hot-swaps the live value. Idempotent.
+func (m *ConfigManager) Reload(ctx context.Context) error {
+	next, err := loadSchedConfig(m.path)
+	if err != nil {
+		return err
+	}
+	m.swap(ctx, next)
+	return nil
+}
+
+// swap atomically replaces the live config and records a diff in the audit
+// log when the config actually changed.
+func (m *ConfigManager) swap(ctx context.Context, next *SchedConfig) {
+	old := m.cfg.Swap(next)
+	if m.auditFn == nil {
+		return
+	}
+	oldJSON, _ := json.Marshal(old)
+	newJSON, _ := json.Marshal(next)
+	if string(oldJSON) == string(newJSON) {
+		return // no change; skip audit
+	}
+	m.auditFn(ctx, string(oldJSON), string(newJSON))
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Loading logic
+// ─────────────────────────────────────────────────────────────────────────────
+
+// loadSchedConfig builds a SchedConfig using the layering strategy:
+// defaults → YAML file → env-var overrides.
+func loadSchedConfig(path string) (*SchedConfig, error) {
+	cfg := DefaultSchedConfig()
+
+	// Layer 2: optional YAML file.
+	if path != "" {
+		data, err := os.ReadFile(path)
+		if err != nil && !os.IsNotExist(err) {
+			return nil, fmt.Errorf("sched config: read %s: %w", path, err)
+		}
+		if err == nil {
+			if err := yaml.Unmarshal(data, &cfg); err != nil {
+				return nil, fmt.Errorf("sched config: parse %s: %w", path, err)
+			}
+		}
+	}
+
+	// Layer 3: environment variable overrides.
+	applySchedEnvOverrides(&cfg)
+
+	return &cfg, nil
+}
+
+// applySchedEnvOverrides overlays environment-variable values on top of cfg.
+// Only non-empty env vars are applied; empty vars preserve the current value.
+func applySchedEnvOverrides(cfg *SchedConfig) {
+	// ── 15D detect ────────────────────────────────────────────────────────────
+	if v := posIntEnv("DETECT_DOMESTIC_FAIL_NUM"); v > 0 {
+		cfg.Detect.DomesticFailNumerator = v
+	}
+	if v := posIntEnv("DETECT_DOMESTIC_FAIL_DEN"); v > 0 {
+		cfg.Detect.DomesticFailDenominator = v
+	}
+	if v := posIntEnv("DETECT_SUSPECT_STREAK_MIN"); v > 0 {
+		cfg.Detect.SuspectStreakMin = v
+	}
+	if v := posFloatEnv("DETECT_TRAFFIC_DROP_THRESHOLD"); v > 0 {
+		cfg.Detect.TrafficDropThreshold = v
+	}
+	if v := posIntEnv("DETECT_TRAFFIC_BASELINE_MIN"); v > 0 {
+		cfg.Detect.TrafficBaselineMin = v
+	}
+	if v := posIntEnv("DETECT_CONFIRMED_STREAK_MIN"); v > 0 {
+		cfg.Detect.ConfirmedStreakMin = v
+	}
+	if v := posIntEnv("DETECT_RECOVER_STREAK_MIN"); v > 0 {
+		cfg.Detect.RecoverStreakMin = v
+	}
+	if v := posIntEnv("DETECT_SUSPECT_WEIGHT"); v > 0 {
+		cfg.Detect.SuspectWeight = v
+	}
+
+	// ── 15F capacity / breaker ────────────────────────────────────────────────
+	if v := posFloatEnv("CAP_WATERMARK_THRESHOLD"); v > 0 {
+		cfg.Capacity.WatermarkThreshold = v
+	}
+	if v := posIntEnv("CAP_ALERT_SUPPRESS_MIN"); v > 0 {
+		cfg.Capacity.AlertSuppressMin = v
+	}
+	if v := posIntEnv("CAP_BREAKER_WINDOW_MIN"); v > 0 {
+		cfg.Capacity.BreakerWindowMin = v
+	}
+	if v := posIntEnv("CAP_BREAKER_FRACTION_PCT"); v > 0 {
+		cfg.Capacity.BreakerFractionPct = v
+	}
+	if v := posIntEnv("CAP_BREAKER_MIN_N"); v > 0 {
+		cfg.Capacity.BreakerMinN = v
+	}
+	if v := posIntEnv("CAP_MANUAL_ALERT_THRESH"); v > 0 {
+		cfg.Capacity.ManualAlertThresh = v
+	}
+	if v := os.Getenv("CAP_EXPECTED_PROBE_IDS"); v != "" {
+		cfg.Capacity.ExpectedProbeIDs = splitCSVEnv(v)
+	}
+
+	// ── 15E probe / grayscale ─────────────────────────────────────────────────
+	if v := posIntEnv("PROBE_GRAYSCALE_INTERVAL_H"); v > 0 {
+		cfg.Probe.GrayscaleIntervalHours = v
+	}
+	if v := posIntEnv("PROBE_TIMEOUT_MIN"); v > 0 {
+		cfg.Probe.ProbeTimeoutMin = v
+	}
+	if v := posIntEnv("PROBE_MAX_ATTEMPTS"); v > 0 {
+		cfg.Probe.MaxAttempts = v
+	}
+	if v := posIntEnv("PROBE_CYCLES_REQUIRED"); v > 0 {
+		cfg.Probe.ProbeCyclesRequired = v
+	}
+}
+
+// posIntEnv reads key as a positive integer. Returns 0 if the env var is
+// unset, empty, zero, or non-parseable (logs a warning for parse errors).
+func posIntEnv(key string) int {
+	v := os.Getenv(key)
+	if v == "" {
+		return 0
+	}
+	n, err := strconv.Atoi(v)
+	if err != nil || n <= 0 {
+		if err != nil {
+			slog.Warn("sched config: invalid integer env var", "key", key, "value", v)
+		}
+		return 0
+	}
+	return n
+}
+
+// posFloatEnv reads key as a positive float64. Returns 0 on error.
+func posFloatEnv(key string) float64 {
+	v := os.Getenv(key)
+	if v == "" {
+		return 0
+	}
+	f, err := strconv.ParseFloat(v, 64)
+	if err != nil || f <= 0 {
+		if err != nil {
+			slog.Warn("sched config: invalid float env var", "key", key, "value", v)
+		}
+		return 0
+	}
+	return f
+}
+
+// splitCSVEnv splits a comma-separated env-var value, trimming whitespace and
+// dropping empty fields.
+func splitCSVEnv(s string) []string {
+	parts := strings.Split(s, ",")
+	out := parts[:0]
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}