feat(scheduler): 容量水位监控 + 熔断器 + 阈值热加载 [tsk_NPgPRxBGv0g9]

实现 15F 任务的三个核心组件： capacity.go — CapacityMonitor（每 1-2 min 由 CapacityLoop 调用） - 池水位检查：up/target < 70% 时发告警，10 min 内降频去重 - 探针失联检查：扫 probe:hb:{probeId}，缺失发「探针失联」事件 - 驱动 Grayscale.Advance 推进养机档位 breaker.go — RedisBreaker（实现 Breaker 接口） - 滑窗 ZSET sched:breaker:{tier}:{region}，窗口 1h - 阈值 N = ceil(池容量 × 30%)，下限 3 - 窗口内计数 ≥ N → 置 tripped 标记位 + critical 告警 - Allow 恒 false 直到窗口滑出自动恢复，或管理员调 Reset - Reset 清除 ZSET + trip flag，写 audit_log config.go — SchedConfig + ConfigManager（全树阈值热加载） - SchedConfig 集中定义 15D/15E/15F 所有阈值 - 来源：默认值 → YAML 文件 → 环境变量（三层叠加） - 监听 SIGHUP，原子替换（atomic.Pointer[SchedConfig]） - 每次变更 diff 写 audit_log（AuditFn 回调）测试（24 个用例，全绿）： - 窗口内第 N 次 Record 后 Allow 返回 false - 窗口滑出后 Allow 自动恢复 - Reset 立即恢复 + audit 条目 - 下限 3 在小池（target=2）生效 - 65% 水位 → 告警；10 min 内不重复 - 探针心跳缺失 → 失联事件；存在 → 无告警 - YAML + env 覆盖 + SIGHUP Reload → 新阈值即时生效 + diff 审计 - 替换 stub 后 15E 全部集成测试仍绿 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-16 00:31:58 +08:00
parent cadd527680
commit 3d9ccff6e9
6 changed files with 1793 additions and 0 deletions
@@ -0,0 +1,307 @@
+package orchestrate
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"math"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+
+	"github.com/wangjia/pangolin/server/internal/idgen"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Redis key constants
+// ─────────────────────────────────────────────────────────────────────────────
+
+const (
+	// breakerWindowKey is the sliding-window ZSET key prefix.
+	// Full key: sched:breaker:{tier}:{region}
+	// Schema: member = replacement UUID, score = Unix timestamp (seconds).
+	breakerWindowKey = "sched:breaker:"
+
+	// breakerTripKey is the trip-flag key prefix.
+	// Full key: sched:breaker:trip:{tier}:{region}
+	// TTL = BreakerWindowMin. Used for: alert dedup + admin visibility.
+	// The key auto-expires when the sliding window clears (enabling auto-recovery).
+	breakerTripKey = "sched:breaker:trip:"
+)
+
+// ─────────────────────────────────────────────────────────────────────────────
+// RedisBreaker
+// ─────────────────────────────────────────────────────────────────────────────
+
+// RedisBreaker is the production 15F circuit-breaker. It satisfies the
+// Breaker interface (Allow + Record) and additionally exposes Reset for the
+// admin handler (task #8).
+//
+// Mechanism:
+//
+//	ZSET sched:breaker:{tier}:{region}
+//	  member = replacement UUID (unique per completed replacement)
+//	  score  = Unix timestamp of completion
+//
+//	Window: BreakerWindowMin (default 60 min). Each Allow call
+//	  runs ZREMRANGEBYSCORE first to expire out-of-window entries.
+//
+//	Threshold N:
+//	  N = ceil(poolTarget × BreakerFractionPct / 100), min BreakerMinN (3).
+//
+//	Trip:
+//	  When window-count ≥ N: Allow returns false. The first detection in a
+//	  window epoch sets a trip-flag key with TTL = BreakerWindowMin and emits
+//	  a critical alert (idempotent via SetNX).
+//
+//	Auto-recovery:
+//	  After BreakerWindowMin minutes with no new Record calls, all ZSET entries
+//	  expire out of the window and the trip flag TTL also expires → Allow true.
+//
+//	Manual reset:
+//	  Admin calls Reset, which deletes the trip flag and the ZSET immediately.
+type RedisBreaker struct {
+	rdb      *redis.Client
+	pools    PoolReader
+	notifier Notifier
+	lc       LifecycleService // for audit log; may be nil
+	cfgMgr   *ConfigManager
+	clock    Clock
+}
+
+// RedisBreakerConfig holds all dependencies for NewRedisBreaker.
+type RedisBreakerConfig struct {
+	RDB      *redis.Client
+	Pools    PoolReader
+	Notifier Notifier
+	LC       LifecycleService // for audit log; may be nil
+	CfgMgr   *ConfigManager
+	Clock    Clock
+}
+
+// NewRedisBreaker constructs a RedisBreaker. If Clock is nil, RealClock is used.
+func NewRedisBreaker(cfg RedisBreakerConfig) *RedisBreaker {
+	if cfg.Clock == nil {
+		cfg.Clock = RealClock{}
+	}
+	return &RedisBreaker{
+		rdb:      cfg.RDB,
+		pools:    cfg.Pools,
+		notifier: cfg.Notifier,
+		lc:       cfg.LC,
+		cfgMgr:   cfg.CfgMgr,
+		clock:    cfg.Clock,
+	}
+}
+
+// Allow implements Breaker. Returns false when the pool's sliding-window
+// replacement count has reached the threshold (breaker is tripped), blocking
+// further replacements from starting.
+//
+// On Redis errors the breaker fails open (returns true) to avoid a single
+// point of failure halting all replacements.
+func (b *RedisBreaker) Allow(tier, region string) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	defer cancel()
+
+	cfg := b.cfgMgr.Current()
+	window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute
+	threshold := b.computeThreshold(ctx, tier, region, cfg)
+
+	zKey := breakerWindowKey + tier + ":" + region
+	tripKey := breakerTripKey + tier + ":" + region
+	cutoff := b.clock.Now().Add(-window)
+
+	// Expire entries outside the sliding window.
+	cutoffStr := fmt.Sprintf("%d", cutoff.Unix())
+	if err := b.rdb.ZRemRangeByScore(ctx, zKey, "-inf", cutoffStr).Err(); err != nil {
+		slog.Error("breaker: ZREMRANGEBYSCORE failed; failing open",
+			"tier", tier, "region", region, "error", err)
+		return true // fail open
+	}
+
+	// Count remaining (in-window) entries.
+	count, err := b.rdb.ZCard(ctx, zKey).Result()
+	if err != nil {
+		slog.Error("breaker: ZCARD failed; failing open",
+			"tier", tier, "region", region, "error", err)
+		return true // fail open
+	}
+
+	if count < int64(threshold) {
+		return true // below threshold — allow replacement
+	}
+
+	// Count ≥ threshold: trip. Emit alert exactly once per window epoch via SetNX.
+	set, setErr := b.rdb.SetNX(ctx, tripKey, "1", window).Result()
+	if setErr != nil {
+		slog.Error("breaker: trip flag set failed",
+			"tier", tier, "region", region, "error", setErr)
+	}
+	if set {
+		b.emitTripAlert(ctx, tier, region, int(count), threshold, cfg)
+	}
+
+	slog.Warn("breaker: replacement blocked — circuit open",
+		"tier", tier, "region", region,
+		"window_count", count, "threshold", threshold,
+	)
+	return false
+}
+
+// Record implements Breaker. Records a completed replacement for the given
+// pool in the sliding-window ZSET. Called by the orchestrator after the old
+// node is successfully destroyed.
+//
+// Record also checks whether the count has just reached the threshold and,
+// if so, trips the breaker and emits a critical alert.
+func (b *RedisBreaker) Record(tier, region string) {
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	defer cancel()
+
+	cfg := b.cfgMgr.Current()
+	window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute
+	threshold := b.computeThreshold(ctx, tier, region, cfg)
+
+	zKey := breakerWindowKey + tier + ":" + region
+	tripKey := breakerTripKey + tier + ":" + region
+	now := b.clock.Now()
+
+	// Add a unique entry for this replacement.
+	member := idgen.NewString()
+	if err := b.rdb.ZAdd(ctx, zKey, redis.Z{
+		Score:  float64(now.Unix()),
+		Member: member,
+	}).Err(); err != nil {
+		slog.Error("breaker: ZADD failed",
+			"tier", tier, "region", region, "error", err)
+		return
+	}
+
+	// Check if we have just reached the threshold; trip once per epoch.
+	count, err := b.rdb.ZCard(ctx, zKey).Result()
+	if err != nil {
+		slog.Error("breaker: ZCARD after record failed",
+			"tier", tier, "region", region, "error", err)
+		return
+	}
+
+	if count >= int64(threshold) {
+		set, setErr := b.rdb.SetNX(ctx, tripKey, "1", window).Result()
+		if setErr != nil {
+			slog.Error("breaker: trip flag on record failed",
+				"tier", tier, "region", region, "error", setErr)
+			return
+		}
+		if set {
+			// First trip in this window epoch: emit critical alert.
+			b.emitTripAlert(ctx, tier, region, int(count), threshold, cfg)
+		}
+	}
+}
+
+// Reset clears a tripped breaker for the given pool by deleting both the trip
+// flag and the sliding-window ZSET. This lets new replacements proceed
+// immediately without waiting for the window to auto-expire.
+//
+// An audit log entry is written with actor and action metadata.
+// Reset is not part of the Breaker interface; it is called only by the admin
+// handler (task #8).
+func (b *RedisBreaker) Reset(ctx context.Context, tier, region, actor string) error {
+	zKey := breakerWindowKey + tier + ":" + region
+	tripKey := breakerTripKey + tier + ":" + region
+
+	pipe := b.rdb.Pipeline()
+	pipe.Del(ctx, tripKey)
+	pipe.Del(ctx, zKey)
+	if _, err := pipe.Exec(ctx); err != nil {
+		return fmt.Errorf("breaker: reset %s/%s: %w", tier, region, err)
+	}
+
+	slog.Info("breaker: manually reset by operator",
+		"tier", tier, "region", region, "actor", actor)
+
+	// Audit trail.
+	meta := fmt.Sprintf(
+		`{"tier":%q,"region":%q,"actor":%q,"action":"breaker_reset"}`,
+		tier, region, actor,
+	)
+	if b.lc != nil {
+		if err := b.lc.WriteAuditLog(
+			ctx, actor, "breaker_reset", "pool:"+tier+":"+region, meta,
+		); err != nil {
+			slog.Error("breaker: write audit log failed", "error", err)
+		}
+	}
+	return nil
+}
+
+// IsTripped reports whether the trip flag is currently set for the pool.
+// Useful for admin status queries.
+func (b *RedisBreaker) IsTripped(ctx context.Context, tier, region string) (bool, error) {
+	tripKey := breakerTripKey + tier + ":" + region
+	exists, err := b.rdb.Exists(ctx, tripKey).Result()
+	if err != nil {
+		return false, fmt.Errorf("breaker: IsTripped %s/%s: %w", tier, region, err)
+	}
+	return exists > 0, nil
+}
+
+// WindowCount returns the number of completed replacements currently recorded
+// in the sliding window for the pool. Useful for admin status queries.
+func (b *RedisBreaker) WindowCount(ctx context.Context, tier, region string) (int64, error) {
+	cfg := b.cfgMgr.Current()
+	window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute
+	cutoff := b.clock.Now().Add(-window)
+	cutoffStr := fmt.Sprintf("%d", cutoff.Unix())
+
+	zKey := breakerWindowKey + tier + ":" + region
+	if err := b.rdb.ZRemRangeByScore(ctx, zKey, "-inf", cutoffStr).Err(); err != nil {
+		return 0, fmt.Errorf("breaker: WindowCount cleanup %s/%s: %w", tier, region, err)
+	}
+	count, err := b.rdb.ZCard(ctx, zKey).Result()
+	if err != nil {
+		return 0, fmt.Errorf("breaker: WindowCount ZCARD %s/%s: %w", tier, region, err)
+	}
+	return count, nil
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+// computeThreshold returns the trip threshold for a pool:
+//
+//	N = ceil(poolTarget × BreakerFractionPct / 100), minimum BreakerMinN.
+//
+// Falls back to BreakerMinN when the pool cannot be found or has zero target.
+func (b *RedisBreaker) computeThreshold(ctx context.Context, tier, region string, cfg *SchedConfig) int {
+	_, target, err := b.pools.PoolCapacity(ctx, tier, region)
+	if err != nil || target <= 0 {
+		return cfg.Capacity.BreakerMinN
+	}
+	n := int(math.Ceil(float64(target) * float64(cfg.Capacity.BreakerFractionPct) / 100.0))
+	if n < cfg.Capacity.BreakerMinN {
+		n = cfg.Capacity.BreakerMinN
+	}
+	return n
+}
+
+// emitTripAlert sends a critical alert to the Notifier for a tripped breaker.
+func (b *RedisBreaker) emitTripAlert(ctx context.Context, tier, region string, count, threshold int, cfg *SchedConfig) {
+	poolID := "pool:" + tier + ":" + region
+	reason := fmt.Sprintf(
+		"circuit breaker tripped: %d replacements in %d-min window (threshold=%d) — manual Reset required",
+		count, cfg.Capacity.BreakerWindowMin, threshold,
+	)
+	slog.Error("breaker: circuit breaker tripped",
+		"tier", tier, "region", region,
+		"window_count", count, "threshold", threshold,
+	)
+	if b.notifier != nil {
+		if err := b.notifier.NotifyFault(ctx, poolID, reason); err != nil {
+			slog.Error("breaker: emit trip alert failed",
+				"pool", poolID, "error", err)
+		}
+	}
+}