package orchestrate import ( "context" "fmt" "log/slog" "math" "time" "github.com/redis/go-redis/v9" "github.com/wangjia/pangolin/server/internal/alert" "github.com/wangjia/pangolin/server/internal/idgen" ) // ───────────────────────────────────────────────────────────────────────────── // Redis key constants // ───────────────────────────────────────────────────────────────────────────── const ( // breakerWindowKey is the sliding-window ZSET key prefix. // Full key: sched:breaker:{tier}:{region} // Schema: member = replacement UUID, score = Unix timestamp (seconds). breakerWindowKey = "sched:breaker:" // breakerTripKey is the trip-flag key prefix. // Full key: sched:breaker:trip:{tier}:{region} // TTL = BreakerWindowMin. Used for: alert dedup + admin visibility. // The key auto-expires when the sliding window clears (enabling auto-recovery). breakerTripKey = "sched:breaker:trip:" ) // ───────────────────────────────────────────────────────────────────────────── // RedisBreaker // ───────────────────────────────────────────────────────────────────────────── // RedisBreaker is the production 15F circuit-breaker. It satisfies the // Breaker interface (Allow + Record) and additionally exposes Reset for the // admin handler (task #8). // // Mechanism: // // ZSET sched:breaker:{tier}:{region} // member = replacement UUID (unique per completed replacement) // score = Unix timestamp of completion // // Window: BreakerWindowMin (default 60 min). Each Allow call // runs ZREMRANGEBYSCORE first to expire out-of-window entries. // // Threshold N: // N = ceil(poolTarget × BreakerFractionPct / 100), min BreakerMinN (3). // // Trip: // When window-count ≥ N: Allow returns false. The first detection in a // window epoch sets a trip-flag key with TTL = BreakerWindowMin and emits // a critical alert (idempotent via SetNX). // // Auto-recovery: // After BreakerWindowMin minutes with no new Record calls, all ZSET entries // expire out of the window and the trip flag TTL also expires → Allow true. // // Manual reset: // Admin calls Reset, which deletes the trip flag and the ZSET immediately. type RedisBreaker struct { rdb *redis.Client pools PoolReader notifier Notifier lc LifecycleService // for audit log; may be nil cfgMgr *ConfigManager clock Clock } // RedisBreakerConfig holds all dependencies for NewRedisBreaker. type RedisBreakerConfig struct { RDB *redis.Client Pools PoolReader Notifier Notifier LC LifecycleService // for audit log; may be nil CfgMgr *ConfigManager Clock Clock } // NewRedisBreaker constructs a RedisBreaker. If Clock is nil, RealClock is used. func NewRedisBreaker(cfg RedisBreakerConfig) *RedisBreaker { if cfg.Clock == nil { cfg.Clock = RealClock{} } return &RedisBreaker{ rdb: cfg.RDB, pools: cfg.Pools, notifier: cfg.Notifier, lc: cfg.LC, cfgMgr: cfg.CfgMgr, clock: cfg.Clock, } } // Allow implements Breaker. Returns false when the pool's sliding-window // replacement count has reached the threshold (breaker is tripped), blocking // further replacements from starting. // // On Redis errors the breaker fails open (returns true) to avoid a single // point of failure halting all replacements. func (b *RedisBreaker) Allow(tier, region string) bool { ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() cfg := b.cfgMgr.Current() window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute threshold := b.computeThreshold(ctx, tier, region, cfg) zKey := breakerWindowKey + tier + ":" + region tripKey := breakerTripKey + tier + ":" + region cutoff := b.clock.Now().Add(-window) // Expire entries outside the sliding window. cutoffStr := fmt.Sprintf("%d", cutoff.Unix()) if err := b.rdb.ZRemRangeByScore(ctx, zKey, "-inf", cutoffStr).Err(); err != nil { slog.Error("breaker: ZREMRANGEBYSCORE failed; failing open", "tier", tier, "region", region, "error", err) return true // fail open } // Count remaining (in-window) entries. count, err := b.rdb.ZCard(ctx, zKey).Result() if err != nil { slog.Error("breaker: ZCARD failed; failing open", "tier", tier, "region", region, "error", err) return true // fail open } if count < int64(threshold) { return true // below threshold — allow replacement } // Count ≥ threshold: trip. Emit alert exactly once per window epoch via SetNX. set, setErr := b.rdb.SetNX(ctx, tripKey, "1", window).Result() if setErr != nil { slog.Error("breaker: trip flag set failed", "tier", tier, "region", region, "error", setErr) } if set { b.emitTripAlert(ctx, tier, region, int(count), threshold, cfg) } slog.Warn("breaker: replacement blocked — circuit open", "tier", tier, "region", region, "window_count", count, "threshold", threshold, ) return false } // Record implements Breaker. Records a completed replacement for the given // pool in the sliding-window ZSET. Called by the orchestrator after the old // node is successfully destroyed. // // Record also checks whether the count has just reached the threshold and, // if so, trips the breaker and emits a critical alert. func (b *RedisBreaker) Record(tier, region string) { ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() cfg := b.cfgMgr.Current() window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute threshold := b.computeThreshold(ctx, tier, region, cfg) zKey := breakerWindowKey + tier + ":" + region tripKey := breakerTripKey + tier + ":" + region now := b.clock.Now() // Add a unique entry for this replacement. member := idgen.NewString() if err := b.rdb.ZAdd(ctx, zKey, redis.Z{ Score: float64(now.Unix()), Member: member, }).Err(); err != nil { slog.Error("breaker: ZADD failed", "tier", tier, "region", region, "error", err) return } // Check if we have just reached the threshold; trip once per epoch. count, err := b.rdb.ZCard(ctx, zKey).Result() if err != nil { slog.Error("breaker: ZCARD after record failed", "tier", tier, "region", region, "error", err) return } if count >= int64(threshold) { set, setErr := b.rdb.SetNX(ctx, tripKey, "1", window).Result() if setErr != nil { slog.Error("breaker: trip flag on record failed", "tier", tier, "region", region, "error", setErr) return } if set { // First trip in this window epoch: emit critical alert. b.emitTripAlert(ctx, tier, region, int(count), threshold, cfg) } } } // Reset clears a tripped breaker for the given pool by deleting both the trip // flag and the sliding-window ZSET. This lets new replacements proceed // immediately without waiting for the window to auto-expire. // // An audit log entry is written with actor and action metadata. // Reset is not part of the Breaker interface; it is called only by the admin // handler (task #8). func (b *RedisBreaker) Reset(ctx context.Context, tier, region, actor string) error { zKey := breakerWindowKey + tier + ":" + region tripKey := breakerTripKey + tier + ":" + region pipe := b.rdb.Pipeline() pipe.Del(ctx, tripKey) pipe.Del(ctx, zKey) if _, err := pipe.Exec(ctx); err != nil { return fmt.Errorf("breaker: reset %s/%s: %w", tier, region, err) } slog.Info("breaker: manually reset by operator", "tier", tier, "region", region, "actor", actor) // Audit trail. meta := fmt.Sprintf( `{"tier":%q,"region":%q,"actor":%q,"action":"breaker_reset"}`, tier, region, actor, ) if b.lc != nil { if err := b.lc.WriteAuditLog( ctx, actor, "breaker_reset", "pool:"+tier+":"+region, meta, ); err != nil { slog.Error("breaker: write audit log failed", "error", err) } } return nil } // IsTripped reports whether the trip flag is currently set for the pool. // Useful for admin status queries. func (b *RedisBreaker) IsTripped(ctx context.Context, tier, region string) (bool, error) { tripKey := breakerTripKey + tier + ":" + region exists, err := b.rdb.Exists(ctx, tripKey).Result() if err != nil { return false, fmt.Errorf("breaker: IsTripped %s/%s: %w", tier, region, err) } return exists > 0, nil } // WindowCount returns the number of completed replacements currently recorded // in the sliding window for the pool. Useful for admin status queries. func (b *RedisBreaker) WindowCount(ctx context.Context, tier, region string) (int64, error) { cfg := b.cfgMgr.Current() window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute cutoff := b.clock.Now().Add(-window) cutoffStr := fmt.Sprintf("%d", cutoff.Unix()) zKey := breakerWindowKey + tier + ":" + region if err := b.rdb.ZRemRangeByScore(ctx, zKey, "-inf", cutoffStr).Err(); err != nil { return 0, fmt.Errorf("breaker: WindowCount cleanup %s/%s: %w", tier, region, err) } count, err := b.rdb.ZCard(ctx, zKey).Result() if err != nil { return 0, fmt.Errorf("breaker: WindowCount ZCARD %s/%s: %w", tier, region, err) } return count, nil } // ───────────────────────────────────────────────────────────────────────────── // Helpers // ───────────────────────────────────────────────────────────────────────────── // computeThreshold returns the trip threshold for a pool: // // N = ceil(poolTarget × BreakerFractionPct / 100), minimum BreakerMinN. // // Falls back to BreakerMinN when the pool cannot be found or has zero target. func (b *RedisBreaker) computeThreshold(ctx context.Context, tier, region string, cfg *SchedConfig) int { _, target, err := b.pools.PoolCapacity(ctx, tier, region) if err != nil || target <= 0 { return cfg.Capacity.BreakerMinN } n := int(math.Ceil(float64(target) * float64(cfg.Capacity.BreakerFractionPct) / 100.0)) if n < cfg.Capacity.BreakerMinN { n = cfg.Capacity.BreakerMinN } return n } // emitTripAlert sends a critical alert to the Notifier for a tripped breaker. func (b *RedisBreaker) emitTripAlert(ctx context.Context, tier, region string, count, threshold int, cfg *SchedConfig) { poolID := "pool:" + tier + ":" + region reason := fmt.Sprintf( "circuit breaker tripped: %d replacements in %d-min window (threshold=%d) — manual Reset required", count, cfg.Capacity.BreakerWindowMin, threshold, ) slog.Error("breaker: circuit breaker tripped", "tier", tier, "region", region, "window_count", count, "threshold", threshold, ) if b.notifier != nil { event := alert.NewEvent(alert.EventTypeBreakerTripped, poolID, map[string]string{"reason": reason}) event.Pool = poolID if err := b.notifier.Notify(ctx, event); err != nil { slog.Error("breaker: emit trip alert failed", "pool", poolID, "error", err) } } }