pangolin/server/internal/scheduler/orchestrate/breaker.go

package orchestrate

import (
	"context"
	"fmt"
	"log/slog"
	"math"
	"time"

	"github.com/redis/go-redis/v9"

	"github.com/wangjia/pangolin/server/internal/alert"
	"github.com/wangjia/pangolin/server/internal/idgen"
)

// ─────────────────────────────────────────────────────────────────────────────
// Redis key constants
// ─────────────────────────────────────────────────────────────────────────────

const (
	// breakerWindowKey is the sliding-window ZSET key prefix.
	// Full key: sched:breaker:{tier}:{region}
	// Schema: member = replacement UUID, score = Unix timestamp (seconds).
	breakerWindowKey = "sched:breaker:"

	// breakerTripKey is the trip-flag key prefix.
	// Full key: sched:breaker:trip:{tier}:{region}
	// TTL = BreakerWindowMin. Used for: alert dedup + admin visibility.
	// The key auto-expires when the sliding window clears (enabling auto-recovery).
	breakerTripKey = "sched:breaker:trip:"
)

// ─────────────────────────────────────────────────────────────────────────────
// RedisBreaker
// ─────────────────────────────────────────────────────────────────────────────

// RedisBreaker is the production 15F circuit-breaker. It satisfies the
// Breaker interface (Allow + Record) and additionally exposes Reset for the
// admin handler (task #8).
//
// Mechanism:
//
//	ZSET sched:breaker:{tier}:{region}
//	  member = replacement UUID (unique per completed replacement)
//	  score  = Unix timestamp of completion
//
//	Window: BreakerWindowMin (default 60 min). Each Allow call
//	  runs ZREMRANGEBYSCORE first to expire out-of-window entries.
//
//	Threshold N:
//	  N = ceil(poolTarget × BreakerFractionPct / 100), min BreakerMinN (3).
//
//	Trip:
//	  When window-count ≥ N: Allow returns false. The first detection in a
//	  window epoch sets a trip-flag key with TTL = BreakerWindowMin and emits
//	  a critical alert (idempotent via SetNX).
//
//	Auto-recovery:
//	  After BreakerWindowMin minutes with no new Record calls, all ZSET entries
//	  expire out of the window and the trip flag TTL also expires → Allow true.
//
//	Manual reset:
//	  Admin calls Reset, which deletes the trip flag and the ZSET immediately.
type RedisBreaker struct {
	rdb      *redis.Client
	pools    PoolReader
	notifier Notifier
	lc       LifecycleService // for audit log; may be nil
	cfgMgr   *ConfigManager
	clock    Clock
}

// RedisBreakerConfig holds all dependencies for NewRedisBreaker.
type RedisBreakerConfig struct {
	RDB      *redis.Client
	Pools    PoolReader
	Notifier Notifier
	LC       LifecycleService // for audit log; may be nil
	CfgMgr   *ConfigManager
	Clock    Clock
}

// NewRedisBreaker constructs a RedisBreaker. If Clock is nil, RealClock is used.
func NewRedisBreaker(cfg RedisBreakerConfig) *RedisBreaker {
	if cfg.Clock == nil {
		cfg.Clock = RealClock{}
	}
	return &RedisBreaker{
		rdb:      cfg.RDB,
		pools:    cfg.Pools,
		notifier: cfg.Notifier,
		lc:       cfg.LC,
		cfgMgr:   cfg.CfgMgr,
		clock:    cfg.Clock,
	}
}

// Allow implements Breaker. Returns false when the pool's sliding-window
// replacement count has reached the threshold (breaker is tripped), blocking
// further replacements from starting.
//
// On Redis errors the breaker fails open (returns true) to avoid a single
// point of failure halting all replacements.
func (b *RedisBreaker) Allow(tier, region string) bool {
	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
	defer cancel()

	cfg := b.cfgMgr.Current()
	window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute
	threshold := b.computeThreshold(ctx, tier, region, cfg)

	zKey := breakerWindowKey + tier + ":" + region
	tripKey := breakerTripKey + tier + ":" + region
	cutoff := b.clock.Now().Add(-window)

	// Expire entries outside the sliding window.
	cutoffStr := fmt.Sprintf("%d", cutoff.Unix())
	if err := b.rdb.ZRemRangeByScore(ctx, zKey, "-inf", cutoffStr).Err(); err != nil {
		slog.Error("breaker: ZREMRANGEBYSCORE failed; failing open",
			"tier", tier, "region", region, "error", err)
		return true // fail open
	}

	// Count remaining (in-window) entries.
	count, err := b.rdb.ZCard(ctx, zKey).Result()
	if err != nil {
		slog.Error("breaker: ZCARD failed; failing open",
			"tier", tier, "region", region, "error", err)
		return true // fail open
	}

	if count < int64(threshold) {
		return true // below threshold — allow replacement
	}

	// Count ≥ threshold: trip. Emit alert exactly once per window epoch via SetNX.
	set, setErr := b.rdb.SetNX(ctx, tripKey, "1", window).Result()
	if setErr != nil {
		slog.Error("breaker: trip flag set failed",
			"tier", tier, "region", region, "error", setErr)
	}
	if set {
		b.emitTripAlert(ctx, tier, region, int(count), threshold, cfg)
	}

	slog.Warn("breaker: replacement blocked — circuit open",
		"tier", tier, "region", region,
		"window_count", count, "threshold", threshold,
	)
	return false
}

// Record implements Breaker. Records a completed replacement for the given
// pool in the sliding-window ZSET. Called by the orchestrator after the old
// node is successfully destroyed.
//
// Record also checks whether the count has just reached the threshold and,
// if so, trips the breaker and emits a critical alert.
func (b *RedisBreaker) Record(tier, region string) {
	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
	defer cancel()

	cfg := b.cfgMgr.Current()
	window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute
	threshold := b.computeThreshold(ctx, tier, region, cfg)

	zKey := breakerWindowKey + tier + ":" + region
	tripKey := breakerTripKey + tier + ":" + region
	now := b.clock.Now()

	// Add a unique entry for this replacement.
	member := idgen.NewString()
	if err := b.rdb.ZAdd(ctx, zKey, redis.Z{
		Score:  float64(now.Unix()),
		Member: member,
	}).Err(); err != nil {
		slog.Error("breaker: ZADD failed",
			"tier", tier, "region", region, "error", err)
		return
	}

	// Check if we have just reached the threshold; trip once per epoch.
	count, err := b.rdb.ZCard(ctx, zKey).Result()
	if err != nil {
		slog.Error("breaker: ZCARD after record failed",
			"tier", tier, "region", region, "error", err)
		return
	}

	if count >= int64(threshold) {
		set, setErr := b.rdb.SetNX(ctx, tripKey, "1", window).Result()
		if setErr != nil {
			slog.Error("breaker: trip flag on record failed",
				"tier", tier, "region", region, "error", setErr)
			return
		}
		if set {
			// First trip in this window epoch: emit critical alert.
			b.emitTripAlert(ctx, tier, region, int(count), threshold, cfg)
		}
	}
}

// Reset clears a tripped breaker for the given pool by deleting both the trip
// flag and the sliding-window ZSET. This lets new replacements proceed
// immediately without waiting for the window to auto-expire.
//
// An audit log entry is written with actor and action metadata.
// Reset is not part of the Breaker interface; it is called only by the admin
// handler (task #8).
func (b *RedisBreaker) Reset(ctx context.Context, tier, region, actor string) error {
	zKey := breakerWindowKey + tier + ":" + region
	tripKey := breakerTripKey + tier + ":" + region

	pipe := b.rdb.Pipeline()
	pipe.Del(ctx, tripKey)
	pipe.Del(ctx, zKey)
	if _, err := pipe.Exec(ctx); err != nil {
		return fmt.Errorf("breaker: reset %s/%s: %w", tier, region, err)
	}

	slog.Info("breaker: manually reset by operator",
		"tier", tier, "region", region, "actor", actor)

	// Audit trail.
	meta := fmt.Sprintf(
		`{"tier":%q,"region":%q,"actor":%q,"action":"breaker_reset"}`,
		tier, region, actor,
	)
	if b.lc != nil {
		if err := b.lc.WriteAuditLog(
			ctx, actor, "breaker_reset", "pool:"+tier+":"+region, meta,
		); err != nil {
			slog.Error("breaker: write audit log failed", "error", err)
		}
	}
	return nil
}

// IsTripped reports whether the trip flag is currently set for the pool.
// Useful for admin status queries.
func (b *RedisBreaker) IsTripped(ctx context.Context, tier, region string) (bool, error) {
	tripKey := breakerTripKey + tier + ":" + region
	exists, err := b.rdb.Exists(ctx, tripKey).Result()
	if err != nil {
		return false, fmt.Errorf("breaker: IsTripped %s/%s: %w", tier, region, err)
	}
	return exists > 0, nil
}

// WindowCount returns the number of completed replacements currently recorded
// in the sliding window for the pool. Useful for admin status queries.
func (b *RedisBreaker) WindowCount(ctx context.Context, tier, region string) (int64, error) {
	cfg := b.cfgMgr.Current()
	window := time.Duration(cfg.Capacity.BreakerWindowMin) * time.Minute
	cutoff := b.clock.Now().Add(-window)
	cutoffStr := fmt.Sprintf("%d", cutoff.Unix())

	zKey := breakerWindowKey + tier + ":" + region
	if err := b.rdb.ZRemRangeByScore(ctx, zKey, "-inf", cutoffStr).Err(); err != nil {
		return 0, fmt.Errorf("breaker: WindowCount cleanup %s/%s: %w", tier, region, err)
	}
	count, err := b.rdb.ZCard(ctx, zKey).Result()
	if err != nil {
		return 0, fmt.Errorf("breaker: WindowCount ZCARD %s/%s: %w", tier, region, err)
	}
	return count, nil
}

// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────

// computeThreshold returns the trip threshold for a pool:
//
//	N = ceil(poolTarget × BreakerFractionPct / 100), minimum BreakerMinN.
//
// Falls back to BreakerMinN when the pool cannot be found or has zero target.
func (b *RedisBreaker) computeThreshold(ctx context.Context, tier, region string, cfg *SchedConfig) int {
	_, target, err := b.pools.PoolCapacity(ctx, tier, region)
	if err != nil || target <= 0 {
		return cfg.Capacity.BreakerMinN
	}
	n := int(math.Ceil(float64(target) * float64(cfg.Capacity.BreakerFractionPct) / 100.0))
	if n < cfg.Capacity.BreakerMinN {
		n = cfg.Capacity.BreakerMinN
	}
	return n
}

// emitTripAlert sends a critical alert to the Notifier for a tripped breaker.
func (b *RedisBreaker) emitTripAlert(ctx context.Context, tier, region string, count, threshold int, cfg *SchedConfig) {
	poolID := "pool:" + tier + ":" + region
	reason := fmt.Sprintf(
		"circuit breaker tripped: %d replacements in %d-min window (threshold=%d) — manual Reset required",
		count, cfg.Capacity.BreakerWindowMin, threshold,
	)
	slog.Error("breaker: circuit breaker tripped",
		"tier", tier, "region", region,
		"window_count", count, "threshold", threshold,
	)
	if b.notifier != nil {
		event := alert.NewEvent(alert.EventTypeBreakerTripped, poolID, map[string]string{"reason": reason})
		event.Pool = poolID
		if err := b.notifier.Notify(ctx, event); err != nil {
			slog.Error("breaker: emit trip alert failed",
				"pool", poolID, "error", err)
		}
	}
}