pangolin/server/internal/scheduler/scheduler.go

package scheduler

import (
	"context"
	"log/slog"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/google/uuid"
	"github.com/redis/go-redis/v9"
)

// ─────────────────────────────────────────────────────────────────────────────
// Leader-election Redis key helper
// ─────────────────────────────────────────────────────────────────────────────

// leaderKey returns the Redis key for a loop's leader election.
// Format: sched:leader:{loopName}
func leaderKey(loop string) string { return "sched:leader:" + loop }

// ─────────────────────────────────────────────────────────────────────────────
// Config
// ─────────────────────────────────────────────────────────────────────────────

// Config holds all scheduler dependencies and tunable timing parameters.
// Zero values for duration fields use the production defaults listed in the
// field comments.  All wired components (Engine, Replacer, …) are required
// unless the field is explicitly marked optional.
type Config struct {
	// RDB is the Redis client used for leader-election keys.  Required.
	RDB *redis.Client

	// InstanceID uniquely identifies this process instance across the fleet.
	// If empty, a random UUID is generated by New.
	InstanceID string

	// ── Wired engine components ──────────────────────────────────────────────

	// Engine is the 15D detection engine.  Required.
	Engine DetectEngine

	// Replacer is the 15E replacement-orchestration Replacer.  Required.
	Replacer OrchestrateReplacer

	// Grayscale is the 15E grayscale warm-up driver.  Required.
	Grayscale OrchestrateGrayscale

	// Capacity is the 15F capacity/circuit-breaker service.
	// Optional — StubCapacityService is used when nil.
	Capacity CapacityService

	// ProbeStore reads probe-agent heartbeat liveness (15A).
	// Optional — probe disconnection checking is disabled when nil.
	ProbeStore ProbeStateReader

	// KnownProbeIDs is the set of probe-agent IDs expected to be alive.
	// When any ID in this list is absent from ProbeStore.AliveProbes, the
	// Notifier is called with reason "probe_disconnected".
	KnownProbeIDs []string

	// Prober is the 15C third-party dial-test driver.
	// Optional — 15C probing is disabled when nil.
	Prober ThirdPartyProber

	// Targets supplies the list of nodes to probe via Prober.
	// Required when Prober is non-nil.
	Targets TargetProvider

	// Notifier is the 15G alerting sink.
	// Optional — LogNotifier is used when nil.
	Notifier Notifier

	// ── Tick intervals (hot-reloadable from 15F config) ─────────────────────
	// Zero → production default (shown in comments).

	DetectInterval      time.Duration // 5 min
	OrchestrateInterval time.Duration // 30 s
	CapacityInterval    time.Duration // 90 s
	ThirdPartyInterval  time.Duration // 10 min — interval between 15C RunOnce calls

	// ── Leader-election timing ────────────────────────────────────────────────
	// Override only in tests (shorter values speed up convergence).

	LeaseTTL    time.Duration // 30 s  — Redis PEXPIRE on the leader key
	RenewPeriod time.Duration // 10 s  — how often the leader renews its lease
	RetryPeriod time.Duration // 5 s   — how often a non-leader tries to acquire
	TickTimeout time.Duration // 30 s  — per-tick context hard-timeout
}

// applyDefaults fills zero-value duration fields with production defaults.
func (c *Config) applyDefaults() {
	if c.InstanceID == "" {
		c.InstanceID = uuid.NewString()
	}
	if c.DetectInterval == 0 {
		c.DetectInterval = 5 * time.Minute
	}
	if c.OrchestrateInterval == 0 {
		c.OrchestrateInterval = 30 * time.Second
	}
	if c.CapacityInterval == 0 {
		c.CapacityInterval = 90 * time.Second
	}
	if c.ThirdPartyInterval == 0 {
		c.ThirdPartyInterval = 10 * time.Minute
	}
	if c.LeaseTTL == 0 {
		c.LeaseTTL = 30 * time.Second
	}
	if c.RenewPeriod == 0 {
		c.RenewPeriod = 10 * time.Second
	}
	if c.RetryPeriod == 0 {
		c.RetryPeriod = 5 * time.Second
	}
	if c.TickTimeout == 0 {
		c.TickTimeout = 30 * time.Second
	}
	if c.Notifier == nil {
		c.Notifier = LogNotifier{}
	}
	if c.Capacity == nil {
		c.Capacity = StubCapacityService{}
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Scheduler
// ─────────────────────────────────────────────────────────────────────────────

// Scheduler runs three leader-elected loops (DetectLoop / OrchestrateLoop /
// CapacityLoop) plus an optional unguarded 15C third-party-prober goroutine.
//
// Design rationale for three independent leader leases (one per loop):
//   - A slow Detect tick (e.g., DB latency spike) must not block Orchestrate
//     progress on other in-flight replacements.
//   - Each loop failing over independently avoids a partial-leader split where
//     a crashing replica holds only one of three leases.
//   - Raft-style leadership is overkill for these loosely coupled loops, which
//     are each idempotent and use Redis-side locking for concurrency safety.
type Scheduler struct {
	rdb  *redis.Client
	cfg  Config
}

// New creates a Scheduler from cfg, applying production defaults for any nil /
// zero fields.
func New(cfg Config) *Scheduler {
	cfg.applyDefaults()
	return &Scheduler{rdb: cfg.RDB, cfg: cfg}
}

// Run starts the scheduler with OS signal handling (SIGTERM / SIGINT) and
// blocks until the scheduler has shut down.  For production use from main().
func (s *Scheduler) Run(ctx context.Context) error {
	sigCtx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
	defer stop()
	return s.Start(sigCtx)
}

// Start starts the scheduler goroutines and blocks until ctx is cancelled.
// Unlike Run, it does not install OS signal handlers — useful for testing.
func (s *Scheduler) Start(ctx context.Context) error {
	var wg sync.WaitGroup

	// ── Three leader-elected loops ─────────────────────────────────────────────
	//
	// Each loop acquires its own independent Redis leader key so that a slow
	// tick in one loop does not delay the others.

	wg.Add(3)
	go func() {
		defer wg.Done()
		s.runLoop(ctx, "detect", s.cfg.DetectInterval, s.cfg.Engine.Tick)
	}()
	go func() {
		defer wg.Done()
		s.runLoop(ctx, "orchestrate", s.cfg.OrchestrateInterval, s.cfg.Replacer.Tick)
	}()
	go func() {
		defer wg.Done()
		s.runLoop(ctx, "capacity", s.cfg.CapacityInterval, s.capacityTick)
	}()

	// ── 15C third-party prober (no leader constraint) ─────────────────────────
	//
	// Design choice: run on ALL replicas without leader election because:
	//   1. Alibaba Cloud API calls are idempotent: same (node, ISP) pair creates
	//      a new one-shot task; the result is written to Redis via SET (overwrite)
	//      so duplicate writes produce the same snapshot with a slightly later
	//      ReceivedAt — harmless.
	//   2. Running on every replica improves data collection reliability: if one
	//      replica crashes mid-cycle, the others continue gathering snapshots.
	//   3. The detect engine (15D) aggregates snapshots regardless of source;
	//      receiving the same data twice per window has no logical effect.
	//
	// If billing for N×API calls becomes a concern, add leader election here by
	// wrapping the loop body in tryRunAsLeader("prober").
	if s.cfg.Prober != nil && s.cfg.Targets != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			s.runThirdPartyProber(ctx)
		}()
	}

	// Wait for ctx cancellation (SIGTERM / test cancel).
	<-ctx.Done()
	slog.Info("scheduler: shutdown signal received — waiting for in-flight ticks to finish",
		"instance", s.cfg.InstanceID)

	// 30-second hard timeout: all loops must finish their current tick within
	// this window.  Leader keys are released by each loop's defer (see
	// runLeaderLoop → releaseLease) so followers can take over immediately.
	allStopped := make(chan struct{})
	go func() {
		wg.Wait()
		close(allStopped)
	}()

	shutdownTimeout := s.cfg.TickTimeout // reuse tick hard-timeout as shutdown budget
	select {
	case <-allStopped:
		slog.Info("scheduler: graceful shutdown complete", "instance", s.cfg.InstanceID)
	case <-time.After(shutdownTimeout):
		slog.Warn("scheduler: shutdown timeout exceeded — forcing exit",
			"timeout", shutdownTimeout, "instance", s.cfg.InstanceID)
	}
	return nil
}

// ─────────────────────────────────────────────────────────────────────────────
// Leader-elected loop driver
// ─────────────────────────────────────────────────────────────────────────────

// runLoop is the generic leader-elected loop driver.
//
//   - Non-leader: polls every RetryPeriod to acquire sched:leader:{name}.
//   - Leader: ticks at interval; a background goroutine renews the lease every
//     RenewPeriod.  Lost lease → return (exit from runLeaderLoop).
//
// The function loops continuously until ctx is Done.
func (s *Scheduler) runLoop(ctx context.Context, name string, interval time.Duration, tick func(context.Context) error) {
	key := leaderKey(name)

	for {
		// Try to acquire the leader lease (SET NX PX <leaseTTL ms>).
		ok, err := s.rdb.SetNX(ctx, key, s.cfg.InstanceID, s.cfg.LeaseTTL).Result()
		switch {
		case err != nil:
			// On a cancelled context SetNX errors without acquiring anything,
			// so it is safe to exit silently. Any other error is logged and retried.
			if ctx.Err() != nil {
				return
			}
			slog.Error("scheduler: leader setnx error", "loop", name, "error", err)
		case ok:
			// Lease acquired. Always enter runLeaderLoop — even if ctx was
			// cancelled in the race immediately after SetNX — because its defer
			// is what releases the key. Returning here on ctx.Err() would leak
			// the just-acquired leader key (no clean DEL on shutdown).
			slog.Info("scheduler: acquired leader lease", "loop", name, "instance", s.cfg.InstanceID)
			s.runLeaderLoop(ctx, name, interval, tick, key)
			slog.Info("scheduler: relinquished leader", "loop", name, "instance", s.cfg.InstanceID)
		}

		// Not leader (or just stepped down): wait before retrying.
		select {
		case <-ctx.Done():
			return
		case <-time.After(s.cfg.RetryPeriod):
		}
	}
}

// runLeaderLoop ticks at interval while holding the leader lease.
// It starts a keepLease goroutine; when the lease is lost (renewal failure or
// key taken by a competitor), it returns so runLoop can retry.
//
// Graceful-exit guarantee: when ctx is cancelled, runLeaderLoop waits for any
// in-flight tick to finish (the tick is given its own TickTimeout context, not
// the parent ctx) before calling defer → releaseLease.  This ensures:
//   - No half-executed migrate/notify steps are left in Redis.
//   - The leader key is actively released (DEL) on clean shutdown, so the
//     follower can take over within RetryPeriod rather than LeaseTTL.
func (s *Scheduler) runLeaderLoop(
	ctx context.Context,
	name string,
	interval time.Duration,
	tick func(context.Context) error,
	leaseKey string,
) {
	// Renewal goroutine: signals via leaseLost when it can no longer renew.
	renewCtx, cancelRenew := context.WithCancel(ctx)
	leaseLost := make(chan struct{})
	go func() {
		defer close(leaseLost)
		s.keepLease(renewCtx, leaseKey, name)
	}()

	// On exit: stop the renewer and release the key so followers converge faster.
	defer func() {
		cancelRenew()
		<-leaseLost // wait for renewer before releasing
		// Use a fresh Background context: the original ctx may already be Done.
		s.releaseLease(context.Background(), leaseKey, name)
	}()

	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			// Shutdown requested: exit (defer will release lease).
			// If a tick was already dispatched before this select was reached,
			// it is still running under its own TickTimeout context and will
			// complete before this goroutine is joined by Start's wg.Wait().
			return

		case <-leaseLost:
			slog.Warn("scheduler: leader lease lost — stepping down", "loop", name)
			return

		case <-ticker.C:
			// Run tick with a private context so SIGTERM / ctx cancel does not
			// abort it mid-flight.  Idempotency (15D optimistic lock, 15E
			// idempotency keys) ensures a tick that completes after the leader
			// key expires is safe.
			tickCtx, cancelTick := context.WithTimeout(context.Background(), s.cfg.TickTimeout)
			if err := tick(tickCtx); err != nil {
				slog.Error("scheduler: tick error", "loop", name, "error", err)
			}
			cancelTick()

			// If parent was cancelled while we were ticking, exit now instead
			// of waiting for the next ticker.C.
			if ctx.Err() != nil {
				return
			}
		}
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Lease renewal / release
// ─────────────────────────────────────────────────────────────────────────────

// Lua scripts are pre-evaluated to validate key ownership before each
// PEXPIRE / DEL so we never extend or delete a key owned by another instance.

const luaRenew = `
if redis.call('GET', KEYS[1]) == ARGV[1] then
    return redis.call('PEXPIRE', KEYS[1], ARGV[2])
else
    return 0
end`

const luaRelease = `
if redis.call('GET', KEYS[1]) == ARGV[1] then
    return redis.call('DEL', KEYS[1])
else
    return 0
end`

// keepLease renews the leader key every RenewPeriod using the Lua renewal
// script.  Returns when renewal fails (key gone or taken) or ctx is Done.
func (s *Scheduler) keepLease(ctx context.Context, key, name string) {
	ticker := time.NewTicker(s.cfg.RenewPeriod)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			res, err := s.rdb.Eval(
				ctx, luaRenew, []string{key},
				s.cfg.InstanceID,
				s.cfg.LeaseTTL.Milliseconds(),
			).Result()
			if err != nil {
				if ctx.Err() != nil {
					return // context cancelled: not a real failure
				}
				slog.Error("scheduler: lease renewal error", "loop", name, "error", err)
				return // treat Redis error as lease loss
			}
			if res.(int64) == 0 {
				slog.Warn("scheduler: lease renewal returned 0 — key gone or taken",
					"loop", name, "key", key)
				return
			}
		}
	}
}

// releaseLease deletes the leader key only if it still belongs to this
// instance, using the Lua release script.
func (s *Scheduler) releaseLease(ctx context.Context, key, name string) {
	res, err := s.rdb.Eval(ctx, luaRelease, []string{key}, s.cfg.InstanceID).Result()
	if err != nil {
		slog.Error("scheduler: release lease error", "loop", name, "error", err)
		return
	}
	if res.(int64) == 1 {
		slog.Info("scheduler: leader key released",
			"loop", name, "instance", s.cfg.InstanceID)
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Capacity tick (CapacityLoop work function)
// ─────────────────────────────────────────────────────────────────────────────

// capacityTick is the work function for CapacityLoop.  It runs:
//  1. 15F capacity / watermark check.
//  2. 15E grayscale weight advancement.
//  3. 15F circuit-breaker counter decay.
//  4. 15A probe-agent disconnection check → 15G alert.
//
// Each step is independent; a failure in one does not abort the others.
func (s *Scheduler) capacityTick(ctx context.Context) error {
	// 1. Capacity / watermark check (15F).
	if err := s.cfg.Capacity.Check(ctx); err != nil {
		slog.Error("scheduler: capacity check error", "error", err)
		// Non-fatal: continue.
	}

	// 2. Grayscale warm-up advancement (15E).
	if err := s.cfg.Grayscale.Advance(ctx); err != nil {
		slog.Error("scheduler: grayscale advance error", "error", err)
	}

	// 3. Circuit-breaker counter decay (15F).
	if err := s.cfg.Capacity.Decay(ctx); err != nil {
		slog.Error("scheduler: capacity decay error", "error", err)
	}

	// 4. Probe-agent disconnection check (15A probe store → 15G alert).
	if s.cfg.ProbeStore != nil && len(s.cfg.KnownProbeIDs) > 0 {
		s.checkProbeConnections(ctx)
	}

	return nil
}

// checkProbeConnections fires a Notifier alert for every known probe agent
// whose heartbeat key has expired (i.e. is absent from AliveProbes).
func (s *Scheduler) checkProbeConnections(ctx context.Context) {
	alive, err := s.cfg.ProbeStore.AliveProbes(ctx)
	if err != nil {
		slog.Error("scheduler: AliveProbes error", "error", err)
		return
	}

	aliveSet := make(map[string]struct{}, len(alive))
	for _, id := range alive {
		aliveSet[id] = struct{}{}
	}

	for _, id := range s.cfg.KnownProbeIDs {
		if _, ok := aliveSet[id]; !ok {
			if notifyErr := s.cfg.Notifier.NotifyFault(ctx, id, "probe_disconnected"); notifyErr != nil {
				slog.Error("scheduler: notify probe disconnect",
					"probe_id", id, "error", notifyErr)
			}
		}
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Third-party prober goroutine (15C, no leader constraint)
// ─────────────────────────────────────────────────────────────────────────────

// runThirdPartyProber fires Prober.RunOnce at ThirdPartyInterval on every
// replica (no leader election — see design comment in Start).
func (s *Scheduler) runThirdPartyProber(ctx context.Context) {
	ticker := time.NewTicker(s.cfg.ThirdPartyInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			targets, err := s.cfg.Targets.ListProbeTargets(ctx)
			if err != nil {
				slog.Error("scheduler: list probe targets for 15C", "error", err)
				continue
			}
			if len(targets) == 0 {
				continue
			}
			// Use a fresh context so ctx cancellation does not abort a running
			// prober call (the degradation contract: result is "no data" on
			// context error, which 15D treats as unknown, not failure).
			probeCtx, cancel := context.WithTimeout(context.Background(), s.cfg.TickTimeout)
			if err := s.cfg.Prober.RunOnce(probeCtx, targets); err != nil {
				slog.Error("scheduler: 15C prober error", "error", err)
			}
			cancel()
		}
	}
}