pangolin/server/internal/scheduler/orchestrate/replacer_test.go

package orchestrate_test

import (
	"context"
	"encoding/json"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/alicebob/miniredis/v2"
	"github.com/redis/go-redis/v9"

	"github.com/wangjia/pangolin/server/internal/alert"
	"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
)

// ─────────────────────────────────────────────────────────────────────────────
// In-process Redis
// ─────────────────────────────────────────────────────────────────────────────

func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
	t.Helper()
	mr := miniredis.RunT(t)
	rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
	return rdb, mr
}

// ─────────────────────────────────────────────────────────────────────────────
// Mock ProvisionService
// ─────────────────────────────────────────────────────────────────────────────

type createCall struct {
	Spec           orchestrate.NodeSpec
	IdempotencyKey string
	ReturnedID     string
}

type mockProvision struct {
	mu  sync.Mutex
	seq int

	providers    []orchestrate.ProviderInfo
	createCalls  []createCall
	destroyCalls []string
	createErr    error

	// idem maps idempotency key → nodeID (simulates provider idempotency).
	idem map[string]string
}

func newMockProvision(providers ...orchestrate.ProviderInfo) *mockProvision {
	return &mockProvision{
		providers: providers,
		idem:      map[string]string{},
	}
}

func (m *mockProvision) CreateNode(_ context.Context, spec orchestrate.NodeSpec, idemKey string) (string, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.createErr != nil {
		return "", m.createErr
	}
	// Idempotency replay.
	if existing, ok := m.idem[idemKey]; ok {
		m.createCalls = append(m.createCalls, createCall{spec, idemKey, existing})
		return existing, nil
	}
	m.seq++
	nodeID := fmt.Sprintf("new-node-%d", m.seq)
	m.idem[idemKey] = nodeID
	m.createCalls = append(m.createCalls, createCall{spec, idemKey, nodeID})
	return nodeID, nil
}

func (m *mockProvision) DestroyNode(_ context.Context, nodeID string) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.destroyCalls = append(m.destroyCalls, nodeID)
	return nil
}

func (m *mockProvision) RotateIP(_ context.Context, _ string) (string, error) { return "", nil }

func (m *mockProvision) ListProviders(_ context.Context, _, _ string) ([]orchestrate.ProviderInfo, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	return append([]orchestrate.ProviderInfo{}, m.providers...), nil
}

func (m *mockProvision) createCount() int {
	m.mu.Lock()
	defer m.mu.Unlock()
	return len(m.createCalls)
}

func (m *mockProvision) destroyCount() int {
	m.mu.Lock()
	defer m.mu.Unlock()
	return len(m.destroyCalls)
}

// ─────────────────────────────────────────────────────────────────────────────
// Mock LifecycleService
// ─────────────────────────────────────────────────────────────────────────────

type transitionEvent struct {
	NodeID, From, To string
}

type mockLC struct {
	mu          sync.Mutex
	nodes       map[string]*orchestrate.NodeInfo
	weights     map[string]int
	version     int64
	auditLogs   []string
	transitions []transitionEvent
}

func newMockLC(nodes ...*orchestrate.NodeInfo) *mockLC {
	m := &mockLC{
		nodes:   make(map[string]*orchestrate.NodeInfo),
		weights: make(map[string]int),
	}
	for _, n := range nodes {
		cp := *n
		m.nodes[n.ID] = &cp
	}
	return m
}

func (m *mockLC) addNode(n *orchestrate.NodeInfo) {
	m.mu.Lock()
	defer m.mu.Unlock()
	cp := *n
	m.nodes[n.ID] = &cp
}

func (m *mockLC) GetNode(_ context.Context, nodeID string) (*orchestrate.NodeInfo, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	n, ok := m.nodes[nodeID]
	if !ok {
		return nil, nil
	}
	cp := *n
	return &cp, nil
}

func (m *mockLC) TransitionStatus(_ context.Context, nodeID, from, to string, _ map[string]any) (int, error) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.transitions = append(m.transitions, transitionEvent{nodeID, from, to})
	return 1, nil // always succeed in tests
}

func (m *mockLC) SetWeight(_ context.Context, nodeID string, weight int) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.weights[nodeID] = weight
	return nil
}

func (m *mockLC) BumpVersion(_ context.Context) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.version++
	return nil
}

func (m *mockLC) WriteAuditLog(_ context.Context, actor, action, target, _ string) error {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.auditLogs = append(m.auditLogs, actor+"|"+action+"|"+target)
	return nil
}

func (m *mockLC) weightOf(nodeID string) int {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.weights[nodeID]
}

func (m *mockLC) versionOf() int64 {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.version
}

func (m *mockLC) hasAudit(entry string) bool {
	m.mu.Lock()
	defer m.mu.Unlock()
	for _, l := range m.auditLogs {
		if l == entry {
			return true
		}
	}
	return false
}

// ─────────────────────────────────────────────────────────────────────────────
// Mock ProbeSnapshotter
// ─────────────────────────────────────────────────────────────────────────────

type mockSnaps struct {
	mu   sync.Mutex
	data map[string]map[string]probe.ProbeSnapshot
}

func (s *mockSnaps) SnapshotsByNode(_ context.Context, nodeID string) (map[string]probe.ProbeSnapshot, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	d, ok := s.data[nodeID]
	if !ok {
		return nil, nil
	}
	out := make(map[string]probe.ProbeSnapshot, len(d))
	for k, v := range d {
		out[k] = v
	}
	return out, nil
}

func (s *mockSnaps) setPass(nodeID string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.data == nil {
		s.data = make(map[string]map[string]probe.ProbeSnapshot)
	}
	s.data[nodeID] = passingSnapshots()
}

// passingSnapshots returns a snapshot set where 2/3 domestic ISPs pass + overseas OK.
func passingSnapshots() map[string]probe.ProbeSnapshot {
	return map[string]probe.ProbeSnapshot{
		"CN:Telecom": {
			Vantage: probe.VantagePoint{Country: "CN", ISP: "ChinaTelecom"},
			Report: probe.NodeReport{
				L1: probe.L1Result{OK: true},
				L3: &probe.L3Result{OK: true},
			},
		},
		"CN:Unicom": {
			Vantage: probe.VantagePoint{Country: "CN", ISP: "ChinaUnicom"},
			Report: probe.NodeReport{
				L1: probe.L1Result{OK: true},
				L3: &probe.L3Result{OK: true},
			},
		},
		"CN:Mobile": {
			Vantage: probe.VantagePoint{Country: "CN", ISP: "ChinaMobile"},
			Report: probe.NodeReport{
				L1: probe.L1Result{OK: false},
				L3: &probe.L3Result{OK: false},
			},
		},
		"SG:AWS": {
			Vantage: probe.VantagePoint{Country: "SG", ISP: "AWS"},
			Report:  probe.NodeReport{L1: probe.L1Result{OK: true}},
		},
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Mock Notifier
// ─────────────────────────────────────────────────────────────────────────────

type mockNotifier struct {
	mu     sync.Mutex
	events []alert.Event
}

func (n *mockNotifier) Notify(_ context.Context, e alert.Event) error {
	n.mu.Lock()
	defer n.mu.Unlock()
	n.events = append(n.events, e)
	return nil
}

// count returns the number of Notify calls received.
func (n *mockNotifier) count() int {
	n.mu.Lock()
	defer n.mu.Unlock()
	return len(n.events)
}

// countByType returns the number of Notify calls with the given EventType.
func (n *mockNotifier) countByType(t alert.EventType) int {
	n.mu.Lock()
	defer n.mu.Unlock()
	c := 0
	for _, e := range n.events {
		if e.Type == t {
			c++
		}
	}
	return c
}

// ─────────────────────────────────────────────────────────────────────────────
// Mock Clock
// ─────────────────────────────────────────────────────────────────────────────

type mockClock struct {
	mu  sync.Mutex
	now time.Time
}

func newMockClock(t time.Time) *mockClock { return &mockClock{now: t} }

func (c *mockClock) Now() time.Time {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.now
}

func (c *mockClock) advance(d time.Duration) {
	c.mu.Lock()
	defer c.mu.Unlock()
	c.now = c.now.Add(d)
}

// ─────────────────────────────────────────────────────────────────────────────
// Test harness
// ─────────────────────────────────────────────────────────────────────────────

type harness struct {
	rdb      *redis.Client
	prov     *mockProvision
	lc       *mockLC
	snaps    *mockSnaps
	notifier *mockNotifier
	clock    *mockClock
	replacer *orchestrate.Replacer
}

func newHarness(t *testing.T, nodes ...*orchestrate.NodeInfo) *harness {
	t.Helper()
	rdb, _ := newTestRedis(t)
	prov := newMockProvision(
		orchestrate.ProviderInfo{ID: "provider-A"},
		orchestrate.ProviderInfo{ID: "provider-B"},
		orchestrate.ProviderInfo{ID: "provider-C"},
	)
	lc := newMockLC(nodes...)
	snaps := &mockSnaps{}
	notifier := &mockNotifier{}
	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())

	r := orchestrate.NewReplacer(orchestrate.Config{
		RDB:      rdb,
		Prov:     prov,
		LC:       lc,
		Snaps:    snaps,
		Notifier: notifier,
		Clock:    clock,
	})
	return &harness{rdb: rdb, prov: prov, lc: lc, snaps: snaps, notifier: notifier, clock: clock, replacer: r}
}

func (h *harness) pushQueue(t *testing.T, oldNodeID, replacementUUID string) {
	t.Helper()
	data, _ := json.Marshal(map[string]string{
		"nodeId":          oldNodeID,
		"replacementUuid": replacementUUID,
	})
	if err := h.rdb.LPush(context.Background(), "detect:replace:queue", data).Err(); err != nil {
		t.Fatalf("pushQueue: %v", err)
	}
}

func (h *harness) tick(t *testing.T) {
	t.Helper()
	if err := h.replacer.Tick(context.Background()); err != nil {
		t.Fatalf("Tick() error: %v", err)
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// State-machine phase explanation (for reference):
//
//   Tick 1: drainQueue creates pending record → advanceAll: stepPending → creating
//   Tick 2: stepCreating → CreateNode called → probing
//   Tick 3: stepProbing (probeStreak=1, need 2)
//   Tick 4: stepProbing (probeStreak=2) → activating
//   Tick 5: stepActivating → SetWeight(10)+TransitionStatus+BumpVersion+grayscale → draining_old
//   Tick 6: stepDrainingOld → DestroyNode(oldNode) → done
// ─────────────────────────────────────────────────────────────────────────────

// ─────────────────────────────────────────────────────────────────────────────
// Test: happy-path replacement (6 ticks)
// ─────────────────────────────────────────────────────────────────────────────

func TestHappyPathReplacement(t *testing.T) {
	const (
		oldNode         = "old-node-001"
		replacementUUID = "uuid-happy"
	)
	h := newHarness(t,
		&orchestrate.NodeInfo{ID: oldNode, Tier: "free", Region: "hkg", Role: "entry"},
	)
	h.pushQueue(t, oldNode, replacementUUID)

	// Tick 1: pending → creating (CreateNode not yet called).
	h.tick(t)
	if h.prov.createCount() != 0 {
		t.Errorf("tick 1: CreateNode should not be called yet; got %d calls", h.prov.createCount())
	}

	// Tick 2: creating → CreateNode → probing.
	h.tick(t)
	if h.prov.createCount() != 1 {
		t.Fatalf("tick 2: CreateNode calls = %d; want 1", h.prov.createCount())
	}
	call0 := h.prov.createCalls[0]
	// The idempotency key for attempt 0 must equal the replacement UUID.
	if call0.IdempotencyKey != replacementUUID {
		t.Errorf("CreateNode idempotency key = %q; want %q", call0.IdempotencyKey, replacementUUID)
	}

	newNodeID := call0.ReturnedID
	// Simulate agent self-registration: add the new node to the lifecycle mock.
	h.lc.addNode(&orchestrate.NodeInfo{ID: newNodeID, Tier: "free", Region: "hkg"})
	// Set probe snapshots to PASS for the new node.
	h.snaps.setPass(newNodeID)

	h.tick(t) // Tick 3: probing streak=1 (< 2 required).
	h.tick(t) // Tick 4: probing streak=2 → phase saved as activating.
	h.tick(t) // Tick 5: activating → SetWeight(10) + TransitionStatus(probing→up) + BumpVersion + grayscale → draining_old.

	// After stepActivating:
	if w := h.lc.weightOf(newNodeID); w != 10 {
		t.Errorf("new node weight after activating = %d; want 10", w)
	}
	if h.lc.versionOf() == 0 {
		t.Error("BumpVersion was not called after activating")
	}

	h.tick(t) // Tick 6: draining_old → DestroyNode(oldNode) → done.

	// Old node must be destroyed.
	oldDestroyed := false
	for _, id := range h.prov.destroyCalls {
		if id == oldNode {
			oldDestroyed = true
			break
		}
	}
	if !oldDestroyed {
		t.Errorf("old node %s not destroyed; destroyCalls = %v", oldNode, h.prov.destroyCalls)
	}

	// Grayscale record must exist for the new node.
	if _, err := h.rdb.Get(context.Background(), "sched:gray:"+newNodeID).Result(); err == redis.Nil {
		t.Errorf("grayscale record for %s not found", newNodeID)
	}

	// Audit log must contain replacement_done.
	if !h.lc.hasAudit("orchestrate|replacement_done|node:" + oldNode) {
		t.Errorf("replacement_done audit log not found; got: %v", h.lc.auditLogs)
	}

	// New node must NOT have been destroyed.
	for _, id := range h.prov.destroyCalls {
		if id == newNodeID {
			t.Errorf("new node %s was incorrectly destroyed", newNodeID)
		}
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Test: probe failure exhausts MaxAttempts → failed + alert (7 ticks + 3 clock advances)
// ─────────────────────────────────────────────────────────────────────────────

func TestProbeFailMaxAttempts(t *testing.T) {
	const (
		oldNode         = "old-node-fail"
		replacementUUID = "uuid-fail"
	)
	h := newHarness(t,
		&orchestrate.NodeInfo{ID: oldNode, Tier: "free", Region: "hkg", Role: "entry"},
	)
	h.pushQueue(t, oldNode, replacementUUID)

	ctx := context.Background()

	// ── Attempt 0 ────────────────────────────────────────────────────────────

	h.tick(t) // Tick 1: pending → creating.
	h.tick(t) // Tick 2: creating → CreateNode(idem=uuid) → probing.

	if h.prov.createCount() != 1 {
		t.Fatalf("attempt 0: CreateNode calls = %d; want 1", h.prov.createCount())
	}
	if h.prov.createCalls[0].IdempotencyKey != replacementUUID {
		t.Errorf("attempt 0 idem key = %q; want %q",
			h.prov.createCalls[0].IdempotencyKey, replacementUUID)
	}

	newNode0 := h.prov.createCalls[0].ReturnedID
	h.snaps.setPass(newNode0) // even if snapshots pass, timeout overrides

	h.clock.advance(orchestrate.ProbeTimeout + time.Second) // cause timeout

	h.tick(t) // Tick 3: probing → timeout → DestroyNode(newNode0) → creating (attempts=1).

	if h.prov.destroyCount() != 1 {
		t.Fatalf("after attempt 0 timeout: DestroyNode calls = %d; want 1", h.prov.destroyCount())
	}
	if h.prov.destroyCalls[0] != newNode0 {
		t.Errorf("attempt 0: expected %s destroyed; got %s", newNode0, h.prov.destroyCalls[0])
	}
	// Old node must NOT have been destroyed yet.
	for _, id := range h.prov.destroyCalls {
		if id == oldNode {
			t.Errorf("old node destroyed during attempt 0 (want: only after success)")
		}
	}

	// ── Attempt 1 ────────────────────────────────────────────────────────────

	h.tick(t) // Tick 4: creating → CreateNode(idem=uuid:retry:1) → probing.

	if h.prov.createCount() != 2 {
		t.Fatalf("attempt 1: CreateNode calls = %d; want 2", h.prov.createCount())
	}
	call1 := h.prov.createCalls[1]
	if call1.IdempotencyKey != replacementUUID+":retry:1" {
		t.Errorf("attempt 1 idem key = %q; want %q:retry:1",
			call1.IdempotencyKey, replacementUUID)
	}

	newNode1 := call1.ReturnedID
	h.snaps.setPass(newNode1)
	h.clock.advance(orchestrate.ProbeTimeout + time.Second)

	h.tick(t) // Tick 5: probing → timeout → DestroyNode(newNode1) → creating (attempts=2).

	if h.prov.destroyCount() != 2 {
		t.Fatalf("after attempt 1 timeout: DestroyNode calls = %d; want 2", h.prov.destroyCount())
	}

	// ── Attempt 2 (final) ────────────────────────────────────────────────────

	h.tick(t) // Tick 6: creating → CreateNode(idem=uuid:retry:2) → probing.

	if h.prov.createCount() != 3 {
		t.Fatalf("attempt 2: CreateNode calls = %d; want 3", h.prov.createCount())
	}
	call2 := h.prov.createCalls[2]
	if call2.IdempotencyKey != replacementUUID+":retry:2" {
		t.Errorf("attempt 2 idem key = %q; want %q:retry:2",
			call2.IdempotencyKey, replacementUUID)
	}

	newNode2 := call2.ReturnedID
	h.snaps.setPass(newNode2)
	h.clock.advance(orchestrate.ProbeTimeout + time.Second)

	h.tick(t) // Tick 7: probing → timeout → DestroyNode(newNode2) → FAILED.

	if h.prov.destroyCount() != 3 {
		t.Fatalf("after attempt 2 timeout: DestroyNode calls = %d; want 3", h.prov.destroyCount())
	}

	// Old node must never have been destroyed.
	for _, id := range h.prov.destroyCalls {
		if id == oldNode {
			t.Errorf("old node %s was incorrectly destroyed during failed replacement", oldNode)
		}
	}

	// Notify(EventTypeReplenishFailed) must be called exactly once.
	if n := h.notifier.countByType(alert.EventTypeReplenishFailed); n != 1 {
		t.Errorf("Notify(ReplenishFailed) calls = %d; want 1", n)
	}

	// Record must be in failed phase.
	raw, err := h.rdb.Get(ctx, "sched:replace:"+replacementUUID).Result()
	if err != nil {
		t.Fatalf("read final record: %v", err)
	}
	var p struct {
		Phase orchestrate.Phase `json:"phase"`
	}
	if err := json.Unmarshal([]byte(raw), &p); err != nil {
		t.Fatalf("unmarshal phase: %v", err)
	}
	if p.Phase != orchestrate.PhaseFailed {
		t.Errorf("final phase = %q; want %q", p.Phase, orchestrate.PhaseFailed)
	}

	// Each attempt must have used a different provider (rotation check).
	p0 := h.prov.createCalls[0].Spec.ProviderID
	p1 := h.prov.createCalls[1].Spec.ProviderID
	p2 := h.prov.createCalls[2].Spec.ProviderID
	if p0 == p1 || p1 == p2 || p0 == p2 {
		t.Errorf("providers should differ across retries; got [%s, %s, %s]", p0, p1, p2)
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Test: crash recovery — CreateNode NOT called again when creating is complete
//
// Simulates a crash where CreateNode succeeded and newNode was set in memory
// but the record was NOT yet saved with phase=probing. On restart the record
// shows phase=creating with newNode set; we expect the Replacer to resume
// from probing without calling CreateNode again.
// ─────────────────────────────────────────────────────────────────────────────

func TestCrashRecovery(t *testing.T) {
	const (
		oldNode          = "old-node-crash"
		replacementUUID  = "uuid-crash"
		simulatedNewNode = "new-node-crash"
	)
	h := newHarness(t,
		&orchestrate.NodeInfo{ID: oldNode, Tier: "free", Region: "hkg", Role: "entry"},
	)

	ctx := context.Background()

	// Inject a pre-crash record: phase=creating, newNode already set.
	// This models a crash AFTER CreateNode returned but BEFORE the record was
	// saved with phase=probing.
	type crashRecord struct {
		Phase             string    `json:"phase"`
		OldNode           string    `json:"oldNode"`
		NewNode           string    `json:"newNode"`
		CurrentProviderID string    `json:"currentProviderId"`
		Attempts          int       `json:"attempts"`
		ProviderTried     []string  `json:"providerTried"`
		ProbeStreak       int       `json:"probeStreak"`
		PhaseStartedAt    time.Time `json:"phaseStartedAt"`
		UpdatedAt         time.Time `json:"updatedAt"`
	}
	recJSON, _ := json.Marshal(crashRecord{
		Phase:          "creating",
		OldNode:        oldNode,
		NewNode:        simulatedNewNode,
		ProviderTried:  []string{},
		PhaseStartedAt: h.clock.Now(),
		UpdatedAt:      h.clock.Now(),
	})
	if err := h.rdb.Set(ctx, "sched:replace:"+replacementUUID, recJSON, 0).Err(); err != nil {
		t.Fatalf("inject crash record: %v", err)
	}
	if err := h.rdb.SAdd(ctx, "sched:replace:index", replacementUUID).Err(); err != nil {
		t.Fatalf("inject index: %v", err)
	}

	// Add the simulated new node to the lifecycle mock.
	h.lc.addNode(&orchestrate.NodeInfo{ID: simulatedNewNode, Tier: "free", Region: "hkg"})
	h.snaps.setPass(simulatedNewNode)

	// Tick 1 ("restart"): phase=creating, newNode already set → skip CreateNode → probing.
	h.tick(t)
	if h.prov.createCount() != 0 {
		t.Fatalf("tick 1 (recovery): CreateNode called %d times; want 0", h.prov.createCount())
	}

	h.tick(t) // Tick 2: probing streak=1.
	h.tick(t) // Tick 3: probing streak=2 → activating.
	h.tick(t) // Tick 4: activating → SetWeight(10) + TransitionStatus + BumpVersion + grayscale → draining_old.
	h.tick(t) // Tick 5: draining_old → DestroyNode(oldNode) → done.

	// CreateNode must never have been called.
	if h.prov.createCount() != 0 {
		t.Errorf("CreateNode called %d times during crash recovery; want 0", h.prov.createCount())
	}

	// New node must be at weight 10.
	if w := h.lc.weightOf(simulatedNewNode); w != 10 {
		t.Errorf("new node weight = %d; want 10", w)
	}

	// Old node must be destroyed.
	oldDestroyed := false
	for _, id := range h.prov.destroyCalls {
		if id == oldNode {
			oldDestroyed = true
			break
		}
	}
	if !oldDestroyed {
		t.Errorf("old node %s not destroyed in crash-recovery path; destroyCalls = %v",
			oldNode, h.prov.destroyCalls)
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Test: idempotency — duplicate queue entries produce a single replacement
// ─────────────────────────────────────────────────────────────────────────────

func TestQueueReplayIdempotent(t *testing.T) {
	const (
		oldNode         = "old-node-idem"
		replacementUUID = "uuid-idem"
	)
	h := newHarness(t,
		&orchestrate.NodeInfo{ID: oldNode, Tier: "free", Region: "hkg"},
	)

	// Push the same entry twice (duplicate delivery).
	h.pushQueue(t, oldNode, replacementUUID)
	h.pushQueue(t, oldNode, replacementUUID)

	// Tick 1: both entries drained; SetNX ensures a single record; stepPending → creating.
	h.tick(t)
	if h.prov.createCount() != 0 {
		t.Errorf("tick 1: CreateNode should not be called yet; got %d", h.prov.createCount())
	}

	// Tick 2: creating → CreateNode called exactly once.
	h.tick(t)
	if h.prov.createCount() != 1 {
		t.Errorf("CreateNode calls = %d after two queue entries; want 1", h.prov.createCount())
	}

	// Only one UUID must be in the in-flight index.
	members, err := h.rdb.SMembers(context.Background(), "sched:replace:index").Result()
	if err != nil {
		t.Fatalf("smembers: %v", err)
	}
	if len(members) != 1 {
		t.Errorf("index member count = %d; want 1", len(members))
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Test: grayscale weight ramp 10 → 25 → 50 → 75 → 100 (each step = 6 h)
// ─────────────────────────────────────────────────────────────────────────────

func TestGrayscaleAdvance(t *testing.T) {
	const nodeID = "gray-node-001"

	rdb, _ := newTestRedis(t)
	lc := newMockLC(&orchestrate.NodeInfo{ID: nodeID})
	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())

	// Seed a grayscale record at weight 10.
	grayKey := "sched:gray:" + nodeID
	type grayRec struct {
		NodeID         string    `json:"nodeId"`
		CurrentWeight  int       `json:"currentWeight"`
		StartedAt      time.Time `json:"startedAt"`
		LastAdvancedAt time.Time `json:"lastAdvancedAt"`
	}
	data, _ := json.Marshal(grayRec{
		NodeID:         nodeID,
		CurrentWeight:  10,
		StartedAt:      clock.Now(),
		LastAdvancedAt: clock.Now(),
	})
	if err := rdb.Set(context.Background(), grayKey, data, 0).Err(); err != nil {
		t.Fatalf("seed gray record: %v", err)
	}

	g := orchestrate.NewGrayscale(rdb, lc, clock)
	ctx := context.Background()

	// Advance the clock by GrayscaleInterval (6 h) and call Advance() four times.
	// Expected progression: 10 → 25 → 50 → 75 → 100 (record deleted).
	wantWeights := []int{25, 50, 75, 100}

	for step, want := range wantWeights {
		clock.advance(orchestrate.GrayscaleInterval)

		if err := g.Advance(ctx); err != nil {
			t.Fatalf("step %d Advance(): %v", step, err)
		}

		if got := lc.weightOf(nodeID); got != want {
			t.Errorf("step %d: weight = %d; want %d", step, got, want)
		}

		// Audit log must record the weight advance.
		auditKey := "grayscale|weight_advanced|node:" + nodeID
		if !lc.hasAudit(auditKey) {
			t.Errorf("step %d: audit log %q not found; logs = %v", step, auditKey, lc.auditLogs)
		}
	}

	// After weight 100, the grayscale record must be deleted.
	if _, err := rdb.Get(ctx, grayKey).Result(); err != redis.Nil {
		t.Error("grayscale record should be deleted after weight 100")
	}

	// Final weight must be 100.
	if lc.weightOf(nodeID) != 100 {
		t.Errorf("final weight = %d; want 100", lc.weightOf(nodeID))
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// Test: grayscale does not advance before the interval elapses
// ─────────────────────────────────────────────────────────────────────────────

func TestGrayscaleNoAdvanceBeforeInterval(t *testing.T) {
	const nodeID = "gray-node-wait"

	rdb, _ := newTestRedis(t)
	lc := newMockLC(&orchestrate.NodeInfo{ID: nodeID})
	clock := newMockClock(time.Unix(1_700_000_000, 0).UTC())

	grayKey := "sched:gray:" + nodeID
	type grayRec struct {
		NodeID         string    `json:"nodeId"`
		CurrentWeight  int       `json:"currentWeight"`
		StartedAt      time.Time `json:"startedAt"`
		LastAdvancedAt time.Time `json:"lastAdvancedAt"`
	}
	data, _ := json.Marshal(grayRec{
		NodeID:         nodeID,
		CurrentWeight:  10,
		StartedAt:      clock.Now(),
		LastAdvancedAt: clock.Now(),
	})
	rdb.Set(context.Background(), grayKey, data, 0)

	g := orchestrate.NewGrayscale(rdb, lc, clock)

	// Advance only 5 h (< 6 h interval).
	clock.advance(5 * time.Hour)

	if err := g.Advance(context.Background()); err != nil {
		t.Fatalf("Advance(): %v", err)
	}

	// Weight must NOT have been updated.
	if w := lc.weightOf(nodeID); w != 0 {
		t.Errorf("weight was updated too early: got %d; want 0", w)
	}
}