pangolin/server/internal/provision/replace_test.go

package provision

import (
	"context"
	"errors"
	"testing"
)

// TestReplace_MakeBeforeBreak asserts the full state-machine migration and that
// capacity never dips: the new node reaches `up` BEFORE the old node leaves the
// directory (enters `draining`).
func TestReplace_MakeBeforeBreak(t *testing.T) {
	h := newHarness(t)
	ctx := context.Background()

	old, err := h.svc.CreateNode(ctx, proSpec(), "old-node")
	if err != nil {
		t.Fatalf("CreateNode old: %v", err)
	}
	if err := h.store.UpdateNodeStatus(ctx, old.ID, StatusUp); err != nil {
		t.Fatalf("seed up: %v", err)
	}
	bootsBefore := h.adapter.createCount()

	res, err := h.svc.Replace(ctx, old.ID, "")
	if err != nil {
		t.Fatalf("Replace: %v", err)
	}
	if res.NewNodeID == 0 || res.NewNodeID == old.ID {
		t.Fatalf("bad new node id: %d", res.NewNodeID)
	}
	if h.adapter.createCount() != bootsBefore+1 {
		t.Errorf("expected exactly one new boot, got delta %d", h.adapter.createCount()-bootsBefore)
	}

	// Capacity invariant: new node `up` seq < old node `draining` seq.
	newUpSeq := h.store.firstSeqForStatus(res.NewNodeID, StatusUp)
	oldDrainSeq := h.store.firstSeqForStatus(old.ID, StatusDraining)
	if newUpSeq < 0 {
		t.Fatal("new node never reached up")
	}
	if oldDrainSeq < 0 {
		t.Fatal("old node never drained")
	}
	if !(newUpSeq < oldDrainSeq) {
		t.Errorf("capacity dipped: new up seq %d not before old draining seq %d", newUpSeq, oldDrainSeq)
	}

	// Old node finally destroyed.
	oldNode, _ := h.store.GetNode(ctx, old.ID)
	if oldNode.Status != StatusDestroyed {
		t.Errorf("old node status = %s, want destroyed", oldNode.Status)
	}
	newNode, _ := h.store.GetNode(ctx, res.NewNodeID)
	if newNode.Status != StatusUp {
		t.Errorf("new node status = %s, want up", newNode.Status)
	}
	// pro pool drains on a timer.
	if len(h.clock.slept) == 0 {
		t.Error("expected a drain sleep for premium pool")
	}
}

// TestReplace_FreeHardCut: consumable/free pool drains by hard cut (no sleep).
func TestReplace_FreeHardCut(t *testing.T) {
	h := newHarness(t)
	ctx := context.Background()
	old, _ := h.svc.CreateNode(ctx, freeSpec(), "old-free")
	_ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp)

	if _, err := h.svc.Replace(ctx, old.ID, ""); err != nil {
		t.Fatalf("Replace: %v", err)
	}
	if len(h.clock.slept) != 0 {
		t.Errorf("free pool should hard-cut, but slept %v", h.clock.slept)
	}
}

// TestReplace_ProbeTimeout: probing fails → new node destroyed, fail counted,
// alert fired, replacement marked failed, old node untouched.
func TestReplace_ProbeTimeout(t *testing.T) {
	h := newHarness(t)
	ctx := context.Background()
	h.prober.err = errors.New("probe timeout")

	old, _ := h.svc.CreateNode(ctx, proSpec(), "old-pt")
	_ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp)

	_, err := h.svc.Replace(ctx, old.ID, "")
	if err == nil {
		t.Fatal("expected probe timeout error")
	}
	if h.svc.FailCount(PoolPremium) != 1 {
		t.Errorf("fail count = %d, want 1", h.svc.FailCount(PoolPremium))
	}
	if h.alert.count(AlertProbeTimeout) != 1 {
		t.Errorf("probe_timeout alerts = %d, want 1", h.alert.count(AlertProbeTimeout))
	}
	// Old node must still be up (capacity protected — we did not drain it).
	oldNode, _ := h.store.GetNode(ctx, old.ID)
	if oldNode.Status != StatusUp {
		t.Errorf("old node status = %s, want up (untouched)", oldNode.Status)
	}
	// The failed new node should be destroyed (bad IP not left running).
	if h.adapter.destroyCount() != 1 {
		t.Errorf("failed new node not destroyed: destroy count %d", h.adapter.destroyCount())
	}
}

// TestReplace_CrashResume: a replacement that crashed after booting the new node
// (idempotency key already saved) must resume WITHOUT a second boot.
func TestReplace_CrashResume(t *testing.T) {
	h := newHarness(t)
	ctx := context.Background()

	old, _ := h.svc.CreateNode(ctx, freeSpec(), "old-cr")
	_ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp)

	ruuid := "fixed-replacement-uuid"
	// Simulate the pre-crash state: the new node was already booted under the
	// orchestration's idempotency key, and a running replacement record exists
	// at the open_new step (new_node_id not yet persisted).
	spec, err := h.svc.replacementSpec(ctx, mustNode(t, h, old.ID), PoolConsumable)
	if err != nil {
		t.Fatalf("replacementSpec: %v", err)
	}
	preNode, err := h.svc.CreateNode(ctx, spec, ruuid+":create")
	if err != nil {
		t.Fatalf("pre-boot new node: %v", err)
	}
	if err := h.store.CreateReplacement(ctx, &Replacement{
		UUID: ruuid, OldNodeID: old.ID, Pool: PoolConsumable,
		Status: ReplaceRunning, Step: StepOpenNew,
	}); err != nil {
		t.Fatalf("seed replacement: %v", err)
	}
	bootsBefore := h.adapter.createCount() // old + pre-booted new = 2

	res, err := h.svc.Replace(ctx, old.ID, ruuid)
	if err != nil {
		t.Fatalf("resume Replace: %v", err)
	}
	if h.adapter.createCount() != bootsBefore {
		t.Errorf("resume re-booted a node: delta %d (want 0)", h.adapter.createCount()-bootsBefore)
	}
	if res.NewNodeID != preNode.ID {
		t.Errorf("resume used a different new node: got %d, want %d", res.NewNodeID, preNode.ID)
	}
	oldNode, _ := h.store.GetNode(ctx, old.ID)
	if oldNode.Status != StatusDestroyed {
		t.Errorf("old node not destroyed after resume: %s", oldNode.Status)
	}
}

// TestReplace_ReplayCompleted: replaying a finished replacement is a no-op.
func TestReplace_ReplayCompleted(t *testing.T) {
	h := newHarness(t)
	ctx := context.Background()
	old, _ := h.svc.CreateNode(ctx, freeSpec(), "old-rc")
	_ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp)

	res1, err := h.svc.Replace(ctx, old.ID, "")
	if err != nil {
		t.Fatalf("Replace: %v", err)
	}
	boots := h.adapter.createCount()
	destroys := h.adapter.destroyCount()

	res2, err := h.svc.Replace(ctx, old.ID, res1.ReplacementUUID)
	if err != nil {
		t.Fatalf("replay Replace: %v", err)
	}
	if res2.NewNodeID != res1.NewNodeID {
		t.Errorf("replay new node id mismatch: %d vs %d", res2.NewNodeID, res1.NewNodeID)
	}
	if h.adapter.createCount() != boots || h.adapter.destroyCount() != destroys {
		t.Errorf("replay caused vendor side effects: boots %d->%d destroys %d->%d",
			boots, h.adapter.createCount(), destroys, h.adapter.destroyCount())
	}
}

// TestRotatePool replaces every up node in a pool with bounded concurrency.
func TestRotatePool(t *testing.T) {
	h := newHarness(t)
	ctx := context.Background()
	a, _ := h.svc.CreateNode(ctx, freeSpec(), "rp-a")
	b, _ := h.svc.CreateNode(ctx, freeSpec(), "rp-b")
	_ = h.store.UpdateNodeStatus(ctx, a.ID, StatusUp)
	_ = h.store.UpdateNodeStatus(ctx, b.ID, StatusUp)

	results, err := h.svc.RotatePool(ctx, PoolConsumable, 2)
	if err != nil {
		t.Fatalf("RotatePool: %v", err)
	}
	if len(results) != 2 {
		t.Fatalf("results = %d, want 2", len(results))
	}
	for _, id := range []int64{a.ID, b.ID} {
		n, _ := h.store.GetNode(ctx, id)
		if n.Status != StatusDestroyed {
			t.Errorf("node %d status = %s, want destroyed", id, n.Status)
		}
	}
}

func mustNode(t *testing.T, h *harness, id int64) *Node {
	t.Helper()
	n, err := h.store.GetNode(context.Background(), id)
	if err != nil || n == nil {
		t.Fatalf("get node %d: %v", id, err)
	}
	return n
}