package provision import ( "context" "errors" "testing" ) // TestReplace_MakeBeforeBreak asserts the full state-machine migration and that // capacity never dips: the new node reaches `up` BEFORE the old node leaves the // directory (enters `draining`). func TestReplace_MakeBeforeBreak(t *testing.T) { h := newHarness(t) ctx := context.Background() old, err := h.svc.CreateNode(ctx, proSpec(), "old-node") if err != nil { t.Fatalf("CreateNode old: %v", err) } if err := h.store.UpdateNodeStatus(ctx, old.ID, StatusUp); err != nil { t.Fatalf("seed up: %v", err) } bootsBefore := h.adapter.createCount() res, err := h.svc.Replace(ctx, old.ID, "") if err != nil { t.Fatalf("Replace: %v", err) } if res.NewNodeID == 0 || res.NewNodeID == old.ID { t.Fatalf("bad new node id: %d", res.NewNodeID) } if h.adapter.createCount() != bootsBefore+1 { t.Errorf("expected exactly one new boot, got delta %d", h.adapter.createCount()-bootsBefore) } // Capacity invariant: new node `up` seq < old node `draining` seq. newUpSeq := h.store.firstSeqForStatus(res.NewNodeID, StatusUp) oldDrainSeq := h.store.firstSeqForStatus(old.ID, StatusDraining) if newUpSeq < 0 { t.Fatal("new node never reached up") } if oldDrainSeq < 0 { t.Fatal("old node never drained") } if !(newUpSeq < oldDrainSeq) { t.Errorf("capacity dipped: new up seq %d not before old draining seq %d", newUpSeq, oldDrainSeq) } // Old node finally destroyed. oldNode, _ := h.store.GetNode(ctx, old.ID) if oldNode.Status != StatusDestroyed { t.Errorf("old node status = %s, want destroyed", oldNode.Status) } newNode, _ := h.store.GetNode(ctx, res.NewNodeID) if newNode.Status != StatusUp { t.Errorf("new node status = %s, want up", newNode.Status) } // pro pool drains on a timer. if len(h.clock.slept) == 0 { t.Error("expected a drain sleep for premium pool") } } // TestReplace_FreeHardCut: consumable/free pool drains by hard cut (no sleep). func TestReplace_FreeHardCut(t *testing.T) { h := newHarness(t) ctx := context.Background() old, _ := h.svc.CreateNode(ctx, freeSpec(), "old-free") _ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp) if _, err := h.svc.Replace(ctx, old.ID, ""); err != nil { t.Fatalf("Replace: %v", err) } if len(h.clock.slept) != 0 { t.Errorf("free pool should hard-cut, but slept %v", h.clock.slept) } } // TestReplace_ProbeTimeout: probing fails → new node destroyed, fail counted, // alert fired, replacement marked failed, old node untouched. func TestReplace_ProbeTimeout(t *testing.T) { h := newHarness(t) ctx := context.Background() h.prober.err = errors.New("probe timeout") old, _ := h.svc.CreateNode(ctx, proSpec(), "old-pt") _ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp) _, err := h.svc.Replace(ctx, old.ID, "") if err == nil { t.Fatal("expected probe timeout error") } if h.svc.FailCount(PoolPremium) != 1 { t.Errorf("fail count = %d, want 1", h.svc.FailCount(PoolPremium)) } if h.alert.count(AlertProbeTimeout) != 1 { t.Errorf("probe_timeout alerts = %d, want 1", h.alert.count(AlertProbeTimeout)) } // Old node must still be up (capacity protected — we did not drain it). oldNode, _ := h.store.GetNode(ctx, old.ID) if oldNode.Status != StatusUp { t.Errorf("old node status = %s, want up (untouched)", oldNode.Status) } // The failed new node should be destroyed (bad IP not left running). if h.adapter.destroyCount() != 1 { t.Errorf("failed new node not destroyed: destroy count %d", h.adapter.destroyCount()) } } // TestReplace_CrashResume: a replacement that crashed after booting the new node // (idempotency key already saved) must resume WITHOUT a second boot. func TestReplace_CrashResume(t *testing.T) { h := newHarness(t) ctx := context.Background() old, _ := h.svc.CreateNode(ctx, freeSpec(), "old-cr") _ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp) ruuid := "fixed-replacement-uuid" // Simulate the pre-crash state: the new node was already booted under the // orchestration's idempotency key, and a running replacement record exists // at the open_new step (new_node_id not yet persisted). spec, err := h.svc.replacementSpec(ctx, mustNode(t, h, old.ID), PoolConsumable) if err != nil { t.Fatalf("replacementSpec: %v", err) } preNode, err := h.svc.CreateNode(ctx, spec, ruuid+":create") if err != nil { t.Fatalf("pre-boot new node: %v", err) } if err := h.store.CreateReplacement(ctx, &Replacement{ UUID: ruuid, OldNodeID: old.ID, Pool: PoolConsumable, Status: ReplaceRunning, Step: StepOpenNew, }); err != nil { t.Fatalf("seed replacement: %v", err) } bootsBefore := h.adapter.createCount() // old + pre-booted new = 2 res, err := h.svc.Replace(ctx, old.ID, ruuid) if err != nil { t.Fatalf("resume Replace: %v", err) } if h.adapter.createCount() != bootsBefore { t.Errorf("resume re-booted a node: delta %d (want 0)", h.adapter.createCount()-bootsBefore) } if res.NewNodeID != preNode.ID { t.Errorf("resume used a different new node: got %d, want %d", res.NewNodeID, preNode.ID) } oldNode, _ := h.store.GetNode(ctx, old.ID) if oldNode.Status != StatusDestroyed { t.Errorf("old node not destroyed after resume: %s", oldNode.Status) } } // TestReplace_ReplayCompleted: replaying a finished replacement is a no-op. func TestReplace_ReplayCompleted(t *testing.T) { h := newHarness(t) ctx := context.Background() old, _ := h.svc.CreateNode(ctx, freeSpec(), "old-rc") _ = h.store.UpdateNodeStatus(ctx, old.ID, StatusUp) res1, err := h.svc.Replace(ctx, old.ID, "") if err != nil { t.Fatalf("Replace: %v", err) } boots := h.adapter.createCount() destroys := h.adapter.destroyCount() res2, err := h.svc.Replace(ctx, old.ID, res1.ReplacementUUID) if err != nil { t.Fatalf("replay Replace: %v", err) } if res2.NewNodeID != res1.NewNodeID { t.Errorf("replay new node id mismatch: %d vs %d", res2.NewNodeID, res1.NewNodeID) } if h.adapter.createCount() != boots || h.adapter.destroyCount() != destroys { t.Errorf("replay caused vendor side effects: boots %d->%d destroys %d->%d", boots, h.adapter.createCount(), destroys, h.adapter.destroyCount()) } } // TestRotatePool replaces every up node in a pool with bounded concurrency. func TestRotatePool(t *testing.T) { h := newHarness(t) ctx := context.Background() a, _ := h.svc.CreateNode(ctx, freeSpec(), "rp-a") b, _ := h.svc.CreateNode(ctx, freeSpec(), "rp-b") _ = h.store.UpdateNodeStatus(ctx, a.ID, StatusUp) _ = h.store.UpdateNodeStatus(ctx, b.ID, StatusUp) results, err := h.svc.RotatePool(ctx, PoolConsumable, 2) if err != nil { t.Fatalf("RotatePool: %v", err) } if len(results) != 2 { t.Fatalf("results = %d, want 2", len(results)) } for _, id := range []int64{a.ID, b.ID} { n, _ := h.store.GetNode(ctx, id) if n.Status != StatusDestroyed { t.Errorf("node %d status = %s, want destroyed", id, n.Status) } } } func mustNode(t *testing.T, h *harness, id int64) *Node { t.Helper() n, err := h.store.GetNode(context.Background(), id) if err != nil || n == nil { t.Fatalf("get node %d: %v", id, err) } return n }