//go:build integration // Integration tests for the Lifecycle state machine. // // Required: a running MySQL 8 instance with an empty database. // Set PANGOLIN_TEST_DSN to its DSN, e.g.: // // export PANGOLIN_TEST_DSN="root:secret@tcp(127.0.0.1:3306)/pangolin_test?parseTime=true" // go test -tags integration ./internal/nodes/... -run TestLifecycle -v // // Tests use miniredis in-process for the CRL — no separate Redis is needed. package nodes_test import ( "context" "database/sql" "errors" "os" "sync" "testing" "github.com/alicebob/miniredis/v2" "github.com/redis/go-redis/v9" "github.com/wangjia/pangolin/server/internal/config" "github.com/wangjia/pangolin/server/internal/mtls" "github.com/wangjia/pangolin/server/internal/nodes" "github.com/wangjia/pangolin/server/internal/store" ) // ─── Test infrastructure ───────────────────────────────────────────────────── // lifecycleFixture holds all resources for one lifecycle integration test. type lifecycleFixture struct { lc *nodes.Lifecycle db *sql.DB mr *miniredis.Miniredis rdb *redis.Client // same underlying server as mr } // requireDSN reads PANGOLIN_TEST_DSN from the environment. // If not set, the test is skipped with an informative message. // The DSN must include multiStatements=true (set automatically by TestMain // when it starts its own container; callers providing an external DSN should // include it too). func requireDSN(t *testing.T) string { t.Helper() dsn := os.Getenv("PANGOLIN_TEST_DSN") if dsn == "" { t.Skip("set PANGOLIN_TEST_DSN to run lifecycle integration tests " + "(e.g. root:secret@tcp(127.0.0.1:3306)/pangolin_test?multiStatements=true)") } return dsn } // setupLifecycle opens the MySQL database identified by PANGOLIN_TEST_DSN, // applies all migrations, seeds a provider row, and wires up a Lifecycle // backed by miniredis for the CRL. // // All resources are cleaned up via t.Cleanup. // // Implementation note: golang-migrate's MySQL driver calls (*sql.DB).Close() // when its own Close() is invoked (via m.Close() in newMigrator's cleanup). // To avoid having the migration step close the connection that the test will // use, we open a dedicated short-lived *sql.DB just for MigrateUp, close it // immediately after, and then open a fresh *sql.DB for test assertions. func setupLifecycle(t *testing.T) *lifecycleFixture { t.Helper() dsn := requireDSN(t) ctx := context.Background() // 1. Run migrations on a dedicated connection that golang-migrate will close. migDB, err := store.Open(&config.Config{DSN: dsn}) if err != nil { t.Fatalf("store.Open (migrate): %v", err) } if err := store.MigrateUp(migDB, "mysql"); err != nil { _ = migDB.Close() t.Fatalf("MigrateUp: %v", err) } _ = migDB.Close() // golang-migrate already closed it; this is a no-op but explicit. // 2. Open a fresh connection for the actual test operations. db, err := store.Open(&config.Config{DSN: dsn}) if err != nil { t.Fatalf("store.Open: %v", err) } t.Cleanup(func() { _ = db.Close() }) // Clean up nodes/events from any previous run so tests are idempotent. if _, err := db.ExecContext(ctx, `DELETE FROM node_events`); err != nil { t.Fatalf("truncate node_events: %v", err) } if _, err := db.ExecContext(ctx, `DELETE FROM nodes`); err != nil { t.Fatalf("truncate nodes: %v", err) } // Seed a provider row — nodes.provider_id is a FK. if _, err := db.ExecContext(ctx, `INSERT INTO providers (name, api_kind, regions, pool) VALUES ('test-provider', 'vultr', '["HK"]', 'consumable') ON DUPLICATE KEY UPDATE name = name`, ); err != nil { t.Fatalf("seed provider: %v", err) } mr := miniredis.RunT(t) rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()}) t.Cleanup(func() { _ = rdb.Close() }) crl := mtls.NewCRL(rdb, nil) // nil db — only Redis revocation needed in tests lc := nodes.NewLifecycle(db, crl, rdb) return &lifecycleFixture{lc: lc, db: db, mr: mr, rdb: rdb} } // insertNode inserts a test node with the given uuid and initial status. func insertNode(t *testing.T, db *sql.DB, uuid, status string) { t.Helper() ctx := context.Background() var providerID int64 if err := db.QueryRowContext(ctx, `SELECT id FROM providers LIMIT 1`, ).Scan(&providerID); err != nil { t.Fatalf("get provider_id: %v", err) } _, err := db.ExecContext(ctx, ` INSERT INTO nodes (uuid, region, name_zh, name_en, tier, endpoint, reality_pbk, reality_sni, provider_id, status) VALUES (?, 'HK', '香港测试', 'HK Test', 'free', '1.2.3.4:443', 'fake-pbk', 'sni.example.com', ?, ?) ON DUPLICATE KEY UPDATE status = VALUES(status), weight = 100 `, uuid, providerID, status) if err != nil { t.Fatalf("insertNode(%s, %s): %v", uuid, status, err) } } // nodeStatus reads the current status of a node by UUID. func nodeStatus(t *testing.T, db *sql.DB, uuid string) string { t.Helper() var status string if err := db.QueryRowContext(context.Background(), `SELECT status FROM nodes WHERE uuid = ?`, uuid, ).Scan(&status); err != nil { t.Fatalf("nodeStatus(%s): %v", uuid, err) } return status } // nodeWeight reads the current weight of a node by UUID. func nodeWeight(t *testing.T, db *sql.DB, uuid string) int { t.Helper() var w int if err := db.QueryRowContext(context.Background(), `SELECT weight FROM nodes WHERE uuid = ?`, uuid, ).Scan(&w); err != nil { t.Fatalf("nodeWeight(%s): %v", uuid, err) } return w } // eventCount returns the number of node_events rows for the node with the given UUID. func eventCount(t *testing.T, db *sql.DB, uuid string) int { t.Helper() var count int if err := db.QueryRowContext(context.Background(), ` SELECT COUNT(*) FROM node_events ne JOIN nodes n ON n.id = ne.node_id WHERE n.uuid = ? `, uuid).Scan(&count); err != nil { t.Fatalf("eventCount(%s): %v", uuid, err) } return count } // dirVersion reads the current directory_version. func dirVersion(t *testing.T, db *sql.DB) int64 { t.Helper() var v int64 if err := db.QueryRowContext(context.Background(), `SELECT version FROM directory_version WHERE id = 1`, ).Scan(&v); err != nil { t.Fatalf("dirVersion: %v", err) } return v } // ─── Category 1: all valid transitions succeed ─────────────────────────────── // TestLifecycle_AllValidTransitions verifies that every legal transition: // - Updates the node's status to the target value. // - Inserts exactly one node_events row. // - Increments the directory_version by 1. func TestLifecycle_AllValidTransitions(t *testing.T) { f := setupLifecycle(t) ctx := context.Background() detail := map[string]any{"source": "integration-test"} cases := []struct { name string uuid string start string fn func(string) error wantStatus string }{ { name: "provisioning→probing", uuid: "lc-probing", start: "provisioning", fn: func(u string) error { return f.lc.MarkProbing(ctx, u, detail) }, wantStatus: "probing", }, { name: "probing→up", uuid: "lc-up", start: "probing", fn: func(u string) error { return f.lc.MarkUp(ctx, u, detail) }, wantStatus: "up", }, { name: "up→draining", uuid: "lc-draining", start: "up", fn: func(u string) error { return f.lc.MarkDraining(ctx, u, detail) }, wantStatus: "draining", }, { name: "draining→down", uuid: "lc-down-from-drain", start: "draining", fn: func(u string) error { return f.lc.MarkDown(ctx, u, detail) }, wantStatus: "down", }, { name: "up→down (skip draining, blocked confirmed)", uuid: "lc-down-from-up", start: "up", fn: func(u string) error { return f.lc.MarkDown(ctx, u, detail) }, wantStatus: "down", }, { name: "probing→destroyed (bad IP, never pooled)", uuid: "lc-destroyed-probe", start: "probing", fn: func(u string) error { return f.lc.MarkDestroyed(ctx, u, detail) }, wantStatus: "destroyed", }, { name: "down→destroyed (permanent retirement)", uuid: "lc-destroyed-down", start: "down", fn: func(u string) error { return f.lc.MarkDestroyed(ctx, u, detail) }, wantStatus: "destroyed", }, } v0 := dirVersion(t, f.db) for _, tc := range cases { tc := tc t.Run(tc.name, func(t *testing.T) { insertNode(t, f.db, tc.uuid, tc.start) vBefore := dirVersion(t, f.db) if err := tc.fn(tc.uuid); err != nil { t.Fatalf("transition: %v", err) } // Status must have reached the target state. if got := nodeStatus(t, f.db, tc.uuid); got != tc.wantStatus { t.Errorf("status = %q, want %q", got, tc.wantStatus) } // Exactly one event row per transition. if n := eventCount(t, f.db, tc.uuid); n != 1 { t.Errorf("event_count = %d, want 1", n) } // Directory version incremented by exactly 1. vAfter := dirVersion(t, f.db) if vAfter != vBefore+1 { t.Errorf("version delta = %d, want 1 (before=%d after=%d)", vAfter-vBefore, vBefore, vAfter) } }) } // Total version bumps must equal the number of valid transitions exercised. if got := dirVersion(t, f.db); got != v0+int64(len(cases)) { t.Errorf("total version = %d, want %d (seed=%d + %d transitions)", got, v0+int64(len(cases)), v0, len(cases)) } } // ─── Category 2: invalid transitions return ErrInvalidTransition ───────────── // TestLifecycle_InvalidTransitions verifies that illegal moves return // ErrInvalidTransition and leave the node entirely unchanged (no event, no bump). func TestLifecycle_InvalidTransitions(t *testing.T) { f := setupLifecycle(t) ctx := context.Background() detail := map[string]any{} cases := []struct { name string uuid string start string fn func(string) error }{ { name: "down→up", uuid: "lc-inv-down-up", start: "down", fn: func(u string) error { return f.lc.MarkUp(ctx, u, detail) }, }, { name: "destroyed→probing", uuid: "lc-inv-dest-probing", start: "destroyed", fn: func(u string) error { return f.lc.MarkProbing(ctx, u, detail) }, }, { name: "destroyed→up", uuid: "lc-inv-dest-up", start: "destroyed", fn: func(u string) error { return f.lc.MarkUp(ctx, u, detail) }, }, { name: "destroyed→draining", uuid: "lc-inv-dest-draining", start: "destroyed", fn: func(u string) error { return f.lc.MarkDraining(ctx, u, detail) }, }, { name: "destroyed→down", uuid: "lc-inv-dest-down", start: "destroyed", fn: func(u string) error { return f.lc.MarkDown(ctx, u, detail) }, }, { name: "provisioning→up", uuid: "lc-inv-prov-up", start: "provisioning", fn: func(u string) error { return f.lc.MarkUp(ctx, u, detail) }, }, { name: "up→probing (backward)", uuid: "lc-inv-up-probing", start: "up", fn: func(u string) error { return f.lc.MarkProbing(ctx, u, detail) }, }, { name: "down→draining (backward)", uuid: "lc-inv-down-draining", start: "down", fn: func(u string) error { return f.lc.MarkDraining(ctx, u, detail) }, }, } for _, tc := range cases { tc := tc t.Run(tc.name, func(t *testing.T) { insertNode(t, f.db, tc.uuid, tc.start) vBefore := dirVersion(t, f.db) err := tc.fn(tc.uuid) if !errors.Is(err, nodes.ErrInvalidTransition) { t.Errorf("got %v, want ErrInvalidTransition", err) } // Status must be unchanged. if got := nodeStatus(t, f.db, tc.uuid); got != tc.start { t.Errorf("status changed to %q after invalid transition; want %q", got, tc.start) } // No event must have been written. if n := eventCount(t, f.db, tc.uuid); n != 0 { t.Errorf("event_count = %d after invalid transition; want 0", n) } // Directory version must be unchanged. if vAfter := dirVersion(t, f.db); vAfter != vBefore { t.Errorf("directory_version changed from %d to %d after invalid transition", vBefore, vAfter) } }) } } // ─── Category 3: concurrent write — exactly one succeeds ────────────────────── // TestLifecycle_ConcurrentWrite launches two goroutines that both attempt the // same transition simultaneously and verifies that exactly one succeeds // (optimistic-lock guarantee via UPDATE … WHERE status=). func TestLifecycle_ConcurrentWrite(t *testing.T) { f := setupLifecycle(t) ctx := context.Background() detail := map[string]any{"concurrent": true} const uuid = "lc-concurrent-destroy" insertNode(t, f.db, uuid, "down") var wg sync.WaitGroup errs := make([]error, 2) for i := range errs { wg.Add(1) i := i go func() { defer wg.Done() errs[i] = f.lc.MarkDestroyed(ctx, uuid, detail) }() } wg.Wait() successes := 0 for _, err := range errs { if err == nil { successes++ } } if successes != 1 { t.Errorf("concurrent MarkDestroyed: %d successes, want exactly 1 (errs: %v | %v)", successes, errs[0], errs[1]) } // Final state: destroyed, one event. if got := nodeStatus(t, f.db, uuid); got != "destroyed" { t.Errorf("final status = %q, want destroyed", got) } if n := eventCount(t, f.db, uuid); n != 1 { t.Errorf("event_count = %d after concurrent destroy; want 1", n) } } // ─── Category 4: MarkDestroyed post-commit hooks ────────────────────────────── // TestLifecycle_MarkDestroyed_PostHooks verifies that after a successful // MarkDestroyed call: // - mtls.CRL.IsRevoked(nodeUUID) returns true. // - Redis keys node:load:{uuid} and node:cmdq:{uuid} are deleted. func TestLifecycle_MarkDestroyed_PostHooks(t *testing.T) { f := setupLifecycle(t) ctx := context.Background() const uuid = "lc-destroyed-hooks" insertNode(t, f.db, uuid, "down") // Pre-populate the Redis keys that MarkDestroyed should clean up. loadKey := "node:load:" + uuid cmdqKey := "node:cmdq:" + uuid if err := f.rdb.Set(ctx, loadKey, "dummy-load", 0).Err(); err != nil { t.Fatalf("pre-seed load key: %v", err) } if err := f.rdb.ZAdd(ctx, cmdqKey, redis.Z{Score: 1, Member: "dummy-cmd"}).Err(); err != nil { t.Fatalf("pre-seed cmdq key: %v", err) } // Sanity-check: both keys exist before the call. if n, err := f.rdb.Exists(ctx, loadKey, cmdqKey).Result(); err != nil || n != 2 { t.Fatalf("pre-seed check: exists=%d err=%v (want 2 keys)", n, err) } if err := f.lc.MarkDestroyed(ctx, uuid, nil); err != nil { t.Fatalf("MarkDestroyed: %v", err) } // Certificate must be revoked: verify via an independent CRL view backed // by the same miniredis instance. crl2 := mtls.NewCRL(f.rdb, nil) if !crl2.IsRevoked(uuid) { t.Error("IsRevoked = false after MarkDestroyed; want true") } // Both Redis keys must be gone. if n, err := f.rdb.Exists(ctx, loadKey, cmdqKey).Result(); err != nil { t.Fatalf("redis Exists after destroy: %v", err) } else if n != 0 { t.Errorf("redis keys still exist after MarkDestroyed: %d key(s) remain", n) } } // ─── Category 5: MarkBlockedSuspect — weight=10, status unchanged ───────────── // TestLifecycle_MarkBlockedSuspect verifies that MarkBlockedSuspect: // - Does NOT change the node's status. // - Sets the node's weight to 10. // - Writes exactly one node_events row with event=blocked_suspect. // - Bumps the directory version by 1. func TestLifecycle_MarkBlockedSuspect(t *testing.T) { f := setupLifecycle(t) ctx := context.Background() const uuid = "lc-suspect" insertNode(t, f.db, uuid, "up") vBefore := dirVersion(t, f.db) if err := f.lc.MarkBlockedSuspect(ctx, uuid, map[string]any{ "reason": "domestic probes failing", "failure_streak": 3, }); err != nil { t.Fatalf("MarkBlockedSuspect: %v", err) } // Status must remain "up". if got := nodeStatus(t, f.db, uuid); got != "up" { t.Errorf("status = %q after MarkBlockedSuspect; want up", got) } // Weight must be 10. if w := nodeWeight(t, f.db, uuid); w != 10 { t.Errorf("weight = %d after MarkBlockedSuspect; want 10", w) } // Exactly one event row. if n := eventCount(t, f.db, uuid); n != 1 { t.Errorf("event_count = %d after MarkBlockedSuspect; want 1", n) } // Directory version must have been bumped by 1. if vAfter := dirVersion(t, f.db); vAfter != vBefore+1 { t.Errorf("directory_version: want %d, got %d", vBefore+1, vAfter) } } // TestLifecycle_MarkBlockedSuspect_Idempotent verifies that calling // MarkBlockedSuspect twice keeps weight at 10 and writes a second event. func TestLifecycle_MarkBlockedSuspect_Idempotent(t *testing.T) { f := setupLifecycle(t) ctx := context.Background() const uuid = "lc-suspect-idem" insertNode(t, f.db, uuid, "up") for i := 1; i <= 2; i++ { if err := f.lc.MarkBlockedSuspect(ctx, uuid, map[string]any{"streak": i}); err != nil { t.Fatalf("call %d: MarkBlockedSuspect: %v", i, err) } } if w := nodeWeight(t, f.db, uuid); w != 10 { t.Errorf("weight = %d after two calls; want 10", w) } if n := eventCount(t, f.db, uuid); n != 2 { t.Errorf("event_count = %d; want 2 (one per call)", n) } }