feat(backend): admin 手动运维接真实 lifecycle/provision
管理后台节点页的「拉黑/恢复(draining/up)」与「一键换机(replace)」此前 注入的是 StubLifecycle/StubProvision(Ready()=false,UI 置灰、操作返回 服务不可用)。本次接真实实现: - RealLifecycle:复用 scheduler 的乐观锁状态流转(UPDATE nodes WHERE status=from + node_events + directory_version bump),手动运维与调度 自动循环共用同一条 canonical 路径;读当前状态作 from 守卫,幂等安全。 - RealProvision:委托 provision.Service.Replace 的 make-before-break 换机 (先拉起新节点再 drain 旧节点,容量不掉)。 - BuildServices:lifecycle 恒为真实;provision 在服务初始化成功时为真实, 否则回退 not-ready stub(无厂商凭证时点击换机给出明确错误)。 补 services_real_test.go:换机委托(空 UUID 起新编排)、错误透传、 非法目标状态拒绝、Ready 标志。server 全量 23 包测试通过。 e2e(真实换机)待机群 + 厂商凭证。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@ import (
|
|||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
"github.com/redis/go-redis/v9"
|
"github.com/redis/go-redis/v9"
|
||||||
"github.com/wangjia/pangolin/server/internal/codes"
|
"github.com/wangjia/pangolin/server/internal/codes"
|
||||||
|
"github.com/wangjia/pangolin/server/internal/provision"
|
||||||
|
"github.com/wangjia/pangolin/server/internal/provision/providers"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewRouter wires the full admin HTTP handler chain:
|
// NewRouter wires the full admin HTTP handler chain:
|
||||||
@@ -46,17 +48,28 @@ func NewRouter(h *Handlers, sessions *SessionStore, cfg *Config, sec *SecurityLo
|
|||||||
return r
|
return r
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildServices assembles the downstream services. The codes service (#3) is
|
// BuildServices assembles the downstream services. Codes (#3) and lifecycle (#5)
|
||||||
// real; lifecycle (#5) and provisioning (#14) are stubs until those modules
|
// are always real (DB-backed). Provisioning (#14) is real when the provision
|
||||||
// land — the UI greys out their controls (Ready()==false).
|
// service initializes; otherwise it falls back to the not-ready stub and the UI
|
||||||
|
// greys out the replace control (Ready()==false).
|
||||||
func BuildServices(db *sql.DB, rdb *redis.Client, failMax int, lockDur time.Duration) Services {
|
func BuildServices(db *sql.DB, rdb *redis.Client, failMax int, lockDur time.Duration) Services {
|
||||||
codeStore := codes.NewStore(db)
|
codeStore := codes.NewStore(db)
|
||||||
codeSvc := codes.NewService(codeStore, rdb, failMax, lockDur)
|
codeSvc := codes.NewService(codeStore, rdb, failMax, lockDur)
|
||||||
return Services{
|
svc := Services{
|
||||||
Codes: NewCodesAdapter(codeSvc, codeStore),
|
Codes: NewCodesAdapter(codeSvc, codeStore),
|
||||||
Lifecycle: NewStubLifecycle(),
|
Lifecycle: NewRealLifecycle(db),
|
||||||
Provision: NewStubProvision(),
|
Provision: NewStubProvision(),
|
||||||
}
|
}
|
||||||
|
provSvc, err := provision.NewService(provision.Config{
|
||||||
|
Store: provision.NewMySQLStore(db),
|
||||||
|
Adapters: providers.NewRegistry(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("admin: provision service init failed (%v) — replace control disabled", err)
|
||||||
|
} else {
|
||||||
|
svc.Provision = NewRealProvision(provSvc)
|
||||||
|
}
|
||||||
|
return svc
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewHandler builds the complete admin http.Handler from its dependencies.
|
// NewHandler builds the complete admin http.Handler from its dependencies.
|
||||||
|
|||||||
@@ -2,10 +2,16 @@ package admin
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/wangjia/pangolin/server/internal/codes"
|
"github.com/wangjia/pangolin/server/internal/codes"
|
||||||
|
"github.com/wangjia/pangolin/server/internal/provision"
|
||||||
|
"github.com/wangjia/pangolin/server/internal/scheduler"
|
||||||
|
"github.com/wangjia/pangolin/server/internal/scheduler/orchestrate"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrServiceUnavailable is returned by stub services whose real implementation
|
// ErrServiceUnavailable is returned by stub services whose real implementation
|
||||||
@@ -134,6 +140,55 @@ func (StubLifecycle) TransitionStatus(context.Context, int64, string, string) er
|
|||||||
// Ready reports false.
|
// Ready reports false.
|
||||||
func (StubLifecycle) Ready() bool { return false }
|
func (StubLifecycle) Ready() bool { return false }
|
||||||
|
|
||||||
|
// RealLifecycle is the SQL-backed lifecycle service. It reuses the scheduler's
|
||||||
|
// canonical optimistic-lock transition (UPDATE nodes ... WHERE status=from, plus
|
||||||
|
// a node_events row and a directory_version bump so clients re-fetch), so manual
|
||||||
|
// admin transitions and the autonomous scheduler loop share one code path.
|
||||||
|
type RealLifecycle struct {
|
||||||
|
db *sql.DB
|
||||||
|
lc orchestrate.LifecycleService
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRealLifecycle wires the lifecycle adapter over db.
|
||||||
|
func NewRealLifecycle(db *sql.DB) *RealLifecycle {
|
||||||
|
return &RealLifecycle{
|
||||||
|
db: db,
|
||||||
|
lc: scheduler.NewOrchestrateLifecycle(scheduler.NewSQLLifecycle(db, nil)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TransitionStatus moves nodeID to target ("draining" | "up"). It reads the
|
||||||
|
// current status as the optimistic-lock guard, so the transition is a safe
|
||||||
|
// no-op if the node already moved.
|
||||||
|
func (l *RealLifecycle) TransitionStatus(ctx context.Context, nodeID int64, target, actor string) error {
|
||||||
|
if target != "draining" && target != "up" {
|
||||||
|
return fmt.Errorf("admin: unsupported target status %q", target)
|
||||||
|
}
|
||||||
|
var from string
|
||||||
|
switch err := l.db.QueryRowContext(ctx, `SELECT status FROM nodes WHERE id = ?`, nodeID).Scan(&from); err {
|
||||||
|
case nil:
|
||||||
|
case sql.ErrNoRows:
|
||||||
|
return fmt.Errorf("admin: node %d not found", nodeID)
|
||||||
|
default:
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if from == target {
|
||||||
|
return nil // already in target state — idempotent no-op
|
||||||
|
}
|
||||||
|
n, err := l.lc.TransitionStatus(ctx, strconv.FormatInt(nodeID, 10), from, target,
|
||||||
|
map[string]any{"actor": actor, "source": "admin"})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if n == 0 {
|
||||||
|
return fmt.Errorf("admin: node %d status changed concurrently", nodeID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ready reports true — the lifecycle service only needs the database.
|
||||||
|
func (*RealLifecycle) Ready() bool { return true }
|
||||||
|
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
// Node provisioning service (#14) — not yet implemented
|
// Node provisioning service (#14) — not yet implemented
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
@@ -159,6 +214,34 @@ func (StubProvision) Replace(context.Context, int64, string) error { return ErrS
|
|||||||
// Ready reports false.
|
// Ready reports false.
|
||||||
func (StubProvision) Ready() bool { return false }
|
func (StubProvision) Ready() bool { return false }
|
||||||
|
|
||||||
|
// provReplacer is the slice of provision.Service the admin replace control needs
|
||||||
|
// (declared as an interface so tests can fake it without a database).
|
||||||
|
type provReplacer interface {
|
||||||
|
Replace(ctx context.Context, nodeID int64, replacementUUID string) (*provision.ReplaceResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RealProvision drives a one-click make-before-break node replacement via the
|
||||||
|
// real provision.Service (doc/04 §4.1): a fresh node is brought UP before the
|
||||||
|
// old one drains, so capacity never dips.
|
||||||
|
type RealProvision struct {
|
||||||
|
svc provReplacer
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRealProvision wires the provision adapter over the real service.
|
||||||
|
func NewRealProvision(svc provReplacer) *RealProvision { return &RealProvision{svc: svc} }
|
||||||
|
|
||||||
|
// Replace provisions a replacement for nodeID. An empty replacementUUID starts a
|
||||||
|
// fresh orchestration (the service mints its own idempotency key). The actor is
|
||||||
|
// recorded in the admin audit log by the caller.
|
||||||
|
func (p *RealProvision) Replace(ctx context.Context, nodeID int64, _ string) error {
|
||||||
|
_, err := p.svc.Replace(ctx, nodeID, "")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ready reports true — the provision service is wired. Without vendor
|
||||||
|
// credentials the Replace call fails with a clear error when invoked.
|
||||||
|
func (*RealProvision) Ready() bool { return true }
|
||||||
|
|
||||||
// Services bundles the three downstream services the admin UI depends on.
|
// Services bundles the three downstream services the admin UI depends on.
|
||||||
type Services struct {
|
type Services struct {
|
||||||
Codes CodesService
|
Codes CodesService
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package admin
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/wangjia/pangolin/server/internal/provision"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeReplacer records the arguments passed to Replace and returns canned values.
|
||||||
|
type fakeReplacer struct {
|
||||||
|
gotNodeID int64
|
||||||
|
gotUUID string
|
||||||
|
res *provision.ReplaceResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeReplacer) Replace(_ context.Context, nodeID int64, uuid string) (*provision.ReplaceResult, error) {
|
||||||
|
f.gotNodeID, f.gotUUID = nodeID, uuid
|
||||||
|
return f.res, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRealProvision_Replace_DelegatesWithFreshUUID(t *testing.T) {
|
||||||
|
f := &fakeReplacer{res: &provision.ReplaceResult{NewNodeID: 99}}
|
||||||
|
p := NewRealProvision(f)
|
||||||
|
|
||||||
|
if err := p.Replace(context.Background(), 42, "operator"); err != nil {
|
||||||
|
t.Fatalf("Replace: %v", err)
|
||||||
|
}
|
||||||
|
if f.gotNodeID != 42 {
|
||||||
|
t.Errorf("nodeID = %d, want 42", f.gotNodeID)
|
||||||
|
}
|
||||||
|
// Admin always starts a fresh orchestration: the service mints its own key.
|
||||||
|
if f.gotUUID != "" {
|
||||||
|
t.Errorf("replacementUUID = %q, want empty (fresh start)", f.gotUUID)
|
||||||
|
}
|
||||||
|
if !p.Ready() {
|
||||||
|
t.Error("RealProvision.Ready() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRealProvision_Replace_PropagatesError(t *testing.T) {
|
||||||
|
sentinel := errors.New("no vendor credentials")
|
||||||
|
p := NewRealProvision(&fakeReplacer{err: sentinel})
|
||||||
|
if err := p.Replace(context.Background(), 7, "operator"); !errors.Is(err, sentinel) {
|
||||||
|
t.Errorf("Replace error = %v, want %v", err, sentinel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRealLifecycle_RejectsUnsupportedTarget(t *testing.T) {
|
||||||
|
// Target validation happens before any DB access, so a nil db is safe here.
|
||||||
|
l := &RealLifecycle{}
|
||||||
|
if err := l.TransitionStatus(context.Background(), 1, "destroyed", "operator"); err == nil {
|
||||||
|
t.Error("TransitionStatus with target=destroyed should error, got nil")
|
||||||
|
}
|
||||||
|
if !l.Ready() {
|
||||||
|
t.Error("RealLifecycle.Ready() = false, want true")
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user