Files
pangolin/server/internal/provision/fakes_test.go
wangjia 3d5bac66b4 feat(provision): 弹性节点基建 Terraform + 一键更换 (tsk_6u0FxmbC7Yeq)
IaC 面 (infra/) 与控制面 (server/internal/provision) 双产出,落地 doc/04 §4
「节点是牲口」弹性拓扑与 make-before-break 一键更换。

server/internal/provision:
- CloudAdapter 适配层 + Registry(首发 vultr 消耗品池 / hetzner 精品池各一);
  厂商凭证仅从 PROVISION_<VENDOR>_* env 注入,不入库不入 git。
- ProvisionService:CreateNode(幂等键重放不重复开机)、DestroyNode(幂等)、
  RotateIP(换 IP 不换机 + version bump)、ListProviders。
- Replace 一键更换:先建后拆,新机 up 先于旧机 draining(容量不下降),
  replacement_uuid 幂等键 + replacements 表分步记录,崩溃可续跑不重复。
- RotatePool:池内滚动轮换,并发度 1–2。
- cmd/nodectl CLI:create/destroy/rotate-ip/replace/rotate-pool/providers。
- 单测(mock 厂商 API + 内存 Store):幂等重放、make-before-break 时序断言、
  开机失败/探活超时→destroyed+失败计数+告警钩子、崩溃续跑、RotatePool。

infra/:
- terraform/:探针机 + 控制面基线模块化(probe / control-plane)+ README,
  低频基线进 state,节点不进 Terraform。
- cloud-init/node.yaml.tmpl:节点引导模板(注入一次性 bootstrap token,task #5)。
- identity-isolation.md:身份隔离登记表(doc/06 §2 红线,无任何凭证)。

migrations/000008:nodes 增 provider_instance_id/elastic_ip_id、node_events
增 ip_rotated、provision_idempotency / replacements 表(附加式,不动现网)。

红线:仅面向新厂商池,绝不纳管现网生产 EC2(deploy/ marzban)。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-13 14:23:39 +08:00

463 lines
11 KiB
Go

package provision
import (
"context"
"fmt"
"sync"
"time"
)
// ---------------------------------------------------------------------------
// fakeStore — an in-memory Store that records the ordered sequence of status
// transitions so tests can assert make-before-break timing.
// ---------------------------------------------------------------------------
type statusTransition struct {
Seq int
NodeID int64
Status Status
}
type fakeStore struct {
mu sync.Mutex
nodes map[int64]*Node
byUUID map[string]int64
providers map[int64]*Provider
idem map[string]string
replaces map[string]*Replacement
events []struct {
NodeID int64
Event Event
Detail string
}
audits []string
transitions []statusTransition
nextNodeID int64
version int64
seq int
}
func newFakeStore() *fakeStore {
return &fakeStore{
nodes: map[int64]*Node{},
byUUID: map[string]int64{},
providers: map[int64]*Provider{},
idem: map[string]string{},
replaces: map[string]*Replacement{},
nextNodeID: 0,
version: 1,
}
}
func (f *fakeStore) addProvider(p *Provider) {
f.mu.Lock()
defer f.mu.Unlock()
f.providers[p.ID] = p
}
func (f *fakeStore) InsertNode(_ context.Context, n *Node) (int64, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.nextNodeID++
id := f.nextNodeID
cp := *n
cp.ID = id
if cp.Status == "" {
cp.Status = StatusProvisioning
}
cp.CreatedAt = time.Now()
f.nodes[id] = &cp
f.byUUID[cp.UUID] = id
f.seq++
f.transitions = append(f.transitions, statusTransition{f.seq, id, cp.Status})
return id, nil
}
func (f *fakeStore) GetNode(_ context.Context, id int64) (*Node, error) {
f.mu.Lock()
defer f.mu.Unlock()
n, ok := f.nodes[id]
if !ok {
return nil, nil
}
cp := *n
return &cp, nil
}
func (f *fakeStore) GetNodeByUUID(_ context.Context, uuid string) (*Node, error) {
f.mu.Lock()
id, ok := f.byUUID[uuid]
f.mu.Unlock()
if !ok {
return nil, nil
}
return f.GetNode(context.Background(), id)
}
func (f *fakeStore) UpdateNodeStatus(_ context.Context, id int64, status Status) error {
f.mu.Lock()
defer f.mu.Unlock()
n, ok := f.nodes[id]
if !ok {
return fmt.Errorf("fakeStore: node %d not found", id)
}
n.Status = status
f.seq++
f.transitions = append(f.transitions, statusTransition{f.seq, id, status})
return nil
}
func (f *fakeStore) UpdateNodeEndpoint(_ context.Context, id int64, endpoint string) error {
f.mu.Lock()
defer f.mu.Unlock()
if n, ok := f.nodes[id]; ok {
n.Endpoint = endpoint
}
return nil
}
func (f *fakeStore) SetNodeInstance(_ context.Context, id int64, instanceID, endpoint string) error {
f.mu.Lock()
defer f.mu.Unlock()
if n, ok := f.nodes[id]; ok {
n.ProviderInstanceID = instanceID
n.Endpoint = endpoint
}
return nil
}
func (f *fakeStore) SetNodeWeight(_ context.Context, id int64, weight int) error {
f.mu.Lock()
defer f.mu.Unlock()
if n, ok := f.nodes[id]; ok {
n.Weight = weight
}
return nil
}
func (f *fakeStore) ListNodesByPool(_ context.Context, pool Pool, status Status) ([]*Node, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []*Node
for _, n := range f.nodes {
p := f.providers[n.ProviderID]
if p == nil || p.Pool != pool {
continue
}
if status != "" && n.Status != status {
continue
}
cp := *n
out = append(out, &cp)
}
return out, nil
}
func (f *fakeStore) ListProviders(_ context.Context, pool Pool) ([]*Provider, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []*Provider
for _, p := range f.providers {
if !p.Enabled {
continue
}
if pool != "" && p.Pool != pool {
continue
}
cp := *p
out = append(out, &cp)
}
return out, nil
}
func (f *fakeStore) GetProvider(_ context.Context, id int64) (*Provider, error) {
f.mu.Lock()
defer f.mu.Unlock()
p, ok := f.providers[id]
if !ok {
return nil, nil
}
cp := *p
return &cp, nil
}
func (f *fakeStore) WriteNodeEvent(_ context.Context, nodeID int64, event Event, detail string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.events = append(f.events, struct {
NodeID int64
Event Event
Detail string
}{nodeID, event, detail})
return nil
}
func (f *fakeStore) WriteAuditLog(_ context.Context, actor, action, target, meta string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.audits = append(f.audits, fmt.Sprintf("%s|%s|%s", actor, action, target))
return nil
}
func (f *fakeStore) BumpDirectoryVersion(_ context.Context) (int64, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.version++
return f.version, nil
}
func (f *fakeStore) LookupIdempotency(_ context.Context, key string) (string, bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.idem[key]
return v, ok, nil
}
func (f *fakeStore) SaveIdempotency(_ context.Context, key, nodeUUID string) error {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.idem[key]; !ok {
f.idem[key] = nodeUUID
}
return nil
}
func (f *fakeStore) CreateReplacement(_ context.Context, r *Replacement) error {
f.mu.Lock()
defer f.mu.Unlock()
cp := *r
f.replaces[r.UUID] = &cp
return nil
}
func (f *fakeStore) GetReplacement(_ context.Context, uuid string) (*Replacement, error) {
f.mu.Lock()
defer f.mu.Unlock()
r, ok := f.replaces[uuid]
if !ok {
return nil, nil
}
cp := *r
return &cp, nil
}
func (f *fakeStore) UpdateReplacement(_ context.Context, r *Replacement) error {
f.mu.Lock()
defer f.mu.Unlock()
cp := *r
f.replaces[r.UUID] = &cp
return nil
}
// helpers for assertions
func (f *fakeStore) countEvents(e Event) int {
f.mu.Lock()
defer f.mu.Unlock()
n := 0
for _, ev := range f.events {
if ev.Event == e {
n++
}
}
return n
}
// firstSeqForStatus returns the seq of the first transition of nodeID into status,
// or -1 if it never happened.
func (f *fakeStore) firstSeqForStatus(nodeID int64, status Status) int {
f.mu.Lock()
defer f.mu.Unlock()
for _, t := range f.transitions {
if t.NodeID == nodeID && t.Status == status {
return t.Seq
}
}
return -1
}
// ---------------------------------------------------------------------------
// fakeAdapter / fakeFactory
// ---------------------------------------------------------------------------
type fakeAdapter struct {
mu sync.Mutex
createCalls int
destroyCalls int
createErr error
elastic bool
nextIP int
created []string // instance IDs created
destroyed []string
}
func newFakeAdapter() *fakeAdapter { return &fakeAdapter{elastic: true} }
func (a *fakeAdapter) Kind() string { return "fake" }
func (a *fakeAdapter) SupportsElasticIP() bool { return a.elastic }
func (a *fakeAdapter) CreateInstance(_ context.Context, in CreateInput) (*Instance, error) {
a.mu.Lock()
defer a.mu.Unlock()
a.createCalls++
if a.createErr != nil {
return nil, a.createErr
}
a.nextIP++
id := fmt.Sprintf("inst-%d", a.createCalls)
a.created = append(a.created, id)
return &Instance{ID: id, IP: fmt.Sprintf("203.0.113.%d", a.nextIP), Region: in.Region}, nil
}
func (a *fakeAdapter) DestroyInstance(_ context.Context, instanceID string) error {
a.mu.Lock()
defer a.mu.Unlock()
a.destroyCalls++
a.destroyed = append(a.destroyed, instanceID)
return nil
}
func (a *fakeAdapter) AttachIP(_ context.Context, _ string) (string, error) {
a.mu.Lock()
defer a.mu.Unlock()
a.nextIP++
return fmt.Sprintf("198.51.100.%d", a.nextIP), nil
}
func (a *fakeAdapter) ListRegions(context.Context) ([]Region, error) {
return []Region{{ID: "hkg", Country: "HK"}}, nil
}
func (a *fakeAdapter) createCount() int {
a.mu.Lock()
defer a.mu.Unlock()
return a.createCalls
}
func (a *fakeAdapter) destroyCount() int {
a.mu.Lock()
defer a.mu.Unlock()
return a.destroyCalls
}
type fakeFactory struct{ a CloudAdapter }
func (f fakeFactory) For(*Provider) (CloudAdapter, error) { return f.a, nil }
// ---------------------------------------------------------------------------
// fakeProber / fakeClock / fakeAlert / fakeTokens
// ---------------------------------------------------------------------------
type fakeProber struct {
err error
calls int
}
func (p *fakeProber) WaitReady(ctx context.Context, _ *Node) error {
p.calls++
if p.err != nil {
return p.err
}
return nil
}
// fakeClock records sleeps without actually blocking.
type fakeClock struct {
now time.Time
slept []time.Duration
}
func (c *fakeClock) Now() time.Time { return c.now }
func (c *fakeClock) Sleep(_ context.Context, d time.Duration) error {
c.slept = append(c.slept, d)
c.now = c.now.Add(d)
return nil
}
type fakeAlert struct {
mu sync.Mutex
alerts []Alert
}
func (a *fakeAlert) Fire(_ context.Context, al Alert) {
a.mu.Lock()
defer a.mu.Unlock()
a.alerts = append(a.alerts, al)
}
func (a *fakeAlert) count(kind AlertKind) int {
a.mu.Lock()
defer a.mu.Unlock()
n := 0
for _, al := range a.alerts {
if al.Kind == kind {
n++
}
}
return n
}
type fakeTokens struct{ calls int }
func (t *fakeTokens) IssueToken(context.Context, string) (string, error) {
t.calls++
return fmt.Sprintf("token-%d", t.calls), nil
}
// ---------------------------------------------------------------------------
// test harness builder
// ---------------------------------------------------------------------------
type harness struct {
svc *Service
store *fakeStore
adapter *fakeAdapter
prober *fakeProber
clock *fakeClock
alert *fakeAlert
tokens *fakeTokens
}
func newHarness(t interface{ Fatalf(string, ...any) }) *harness {
store := newFakeStore()
// One consumable + one premium provider.
store.addProvider(&Provider{ID: 1, Name: "v", APIKind: "fake", Pool: PoolConsumable, Enabled: true})
store.addProvider(&Provider{ID: 2, Name: "h", APIKind: "fake", Pool: PoolPremium, Enabled: true})
adapter := newFakeAdapter()
prober := &fakeProber{}
clock := &fakeClock{now: time.Unix(1700000000, 0).UTC()}
alert := &fakeAlert{}
tokens := &fakeTokens{}
renderer, err := NewTemplateRendererFromString("uuid={{.NodeUUID}} token={{.BootstrapToken}}")
if err != nil {
t.Fatalf("renderer: %v", err)
}
svc, err := NewService(Config{
Store: store,
Adapters: fakeFactory{adapter},
Tokens: tokens,
Renderer: renderer,
Prober: prober,
Alert: alert,
Clock: clock,
ControlPlaneURL: "https://cp.example",
DrainTimeout: 30 * time.Minute,
ProbeTimeout: time.Minute,
})
if err != nil {
t.Fatalf("NewService: %v", err)
}
return &harness{svc, store, adapter, prober, clock, alert, tokens}
}
func proSpec() NodeSpec {
return NodeSpec{Region: "hkg", Tier: TierPro, ProviderID: 2, Plan: "cpx11", RealityPBK: "pbk", RealitySNI: "www.example.com", HY2Port: 443, NameZH: "香港", NameEn: "HK"}
}
func freeSpec() NodeSpec {
return NodeSpec{Region: "hkg", Tier: TierFree, ProviderID: 1, Plan: "vc2-1c-1gb", RealityPBK: "pbk", RealitySNI: "www.example.com", HY2Port: 443, NameZH: "香港", NameEn: "HK"}
}