feat(provision): 弹性节点基建 Terraform + 一键更换 (tsk_6u0FxmbC7Yeq)

IaC 面 (infra/) 与控制面 (server/internal/provision) 双产出，落地 doc/04 §4
「节点是牲口」弹性拓扑与 make-before-break 一键更换。

server/internal/provision:
- CloudAdapter 适配层 + Registry（首发 vultr 消耗品池 / hetzner 精品池各一）；
  厂商凭证仅从 PROVISION_<VENDOR>_* env 注入，不入库不入 git。
- ProvisionService：CreateNode(幂等键重放不重复开机)、DestroyNode(幂等)、
  RotateIP(换 IP 不换机 + version bump)、ListProviders。
- Replace 一键更换：先建后拆，新机 up 先于旧机 draining（容量不下降），
  replacement_uuid 幂等键 + replacements 表分步记录，崩溃可续跑不重复。
- RotatePool：池内滚动轮换，并发度 1–2。
- cmd/nodectl CLI：create/destroy/rotate-ip/replace/rotate-pool/providers。
- 单测（mock 厂商 API + 内存 Store）：幂等重放、make-before-break 时序断言、
  开机失败/探活超时→destroyed+失败计数+告警钩子、崩溃续跑、RotatePool。

infra/:
- terraform/：探针机 + 控制面基线模块化（probe / control-plane）+ README，
  低频基线进 state，节点不进 Terraform。
- cloud-init/node.yaml.tmpl：节点引导模板（注入一次性 bootstrap token，task #5）。
- identity-isolation.md：身份隔离登记表（doc/06 §2 红线，无任何凭证）。

migrations/000008：nodes 增 provider_instance_id/elastic_ip_id、node_events
增 ip_rotated、provision_idempotency / replacements 表（附加式，不动现网）。

红线：仅面向新厂商池，绝不纳管现网生产 EC2(deploy/ marzban)。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

server/internal/provision/providers/doc.go

@@ -0,0 +1,16 @@
+// Package providers holds the concrete CloudAdapter implementations behind the
+// provision.AdapterFactory boundary (doc/04 §4: "厂商 API 适配层").
+//
+// First launch ships two vendors — one per pool (doc/04 §5.2):
+//
+//   - vultr   — consumable pool (entry/free): cheap, hourly-billed small vendor.
+//   - hetzner — premium pool (exit / pro entry): stable, good native IPs.
+//
+// CREDENTIAL RED LINE (doc/06 §2): adapters read their API credentials ONLY
+// from independent secrets (environment variables here). Credentials are never
+// stored in the providers table, never logged, and never committed to git. The
+// Registry binds providers.api_kind → adapter at wiring time.
+//
+// Identity isolation: each vendor MUST use a fully independent account / email /
+// crypto payment, registered in infra/identity-isolation.md.
+package providers

server/internal/provision/providers/hetzner.go

@@ -0,0 +1,154 @@
+package providers
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/wangjia/pangolin/server/internal/provision"
+)
+
+// hetznerAdapter implements provision.CloudAdapter against the Hetzner Cloud
+// API v1. Premium pool: stable vendor, good native IPs, floating-IP capable.
+//
+// Credentials: PROVISION_HETZNER_API_TOKEN (env, independent secret).
+// Default image: PROVISION_HETZNER_IMAGE (defaults to "debian-12").
+type hetznerAdapter struct {
+	c     *httpClient
+	image string
+}
+
+func newHetznerFromEnv() (provision.CloudAdapter, error) {
+	token, err := secret("PROVISION_HETZNER_API_TOKEN")
+	if err != nil {
+		return nil, err
+	}
+	image := os.Getenv("PROVISION_HETZNER_IMAGE")
+	if image == "" {
+		image = "debian-12"
+	}
+	return &hetznerAdapter{
+		c:     newHTTPClient("https://api.hetzner.cloud/v1", token),
+		image: image,
+	}, nil
+}
+
+func (a *hetznerAdapter) Kind() string            { return "hetzner" }
+func (a *hetznerAdapter) SupportsElasticIP() bool { return true } // floating IPs
+
+func (a *hetznerAdapter) CreateInstance(ctx context.Context, in provision.CreateInput) (*provision.Instance, error) {
+	body := map[string]any{
+		"name":               sanitizeName(in.Label),
+		"server_type":        in.Plan,
+		"image":              a.image,
+		"location":           in.Region,
+		"user_data":          in.UserData, // Hetzner accepts raw cloud-init
+		"start_after_create": true,
+		"labels":             labelsFromTags(in.Tags),
+	}
+	if len(in.SSHKeyIDs) > 0 {
+		body["ssh_keys"] = in.SSHKeyIDs
+	}
+	var resp struct {
+		Server struct {
+			ID        int64 `json:"id"`
+			PublicNet struct {
+				IPv4 struct {
+					IP string `json:"ip"`
+				} `json:"ipv4"`
+			} `json:"public_net"`
+			Datacenter struct {
+				Location struct {
+					Name string `json:"name"`
+				} `json:"location"`
+			} `json:"datacenter"`
+		} `json:"server"`
+	}
+	if err := a.c.do(ctx, "POST", "/servers", body, &resp); err != nil {
+		return nil, err
+	}
+	return &provision.Instance{
+		ID:     strconv.FormatInt(resp.Server.ID, 10),
+		IP:     resp.Server.PublicNet.IPv4.IP,
+		Region: resp.Server.Datacenter.Location.Name,
+	}, nil
+}
+
+func (a *hetznerAdapter) DestroyInstance(ctx context.Context, instanceID string) error {
+	err := a.c.do(ctx, "DELETE", "/servers/"+instanceID, nil, nil)
+	if err != nil && isNotFound(err) {
+		return nil // idempotent
+	}
+	return err
+}
+
+func (a *hetznerAdapter) AttachIP(ctx context.Context, instanceID string) (string, error) {
+	id, err := strconv.ParseInt(instanceID, 10, 64)
+	if err != nil {
+		return "", err
+	}
+	var resp struct {
+		FloatingIP struct {
+			ID int64  `json:"id"`
+			IP string `json:"ip"`
+		} `json:"floating_ip"`
+	}
+	if err := a.c.do(ctx, "POST", "/floating_ips", map[string]any{
+		"type":        "ipv4",
+		"server":      id,
+		"description": "rotate-" + instanceID,
+	}, &resp); err != nil {
+		return "", err
+	}
+	return resp.FloatingIP.IP, nil
+}
+
+func (a *hetznerAdapter) ListRegions(ctx context.Context) ([]provision.Region, error) {
+	var resp struct {
+		Locations []struct {
+			Name    string `json:"name"`
+			Country string `json:"country"`
+			City    string `json:"city"`
+		} `json:"locations"`
+	}
+	if err := a.c.do(ctx, "GET", "/locations", nil, &resp); err != nil {
+		return nil, err
+	}
+	out := make([]provision.Region, 0, len(resp.Locations))
+	for _, l := range resp.Locations {
+		out = append(out, provision.Region{ID: l.Name, Country: l.Country, City: l.City})
+	}
+	return out, nil
+}
+
+// --- helpers shared with vultr ---
+
+func atoiDefault(s string, def int) int {
+	if s == "" {
+		return def
+	}
+	n, err := strconv.Atoi(s)
+	if err != nil {
+		return def
+	}
+	return n
+}
+
+func isNotFound(err error) bool {
+	return err != nil && (strings.Contains(err.Error(), "status 404") || strings.Contains(err.Error(), "not_found"))
+}
+
+// sanitizeName makes a UUID acceptable as a Hetzner server name (RFC1123-ish).
+func sanitizeName(s string) string {
+	s = strings.ToLower(s)
+	return "node-" + s
+}
+
+func labelsFromTags(tags []string) map[string]string {
+	m := map[string]string{}
+	for i, t := range tags {
+		m["tag"+strconv.Itoa(i)] = t
+	}
+	return m
+}

server/internal/provision/providers/registry.go

@@ -0,0 +1,121 @@
+package providers
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/wangjia/pangolin/server/internal/provision"
+)
+
+// Factory builds a CloudAdapter, reading credentials from independent secrets
+// (env). It returns provision.ErrNoCredentials when the required secret is unset.
+type Factory func() (provision.CloudAdapter, error)
+
+// builtins maps api_kind → Factory. Extend here when onboarding a vendor.
+var builtins = map[string]Factory{
+	"vultr":   newVultrFromEnv,
+	"hetzner": newHetznerFromEnv,
+}
+
+// Registry resolves provision.Provider rows to live adapters. It satisfies
+// provision.AdapterFactory and caches one adapter per api_kind.
+type Registry struct {
+	factories map[string]Factory
+	cache     map[string]provision.CloudAdapter
+}
+
+// NewRegistry returns a Registry backed by the built-in vendor factories.
+func NewRegistry() *Registry {
+	fs := make(map[string]Factory, len(builtins))
+	for k, v := range builtins {
+		fs[k] = v
+	}
+	return &Registry{factories: fs, cache: map[string]provision.CloudAdapter{}}
+}
+
+// Register adds or overrides a factory for api_kind (used in tests / extension).
+func (r *Registry) Register(apiKind string, f Factory) { r.factories[apiKind] = f }
+
+// For resolves the adapter for a provider row.
+func (r *Registry) For(p *provision.Provider) (provision.CloudAdapter, error) {
+	if a, ok := r.cache[p.APIKind]; ok {
+		return a, nil
+	}
+	f, ok := r.factories[p.APIKind]
+	if !ok {
+		return nil, fmt.Errorf("providers: no adapter registered for api_kind %q", p.APIKind)
+	}
+	a, err := f()
+	if err != nil {
+		return nil, err
+	}
+	r.cache[p.APIKind] = a
+	return a, nil
+}
+
+var _ provision.AdapterFactory = (*Registry)(nil)
+
+// --- shared HTTP helper ---
+
+// httpClient is a small JSON REST helper shared by the adapters.
+type httpClient struct {
+	base   string
+	bearer string
+	hc     *http.Client
+}
+
+func newHTTPClient(base, bearer string) *httpClient {
+	return &httpClient{base: base, bearer: bearer, hc: &http.Client{Timeout: 30 * time.Second}}
+}
+
+// do issues an authenticated JSON request and decodes the response into out
+// (out may be nil). It returns an error on any non-2xx status.
+func (c *httpClient) do(ctx context.Context, method, path string, body, out any) error {
+	var rdr io.Reader
+	if body != nil {
+		buf, err := json.Marshal(body)
+		if err != nil {
+			return fmt.Errorf("providers: marshal body: %w", err)
+		}
+		rdr = bytes.NewReader(buf)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, c.base+path, rdr)
+	if err != nil {
+		return fmt.Errorf("providers: build request: %w", err)
+	}
+	req.Header.Set("Authorization", "Bearer "+c.bearer)
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	resp, err := c.hc.Do(req)
+	if err != nil {
+		return fmt.Errorf("providers: %s %s: %w", method, path, err)
+	}
+	defer resp.Body.Close()
+	data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		// Never echo credentials; only status + vendor body (no auth header).
+		return fmt.Errorf("providers: %s %s: status %d: %s", method, path, resp.StatusCode, string(data))
+	}
+	if out != nil && len(data) > 0 {
+		if err := json.Unmarshal(data, out); err != nil {
+			return fmt.Errorf("providers: decode response: %w", err)
+		}
+	}
+	return nil
+}
+
+// secret reads an env-injected credential, returning ErrNoCredentials if unset.
+func secret(env string) (string, error) {
+	v := os.Getenv(env)
+	if v == "" {
+		return "", fmt.Errorf("%w (env %s)", provision.ErrNoCredentials, env)
+	}
+	return v, nil
+}

server/internal/provision/providers/registry_test.go

@@ -0,0 +1,76 @@
+package providers
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/wangjia/pangolin/server/internal/provision"
+)
+
+func TestRegistry_UnknownKind(t *testing.T) {
+	r := NewRegistry()
+	_, err := r.For(&provision.Provider{APIKind: "does-not-exist"})
+	if err == nil {
+		t.Fatal("expected error for unknown api_kind")
+	}
+}
+
+func TestRegistry_NoCredentials(t *testing.T) {
+	// Ensure the env vars are unset for a clean assertion.
+	t.Setenv("PROVISION_VULTR_API_KEY", "")
+	t.Setenv("PROVISION_HETZNER_API_TOKEN", "")
+
+	r := NewRegistry()
+	if _, err := r.For(&provision.Provider{APIKind: "vultr"}); !errors.Is(err, provision.ErrNoCredentials) {
+		t.Errorf("vultr without creds: err = %v, want ErrNoCredentials", err)
+	}
+	if _, err := r.For(&provision.Provider{APIKind: "hetzner"}); !errors.Is(err, provision.ErrNoCredentials) {
+		t.Errorf("hetzner without creds: err = %v, want ErrNoCredentials", err)
+	}
+}
+
+func TestRegistry_RegisterAndCache(t *testing.T) {
+	r := NewRegistry()
+	built := 0
+	r.Register("fake", func() (provision.CloudAdapter, error) {
+		built++
+		return stubAdapter{}, nil
+	})
+	p := &provision.Provider{APIKind: "fake"}
+	if _, err := r.For(p); err != nil {
+		t.Fatalf("For: %v", err)
+	}
+	if _, err := r.For(p); err != nil {
+		t.Fatalf("For (cached): %v", err)
+	}
+	if built != 1 {
+		t.Errorf("factory built %d times, want 1 (cached)", built)
+	}
+}
+
+func TestRegistry_CredentialsFromEnv(t *testing.T) {
+	t.Setenv("PROVISION_VULTR_API_KEY", "secret-key")
+	r := NewRegistry()
+	a, err := r.For(&provision.Provider{APIKind: "vultr"})
+	if err != nil {
+		t.Fatalf("vultr with creds: %v", err)
+	}
+	if a.Kind() != "vultr" {
+		t.Errorf("kind = %s, want vultr", a.Kind())
+	}
+	if !a.SupportsElasticIP() {
+		t.Error("vultr should support elastic IP (reserved IPs)")
+	}
+}
+
+type stubAdapter struct{}
+
+func (stubAdapter) Kind() string            { return "fake" }
+func (stubAdapter) SupportsElasticIP() bool { return false }
+func (stubAdapter) CreateInstance(context.Context, provision.CreateInput) (*provision.Instance, error) {
+	return &provision.Instance{ID: "x"}, nil
+}
+func (stubAdapter) DestroyInstance(context.Context, string) error           { return nil }
+func (stubAdapter) AttachIP(context.Context, string) (string, error)        { return "", nil }
+func (stubAdapter) ListRegions(context.Context) ([]provision.Region, error) { return nil, nil }

server/internal/provision/providers/vultr.go

@@ -0,0 +1,110 @@
+package providers
+
+import (
+	"context"
+	"encoding/base64"
+	"os"
+
+	"github.com/wangjia/pangolin/server/internal/provision"
+)
+
+// vultrAdapter implements provision.CloudAdapter against the Vultr API v2.
+// Consumable pool: cheap, hourly-billed, reserved-IP capable.
+//
+// Credentials: PROVISION_VULTR_API_KEY (env, independent secret).
+// Default OS image: PROVISION_VULTR_OS_ID (defaults to a current Debian image).
+type vultrAdapter struct {
+	c    *httpClient
+	osID int
+}
+
+func newVultrFromEnv() (provision.CloudAdapter, error) {
+	key, err := secret("PROVISION_VULTR_API_KEY")
+	if err != nil {
+		return nil, err
+	}
+	osID := atoiDefault(os.Getenv("PROVISION_VULTR_OS_ID"), 2136) // 2136 = Debian 12 x64
+	return &vultrAdapter{
+		c:    newHTTPClient("https://api.vultr.com/v2", key),
+		osID: osID,
+	}, nil
+}
+
+func (a *vultrAdapter) Kind() string            { return "vultr" }
+func (a *vultrAdapter) SupportsElasticIP() bool { return true } // reserved IPs
+
+func (a *vultrAdapter) CreateInstance(ctx context.Context, in provision.CreateInput) (*provision.Instance, error) {
+	body := map[string]any{
+		"region":    in.Region,
+		"plan":      in.Plan,
+		"os_id":     a.osID,
+		"label":     in.Label,
+		"hostname":  in.Label,
+		"user_data": base64.StdEncoding.EncodeToString([]byte(in.UserData)),
+		"tags":      in.Tags,
+		"backups":   "disabled",
+	}
+	if len(in.SSHKeyIDs) > 0 {
+		body["sshkey_id"] = in.SSHKeyIDs
+	}
+	var resp struct {
+		Instance struct {
+			ID     string `json:"id"`
+			MainIP string `json:"main_ip"`
+			Region string `json:"region"`
+		} `json:"instance"`
+	}
+	if err := a.c.do(ctx, "POST", "/instances", body, &resp); err != nil {
+		return nil, err
+	}
+	return &provision.Instance{
+		ID:     resp.Instance.ID,
+		IP:     resp.Instance.MainIP,
+		Region: resp.Instance.Region,
+	}, nil
+}
+
+func (a *vultrAdapter) DestroyInstance(ctx context.Context, instanceID string) error {
+	err := a.c.do(ctx, "DELETE", "/instances/"+instanceID, nil, nil)
+	if err != nil && isNotFound(err) {
+		return nil // idempotent: already gone
+	}
+	return err
+}
+
+func (a *vultrAdapter) AttachIP(ctx context.Context, instanceID string) (string, error) {
+	// Allocate a reserved IPv4, then attach it to the instance.
+	var created struct {
+		ReservedIP struct {
+			ID     string `json:"id"`
+			Subnet string `json:"subnet"`
+		} `json:"reserved_ip"`
+	}
+	if err := a.c.do(ctx, "POST", "/reserved-ips", map[string]any{
+		"region":      "",
+		"ip_type":     "v4",
+		"label":       "rotate-" + instanceID,
+		"instance_id": instanceID,
+	}, &created); err != nil {
+		return "", err
+	}
+	return created.ReservedIP.Subnet, nil
+}
+
+func (a *vultrAdapter) ListRegions(ctx context.Context) ([]provision.Region, error) {
+	var resp struct {
+		Regions []struct {
+			ID      string `json:"id"`
+			Country string `json:"country"`
+			City    string `json:"city"`
+		} `json:"regions"`
+	}
+	if err := a.c.do(ctx, "GET", "/regions", nil, &resp); err != nil {
+		return nil, err
+	}
+	out := make([]provision.Region, 0, len(resp.Regions))
+	for _, r := range resp.Regions {
+		out = append(out, provision.Region{ID: r.ID, Country: r.Country, City: r.City})
+	}
+	return out, nil
+}