pangolin/server/internal/scheduler/orchestrate/deps.go

// Package orchestrate implements the replacement orchestration state machine
// (task 15E). It is driven every 30 s by the OrchestrateLoop (task 15H).
//
// Key components:
//   - Replacer.Tick   — drains the 15D replace queue and advances in-flight records.
//   - Grayscale.Advance — gradually ramps new/recovered node weight 10→25→50→75→100.
package orchestrate

import (
	"context"
	"log/slog"
	"time"

	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
)

// ─────────────────────────────────────────────────────────────────────────────
// Phase constants
// ─────────────────────────────────────────────────────────────────────────────

// Phase is the orchestration phase of a replacement record.
type Phase string

const (
	PhasePending     Phase = "pending"      // waiting for breaker / quota approval
	PhaseCreating    Phase = "creating"     // CreateNode called; waiting for new node
	PhaseProbing     Phase = "probing"      // waiting for consecutive probe passes
	PhaseActivating  Phase = "activating"   // promoting new node; starting grayscale
	PhaseDrainingOld Phase = "draining_old" // destroying old (already-down) node
	PhaseDone        Phase = "done"         // finished successfully
	PhaseFailed      Phase = "failed"       // terminal failure; human review needed
)

// ─────────────────────────────────────────────────────────────────────────────
// Tunable constants
// ─────────────────────────────────────────────────────────────────────────────

const (
	// MaxAttempts is the maximum number of create+probe tries before marking failed.
	MaxAttempts = 3

	// ProbeCyclesRequired is the number of consecutive Tick cycles with a passing
	// probe snapshot required before the new node is promoted to "up".
	ProbeCyclesRequired = 2

	// ProbeTimeout is the maximum time allowed in the probing phase per attempt.
	ProbeTimeout = 15 * time.Minute

	// GrayscaleInterval is the time between successive weight ramp steps.
	GrayscaleInterval = 6 * time.Hour
)

// GrayscaleWeights is the weight ladder for the warmup ramp.
// New / recovered nodes start at 10 and advance every GrayscaleInterval.
var GrayscaleWeights = []int{10, 25, 50, 75, 100}

// ─────────────────────────────────────────────────────────────────────────────
// Data types
// ─────────────────────────────────────────────────────────────────────────────

// NodeSpec describes the replacement node to be provisioned.
type NodeSpec struct {
	Tier       string
	Region     string
	Role       string
	ProviderID string // preferred provider for this attempt
	RealitySNI string // SNI rotation
	RealityPBK string
	HY2Port    int
	NameZH     string
	NameEn     string
	Tags       []string
}

// NodeInfo holds the properties of an existing node needed to build a
// replacement spec or to look up tier / region for the breaker.
type NodeInfo struct {
	ID         string
	Tier       string
	Region     string
	Role       string
	ProviderID string
	RealitySNI string
	RealityPBK string
	HY2Port    int
	NameZH     string
	NameEn     string
	Tags       []string
}

// ProviderInfo describes a cloud provider available for provisioning.
type ProviderInfo struct {
	ID      string   // provider identifier
	Regions []string // supported regions (empty means all regions)
}

// ─────────────────────────────────────────────────────────────────────────────
// Service interfaces
// ─────────────────────────────────────────────────────────────────────────────

// ProvisionService is the #14 provisioning interface.
// The real implementation is provided by task #14; a mock is used in tests.
type ProvisionService interface {
	// CreateNode provisions a new node. idempotencyKey makes the call crash-safe:
	// replaying the same key returns the already-created node without booting again.
	CreateNode(ctx context.Context, spec NodeSpec, idempotencyKey string) (nodeID string, err error)

	// DestroyNode tears down a node and releases its IP.
	DestroyNode(ctx context.Context, nodeID string) error

	// RotateIP swaps the elastic IP on an existing node without re-creating it.
	RotateIP(ctx context.Context, nodeID string) (newNodeID string, err error)

	// ListProviders returns providers available for the given tier and region.
	// The returned slice is ordered; callers use rotation for provider selection.
	ListProviders(ctx context.Context, tier, region string) ([]ProviderInfo, error)
}

// LifecycleService provides node lifecycle management (real impl: task #5).
type LifecycleService interface {
	// GetNode returns the current properties of the node, or nil if not found.
	GetNode(ctx context.Context, nodeID string) (*NodeInfo, error)

	// TransitionStatus performs an optimistic-lock status transition.
	// Returns (1, nil) on success, (0, nil) on lock conflict.
	TransitionStatus(ctx context.Context, nodeID string, from, to string, detail map[string]any) (int, error)

	// SetWeight updates the routing weight for the node.
	SetWeight(ctx context.Context, nodeID string, weight int) error

	// BumpVersion increments the global directory version so clients re-fetch.
	BumpVersion(ctx context.Context) error

	// WriteAuditLog records an audit trail entry.
	WriteAuditLog(ctx context.Context, actor, action, target, meta string) error
}

// ProbeSnapshotter reads the most-recent probe snapshots for a node (15A store).
type ProbeSnapshotter interface {
	SnapshotsByNode(ctx context.Context, nodeID string) (map[string]probe.ProbeSnapshot, error)
}

// ─────────────────────────────────────────────────────────────────────────────
// Breaker (15F stub)
// ─────────────────────────────────────────────────────────────────────────────

// Breaker is the 15F circuit-breaker interface.
type Breaker interface {
	// Allow returns true if a new replacement is permitted to proceed.
	Allow(tier, region string) bool

	// Record increments the failure counter after a replaced node has been destroyed.
	Record(tier, region string)
}

// StubBreaker always permits replacements.
// TODO(15F): replace with the real Breaker once task 15F is implemented.
type StubBreaker struct{}

// Allow implements Breaker; always returns true.
func (StubBreaker) Allow(_, _ string) bool { return true }

// Record implements Breaker; no-op until 15F is implemented.
func (StubBreaker) Record(_, _ string) {}

// ─────────────────────────────────────────────────────────────────────────────
// Notifier (15G stub)
// ─────────────────────────────────────────────────────────────────────────────

// Notifier is the 15G alerting interface.
type Notifier interface {
	NotifyFault(ctx context.Context, nodeID, reason string) error
}

// LogNotifier logs faults via slog. Used when no real notifier is wired.
type LogNotifier struct{}

// NotifyFault implements Notifier.
func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
	slog.Warn("orchestrate: replacement failed — manual review required",
		"node_id", nodeID,
		"reason", reason,
	)
	return nil
}

// ─────────────────────────────────────────────────────────────────────────────
// Clock (for testability)
// ─────────────────────────────────────────────────────────────────────────────

// Clock abstracts wall-clock time so tests can fast-forward without sleeping.
type Clock interface {
	Now() time.Time
}

// RealClock is the production clock.
type RealClock struct{}

// Now implements Clock.
func (RealClock) Now() time.Time { return time.Now().UTC() }