// Package orchestrate implements the replacement orchestration state machine // (task 15E). It is driven every 30 s by the OrchestrateLoop (task 15H). // // Key components: // - Replacer.Tick — drains the 15D replace queue and advances in-flight records. // - Grayscale.Advance — gradually ramps new/recovered node weight 10→25→50→75→100. package orchestrate import ( "context" "time" "github.com/wangjia/pangolin/server/internal/alert" "github.com/wangjia/pangolin/server/internal/scheduler/probe" ) // ───────────────────────────────────────────────────────────────────────────── // Phase constants // ───────────────────────────────────────────────────────────────────────────── // Phase is the orchestration phase of a replacement record. type Phase string const ( PhasePending Phase = "pending" // waiting for breaker / quota approval PhaseCreating Phase = "creating" // CreateNode called; waiting for new node PhaseProbing Phase = "probing" // waiting for consecutive probe passes PhaseActivating Phase = "activating" // promoting new node; starting grayscale PhaseDrainingOld Phase = "draining_old" // destroying old (already-down) node PhaseDone Phase = "done" // finished successfully PhaseFailed Phase = "failed" // terminal failure; human review needed ) // ───────────────────────────────────────────────────────────────────────────── // Tunable constants // ───────────────────────────────────────────────────────────────────────────── const ( // MaxAttempts is the maximum number of create+probe tries before marking failed. MaxAttempts = 3 // ProbeCyclesRequired is the number of consecutive Tick cycles with a passing // probe snapshot required before the new node is promoted to "up". ProbeCyclesRequired = 2 // ProbeTimeout is the maximum time allowed in the probing phase per attempt. ProbeTimeout = 15 * time.Minute // GrayscaleInterval is the time between successive weight ramp steps. GrayscaleInterval = 6 * time.Hour ) // GrayscaleWeights is the weight ladder for the warmup ramp. // New / recovered nodes start at 10 and advance every GrayscaleInterval. var GrayscaleWeights = []int{10, 25, 50, 75, 100} // ───────────────────────────────────────────────────────────────────────────── // Data types // ───────────────────────────────────────────────────────────────────────────── // NodeSpec describes the replacement node to be provisioned. type NodeSpec struct { Tier string Region string Role string ProviderID string // preferred provider for this attempt RealitySNI string // SNI rotation RealityPBK string HY2Port int NameZH string NameEn string Tags []string } // NodeInfo holds the properties of an existing node needed to build a // replacement spec or to look up tier / region for the breaker. type NodeInfo struct { ID string Tier string Region string Role string ProviderID string RealitySNI string RealityPBK string HY2Port int NameZH string NameEn string Tags []string } // ProviderInfo describes a cloud provider available for provisioning. type ProviderInfo struct { ID string // provider identifier Regions []string // supported regions (empty means all regions) } // ───────────────────────────────────────────────────────────────────────────── // Service interfaces // ───────────────────────────────────────────────────────────────────────────── // ProvisionService is the #14 provisioning interface. // The real implementation is provided by task #14; a mock is used in tests. type ProvisionService interface { // CreateNode provisions a new node. idempotencyKey makes the call crash-safe: // replaying the same key returns the already-created node without booting again. CreateNode(ctx context.Context, spec NodeSpec, idempotencyKey string) (nodeID string, err error) // DestroyNode tears down a node and releases its IP. DestroyNode(ctx context.Context, nodeID string) error // RotateIP swaps the elastic IP on an existing node without re-creating it. RotateIP(ctx context.Context, nodeID string) (newNodeID string, err error) // ListProviders returns providers available for the given tier and region. // The returned slice is ordered; callers use rotation for provider selection. ListProviders(ctx context.Context, tier, region string) ([]ProviderInfo, error) } // LifecycleService provides node lifecycle management (real impl: task #5). type LifecycleService interface { // GetNode returns the current properties of the node, or nil if not found. GetNode(ctx context.Context, nodeID string) (*NodeInfo, error) // TransitionStatus performs an optimistic-lock status transition. // Returns (1, nil) on success, (0, nil) on lock conflict. TransitionStatus(ctx context.Context, nodeID string, from, to string, detail map[string]any) (int, error) // SetWeight updates the routing weight for the node. SetWeight(ctx context.Context, nodeID string, weight int) error // BumpVersion increments the global directory version so clients re-fetch. BumpVersion(ctx context.Context) error // WriteAuditLog records an audit trail entry. WriteAuditLog(ctx context.Context, actor, action, target, meta string) error } // ProbeSnapshotter reads the most-recent probe snapshots for a node (15A store). type ProbeSnapshotter interface { SnapshotsByNode(ctx context.Context, nodeID string) (map[string]probe.ProbeSnapshot, error) } // ───────────────────────────────────────────────────────────────────────────── // Breaker (15F stub) // ───────────────────────────────────────────────────────────────────────────── // Breaker is the 15F circuit-breaker interface. type Breaker interface { // Allow returns true if a new replacement is permitted to proceed. Allow(tier, region string) bool // Record increments the failure counter after a replaced node has been destroyed. Record(tier, region string) } // StubBreaker always permits replacements. // TODO(15F): replace with the real Breaker once task 15F is implemented. type StubBreaker struct{} // Allow implements Breaker; always returns true. func (StubBreaker) Allow(_, _ string) bool { return true } // Record implements Breaker; no-op until 15F is implemented. func (StubBreaker) Record(_, _ string) {} // ───────────────────────────────────────────────────────────────────────────── // Notifier (15G) // ───────────────────────────────────────────────────────────────────────────── // Notifier is the 15G unified alert outlet used by the orchestrator. // It is satisfied by alert.Notifier (TG implementation) and alert.LogNotifier // (fallback / development stub). type Notifier = alert.Notifier // LogNotifier is re-exported for callers that need a no-op Notifier without // importing the alert package directly. type LogNotifier = alert.LogNotifier // ───────────────────────────────────────────────────────────────────────────── // Clock (for testability) // ───────────────────────────────────────────────────────────────────────────── // Clock abstracts wall-clock time so tests can fast-forward without sleeping. type Clock interface { Now() time.Time } // RealClock is the production clock. type RealClock struct{} // Now implements Clock. func (RealClock) Now() time.Time { return time.Now().UTC() }