ebc9c1e702
## 内容 ### server/internal/scheduler/deps.go 新增 scheduler 级接口文件,定义所有外部依赖接口: - DetectEngine / OrchestrateReplacer / OrchestrateGrayscale(15D/15E) - CapacityService + StubCapacityService(15F 存根) - ProbeStateReader(15A)、ThirdPartyProber / TargetProvider(15C) - Notifier + LogNotifier(15G)、StaticTargetProvider ### server/internal/scheduler/scheduler.go Scheduler 主体实现: - Config 结构体:注入所有依赖 + 可配置的选主时间参数(方便测试) - Run(ctx) = 安装 SIGTERM/SIGINT 信号 + Start(ctx) - Start(ctx) = 启动三 loop goroutine + 可选 15C 采集 goroutine + 30s 优雅退出 - runLoop: 非主实例每 5s 抢租约(SET NX PX 30000) - runLeaderLoop: 主实例按 interval tick,defer 释放租约 - keepLease: 每 10s Lua 校验后续租(PEXPIRE),失败即退出 - releaseLease: Lua 校验后 DEL,加速接管 - capacityTick: 15F check → 15E grayscale.Advance → 15F decay → 探针失联告警 - runThirdPartyProber: 无主从约束全副本跑(理由注释在代码中) ### server/internal/scheduler/wiring.go BuildStubConfig(): 用存根 LC/Prov 构建可运行的 Config, 供 server main 和集成测试使用;待 #5/#14 就绪后替换真实实现。 ### server/internal/scheduler/scheduler_test.go (go test -race 全通过) - TestLeaderElection: 两实例竞争,任意时刻只有一个在 tick - TestFollowerTakeover: 主实例退出后备实例在 ≤LeaseTTL 内接管 - TestGracefulShutdown: ctx 取消后三个 leader key 被主动 DEL - TestCapacityTickProbeDisconnect: 失联探针触发 Notifier - TestE2EMockScenario: 端到端 mock 剧本 up→suspect(降权10)→confirmed→down + 替换队列 → CreateNode→probing→activating(weight10+BumpVersion+grayscale)→ draining_old→done,全链 node_events/audit_log 对账 ### server/cmd/server/main.go - 新增 SCHED_ENABLED=true 开关,灰度启动 scheduler - 复用 REDIS_ADDR/REDIS_PASSWORD,probe.Store 共享给 scheduler - 通过 signal.NotifyContext 传递 SIGTERM,确保优雅退出 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
137 lines
7.1 KiB
Go
137 lines
7.1 KiB
Go
// Package scheduler wires together the 15D detection engine, 15E orchestration
|
||
// engine, 15F capacity service, 15A probe store, and 15G notifier into three
|
||
// leader-elected goroutine loops (DetectLoop / OrchestrateLoop / CapacityLoop),
|
||
// plus an optional unguarded 15C third-party prober goroutine.
|
||
//
|
||
// This file declares the narrow dependency interfaces used by Scheduler. Real
|
||
// implementations satisfy these interfaces; test mocks do too.
|
||
package scheduler
|
||
|
||
import (
|
||
"context"
|
||
"log/slog"
|
||
|
||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||
)
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
// 15D – detection engine
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
// DetectEngine is the 15D interface. *detect.Engine satisfies it.
|
||
// Tick is called by DetectLoop on every 5-minute cycle.
|
||
type DetectEngine interface {
|
||
Tick(ctx context.Context) error
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
// 15E – orchestration engine
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
// OrchestrateReplacer is the 15E replacement-orchestration interface.
|
||
// *orchestrate.Replacer satisfies it.
|
||
// Tick is called by OrchestrateLoop every 30 seconds.
|
||
type OrchestrateReplacer interface {
|
||
Tick(ctx context.Context) error
|
||
}
|
||
|
||
// OrchestrateGrayscale is the 15E grayscale warm-up interface.
|
||
// *orchestrate.Grayscale satisfies it.
|
||
// Advance is called by CapacityLoop every 1–2 minutes.
|
||
type OrchestrateGrayscale interface {
|
||
Advance(ctx context.Context) error
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
// 15F – capacity service (stub until task 15F is delivered)
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
// CapacityService is the 15F interface for watermark/quota checking and
|
||
// circuit-breaker time-decay. Replace StubCapacityService with the real
|
||
// 15F implementation once that task is complete.
|
||
type CapacityService interface {
|
||
// Check verifies that the node-pool watermark and per-region quotas are
|
||
// within acceptable bounds. Non-fatal: errors are logged, not returned.
|
||
Check(ctx context.Context) error
|
||
|
||
// Decay applies time-based decay to circuit-breaker event counters so that
|
||
// temporary bursts do not permanently block replacements.
|
||
Decay(ctx context.Context) error
|
||
}
|
||
|
||
// StubCapacityService is a no-op CapacityService used until task 15F is ready.
|
||
type StubCapacityService struct{}
|
||
|
||
// Check implements CapacityService (always succeeds).
|
||
func (StubCapacityService) Check(_ context.Context) error { return nil }
|
||
|
||
// Decay implements CapacityService (always succeeds).
|
||
func (StubCapacityService) Decay(_ context.Context) error { return nil }
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
// 15A – probe state reader
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
// ProbeStateReader reads the heartbeat liveness of probe agents from the 15A
|
||
// Redis store. *probe.Store satisfies it (via AliveProbes).
|
||
type ProbeStateReader interface {
|
||
// AliveProbes returns the IDs of probe agents that have sent a heartbeat
|
||
// within the last heartbeatTTL window (15 min). An empty result means
|
||
// "no recent heartbeats", not "all probes are down".
|
||
AliveProbes(ctx context.Context) ([]string, error)
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
// 15C – third-party prober
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
// ThirdPartyProber is the 15C interface.
|
||
// *probe.AliyunSyntheticAgent satisfies it.
|
||
type ThirdPartyProber interface {
|
||
// RunOnce issues a full multi-ISP probe cycle for all targets and writes
|
||
// results to Redis via the 15A probe store.
|
||
RunOnce(ctx context.Context, targets []probe.ProbeTarget) error
|
||
}
|
||
|
||
// TargetProvider supplies the current list of active nodes to be probed via
|
||
// the 15C third-party service. Typically backed by a DB query; a static
|
||
// implementation is available as StaticTargetProvider.
|
||
type TargetProvider interface {
|
||
ListProbeTargets(ctx context.Context) ([]probe.ProbeTarget, error)
|
||
}
|
||
|
||
// StaticTargetProvider returns a fixed, pre-configured list of probe targets.
|
||
// Useful for initial deployment and integration tests before a DB-backed
|
||
// provider is available.
|
||
type StaticTargetProvider struct {
|
||
Targets []probe.ProbeTarget
|
||
}
|
||
|
||
// ListProbeTargets implements TargetProvider.
|
||
func (p StaticTargetProvider) ListProbeTargets(_ context.Context) ([]probe.ProbeTarget, error) {
|
||
return p.Targets, nil
|
||
}
|
||
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
// 15G – notifier
|
||
// ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
// Notifier is the 15G alerting interface. The real implementation sends a
|
||
// Telegram (or similar) alert; LogNotifier is the stub.
|
||
type Notifier interface {
|
||
NotifyFault(ctx context.Context, nodeID, reason string) error
|
||
}
|
||
|
||
// LogNotifier is a Notifier stub that writes to slog.
|
||
// Used when no real 15G notifier is wired.
|
||
type LogNotifier struct{}
|
||
|
||
// NotifyFault implements Notifier.
|
||
func (LogNotifier) NotifyFault(_ context.Context, nodeID, reason string) error {
|
||
slog.Warn("scheduler: node fault — manual review required",
|
||
"node_id", nodeID,
|
||
"reason", reason,
|
||
)
|
||
return nil
|
||
}
|