feat(scheduler): 装配 scheduler — 三 loop + Redis 选主 + 优雅退出 [tsk_rtYVDLkmc5mo]
## 内容 ### server/internal/scheduler/deps.go 新增 scheduler 级接口文件,定义所有外部依赖接口: - DetectEngine / OrchestrateReplacer / OrchestrateGrayscale(15D/15E) - CapacityService + StubCapacityService(15F 存根) - ProbeStateReader(15A)、ThirdPartyProber / TargetProvider(15C) - Notifier + LogNotifier(15G)、StaticTargetProvider ### server/internal/scheduler/scheduler.go Scheduler 主体实现: - Config 结构体:注入所有依赖 + 可配置的选主时间参数(方便测试) - Run(ctx) = 安装 SIGTERM/SIGINT 信号 + Start(ctx) - Start(ctx) = 启动三 loop goroutine + 可选 15C 采集 goroutine + 30s 优雅退出 - runLoop: 非主实例每 5s 抢租约(SET NX PX 30000) - runLeaderLoop: 主实例按 interval tick,defer 释放租约 - keepLease: 每 10s Lua 校验后续租(PEXPIRE),失败即退出 - releaseLease: Lua 校验后 DEL,加速接管 - capacityTick: 15F check → 15E grayscale.Advance → 15F decay → 探针失联告警 - runThirdPartyProber: 无主从约束全副本跑(理由注释在代码中) ### server/internal/scheduler/wiring.go BuildStubConfig(): 用存根 LC/Prov 构建可运行的 Config, 供 server main 和集成测试使用;待 #5/#14 就绪后替换真实实现。 ### server/internal/scheduler/scheduler_test.go (go test -race 全通过) - TestLeaderElection: 两实例竞争,任意时刻只有一个在 tick - TestFollowerTakeover: 主实例退出后备实例在 ≤LeaseTTL 内接管 - TestGracefulShutdown: ctx 取消后三个 leader key 被主动 DEL - TestCapacityTickProbeDisconnect: 失联探针触发 Notifier - TestE2EMockScenario: 端到端 mock 剧本 up→suspect(降权10)→confirmed→down + 替换队列 → CreateNode→probing→activating(weight10+BumpVersion+grayscale)→ draining_old→done,全链 node_events/audit_log 对账 ### server/cmd/server/main.go - 新增 SCHED_ENABLED=true 开关,灰度启动 scheduler - 复用 REDIS_ADDR/REDIS_PASSWORD,probe.Store 共享给 scheduler - 通过 signal.NotifyContext 传递 SIGTERM,确保优雅退出 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -10,6 +10,8 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
@@ -24,6 +26,7 @@ import (
|
||||
"github.com/wangjia/pangolin/server/internal/nodes"
|
||||
agentv1 "github.com/wangjia/pangolin/server/internal/pb/agentv1"
|
||||
"github.com/wangjia/pangolin/server/internal/redisutil"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler"
|
||||
"github.com/wangjia/pangolin/server/internal/scheduler/probe"
|
||||
)
|
||||
|
||||
@@ -66,6 +69,7 @@ func main() {
|
||||
// If PROBE_SECRETS is empty the route is not registered and the server
|
||||
// starts normally without the probe ingest endpoint.
|
||||
probeSecretsJSON := os.Getenv("PROBE_SECRETS")
|
||||
var sharedProbeStore *probe.Store // may be re-used by the scheduler
|
||||
if probeSecretsJSON != "" {
|
||||
redisAddr := getenvDefault("REDIS_ADDR", "127.0.0.1:6379")
|
||||
rdb, err := redisutil.New(redisAddr, os.Getenv("REDIS_PASSWORD"), 0)
|
||||
@@ -78,12 +82,58 @@ func main() {
|
||||
}
|
||||
reg := probe.NewMapRegistry(secretMap)
|
||||
st := probe.NewStore(rdb)
|
||||
sharedProbeStore = st
|
||||
h := probe.NewIngestHandler(reg, st)
|
||||
r.Post("/probe/report", h.ServeHTTP)
|
||||
log.Printf("probe ingest route registered (%d probe(s))", len(secretMap))
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Scheduler (optional) ─────────────────────────────────────────────────
|
||||
//
|
||||
// Set SCHED_ENABLED=true to start the three leader-elected scheduler loops
|
||||
// (DetectLoop / OrchestrateLoop / CapacityLoop) in this process alongside the
|
||||
// HTTP and gRPC servers.
|
||||
//
|
||||
// Dependencies:
|
||||
// REDIS_ADDR, REDIS_PASSWORD – shared with the probe route (new client if
|
||||
// the probe route is disabled).
|
||||
//
|
||||
// Rollback: set SCHED_ENABLED=false (or unset it) and restart; all other
|
||||
// server functionality is unaffected.
|
||||
if os.Getenv("SCHED_ENABLED") == "true" {
|
||||
redisAddr := getenvDefault("REDIS_ADDR", "127.0.0.1:6379")
|
||||
schedRDB, err := redisutil.New(redisAddr, os.Getenv("REDIS_PASSWORD"), 0)
|
||||
if err != nil {
|
||||
log.Printf("scheduler: redis connect failed (%v) — scheduler disabled", err)
|
||||
} else {
|
||||
// Use the shared probe store if the probe route is already wired up;
|
||||
// otherwise create a standalone store backed by the scheduler Redis client.
|
||||
ps := sharedProbeStore
|
||||
if ps == nil {
|
||||
ps = probe.NewStore(schedRDB)
|
||||
}
|
||||
cfg := scheduler.BuildStubConfig(schedRDB, ps)
|
||||
// TODO(#5): replace stub lifecycle with real LifecycleService
|
||||
// TODO(#14): replace stub provision with real ProvisionService
|
||||
sched := scheduler.New(cfg)
|
||||
|
||||
// The scheduler installs its own SIGTERM/SIGINT handler via Run().
|
||||
// We also respect the Go context tree: sigCtx cancels when the process
|
||||
// receives a signal, giving the scheduler up to TickTimeout (30 s) to
|
||||
// finish any in-flight tick before the OS kills the process.
|
||||
sigCtx, stopSig := signal.NotifyContext(context.Background(),
|
||||
os.Interrupt, syscall.SIGTERM)
|
||||
go func() {
|
||||
defer stopSig()
|
||||
if err := sched.Start(sigCtx); err != nil {
|
||||
slog.Error("scheduler: Start error", "error", err)
|
||||
}
|
||||
}()
|
||||
log.Printf("scheduler started (SCHED_ENABLED=true, stub lifecycle/provision)")
|
||||
}
|
||||
}
|
||||
|
||||
// Mount all /v1/... routes. HandlerFromMuxWithBaseURL registers every route
|
||||
// from the OpenAPI spec onto the provided chi router with the given prefix,
|
||||
// so the router itself is the http.Handler we serve.
|
||||
|
||||
Reference in New Issue
Block a user