fix(scheduler): 修复 #15H 两个 flaky leader-election 测试的根因
TestGracefulShutdown(真 bug):runLoop 在 SetNX 获取 leader 锁后、进入
runLeaderLoop(其 defer 负责释放)之前,若 ctx 恰好在此刻 cancel,
`if ctx.Err() != nil { return }` 会丢弃已获取的 key 而不释放 →
关停后 leader 键残留(DEL 从未执行)。早 cancel 时三个 loop 各自命中此竞态,
故残留的键不固定。修复:将 ctx.Err() 早退限定在 SetNX 出错(未获取)的分支;
一旦获取成功就必定进入 runLeaderLoop,由其 defer 保证释放。
TestFollowerTakeover(测试设计竞态):测试同时启动 leader/follower 两实例却
假定名为 "leader" 的实例赢得选举——而选举是先到先得,"follower" 可能先抢到
detect 锁,导致 engLeader 永不 tick("leader never ticked")。修复:先单独
启动 leader 并等其 tick(确认占锁),再启动 follower,消除选举非确定性。
两修复均移除原 t.Skip,恢复测试。验证:各自隔离 12/12 通过、scheduler 全包
10/10、全量 go test ./... 3 连跑 0 失败、scheduler -race 无数据竞争。
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -249,12 +249,19 @@ func (s *Scheduler) runLoop(ctx context.Context, name string, interval time.Dura
|
||||
for {
|
||||
// Try to acquire the leader lease (SET NX PX <leaseTTL ms>).
|
||||
ok, err := s.rdb.SetNX(ctx, key, s.cfg.InstanceID, s.cfg.LeaseTTL).Result()
|
||||
if ctx.Err() != nil {
|
||||
return // context cancelled — don't log spurious errors
|
||||
}
|
||||
if err != nil {
|
||||
switch {
|
||||
case err != nil:
|
||||
// On a cancelled context SetNX errors without acquiring anything,
|
||||
// so it is safe to exit silently. Any other error is logged and retried.
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
slog.Error("scheduler: leader setnx error", "loop", name, "error", err)
|
||||
} else if ok {
|
||||
case ok:
|
||||
// Lease acquired. Always enter runLeaderLoop — even if ctx was
|
||||
// cancelled in the race immediately after SetNX — because its defer
|
||||
// is what releases the key. Returning here on ctx.Err() would leak
|
||||
// the just-acquired leader key (no clean DEL on shutdown).
|
||||
slog.Info("scheduler: acquired leader lease", "loop", name, "instance", s.cfg.InstanceID)
|
||||
s.runLeaderLoop(ctx, name, interval, tick, key)
|
||||
slog.Info("scheduler: relinquished leader", "loop", name, "instance", s.cfg.InstanceID)
|
||||
|
||||
Reference in New Issue
Block a user