Files
wangjia 2080062d8c feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗
盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通)
而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。

控制面:
- agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖),
  经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。
- NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。
- effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode
  统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。

客户端(connection_provider 连通看门狗):
- 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。
- 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。
- 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。

测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗
智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-29 13:19:22 +08:00

218 lines
6.1 KiB
Go

package agentd
import (
"context"
"errors"
"math/rand"
"sync/atomic"
"time"
agentv1 "github.com/wangjia/pangolin/server/internal/pb/agentv1"
)
// errResync is returned within a session to force a reconnect + full Register.
var errResync = errors.New("agentd: full resync required")
// Agent is the top-level node daemon. Construct with New and drive with Run.
type Agent struct {
cfg Config
sb *SingBox
nodeUUID atomic.Value // string; written once by Run, read by all loops + tests
dial Dialer
enrollDial EnrollDialer
load LoadSource
usage UsageSource
health HealthSource
clock func() time.Time
lastCommandID atomic.Int64
shutdown context.CancelFunc
restarter Restarter
}
// Option customises an Agent (primarily for tests / dependency injection).
type Option func(*Agent)
func WithRestarter(r Restarter) Option { return func(a *Agent) { a.restarter = r } }
func WithDialer(d Dialer) Option { return func(a *Agent) { a.dial = d } }
func WithEnrollDialer(d EnrollDialer) Option { return func(a *Agent) { a.enrollDial = d } }
func WithLoadSource(l LoadSource) Option { return func(a *Agent) { a.load = l } }
func WithUsageSource(u UsageSource) Option { return func(a *Agent) { a.usage = u } }
func WithHealthSource(h HealthSource) Option { return func(a *Agent) { a.health = h } }
func WithClock(f func() time.Time) Option { return func(a *Agent) { a.clock = f } }
// New builds an Agent. Production callers pass nothing extra and get mTLS dialers;
// tests inject bufconn dialers, fake restarters, etc.
func New(cfg Config, opts ...Option) *Agent {
cfg = cfg.withDefaults()
a := &Agent{cfg: cfg, clock: time.Now}
for _, o := range opts {
o(a)
}
if a.dial == nil {
if cfg.Insecure {
a.dial = insecureDialer(cfg.ControlPlaneAddr)
} else {
a.dial = NewMTLSDialer(cfg)
}
}
if a.enrollDial == nil {
if cfg.Insecure {
a.enrollDial = EnrollDialer(insecureDialer(cfg.ControlPlaneAddr))
} else {
a.enrollDial = NewEnrollDialer(cfg)
}
}
a.sb = NewSingBox(cfg, a.restarter)
a.sb.clock = a.clock
if a.load == nil {
a.load = defaultLoadSource{sb: a.sb}
}
if a.usage == nil {
a.usage = nopUsageSource{}
}
if a.health == nil {
a.health = newClashHealthSource()
}
return a
}
// SingBox exposes the underlying manager (tests / introspection).
func (a *Agent) SingBox() *SingBox { return a.sb }
// UseClashUsage wires the legacy usage source: reads node-total traffic from the
// local clash_api and reports per-window deltas (split across dp_uuids — exact
// for one active user, approximate for many). Superseded by UseV2RayUsage.
func (a *Agent) UseClashUsage() { a.usage = newClashUsageSource(a.sb) }
// UseV2RayUsage wires the accurate usage source: reads PER-USER traffic from the
// local v2ray_api StatsService (one counter pair per dp_uuid). Production main
// enables this; tests leave it off.
func (a *Agent) UseV2RayUsage() { a.usage = newV2RayUsageSource() }
// NodeUUID returns the enrolled node UUID (empty until Run enrolls).
func (a *Agent) NodeUUID() string {
v, _ := a.nodeUUID.Load().(string)
return v
}
// Run enrolls (if needed), restores state, then maintains the control-plane
// connection with exponential-backoff reconnect until ctx is cancelled.
func (a *Agent) Run(ctx context.Context) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
a.shutdown = cancel
uuid, err := EnsureEnrolled(ctx, a.cfg, a.enrollDial)
if err != nil {
return err
}
a.nodeUUID.Store(uuid)
if err := a.sb.LoadState(); err != nil {
return err
}
go a.sb.Run(ctx)
go a.runTTL(ctx)
a.sb.markDirty() // render whatever we restored before the first connect
backoff := a.cfg.BackoffMin
for {
if ctx.Err() != nil {
return nil
}
registered, err := a.runSession(ctx)
if ctx.Err() != nil {
return nil
}
if registered {
backoff = a.cfg.BackoffMin // healthy session → reset backoff
}
if err != nil && !errors.Is(err, errResync) {
logf("session ended: %v; reconnecting in %s", err, backoff)
}
if !sleepCtx(ctx, jitter(backoff)) {
return nil
}
backoff = nextBackoff(backoff, a.cfg.BackoffMax)
}
}
// runSession establishes one connection: Register (full snapshot), then runs
// heartbeat/subscribe/usage until the first of them fails. Returns whether
// Register succeeded (used to reset backoff).
func (a *Agent) runSession(ctx context.Context) (registered bool, err error) {
conn, err := a.dial(ctx)
if err != nil {
return false, err
}
defer conn.Close()
client := agentv1.NewAgentServiceClient(conn)
snap, err := client.Register(ctx, &agentv1.RegisterRequest{
NodeUUID: a.NodeUUID(),
AgentVersion: a.cfg.AgentVersion,
LocalConfigVersion: a.sb.ConfigVersion(),
})
if err != nil {
return false, err
}
// Full snapshot is authoritative: overwrite local credential table + inbounds.
a.sb.ApplyConfig(snap, true)
if snap.LastCommandID > a.lastCommandID.Load() {
a.lastCommandID.Store(snap.LastCommandID)
}
sctx, cancel := context.WithCancel(ctx)
defer cancel()
errc := make(chan error, 3)
go func() { errc <- a.runHeartbeat(sctx, client) }()
go func() { errc <- a.runSubscribe(sctx, client) }()
go func() { errc <- a.runUsage(sctx, client) }()
first := <-errc
cancel()
<-errc
<-errc
return true, first
}
// ─── backoff helpers ─────────────────────────────────────────────────────────
func nextBackoff(cur, max time.Duration) time.Duration {
next := cur * 2
if next > max {
next = max
}
return next
}
// jitter applies ±20% randomisation to avoid thundering-herd reconnects.
func jitter(d time.Duration) time.Duration {
if d <= 0 {
return 0
}
delta := float64(d) * 0.2
return d + time.Duration((rand.Float64()*2-1)*delta)
}
// sleepCtx sleeps for d unless ctx is cancelled first. Returns false if cancelled.
func sleepCtx(ctx context.Context, d time.Duration) bool {
if d <= 0 {
return ctx.Err() == nil
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return false
case <-t.C:
return true
}
}