feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗
盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通) 而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。 控制面: - agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖), 经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。 - NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。 - effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode 统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。 客户端(connection_provider 连通看门狗): - 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。 - 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。 - 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。 测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗 智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,7 @@ package agentd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
agentv1 "github.com/wangjia/pangolin/server/internal/pb/agentv1"
|
||||
@@ -23,6 +24,49 @@ func (d defaultLoadSource) Load() (int32, int64, int64, float64) {
|
||||
return int32(d.sb.OnlinePeers()), 0, 0, 0
|
||||
}
|
||||
|
||||
// HealthSource reports whether the local data plane (sing-box) is healthy, sent
|
||||
// in each heartbeat so the control plane can mark a node down when its sing-box
|
||||
// is dead even though the agent's gRPC stream is still up.
|
||||
type HealthSource interface{ Healthy() bool }
|
||||
|
||||
// clashHealthSource probes sing-box's clash_api /version. A reachable, 200-OK API
|
||||
// means the sing-box process is alive and its config loaded. Applies a 2-strike
|
||||
// debounce: a single failed probe is tolerated (covers the sing-box restart
|
||||
// window); two consecutive failures report unhealthy.
|
||||
type clashHealthSource struct {
|
||||
probe func() bool // single probe, true = healthy; injectable for tests
|
||||
fails int
|
||||
}
|
||||
|
||||
func newClashHealthSource() *clashHealthSource {
|
||||
client := &http.Client{Timeout: 2 * time.Second}
|
||||
return &clashHealthSource{probe: func() bool {
|
||||
req, err := http.NewRequestWithContext(
|
||||
context.Background(), http.MethodGet, "http://"+clashAPIAddr+"/version", nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if clashAPISecret != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+clashAPISecret)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}}
|
||||
}
|
||||
|
||||
func (h *clashHealthSource) Healthy() bool {
|
||||
if h.probe() {
|
||||
h.fails = 0
|
||||
return true
|
||||
}
|
||||
h.fails++
|
||||
return h.fails < 2 // tolerate 1 failure; unhealthy from the 2nd consecutive
|
||||
}
|
||||
|
||||
// runHeartbeat sends a heartbeat every cfg.HeartbeatInterval until ctx is
|
||||
// cancelled or an RPC error occurs. A need_full_resync response returns
|
||||
// errResync so the session re-Registers and overwrites local state.
|
||||
@@ -51,6 +95,7 @@ func (a *Agent) sendHeartbeat(ctx context.Context, client agentv1.AgentServiceCl
|
||||
BandwidthDownBps: down,
|
||||
CPUPercent: cpu,
|
||||
TimestampUnix: a.clock().Unix(),
|
||||
DataPlaneHealthy: a.health.Healthy(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Reference in New Issue
Block a user