feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗

盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通)
而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。

控制面:
- agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖),
  经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。
- NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。
- effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode
  统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。

客户端(connection_provider 连通看门狗):
- 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。
- 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。
- 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。

测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗
智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-29 13:19:22 +08:00
parent 6249d5b2ea
commit 2080062d8c
15 changed files with 448 additions and 29 deletions
+38 -11
View File
@@ -1,6 +1,7 @@
package httpapi
import (
"context"
"encoding/json"
"log/slog"
"net/http"
@@ -23,19 +24,40 @@ const (
freeMinuteTTL = time.Minute
)
// nodeLoadReader reads a node's last-reported runtime load (for the data-plane
// health flag). *nodes.LoadCache satisfies it; nil = treat all nodes healthy.
type nodeLoadReader interface {
Get(ctx context.Context, nodeUUID string) (*nodes.NodeLoad, bool, error)
}
// NodeAPI serves the /v1/nodes endpoints.
type NodeAPI struct {
store nodes.NodeStore
hub *nodes.Hub
load nodeLoadReader
deriveKey string
// rulesBaseURL 是控制面对外公网基址(PANGOLIN_PUBLIC_URL),供国内分流的
// rule_set .srs 下载用;空则分流不生效。
rulesBaseURL string
}
// NewNodeAPI creates a NodeAPI.
func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, deriveKey, rulesBaseURL string) *NodeAPI {
return &NodeAPI{store: store, hub: hub, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
// NewNodeAPI creates a NodeAPI. load may be nil (then all nodes are treated as
// data-plane healthy — agent gRPC liveness still gates status).
func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, load nodeLoadReader, deriveKey, rulesBaseURL string) *NodeAPI {
return &NodeAPI{store: store, hub: hub, load: load, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
}
// dataPlaneHealthy reports the node's last sing-box health (default true when
// unknown: just registered / load expired / read error — agent liveness gates).
func (a *NodeAPI) dataPlaneHealthy(ctx context.Context, nodeUUID string) bool {
if a.load == nil {
return true
}
l, ok, err := a.load.Get(ctx, nodeUUID)
if err != nil || !ok || l == nil {
return true
}
return l.DataPlaneHealthy
}
// ─── GET /v1/nodes ───────────────────────────────────────────────────────────
@@ -53,12 +75,13 @@ type nodeResponse struct {
}
// effectiveNodeStatus downgrades a DB-'up' node to "down" when its agent is
// offline. The DB status column reflects provisioning/scheduler state, NOT agent
// liveness — without this, a node whose agent has been offline for days still
// shows healthy (the gap that hid a 6-day agent outage and let clients "connect"
// to a node that couldn't serve them).
func effectiveNodeStatus(dbStatus string, agentOnline bool) string {
if dbStatus == "up" && !agentOnline {
// offline OR its data plane (sing-box) is unhealthy. The DB status column reflects
// provisioning/scheduler state, NOT live health — without this, a node whose agent
// has been offline for days, or whose sing-box has crashed while the agent stays
// connected, still shows healthy and lets clients "connect" to a node that can't
// serve them.
func effectiveNodeStatus(dbStatus string, agentOnline, dataPlaneHealthy bool) string {
if dbStatus == "up" && (!agentOnline || !dataPlaneHealthy) {
return "down"
}
return dbStatus
@@ -85,7 +108,9 @@ func (a *NodeAPI) ListNodes(w http.ResponseWriter, r *http.Request) {
NameZH: n.NameZH,
NameEN: n.NameEN,
Tier: n.Tier,
Status: effectiveNodeStatus(n.Status, a.hub == nil || a.hub.IsOnline(n.UUID)),
Status: effectiveNodeStatus(n.Status,
a.hub == nil || a.hub.IsOnline(n.UUID),
a.dataPlaneHealthy(r.Context(), n.UUID)),
Host: host,
Port: port,
})
@@ -188,7 +213,9 @@ func (a *NodeAPI) ConnectNode(w http.ResponseWriter, r *http.Request) {
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
if node == nil || node.Status != "up" {
if node == nil || effectiveNodeStatus(node.Status,
a.hub == nil || a.hub.IsOnline(nodeUUID),
a.dataPlaneHealthy(r.Context(), nodeUUID)) != "up" {
apierr.WriteJSON(w, http.StatusNotFound, apierr.ErrNotFound)
return
}