feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗

盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通) 而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。控制面: - agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖), 经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。 - NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。 - effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode 统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。客户端(connection_provider 连通看门狗): - 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。 - 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。 - 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-29 13:19:22 +08:00
parent 6249d5b2ea
commit 2080062d8c
15 changed files with 448 additions and 29 deletions
@@ -1,6 +1,7 @@
 package httpapi

 import (
+	"context"
 	"encoding/json"
 	"log/slog"
 	"net/http"
@@ -23,19 +24,40 @@ const (
 	freeMinuteTTL = time.Minute
 )

+// nodeLoadReader reads a node's last-reported runtime load (for the data-plane
+// health flag). *nodes.LoadCache satisfies it; nil = treat all nodes healthy.
+type nodeLoadReader interface {
+	Get(ctx context.Context, nodeUUID string) (*nodes.NodeLoad, bool, error)
+}
+
 // NodeAPI serves the /v1/nodes endpoints.
 type NodeAPI struct {
 	store     nodes.NodeStore
 	hub       *nodes.Hub
+	load      nodeLoadReader
 	deriveKey string
 	// rulesBaseURL 是控制面对外公网基址(PANGOLIN_PUBLIC_URL),供国内分流的
 	// rule_set .srs 下载用;空则分流不生效。
 	rulesBaseURL string
 }

-// NewNodeAPI creates a NodeAPI.
-func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, deriveKey, rulesBaseURL string) *NodeAPI {
-	return &NodeAPI{store: store, hub: hub, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
+// NewNodeAPI creates a NodeAPI. load may be nil (then all nodes are treated as
+// data-plane healthy — agent gRPC liveness still gates status).
+func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, load nodeLoadReader, deriveKey, rulesBaseURL string) *NodeAPI {
+	return &NodeAPI{store: store, hub: hub, load: load, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
+}
+
+// dataPlaneHealthy reports the node's last sing-box health (default true when
+// unknown: just registered / load expired / read error — agent liveness gates).
+func (a *NodeAPI) dataPlaneHealthy(ctx context.Context, nodeUUID string) bool {
+	if a.load == nil {
+		return true
+	}
+	l, ok, err := a.load.Get(ctx, nodeUUID)
+	if err != nil || !ok || l == nil {
+		return true
+	}
+	return l.DataPlaneHealthy
 }

 // ─── GET /v1/nodes ───────────────────────────────────────────────────────────
@@ -53,12 +75,13 @@ type nodeResponse struct {
 }

 // effectiveNodeStatus downgrades a DB-'up' node to "down" when its agent is
-// offline. The DB status column reflects provisioning/scheduler state, NOT agent
-// liveness — without this, a node whose agent has been offline for days still
-// shows healthy (the gap that hid a 6-day agent outage and let clients "connect"
-// to a node that couldn't serve them).
-func effectiveNodeStatus(dbStatus string, agentOnline bool) string {
-	if dbStatus == "up" && !agentOnline {
+// offline OR its data plane (sing-box) is unhealthy. The DB status column reflects
+// provisioning/scheduler state, NOT live health — without this, a node whose agent
+// has been offline for days, or whose sing-box has crashed while the agent stays
+// connected, still shows healthy and lets clients "connect" to a node that can't
+// serve them.
+func effectiveNodeStatus(dbStatus string, agentOnline, dataPlaneHealthy bool) string {
+	if dbStatus == "up" && (!agentOnline || !dataPlaneHealthy) {
 		return "down"
 	}
 	return dbStatus
@@ -85,7 +108,9 @@ func (a *NodeAPI) ListNodes(w http.ResponseWriter, r *http.Request) {
 			NameZH: n.NameZH,
 			NameEN: n.NameEN,
 			Tier:   n.Tier,
-			Status: effectiveNodeStatus(n.Status, a.hub == nil || a.hub.IsOnline(n.UUID)),
+			Status: effectiveNodeStatus(n.Status,
+				a.hub == nil || a.hub.IsOnline(n.UUID),
+				a.dataPlaneHealthy(r.Context(), n.UUID)),
 			Host:   host,
 			Port:   port,
 		})
@@ -188,7 +213,9 @@ func (a *NodeAPI) ConnectNode(w http.ResponseWriter, r *http.Request) {
 		apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
 		return
 	}
-	if node == nil || node.Status != "up" {
+	if node == nil || effectiveNodeStatus(node.Status,
+		a.hub == nil || a.hub.IsOnline(nodeUUID),
+		a.dataPlaneHealthy(r.Context(), nodeUUID)) != "up" {
 		apierr.WriteJSON(w, http.StatusNotFound, apierr.ErrNotFound)
 		return
 	}
@@ -1,25 +1,69 @@
 package httpapi

-import "testing"
+import (
+	"context"
+	"errors"
+	"testing"

-// TestEffectiveNodeStatus guards the "don't show a node healthy when its agent is
-// offline" fix: a DB-'up' node with an offline agent must report "down".
-func TestEffectiveNodeStatus(t *testing.T) {
+	"github.com/wangjia/pangolin/server/internal/nodes"
+)
+
+type fakeLoadReader struct {
+	load *nodes.NodeLoad
+	ok   bool
+	err  error
+}
+
+func (f fakeLoadReader) Get(context.Context, string) (*nodes.NodeLoad, bool, error) {
+	return f.load, f.ok, f.err
+}
+
+// TestDataPlaneHealthy: unknown (nil reader / missing / error) defaults to healthy
+// (agent liveness gates); a present load returns its reported flag.
+func TestDataPlaneHealthy(t *testing.T) {
+	ctx := context.Background()
 	cases := []struct {
-		name   string
-		db     string
-		online bool
-		want   string
+		name string
+		api  *NodeAPI
+		want bool
 	}{
-		{"up + agent online", "up", true, "up"},
-		{"up + agent offline → down", "up", false, "down"}, // the core fix
-		{"draining untouched when offline", "draining", false, "draining"},
-		{"down stays down", "down", true, "down"},
+		{"nil reader → healthy", &NodeAPI{}, true},
+		{"missing load → healthy", &NodeAPI{load: fakeLoadReader{ok: false}}, true},
+		{"read error → healthy", &NodeAPI{load: fakeLoadReader{err: errors.New("boom")}}, true},
+		{"present healthy", &NodeAPI{load: fakeLoadReader{load: &nodes.NodeLoad{DataPlaneHealthy: true}, ok: true}}, true},
+		{"present unhealthy → false", &NodeAPI{load: fakeLoadReader{load: &nodes.NodeLoad{DataPlaneHealthy: false}, ok: true}}, false},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			if got := effectiveNodeStatus(c.db, c.online); got != c.want {
-				t.Errorf("effectiveNodeStatus(%q, %v) = %q, want %q", c.db, c.online, got, c.want)
+			if got := c.api.dataPlaneHealthy(ctx, "n1"); got != c.want {
+				t.Errorf("dataPlaneHealthy = %v, want %v", got, c.want)
+			}
+		})
+	}
+}
+
+// TestEffectiveNodeStatus guards the "don't show a node healthy when it can't
+// serve" fix: a DB-'up' node reports "down" when its agent is offline OR its data
+// plane (sing-box) is unhealthy.
+func TestEffectiveNodeStatus(t *testing.T) {
+	cases := []struct {
+		name    string
+		db      string
+		online  bool
+		healthy bool
+		want    string
+	}{
+		{"up + agent online + healthy", "up", true, true, "up"},
+		{"up + agent offline → down", "up", false, true, "down"},          // agent-liveness fix
+		{"up + agent online + dp unhealthy → down", "up", true, false, "down"}, // data-plane fix
+		{"up + offline + unhealthy → down", "up", false, false, "down"},
+		{"draining untouched", "draining", false, false, "draining"},
+		{"down stays down", "down", true, true, "down"},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			if got := effectiveNodeStatus(c.db, c.online, c.healthy); got != c.want {
+				t.Errorf("effectiveNodeStatus(%q, %v, %v) = %q, want %q", c.db, c.online, c.healthy, got, c.want)
 			}
 		})
 	}