feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗
盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通) 而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。 控制面: - agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖), 经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。 - NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。 - effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode 统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。 客户端(connection_provider 连通看门狗): - 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。 - 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。 - 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。 测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗 智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -23,19 +24,40 @@ const (
|
||||
freeMinuteTTL = time.Minute
|
||||
)
|
||||
|
||||
// nodeLoadReader reads a node's last-reported runtime load (for the data-plane
|
||||
// health flag). *nodes.LoadCache satisfies it; nil = treat all nodes healthy.
|
||||
type nodeLoadReader interface {
|
||||
Get(ctx context.Context, nodeUUID string) (*nodes.NodeLoad, bool, error)
|
||||
}
|
||||
|
||||
// NodeAPI serves the /v1/nodes endpoints.
|
||||
type NodeAPI struct {
|
||||
store nodes.NodeStore
|
||||
hub *nodes.Hub
|
||||
load nodeLoadReader
|
||||
deriveKey string
|
||||
// rulesBaseURL 是控制面对外公网基址(PANGOLIN_PUBLIC_URL),供国内分流的
|
||||
// rule_set .srs 下载用;空则分流不生效。
|
||||
rulesBaseURL string
|
||||
}
|
||||
|
||||
// NewNodeAPI creates a NodeAPI.
|
||||
func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, deriveKey, rulesBaseURL string) *NodeAPI {
|
||||
return &NodeAPI{store: store, hub: hub, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
|
||||
// NewNodeAPI creates a NodeAPI. load may be nil (then all nodes are treated as
|
||||
// data-plane healthy — agent gRPC liveness still gates status).
|
||||
func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, load nodeLoadReader, deriveKey, rulesBaseURL string) *NodeAPI {
|
||||
return &NodeAPI{store: store, hub: hub, load: load, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
|
||||
}
|
||||
|
||||
// dataPlaneHealthy reports the node's last sing-box health (default true when
|
||||
// unknown: just registered / load expired / read error — agent liveness gates).
|
||||
func (a *NodeAPI) dataPlaneHealthy(ctx context.Context, nodeUUID string) bool {
|
||||
if a.load == nil {
|
||||
return true
|
||||
}
|
||||
l, ok, err := a.load.Get(ctx, nodeUUID)
|
||||
if err != nil || !ok || l == nil {
|
||||
return true
|
||||
}
|
||||
return l.DataPlaneHealthy
|
||||
}
|
||||
|
||||
// ─── GET /v1/nodes ───────────────────────────────────────────────────────────
|
||||
@@ -53,12 +75,13 @@ type nodeResponse struct {
|
||||
}
|
||||
|
||||
// effectiveNodeStatus downgrades a DB-'up' node to "down" when its agent is
|
||||
// offline. The DB status column reflects provisioning/scheduler state, NOT agent
|
||||
// liveness — without this, a node whose agent has been offline for days still
|
||||
// shows healthy (the gap that hid a 6-day agent outage and let clients "connect"
|
||||
// to a node that couldn't serve them).
|
||||
func effectiveNodeStatus(dbStatus string, agentOnline bool) string {
|
||||
if dbStatus == "up" && !agentOnline {
|
||||
// offline OR its data plane (sing-box) is unhealthy. The DB status column reflects
|
||||
// provisioning/scheduler state, NOT live health — without this, a node whose agent
|
||||
// has been offline for days, or whose sing-box has crashed while the agent stays
|
||||
// connected, still shows healthy and lets clients "connect" to a node that can't
|
||||
// serve them.
|
||||
func effectiveNodeStatus(dbStatus string, agentOnline, dataPlaneHealthy bool) string {
|
||||
if dbStatus == "up" && (!agentOnline || !dataPlaneHealthy) {
|
||||
return "down"
|
||||
}
|
||||
return dbStatus
|
||||
@@ -85,7 +108,9 @@ func (a *NodeAPI) ListNodes(w http.ResponseWriter, r *http.Request) {
|
||||
NameZH: n.NameZH,
|
||||
NameEN: n.NameEN,
|
||||
Tier: n.Tier,
|
||||
Status: effectiveNodeStatus(n.Status, a.hub == nil || a.hub.IsOnline(n.UUID)),
|
||||
Status: effectiveNodeStatus(n.Status,
|
||||
a.hub == nil || a.hub.IsOnline(n.UUID),
|
||||
a.dataPlaneHealthy(r.Context(), n.UUID)),
|
||||
Host: host,
|
||||
Port: port,
|
||||
})
|
||||
@@ -188,7 +213,9 @@ func (a *NodeAPI) ConnectNode(w http.ResponseWriter, r *http.Request) {
|
||||
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
|
||||
return
|
||||
}
|
||||
if node == nil || node.Status != "up" {
|
||||
if node == nil || effectiveNodeStatus(node.Status,
|
||||
a.hub == nil || a.hub.IsOnline(nodeUUID),
|
||||
a.dataPlaneHealthy(r.Context(), nodeUUID)) != "up" {
|
||||
apierr.WriteJSON(w, http.StatusNotFound, apierr.ErrNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user