Files
pangolin/server/internal/httpapi/nodes.go
T
wangjia 2080062d8c feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗
盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通)
而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。

控制面:
- agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖),
  经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。
- NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。
- effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode
  统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。

客户端(connection_provider 连通看门狗):
- 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。
- 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。
- 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。

测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗
智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-29 13:19:22 +08:00

328 lines
12 KiB
Go

package httpapi
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"github.com/wangjia/pangolin/server/internal/apierr"
"github.com/wangjia/pangolin/server/internal/auth"
"github.com/wangjia/pangolin/server/internal/nodes"
agentv1 "github.com/wangjia/pangolin/server/internal/pb/agentv1"
)
const (
// paidCredentialTTL is the default connect credential lifetime for paid users.
paidCredentialTTL = 24 * time.Hour
// freeCredentialTTL is the per-minute TTL for free users (per remaining minutes).
freeMinuteTTL = time.Minute
)
// nodeLoadReader reads a node's last-reported runtime load (for the data-plane
// health flag). *nodes.LoadCache satisfies it; nil = treat all nodes healthy.
type nodeLoadReader interface {
Get(ctx context.Context, nodeUUID string) (*nodes.NodeLoad, bool, error)
}
// NodeAPI serves the /v1/nodes endpoints.
type NodeAPI struct {
store nodes.NodeStore
hub *nodes.Hub
load nodeLoadReader
deriveKey string
// rulesBaseURL 是控制面对外公网基址(PANGOLIN_PUBLIC_URL),供国内分流的
// rule_set .srs 下载用;空则分流不生效。
rulesBaseURL string
}
// NewNodeAPI creates a NodeAPI. load may be nil (then all nodes are treated as
// data-plane healthy — agent gRPC liveness still gates status).
func NewNodeAPI(store nodes.NodeStore, hub *nodes.Hub, load nodeLoadReader, deriveKey, rulesBaseURL string) *NodeAPI {
return &NodeAPI{store: store, hub: hub, load: load, deriveKey: deriveKey, rulesBaseURL: rulesBaseURL}
}
// dataPlaneHealthy reports the node's last sing-box health (default true when
// unknown: just registered / load expired / read error — agent liveness gates).
func (a *NodeAPI) dataPlaneHealthy(ctx context.Context, nodeUUID string) bool {
if a.load == nil {
return true
}
l, ok, err := a.load.Get(ctx, nodeUUID)
if err != nil || !ok || l == nil {
return true
}
return l.DataPlaneHealthy
}
// ─── GET /v1/nodes ───────────────────────────────────────────────────────────
type nodeResponse struct {
ID string `json:"id"` // UUID (used as path param for connect)
Region string `json:"region"` // HK / JP / SG / US
NameZH string `json:"name_zh"`
NameEN string `json:"name_en"`
Tier string `json:"tier"` // "free" | "pro"
Status string `json:"status"` // "up" only when the agent is also online; "down" otherwise
// host/port 暴露节点入口,供客户端实测真实延迟(per-client ping 服务端无法代知)。
Host string `json:"host"`
Port int `json:"port"`
}
// effectiveNodeStatus downgrades a DB-'up' node to "down" when its agent is
// offline OR its data plane (sing-box) is unhealthy. The DB status column reflects
// provisioning/scheduler state, NOT live health — without this, a node whose agent
// has been offline for days, or whose sing-box has crashed while the agent stays
// connected, still shows healthy and lets clients "connect" to a node that can't
// serve them.
func effectiveNodeStatus(dbStatus string, agentOnline, dataPlaneHealthy bool) string {
if dbStatus == "up" && (!agentOnline || !dataPlaneHealthy) {
return "down"
}
return dbStatus
}
// ListNodes handles GET /v1/nodes.
func (a *NodeAPI) ListNodes(w http.ResponseWriter, r *http.Request) {
nodeRows, err := a.store.ListUp(r.Context())
if err != nil {
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
resp := make([]nodeResponse, 0, len(nodeRows))
for _, n := range nodeRows {
host, portStr := splitHostPort(n.Endpoint)
if host == "" {
host = n.Endpoint
}
port, _ := strconv.Atoi(portStr)
resp = append(resp, nodeResponse{
ID: n.UUID,
Region: n.Region,
NameZH: n.NameZH,
NameEN: n.NameEN,
Tier: n.Tier,
Status: effectiveNodeStatus(n.Status,
a.hub == nil || a.hub.IsOnline(n.UUID),
a.dataPlaneHealthy(r.Context(), n.UUID)),
Host: host,
Port: port,
})
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
_ = json.NewEncoder(w).Encode(map[string]any{"nodes": resp})
}
// ─── POST /v1/nodes/{id}/connect ─────────────────────────────────────────────
type connectRequest struct {
DeviceID string `json:"device_id"`
}
// ConnectNode handles POST /v1/nodes/{id}/connect.
func (a *NodeAPI) ConnectNode(w http.ResponseWriter, r *http.Request) {
uid, ok := auth.UserIDFromContext(r.Context())
if !ok {
apierr.WriteJSON(w, http.StatusUnauthorized, apierr.ErrUnauthorized)
return
}
nodeUUID := chi.URLParam(r, "id")
if nodeUUID == "" {
apierr.WriteJSON(w, http.StatusBadRequest, apierr.ErrBadRequest)
return
}
var req connectRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 8*1024)).Decode(&req); err != nil || strings.TrimSpace(req.DeviceID) == "" {
apierr.WriteJSON(w, http.StatusBadRequest, &apierr.Error{
Code: "BAD_REQUEST",
MessageZH: "缺少 device_id",
MessageEn: "Missing device_id",
})
return
}
// 1. Load user entitlement (dp_uuid + plan).
ent, err := a.store.EntitlementForUser(r.Context(), uid)
if err != nil {
slog.Error("connect: entitlement load failed", "user", uid, "err", err)
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
if ent == nil {
apierr.WriteJSON(w, http.StatusUnauthorized, apierr.ErrUnauthorized)
return
}
// 1.5 GB 综合配额卡控(todo #5 Phase 2):按账户当日综合流量卡,超 plan.daily_mb 即拒。
// 对免费(与分钟门双卡)与付费(高上限防滥用)统一生效;daily_mb NULL = 不限。
if ent.DailyMB.Valid {
usedBytes, qerr := a.store.AccountDayBytes(r.Context(), uid, time.Now().UTC())
if qerr != nil {
slog.Error("connect: account day bytes failed", "user", uid, "err", qerr)
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
if usedBytes >= ent.DailyMB.Int64*(1<<20) {
apierr.WriteJSON(w, http.StatusForbidden, apierr.ErrQuotaExhausted)
return
}
}
// 2. Determine TTL from plan.
var ttl time.Duration
if ent.AdGate {
// Free plan: flat 10-minute session for MVP (full ad-gate in a later pass).
dm := int64(10)
if ent.DailyMinutes.Valid {
dm = ent.DailyMinutes.Int64
}
if dm <= 0 {
apierr.WriteJSON(w, http.StatusForbidden, apierr.ErrQuotaExhausted)
return
}
ttl = time.Duration(dm) * freeMinuteTTL
} else {
ttl = paidCredentialTTL
}
expiresAt := time.Now().UTC().Add(ttl)
// 2.5 Per-device dp_uuid (todo #5 Phase 2): each device gets its own data-plane
// credential so the node reports per-device traffic counters. Falls back to the
// account-level dp_uuid when the device isn't registered yet (legacy clients).
dpUUID := ent.DpUUID
if devDp, _, derr := a.store.EnsureDeviceDpUUID(r.Context(), uid, req.DeviceID); derr == nil && devDp != "" {
dpUUID = devDp
} else if derr != nil {
slog.Info("connect: per-device dp_uuid unavailable, using account credential",
"user", uid, "device", req.DeviceID, "err", derr.Error())
}
// 3. Resolve node.
node, err := a.store.NodeByUUID(r.Context(), nodeUUID)
if err != nil {
slog.Error("connect: node load failed", "node", nodeUUID, "err", err)
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
if node == nil || effectiveNodeStatus(node.Status,
a.hub == nil || a.hub.IsOnline(nodeUUID),
a.dataPlaneHealthy(r.Context(), nodeUUID)) != "up" {
apierr.WriteJSON(w, http.StatusNotFound, apierr.ErrNotFound)
return
}
// 4. Build the agentv1.Credential.
cred := &agentv1.Credential{
DpUUID: dpUUID,
Protocol: agentv1.ProtocolBoth,
Flow: "xtls-rprx-vision",
ExpiresAtUnix: expiresAt.Unix(),
}
// 5. Refuse early if the node's agent is offline. A credential we can't push
// now leaves the client "connected" to a node that rejects its UUID — the
// exact failure that hid a 6-day agent outage. Persist it so the agent
// applies it on resync, then signal unavailable instead of faking success.
if !a.hub.IsOnline(node.UUID) {
if persistErr := a.store.PersistCredential(r.Context(), node.ID, cred, expiresAt); persistErr != nil {
slog.Error("connect: persist credential (agent offline) failed", "node", nodeUUID, "user", uid, "err", persistErr)
}
slog.Warn("connect: node agent offline, refusing connect", "node", nodeUUID, "user", uid)
apierr.WriteJSON(w, http.StatusServiceUnavailable, apierr.ErrNodeUnavailable)
return
}
// 6. Push to the node agent via Hub. The agent is online, so a push failure is
// a real error — surface it, don't pretend the tunnel will work.
if pushErr := a.hub.Push(r.Context(), node.UUID, &agentv1.Command{
Type: agentv1.CommandTypeUpsert,
Upsert: &agentv1.UpsertPayload{Credential: cred},
}); pushErr != nil {
slog.Error("connect: push credential to agent failed", "node", nodeUUID, "user", uid, "err", pushErr)
apierr.WriteJSON(w, http.StatusServiceUnavailable, apierr.ErrNodeUnavailable)
return
}
// 7. Persist credential for agent resync (best-effort; the push above already
// delivered it to the live agent).
if persistErr := a.store.PersistCredential(r.Context(), node.ID, cred, expiresAt); persistErr != nil {
slog.Error("connect: persist credential failed", "node", nodeUUID, "user", uid, "err", persistErr)
}
// 7. Render and return the full sing-box CLIENT config JSON.
// split_cn=1/true → 国内 IP/域名直连(#5);客户端按 smartRoute 偏好传。
splitCN := r.URL.Query().Get("split_cn") == "1" || r.URL.Query().Get("split_cn") == "true"
cfgJSON, renderErr := BuildClientConfig(node, dpUUID, a.deriveKey,
ClientConfigOpts{SplitCN: splitCN, RulesBaseURL: a.rulesBaseURL})
if renderErr != nil {
slog.Error("connect: build client config failed", "node", nodeUUID, "err", renderErr)
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
// 可观测:国内分流是否真正生效(split_active=两个条件都满足才渲染 rule_set)。
slog.Info("client config rendered", "node", nodeUUID, "split_cn", splitCN,
"rules_base_set", a.rulesBaseURL != "",
"split_active", splitCN && a.rulesBaseURL != "", "bytes", len(cfgJSON))
w.Header().Set("Content-Type", "application/json; charset=utf-8")
_, _ = w.Write(cfgJSON)
}
// ─── POST /v1/nodes/{id}/disconnect ──────────────────────────────────────────
// DisconnectNode handles POST /v1/nodes/{id}/disconnect.
func (a *NodeAPI) DisconnectNode(w http.ResponseWriter, r *http.Request) {
uid, ok := auth.UserIDFromContext(r.Context())
if !ok {
apierr.WriteJSON(w, http.StatusUnauthorized, apierr.ErrUnauthorized)
return
}
nodeUUID := chi.URLParam(r, "id")
if nodeUUID == "" {
apierr.WriteJSON(w, http.StatusBadRequest, apierr.ErrBadRequest)
return
}
// Load dp_uuid.
ent, err := a.store.EntitlementForUser(r.Context(), uid)
if err != nil {
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
if ent == nil {
w.WriteHeader(http.StatusNoContent) // nothing to revoke
return
}
node, err := a.store.NodeByUUID(r.Context(), nodeUUID)
if err != nil {
apierr.WriteJSON(w, http.StatusInternalServerError, apierr.ErrInternal)
return
}
if node == nil {
w.WriteHeader(http.StatusNoContent)
return
}
// Push revoke command.
_ = a.hub.Push(r.Context(), node.UUID, &agentv1.Command{
Type: agentv1.CommandTypeRevoke,
Revoke: &agentv1.RevokePayload{DpUUID: ent.DpUUID},
})
// Delete persisted credential.
_ = a.store.DeleteCredential(r.Context(), node.ID, ent.DpUUID)
w.WriteHeader(http.StatusNoContent)
}