wangjia
2080062d8c
盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通) 而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。 控制面: - agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖), 经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。 - NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。 - effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode 统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。 客户端(connection_provider 连通看门狗): - 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。 - 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。 - 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。 测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗 智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
98 lines
2.9 KiB
Go
98 lines
2.9 KiB
Go
package nodes
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
const (
|
|
// nodeLoadKeyPrefix is the Redis HASH key prefix for per-node runtime load metrics.
|
|
// Full key: node:load:{node_uuid} (doc/03 §4)
|
|
nodeLoadKeyPrefix = "node:load:"
|
|
|
|
// nodeLoadTTL is the TTL on each node:load hash. A node that stops sending
|
|
// heartbeats has its entry automatically evicted after 90 seconds.
|
|
nodeLoadTTL = 90 * time.Second
|
|
)
|
|
|
|
// NodeLoad holds the runtime metrics reported in each heartbeat.
|
|
type NodeLoad struct {
|
|
OnlineCount int64
|
|
BandwidthUpBps int64
|
|
BandwidthDownBps int64
|
|
CPUPercent float64
|
|
// DataPlaneHealthy is the agent's last sing-box health probe result. The
|
|
// control plane downgrades a node to "down" when this is false (even if the
|
|
// agent's gRPC stream is still up). Absent in older hashes → read as true.
|
|
DataPlaneHealthy bool
|
|
}
|
|
|
|
// LoadCache reads and writes node:load Redis hashes.
|
|
// It accepts redis.Cmdable so it can be used with a *redis.Client or pipeline.
|
|
type LoadCache struct {
|
|
rdb redis.Cmdable
|
|
}
|
|
|
|
// NewLoadCache creates a LoadCache backed by rdb.
|
|
func NewLoadCache(rdb redis.Cmdable) *LoadCache {
|
|
return &LoadCache{rdb: rdb}
|
|
}
|
|
|
|
// Set writes nodeUUID's current load metrics to Redis and refreshes the TTL.
|
|
// The hash fields match the names in doc/03 §4: online_count, bw_up, bw_down, cpu.
|
|
func (c *LoadCache) Set(ctx context.Context, nodeUUID string, load NodeLoad) error {
|
|
key := nodeLoadKeyPrefix + nodeUUID
|
|
dpHealthy := 0
|
|
if load.DataPlaneHealthy {
|
|
dpHealthy = 1
|
|
}
|
|
pipe := c.rdb.Pipeline()
|
|
pipe.HSet(ctx, key,
|
|
"online_count", load.OnlineCount,
|
|
"bw_up", load.BandwidthUpBps,
|
|
"bw_down", load.BandwidthDownBps,
|
|
"cpu", load.CPUPercent,
|
|
"dp_healthy", dpHealthy,
|
|
)
|
|
pipe.Expire(ctx, key, nodeLoadTTL)
|
|
if _, err := pipe.Exec(ctx); err != nil {
|
|
return fmt.Errorf("nodes.LoadCache.Set: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Get reads nodeUUID's last-known load. Returns (nil, false, nil) if the key has
|
|
// expired or was never set. Fields missing from the hash default to zero.
|
|
func (c *LoadCache) Get(ctx context.Context, nodeUUID string) (*NodeLoad, bool, error) {
|
|
key := nodeLoadKeyPrefix + nodeUUID
|
|
vals, err := c.rdb.HGetAll(ctx, key).Result()
|
|
if err != nil {
|
|
return nil, false, fmt.Errorf("nodes.LoadCache.Get: %w", err)
|
|
}
|
|
if len(vals) == 0 {
|
|
return nil, false, nil
|
|
}
|
|
// Default healthy: hashes written before dp_healthy existed (or by an older
|
|
// control plane during rollout) must not be read as "unhealthy" → false-down.
|
|
l := NodeLoad{DataPlaneHealthy: true}
|
|
if v, ok := vals["dp_healthy"]; ok {
|
|
l.DataPlaneHealthy = v == "1"
|
|
}
|
|
if v, ok := vals["online_count"]; ok {
|
|
_, _ = fmt.Sscan(v, &l.OnlineCount)
|
|
}
|
|
if v, ok := vals["bw_up"]; ok {
|
|
_, _ = fmt.Sscan(v, &l.BandwidthUpBps)
|
|
}
|
|
if v, ok := vals["bw_down"]; ok {
|
|
_, _ = fmt.Sscan(v, &l.BandwidthDownBps)
|
|
}
|
|
if v, ok := vals["cpu"]; ok {
|
|
_, _ = fmt.Sscan(v, &l.CPUPercent)
|
|
}
|
|
return &l, true, nil
|
|
}
|