feat: 数据面故障韧性 — 节点健康上报(控制面) + 客户端连通看门狗

盲区:节点 status 只看 agent 的 gRPC 在线,sing-box 数据面坏了(崩/配置坏/数据口不通)
而 agent 仍在线时,节点仍显示 up、connect 放行、客户端「已连接」但流量全失败,且不自愈。

控制面:
- agent 每心跳探 sing-box clash_api /version(2s 超时,连续 2 次失败才报不健康,防抖),
  经新增 HeartbeatRequest.data_plane_healthy 上报(手写 agentv1 契约 + proto 同步;JSON codec)。
- NodeLoad 加 DataPlaneHealthy(随 load 写 Redis;字段缺失默认 healthy 防滚动期误杀)。
- effectiveNodeStatus 扩为 (dbStatus, agentOnline, dataPlaneHealthy);ListNodes 与 ConnectNode
  统一改用它 → 数据面坏的节点列表置灰 + connect 返回 404 拦截。

客户端(connection_provider 连通看门狗):
- 连上后每 15s 经隧道 HTTP 探海外 generate_204(可注入),连续 3 次失败判当前节点不可用。
- 智能选择 → 自动切到其他最优可用节点重连;手动选定 → 断开并提示「节点异常」(尊重用户选择)。
- 新增 nodeUnhealthySwitched/nodeUnhealthyError 文案。

测试:agent 健康探测防抖、effectiveNodeStatus 真值表、dataPlaneHealthy 助手、看门狗
智能切/手动断/健康不触发;go test ./... 与 flutter test 全绿。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-29 13:19:22 +08:00
parent 6249d5b2ea
commit 2080062d8c
15 changed files with 448 additions and 29 deletions
+3
View File
@@ -56,6 +56,9 @@ abstract class AppText {
String nodeSwitchTitle(String name);
String get nodeSwitchBody;
String get nodeSwitchConfirm;
// 连通看门狗:当前节点数据面不可用时的提示
String get nodeUnhealthySwitched; // 智能选择已自动切换
String get nodeUnhealthyError; // 手动节点:断开并提示
String get smartSub;
String get recommended;
+4
View File
@@ -70,6 +70,10 @@ class StringsEn extends AppText {
@override
String get nodeSwitchConfirm => 'Switch';
@override
String get nodeUnhealthySwitched => 'Current node unreachable — switched automatically';
@override
String get nodeUnhealthyError => 'Current node is unreachable — reconnect or switch node';
@override
String get smartSub => 'Picks the best node for your network';
@override
String get recommended => 'Recommended';
+4
View File
@@ -69,6 +69,10 @@ class StringsZh extends AppText {
@override
String get nodeSwitchConfirm => '切换';
@override
String get nodeUnhealthySwitched => '当前节点异常,已自动切换';
@override
String get nodeUnhealthyError => '当前节点异常,请重连或更换节点';
@override
String get smartSub => '根据当前网络环境,自动选择最优节点';
@override
String get recommended => '推荐';
+113 -2
View File
@@ -6,12 +6,14 @@
// - 严禁乐观翻转:VpnPhase.on 必须由 bridge.statusStream 确认后才置。
// - 状态来源:bridge.statusStream(来自内核回调),非 Timer 模拟。
import 'dart:async';
import 'dart:io';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import '../bridge/vpn_bridge.dart';
import '../bridge/vpn_bridge_provider.dart';
import '../l10n/app_text.dart';
import '../models/node.dart';
import '../services/api_config.dart';
import '../services/connect_api.dart';
import '../services/device_identity.dart';
@@ -53,19 +55,59 @@ class ConnectionState {
int get hashCode => Object.hash(phase, elapsed, error);
}
// ── 连通看门狗 ─────────────────────────────────────────────────────
//
// 已连接(本地 TUN 已起)≠ 远端节点数据面可用:节点 sing-box 崩了/数据口不通而
// agent 仍在线时,客户端会一直显示「已连接」但流量全失败。看门狗周期性经隧道做
// 连通性探测,连续失败即判当前节点不可用 → 智能选择自动切节点 / 手动选定的只断开提示。
/// 连通探测接缝:true=经隧道能到外网。默认走海外 204 端点;测试可注入假实现。
typedef HealthProber = Future<bool> Function();
const _kWatchdogInterval = Duration(seconds: 15);
const _kHealthFailLimit = 3; // 连续 N 次失败(≈45s)才判定不可用,防抖
/// 默认探测:GET 海外 generate_204(全局 TUN 下走隧道 → 真实检验代理出网)。
/// 任一端点 2xx/204 即视为通;多端点容错,避免单点误判。
Future<bool> defaultHealthProbe() async {
const urls = [
'https://www.gstatic.com/generate_204',
'https://cp.cloudflare.com/generate_204',
];
for (final u in urls) {
final client = HttpClient()..connectionTimeout = const Duration(seconds: 4);
try {
final req = await client.getUrl(Uri.parse(u)).timeout(const Duration(seconds: 5));
final resp = await req.close().timeout(const Duration(seconds: 5));
await resp.drain<void>();
if (resp.statusCode >= 200 && resp.statusCode < 300) return true;
} catch (_) {
// 该端点不通,试下一个
} finally {
client.close(force: true);
}
}
return false;
}
// ── 状态机 ───────────────────────────────────────────────────────
class ConnectionController extends StateNotifier<ConnectionState> {
ConnectionController(this._ref, this._bridge)
: super(const ConnectionState(phase: VpnPhase.off)) {
ConnectionController(this._ref, this._bridge, {HealthProber? prober})
: _prober = prober ?? defaultHealthProbe,
super(const ConnectionState(phase: VpnPhase.off)) {
// 订阅桥状态流:状态由内核事件驱动,严禁 UI 乐观翻转。
_statusSub = _bridge.statusStream.listen(_onKernelStatus);
}
final Ref _ref;
final VpnBridge _bridge;
final HealthProber _prober;
StreamSubscription<VpnStatus>? _statusSub;
Timer? _elapsed;
Timer? _watchdog;
int _healthFails = 0;
bool _probing = false;
ConnectApi? _api;
// ── 公有 API ───────────────────────────────────────────────────
@@ -167,16 +209,84 @@ class ConnectionController extends StateNotifier<ConnectionState> {
case VpnStatus.on:
state = state.copyWith(phase: VpnPhase.on);
_startElapsed();
_startWatchdog();
case VpnStatus.connecting:
state = state.copyWith(phase: VpnPhase.connecting);
_stopElapsed();
_stopWatchdog();
case VpnStatus.off:
case VpnStatus.error:
state = const ConnectionState(phase: VpnPhase.off);
_stopElapsed();
_stopWatchdog();
}
}
// ── 连通看门狗 ───────────────────────────────────────────────
void _startWatchdog() {
_watchdog?.cancel();
_healthFails = 0;
_watchdog = Timer.periodic(_kWatchdogInterval, (_) => _checkHealth());
}
void _stopWatchdog() {
_watchdog?.cancel();
_watchdog = null;
_healthFails = 0;
_probing = false;
}
Future<void> _checkHealth() async {
if (_probing || !mounted || state.phase != VpnPhase.on) return;
_probing = true;
final ok = await _prober();
_probing = false;
if (!mounted || state.phase != VpnPhase.on) return;
if (ok) {
_healthFails = 0;
return;
}
_healthFails++;
if (_healthFails >= _kHealthFailLimit) {
_stopWatchdog();
await _onNodeUnhealthy();
}
}
/// 当前节点连续探测失败:智能选择 → 切到其他最优可用节点重连;手动选定 → 断开并提示。
Future<void> _onNodeUnhealthy() async {
final t = _ref.read(appTextProvider);
final smart = _ref.read(selectedNodeCodeProvider) == kSmartNodeCode;
final altCode = smart ? _pickAlternativeCode(_ref.read(effectiveNodeProvider).code) : null;
if (smart && altCode != null) {
_ref.read(selectedNodeCodeProvider.notifier).state = altCode;
await _disconnect();
await _connect();
// 重连进行中给出「已自动切换」提示(connecting 态显示,连上后随 copyWith 清除)。
if (mounted && state.phase != VpnPhase.off) {
state = ConnectionState(phase: state.phase, error: t.nodeUnhealthySwitched);
}
return;
}
// 手动选定节点(或智能模式无其他可用节点):断开并提示,尊重用户选择、不自动换。
await _disconnect();
if (mounted) state = ConnectionState(phase: VpnPhase.off, error: t.nodeUnhealthyError);
}
/// 选延迟最优、可用(status up)、非当前节点的 code;无则 null。
String? _pickAlternativeCode(String excludeCode) {
final nodes = (_ref.read(nodesProvider).valueOrNull ?? const [])
.where((n) => !n.isDown && n.uuid.isNotEmpty && n.code != excludeCode)
.toList();
if (nodes.isEmpty) return null;
nodes.sort((a, b) {
final pa = a.ping > 0 ? a.ping : 1 << 30;
final pb = b.ping > 0 ? b.ping : 1 << 30;
return pa.compareTo(pb);
});
return nodes.first.code;
}
void _startElapsed() {
_elapsed?.cancel();
_elapsed = Timer.periodic(const Duration(seconds: 1), (_) {
@@ -195,6 +305,7 @@ class ConnectionController extends StateNotifier<ConnectionState> {
void dispose() {
_statusSub?.cancel();
_stopElapsed();
_stopWatchdog();
_api?.dispose();
super.dispose();
}
@@ -0,0 +1,132 @@
// connection_watchdog_test.dart — 连通看门狗:已连接后探测连续失败时,
// 智能选择自动切到其他节点;手动选定的节点只断开并提示。
import 'dart:async';
import 'package:flutter/widgets.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:pangolin_vpn/bridge/vpn_bridge.dart';
import 'package:pangolin_vpn/bridge/vpn_bridge_provider.dart';
import 'package:pangolin_vpn/l10n/strings_zh.dart';
import 'package:pangolin_vpn/models/node.dart';
import 'package:pangolin_vpn/services/connect_api.dart';
import 'package:pangolin_vpn/state/connection_provider.dart';
import 'package:pangolin_vpn/state/nodes_provider.dart';
// 可控假桥:能手动推 VpnStatus,无内部计时器(避免 pending timer)。
class _FakeBridge implements VpnBridge {
final _status = StreamController<VpnStatus>.broadcast();
final _stats = StreamController<VpnStatsEvent>.broadcast();
void emit(VpnStatus s) => _status.add(s);
@override
Stream<VpnStatus> get statusStream => _status.stream;
@override
Stream<VpnStatsEvent> get statsStream => _stats.stream;
@override
Future<void> start(String configJson) async {}
@override
Future<void> stop() async {}
@override
Future<VpnStatus> getStatus() async => VpnStatus.off;
@override
Future<void> selectOutbound(String tag) async {}
@override
Future<String> getActiveOutbound() async => 'auto';
@override
Future<void> setKillSwitch({required bool on}) async {}
@override
void dispose() {
_status.close();
_stats.close();
}
}
// 重连时 fetchConfig 立即抛错,绕开真实网络(不留超时计时器)。
class _FakeConnectApi extends ConnectApi {
_FakeConnectApi() : super(baseUrl: 'http://test.local', authToken: '');
@override
Future<String> fetchConfig({required String nodeId, required String deviceId, bool splitCN = false}) async {
throw const ConnectApiException(statusCode: 0, messageZh: 'x', messageEn: 'x');
}
@override
void dispose() {}
}
class _StubNodes extends NodesNotifier {
@override
Future<List<Node>> build() async => const [
Node(code: 'HK', nameZh: '香港', nameEn: 'Hong Kong', ping: 20, uuid: 'hk-uuid', host: 'hk', port: 443),
Node(code: 'JP', nameZh: '东京', nameEn: 'Tokyo', ping: 35, uuid: 'jp-uuid', host: 'jp', port: 443),
];
@override
Future<void> refresh() async {}
}
void main() {
const t = StringsZh();
ProviderContainer makeContainer(_FakeBridge bridge, {required bool healthy}) =>
ProviderContainer(overrides: [
vpnBridgeProvider.overrideWithValue(bridge),
nodesProvider.overrideWith(_StubNodes.new),
connectApiFactoryProvider.overrideWithValue((_) => _FakeConnectApi()),
connectionProvider.overrideWith((ref) => ConnectionController(
ref, ref.watch(vpnBridgeProvider),
prober: () async => healthy,
)),
]);
// 驱动到 on,再走 3 个看门狗周期(每个 15s)。
Future<void> driveOnAndProbe(WidgetTester tester, ProviderContainer c, _FakeBridge bridge) async {
c.read(nodesProvider); // 触发(异步)节点加载
await tester.pump(); // 等 valueOrNull 就绪(智能切换要读节点列表)
c.read(connectionProvider); // 建 controller(订阅状态流)
bridge.emit(VpnStatus.on);
await tester.pump();
expect(c.read(connectionProvider).phase, VpnPhase.on);
for (var i = 0; i < 3; i++) {
await tester.pump(const Duration(seconds: 15));
await tester.pump();
}
await tester.pump();
}
testWidgets('智能模式:探测连续失败 → 自动切到其他最优节点', (tester) async {
final bridge = _FakeBridge();
final c = makeContainer(bridge, healthy: false);
addTearDown(bridge.dispose);
addTearDown(c.dispose);
await tester.pumpWidget(UncontrolledProviderScope(container: c, child: const SizedBox()));
// 默认即智能选择(selectedNodeCode = AUTO);有效节点取最小延迟 HK,故应切到 JP。
await driveOnAndProbe(tester, c, bridge);
expect(c.read(selectedNodeCodeProvider), 'JP', reason: '智能模式应自动切到其他最优节点');
});
testWidgets('手动选定节点:探测连续失败 → 断开并提示,不自动换', (tester) async {
final bridge = _FakeBridge();
final c = makeContainer(bridge, healthy: false);
addTearDown(bridge.dispose);
addTearDown(c.dispose);
await tester.pumpWidget(UncontrolledProviderScope(container: c, child: const SizedBox()));
c.read(selectedNodeCodeProvider.notifier).state = 'HK'; // 手动选定 HK
await driveOnAndProbe(tester, c, bridge);
expect(c.read(selectedNodeCodeProvider), 'HK', reason: '手动模式不应自动换节点');
final st = c.read(connectionProvider);
expect(st.phase, VpnPhase.off, reason: '应断开');
expect(st.error, t.nodeUnhealthyError, reason: '应给出节点异常提示');
});
testWidgets('探测正常:不触发任何动作', (tester) async {
final bridge = _FakeBridge();
final c = makeContainer(bridge, healthy: true);
addTearDown(bridge.dispose);
addTearDown(c.dispose);
await tester.pumpWidget(UncontrolledProviderScope(container: c, child: const SizedBox()));
c.read(selectedNodeCodeProvider.notifier).state = 'HK';
await driveOnAndProbe(tester, c, bridge);
expect(c.read(connectionProvider).phase, VpnPhase.on, reason: '健康时应保持连接');
expect(c.read(selectedNodeCodeProvider), 'HK');
bridge.emit(VpnStatus.off); // 收尾:停看门狗/计时器,避免 pending timer
await tester.pump();
});
}