pangolin/server/internal/scheduler/detect/engine_test.go

package detect_test

import (
	"context"
	"encoding/json"
	"testing"
	"time"

	"github.com/alicebob/miniredis/v2"
	"github.com/redis/go-redis/v9"

	"github.com/wangjia/pangolin/server/internal/alert"
	"github.com/wangjia/pangolin/server/internal/scheduler/detect"
	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
)

// ─────────────────────────────────────────────────────────────────────────────
// Test helpers
// ─────────────────────────────────────────────────────────────────────────────

// mockSnapshotter implements detect.ProbeSnapshotter for unit tests.
type mockSnapshotter struct {
	data map[string]map[string]probe.ProbeSnapshot
}

func (m *mockSnapshotter) SnapshotsByNode(_ context.Context, nodeID string) (map[string]probe.ProbeSnapshot, error) {
	if snaps, ok := m.data[nodeID]; ok {
		return snaps, nil
	}
	return nil, nil
}

// recordingNotifier records Notify calls for assertion.
type recordingNotifier struct {
	events []alert.Event
}

func (r *recordingNotifier) Notify(_ context.Context, e alert.Event) error {
	r.events = append(r.events, e)
	return nil
}

// hasFaultEvent returns true if any recorded event has type EventTypeFault.
func (r *recordingNotifier) hasFaultEvent() bool {
	for _, e := range r.events {
		if e.Type == alert.EventTypeFault {
			return true
		}
	}
	return false
}

// newTestRedis creates an in-process Redis (miniredis) and returns a connected
// client plus a cleanup function.  Tests must call cleanup() at the end.
func newTestRedis(t *testing.T) (*redis.Client, *miniredis.Miniredis) {
	t.Helper()
	mr := miniredis.RunT(t)
	rdb := redis.NewClient(&redis.Options{Addr: mr.Addr()})
	return rdb, mr
}

// newEngine wires up an Engine with the given mock components.
func newEngine(
	t *testing.T,
	rdb *redis.Client,
	lc *detect.MockLifecycle,
	snaps *mockSnapshotter,
	notifier detect.Notifier,
	cfg *detect.DetectConfig,
) *detect.Engine {
	t.Helper()
	streaks := detect.NewStreakStore(rdb)
	return detect.NewEngine(snaps, lc, streaks, rdb, notifier, cfg)
}

// snap builds a probe.ProbeSnapshot for one (node, vantage) pair.
func snap(country, isp string, l1OK bool, l3OK *bool) probe.ProbeSnapshot {
	r := probe.NodeReport{L1: probe.L1Result{OK: l1OK}}
	if l3OK != nil {
		r.L3 = &probe.L3Result{OK: *l3OK}
	}
	return probe.ProbeSnapshot{
		Vantage: probe.VantagePoint{Country: country, ISP: isp},
		Report:  r,
	}
}

func boolPtr(b bool) *bool { return &b }

// cnFail returns a domestic (CN) probe snapshot where L3 is failing.
func cnFail(isp string) probe.ProbeSnapshot {
	return snap("CN", isp, true, boolPtr(false))
}

// cnOK returns a domestic (CN) probe snapshot where L3 passes.
func cnOK(isp string) probe.ProbeSnapshot {
	return snap("CN", isp, true, boolPtr(true))
}

// overseasOK returns an overseas probe snapshot where L1 passes.
func overseasOK() probe.ProbeSnapshot {
	return snap("SG", "AWS", true, nil)
}

// overseasFail returns an overseas probe snapshot where L1 fails.
func overseasFail() probe.ProbeSnapshot {
	return snap("SG", "AWS", false, nil)
}

// snapsForNode builds the map that mockSnapshotter.data["nodeID"] expects.
func snapsForNode(snaps ...probe.ProbeSnapshot) map[string]probe.ProbeSnapshot {
	m := make(map[string]probe.ProbeSnapshot, len(snaps))
	for i, s := range snaps {
		key := s.Vantage.Country + ":" + s.Vantage.ISP
		_ = i
		m[key] = s
	}
	return m
}

// ─────────────────────────────────────────────────────────────────────────────
// Core rule tests (table-driven)
// ─────────────────────────────────────────────────────────────────────────────

const nodeID = "node-001"

// TestRules covers the five classification rules and their boundary conditions
// through table-driven subtests.
func TestRules(t *testing.T) {
	cfg := detect.DefaultConfig()
	ctx := context.Background()

	type tick struct {
		// snaps to present on this tick (keyed by node ID)
		snaps map[string]probe.ProbeSnapshot
	}

	tests := []struct {
		name string
		// initial node status before any ticks
		initialStatus detect.NodeStatus
		// ticks is the sequence of probe states to feed to the engine
		ticks []tick
		// wantStatus is the expected node status after all ticks
		wantStatus detect.NodeStatus
		// wantEvents are (From, To) pairs expected in the lifecycle event log
		wantEvents [][2]detect.NodeStatus
		// wantFaultNotified indicates whether NotifyFault should have been called
		wantFaultNotified bool
	}{
		{
			// Exactly 2 out of 3 ISPs fail → meets the ≥2/3 threshold.
			// After 2 consecutive failing cycles (10 min) the node becomes suspect.
			name:          "exactly_2_of_3_ISPs_fail_after_2_cycles",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusBlockedSuspect,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusUp, detect.StatusBlockedSuspect},
			},
		},
		{
			// 1 out of 3 ISPs fail → below the 2/3 threshold → no transition.
			name:          "1_of_3_ISPs_fail_no_suspect",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnOK("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnOK("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusUp,
			wantEvents: nil,
		},
		{
			// 9 min (= 1 complete 5-min cycle): fail_streak reaches 1, not yet 2.
			// No transition expected after exactly one failing tick.
			name:          "9min_one_cycle_no_transition",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusUp,
			wantEvents: nil,
		},
		{
			// 10 min (= 2 complete 5-min cycles): fail_streak reaches 2 ≥ SuspectStreakMin.
			// Transition to blocked_suspect expected.
			name:          "10min_two_cycles_triggers_suspect",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusBlockedSuspect,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusUp, detect.StatusBlockedSuspect},
			},
		},
		{
			// Traffic warning: drop ≥ 80 % + baseline ≥ 20 → effectiveSuspectStreakMin = 1.
			// A single failing tick is enough to enter blocked_suspect.
			name:          "traffic_warning_relaxes_threshold_to_1_cycle",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				// The MockLifecycle will have loads set before running this test;
				// we pass snaps with 2/3 ISPs failing + overseas OK.
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusBlockedSuspect,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusUp, detect.StatusBlockedSuspect},
			},
		},
		{
			// Confirmed: node spends 6 consecutive cycles in blocked_suspect
			// with domestic still failing → transitions to blocked_confirmed → down.
			name:          "suspect_6_cycles_triggers_confirmed",
			initialStatus: detect.StatusBlockedSuspect,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusDown,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusBlockedSuspect, detect.StatusBlockedConfirmed},
				{detect.StatusBlockedConfirmed, detect.StatusDown},
			},
		},
		{
			// Fault: domestic + overseas both fail → fault rule fires, no state
			// transition, NotifyFault is called exactly once.
			name:          "domestic_and_overseas_fail_triggers_fault_not_block",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), overseasFail())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), overseasFail())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), overseasFail())},
			},
			wantStatus:        detect.StatusUp,  // no transition
			wantEvents:        nil,               // no lifecycle transitions
			wantFaultNotified: true,
		},
		{
			// Recover: 2 consecutive cycles with domestic passing while in
			// blocked_suspect → transitions back to up.
			name:          "recover_2_cycles_back_to_up",
			initialStatus: detect.StatusBlockedSuspect,
			ticks: []tick{
				{snaps: snapsForNode(cnOK("ChinaTelecom"), cnOK("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnOK("ChinaTelecom"), cnOK("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusUp,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusBlockedSuspect, detect.StatusUp},
			},
		},
		{
			// Recovery not yet reached: 1 cycle passing while in blocked_suspect
			// → still in blocked_suspect.
			name:          "recover_1_cycle_not_yet_recovered",
			initialStatus: detect.StatusBlockedSuspect,
			ticks: []tick{
				{snaps: snapsForNode(cnOK("ChinaTelecom"), cnOK("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusBlockedSuspect,
			wantEvents: nil,
		},
		{
			// Third-party ("3rd-") ISP prefix is stripped before grouping:
			// "3rd-ChinaTelecom" and "ChinaTelecom" must count as the same ISP.
			// Two ISPs fail out of three → triggers suspect.
			name:          "third_party_isp_prefix_merged",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				{snaps: snapsForNode(cnFail("3rd-ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("3rd-ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusBlockedSuspect,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusUp, detect.StatusBlockedSuspect},
			},
		},
		{
			// L3 failure overrides L1/L2: even if L1 is OK the ISP is counted
			// as failing when L3 is explicitly false.
			name:          "l3_failure_overrides_l1_l2",
			initialStatus: detect.StatusUp,
			ticks: []tick{
				// L1 passes but L3 fails for two ISPs → ≥2/3 threshold met.
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
				{snaps: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())},
			},
			wantStatus: detect.StatusBlockedSuspect,
			wantEvents: [][2]detect.NodeStatus{
				{detect.StatusUp, detect.StatusBlockedSuspect},
			},
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			rdb, _ := newTestRedis(t)

			lc := detect.NewMockLifecycle([]detect.NodeInfo{
				{ID: nodeID, Status: tc.initialStatus, Weight: 100},
			})

			// For the traffic-warning test, set up load history showing a large drop.
			// Older half: 100 connections.  Newer half: 20 connections.
			// Drop = 80 % ≥ TrafficDropThreshold (80 %).
			// Current baseline = 20 ≥ TrafficBaselineMin (20) → warning applies.
			if tc.name == "traffic_warning_relaxes_threshold_to_1_cycle" {
				now := time.Now().Unix()
				lc.SetLoads(nodeID, []detect.LoadPoint{
					{Timestamp: now - 14*60, Online: 100},
					{Timestamp: now - 10*60, Online: 100},
					{Timestamp: now - 5*60, Online: 20},
					{Timestamp: now - 60, Online: 20},
				})
			}

			notifier := &recordingNotifier{}
			snapper := &mockSnapshotter{data: make(map[string]map[string]probe.ProbeSnapshot)}
			eng := newEngine(t, rdb, lc, snapper, notifier, &cfg)

			for i, tick := range tc.ticks {
				snapper.data[nodeID] = tick.snaps
				if err := eng.Tick(ctx); err != nil {
					t.Fatalf("tick %d: Tick() error: %v", i, err)
				}
			}

			// Verify final node status.
			if got := lc.NodeStatus(nodeID); got != tc.wantStatus {
				t.Errorf("node status = %q, want %q", got, tc.wantStatus)
			}

			// Verify lifecycle events (from/to pairs only; detail checked separately).
			events := lc.Events()
			if len(events) != len(tc.wantEvents) {
				t.Errorf("got %d events, want %d: %v", len(events), len(tc.wantEvents), events)
			} else {
				for i, ev := range events {
					want := tc.wantEvents[i]
					if ev.From != want[0] || ev.To != want[1] {
						t.Errorf("event[%d]: from=%q to=%q, want from=%q to=%q",
							i, ev.From, ev.To, want[0], want[1])
					}
				}
			}

			// Verify fault notification.
			if tc.wantFaultNotified && !notifier.hasFaultEvent() {
				t.Error("expected fault Notify event to be recorded, but it was not")
			}
			if !tc.wantFaultNotified && notifier.hasFaultEvent() {
				t.Errorf("unexpected fault Notify events: %v", notifier.events)
			}
		})
	}
}

// TestEventDetailContainsFromTo verifies that every TransitionStatus call
// includes "from" and "to" keys in its detail map (node_events audit trail).
func TestEventDetailContainsFromTo(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()
	rdb, _ := newTestRedis(t)

	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	snapper := &mockSnapshotter{
		data: map[string]map[string]probe.ProbeSnapshot{
			nodeID: snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK()),
		},
	}
	eng := newEngine(t, rdb, lc, snapper, nil, &cfg)

	// Two ticks to trigger suspect.
	if err := eng.Tick(ctx); err != nil {
		t.Fatal(err)
	}
	if err := eng.Tick(ctx); err != nil {
		t.Fatal(err)
	}

	events := lc.Events()
	if len(events) == 0 {
		t.Fatal("no events recorded")
	}
	for _, ev := range events {
		if _, ok := ev.Detail["from"]; !ok {
			t.Errorf("event %v→%v detail missing 'from'", ev.From, ev.To)
		}
		if _, ok := ev.Detail["to"]; !ok {
			t.Errorf("event %v→%v detail missing 'to'", ev.From, ev.To)
		}
	}
}

// TestOptimisticLockConflict verifies that when TransitionStatus returns 0
// (lock conflict), the current cycle is abandoned without any side-effects,
// and the engine is idempotent — the next tick re-attempts cleanly.
func TestOptimisticLockConflict(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()
	rdb, _ := newTestRedis(t)

	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	// Register a conflict for the up→suspect transition.
	lc.SetConflict(nodeID, detect.StatusUp, detect.StatusBlockedSuspect)

	failSnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: failSnaps}}
	eng := newEngine(t, rdb, lc, snapper, nil, &cfg)

	// Run two ticks — normally enough to trigger suspect.
	if err := eng.Tick(ctx); err != nil {
		t.Fatal(err)
	}
	if err := eng.Tick(ctx); err != nil {
		t.Fatal(err)
	}

	// Conflict registered: status must remain "up", no events logged.
	if got := lc.NodeStatus(nodeID); got != detect.StatusUp {
		t.Errorf("status = %q after conflict; want %q", got, detect.StatusUp)
	}
	if events := lc.Events(); len(events) != 0 {
		t.Errorf("expected 0 events after conflict, got %d: %v", len(events), events)
	}

	// Clear conflict, run one more tick → should transition now (streak already at 1).
	lc.ClearConflict(nodeID, detect.StatusUp, detect.StatusBlockedSuspect)
	if err := eng.Tick(ctx); err != nil {
		t.Fatal(err)
	}
	if got := lc.NodeStatus(nodeID); got != detect.StatusBlockedSuspect {
		t.Errorf("status = %q after conflict cleared; want %q", got, detect.StatusBlockedSuspect)
	}
}

// TestStreakPersistenceAcrossRestart verifies that streak counters survive a
// simulated process restart: a new Engine reading the same Redis should
// continue from the saved streak rather than starting from zero.
func TestStreakPersistenceAcrossRestart(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig() // SuspectStreakMin = 2

	rdb, _ := newTestRedis(t)

	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	failSnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: failSnaps}}

	// First "process instance": run one tick (fail_streak becomes 1).
	eng1 := newEngine(t, rdb, lc, snapper, nil, &cfg)
	if err := eng1.Tick(ctx); err != nil {
		t.Fatal(err)
	}
	if got := lc.NodeStatus(nodeID); got != detect.StatusUp {
		t.Errorf("after tick 1 status = %q; want up", got)
	}

	// "Restart": create a brand-new Engine pointing at the same Redis.
	// The streak (fail_streak=1) must still be present.
	eng2 := newEngine(t, rdb, lc, snapper, nil, &cfg)
	if err := eng2.Tick(ctx); err != nil {
		t.Fatal(err)
	}

	// The second engine picks up fail_streak=1, increments to 2, and fires
	// the suspect transition.
	if got := lc.NodeStatus(nodeID); got != detect.StatusBlockedSuspect {
		t.Errorf("after restart tick status = %q; want blocked_suspect", got)
	}
}

// TestReplaceQueuePopulated verifies that when a node is confirmed, the
// detect:replace:queue Redis list receives a valid JSON entry.
func TestReplaceQueuePopulated(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()

	rdb, _ := newTestRedis(t)
	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusBlockedSuspect, Weight: 10},
	})
	failSnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: failSnaps}}
	eng := newEngine(t, rdb, lc, snapper, nil, &cfg)

	// Run ConfirmedStreakMin (6) ticks.
	for i := 0; i < cfg.ConfirmedStreakMin; i++ {
		if err := eng.Tick(ctx); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
	}

	if got := lc.NodeStatus(nodeID); got != detect.StatusDown {
		t.Errorf("node status = %q after 6 suspect cycles; want down", got)
	}

	// Verify the replace queue has one entry with the correct shape.
	qlen, err := rdb.LLen(ctx, "detect:replace:queue").Result()
	if err != nil {
		t.Fatalf("LLEN: %v", err)
	}
	if qlen != 1 {
		t.Fatalf("replace queue length = %d; want 1", qlen)
	}

	raw, err := rdb.LIndex(ctx, "detect:replace:queue", 0).Result()
	if err != nil {
		t.Fatalf("LINDEX: %v", err)
	}
	var entry struct {
		NodeID          string `json:"nodeId"`
		ReplacementUUID string `json:"replacementUuid"`
	}
	if err := json.Unmarshal([]byte(raw), &entry); err != nil {
		t.Fatalf("unmarshal replace queue entry: %v", err)
	}
	if entry.NodeID != nodeID {
		t.Errorf("replace queue nodeId = %q; want %q", entry.NodeID, nodeID)
	}
	if entry.ReplacementUUID == "" {
		t.Error("replace queue replacementUuid is empty")
	}
}

// TestProbeFreqMarkWritten verifies that the probe:freq:{nodeID} Redis key is
// set when a node transitions to blocked_suspect.
func TestProbeFreqMarkWritten(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()

	rdb, _ := newTestRedis(t)
	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	failSnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: failSnaps}}
	eng := newEngine(t, rdb, lc, snapper, nil, &cfg)

	// Two ticks to trigger suspect.
	for i := 0; i < 2; i++ {
		if err := eng.Tick(ctx); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
	}

	if got := lc.NodeStatus(nodeID); got != detect.StatusBlockedSuspect {
		t.Errorf("status = %q; want blocked_suspect", got)
	}

	freqKey := "probe:freq:" + nodeID
	val, err := rdb.Get(ctx, freqKey).Result()
	if err != nil {
		t.Fatalf("probe freq key not found: %v", err)
	}
	if val != "60" {
		t.Errorf("probe:freq value = %q; want 60", val)
	}
}

// TestFaultNoTransitionNoStreak verifies that the fault rule leaves the node
// status and streaks completely unchanged across multiple ticks, even though
// the domestic probes are failing.
func TestFaultNoTransitionNoStreak(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()

	rdb, _ := newTestRedis(t)
	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	// Both domestic and overseas fail → fault.
	faultSnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), overseasFail())
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: faultSnaps}}
	notifier := &recordingNotifier{}
	eng := newEngine(t, rdb, lc, snapper, notifier, &cfg)

	// Run many ticks — should never transition.
	for i := 0; i < 10; i++ {
		if err := eng.Tick(ctx); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
	}

	if got := lc.NodeStatus(nodeID); got != detect.StatusUp {
		t.Errorf("status = %q; want up (fault must not transition)", got)
	}
	if events := lc.Events(); len(events) != 0 {
		t.Errorf("unexpected events: %v", events)
	}
	if !notifier.hasFaultEvent() {
		t.Error("expected fault Notify event to be recorded at least once")
	}
}

// TestSuspectWeightApplied verifies that when a node enters blocked_suspect,
// its routing weight is reduced to cfg.SuspectWeight (default 10).
func TestSuspectWeightApplied(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()

	rdb, _ := newTestRedis(t)
	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	failSnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"), overseasOK())
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: failSnaps}}
	eng := newEngine(t, rdb, lc, snapper, nil, &cfg)

	for i := 0; i < 2; i++ {
		if err := eng.Tick(ctx); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
	}

	if got := lc.NodeWeight(nodeID); got != cfg.SuspectWeight {
		t.Errorf("weight after suspect = %d; want %d", got, cfg.SuspectWeight)
	}
}

// TestNoOverseasDataNoSuspect verifies that when no overseas probe data is
// available, the engine does not trigger the suspect rule (we cannot confirm
// the node is globally reachable, so we cannot attribute a domestic failure
// to GFW censorship).
func TestNoOverseasDataNoSuspect(t *testing.T) {
	ctx := context.Background()
	cfg := detect.DefaultConfig()

	rdb, _ := newTestRedis(t)
	lc := detect.NewMockLifecycle([]detect.NodeInfo{
		{ID: nodeID, Status: detect.StatusUp, Weight: 100},
	})
	// Only domestic probes, no overseas vantage.
	domesticOnlySnaps := snapsForNode(cnFail("ChinaTelecom"), cnFail("ChinaUnicom"), cnOK("ChinaMobile"))
	snapper := &mockSnapshotter{data: map[string]map[string]probe.ProbeSnapshot{nodeID: domesticOnlySnaps}}
	eng := newEngine(t, rdb, lc, snapper, nil, &cfg)

	for i := 0; i < 5; i++ {
		if err := eng.Tick(ctx); err != nil {
			t.Fatalf("tick %d: %v", i, err)
		}
	}

	if got := lc.NodeStatus(nodeID); got != detect.StatusUp {
		t.Errorf("status = %q; want up (no overseas data → no suspect)", got)
	}
}