package probe import ( "context" "encoding/json" "fmt" "strconv" "strings" "time" "github.com/redis/go-redis/v9" "github.com/wangjia/pangolin/server/internal/alert" ) // Redis TTL constants for the probe subsystem. const ( // snapshotTTL is how long a per-(node,vantage) probe snapshot is retained. // After this window the key expires and 15D must treat it as "no data". snapshotTTL = 30 * time.Minute // heartbeatTTL is how long a probe heartbeat key lives without renewal. // After this window the key expires — meaning "no recent data", NOT "failure". heartbeatTTL = 15 * time.Minute // seenTTL is the lifetime of a replay-prevention key. // Must exceed 2 × timeWindow (2 × 300 s = 600 s); 700 s adds a 100 s buffer. seenTTL = 700 * time.Second ) // Store handles Redis reads and writes for the probe subsystem. // // # Key schema // // probe:{nodeID}:{vantageKey} → ProbeSnapshot JSON, TTL snapshotTTL (30 min) // probe:hb:{probeID} → Unix timestamp string, TTL heartbeatTTL (15 min) // probe:seen:{probeID}:{ts} → "1", TTL seenTTL (700 s), replay prevention // // # Missing-key semantics (IMPORTANT – must not be violated by callers) // // A missing probe:hb:{probeID} key means "no recent heartbeat data". // It must NEVER be interpreted as "probe is down" or folded into a failure // signal. Determination logic (15D) must treat an absent key as unknown, // not as a negative result. // // # Node-ID constraint // // NodeIDs must not be the literal string "hb" or "seen", as those are used as // key-namespace prefixes. In practice node IDs are UUIDs so this is safe. type Store struct { rdb *redis.Client } // NewStore creates a Store backed by the given Redis client. func NewStore(rdb *redis.Client) *Store { return &Store{rdb: rdb} } // -------------------------------------------------------------------------- // Key helpers // -------------------------------------------------------------------------- // snapshotKey returns the Redis key for the latest probe snapshot // for the given (nodeID, vantage) pair. // // Format: probe:{nodeID}:{country}:{region}:{isp} func snapshotKey(nodeID string, v VantagePoint) string { return "probe:" + nodeID + ":" + vantageKey(v) } // vantageKey returns a canonical, colon-safe string for a VantagePoint. // It is used as the trailing segment of a snapshot Redis key. func vantageKey(v VantagePoint) string { return sanitizeKeySegment(v.Country) + ":" + sanitizeKeySegment(v.Region) + ":" + sanitizeKeySegment(v.ISP) } // sanitizeKeySegment replaces characters that would interfere with Redis key // parsing (space, colon) with underscores so they are safe in compound keys. func sanitizeKeySegment(s string) string { s = strings.ReplaceAll(s, " ", "_") s = strings.ReplaceAll(s, ":", "_") return s } // heartbeatKey returns the Redis key for a probe agent heartbeat. func heartbeatKey(probeID string) string { return "probe:hb:" + probeID } // seenKey returns the Redis key used for replay-prevention on a // (probeID, ts) tuple. func seenKey(probeID, ts string) string { return "probe:seen:" + probeID + ":" + ts } // -------------------------------------------------------------------------- // Write path // -------------------------------------------------------------------------- // SaveReports persists one ProbeSnapshot per (nodeID, vantage) pair from the // given report batch and updates the probe heartbeat. // All writes are issued in a single pipeline for efficiency. func (s *Store) SaveReports(ctx context.Context, probeID string, vantage VantagePoint, reports []NodeReport) error { now := time.Now().Unix() pipe := s.rdb.Pipeline() for _, rep := range reports { snap := ProbeSnapshot{ ProbeID: probeID, Vantage: vantage, Report: rep, ReceivedAt: now, } data, err := json.Marshal(snap) if err != nil { return fmt.Errorf("probe: marshal snapshot for node %s: %w", rep.NodeID, err) } pipe.Set(ctx, snapshotKey(rep.NodeID, vantage), data, snapshotTTL) } // Update heartbeat: value is the ingest timestamp so readers can compute // staleness without needing a separate TTL query. pipe.Set(ctx, heartbeatKey(probeID), strconv.FormatInt(now, 10), heartbeatTTL) if _, err := pipe.Exec(ctx); err != nil { return fmt.Errorf("probe: redis pipeline exec: %w", err) } return nil } // CheckAndMarkSeen atomically checks whether the (probeID, ts) pair has been // seen before, and marks it as seen if not. // // Returns (true, nil) – this is a replay; the caller should handle idempotently. // Returns (false, nil) – first time seen; the caller should process normally. func (s *Store) CheckAndMarkSeen(ctx context.Context, probeID, ts string) (bool, error) { key := seenKey(probeID, ts) // SET … NX EX: atomically set iff the key does not exist. set, err := s.rdb.SetNX(ctx, key, "1", seenTTL).Result() if err != nil { return false, fmt.Errorf("probe: seen check: %w", err) } // SetNX returns true if the key was newly created (not a replay). return !set, nil } // -------------------------------------------------------------------------- // Read interfaces consumed by 15D (determination logic) // -------------------------------------------------------------------------- // SnapshotsByNode returns all cached probe snapshots for the given nodeID, // indexed by vantage key string (Country:Region:ISP). // // A missing key — and therefore an empty map — means "no recent probe data // from any vantage point". Callers (15D) must treat this as "unknown", not // as a failure signal. // // Uses SCAN + GET in two passes. For the expected scale (dozens of vantages // per node) this is efficient enough; 15D may add caching on top if needed. func (s *Store) SnapshotsByNode(ctx context.Context, nodeID string) (map[string]ProbeSnapshot, error) { pattern := "probe:" + nodeID + ":*" var keys []string iter := s.rdb.Scan(ctx, 0, pattern, 0).Iterator() for iter.Next(ctx) { keys = append(keys, iter.Val()) } if err := iter.Err(); err != nil { return nil, fmt.Errorf("probe: scan snapshots for node %s: %w", nodeID, err) } result := make(map[string]ProbeSnapshot, len(keys)) prefix := "probe:" + nodeID + ":" for _, k := range keys { val, err := s.rdb.Get(ctx, k).Result() if err == redis.Nil { // Key expired between SCAN and GET – not an error. continue } if err != nil { return nil, fmt.Errorf("probe: get snapshot %s: %w", k, err) } var snap ProbeSnapshot if err := json.Unmarshal([]byte(val), &snap); err != nil { // Corrupted data – log-worthy but non-fatal; skip this entry. continue } vk := strings.TrimPrefix(k, prefix) result[vk] = snap } return result, nil } // AliveProbes returns the IDs of probe agents that have sent a heartbeat // within the last heartbeatTTL window (15 min). // // An empty result means "no probes have reported recently" – not that all // probes are down. Missing heartbeat keys must never be folded into a // failure signal by callers. func (s *Store) AliveProbes(ctx context.Context) ([]string, error) { var ids []string iter := s.rdb.Scan(ctx, 0, "probe:hb:*", 0).Iterator() for iter.Next(ctx) { k := iter.Val() ids = append(ids, strings.TrimPrefix(k, "probe:hb:")) } if err := iter.Err(); err != nil { return nil, fmt.Errorf("probe: scan heartbeats: %w", err) } return ids, nil } // CheckHeartbeats scans all known probe heartbeat keys and fires an // EventTypeHeartbeatMissing alert via notifier for every probe whose last // heartbeat timestamp is older than threshold. // // This is called by 15H (DetectLoop / assembly) on a periodic basis. // Missing-key semantics apply: a key that expired (TTL elapsed) is not seen // at all — only keys that exist but carry a stale timestamp are reported. // // threshold should be ≥90 s per the operational SLO. func (s *Store) CheckHeartbeats(ctx context.Context, threshold time.Duration, notifier alert.Notifier) error { now := time.Now().Unix() cutoff := now - int64(threshold.Seconds()) var scanErr error iter := s.rdb.Scan(ctx, 0, "probe:hb:*", 0).Iterator() for iter.Next(ctx) { k := iter.Val() probeID := strings.TrimPrefix(k, "probe:hb:") val, err := s.rdb.Get(ctx, k).Result() if err != nil { // Key may have expired between SCAN and GET; skip. continue } ts, err := strconv.ParseInt(val, 10, 64) if err != nil { continue // corrupt value; ignore } if ts < cutoff { // Heartbeat is stale: emit alert. staleSecs := now - ts ev := alert.NewEvent(alert.EventTypeHeartbeatMissing, probeID, map[string]string{ "stale_seconds": strconv.FormatInt(staleSecs, 10), "threshold_s": strconv.FormatInt(int64(threshold.Seconds()), 10), }) if notifyErr := notifier.Notify(ctx, ev); notifyErr != nil { // Log but continue checking other probes. scanErr = fmt.Errorf("probe: notify heartbeat missing for %s: %w", probeID, notifyErr) } } } if err := iter.Err(); err != nil { return fmt.Errorf("probe: scan heartbeats: %w", err) } return scanErr }