package nodes import ( "context" "encoding/json" "fmt" "log/slog" "strconv" "sync" "github.com/redis/go-redis/v9" agentv1 "github.com/wangjia/pangolin/server/internal/pb/agentv1" ) const ( // cmdQueuePrefix is the Redis ZSET key for each node's pending command queue. // Score = command_id (float64 cast of int64); member = JSON-encoded Command. // Commands remain in the queue until Ack'd; reconnecting nodes replay from afterID. cmdQueuePrefix = "node:cmdq:" // cmdPubChannel is the Redis Pub/Sub channel for cross-instance command delivery. // Published whenever a command cannot be delivered locally (node connected elsewhere). cmdPubChannel = "node:cmd" // cmdIDKey is the Redis counter for auto-generating monotonically increasing command IDs. cmdIDKey = "pangolin:node:next_cmdid" // subChanCap is the size of each local subscriber's delivery channel buffer. subChanCap = 256 ) // pubSubPayload is the message body published to cmdPubChannel. type pubSubPayload struct { NodeUUID string `json:"n"` Cmd *agentv1.Command `json:"c"` } // Hub routes commands to connected nodes. // // Persistence: every pushed command is stored in a per-node Redis ZSET // (node:cmdq:{uuid}) so the queue survives instance restarts. // // Local delivery: each active Subscribe stream holds a buffered channel. // When a node is connected locally, commands are sent directly to that channel. // // Cross-instance delivery: when the target node is not connected to this instance, // the command is published on cmdPubChannel so any other instance holding the node's // stream can pick it up and forward it. // // At-least-once semantics: commands are removed from the queue only on Ack. // A reconnecting node replays from its last acknowledged command_id. type Hub struct { mu sync.RWMutex subs map[string]chan *agentv1.Command rdb *redis.Client } // NewHub creates a Hub backed by rdb. // Call Start(ctx) to activate the cross-instance pub/sub goroutine. func NewHub(rdb *redis.Client) *Hub { return &Hub{ subs: make(map[string]chan *agentv1.Command), rdb: rdb, } } // Start begins the background pub/sub subscriber goroutine. // It runs until ctx is cancelled. func (h *Hub) Start(ctx context.Context) { go h.runPubSub(ctx) } func (h *Hub) runPubSub(ctx context.Context) { sub := h.rdb.Subscribe(ctx, cmdPubChannel) defer func() { _ = sub.Close() }() ch := sub.Channel() for { select { case <-ctx.Done(): return case msg, ok := <-ch: if !ok { return } var p pubSubPayload if err := json.Unmarshal([]byte(msg.Payload), &p); err != nil { slog.Warn("nodes.hub: bad pub/sub message", "err", err) continue } h.deliverLocal(p.NodeUUID, p.Cmd) } } } // deliverLocal sends cmd to the local subscriber for nodeUUID (non-blocking). // Returns true if the command was placed on the channel. func (h *Hub) deliverLocal(nodeUUID string, cmd *agentv1.Command) bool { h.mu.RLock() ch, ok := h.subs[nodeUUID] h.mu.RUnlock() if !ok { return false } select { case ch <- cmd: return true default: slog.Warn("nodes.hub: subscriber channel full, dropping command", "node_uuid", nodeUUID, "command_id", cmd.CommandID) return false } } // Register adds a local subscriber for nodeUUID and returns: // - ch: the receive channel on which live commands will arrive. // - done: cleanup function that must be called when the stream ends. // // Register must be called before Replay to avoid missing commands that arrive // between the two operations. func (h *Hub) Register(nodeUUID string) (<-chan *agentv1.Command, func()) { ch := make(chan *agentv1.Command, subChanCap) h.mu.Lock() h.subs[nodeUUID] = ch h.mu.Unlock() return ch, func() { h.mu.Lock() // Guard against re-registration: only delete if this is still the active channel. if h.subs[nodeUUID] == ch { delete(h.subs, nodeUUID) } h.mu.Unlock() } } // IsOnline reports whether an agent stream for nodeUUID is currently registered // on THIS instance — the authoritative agent-liveness signal for single-instance // deployments (the default). Used to refuse a connect / mark a node unhealthy // when its agent is offline, instead of handing out a credential nothing applies. // // Multi-instance note: an agent connected to a *different* instance is not in // this instance's subs map, so a correct multi-instance answer needs a // Redis-backed presence layer (deferred; prod runs single-instance). func (h *Hub) IsOnline(nodeUUID string) bool { h.mu.RLock() _, ok := h.subs[nodeUUID] h.mu.RUnlock() return ok } // Replay returns all unacked commands for nodeUUID with command_id > afterID, ordered ascending. // This is called during Subscribe to re-send commands the agent may have missed. func (h *Hub) Replay(ctx context.Context, nodeUUID string, afterID int64) ([]*agentv1.Command, error) { key := cmdQueuePrefix + nodeUUID min := fmt.Sprintf("(%d", afterID) // exclusive: score > afterID strs, err := h.rdb.ZRangeByScore(ctx, key, &redis.ZRangeBy{ Min: min, Max: "+inf", }).Result() if err != nil { return nil, fmt.Errorf("nodes.hub.Replay: %w", err) } cmds := make([]*agentv1.Command, 0, len(strs)) for _, s := range strs { var c agentv1.Command if err := json.Unmarshal([]byte(s), &c); err != nil { slog.Warn("nodes.hub: corrupt command in queue, skipping", "err", err, "node_uuid", nodeUUID) continue } cmds = append(cmds, &c) } return cmds, nil } // Push enqueues cmd for nodeUUID. // If cmd.CommandID == 0, a new monotonic ID is assigned via Redis INCR. // The command is persisted in the node's Redis queue then delivered: // - directly to the local subscriber channel if this instance holds the stream, // - otherwise published on cmdPubChannel for cross-instance delivery. func (h *Hub) Push(ctx context.Context, nodeUUID string, cmd *agentv1.Command) error { // Assign ID if not set. if cmd.CommandID == 0 { id, err := h.rdb.Incr(ctx, cmdIDKey).Result() if err != nil { return fmt.Errorf("nodes.hub.Push: generate id: %w", err) } cmd.CommandID = id } // Persist to Redis ZSET (durable, survives instance restart). data, err := json.Marshal(cmd) if err != nil { return fmt.Errorf("nodes.hub.Push: marshal: %w", err) } if err := h.rdb.ZAdd(ctx, cmdQueuePrefix+nodeUUID, redis.Z{ Score: float64(cmd.CommandID), Member: string(data), }).Err(); err != nil { return fmt.Errorf("nodes.hub.Push: zadd: %w", err) } // Try local delivery first (fastest path, no pub/sub latency). if h.deliverLocal(nodeUUID, cmd) { return nil } // Publish for cross-instance delivery. If the node is not connected anywhere, // this is a no-op and the node will pick up the command on next reconnect via Replay. payload, err := json.Marshal(pubSubPayload{NodeUUID: nodeUUID, Cmd: cmd}) if err != nil { return fmt.Errorf("nodes.hub.Push: marshal pubsub: %w", err) } if err := h.rdb.Publish(ctx, cmdPubChannel, payload).Err(); err != nil { // Non-fatal: command is already durable. slog.Warn("nodes.hub: publish failed (command still queued)", "node_uuid", nodeUUID, "command_id", cmd.CommandID, "err", err) } return nil } // Broadcast pushes cmd to each nodeUUID with a freshly assigned command_id per node. // A Broadcast cmd should be passed with CommandID = 0. func (h *Hub) Broadcast(ctx context.Context, nodeUUIDs []string, cmd *agentv1.Command) error { for _, uuid := range nodeUUIDs { clone := *cmd clone.CommandID = 0 // each push gets an independent ID if err := h.Push(ctx, uuid, &clone); err != nil { slog.Warn("nodes.hub.Broadcast: push failed", "node_uuid", uuid, "err", err) } } return nil } // Ack removes commandID from the pending queue for nodeUUID. // A second Ack for the same commandID is a no-op (idempotent). func (h *Hub) Ack(ctx context.Context, nodeUUID string, commandID int64) error { score := strconv.FormatInt(commandID, 10) return h.rdb.ZRemRangeByScore(ctx, cmdQueuePrefix+nodeUUID, score, score).Err() }