feat(nodes): gRPC server + Hub + Redis pub/sub cross-instance delivery (tsk__58l3wTLvaSn)

Implements AgentService gRPC server with all 6 RPCs (Enroll/Register/Heartbeat/
Subscribe/Ack/ReportUsage), Hub command routing with Redis ZSET at-least-once
persistence and cross-instance pub/sub delivery, LoadCache for node:load metrics,
NodeStore SQL interface + MySQL implementation, and full mTLS gRPC listener in main.

Integration tests: 18 tests covering full Enroll→Register→Heartbeat→Subscribe→Ack
flow, reconnect resume with last_command_id, and cross-instance pub/sub delivery
via bufconn + miniredis + mockNodeStore + real mTLS certificates.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

server/cmd/server/main.go

@@ -1,20 +1,31 @@
 package main
 
 import (
+	"context"
+	"crypto/tls"
 	"encoding/json"
 	"flag"
 	"log"
+	"log/slog"
+	"net"
 	"net/http"
 	"os"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+
+	"github.com/wangjia/pangolin/server/internal/db"
+	"github.com/wangjia/pangolin/server/internal/mtls"
+	"github.com/wangjia/pangolin/server/internal/nodes"
+	agentv1 "github.com/wangjia/pangolin/server/internal/pb/agentv1"
 	"github.com/wangjia/pangolin/server/internal/redisutil"
 	"github.com/wangjia/pangolin/server/internal/scheduler/probe"
 )
 
 func main() {
-	addr := flag.String("addr", "", "listen address (default :8080, overridden by ADDR env)")
+	addr := flag.String("addr", "", "HTTP listen address (default :8080, overridden by ADDR env)")
 	flag.Parse()
 
 	if *addr == "" {
@@ -25,6 +36,8 @@ func main() {
 		}
 	}
 
+	// ─── HTTP server ──────────────────────────────────────────────────────────
+
 	r := chi.NewRouter()
 	r.Use(middleware.Logger)
 	r.Use(middleware.Recoverer)
@@ -36,14 +49,6 @@ func main() {
 	})
 
 	// --- Probe ingest route (optional) ---
-	// Requires:
-	//   PROBE_SECRETS  – JSON object mapping probeId → HMAC secret, e.g.
-	//                    '{"probe-sg-01":"s3cr3t1","probe-jp-01":"s3cr3t2"}'
-	//   REDIS_ADDR     – Redis address, default 127.0.0.1:6379
-	//   REDIS_PASSWORD – Redis password (optional)
-	//
-	// If PROBE_SECRETS is empty the route is not registered and the server
-	// starts normally without the probe ingest endpoint.
 	probeSecretsJSON := os.Getenv("PROBE_SECRETS")
 	if probeSecretsJSON != "" {
 		redisAddr := getenvDefault("REDIS_ADDR", "127.0.0.1:6379")
@@ -63,12 +68,101 @@ func main() {
 		}
 	}
 
+	// ─── gRPC agent server (optional) ────────────────────────────────────────
+	// All five env vars must be set to activate the gRPC listener.
+
+	grpcAddr := os.Getenv("GRPC_ADDR")
+	caKeyPath := os.Getenv("CA_KEY_PATH")
+	caCertPath := os.Getenv("CA_CERT_PATH")
+	grpcCertPath := os.Getenv("GRPC_CERT_PATH")
+	grpcKeyPath := os.Getenv("GRPC_KEY_PATH")
+	dbDSN := os.Getenv("DB_DSN")
+
+	if grpcAddr != "" && caKeyPath != "" && caCertPath != "" &&
+		grpcCertPath != "" && grpcKeyPath != "" && dbDSN != "" {
+		redisAddr := getenvDefault("REDIS_ADDR", "127.0.0.1:6379")
+		go startGRPC(grpcAddr, caKeyPath, caCertPath, grpcCertPath, grpcKeyPath,
+			redisAddr, os.Getenv("REDIS_PASSWORD"), dbDSN)
+	} else if grpcAddr != "" {
+		slog.Warn("grpc server disabled: one or more required env vars are missing",
+			"GRPC_ADDR", grpcAddr,
+			"CA_KEY_PATH_set", caKeyPath != "",
+			"CA_CERT_PATH_set", caCertPath != "",
+			"GRPC_CERT_PATH_set", grpcCertPath != "",
+			"GRPC_KEY_PATH_set", grpcKeyPath != "",
+			"DB_DSN_set", dbDSN != "")
+	}
+
+	// ─── Start HTTP server (blocking) ────────────────────────────────────────
+
 	log.Printf("pangolin server listening on %s", *addr)
 	if err := http.ListenAndServe(*addr, r); err != nil {
 		log.Fatalf("server error: %v", err)
 	}
 }
 
+// startGRPC initialises and starts the mTLS gRPC agent server.
+// Must be called in a goroutine. Calls log.Fatalf on unrecoverable errors.
+func startGRPC(
+	addr, caKeyPath, caCertPath, grpcCertPath, grpcKeyPath,
+	redisAddr, redisPassword, dbDSN string,
+) {
+	// Pangolin Node CA (signs + verifies node client certs).
+	ca, err := mtls.NewCA(mtls.CAConfig{
+		KeyPath:  caKeyPath,
+		CertPath: caCertPath,
+	})
+	if err != nil {
+		log.Fatalf("grpc: load CA: %v", err)
+	}
+
+	// Redis (hub command queue, load cache, CRL, bootstrap tokens).
+	rdb, err := redisutil.New(redisAddr, redisPassword, 0)
+	if err != nil {
+		log.Fatalf("grpc: redis connect: %v", err)
+	}
+
+	// CRL + one-time bootstrap token manager.
+	crl := mtls.NewCRL(rdb, nil)
+	tokens := mtls.NewBootstrapTokenManager(rdb)
+
+	// MySQL for node catalogue and usage accumulation.
+	sqlDB, err := db.Open(dbDSN)
+	if err != nil {
+		log.Fatalf("grpc: db open: %v", err)
+	}
+
+	// Nodes service (wires Hub, LoadCache, Handler).
+	store := nodes.NewSQLNodeStore(sqlDB)
+	svc := nodes.NewService(ca, tokens, rdb, store)
+	svc.Hub().Start(context.Background()) // pub/sub goroutine runs for process lifetime
+
+	// gRPC transport TLS (server's own cert, typically from Let's Encrypt).
+	serverCert, err := tls.LoadX509KeyPair(grpcCertPath, grpcKeyPath)
+	if err != nil {
+		log.Fatalf("grpc: load transport TLS cert: %v", err)
+	}
+	tlsCfg := mtls.NewServerTLSConfig(ca, crl)
+	tlsCfg.Certificates = []tls.Certificate{serverCert}
+
+	// Build gRPC server with mTLS transport + identity interceptors.
+	srv := grpc.NewServer(
+		grpc.Creds(credentials.NewTLS(tlsCfg)),
+		grpc.ChainUnaryInterceptor(mtls.UnaryServerInterceptor()),
+		grpc.ChainStreamInterceptor(mtls.StreamServerInterceptor()),
+	)
+	agentv1.RegisterAgentServiceServer(srv, svc.Handler())
+
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalf("grpc: listen %s: %v", addr, err)
+	}
+	slog.Info("grpc agent server listening", "addr", addr)
+	if err := srv.Serve(lis); err != nil {
+		log.Fatalf("grpc: serve: %v", err)
+	}
+}
+
 func getenvDefault(key, def string) string {
 	if v := os.Getenv(key); v != "" {
 		return v