pangolin/server/internal/scheduler/orchestrate/config.go

package orchestrate

import (
	"context"
	"encoding/json"
	"fmt"
	"log/slog"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync/atomic"
	"syscall"

	"gopkg.in/yaml.v3"
)

// ─────────────────────────────────────────────────────────────────────────────
// SchedConfig — master configuration tree
// ─────────────────────────────────────────────────────────────────────────────

// SchedConfig is the single source of truth for all scheduler thresholds.
// It covers: 15D detection, 15F capacity/breaker, and 15E probe/grayscale.
//
// Values are layered in order:
//  1. Defaults (DefaultSchedConfig).
//  2. Optional YAML file (path set via SCHED_CONFIG_FILE env var, or passed
//     to NewConfigManager).
//  3. Environment variable overrides (applied on top of YAML).
//
// The live config is accessed via ConfigManager.Current() — a lock-free
// atomic.Pointer read. Hot reload is triggered by SIGHUP; each reload
// computes a JSON diff and calls the provided AuditFn.
type SchedConfig struct {
	// Detect holds 15D detection-engine thresholds.
	Detect DetectSection `yaml:"detect"`
	// Capacity holds 15F capacity-monitor and breaker thresholds.
	Capacity CapacitySection `yaml:"capacity"`
	// Probe holds 15E probe-window and grayscale-ramp settings.
	Probe ProbeSection `yaml:"probe"`
}

// DetectSection mirrors detect.DetectConfig; task 15H bridges the two at
// wiring time.
type DetectSection struct {
	// DomesticFailNumerator / DomesticFailDenominator define the ISP-failure
	// fraction that triggers the suspect rule (default: 2/3).
	DomesticFailNumerator   int `yaml:"domestic_fail_numerator"`
	DomesticFailDenominator int `yaml:"domestic_fail_denominator"`

	// SuspectStreakMin is the number of consecutive failing cycles before a
	// node enters blocked_suspect (default: 2).
	SuspectStreakMin int `yaml:"suspect_streak_min"`

	// TrafficDropThreshold is the minimum percentage drop in online connections
	// over 15 min that activates the traffic-warning rule (default: 80.0).
	TrafficDropThreshold float64 `yaml:"traffic_drop_threshold"`

	// TrafficBaselineMin is the minimum current online-connection count for
	// the traffic-warning relaxation to apply (default: 20).
	TrafficBaselineMin int `yaml:"traffic_baseline_min"`

	// ConfirmedStreakMin is the number of consecutive cycles in blocked_suspect
	// before promotion to blocked_confirmed (default: 6).
	ConfirmedStreakMin int `yaml:"confirmed_streak_min"`

	// RecoverStreakMin is the number of consecutive passing cycles while in
	// blocked_suspect required to recover to up (default: 2).
	RecoverStreakMin int `yaml:"recover_streak_min"`

	// SuspectWeight is the routing weight applied when a node first enters
	// blocked_suspect (default: 10).
	SuspectWeight int `yaml:"suspect_weight"`
}

// CapacitySection holds 15F capacity-monitor and circuit-breaker thresholds.
type CapacitySection struct {
	// WatermarkThreshold is the minimum pool fill-rate (upCount/target) before
	// an alert is emitted (default: 0.70 = 70%).
	WatermarkThreshold float64 `yaml:"watermark_threshold"`

	// AlertSuppressMin is the minimum gap in minutes between repeated alerts
	// for the same pool or probe (default: 10 min).
	AlertSuppressMin int `yaml:"alert_suppress_min"`

	// BreakerWindowMin is the sliding-window duration in minutes for the
	// circuit breaker (default: 60 min = 1 h).
	BreakerWindowMin int `yaml:"breaker_window_min"`

	// BreakerFractionPct is the percentage of pool target capacity that, when
	// replaced within one window, trips the breaker (default: 30).
	BreakerFractionPct int `yaml:"breaker_fraction_pct"`

	// BreakerMinN is the absolute lower bound for the breaker trip threshold
	// (default: 3). Overrides fraction when fraction yields a smaller number.
	BreakerMinN int `yaml:"breaker_min_n"`

	// ManualAlertThresh is the consecutive probe-failure count for a single
	// node that triggers a 转人工 (escalate-to-human) alert. Should match
	// 15E's MaxAttempts (default: 3).
	ManualAlertThresh int `yaml:"manual_alert_thresh"`

	// ExpectedProbeIDs is the list of probe-agent IDs that must maintain live
	// heartbeat keys in Redis. A missing key triggers a 「探针失联」alert.
	ExpectedProbeIDs []string `yaml:"expected_probe_ids"`
}

// ProbeSection holds 15E probe-window and grayscale-ramp settings.
type ProbeSection struct {
	// GrayscaleIntervalHours is the time in hours between successive grayscale
	// weight-ramp steps (default: 6 h).
	GrayscaleIntervalHours int `yaml:"grayscale_interval_hours"`

	// ProbeTimeoutMin is the maximum time in minutes allowed for a node to
	// pass probing per attempt (default: 15 min).
	ProbeTimeoutMin int `yaml:"probe_timeout_min"`

	// MaxAttempts is the maximum number of create+probe attempts per
	// replacement record before marking it failed (default: 3).
	MaxAttempts int `yaml:"max_attempts"`

	// ProbeCyclesRequired is the number of consecutive passing probe ticks
	// needed before a new node is promoted to up (default: 2).
	ProbeCyclesRequired int `yaml:"probe_cycles_required"`
}

// DefaultSchedConfig returns a SchedConfig pre-filled with production defaults.
func DefaultSchedConfig() SchedConfig {
	return SchedConfig{
		Detect: DetectSection{
			DomesticFailNumerator:   2,
			DomesticFailDenominator: 3,
			SuspectStreakMin:        2,
			TrafficDropThreshold:    80.0,
			TrafficBaselineMin:      20,
			ConfirmedStreakMin:       6,
			RecoverStreakMin:        2,
			SuspectWeight:           10,
		},
		Capacity: CapacitySection{
			WatermarkThreshold: 0.70,
			AlertSuppressMin:   10,
			BreakerWindowMin:   60,
			BreakerFractionPct: 30,
			BreakerMinN:        3,
			ManualAlertThresh:  3,
		},
		Probe: ProbeSection{
			GrayscaleIntervalHours: 6,
			ProbeTimeoutMin:        15,
			MaxAttempts:            3,
			ProbeCyclesRequired:    2,
		},
	}
}

// ─────────────────────────────────────────────────────────────────────────────
// ConfigManager
// ─────────────────────────────────────────────────────────────────────────────

// AuditFn is called after each successful config reload with the old and new
// configurations serialised as JSON strings. The caller is responsible for
// deciding how to persist the diff (e.g. via LifecycleService.WriteAuditLog).
// May be nil to skip auditing.
type AuditFn func(ctx context.Context, oldJSON, newJSON string)

// ConfigManager holds the live SchedConfig and supports SIGHUP-triggered
// hot reload. Config reads are lock-free (atomic.Pointer).
//
// Usage:
//
//	mgr := NewConfigManager("/etc/pangolin/sched.yaml", auditFn)
//	if err := mgr.Load(); err != nil { log.Fatal(err) }
//	mgr.WatchSIGHUP(ctx)
//	cfg := mgr.Current()
type ConfigManager struct {
	path    string              // optional YAML file path (may be empty)
	cfg     atomic.Pointer[SchedConfig]
	auditFn AuditFn
}

// NewConfigManager creates a ConfigManager.
//
//   path:    optional path to the YAML config file. If SCHED_CONFIG_FILE is
//            set in the environment it overrides this argument.
//   auditFn: called on every reload when the config changes; may be nil.
//
// The manager is seeded with DefaultSchedConfig so Current() is never nil.
func NewConfigManager(path string, fn AuditFn) *ConfigManager {
	if p := os.Getenv("SCHED_CONFIG_FILE"); p != "" {
		path = p
	}
	m := &ConfigManager{path: path, auditFn: fn}
	dflt := DefaultSchedConfig()
	m.cfg.Store(&dflt)
	return m
}

// Current returns the live config. Never nil.
func (m *ConfigManager) Current() *SchedConfig {
	return m.cfg.Load()
}

// Load reads the config (YAML file if configured, then env overrides) and
// atomically replaces the live config. Safe to call multiple times.
func (m *ConfigManager) Load() error {
	next, err := loadSchedConfig(m.path)
	if err != nil {
		return err
	}
	m.swap(context.Background(), next)
	return nil
}

// WatchSIGHUP starts a goroutine that calls Reload whenever SIGHUP is
// received. The goroutine stops when ctx is cancelled.
func (m *ConfigManager) WatchSIGHUP(ctx context.Context) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGHUP)
	go func() {
		defer signal.Stop(ch)
		for {
			select {
			case <-ctx.Done():
				return
			case <-ch:
				if err := m.Reload(ctx); err != nil {
					slog.Error("sched config: SIGHUP reload failed", "error", err)
				} else {
					slog.Info("sched config: reloaded via SIGHUP")
				}
			}
		}
	}()
}

// Reload re-reads the config and hot-swaps the live value. Idempotent.
func (m *ConfigManager) Reload(ctx context.Context) error {
	next, err := loadSchedConfig(m.path)
	if err != nil {
		return err
	}
	m.swap(ctx, next)
	return nil
}

// swap atomically replaces the live config and records a diff in the audit
// log when the config actually changed.
func (m *ConfigManager) swap(ctx context.Context, next *SchedConfig) {
	old := m.cfg.Swap(next)
	if m.auditFn == nil {
		return
	}
	oldJSON, _ := json.Marshal(old)
	newJSON, _ := json.Marshal(next)
	if string(oldJSON) == string(newJSON) {
		return // no change; skip audit
	}
	m.auditFn(ctx, string(oldJSON), string(newJSON))
}

// ─────────────────────────────────────────────────────────────────────────────
// Loading logic
// ─────────────────────────────────────────────────────────────────────────────

// loadSchedConfig builds a SchedConfig using the layering strategy:
// defaults → YAML file → env-var overrides.
func loadSchedConfig(path string) (*SchedConfig, error) {
	cfg := DefaultSchedConfig()

	// Layer 2: optional YAML file.
	if path != "" {
		data, err := os.ReadFile(path)
		if err != nil && !os.IsNotExist(err) {
			return nil, fmt.Errorf("sched config: read %s: %w", path, err)
		}
		if err == nil {
			if err := yaml.Unmarshal(data, &cfg); err != nil {
				return nil, fmt.Errorf("sched config: parse %s: %w", path, err)
			}
		}
	}

	// Layer 3: environment variable overrides.
	applySchedEnvOverrides(&cfg)

	return &cfg, nil
}

// applySchedEnvOverrides overlays environment-variable values on top of cfg.
// Only non-empty env vars are applied; empty vars preserve the current value.
func applySchedEnvOverrides(cfg *SchedConfig) {
	// ── 15D detect ────────────────────────────────────────────────────────────
	if v := posIntEnv("DETECT_DOMESTIC_FAIL_NUM"); v > 0 {
		cfg.Detect.DomesticFailNumerator = v
	}
	if v := posIntEnv("DETECT_DOMESTIC_FAIL_DEN"); v > 0 {
		cfg.Detect.DomesticFailDenominator = v
	}
	if v := posIntEnv("DETECT_SUSPECT_STREAK_MIN"); v > 0 {
		cfg.Detect.SuspectStreakMin = v
	}
	if v := posFloatEnv("DETECT_TRAFFIC_DROP_THRESHOLD"); v > 0 {
		cfg.Detect.TrafficDropThreshold = v
	}
	if v := posIntEnv("DETECT_TRAFFIC_BASELINE_MIN"); v > 0 {
		cfg.Detect.TrafficBaselineMin = v
	}
	if v := posIntEnv("DETECT_CONFIRMED_STREAK_MIN"); v > 0 {
		cfg.Detect.ConfirmedStreakMin = v
	}
	if v := posIntEnv("DETECT_RECOVER_STREAK_MIN"); v > 0 {
		cfg.Detect.RecoverStreakMin = v
	}
	if v := posIntEnv("DETECT_SUSPECT_WEIGHT"); v > 0 {
		cfg.Detect.SuspectWeight = v
	}

	// ── 15F capacity / breaker ────────────────────────────────────────────────
	if v := posFloatEnv("CAP_WATERMARK_THRESHOLD"); v > 0 {
		cfg.Capacity.WatermarkThreshold = v
	}
	if v := posIntEnv("CAP_ALERT_SUPPRESS_MIN"); v > 0 {
		cfg.Capacity.AlertSuppressMin = v
	}
	if v := posIntEnv("CAP_BREAKER_WINDOW_MIN"); v > 0 {
		cfg.Capacity.BreakerWindowMin = v
	}
	if v := posIntEnv("CAP_BREAKER_FRACTION_PCT"); v > 0 {
		cfg.Capacity.BreakerFractionPct = v
	}
	if v := posIntEnv("CAP_BREAKER_MIN_N"); v > 0 {
		cfg.Capacity.BreakerMinN = v
	}
	if v := posIntEnv("CAP_MANUAL_ALERT_THRESH"); v > 0 {
		cfg.Capacity.ManualAlertThresh = v
	}
	if v := os.Getenv("CAP_EXPECTED_PROBE_IDS"); v != "" {
		cfg.Capacity.ExpectedProbeIDs = splitCSVEnv(v)
	}

	// ── 15E probe / grayscale ─────────────────────────────────────────────────
	if v := posIntEnv("PROBE_GRAYSCALE_INTERVAL_H"); v > 0 {
		cfg.Probe.GrayscaleIntervalHours = v
	}
	if v := posIntEnv("PROBE_TIMEOUT_MIN"); v > 0 {
		cfg.Probe.ProbeTimeoutMin = v
	}
	if v := posIntEnv("PROBE_MAX_ATTEMPTS"); v > 0 {
		cfg.Probe.MaxAttempts = v
	}
	if v := posIntEnv("PROBE_CYCLES_REQUIRED"); v > 0 {
		cfg.Probe.ProbeCyclesRequired = v
	}
}

// posIntEnv reads key as a positive integer. Returns 0 if the env var is
// unset, empty, zero, or non-parseable (logs a warning for parse errors).
func posIntEnv(key string) int {
	v := os.Getenv(key)
	if v == "" {
		return 0
	}
	n, err := strconv.Atoi(v)
	if err != nil || n <= 0 {
		if err != nil {
			slog.Warn("sched config: invalid integer env var", "key", key, "value", v)
		}
		return 0
	}
	return n
}

// posFloatEnv reads key as a positive float64. Returns 0 on error.
func posFloatEnv(key string) float64 {
	v := os.Getenv(key)
	if v == "" {
		return 0
	}
	f, err := strconv.ParseFloat(v, 64)
	if err != nil || f <= 0 {
		if err != nil {
			slog.Warn("sched config: invalid float env var", "key", key, "value", v)
		}
		return 0
	}
	return f
}

// splitCSVEnv splits a comma-separated env-var value, trimming whitespace and
// dropping empty fields.
func splitCSVEnv(s string) []string {
	parts := strings.Split(s, ",")
	out := parts[:0]
	for _, p := range parts {
		p = strings.TrimSpace(p)
		if p != "" {
			out = append(out, p)
		}
	}
	return out
}