package orchestrate import ( "context" "encoding/json" "fmt" "log/slog" "os" "os/signal" "strconv" "strings" "sync/atomic" "syscall" "gopkg.in/yaml.v3" ) // ───────────────────────────────────────────────────────────────────────────── // SchedConfig — master configuration tree // ───────────────────────────────────────────────────────────────────────────── // SchedConfig is the single source of truth for all scheduler thresholds. // It covers: 15D detection, 15F capacity/breaker, and 15E probe/grayscale. // // Values are layered in order: // 1. Defaults (DefaultSchedConfig). // 2. Optional YAML file (path set via SCHED_CONFIG_FILE env var, or passed // to NewConfigManager). // 3. Environment variable overrides (applied on top of YAML). // // The live config is accessed via ConfigManager.Current() — a lock-free // atomic.Pointer read. Hot reload is triggered by SIGHUP; each reload // computes a JSON diff and calls the provided AuditFn. type SchedConfig struct { // Detect holds 15D detection-engine thresholds. Detect DetectSection `yaml:"detect"` // Capacity holds 15F capacity-monitor and breaker thresholds. Capacity CapacitySection `yaml:"capacity"` // Probe holds 15E probe-window and grayscale-ramp settings. Probe ProbeSection `yaml:"probe"` } // DetectSection mirrors detect.DetectConfig; task 15H bridges the two at // wiring time. type DetectSection struct { // DomesticFailNumerator / DomesticFailDenominator define the ISP-failure // fraction that triggers the suspect rule (default: 2/3). DomesticFailNumerator int `yaml:"domestic_fail_numerator"` DomesticFailDenominator int `yaml:"domestic_fail_denominator"` // SuspectStreakMin is the number of consecutive failing cycles before a // node enters blocked_suspect (default: 2). SuspectStreakMin int `yaml:"suspect_streak_min"` // TrafficDropThreshold is the minimum percentage drop in online connections // over 15 min that activates the traffic-warning rule (default: 80.0). TrafficDropThreshold float64 `yaml:"traffic_drop_threshold"` // TrafficBaselineMin is the minimum current online-connection count for // the traffic-warning relaxation to apply (default: 20). TrafficBaselineMin int `yaml:"traffic_baseline_min"` // ConfirmedStreakMin is the number of consecutive cycles in blocked_suspect // before promotion to blocked_confirmed (default: 6). ConfirmedStreakMin int `yaml:"confirmed_streak_min"` // RecoverStreakMin is the number of consecutive passing cycles while in // blocked_suspect required to recover to up (default: 2). RecoverStreakMin int `yaml:"recover_streak_min"` // SuspectWeight is the routing weight applied when a node first enters // blocked_suspect (default: 10). SuspectWeight int `yaml:"suspect_weight"` } // CapacitySection holds 15F capacity-monitor and circuit-breaker thresholds. type CapacitySection struct { // WatermarkThreshold is the minimum pool fill-rate (upCount/target) before // an alert is emitted (default: 0.70 = 70%). WatermarkThreshold float64 `yaml:"watermark_threshold"` // AlertSuppressMin is the minimum gap in minutes between repeated alerts // for the same pool or probe (default: 10 min). AlertSuppressMin int `yaml:"alert_suppress_min"` // BreakerWindowMin is the sliding-window duration in minutes for the // circuit breaker (default: 60 min = 1 h). BreakerWindowMin int `yaml:"breaker_window_min"` // BreakerFractionPct is the percentage of pool target capacity that, when // replaced within one window, trips the breaker (default: 30). BreakerFractionPct int `yaml:"breaker_fraction_pct"` // BreakerMinN is the absolute lower bound for the breaker trip threshold // (default: 3). Overrides fraction when fraction yields a smaller number. BreakerMinN int `yaml:"breaker_min_n"` // ManualAlertThresh is the consecutive probe-failure count for a single // node that triggers a 转人工 (escalate-to-human) alert. Should match // 15E's MaxAttempts (default: 3). ManualAlertThresh int `yaml:"manual_alert_thresh"` // ExpectedProbeIDs is the list of probe-agent IDs that must maintain live // heartbeat keys in Redis. A missing key triggers a 「探针失联」alert. ExpectedProbeIDs []string `yaml:"expected_probe_ids"` } // ProbeSection holds 15E probe-window and grayscale-ramp settings. type ProbeSection struct { // GrayscaleIntervalHours is the time in hours between successive grayscale // weight-ramp steps (default: 6 h). GrayscaleIntervalHours int `yaml:"grayscale_interval_hours"` // ProbeTimeoutMin is the maximum time in minutes allowed for a node to // pass probing per attempt (default: 15 min). ProbeTimeoutMin int `yaml:"probe_timeout_min"` // MaxAttempts is the maximum number of create+probe attempts per // replacement record before marking it failed (default: 3). MaxAttempts int `yaml:"max_attempts"` // ProbeCyclesRequired is the number of consecutive passing probe ticks // needed before a new node is promoted to up (default: 2). ProbeCyclesRequired int `yaml:"probe_cycles_required"` } // DefaultSchedConfig returns a SchedConfig pre-filled with production defaults. func DefaultSchedConfig() SchedConfig { return SchedConfig{ Detect: DetectSection{ DomesticFailNumerator: 2, DomesticFailDenominator: 3, SuspectStreakMin: 2, TrafficDropThreshold: 80.0, TrafficBaselineMin: 20, ConfirmedStreakMin: 6, RecoverStreakMin: 2, SuspectWeight: 10, }, Capacity: CapacitySection{ WatermarkThreshold: 0.70, AlertSuppressMin: 10, BreakerWindowMin: 60, BreakerFractionPct: 30, BreakerMinN: 3, ManualAlertThresh: 3, }, Probe: ProbeSection{ GrayscaleIntervalHours: 6, ProbeTimeoutMin: 15, MaxAttempts: 3, ProbeCyclesRequired: 2, }, } } // ───────────────────────────────────────────────────────────────────────────── // ConfigManager // ───────────────────────────────────────────────────────────────────────────── // AuditFn is called after each successful config reload with the old and new // configurations serialised as JSON strings. The caller is responsible for // deciding how to persist the diff (e.g. via LifecycleService.WriteAuditLog). // May be nil to skip auditing. type AuditFn func(ctx context.Context, oldJSON, newJSON string) // ConfigManager holds the live SchedConfig and supports SIGHUP-triggered // hot reload. Config reads are lock-free (atomic.Pointer). // // Usage: // // mgr := NewConfigManager("/etc/pangolin/sched.yaml", auditFn) // if err := mgr.Load(); err != nil { log.Fatal(err) } // mgr.WatchSIGHUP(ctx) // cfg := mgr.Current() type ConfigManager struct { path string // optional YAML file path (may be empty) cfg atomic.Pointer[SchedConfig] auditFn AuditFn } // NewConfigManager creates a ConfigManager. // // path: optional path to the YAML config file. If SCHED_CONFIG_FILE is // set in the environment it overrides this argument. // auditFn: called on every reload when the config changes; may be nil. // // The manager is seeded with DefaultSchedConfig so Current() is never nil. func NewConfigManager(path string, fn AuditFn) *ConfigManager { if p := os.Getenv("SCHED_CONFIG_FILE"); p != "" { path = p } m := &ConfigManager{path: path, auditFn: fn} dflt := DefaultSchedConfig() m.cfg.Store(&dflt) return m } // Current returns the live config. Never nil. func (m *ConfigManager) Current() *SchedConfig { return m.cfg.Load() } // Load reads the config (YAML file if configured, then env overrides) and // atomically replaces the live config. Safe to call multiple times. func (m *ConfigManager) Load() error { next, err := loadSchedConfig(m.path) if err != nil { return err } m.swap(context.Background(), next) return nil } // WatchSIGHUP starts a goroutine that calls Reload whenever SIGHUP is // received. The goroutine stops when ctx is cancelled. func (m *ConfigManager) WatchSIGHUP(ctx context.Context) { ch := make(chan os.Signal, 1) signal.Notify(ch, syscall.SIGHUP) go func() { defer signal.Stop(ch) for { select { case <-ctx.Done(): return case <-ch: if err := m.Reload(ctx); err != nil { slog.Error("sched config: SIGHUP reload failed", "error", err) } else { slog.Info("sched config: reloaded via SIGHUP") } } } }() } // Reload re-reads the config and hot-swaps the live value. Idempotent. func (m *ConfigManager) Reload(ctx context.Context) error { next, err := loadSchedConfig(m.path) if err != nil { return err } m.swap(ctx, next) return nil } // swap atomically replaces the live config and records a diff in the audit // log when the config actually changed. func (m *ConfigManager) swap(ctx context.Context, next *SchedConfig) { old := m.cfg.Swap(next) if m.auditFn == nil { return } oldJSON, _ := json.Marshal(old) newJSON, _ := json.Marshal(next) if string(oldJSON) == string(newJSON) { return // no change; skip audit } m.auditFn(ctx, string(oldJSON), string(newJSON)) } // ───────────────────────────────────────────────────────────────────────────── // Loading logic // ───────────────────────────────────────────────────────────────────────────── // loadSchedConfig builds a SchedConfig using the layering strategy: // defaults → YAML file → env-var overrides. func loadSchedConfig(path string) (*SchedConfig, error) { cfg := DefaultSchedConfig() // Layer 2: optional YAML file. if path != "" { data, err := os.ReadFile(path) if err != nil && !os.IsNotExist(err) { return nil, fmt.Errorf("sched config: read %s: %w", path, err) } if err == nil { if err := yaml.Unmarshal(data, &cfg); err != nil { return nil, fmt.Errorf("sched config: parse %s: %w", path, err) } } } // Layer 3: environment variable overrides. applySchedEnvOverrides(&cfg) return &cfg, nil } // applySchedEnvOverrides overlays environment-variable values on top of cfg. // Only non-empty env vars are applied; empty vars preserve the current value. func applySchedEnvOverrides(cfg *SchedConfig) { // ── 15D detect ──────────────────────────────────────────────────────────── if v := posIntEnv("DETECT_DOMESTIC_FAIL_NUM"); v > 0 { cfg.Detect.DomesticFailNumerator = v } if v := posIntEnv("DETECT_DOMESTIC_FAIL_DEN"); v > 0 { cfg.Detect.DomesticFailDenominator = v } if v := posIntEnv("DETECT_SUSPECT_STREAK_MIN"); v > 0 { cfg.Detect.SuspectStreakMin = v } if v := posFloatEnv("DETECT_TRAFFIC_DROP_THRESHOLD"); v > 0 { cfg.Detect.TrafficDropThreshold = v } if v := posIntEnv("DETECT_TRAFFIC_BASELINE_MIN"); v > 0 { cfg.Detect.TrafficBaselineMin = v } if v := posIntEnv("DETECT_CONFIRMED_STREAK_MIN"); v > 0 { cfg.Detect.ConfirmedStreakMin = v } if v := posIntEnv("DETECT_RECOVER_STREAK_MIN"); v > 0 { cfg.Detect.RecoverStreakMin = v } if v := posIntEnv("DETECT_SUSPECT_WEIGHT"); v > 0 { cfg.Detect.SuspectWeight = v } // ── 15F capacity / breaker ──────────────────────────────────────────────── if v := posFloatEnv("CAP_WATERMARK_THRESHOLD"); v > 0 { cfg.Capacity.WatermarkThreshold = v } if v := posIntEnv("CAP_ALERT_SUPPRESS_MIN"); v > 0 { cfg.Capacity.AlertSuppressMin = v } if v := posIntEnv("CAP_BREAKER_WINDOW_MIN"); v > 0 { cfg.Capacity.BreakerWindowMin = v } if v := posIntEnv("CAP_BREAKER_FRACTION_PCT"); v > 0 { cfg.Capacity.BreakerFractionPct = v } if v := posIntEnv("CAP_BREAKER_MIN_N"); v > 0 { cfg.Capacity.BreakerMinN = v } if v := posIntEnv("CAP_MANUAL_ALERT_THRESH"); v > 0 { cfg.Capacity.ManualAlertThresh = v } if v := os.Getenv("CAP_EXPECTED_PROBE_IDS"); v != "" { cfg.Capacity.ExpectedProbeIDs = splitCSVEnv(v) } // ── 15E probe / grayscale ───────────────────────────────────────────────── if v := posIntEnv("PROBE_GRAYSCALE_INTERVAL_H"); v > 0 { cfg.Probe.GrayscaleIntervalHours = v } if v := posIntEnv("PROBE_TIMEOUT_MIN"); v > 0 { cfg.Probe.ProbeTimeoutMin = v } if v := posIntEnv("PROBE_MAX_ATTEMPTS"); v > 0 { cfg.Probe.MaxAttempts = v } if v := posIntEnv("PROBE_CYCLES_REQUIRED"); v > 0 { cfg.Probe.ProbeCyclesRequired = v } } // posIntEnv reads key as a positive integer. Returns 0 if the env var is // unset, empty, zero, or non-parseable (logs a warning for parse errors). func posIntEnv(key string) int { v := os.Getenv(key) if v == "" { return 0 } n, err := strconv.Atoi(v) if err != nil || n <= 0 { if err != nil { slog.Warn("sched config: invalid integer env var", "key", key, "value", v) } return 0 } return n } // posFloatEnv reads key as a positive float64. Returns 0 on error. func posFloatEnv(key string) float64 { v := os.Getenv(key) if v == "" { return 0 } f, err := strconv.ParseFloat(v, 64) if err != nil || f <= 0 { if err != nil { slog.Warn("sched config: invalid float env var", "key", key, "value", v) } return 0 } return f } // splitCSVEnv splits a comma-separated env-var value, trimming whitespace and // dropping empty fields. func splitCSVEnv(s string) []string { parts := strings.Split(s, ",") out := parts[:0] for _, p := range parts { p = strings.TrimSpace(p) if p != "" { out = append(out, p) } } return out }