pangolin/server/internal/scheduler/probe/prober_agent.go

// Package probe – prober_agent.go
//
// # 供应商选型：阿里云云监控·云拨测 (Alibaba Cloud CloudMonitor Synthetic Monitoring)
//
// 选型理由：
//  1. 官方支持电信/联通/移动运营商节点筛选（IspCode 参数：246/247/248）。
//  2. 支持 TCP 连通性探测（TaskType="TCP"），可获取 RTT + 失败原因。
//  3. 覆盖全国 30+ 省份，节点元数据含 ISP 编码和城市。
//  4. RPC 风格 REST API，HMAC-SHA1 签名，无需官方 SDK，依赖只有标准库。
//  5. 按次计费，支持 RAM 子账号（权限最小化，满足运营安全 §06 红线 §2 身份隔离）。
//
// 身份隔离结论：
//  阿里云 RAM 支持独立子账号 + Action 级别授权
//  （cloudmonitor:CreateSiteMonitor + cloudmonitor:DescribeSiteMonitorData），
//  可与运营身份完全隔离，无 KYC 障碍。满足红线要求，无需退纯自建。
//
// API 版本：2019-01-01，RPC 风格
// 参考文档：https://help.aliyun.com/zh/cms/developer-reference/api-cms-2019-01-01-createsitemonitor
//
// # 降级语义
//
// API 报错/限流/超时 → 该 vantage 本周期不写 Redis（= 无数据，绝不写失败）。
// 连续不可用：incrementFailCount + slog.Warn，留给 15G 事件口消费。
package probe

import (
	"context"
	"crypto/hmac"
	"crypto/sha1" //nolint:gosec // Alibaba Cloud ACS v1 signing mandates HMAC-SHA1
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"
	"time"

	"github.com/wangjia/pangolin/server/internal/alert"
)

// --------------------------------------------------------------------------
// Public types
// --------------------------------------------------------------------------

// ProbeTarget describes a single node endpoint to be dial-tested.
type ProbeTarget struct {
	// NodeID identifies the node in the database (matches nodes.id).
	NodeID string

	// Host is the hostname or IP address of the node endpoint.
	Host string

	// Port is the TCP port to probe (e.g. 443).
	Port int
}

// VantageResult is a normalized dial-test result from a single ISP vantage point.
//
// L3 is always absent for third-party dial-tests.  Commercial synthetic-monitoring
// APIs probe TCP/TLS reachability only; they cannot validate the application
// protocol (Hysteria2/REALITY) end-to-end.  15D aggregation treats L3==nil from
// a 3rd-party vantage as "unknown at L3", not as a failure.
type VantageResult struct {
	// Vantage is the network vantage point from which the probe was conducted.
	// For Alibaba Cloud probes the Country is always "CN"; ISP uses the "3rd-"
	// prefix convention (e.g. "3rd-ChinaTelecom") to distinguish from first-party
	// probe agents.  15D may apply a configurable weight multiplier on this prefix.
	Vantage VantagePoint

	// L1 is the TCP connectivity result.
	L1 L1Result

	// L2 is the TLS handshake result.  Nil when L1 failed or the vendor did not
	// perform TLS probing.
	L2 *L2Result

	// L3 is intentionally absent — see package doc above.
}

// ProberAgent abstracts a third-party dial-test provider.
// Implementations must be safe for concurrent use.
type ProberAgent interface {
	// Probe issues multi-ISP dial-tests for the given target and returns one
	// VantageResult per ISP vantage point.
	//
	// Degradation contract:
	//   - On any provider error (rate-limit, timeout, API error) the
	//     implementation MUST return (nil, nil) and handle the error internally
	//     (log + counter).  Callers treat an empty slice as "no data this cycle".
	//   - The implementation MUST NOT return a VantageResult whose L1.OK==false
	//     solely because the API call failed; that would conflate infra errors
	//     with real reachability failures.
	Probe(ctx context.Context, target ProbeTarget) ([]VantageResult, error)
}

// --------------------------------------------------------------------------
// AliyunSyntheticAgent
// --------------------------------------------------------------------------

// aliyunISP maps human-readable ISP names (used in VantageResult) to the
// Alibaba Cloud CloudMonitor ISP code strings.
var aliyunISP = []struct {
	name    string // becomes VantagePoint.ISP with "3rd-" prefix
	ispCode string // Alibaba Cloud IspCode
}{
	{"ChinaTelecom", "246"},
	{"ChinaUnicom", "247"},
	{"ChinaMobile", "248"},
}

// aliyunEndpoint is the CloudMonitor API base URL.
// Overridable in tests via AliyunSyntheticAgent.baseURL.
const aliyunEndpoint = "https://cloudmonitor.cn-hangzhou.aliyuncs.com/"

// probeAgentLostThreshold is the number of consecutive per-ISP API failures
// that must accumulate before an EventTypeProbeAgentLost alert is emitted.
// This mirrors the "连续失败≥3" policy documented in the package comments.
const probeAgentLostThreshold = 3

// AliyunSyntheticAgentConfig holds configuration for AliyunSyntheticAgent.
// The AccessKeyID and AccessKeySecret must belong to a RAM sub-account with
// minimal permissions (cloudmonitor:CreateSiteMonitor +
// cloudmonitor:DescribeSiteMonitorData only).  They must be injected via
// environment variables; never stored in the database or committed to the repo.
type AliyunSyntheticAgentConfig struct {
	// AccessKeyID is the Alibaba Cloud RAM AccessKey ID.
	// Source: env var ALIYUN_PROBE_ACCESS_KEY_ID.
	AccessKeyID string

	// AccessKeySecret is the Alibaba Cloud RAM AccessKey Secret.
	// Source: env var ALIYUN_PROBE_ACCESS_KEY_SECRET.
	AccessKeySecret string

	// PollInterval is how long to wait between polling for task results.
	// Default: 5 seconds.
	PollInterval time.Duration

	// PollTimeout is the maximum time to wait for a single task to complete.
	// Default: 60 seconds.
	PollTimeout time.Duration

	// ProbeCity is the Alibaba Cloud city code used for all ISP probes.
	// Default: "563" (Beijing).  Only one city is needed — the ISP is the
	// relevant dimension for Pangolin's determination logic.
	ProbeCity string

	// BaseURL overrides the default CloudMonitor API endpoint.
	// Leave empty for production; set to a mock server URL in tests.
	BaseURL string

	// HTTPTimeout overrides the HTTP client timeout per call.
	// Zero uses the default of 30 seconds.
	HTTPTimeout time.Duration
}

// AliyunSyntheticAgent implements ProberAgent using Alibaba Cloud CloudMonitor
// Synthetic Monitoring.
type AliyunSyntheticAgent struct {
	cfg        AliyunSyntheticAgentConfig
	store      *Store
	httpClient *http.Client
	baseURL    string // overridable in tests
	failCount  atomic.Int64
	logger     *slog.Logger
	// notifier is the 15G exit channel for EventTypeProbeAgentLost events.
	// Nil means no alerting (development / test with no TG configured).
	notifier alert.Notifier
}

// SetNotifier injects the 15G alert outlet into the agent.
// When not set, no EventTypeProbeAgentLost alerts are emitted (dev/test mode).
// Call before RunOnce / Probe.
func (a *AliyunSyntheticAgent) SetNotifier(n alert.Notifier) {
	a.notifier = n
}

// NewAliyunSyntheticAgent creates an AliyunSyntheticAgent.
// logger may be nil (falls back to slog.Default()).
func NewAliyunSyntheticAgent(cfg AliyunSyntheticAgentConfig, store *Store, logger *slog.Logger) *AliyunSyntheticAgent {
	if cfg.PollInterval == 0 {
		cfg.PollInterval = 5 * time.Second
	}
	if cfg.PollTimeout == 0 {
		cfg.PollTimeout = 60 * time.Second
	}
	if cfg.ProbeCity == "" {
		cfg.ProbeCity = "563" // Beijing
	}
	if logger == nil {
		logger = slog.Default()
	}
	httpTimeout := 30 * time.Second
	if cfg.HTTPTimeout > 0 {
		httpTimeout = cfg.HTTPTimeout
	}
	baseURL := aliyunEndpoint
	if cfg.BaseURL != "" {
		baseURL = cfg.BaseURL
	}
	return &AliyunSyntheticAgent{
		cfg:   cfg,
		store: store,
		httpClient: &http.Client{
			Timeout: httpTimeout,
		},
		baseURL: baseURL,
		logger:  logger,
	}
}

// NewAliyunSyntheticAgentWithOptions is a convenience constructor for tests that
// need to inject a mock server URL without embedding it in the main config type.
// Production code should use NewAliyunSyntheticAgent.
func NewAliyunSyntheticAgentWithOptions(cfg AliyunSyntheticAgentConfig, store *Store, logger *slog.Logger, baseURL string) *AliyunSyntheticAgent {
	cfg.BaseURL = baseURL
	return NewAliyunSyntheticAgent(cfg, store, logger)
}

// SetHTTPClientTimeout replaces the agent's HTTP client timeout.
// Exported only for use in tests; production code should set cfg.HTTPTimeout.
func SetHTTPClientTimeout(a *AliyunSyntheticAgent, d time.Duration) *AliyunSyntheticAgent {
	a.httpClient = &http.Client{Timeout: d}
	return a
}

// RunOnce issues a full multi-ISP probe cycle for all given targets and writes
// results to Redis via store.  Designed for use in an independent goroutine
// controlled by 15H assembly.
//
// Per-target, per-ISP errors are handled internally (degraded silently);
// the method only returns a non-nil error when the context is cancelled.
func (a *AliyunSyntheticAgent) RunOnce(ctx context.Context, targets []ProbeTarget) error {
	for _, t := range targets {
		if ctx.Err() != nil {
			return ctx.Err()
		}
		results, err := a.Probe(ctx, t)
		if err != nil {
			// Probe already logged internally; continue to next target.
			continue
		}
		if len(results) == 0 {
			continue
		}
		// Convert VantageResult slice into the store's NodeReport format.
		// Each ISP vantage is a separate SaveReports call (distinct vantage key).
		for _, vr := range results {
			report := NodeReport{
				NodeID: t.NodeID,
				Ts:     time.Now().Unix(),
				L1:     vr.L1,
				L2:     vr.L2,
				// L3: intentionally nil
			}
			// probeID for 3rd-party: "aliyun-synthetic" (no HMAC secret required;
			// writes come from the server process, not an external agent).
			if wErr := a.store.SaveReports(ctx, "aliyun-synthetic", vr.Vantage, []NodeReport{report}); wErr != nil {
				a.logger.Warn("prober_agent: store write failed",
					"node", t.NodeID, "isp", vr.Vantage.ISP, "error", wErr)
			}
		}
	}
	return nil
}

// Probe implements ProberAgent.  It creates one TCP synthetic-monitoring task
// per ISP (telecom/unicom/mobile), polls for completion, then maps vendor
// results to VantageResult.
//
// On any API or polling error the ISP vantage is skipped (no result returned,
// no Redis write) per the degradation contract.
//
// When consecutive per-ISP API failures reach probeAgentLostThreshold the
// 探针失联 (EventTypeProbeAgentLost) alert is emitted via the injected Notifier.
func (a *AliyunSyntheticAgent) Probe(ctx context.Context, target ProbeTarget) ([]VantageResult, error) {
	var out []VantageResult
	for _, isp := range aliyunISP {
		vr, err := a.probeISP(ctx, target, isp.ispCode, isp.name)
		if err != nil {
			cnt := a.failCount.Add(1)
			a.logger.Warn("prober_agent: ISP probe failed (degraded, no data written)",
				"node", target.NodeID, "isp", isp.name, "error", err,
				"consecutive_failures", cnt)
			// Emit 探针失联 alert when threshold is crossed (15G exit channel).
			if cnt >= probeAgentLostThreshold && a.notifier != nil {
				ev := alert.NewEvent(alert.EventTypeProbeAgentLost, "aliyun-synthetic", map[string]string{
					"consecutive_failures": strconv.FormatInt(cnt, 10),
					"last_isp":             isp.name,
					"last_node":            target.NodeID,
				})
				if notifyErr := a.notifier.Notify(ctx, ev); notifyErr != nil {
					a.logger.Warn("prober_agent: notify probe agent lost", "error", notifyErr)
				}
			}
			// Degradation: skip this vantage this cycle.
			continue
		}
		a.failCount.Store(0) // reset on success
		out = append(out, vr)
	}
	return out, nil
}

// probeISP creates a single-ISP task, polls for results, and maps to VantageResult.
func (a *AliyunSyntheticAgent) probeISP(ctx context.Context, target ProbeTarget, ispCode, ispName string) (VantageResult, error) {
	taskID, err := a.createTask(ctx, target, ispCode)
	if err != nil {
		return VantageResult{}, fmt.Errorf("create task: %w", err)
	}

	result, err := a.pollResult(ctx, taskID)
	if err != nil {
		return VantageResult{}, fmt.Errorf("poll result: %w", err)
	}

	vp := VantagePoint{
		Country: "CN",
		Region:  result.Province,
		ISP:     "3rd-" + ispName,
	}

	vr := VantageResult{Vantage: vp}

	if result.Availability > 0 && result.ResponseTime > 0 {
		vr.L1 = L1Result{OK: true, RttMs: int(result.ResponseTime)}
	} else {
		vr.L1 = L1Result{OK: false, RttMs: 0}
	}

	// Alibaba Cloud TCP synthetic does not return a separate TLS phase result;
	// set L2 only when L1 succeeded and the port is 443 (implies TLS negotiated).
	if vr.L1.OK && target.Port == 443 {
		tlsErr := ""
		if result.ErrorInfo != "" {
			tlsErr = result.ErrorInfo
		}
		vr.L2 = &L2Result{
			OK:   tlsErr == "",
			ALPN: "", // vendor does not expose ALPN
			Err:  tlsErr,
		}
	}

	return vr, nil
}

// --------------------------------------------------------------------------
// Alibaba Cloud CloudMonitor Synthetic API calls
// --------------------------------------------------------------------------

// aliyunTaskResult holds the vendor-specific fields returned by
// DescribeSiteMonitorData.  Only fields needed for VantageResult are kept;
// vendor-specific identifiers do not escape this file.
type aliyunTaskResult struct {
	// Province is the probe node's province name (in Chinese, e.g. "北京").
	Province string
	// ResponseTime is the TCP round-trip time in milliseconds.
	ResponseTime float64
	// Availability is 100.0 for success, 0.0 for failure.
	Availability float64
	// ErrorInfo is a short vendor error string when the probe failed.
	ErrorInfo string
}

// createTask calls CreateSiteMonitor and returns the new task ID.
// address format: "host:port".
func (a *AliyunSyntheticAgent) createTask(ctx context.Context, target ProbeTarget, ispCode string) (string, error) {
	ispCity, _ := json.Marshal([]map[string]string{
		{"City": a.cfg.ProbeCity, "Isp": ispCode},
	})

	params := map[string]string{
		"Action":       "CreateSiteMonitor",
		"TaskName":     fmt.Sprintf("pangolin-%s-%s", target.NodeID, ispCode),
		"Address":      fmt.Sprintf("%s:%d", target.Host, target.Port),
		"TaskType":     "TCP",
		"Interval":     "1",    // one-time task
		"IspCities":    string(ispCity),
		"AlertIds":     "",
	}

	body, err := a.call(ctx, params)
	if err != nil {
		return "", err
	}

	var resp struct {
		Code    string `json:"Code"`
		Message string `json:"Message"`
		Data    struct {
			TaskID string `json:"TaskId"`
		} `json:"Data"`
	}
	if err := json.Unmarshal(body, &resp); err != nil {
		return "", fmt.Errorf("parse CreateSiteMonitor response: %w", err)
	}
	if resp.Code != "200" {
		return "", fmt.Errorf("CreateSiteMonitor error code=%s msg=%s", resp.Code, resp.Message)
	}
	if resp.Data.TaskID == "" {
		return "", fmt.Errorf("CreateSiteMonitor returned empty TaskId")
	}
	return resp.Data.TaskID, nil
}

// pollResult polls DescribeSiteMonitorData until the task produces a result
// or the context / poll timeout is exceeded.
func (a *AliyunSyntheticAgent) pollResult(ctx context.Context, taskID string) (aliyunTaskResult, error) {
	deadline := time.Now().Add(a.cfg.PollTimeout)
	ticker := time.NewTicker(a.cfg.PollInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return aliyunTaskResult{}, ctx.Err()
		case t := <-ticker.C:
			if t.After(deadline) {
				return aliyunTaskResult{}, fmt.Errorf("poll timeout after %s", a.cfg.PollTimeout)
			}
			result, ready, err := a.fetchResult(ctx, taskID)
			if err != nil {
				return aliyunTaskResult{}, err
			}
			if ready {
				return result, nil
			}
		}
	}
}

// fetchResult calls DescribeSiteMonitorData once.
// Returns (result, true, nil) when data is available, or (zero, false, nil) when
// the task is still running.
func (a *AliyunSyntheticAgent) fetchResult(ctx context.Context, taskID string) (aliyunTaskResult, bool, error) {
	now := time.Now()
	params := map[string]string{
		"Action":     "DescribeSiteMonitorData",
		"TaskId":     taskID,
		"MetricName": "Availability",
		"Period":     "60",
		"StartTime":  fmt.Sprintf("%d", now.Add(-2*time.Minute).Unix()*1000),
		"EndTime":    fmt.Sprintf("%d", now.Unix()*1000),
	}

	body, err := a.call(ctx, params)
	if err != nil {
		return aliyunTaskResult{}, false, err
	}

	var resp struct {
		Code    string `json:"Code"`
		Message string `json:"Message"`
		Data    struct {
			List []struct {
				Maximum      float64 `json:"Maximum"`
				Minimum      float64 `json:"Minimum"`
				Average      float64 `json:"Average"`
				Value        float64 `json:"Value"`
				Cnt          int     `json:"Cnt"`
				Province     string  `json:"Province"`
				Isp          string  `json:"Isp"`
				ErrorCode    string  `json:"ErrorCode"`
				ErrorInfo    string  `json:"ErrorInfo"`
			} `json:"List"`
		} `json:"Data"`
	}

	if err := json.Unmarshal(body, &resp); err != nil {
		return aliyunTaskResult{}, false, fmt.Errorf("parse DescribeSiteMonitorData: %w", err)
	}
	if resp.Code != "200" {
		return aliyunTaskResult{}, false, fmt.Errorf("DescribeSiteMonitorData error code=%s msg=%s", resp.Code, resp.Message)
	}
	if len(resp.Data.List) == 0 {
		// Task still running — no data point yet.
		return aliyunTaskResult{}, false, nil
	}

	item := resp.Data.List[0]
	result := aliyunTaskResult{
		Province:     item.Province,
		ResponseTime: item.Average,
		Availability: item.Value,
		ErrorInfo:    item.ErrorInfo,
	}
	return result, true, nil
}

// --------------------------------------------------------------------------
// Alibaba Cloud ACS v1 signing (HMAC-SHA1)
// --------------------------------------------------------------------------

// call signs and executes one Alibaba Cloud CloudMonitor RPC call.
// It adds the common authentication parameters, signs the request, and
// returns the raw response body.
func (a *AliyunSyntheticAgent) call(ctx context.Context, params map[string]string) ([]byte, error) {
	// Merge common authentication parameters.
	p := make(map[string]string, len(params)+8)
	for k, v := range params {
		p[k] = v
	}
	p["AccessKeyId"] = a.cfg.AccessKeyID
	p["SignatureMethod"] = "HMAC-SHA1"
	p["SignatureNonce"] = fmt.Sprintf("%d", time.Now().UnixNano())
	p["SignatureVersion"] = "1.0"
	p["Timestamp"] = time.Now().UTC().Format("2006-01-02T15:04:05Z")
	p["Format"] = "JSON"
	p["Version"] = "2019-01-01"

	// Build sorted query string for signing.
	keys := make([]string, 0, len(p))
	for k := range p {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	var parts []string
	for _, k := range keys {
		parts = append(parts, percentEncode(k)+"="+percentEncode(p[k]))
	}
	queryStr := strings.Join(parts, "&")

	// ACS v1 string to sign: "POST\n%2F\n" + percent-encoded sorted query.
	stringToSign := "POST\n%2F\n" + percentEncode(queryStr)

	// HMAC-SHA1 with key = AccessKeySecret + "&".
	//nolint:gosec // required by Alibaba Cloud ACS v1 spec
	mac := hmac.New(sha1.New, []byte(a.cfg.AccessKeySecret+"&"))
	mac.Write([]byte(stringToSign))
	sig := base64.StdEncoding.EncodeToString(mac.Sum(nil))

	// Build final query string including the signature.
	finalQuery := queryStr + "&Signature=" + percentEncode(sig)

	reqURL := a.baseURL + "?" + finalQuery
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL, nil)
	if err != nil {
		return nil, fmt.Errorf("build request: %w", err)
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	resp, err := a.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("http call: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
		return nil, fmt.Errorf("rate-limited or unavailable: HTTP %d", resp.StatusCode)
	}

	rawBody, err := io.ReadAll(io.LimitReader(resp.Body, 512*1024))
	if err != nil {
		return nil, fmt.Errorf("read response: %w", err)
	}
	return rawBody, nil
}

// percentEncode is RFC 3986 percent-encoding as required by Alibaba Cloud ACS v1.
// It encodes all characters except unreserved ones (A-Z a-z 0-9 - _ . ~).
func percentEncode(s string) string {
	var b strings.Builder
	for _, c := range []byte(s) {
		if isUnreserved(c) {
			b.WriteByte(c)
		} else {
			fmt.Fprintf(&b, "%%%02X", c)
		}
	}
	return b.String()
}

func isUnreserved(c byte) bool {
	return (c >= 'A' && c <= 'Z') ||
		(c >= 'a' && c <= 'z') ||
		(c >= '0' && c <= '9') ||
		c == '-' || c == '_' || c == '.' || c == '~'
}