pangolin/deploy/backup/scripts/verify-rpo-rto.sh

#!/usr/bin/env bash
# verify-rpo-rto.sh —— 实时 RPO/RTO 状态检查
# 用法：
#   verify-rpo-rto.sh           # 完整检查并打印结果
#   verify-rpo-rto.sh --status-only  # 仅打印摘要（容器启动时调用）
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/lib.sh"

STATUS_ONLY="${1:-}"

log_info "RPO/RTO 状态检查"

RPO_TARGET_SECONDS=900    # 15 分钟
RTO_TARGET_HOURS=4

PASS=0
WARN=0
FAIL=0

check() {
    local status="$1"
    local msg="$2"
    case "${status}" in
        pass) PASS=$(( PASS + 1 )); log_ok "  ✅ ${msg}" ;;
        warn) WARN=$(( WARN + 1 )); log_warn "  ⚠️  ${msg}" ;;
        fail) FAIL=$(( FAIL + 1 )); log_error "  ❌ ${msg}" ;;
    esac
}

# ──────────────────────────────────────────────────────
# 1. 检查 S3 最新增量备份的时间
# ──────────────────────────────────────────────────────
check_latest_backup_age() {
    local service="$1"
    local s3_prefix="$2"

    local latest
    latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_prefix}/" \
        --recursive 2>/dev/null \
        | sort | tail -1 | awk '{print $1, $2}' || echo "")"

    if [ -z "${latest}" ]; then
        check fail "${service}: S3 中无备份"
        return
    fi

    local last_date last_time
    last_date=$(echo "${latest}" | awk '{print $1}')
    last_time=$(echo "${latest}" | awk '{print $2}')

    local last_ts
    last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \
               || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \
               || echo 0)"
    local now
    now=$(date +%s)
    local gap=$(( now - last_ts ))
    local gap_min=$(( gap / 60 ))

    if [ "${gap}" -le "${RPO_TARGET_SECONDS}" ]; then
        check pass "${service}: 最新备份距今 ${gap_min}min (RPO ≤ 15min ✓)"
    elif [ "${gap}" -le $(( RPO_TARGET_SECONDS * 2 )) ]; then
        check warn "${service}: 最新备份距今 ${gap_min}min (接近 RPO 限制)"
    else
        check fail "${service}: 最新备份距今 ${gap_min}min (超出 RPO 15min 限制)"
    fi
}

# ──────────────────────────────────────────────────────
# 2. 检查 S3 最新全量备份时间（应为 < 25 小时）
# ──────────────────────────────────────────────────────
check_full_backup_freshness() {
    local service="$1"
    local s3_prefix="$2"

    local latest
    latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_prefix}/" \
        2>/dev/null | sort | tail -1 | awk '{print $1, $2}' || echo "")"

    if [ -z "${latest}" ]; then
        check fail "${service} 全量: S3 中无全量备份"
        return
    fi

    local last_date last_time
    last_date=$(echo "${latest}" | awk '{print $1}')
    last_time=$(echo "${latest}" | awk '{print $2}')

    local last_ts
    last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \
               || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \
               || echo 0)"
    local now
    now=$(date +%s)
    local gap_hours=$(( (now - last_ts) / 3600 ))

    if [ "${gap_hours}" -le 25 ]; then
        check pass "${service} 全量: 最近 ${gap_hours}h 内有全量备份"
    else
        check fail "${service} 全量: 上次全量备份距今 ${gap_hours}h (> 25h)"
    fi
}

# ──────────────────────────────────────────────────────
# 3. 检查 DR 演练报告（最新演练是否在 35 天内）
# ──────────────────────────────────────────────────────
check_dr_drill_freshness() {
    local latest
    latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/" \
        2>/dev/null | sort | tail -1 | awk '{print $1, $2}' || echo "")"

    if [ -z "${latest}" ]; then
        check warn "DR 演练: S3 中无演练报告（首次部署？）"
        return
    fi

    local last_date last_time
    last_date=$(echo "${latest}" | awk '{print $1}')
    last_time=$(echo "${latest}" | awk '{print $2}')

    local last_ts
    last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \
               || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \
               || echo 0)"
    local now
    now=$(date +%s)
    local gap_days=$(( (now - last_ts) / 86400 ))

    if [ "${gap_days}" -le 35 ]; then
        check pass "DR 演练: 上次演练距今 ${gap_days} 天"
    else
        check warn "DR 演练: 上次演练距今 ${gap_days} 天 (> 35 天，请检查)"
    fi
}

# ──────────────────────────────────────────────────────
# 执行检查
# ──────────────────────────────────────────────────────
check_latest_backup_age "MySQL-增量" "mysql/inc"
check_latest_backup_age "PG-增量" "postgres/inc"
check_full_backup_freshness "MySQL" "mysql/full"
check_full_backup_freshness "PostgreSQL" "postgres/full"
check_dr_drill_freshness

# ──────────────────────────────────────────────────────
# 汇总
# ──────────────────────────────────────────────────────
echo ""
log_info "RPO/RTO 检查汇总"
log_info "  通过: ${PASS}  警告: ${WARN}  失败: ${FAIL}"
log_info "  RPO 目标: ≤ 15min  |  RTO 目标: ≤ ${RTO_TARGET_HOURS}h"

if [ "${FAIL}" -gt 0 ]; then
    log_error "状态: 不达标 ❌"
    exit 1
elif [ "${WARN}" -gt 0 ]; then
    log_warn "状态: 警告 ⚠️"
    exit 0
else
    log_ok "状态: 达标 ✅"
    exit 0
fi