pangolin/deploy/backup/scripts/dr-drill.sh

#!/usr/bin/env bash
# dr-drill.sh —— 月度灾备恢复演练
# 目标：RPO ≤ 15min，RTO ≤ 4h
#
# 演练流程：
#   1. 从 S3 列出最新备份，验证时间戳（RPO 检查）
#   2. 下载最新全量备份，age 解密，验证完整性
#   3. 在临时 MySQL 容器中执行完整恢复，验证数据
#   4. 在临时 PostgreSQL 容器中执行恢复，验证数据
#   5. 记录总耗时，与 RTO 4h 对比
#   6. 生成报告，通过 Webhook 推送
#
# 演练使用独立 Docker 容器，不影响生产数据库
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/lib.sh"

DRILL_ID="drill_$(date +%Y%m%d_%H%M%S)"
DRILL_DIR="/tmp/dr_drill_${DRILL_ID}"
REPORT_FILE="${DRILL_DIR}/report.txt"
RTO_TARGET_SECONDS=$(( 4 * 3600 ))   # 4 小时
DRILL_START="$(timer_start)"

mkdir -p "${DRILL_DIR}"

# age 私钥（演练时必须可用）
AGE_PRIVATE_KEY_FILE="${AGE_PRIVATE_KEY_FILE:-/etc/age-private.txt}"

log_info "========== DR 演练开始 [${DRILL_ID}] =========="

# ---------- 报告工具 ----------
PASS_COUNT=0
FAIL_COUNT=0

check_pass() { PASS_COUNT=$(( PASS_COUNT + 1 )); log_ok "  ✅ $*"; echo "PASS: $*" >> "${REPORT_FILE}"; }
check_fail() { FAIL_COUNT=$(( FAIL_COUNT + 1 )); log_error "  ❌ $*"; echo "FAIL: $*" >> "${REPORT_FILE}"; }
section()    { log_info "--- $* ---"; echo "" >> "${REPORT_FILE}"; echo "=== $* ===" >> "${REPORT_FILE}"; }

{
    echo "Pangolin 灾备演练报告"
    echo "演练 ID: ${DRILL_ID}"
    echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')"
    echo "目标: RPO ≤ 15min, RTO ≤ 4h"
    echo ""
} > "${REPORT_FILE}"

# ──────────────────────────────────────────────────────
# 检查 1：RPO 验证（最新增量备份时间戳）
# ──────────────────────────────────────────────────────
section "RPO 验证（目标 ≤ 15min）"
check_rpo() {
    log_info "[RPO] 检查 S3 最新增量备份时间"

    # 列出最新的增量备份文件
    local latest_inc
    latest_inc="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/inc/" \
        --recursive \
        | sort | tail -1 | awk '{print $1, $2}')"

    if [ -z "${latest_inc}" ]; then
        check_fail "S3 中未找到 MySQL 增量备份"
        return 1
    fi

    log_info "[RPO] 最新增量备份: ${latest_inc}"
    local last_date last_time
    last_date=$(echo "${latest_inc}" | awk '{print $1}')
    last_time=$(echo "${latest_inc}" | awk '{print $2}')
    local last_ts
    last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s)"
    local now
    now="$(date +%s)"
    local gap=$(( now - last_ts ))
    local gap_min=$(( gap / 60 ))

    echo "最新增量备份距今: ${gap_min} 分钟" >> "${REPORT_FILE}"

    if [ "${gap}" -le 900 ]; then   # 900 秒 = 15 分钟
        check_pass "RPO 达标: 最新备份距今 ${gap_min} 分钟 (≤ 15min)"
    else
        check_fail "RPO 未达标: 最新备份距今 ${gap_min} 分钟 (> 15min)"
    fi
}
check_rpo || true

# ──────────────────────────────────────────────────────
# 检查 2：备份文件完整性验证
# ──────────────────────────────────────────────────────
section "备份文件完整性"
check_backup_integrity() {
    if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
        check_fail "age 私钥不可用，跳过解密验证（设置 AGE_PRIVATE_KEY 环境变量）"
        return 1
    fi

    log_info "[完整性] 下载并解密最新全量备份"

    # 找到最新全量备份的 S3 key
    local latest_full_key
    latest_full_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/" \
        | sort | tail -1 | awk '{print $4}')"

    if [ -z "${latest_full_key}" ]; then
        check_fail "S3 中未找到 MySQL 全量备份"
        return 1
    fi

    local enc_file="${DRILL_DIR}/${latest_full_key}"
    local dec_file="${enc_file%.age}"

    log_info "[完整性] 下载: ${latest_full_key}"
    AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/${latest_full_key}" \
        "${enc_file}"

    log_info "[完整性] age 解密"
    age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}"
    rm -f "${enc_file}"

    # 验证 tar 归档完整性（不解压）
    if tar -tzf "${dec_file}" > /dev/null 2>&1; then
        check_pass "全量备份 tar.gz 归档完整"
    else
        check_fail "全量备份 tar.gz 归档损坏"
        return 1
    fi

    echo "${dec_file}"
}
FULL_ARCHIVE="$(check_backup_integrity 2>/dev/null)" || FULL_ARCHIVE=""

# ──────────────────────────────────────────────────────
# 检查 3：MySQL 恢复验证（临时容器）
# ──────────────────────────────────────────────────────
section "MySQL 恢复验证"
MYSQL_DRILL_PORT=33306  # 避免与生产 3306 冲突
MYSQL_DRILL_CONTAINER="drill_mysql_${DRILL_ID}"

check_mysql_restore() {
    if [ -z "${FULL_ARCHIVE}" ]; then
        check_fail "全量备份不可用，跳过 MySQL 恢复验证"
        return 1
    fi

    local mysql_restore_dir="${DRILL_DIR}/mysql_restore"
    mkdir -p "${mysql_restore_dir}"

    # 解压备份
    log_info "[MySQL] 解压备份到临时目录"
    tar -xzf "${FULL_ARCHIVE}" -C "${mysql_restore_dir}"

    local backup_subdir
    backup_subdir="$(ls -1 "${mysql_restore_dir}" | head -1)"
    local target_dir="${mysql_restore_dir}/${backup_subdir}"

    # xtrabackup --prepare（无需 MySQL 服务）
    log_info "[MySQL] xtrabackup --prepare 验证"
    if xtrabackup --prepare --target-dir="${target_dir}" 2>&1 | grep -q "completed OK"; then
        check_pass "xtrabackup --prepare 成功（备份一致性验证通过）"
    else
        check_fail "xtrabackup --prepare 失败"
        return 1
    fi

    # 启动临时 MySQL 容器（使用恢复的数据目录）
    log_info "[MySQL] 启动临时 MySQL 容器（端口 ${MYSQL_DRILL_PORT}）"
    docker run -d \
        --name "${MYSQL_DRILL_CONTAINER}" \
        --mount "type=bind,src=${target_dir},dst=/var/lib/mysql" \
        -e MYSQL_ROOT_PASSWORD=drill_tmp_pw \
        -p "127.0.0.1:${MYSQL_DRILL_PORT}:3306" \
        mysql:8.0 \
        --skip-grant-tables 2>/dev/null || {
        check_fail "临时 MySQL 容器启动失败（可能没有 Docker）"
        return 1
    }

    # 等待 MySQL 就绪（最多 60 秒）
    local wait=0
    until mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \
            --user=root --password=drill_tmp_pw \
            -e "SELECT 1" > /dev/null 2>&1 || [ "${wait}" -ge 60 ]; do
        sleep 2
        wait=$(( wait + 2 ))
    done

    if mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \
            --user=root --password=drill_tmp_pw \
            -e "SHOW DATABASES;" > /dev/null 2>&1; then
        check_pass "MySQL 从备份恢复后成功响应查询"
    else
        check_fail "MySQL 恢复后无法执行查询"
    fi

    # 停止并删除临时容器
    docker stop "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true
    docker rm -f "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true
    rm -rf "${mysql_restore_dir}"
}
check_mysql_restore || true

# ──────────────────────────────────────────────────────
# 检查 4：PostgreSQL 恢复验证
# ──────────────────────────────────────────────────────
section "PostgreSQL 恢复验证"
PG_DRILL_PORT=55432
PG_DRILL_CONTAINER="drill_pg_${DRILL_ID}"

check_pg_restore() {
    if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
        check_fail "age 私钥不可用，跳过 PG 恢复验证"
        return 1
    fi

    local first_db
    IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}"
    first_db="${DBS[0]}"
    first_db="$(echo "${first_db}" | tr -d ' ')"

    log_info "[PostgreSQL] 下载最新 ${first_db} 备份"
    local latest_key
    latest_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/" \
        | sort | tail -1 | awk '{print $4}')"

    if [ -z "${latest_key}" ]; then
        check_fail "S3 中未找到 PostgreSQL 备份: ${first_db}"
        return 1
    fi

    local enc_file="${DRILL_DIR}/pg_${latest_key}"
    local dec_file="${enc_file%.age}"

    AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/${latest_key}" \
        "${enc_file}"

    age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}"
    rm -f "${enc_file}"

    # 用 pg_restore --list 验证备份结构（无需启动 PostgreSQL）
    if pg_restore --list "${dec_file}" > /dev/null 2>&1; then
        check_pass "PostgreSQL 备份结构验证通过 (${first_db})"
    else
        check_fail "PostgreSQL 备份文件损坏 (${first_db})"
    fi

    rm -f "${dec_file}"
}
check_pg_restore || true

# ──────────────────────────────────────────────────────
# 汇总报告
# ──────────────────────────────────────────────────────
section "RTO 验证（目标 ≤ 4h）"
ELAPSED=$(timer_elapsed "${DRILL_START}")
DURATION=$(format_duration "${ELAPSED}")

echo "演练总耗时: ${ELAPSED}s (${DURATION})" >> "${REPORT_FILE}"
echo "RTO 目标: ${RTO_TARGET_SECONDS}s (4h)" >> "${REPORT_FILE}"

if [ "${ELAPSED}" -le "${RTO_TARGET_SECONDS}" ]; then
    check_pass "RTO 达标: 演练耗时 ${DURATION} (≤ 4h)"
else
    check_fail "RTO 未达标: 演练耗时 ${DURATION} (> 4h)"
fi

{
    echo ""
    echo "================="
    echo "检查通过: ${PASS_COUNT}"
    echo "检查失败: ${FAIL_COUNT}"
    echo "================="
    echo "结束时间: $(date '+%Y-%m-%d %H:%M:%S')"
    if [ "${FAIL_COUNT}" -eq 0 ]; then
        echo "结论: 演练通过 ✅"
    else
        echo "结论: 演练失败 ❌，请检查上方失败项"
    fi
} >> "${REPORT_FILE}"

# 上传演练报告到 S3
if AWS_PROFILE="${AWS_PROFILE}" aws s3 cp "${REPORT_FILE}" \
    "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt" 2>/dev/null; then
    log_ok "演练报告已上传: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt"
fi

# 打印报告
echo ""
cat "${REPORT_FILE}"

# 清理
rm -rf "${DRILL_DIR}"

# 通知
if [ "${FAIL_COUNT}" -eq 0 ]; then
    log_ok "========== DR 演练通过 [通过: ${PASS_COUNT}，失败: ${FAIL_COUNT}，耗时: ${DURATION}] =========="
    notify "INFO" "月度 DR 演练通过 | ${DRILL_ID} | 耗时 ${DURATION} | 通过 ${PASS_COUNT} 项"
else
    log_error "========== DR 演练有 ${FAIL_COUNT} 项失败 [耗时: ${DURATION}] =========="
    notify "ERROR" "月度 DR 演练失败 | ${DRILL_ID} | 失败 ${FAIL_COUNT} 项，请检查"
    exit 1
fi