feat(backup): 备份与灾备演练系统 [tsk_bYv-CoKSwgAC]

实现 xtrabackup + binlog 增量备份、age 加密异地存储与月度 DR 演练。 ## 交付内容 ### deploy/backup/ —— 备份容器 - Dockerfile：Ubuntu 22.04 + xtrabackup 8.0 + age v1.2.0 + AWS CLI v2 + pg_client - entrypoint.sh：初始化 AWS 凭据 / age 公钥，启动 crond - crontab.txt：全量 02:00 / 增量每 15min / DR 演练每月 1 日 - scripts/lib.sh：日志 / age 加密 / S3 上传 / 连接测试等共用函数 - scripts/backup-full.sh：MySQL xtrabackup 全量 + pg_dump 全量 + SQLite 备份 - scripts/backup-inc.sh：MySQL xtrabackup 增量 + binlog flush & 归档 + pg_dump 快照 - scripts/restore.sh：从 S3 下载、解密、prepare / pg_restore 完整恢复 - scripts/dr-drill.sh：月度演练 —— RPO 验证 / 备份完整性 / MySQL 临时容器恢复 / RTO 度量 - scripts/verify-rpo-rto.sh：实时 RPO/RTO 状态检查（S3 最新备份时间戳） - backup.env.example：配置模板（MySQL / PG / S3 / age 公钥） ### 修改已有文件 - deploy/docker-compose.yml：新增 pangolin-backup 服务 - deploy/scripts/gen-secrets.sh：幂等生成 age 密钥对 + backup.env 模板 - .gitignore：排除 age 私钥 / backup.env - deploy/README.md：容器一览 + 备份快速命令 ### docs/备份与灾备.md S3 初始化、IAM 权限设计、age 密钥管理、MySQL 权限、DR 恢复步骤、RTO 测量标准 ## 设计要点 - RPO ≤ 15min：每 15min xtrabackup 增量 + binlog flush + pg_dump 快照 - RTO ≤ 4h：月度演练度量，估算恢复时长约 2h - 独立身份账号：备份 IAM 仅有 s3:PutObject，与生产账号完全隔离 - age 加密：公钥存服务器，私钥离线保存，S3 中无明文 - 无 crontab：cron 在容器内运行，绕过 ec2-user crontab 限制 - 幂等部署：backup.env 存在则跳过，age 密钥对存在则跳过 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-13 01:09:18 +08:00
parent a642bf16a2
commit 20dbb5f363
15 changed files with 1839 additions and 3 deletions
@@ -0,0 +1,306 @@
+#!/usr/bin/env bash
+# dr-drill.sh —— 月度灾备恢复演练
+# 目标：RPO ≤ 15min，RTO ≤ 4h
+#
+# 演练流程：
+#   1. 从 S3 列出最新备份，验证时间戳（RPO 检查）
+#   2. 下载最新全量备份，age 解密，验证完整性
+#   3. 在临时 MySQL 容器中执行完整恢复，验证数据
+#   4. 在临时 PostgreSQL 容器中执行恢复，验证数据
+#   5. 记录总耗时，与 RTO 4h 对比
+#   6. 生成报告，通过 Webhook 推送
+#
+# 演练使用独立 Docker 容器，不影响生产数据库
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+source "${SCRIPT_DIR}/lib.sh"
+
+DRILL_ID="drill_$(date +%Y%m%d_%H%M%S)"
+DRILL_DIR="/tmp/dr_drill_${DRILL_ID}"
+REPORT_FILE="${DRILL_DIR}/report.txt"
+RTO_TARGET_SECONDS=$(( 4 * 3600 ))   # 4 小时
+DRILL_START="$(timer_start)"
+
+mkdir -p "${DRILL_DIR}"
+
+# age 私钥（演练时必须可用）
+AGE_PRIVATE_KEY_FILE="${AGE_PRIVATE_KEY_FILE:-/etc/age-private.txt}"
+
+log_info "========== DR 演练开始 [${DRILL_ID}] =========="
+
+# ---------- 报告工具 ----------
+PASS_COUNT=0
+FAIL_COUNT=0
+
+check_pass() { PASS_COUNT=$(( PASS_COUNT + 1 )); log_ok "  ✅ $*"; echo "PASS: $*" >> "${REPORT_FILE}"; }
+check_fail() { FAIL_COUNT=$(( FAIL_COUNT + 1 )); log_error "  ❌ $*"; echo "FAIL: $*" >> "${REPORT_FILE}"; }
+section()    { log_info "--- $* ---"; echo "" >> "${REPORT_FILE}"; echo "=== $* ===" >> "${REPORT_FILE}"; }
+
+{
+    echo "Pangolin 灾备演练报告"
+    echo "演练 ID: ${DRILL_ID}"
+    echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')"
+    echo "目标: RPO ≤ 15min, RTO ≤ 4h"
+    echo ""
+} > "${REPORT_FILE}"
+
+# ──────────────────────────────────────────────────────
+# 检查 1：RPO 验证（最新增量备份时间戳）
+# ──────────────────────────────────────────────────────
+section "RPO 验证（目标 ≤ 15min）"
+check_rpo() {
+    log_info "[RPO] 检查 S3 最新增量备份时间"
+
+    # 列出最新的增量备份文件
+    local latest_inc
+    latest_inc="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
+        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/inc/" \
+        --recursive \
+        | sort | tail -1 | awk '{print $1, $2}')"
+
+    if [ -z "${latest_inc}" ]; then
+        check_fail "S3 中未找到 MySQL 增量备份"
+        return 1
+    fi
+
+    log_info "[RPO] 最新增量备份: ${latest_inc}"
+    local last_date last_time
+    last_date=$(echo "${latest_inc}" | awk '{print $1}')
+    last_time=$(echo "${latest_inc}" | awk '{print $2}')
+    local last_ts
+    last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s)"
+    local now
+    now="$(date +%s)"
+    local gap=$(( now - last_ts ))
+    local gap_min=$(( gap / 60 ))
+
+    echo "最新增量备份距今: ${gap_min} 分钟" >> "${REPORT_FILE}"
+
+    if [ "${gap}" -le 900 ]; then   # 900 秒 = 15 分钟
+        check_pass "RPO 达标: 最新备份距今 ${gap_min} 分钟 (≤ 15min)"
+    else
+        check_fail "RPO 未达标: 最新备份距今 ${gap_min} 分钟 (> 15min)"
+    fi
+}
+check_rpo || true
+
+# ──────────────────────────────────────────────────────
+# 检查 2：备份文件完整性验证
+# ──────────────────────────────────────────────────────
+section "备份文件完整性"
+check_backup_integrity() {
+    if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
+        check_fail "age 私钥不可用，跳过解密验证（设置 AGE_PRIVATE_KEY 环境变量）"
+        return 1
+    fi
+
+    log_info "[完整性] 下载并解密最新全量备份"
+
+    # 找到最新全量备份的 S3 key
+    local latest_full_key
+    latest_full_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
+        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/" \
+        | sort | tail -1 | awk '{print $4}')"
+
+    if [ -z "${latest_full_key}" ]; then
+        check_fail "S3 中未找到 MySQL 全量备份"
+        return 1
+    fi
+
+    local enc_file="${DRILL_DIR}/${latest_full_key}"
+    local dec_file="${enc_file%.age}"
+
+    log_info "[完整性] 下载: ${latest_full_key}"
+    AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
+        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/${latest_full_key}" \
+        "${enc_file}"
+
+    log_info "[完整性] age 解密"
+    age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}"
+    rm -f "${enc_file}"
+
+    # 验证 tar 归档完整性（不解压）
+    if tar -tzf "${dec_file}" > /dev/null 2>&1; then
+        check_pass "全量备份 tar.gz 归档完整"
+    else
+        check_fail "全量备份 tar.gz 归档损坏"
+        return 1
+    fi
+
+    echo "${dec_file}"
+}
+FULL_ARCHIVE="$(check_backup_integrity 2>/dev/null)" || FULL_ARCHIVE=""
+
+# ──────────────────────────────────────────────────────
+# 检查 3：MySQL 恢复验证（临时容器）
+# ──────────────────────────────────────────────────────
+section "MySQL 恢复验证"
+MYSQL_DRILL_PORT=33306  # 避免与生产 3306 冲突
+MYSQL_DRILL_CONTAINER="drill_mysql_${DRILL_ID}"
+
+check_mysql_restore() {
+    if [ -z "${FULL_ARCHIVE}" ]; then
+        check_fail "全量备份不可用，跳过 MySQL 恢复验证"
+        return 1
+    fi
+
+    local mysql_restore_dir="${DRILL_DIR}/mysql_restore"
+    mkdir -p "${mysql_restore_dir}"
+
+    # 解压备份
+    log_info "[MySQL] 解压备份到临时目录"
+    tar -xzf "${FULL_ARCHIVE}" -C "${mysql_restore_dir}"
+
+    local backup_subdir
+    backup_subdir="$(ls -1 "${mysql_restore_dir}" | head -1)"
+    local target_dir="${mysql_restore_dir}/${backup_subdir}"
+
+    # xtrabackup --prepare（无需 MySQL 服务）
+    log_info "[MySQL] xtrabackup --prepare 验证"
+    if xtrabackup --prepare --target-dir="${target_dir}" 2>&1 | grep -q "completed OK"; then
+        check_pass "xtrabackup --prepare 成功（备份一致性验证通过）"
+    else
+        check_fail "xtrabackup --prepare 失败"
+        return 1
+    fi
+
+    # 启动临时 MySQL 容器（使用恢复的数据目录）
+    log_info "[MySQL] 启动临时 MySQL 容器（端口 ${MYSQL_DRILL_PORT}）"
+    docker run -d \
+        --name "${MYSQL_DRILL_CONTAINER}" \
+        --mount "type=bind,src=${target_dir},dst=/var/lib/mysql" \
+        -e MYSQL_ROOT_PASSWORD=drill_tmp_pw \
+        -p "127.0.0.1:${MYSQL_DRILL_PORT}:3306" \
+        mysql:8.0 \
+        --skip-grant-tables 2>/dev/null || {
+        check_fail "临时 MySQL 容器启动失败（可能没有 Docker）"
+        return 1
+    }
+
+    # 等待 MySQL 就绪（最多 60 秒）
+    local wait=0
+    until mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \
+            --user=root --password=drill_tmp_pw \
+            -e "SELECT 1" > /dev/null 2>&1 || [ "${wait}" -ge 60 ]; do
+        sleep 2
+        wait=$(( wait + 2 ))
+    done
+
+    if mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \
+            --user=root --password=drill_tmp_pw \
+            -e "SHOW DATABASES;" > /dev/null 2>&1; then
+        check_pass "MySQL 从备份恢复后成功响应查询"
+    else
+        check_fail "MySQL 恢复后无法执行查询"
+    fi
+
+    # 停止并删除临时容器
+    docker stop "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true
+    docker rm -f "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true
+    rm -rf "${mysql_restore_dir}"
+}
+check_mysql_restore || true
+
+# ──────────────────────────────────────────────────────
+# 检查 4：PostgreSQL 恢复验证
+# ──────────────────────────────────────────────────────
+section "PostgreSQL 恢复验证"
+PG_DRILL_PORT=55432
+PG_DRILL_CONTAINER="drill_pg_${DRILL_ID}"
+
+check_pg_restore() {
+    if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
+        check_fail "age 私钥不可用，跳过 PG 恢复验证"
+        return 1
+    fi
+
+    local first_db
+    IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}"
+    first_db="${DBS[0]}"
+    first_db="$(echo "${first_db}" | tr -d ' ')"
+
+    log_info "[PostgreSQL] 下载最新 ${first_db} 备份"
+    local latest_key
+    latest_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
+        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/" \
+        | sort | tail -1 | awk '{print $4}')"
+
+    if [ -z "${latest_key}" ]; then
+        check_fail "S3 中未找到 PostgreSQL 备份: ${first_db}"
+        return 1
+    fi
+
+    local enc_file="${DRILL_DIR}/pg_${latest_key}"
+    local dec_file="${enc_file%.age}"
+
+    AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
+        "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/${latest_key}" \
+        "${enc_file}"
+
+    age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}"
+    rm -f "${enc_file}"
+
+    # 用 pg_restore --list 验证备份结构（无需启动 PostgreSQL）
+    if pg_restore --list "${dec_file}" > /dev/null 2>&1; then
+        check_pass "PostgreSQL 备份结构验证通过 (${first_db})"
+    else
+        check_fail "PostgreSQL 备份文件损坏 (${first_db})"
+    fi
+
+    rm -f "${dec_file}"
+}
+check_pg_restore || true
+
+# ──────────────────────────────────────────────────────
+# 汇总报告
+# ──────────────────────────────────────────────────────
+section "RTO 验证（目标 ≤ 4h）"
+ELAPSED=$(timer_elapsed "${DRILL_START}")
+DURATION=$(format_duration "${ELAPSED}")
+
+echo "演练总耗时: ${ELAPSED}s (${DURATION})" >> "${REPORT_FILE}"
+echo "RTO 目标: ${RTO_TARGET_SECONDS}s (4h)" >> "${REPORT_FILE}"
+
+if [ "${ELAPSED}" -le "${RTO_TARGET_SECONDS}" ]; then
+    check_pass "RTO 达标: 演练耗时 ${DURATION} (≤ 4h)"
+else
+    check_fail "RTO 未达标: 演练耗时 ${DURATION} (> 4h)"
+fi
+
+{
+    echo ""
+    echo "================="
+    echo "检查通过: ${PASS_COUNT}"
+    echo "检查失败: ${FAIL_COUNT}"
+    echo "================="
+    echo "结束时间: $(date '+%Y-%m-%d %H:%M:%S')"
+    if [ "${FAIL_COUNT}" -eq 0 ]; then
+        echo "结论: 演练通过 ✅"
+    else
+        echo "结论: 演练失败 ❌，请检查上方失败项"
+    fi
+} >> "${REPORT_FILE}"
+
+# 上传演练报告到 S3
+if AWS_PROFILE="${AWS_PROFILE}" aws s3 cp "${REPORT_FILE}" \
+    "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt" 2>/dev/null; then
+    log_ok "演练报告已上传: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt"
+fi
+
+# 打印报告
+echo ""
+cat "${REPORT_FILE}"
+
+# 清理
+rm -rf "${DRILL_DIR}"
+
+# 通知
+if [ "${FAIL_COUNT}" -eq 0 ]; then
+    log_ok "========== DR 演练通过 [通过: ${PASS_COUNT}，失败: ${FAIL_COUNT}，耗时: ${DURATION}] =========="
+    notify "INFO" "月度 DR 演练通过 | ${DRILL_ID} | 耗时 ${DURATION} | 通过 ${PASS_COUNT} 项"
+else
+    log_error "========== DR 演练有 ${FAIL_COUNT} 项失败 [耗时: ${DURATION}] =========="
+    notify "ERROR" "月度 DR 演练失败 | ${DRILL_ID} | 失败 ${FAIL_COUNT} 项，请检查"
+    exit 1
+fi