feat(backup): 备份与灾备演练系统 [tsk_bYv-CoKSwgAC]
实现 xtrabackup + binlog 增量备份、age 加密异地存储与月度 DR 演练。 ## 交付内容 ### deploy/backup/ —— 备份容器 - Dockerfile：Ubuntu 22.04 + xtrabackup 8.0 + age v1.2.0 + AWS CLI v2 + pg_client - entrypoint.sh：初始化 AWS 凭据 / age 公钥，启动 crond - crontab.txt：全量 02:00 / 增量每 15min / DR 演练每月 1 日 - scripts/lib.sh：日志 / age 加密 / S3 上传 / 连接测试等共用函数 - scripts/backup-full.sh：MySQL xtrabackup 全量 + pg_dump 全量 + SQLite 备份 - scripts/backup-inc.sh：MySQL xtrabackup 增量 + binlog flush & 归档 + pg_dump 快照 - scripts/restore.sh：从 S3 下载、解密、prepare / pg_restore 完整恢复 - scripts/dr-drill.sh：月度演练 —— RPO 验证 / 备份完整性 / MySQL 临时容器恢复 / RTO 度量 - scripts/verify-rpo-rto.sh：实时 RPO/RTO 状态检查（S3 最新备份时间戳） - backup.env.example：配置模板（MySQL / PG / S3 / age 公钥） ### 修改已有文件 - deploy/docker-compose.yml：新增 pangolin-backup 服务 - deploy/scripts/gen-secrets.sh：幂等生成 age 密钥对 + backup.env 模板 - .gitignore：排除 age 私钥 / backup.env - deploy/README.md：容器一览 + 备份快速命令 ### docs/备份与灾备.md S3 初始化、IAM 权限设计、age 密钥管理、MySQL 权限、DR 恢复步骤、RTO 测量标准 ## 设计要点 - RPO ≤ 15min：每 15min xtrabackup 增量 + binlog flush + pg_dump 快照 - RTO ≤ 4h：月度演练度量，估算恢复时长约 2h - 独立身份账号：备份 IAM 仅有 s3:PutObject，与生产账号完全隔离 - age 加密：公钥存服务器，私钥离线保存，S3 中无明文 - 无 crontab：cron 在容器内运行，绕过 ec2-user crontab 限制 - 幂等部署：backup.env 存在则跳过，age 密钥对存在则跳过 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env bash
|
||||
# backup-inc.sh —— 每 15 分钟增量备份（RPO ≤ 15min）
|
||||
# MySQL：xtrabackup 增量 + binlog flush & 归档
|
||||
# PostgreSQL：pg_dump（轻量，一致性快照）
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/lib.sh"
|
||||
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
START="$(timer_start)"
|
||||
|
||||
log_info "---------- 增量备份开始 ${TIMESTAMP} ----------"
|
||||
|
||||
ERRORS=0
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 1. MySQL 增量备份（xtrabackup incremental）
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_incremental_backup() {
|
||||
# 确定 basedir：优先用最近一次增量，若无则用最近全量
|
||||
local basedir=""
|
||||
if [ -f "${BACKUP_DIR}/mysql/latest-inc.txt" ]; then
|
||||
basedir="$(cat "${BACKUP_DIR}/mysql/latest-inc.txt")"
|
||||
elif [ -f "${BACKUP_DIR}/mysql/latest-full.txt" ]; then
|
||||
basedir="$(cat "${BACKUP_DIR}/mysql/latest-full.txt")"
|
||||
else
|
||||
log_warn "[MySQL] 未找到基础备份，跳过增量（请先执行全量备份）"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -d "${basedir}" ]; then
|
||||
log_warn "[MySQL] 基础备份目录不存在: ${basedir}，跳过增量"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local target="${BACKUP_DIR}/mysql/inc/${TIMESTAMP}"
|
||||
local archive="${BACKUP_DIR}/mysql/inc/${TIMESTAMP}.tar.gz"
|
||||
local s3_key="mysql/inc/${TIMESTAMP}.tar.gz"
|
||||
|
||||
log_info "[MySQL] xtrabackup 增量备份，basedir=${basedir}"
|
||||
mkdir -p "${target}"
|
||||
|
||||
if ! mysql_ping; then
|
||||
log_warn "[MySQL] 无法连接 MySQL，跳过增量"
|
||||
return 1
|
||||
fi
|
||||
|
||||
xtrabackup \
|
||||
--backup \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--target-dir="${target}" \
|
||||
--incremental-basedir="${basedir}" \
|
||||
--compress \
|
||||
--compress-threads=2 \
|
||||
${MYSQL_DATADIR:+--datadir="${MYSQL_DATADIR}"} \
|
||||
2>&1 | tee "${target}/xtrabackup.log"
|
||||
|
||||
tar -czf "${archive}" -C "$(dirname "${target}")" "$(basename "${target}")"
|
||||
|
||||
# 更新最新增量引用
|
||||
echo "${target}" > "${BACKUP_DIR}/mysql/latest-inc.txt"
|
||||
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
rm -f "${archive}"
|
||||
|
||||
# 清理 2 天前的增量目录（全量已上 S3，本地只保留最近供下一次增量）
|
||||
find "${BACKUP_DIR}/mysql/inc" -maxdepth 1 -type d -mtime +2 \
|
||||
-not -path "${BACKUP_DIR}/mysql/inc" \
|
||||
-exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
log_ok "[MySQL] 增量备份完成: ${target}"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 2. MySQL Binlog 归档（FLUSH BINARY LOGS + 复制新 binlog）
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_binlog_backup() {
|
||||
if ! mysql_ping; then
|
||||
log_warn "[MySQL-binlog] 无法连接 MySQL，跳过 binlog 归档"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "[MySQL-binlog] FLUSH BINARY LOGS"
|
||||
mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
-e "FLUSH BINARY LOGS;" 2>/dev/null
|
||||
|
||||
# 从 MySQL 取 binlog 文件列表
|
||||
local binlog_dir="${MYSQL_DATADIR:-/var/lib/mysql}"
|
||||
local dest="${BACKUP_DIR}/mysql/binlog"
|
||||
mkdir -p "${dest}"
|
||||
|
||||
# 使用 mysqlbinlog 工具读取当前活跃 binlog 之前的文件并归档
|
||||
local binlog_index
|
||||
binlog_index="$(mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--batch --skip-column-names \
|
||||
-e "SHOW BINARY LOGS;" 2>/dev/null | awk '{print $1}')"
|
||||
|
||||
if [ -z "${binlog_index}" ]; then
|
||||
log_warn "[MySQL-binlog] 无法获取 binlog 列表（binlog 未开启？），跳过"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 找出尚未归档的 binlog 文件（非最新一个，最新的正在写入）
|
||||
local count
|
||||
count=$(echo "${binlog_index}" | wc -l)
|
||||
# 跳过最后一个（正在写入）
|
||||
local to_archive
|
||||
to_archive=$(echo "${binlog_index}" | head -n $(( count - 1 )))
|
||||
|
||||
for binlog_file in ${to_archive}; do
|
||||
local dest_file="${dest}/${binlog_file}.gz"
|
||||
if [ -f "${dest_file}" ]; then
|
||||
continue # 已归档
|
||||
fi
|
||||
log_info "[MySQL-binlog] 归档 ${binlog_file}"
|
||||
# 使用 mysqlbinlog 以 raw 模式读取 + gzip 压缩
|
||||
mysqlbinlog \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--read-from-remote-server \
|
||||
--raw \
|
||||
--result-file="${dest}/" \
|
||||
"${binlog_file}" 2>/dev/null \
|
||||
&& gzip -f "${dest}/${binlog_file}" \
|
||||
&& encrypt_and_upload "${dest_file}" "mysql/binlog/${TIMESTAMP}_${binlog_file}.gz" \
|
||||
&& rm -f "${dest_file}" \
|
||||
|| log_warn "[MySQL-binlog] ${binlog_file} 归档失败，忽略"
|
||||
done
|
||||
|
||||
# 清理 3 天前的本地 binlog 备份
|
||||
find "${dest}" -name "*.gz" -mtime +3 -delete 2>/dev/null || true
|
||||
log_ok "[MySQL-binlog] binlog 归档完成"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 3. PostgreSQL 增量备份（pg_dump，15min 一次）
|
||||
# ──────────────────────────────────────────────────────
|
||||
postgres_incremental_backup() {
|
||||
if ! pg_ping; then
|
||||
log_warn "[PostgreSQL] 无法连接，跳过增量"
|
||||
return 1
|
||||
fi
|
||||
|
||||
IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}"
|
||||
for DB in "${DBS[@]}"; do
|
||||
DB="$(echo "${DB}" | tr -d ' ')"
|
||||
local archive="${BACKUP_DIR}/postgres/inc_${DB}_${TIMESTAMP}.pgdump"
|
||||
local s3_key="postgres/inc/${DB}/${TIMESTAMP}.pgdump"
|
||||
|
||||
PGPASSWORD="${PG_PASSWORD:-}" pg_dump \
|
||||
--host="${PG_HOST}" \
|
||||
--port="${PG_PORT}" \
|
||||
--username="${PG_USER:-postgres}" \
|
||||
--format=custom \
|
||||
--compress=6 \
|
||||
--no-password \
|
||||
"${DB}" > "${archive}"
|
||||
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
rm -f "${archive}"
|
||||
log_ok "[PostgreSQL] ${DB} 增量快照完成"
|
||||
done
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 执行
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_incremental_backup || { log_error "[MySQL] 增量备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
mysql_binlog_backup || { log_warn "[MySQL] binlog 归档失败"; }
|
||||
postgres_incremental_backup || { log_error "[PG] 增量备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
|
||||
ELAPSED=$(timer_elapsed "${START}")
|
||||
DURATION=$(format_duration "${ELAPSED}")
|
||||
|
||||
if [ "${ERRORS}" -eq 0 ]; then
|
||||
log_ok "---------- 增量备份完成，耗时 ${DURATION} ----------"
|
||||
else
|
||||
log_error "---------- 增量备份完成（${ERRORS} 个失败），耗时 ${DURATION} ----------"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user