feat(backup): 备份与灾备演练系统 [tsk_bYv-CoKSwgAC]
实现 xtrabackup + binlog 增量备份、age 加密异地存储与月度 DR 演练。 ## 交付内容 ### deploy/backup/ —— 备份容器 - Dockerfile：Ubuntu 22.04 + xtrabackup 8.0 + age v1.2.0 + AWS CLI v2 + pg_client - entrypoint.sh：初始化 AWS 凭据 / age 公钥，启动 crond - crontab.txt：全量 02:00 / 增量每 15min / DR 演练每月 1 日 - scripts/lib.sh：日志 / age 加密 / S3 上传 / 连接测试等共用函数 - scripts/backup-full.sh：MySQL xtrabackup 全量 + pg_dump 全量 + SQLite 备份 - scripts/backup-inc.sh：MySQL xtrabackup 增量 + binlog flush & 归档 + pg_dump 快照 - scripts/restore.sh：从 S3 下载、解密、prepare / pg_restore 完整恢复 - scripts/dr-drill.sh：月度演练 —— RPO 验证 / 备份完整性 / MySQL 临时容器恢复 / RTO 度量 - scripts/verify-rpo-rto.sh：实时 RPO/RTO 状态检查（S3 最新备份时间戳） - backup.env.example：配置模板（MySQL / PG / S3 / age 公钥） ### 修改已有文件 - deploy/docker-compose.yml：新增 pangolin-backup 服务 - deploy/scripts/gen-secrets.sh：幂等生成 age 密钥对 + backup.env 模板 - .gitignore：排除 age 私钥 / backup.env - deploy/README.md：容器一览 + 备份快速命令 ### docs/备份与灾备.md S3 初始化、IAM 权限设计、age 密钥管理、MySQL 权限、DR 恢复步骤、RTO 测量标准 ## 设计要点 - RPO ≤ 15min：每 15min xtrabackup 增量 + binlog flush + pg_dump 快照 - RTO ≤ 4h：月度演练度量，估算恢复时长约 2h - 独立身份账号：备份 IAM 仅有 s3:PutObject，与生产账号完全隔离 - age 加密：公钥存服务器，私钥离线保存，S3 中无明文 - 无 crontab：cron 在容器内运行，绕过 ec2-user crontab 限制 - 幂等部署：backup.env 存在则跳过，age 密钥对存在则跳过 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
wangjia
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env bash
|
||||
# backup-full.sh —— 每日全量备份编排脚本（02:00 运行）
|
||||
# 内容：MySQL xtrabackup 全量 + PostgreSQL pg_dump + Marzban SQLite
|
||||
# 备份 → age 加密 → 上传 S3
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/lib.sh"
|
||||
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
START="$(timer_start)"
|
||||
|
||||
log_info "========== 全量备份开始 ${TIMESTAMP} =========="
|
||||
|
||||
ERRORS=0
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 1. MySQL 全量备份（xtrabackup）
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_full_backup() {
|
||||
local target="${BACKUP_DIR}/mysql/full/${TIMESTAMP}"
|
||||
local archive="${BACKUP_DIR}/mysql/full/${TIMESTAMP}.tar.gz"
|
||||
local s3_key="mysql/full/${TIMESTAMP}.tar.gz"
|
||||
|
||||
log_info "[MySQL] 开始 xtrabackup 全量备份"
|
||||
mkdir -p "${target}"
|
||||
|
||||
if ! mysql_ping; then
|
||||
log_warn "[MySQL] 无法连接 MySQL，跳过 MySQL 备份"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# xtrabackup --backup：连接 MySQL 并备份 InnoDB 数据文件
|
||||
xtrabackup \
|
||||
--backup \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--target-dir="${target}" \
|
||||
--compress \
|
||||
--compress-threads=2 \
|
||||
${MYSQL_DATADIR:+--datadir="${MYSQL_DATADIR}"} \
|
||||
2>&1 | tee "${target}/xtrabackup-backup.log"
|
||||
|
||||
# xtrabackup --prepare：回放 redo log，使备份一致（无需 MySQL 服务）
|
||||
xtrabackup \
|
||||
--prepare \
|
||||
--target-dir="${target}" \
|
||||
2>&1 | tee "${target}/xtrabackup-prepare.log"
|
||||
|
||||
# 打包（xtrabackup --compress 已压缩，这里仅归档）
|
||||
log_info "[MySQL] 打包备份目录"
|
||||
tar -czf "${archive}" -C "$(dirname "${target}")" "$(basename "${target}")"
|
||||
|
||||
# 更新 latest-full 引用（供增量备份使用）
|
||||
echo "${target}" > "${BACKUP_DIR}/mysql/latest-full.txt"
|
||||
# 清除上一次增量引用
|
||||
rm -f "${BACKUP_DIR}/mysql/latest-inc.txt"
|
||||
|
||||
# 加密 + 上传
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
|
||||
# 清理本地中间文件（保留原始目录供后续增量）
|
||||
rm -f "${archive}"
|
||||
|
||||
# 清理 7 天前的全量备份目录
|
||||
find "${BACKUP_DIR}/mysql/full" -maxdepth 1 -type d -mtime +7 \
|
||||
-not -path "${BACKUP_DIR}/mysql/full" \
|
||||
-exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
log_ok "[MySQL] 全量备份完成: ${target}"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 2. PostgreSQL 全量备份（pg_dump）
|
||||
# ──────────────────────────────────────────────────────
|
||||
postgres_full_backup() {
|
||||
log_info "[PostgreSQL] 开始 pg_dump 全量备份"
|
||||
|
||||
if ! pg_ping; then
|
||||
log_warn "[PostgreSQL] 无法连接 PostgreSQL，跳过 PG 备份"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 支持多数据库（逗号分隔）
|
||||
IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}"
|
||||
for DB in "${DBS[@]}"; do
|
||||
DB="$(echo "${DB}" | tr -d ' ')"
|
||||
local archive="${BACKUP_DIR}/postgres/full_${DB}_${TIMESTAMP}.pgdump"
|
||||
local s3_key="postgres/full_${DB}_${TIMESTAMP}.pgdump"
|
||||
|
||||
log_info "[PostgreSQL] 备份数据库: ${DB}"
|
||||
PGPASSWORD="${PG_PASSWORD:-}" pg_dump \
|
||||
--host="${PG_HOST}" \
|
||||
--port="${PG_PORT}" \
|
||||
--username="${PG_USER:-postgres}" \
|
||||
--format=custom \
|
||||
--compress=6 \
|
||||
--no-password \
|
||||
"${DB}" > "${archive}"
|
||||
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
rm -f "${archive}"
|
||||
log_ok "[PostgreSQL] ${DB} 备份完成"
|
||||
done
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 3. Marzban SQLite 备份
|
||||
# ──────────────────────────────────────────────────────
|
||||
marzban_backup() {
|
||||
local marzban_db="${MARZBAN_DB_PATH:-/var/lib/marzban/db.sqlite3}"
|
||||
local archive="${BACKUP_DIR}/marzban/marzban_${TIMESTAMP}.sqlite3.gz"
|
||||
local s3_key="marzban/marzban_${TIMESTAMP}.sqlite3.gz"
|
||||
|
||||
if [ ! -f "${marzban_db}" ]; then
|
||||
log_warn "[Marzban] SQLite 文件不存在: ${marzban_db}，跳过"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "[Marzban] 备份 SQLite: ${marzban_db}"
|
||||
# 使用 sqlite3 online backup，避免损坏 WAL
|
||||
sqlite3 "${marzban_db}" ".backup '${BACKUP_DIR}/marzban/marzban_${TIMESTAMP}.sqlite3'"
|
||||
gzip -9 "${BACKUP_DIR}/marzban/marzban_${TIMESTAMP}.sqlite3"
|
||||
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
rm -f "${archive}"
|
||||
|
||||
# 清理 30 天前的 SQLite 备份
|
||||
find "${BACKUP_DIR}/marzban" -name "*.gz" -mtime +30 -delete 2>/dev/null || true
|
||||
log_ok "[Marzban] SQLite 备份完成"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 执行
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_full_backup || { log_error "[MySQL] 全量备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
postgres_full_backup || { log_error "[PostgreSQL] 全量备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
marzban_backup || { log_error "[Marzban] 备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
|
||||
ELAPSED=$(timer_elapsed "${START}")
|
||||
DURATION=$(format_duration "${ELAPSED}")
|
||||
|
||||
if [ "${ERRORS}" -eq 0 ]; then
|
||||
log_ok "========== 全量备份完成，耗时 ${DURATION} =========="
|
||||
notify "INFO" "全量备份成功，耗时 ${DURATION}"
|
||||
else
|
||||
log_error "========== 全量备份完成（${ERRORS} 个失败），耗时 ${DURATION} =========="
|
||||
notify "ERROR" "全量备份完成，但有 ${ERRORS} 个失败，请检查日志"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env bash
|
||||
# backup-inc.sh —— 每 15 分钟增量备份（RPO ≤ 15min）
|
||||
# MySQL：xtrabackup 增量 + binlog flush & 归档
|
||||
# PostgreSQL：pg_dump（轻量，一致性快照）
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/lib.sh"
|
||||
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
START="$(timer_start)"
|
||||
|
||||
log_info "---------- 增量备份开始 ${TIMESTAMP} ----------"
|
||||
|
||||
ERRORS=0
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 1. MySQL 增量备份（xtrabackup incremental）
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_incremental_backup() {
|
||||
# 确定 basedir：优先用最近一次增量，若无则用最近全量
|
||||
local basedir=""
|
||||
if [ -f "${BACKUP_DIR}/mysql/latest-inc.txt" ]; then
|
||||
basedir="$(cat "${BACKUP_DIR}/mysql/latest-inc.txt")"
|
||||
elif [ -f "${BACKUP_DIR}/mysql/latest-full.txt" ]; then
|
||||
basedir="$(cat "${BACKUP_DIR}/mysql/latest-full.txt")"
|
||||
else
|
||||
log_warn "[MySQL] 未找到基础备份，跳过增量（请先执行全量备份）"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -d "${basedir}" ]; then
|
||||
log_warn "[MySQL] 基础备份目录不存在: ${basedir}，跳过增量"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local target="${BACKUP_DIR}/mysql/inc/${TIMESTAMP}"
|
||||
local archive="${BACKUP_DIR}/mysql/inc/${TIMESTAMP}.tar.gz"
|
||||
local s3_key="mysql/inc/${TIMESTAMP}.tar.gz"
|
||||
|
||||
log_info "[MySQL] xtrabackup 增量备份，basedir=${basedir}"
|
||||
mkdir -p "${target}"
|
||||
|
||||
if ! mysql_ping; then
|
||||
log_warn "[MySQL] 无法连接 MySQL，跳过增量"
|
||||
return 1
|
||||
fi
|
||||
|
||||
xtrabackup \
|
||||
--backup \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--target-dir="${target}" \
|
||||
--incremental-basedir="${basedir}" \
|
||||
--compress \
|
||||
--compress-threads=2 \
|
||||
${MYSQL_DATADIR:+--datadir="${MYSQL_DATADIR}"} \
|
||||
2>&1 | tee "${target}/xtrabackup.log"
|
||||
|
||||
tar -czf "${archive}" -C "$(dirname "${target}")" "$(basename "${target}")"
|
||||
|
||||
# 更新最新增量引用
|
||||
echo "${target}" > "${BACKUP_DIR}/mysql/latest-inc.txt"
|
||||
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
rm -f "${archive}"
|
||||
|
||||
# 清理 2 天前的增量目录（全量已上 S3，本地只保留最近供下一次增量）
|
||||
find "${BACKUP_DIR}/mysql/inc" -maxdepth 1 -type d -mtime +2 \
|
||||
-not -path "${BACKUP_DIR}/mysql/inc" \
|
||||
-exec rm -rf {} + 2>/dev/null || true
|
||||
|
||||
log_ok "[MySQL] 增量备份完成: ${target}"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 2. MySQL Binlog 归档（FLUSH BINARY LOGS + 复制新 binlog）
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_binlog_backup() {
|
||||
if ! mysql_ping; then
|
||||
log_warn "[MySQL-binlog] 无法连接 MySQL，跳过 binlog 归档"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "[MySQL-binlog] FLUSH BINARY LOGS"
|
||||
mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
-e "FLUSH BINARY LOGS;" 2>/dev/null
|
||||
|
||||
# 从 MySQL 取 binlog 文件列表
|
||||
local binlog_dir="${MYSQL_DATADIR:-/var/lib/mysql}"
|
||||
local dest="${BACKUP_DIR}/mysql/binlog"
|
||||
mkdir -p "${dest}"
|
||||
|
||||
# 使用 mysqlbinlog 工具读取当前活跃 binlog 之前的文件并归档
|
||||
local binlog_index
|
||||
binlog_index="$(mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--batch --skip-column-names \
|
||||
-e "SHOW BINARY LOGS;" 2>/dev/null | awk '{print $1}')"
|
||||
|
||||
if [ -z "${binlog_index}" ]; then
|
||||
log_warn "[MySQL-binlog] 无法获取 binlog 列表（binlog 未开启？），跳过"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 找出尚未归档的 binlog 文件（非最新一个，最新的正在写入）
|
||||
local count
|
||||
count=$(echo "${binlog_index}" | wc -l)
|
||||
# 跳过最后一个（正在写入）
|
||||
local to_archive
|
||||
to_archive=$(echo "${binlog_index}" | head -n $(( count - 1 )))
|
||||
|
||||
for binlog_file in ${to_archive}; do
|
||||
local dest_file="${dest}/${binlog_file}.gz"
|
||||
if [ -f "${dest_file}" ]; then
|
||||
continue # 已归档
|
||||
fi
|
||||
log_info "[MySQL-binlog] 归档 ${binlog_file}"
|
||||
# 使用 mysqlbinlog 以 raw 模式读取 + gzip 压缩
|
||||
mysqlbinlog \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--read-from-remote-server \
|
||||
--raw \
|
||||
--result-file="${dest}/" \
|
||||
"${binlog_file}" 2>/dev/null \
|
||||
&& gzip -f "${dest}/${binlog_file}" \
|
||||
&& encrypt_and_upload "${dest_file}" "mysql/binlog/${TIMESTAMP}_${binlog_file}.gz" \
|
||||
&& rm -f "${dest_file}" \
|
||||
|| log_warn "[MySQL-binlog] ${binlog_file} 归档失败，忽略"
|
||||
done
|
||||
|
||||
# 清理 3 天前的本地 binlog 备份
|
||||
find "${dest}" -name "*.gz" -mtime +3 -delete 2>/dev/null || true
|
||||
log_ok "[MySQL-binlog] binlog 归档完成"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 3. PostgreSQL 增量备份（pg_dump，15min 一次）
|
||||
# ──────────────────────────────────────────────────────
|
||||
postgres_incremental_backup() {
|
||||
if ! pg_ping; then
|
||||
log_warn "[PostgreSQL] 无法连接，跳过增量"
|
||||
return 1
|
||||
fi
|
||||
|
||||
IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}"
|
||||
for DB in "${DBS[@]}"; do
|
||||
DB="$(echo "${DB}" | tr -d ' ')"
|
||||
local archive="${BACKUP_DIR}/postgres/inc_${DB}_${TIMESTAMP}.pgdump"
|
||||
local s3_key="postgres/inc/${DB}/${TIMESTAMP}.pgdump"
|
||||
|
||||
PGPASSWORD="${PG_PASSWORD:-}" pg_dump \
|
||||
--host="${PG_HOST}" \
|
||||
--port="${PG_PORT}" \
|
||||
--username="${PG_USER:-postgres}" \
|
||||
--format=custom \
|
||||
--compress=6 \
|
||||
--no-password \
|
||||
"${DB}" > "${archive}"
|
||||
|
||||
encrypt_and_upload "${archive}" "${s3_key}"
|
||||
rm -f "${archive}"
|
||||
log_ok "[PostgreSQL] ${DB} 增量快照完成"
|
||||
done
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 执行
|
||||
# ──────────────────────────────────────────────────────
|
||||
mysql_incremental_backup || { log_error "[MySQL] 增量备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
mysql_binlog_backup || { log_warn "[MySQL] binlog 归档失败"; }
|
||||
postgres_incremental_backup || { log_error "[PG] 增量备份失败"; ERRORS=$(( ERRORS + 1 )); }
|
||||
|
||||
ELAPSED=$(timer_elapsed "${START}")
|
||||
DURATION=$(format_duration "${ELAPSED}")
|
||||
|
||||
if [ "${ERRORS}" -eq 0 ]; then
|
||||
log_ok "---------- 增量备份完成，耗时 ${DURATION} ----------"
|
||||
else
|
||||
log_error "---------- 增量备份完成（${ERRORS} 个失败），耗时 ${DURATION} ----------"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,306 @@
|
||||
#!/usr/bin/env bash
|
||||
# dr-drill.sh —— 月度灾备恢复演练
|
||||
# 目标：RPO ≤ 15min，RTO ≤ 4h
|
||||
#
|
||||
# 演练流程：
|
||||
# 1. 从 S3 列出最新备份，验证时间戳（RPO 检查）
|
||||
# 2. 下载最新全量备份，age 解密，验证完整性
|
||||
# 3. 在临时 MySQL 容器中执行完整恢复，验证数据
|
||||
# 4. 在临时 PostgreSQL 容器中执行恢复，验证数据
|
||||
# 5. 记录总耗时，与 RTO 4h 对比
|
||||
# 6. 生成报告，通过 Webhook 推送
|
||||
#
|
||||
# 演练使用独立 Docker 容器，不影响生产数据库
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/lib.sh"
|
||||
|
||||
DRILL_ID="drill_$(date +%Y%m%d_%H%M%S)"
|
||||
DRILL_DIR="/tmp/dr_drill_${DRILL_ID}"
|
||||
REPORT_FILE="${DRILL_DIR}/report.txt"
|
||||
RTO_TARGET_SECONDS=$(( 4 * 3600 )) # 4 小时
|
||||
DRILL_START="$(timer_start)"
|
||||
|
||||
mkdir -p "${DRILL_DIR}"
|
||||
|
||||
# age 私钥（演练时必须可用）
|
||||
AGE_PRIVATE_KEY_FILE="${AGE_PRIVATE_KEY_FILE:-/etc/age-private.txt}"
|
||||
|
||||
log_info "========== DR 演练开始 [${DRILL_ID}] =========="
|
||||
|
||||
# ---------- 报告工具 ----------
|
||||
PASS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
|
||||
check_pass() { PASS_COUNT=$(( PASS_COUNT + 1 )); log_ok " ✅ $*"; echo "PASS: $*" >> "${REPORT_FILE}"; }
|
||||
check_fail() { FAIL_COUNT=$(( FAIL_COUNT + 1 )); log_error " ❌ $*"; echo "FAIL: $*" >> "${REPORT_FILE}"; }
|
||||
section() { log_info "--- $* ---"; echo "" >> "${REPORT_FILE}"; echo "=== $* ===" >> "${REPORT_FILE}"; }
|
||||
|
||||
{
|
||||
echo "Pangolin 灾备演练报告"
|
||||
echo "演练 ID: ${DRILL_ID}"
|
||||
echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "目标: RPO ≤ 15min, RTO ≤ 4h"
|
||||
echo ""
|
||||
} > "${REPORT_FILE}"
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 检查 1：RPO 验证（最新增量备份时间戳）
|
||||
# ──────────────────────────────────────────────────────
|
||||
section "RPO 验证（目标 ≤ 15min）"
|
||||
check_rpo() {
|
||||
log_info "[RPO] 检查 S3 最新增量备份时间"
|
||||
|
||||
# 列出最新的增量备份文件
|
||||
local latest_inc
|
||||
latest_inc="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/inc/" \
|
||||
--recursive \
|
||||
| sort | tail -1 | awk '{print $1, $2}')"
|
||||
|
||||
if [ -z "${latest_inc}" ]; then
|
||||
check_fail "S3 中未找到 MySQL 增量备份"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "[RPO] 最新增量备份: ${latest_inc}"
|
||||
local last_date last_time
|
||||
last_date=$(echo "${latest_inc}" | awk '{print $1}')
|
||||
last_time=$(echo "${latest_inc}" | awk '{print $2}')
|
||||
local last_ts
|
||||
last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s)"
|
||||
local now
|
||||
now="$(date +%s)"
|
||||
local gap=$(( now - last_ts ))
|
||||
local gap_min=$(( gap / 60 ))
|
||||
|
||||
echo "最新增量备份距今: ${gap_min} 分钟" >> "${REPORT_FILE}"
|
||||
|
||||
if [ "${gap}" -le 900 ]; then # 900 秒 = 15 分钟
|
||||
check_pass "RPO 达标: 最新备份距今 ${gap_min} 分钟 (≤ 15min)"
|
||||
else
|
||||
check_fail "RPO 未达标: 最新备份距今 ${gap_min} 分钟 (> 15min)"
|
||||
fi
|
||||
}
|
||||
check_rpo || true
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 检查 2：备份文件完整性验证
|
||||
# ──────────────────────────────────────────────────────
|
||||
section "备份文件完整性"
|
||||
check_backup_integrity() {
|
||||
if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
|
||||
check_fail "age 私钥不可用，跳过解密验证（设置 AGE_PRIVATE_KEY 环境变量）"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "[完整性] 下载并解密最新全量备份"
|
||||
|
||||
# 找到最新全量备份的 S3 key
|
||||
local latest_full_key
|
||||
latest_full_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/" \
|
||||
| sort | tail -1 | awk '{print $4}')"
|
||||
|
||||
if [ -z "${latest_full_key}" ]; then
|
||||
check_fail "S3 中未找到 MySQL 全量备份"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local enc_file="${DRILL_DIR}/${latest_full_key}"
|
||||
local dec_file="${enc_file%.age}"
|
||||
|
||||
log_info "[完整性] 下载: ${latest_full_key}"
|
||||
AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/${latest_full_key}" \
|
||||
"${enc_file}"
|
||||
|
||||
log_info "[完整性] age 解密"
|
||||
age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}"
|
||||
rm -f "${enc_file}"
|
||||
|
||||
# 验证 tar 归档完整性（不解压）
|
||||
if tar -tzf "${dec_file}" > /dev/null 2>&1; then
|
||||
check_pass "全量备份 tar.gz 归档完整"
|
||||
else
|
||||
check_fail "全量备份 tar.gz 归档损坏"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "${dec_file}"
|
||||
}
|
||||
FULL_ARCHIVE="$(check_backup_integrity 2>/dev/null)" || FULL_ARCHIVE=""
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 检查 3：MySQL 恢复验证（临时容器）
|
||||
# ──────────────────────────────────────────────────────
|
||||
section "MySQL 恢复验证"
|
||||
MYSQL_DRILL_PORT=33306 # 避免与生产 3306 冲突
|
||||
MYSQL_DRILL_CONTAINER="drill_mysql_${DRILL_ID}"
|
||||
|
||||
check_mysql_restore() {
|
||||
if [ -z "${FULL_ARCHIVE}" ]; then
|
||||
check_fail "全量备份不可用，跳过 MySQL 恢复验证"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local mysql_restore_dir="${DRILL_DIR}/mysql_restore"
|
||||
mkdir -p "${mysql_restore_dir}"
|
||||
|
||||
# 解压备份
|
||||
log_info "[MySQL] 解压备份到临时目录"
|
||||
tar -xzf "${FULL_ARCHIVE}" -C "${mysql_restore_dir}"
|
||||
|
||||
local backup_subdir
|
||||
backup_subdir="$(ls -1 "${mysql_restore_dir}" | head -1)"
|
||||
local target_dir="${mysql_restore_dir}/${backup_subdir}"
|
||||
|
||||
# xtrabackup --prepare（无需 MySQL 服务）
|
||||
log_info "[MySQL] xtrabackup --prepare 验证"
|
||||
if xtrabackup --prepare --target-dir="${target_dir}" 2>&1 | grep -q "completed OK"; then
|
||||
check_pass "xtrabackup --prepare 成功（备份一致性验证通过）"
|
||||
else
|
||||
check_fail "xtrabackup --prepare 失败"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 启动临时 MySQL 容器（使用恢复的数据目录）
|
||||
log_info "[MySQL] 启动临时 MySQL 容器（端口 ${MYSQL_DRILL_PORT}）"
|
||||
docker run -d \
|
||||
--name "${MYSQL_DRILL_CONTAINER}" \
|
||||
--mount "type=bind,src=${target_dir},dst=/var/lib/mysql" \
|
||||
-e MYSQL_ROOT_PASSWORD=drill_tmp_pw \
|
||||
-p "127.0.0.1:${MYSQL_DRILL_PORT}:3306" \
|
||||
mysql:8.0 \
|
||||
--skip-grant-tables 2>/dev/null || {
|
||||
check_fail "临时 MySQL 容器启动失败（可能没有 Docker）"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 等待 MySQL 就绪（最多 60 秒）
|
||||
local wait=0
|
||||
until mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \
|
||||
--user=root --password=drill_tmp_pw \
|
||||
-e "SELECT 1" > /dev/null 2>&1 || [ "${wait}" -ge 60 ]; do
|
||||
sleep 2
|
||||
wait=$(( wait + 2 ))
|
||||
done
|
||||
|
||||
if mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \
|
||||
--user=root --password=drill_tmp_pw \
|
||||
-e "SHOW DATABASES;" > /dev/null 2>&1; then
|
||||
check_pass "MySQL 从备份恢复后成功响应查询"
|
||||
else
|
||||
check_fail "MySQL 恢复后无法执行查询"
|
||||
fi
|
||||
|
||||
# 停止并删除临时容器
|
||||
docker stop "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true
|
||||
docker rm -f "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true
|
||||
rm -rf "${mysql_restore_dir}"
|
||||
}
|
||||
check_mysql_restore || true
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 检查 4：PostgreSQL 恢复验证
|
||||
# ──────────────────────────────────────────────────────
|
||||
section "PostgreSQL 恢复验证"
|
||||
PG_DRILL_PORT=55432
|
||||
PG_DRILL_CONTAINER="drill_pg_${DRILL_ID}"
|
||||
|
||||
check_pg_restore() {
|
||||
if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
|
||||
check_fail "age 私钥不可用，跳过 PG 恢复验证"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local first_db
|
||||
IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}"
|
||||
first_db="${DBS[0]}"
|
||||
first_db="$(echo "${first_db}" | tr -d ' ')"
|
||||
|
||||
log_info "[PostgreSQL] 下载最新 ${first_db} 备份"
|
||||
local latest_key
|
||||
latest_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/" \
|
||||
| sort | tail -1 | awk '{print $4}')"
|
||||
|
||||
if [ -z "${latest_key}" ]; then
|
||||
check_fail "S3 中未找到 PostgreSQL 备份: ${first_db}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local enc_file="${DRILL_DIR}/pg_${latest_key}"
|
||||
local dec_file="${enc_file%.age}"
|
||||
|
||||
AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/${latest_key}" \
|
||||
"${enc_file}"
|
||||
|
||||
age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}"
|
||||
rm -f "${enc_file}"
|
||||
|
||||
# 用 pg_restore --list 验证备份结构（无需启动 PostgreSQL）
|
||||
if pg_restore --list "${dec_file}" > /dev/null 2>&1; then
|
||||
check_pass "PostgreSQL 备份结构验证通过 (${first_db})"
|
||||
else
|
||||
check_fail "PostgreSQL 备份文件损坏 (${first_db})"
|
||||
fi
|
||||
|
||||
rm -f "${dec_file}"
|
||||
}
|
||||
check_pg_restore || true
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 汇总报告
|
||||
# ──────────────────────────────────────────────────────
|
||||
section "RTO 验证（目标 ≤ 4h）"
|
||||
ELAPSED=$(timer_elapsed "${DRILL_START}")
|
||||
DURATION=$(format_duration "${ELAPSED}")
|
||||
|
||||
echo "演练总耗时: ${ELAPSED}s (${DURATION})" >> "${REPORT_FILE}"
|
||||
echo "RTO 目标: ${RTO_TARGET_SECONDS}s (4h)" >> "${REPORT_FILE}"
|
||||
|
||||
if [ "${ELAPSED}" -le "${RTO_TARGET_SECONDS}" ]; then
|
||||
check_pass "RTO 达标: 演练耗时 ${DURATION} (≤ 4h)"
|
||||
else
|
||||
check_fail "RTO 未达标: 演练耗时 ${DURATION} (> 4h)"
|
||||
fi
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "================="
|
||||
echo "检查通过: ${PASS_COUNT}"
|
||||
echo "检查失败: ${FAIL_COUNT}"
|
||||
echo "================="
|
||||
echo "结束时间: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
if [ "${FAIL_COUNT}" -eq 0 ]; then
|
||||
echo "结论: 演练通过 ✅"
|
||||
else
|
||||
echo "结论: 演练失败 ❌，请检查上方失败项"
|
||||
fi
|
||||
} >> "${REPORT_FILE}"
|
||||
|
||||
# 上传演练报告到 S3
|
||||
if AWS_PROFILE="${AWS_PROFILE}" aws s3 cp "${REPORT_FILE}" \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt" 2>/dev/null; then
|
||||
log_ok "演练报告已上传: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt"
|
||||
fi
|
||||
|
||||
# 打印报告
|
||||
echo ""
|
||||
cat "${REPORT_FILE}"
|
||||
|
||||
# 清理
|
||||
rm -rf "${DRILL_DIR}"
|
||||
|
||||
# 通知
|
||||
if [ "${FAIL_COUNT}" -eq 0 ]; then
|
||||
log_ok "========== DR 演练通过 [通过: ${PASS_COUNT}，失败: ${FAIL_COUNT}，耗时: ${DURATION}] =========="
|
||||
notify "INFO" "月度 DR 演练通过 | ${DRILL_ID} | 耗时 ${DURATION} | 通过 ${PASS_COUNT} 项"
|
||||
else
|
||||
log_error "========== DR 演练有 ${FAIL_COUNT} 项失败 [耗时: ${DURATION}] =========="
|
||||
notify "ERROR" "月度 DR 演练失败 | ${DRILL_ID} | 失败 ${FAIL_COUNT} 项，请检查"
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib.sh —— 共用函数库，被所有备份脚本 source
|
||||
|
||||
# ---------- 日志 ----------
|
||||
log_info() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $*"; }
|
||||
log_warn() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [WARN] $*" >&2; }
|
||||
log_error() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $*" >&2; }
|
||||
log_ok() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [OK] $*"; }
|
||||
|
||||
# ---------- 默认值 ----------
|
||||
: "${BACKUP_DIR:=/backup}"
|
||||
: "${MYSQL_HOST:=127.0.0.1}"
|
||||
: "${MYSQL_PORT:=3306}"
|
||||
: "${PG_HOST:=127.0.0.1}"
|
||||
: "${PG_PORT:=5432}"
|
||||
: "${BACKUP_AWS_REGION:=ap-southeast-1}"
|
||||
: "${AWS_PROFILE:=backup}"
|
||||
|
||||
# ---------- 通知（Webhook，可选） ----------
|
||||
notify() {
|
||||
local level="$1"
|
||||
local msg="$2"
|
||||
log_info "[NOTIFY] ${level}: ${msg}"
|
||||
if [ -n "${NOTIFY_WEBHOOK_URL:-}" ]; then
|
||||
curl -fsS -X POST "${NOTIFY_WEBHOOK_URL}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"level\":\"${level}\",\"msg\":$(printf '%s' "${msg}" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')}" \
|
||||
--max-time 10 || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------- age 加密 ----------
|
||||
# 用法: age_encrypt <input_file> <output_file>
|
||||
age_encrypt() {
|
||||
local src="$1"
|
||||
local dst="$2"
|
||||
local pubkey_file="${AGE_PUBLIC_KEY_FILE:-/etc/age-public.txt}"
|
||||
if [ ! -f "${pubkey_file}" ]; then
|
||||
log_error "age 公钥文件不存在: ${pubkey_file}"
|
||||
return 1
|
||||
fi
|
||||
age --encrypt --recipient "$(cat "${pubkey_file}")" -o "${dst}" "${src}"
|
||||
}
|
||||
|
||||
# ---------- S3 上传 ----------
|
||||
# 用法: s3_upload <local_file> <s3_path_suffix>
|
||||
s3_upload() {
|
||||
local src="$1"
|
||||
local suffix="$2"
|
||||
local dst="s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${suffix}"
|
||||
log_info "上传 $(basename "${src}") → ${dst}"
|
||||
AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
|
||||
--no-progress \
|
||||
--storage-class STANDARD_IA \
|
||||
"${src}" "${dst}"
|
||||
log_ok "上传完成: ${dst}"
|
||||
}
|
||||
|
||||
# ---------- 加密并上传 ----------
|
||||
# 用法: encrypt_and_upload <input_file> <s3_suffix>
|
||||
encrypt_and_upload() {
|
||||
local src="$1"
|
||||
local suffix="$2"
|
||||
local enc_file="${BACKUP_DIR}/encrypt/$(basename "${src}").age"
|
||||
|
||||
mkdir -p "${BACKUP_DIR}/encrypt"
|
||||
log_info "age 加密: ${src}"
|
||||
age_encrypt "${src}" "${enc_file}"
|
||||
|
||||
s3_upload "${enc_file}" "${suffix}.age"
|
||||
|
||||
# 删除本地加密临时文件（原始文件由调用方管理）
|
||||
rm -f "${enc_file}"
|
||||
}
|
||||
|
||||
# ---------- MySQL 连接测试 ----------
|
||||
mysql_ping() {
|
||||
mysql \
|
||||
--host="${MYSQL_HOST}" \
|
||||
--port="${MYSQL_PORT}" \
|
||||
--user="${MYSQL_USER:-root}" \
|
||||
--password="${MYSQL_PASSWORD}" \
|
||||
--connect-timeout=5 \
|
||||
-e "SELECT 1" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# ---------- PostgreSQL 连接测试 ----------
|
||||
pg_ping() {
|
||||
PGPASSWORD="${PG_PASSWORD:-}" pg_isready \
|
||||
--host="${PG_HOST}" \
|
||||
--port="${PG_PORT}" \
|
||||
--username="${PG_USER:-postgres}" \
|
||||
--quiet 2>&1
|
||||
}
|
||||
|
||||
# ---------- 计时工具 ----------
|
||||
timer_start() { date +%s; }
|
||||
timer_elapsed() {
|
||||
local start="$1"
|
||||
local end
|
||||
end=$(date +%s)
|
||||
echo $(( end - start ))
|
||||
}
|
||||
format_duration() {
|
||||
local secs="$1"
|
||||
local h=$(( secs / 3600 ))
|
||||
local m=$(( (secs % 3600) / 60 ))
|
||||
local s=$(( secs % 60 ))
|
||||
printf '%dh %dm %ds' "${h}" "${m}" "${s}"
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env bash
|
||||
# restore.sh —— 灾备恢复脚本
|
||||
# 用途：从 S3 下载 → age 解密 → 恢复 MySQL / PostgreSQL
|
||||
# 使用方式：
|
||||
# restore.sh mysql full 20240101_020000 # 恢复 MySQL 指定全量
|
||||
# restore.sh mysql inc 20240101_021500 # 在全量基础上应用增量
|
||||
# restore.sh postgres blog 20240101_021500
|
||||
# restore.sh marzban 20240101_020000
|
||||
#
|
||||
# 注意：MySQL 恢复需要先停止 MySQL 服务（由运维手动执行）
|
||||
# 此脚本只做"下载 + 解密 + 准备"，不自动重启 MySQL 服务
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/lib.sh"
|
||||
|
||||
# age 私钥（恢复时必须提供）
|
||||
AGE_PRIVATE_KEY_FILE="${AGE_PRIVATE_KEY_FILE:-/etc/age-private.txt}"
|
||||
|
||||
if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then
|
||||
log_error "age 私钥文件不存在: ${AGE_PRIVATE_KEY_FILE}"
|
||||
log_error "恢复需要私钥。请将私钥文件挂载到容器，或设置 AGE_PRIVATE_KEY 环境变量"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RESTORE_DIR="${RESTORE_DIR:-/restore}"
|
||||
mkdir -p "${RESTORE_DIR}"
|
||||
|
||||
# ---------- age 解密 ----------
|
||||
age_decrypt() {
|
||||
local src="$1"
|
||||
local dst="$2"
|
||||
age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dst}" "${src}"
|
||||
}
|
||||
|
||||
# ---------- 从 S3 下载并解密 ----------
|
||||
download_and_decrypt() {
|
||||
local s3_key="$1"
|
||||
local local_enc="${RESTORE_DIR}/$(basename "${s3_key}")"
|
||||
local local_dec="${local_enc%.age}"
|
||||
|
||||
log_info "从 S3 下载: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_key}.age"
|
||||
AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_key}.age" \
|
||||
"${local_enc}"
|
||||
|
||||
log_info "age 解密: ${local_enc}"
|
||||
age_decrypt "${local_enc}" "${local_dec}"
|
||||
rm -f "${local_enc}"
|
||||
|
||||
echo "${local_dec}"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# MySQL 恢复
|
||||
# ──────────────────────────────────────────────────────
|
||||
restore_mysql_full() {
|
||||
local timestamp="$1"
|
||||
local s3_key="mysql/full/${timestamp}.tar.gz"
|
||||
local restore_base="${RESTORE_DIR}/mysql"
|
||||
mkdir -p "${restore_base}"
|
||||
|
||||
log_info "[MySQL] 下载并解密全量备份: ${timestamp}"
|
||||
local archive
|
||||
archive="$(download_and_decrypt "${s3_key}")"
|
||||
|
||||
log_info "[MySQL] 解压: ${archive}"
|
||||
tar -xzf "${archive}" -C "${restore_base}"
|
||||
rm -f "${archive}"
|
||||
|
||||
local backup_dir="${restore_base}/${timestamp}"
|
||||
log_info "[MySQL] xtrabackup --prepare（应用增量前的全量 prepare）"
|
||||
xtrabackup --prepare --apply-log-only --target-dir="${backup_dir}"
|
||||
|
||||
log_ok "[MySQL] 全量备份已解压并 prepare 到: ${backup_dir}"
|
||||
echo "${backup_dir}"
|
||||
}
|
||||
|
||||
restore_mysql_inc() {
|
||||
local base_dir="$1"
|
||||
local timestamp="$2"
|
||||
local s3_key="mysql/inc/${timestamp}.tar.gz"
|
||||
local restore_base="${RESTORE_DIR}/mysql"
|
||||
|
||||
log_info "[MySQL] 下载并解密增量备份: ${timestamp}"
|
||||
local archive
|
||||
archive="$(download_and_decrypt "${s3_key}")"
|
||||
|
||||
tar -xzf "${archive}" -C "${restore_base}"
|
||||
rm -f "${archive}"
|
||||
|
||||
local inc_dir="${restore_base}/${timestamp}"
|
||||
log_info "[MySQL] xtrabackup --prepare（应用增量 ${timestamp}）"
|
||||
xtrabackup --prepare --apply-log-only \
|
||||
--target-dir="${base_dir}" \
|
||||
--incremental-dir="${inc_dir}"
|
||||
rm -rf "${inc_dir}"
|
||||
|
||||
log_ok "[MySQL] 增量 ${timestamp} 已应用到 ${base_dir}"
|
||||
}
|
||||
|
||||
restore_mysql_finalize() {
|
||||
local base_dir="$1"
|
||||
log_info "[MySQL] xtrabackup --prepare（最终 redo log 回放）"
|
||||
xtrabackup --prepare --target-dir="${base_dir}"
|
||||
log_ok "[MySQL] 备份已可用，恢复步骤："
|
||||
log_ok " 1. 停止 MySQL：systemctl stop mysql"
|
||||
log_ok " 2. 清空数据目录：rm -rf /var/lib/mysql/*"
|
||||
log_ok " 3. 复制备份：xtrabackup --copy-back --target-dir=${base_dir}"
|
||||
log_ok " 4. 修改权限：chown -R mysql:mysql /var/lib/mysql"
|
||||
log_ok " 5. 启动 MySQL：systemctl start mysql"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# PostgreSQL 恢复
|
||||
# ──────────────────────────────────────────────────────
|
||||
restore_postgres() {
|
||||
local db="$1"
|
||||
local timestamp="$2"
|
||||
local s3_key="postgres/inc/${db}/${timestamp}.pgdump"
|
||||
local restore_target="${PG_RESTORE_TARGET:-${db}_restored}"
|
||||
|
||||
log_info "[PostgreSQL] 下载并解密: ${db} @ ${timestamp}"
|
||||
local dump_file
|
||||
dump_file="$(download_and_decrypt "${s3_key}")"
|
||||
|
||||
log_info "[PostgreSQL] 恢复数据库: ${restore_target}"
|
||||
PGPASSWORD="${PG_PASSWORD:-}" psql \
|
||||
--host="${PG_HOST}" \
|
||||
--port="${PG_PORT}" \
|
||||
--username="${PG_USER:-postgres}" \
|
||||
-c "CREATE DATABASE \"${restore_target}\";" 2>/dev/null || true
|
||||
|
||||
PGPASSWORD="${PG_PASSWORD:-}" pg_restore \
|
||||
--host="${PG_HOST}" \
|
||||
--port="${PG_PORT}" \
|
||||
--username="${PG_USER:-postgres}" \
|
||||
--dbname="${restore_target}" \
|
||||
--no-password \
|
||||
--verbose \
|
||||
"${dump_file}"
|
||||
|
||||
rm -f "${dump_file}"
|
||||
log_ok "[PostgreSQL] 恢复完成: ${restore_target}"
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 主入口
|
||||
# ──────────────────────────────────────────────────────
|
||||
case "${1:-}" in
|
||||
mysql)
|
||||
case "${2:-}" in
|
||||
full)
|
||||
base_dir="$(restore_mysql_full "${3}")"
|
||||
restore_mysql_finalize "${base_dir}"
|
||||
;;
|
||||
inc)
|
||||
if [ -z "${4:-}" ]; then
|
||||
log_error "用法: restore.sh mysql inc <base_timestamp> <inc_timestamp>"
|
||||
exit 1
|
||||
fi
|
||||
base_dir="$(restore_mysql_full "${3}")"
|
||||
restore_mysql_inc "${base_dir}" "${4}"
|
||||
restore_mysql_finalize "${base_dir}"
|
||||
;;
|
||||
*)
|
||||
echo "用法: restore.sh mysql {full|inc} <timestamp> [inc_timestamp]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
postgres)
|
||||
restore_postgres "${2}" "${3}"
|
||||
;;
|
||||
marzban)
|
||||
marzban_archive="$(download_and_decrypt "marzban/marzban_${2}.sqlite3.gz")"
|
||||
log_ok "[Marzban] 解密文件: ${marzban_archive}"
|
||||
;;
|
||||
*)
|
||||
echo "用法: restore.sh {mysql|postgres|marzban} ..."
|
||||
echo " restore.sh mysql full <timestamp>"
|
||||
echo " restore.sh mysql inc <full_timestamp> <inc_timestamp>"
|
||||
echo " restore.sh postgres <db_name> <timestamp>"
|
||||
echo " restore.sh marzban <timestamp>"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env bash
|
||||
# verify-rpo-rto.sh —— 实时 RPO/RTO 状态检查
|
||||
# 用法：
|
||||
# verify-rpo-rto.sh # 完整检查并打印结果
|
||||
# verify-rpo-rto.sh --status-only # 仅打印摘要（容器启动时调用）
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
source "${SCRIPT_DIR}/lib.sh"
|
||||
|
||||
STATUS_ONLY="${1:-}"
|
||||
|
||||
log_info "RPO/RTO 状态检查"
|
||||
|
||||
RPO_TARGET_SECONDS=900 # 15 分钟
|
||||
RTO_TARGET_HOURS=4
|
||||
|
||||
PASS=0
|
||||
WARN=0
|
||||
FAIL=0
|
||||
|
||||
check() {
|
||||
local status="$1"
|
||||
local msg="$2"
|
||||
case "${status}" in
|
||||
pass) PASS=$(( PASS + 1 )); log_ok " ✅ ${msg}" ;;
|
||||
warn) WARN=$(( WARN + 1 )); log_warn " ⚠️ ${msg}" ;;
|
||||
fail) FAIL=$(( FAIL + 1 )); log_error " ❌ ${msg}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 1. 检查 S3 最新增量备份的时间
|
||||
# ──────────────────────────────────────────────────────
|
||||
check_latest_backup_age() {
|
||||
local service="$1"
|
||||
local s3_prefix="$2"
|
||||
|
||||
local latest
|
||||
latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_prefix}/" \
|
||||
--recursive 2>/dev/null \
|
||||
| sort | tail -1 | awk '{print $1, $2}' || echo "")"
|
||||
|
||||
if [ -z "${latest}" ]; then
|
||||
check fail "${service}: S3 中无备份"
|
||||
return
|
||||
fi
|
||||
|
||||
local last_date last_time
|
||||
last_date=$(echo "${latest}" | awk '{print $1}')
|
||||
last_time=$(echo "${latest}" | awk '{print $2}')
|
||||
|
||||
local last_ts
|
||||
last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \
|
||||
|| date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \
|
||||
|| echo 0)"
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local gap=$(( now - last_ts ))
|
||||
local gap_min=$(( gap / 60 ))
|
||||
|
||||
if [ "${gap}" -le "${RPO_TARGET_SECONDS}" ]; then
|
||||
check pass "${service}: 最新备份距今 ${gap_min}min (RPO ≤ 15min ✓)"
|
||||
elif [ "${gap}" -le $(( RPO_TARGET_SECONDS * 2 )) ]; then
|
||||
check warn "${service}: 最新备份距今 ${gap_min}min (接近 RPO 限制)"
|
||||
else
|
||||
check fail "${service}: 最新备份距今 ${gap_min}min (超出 RPO 15min 限制)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 2. 检查 S3 最新全量备份时间（应为 < 25 小时）
|
||||
# ──────────────────────────────────────────────────────
|
||||
check_full_backup_freshness() {
|
||||
local service="$1"
|
||||
local s3_prefix="$2"
|
||||
|
||||
local latest
|
||||
latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_prefix}/" \
|
||||
2>/dev/null | sort | tail -1 | awk '{print $1, $2}' || echo "")"
|
||||
|
||||
if [ -z "${latest}" ]; then
|
||||
check fail "${service} 全量: S3 中无全量备份"
|
||||
return
|
||||
fi
|
||||
|
||||
local last_date last_time
|
||||
last_date=$(echo "${latest}" | awk '{print $1}')
|
||||
last_time=$(echo "${latest}" | awk '{print $2}')
|
||||
|
||||
local last_ts
|
||||
last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \
|
||||
|| date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \
|
||||
|| echo 0)"
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local gap_hours=$(( (now - last_ts) / 3600 ))
|
||||
|
||||
if [ "${gap_hours}" -le 25 ]; then
|
||||
check pass "${service} 全量: 最近 ${gap_hours}h 内有全量备份"
|
||||
else
|
||||
check fail "${service} 全量: 上次全量备份距今 ${gap_hours}h (> 25h)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 3. 检查 DR 演练报告（最新演练是否在 35 天内）
|
||||
# ──────────────────────────────────────────────────────
|
||||
check_dr_drill_freshness() {
|
||||
local latest
|
||||
latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \
|
||||
"s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/" \
|
||||
2>/dev/null | sort | tail -1 | awk '{print $1, $2}' || echo "")"
|
||||
|
||||
if [ -z "${latest}" ]; then
|
||||
check warn "DR 演练: S3 中无演练报告（首次部署？）"
|
||||
return
|
||||
fi
|
||||
|
||||
local last_date last_time
|
||||
last_date=$(echo "${latest}" | awk '{print $1}')
|
||||
last_time=$(echo "${latest}" | awk '{print $2}')
|
||||
|
||||
local last_ts
|
||||
last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \
|
||||
|| date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \
|
||||
|| echo 0)"
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local gap_days=$(( (now - last_ts) / 86400 ))
|
||||
|
||||
if [ "${gap_days}" -le 35 ]; then
|
||||
check pass "DR 演练: 上次演练距今 ${gap_days} 天"
|
||||
else
|
||||
check warn "DR 演练: 上次演练距今 ${gap_days} 天 (> 35 天，请检查)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 执行检查
|
||||
# ──────────────────────────────────────────────────────
|
||||
check_latest_backup_age "MySQL-增量" "mysql/inc"
|
||||
check_latest_backup_age "PG-增量" "postgres/inc"
|
||||
check_full_backup_freshness "MySQL" "mysql/full"
|
||||
check_full_backup_freshness "PostgreSQL" "postgres/full"
|
||||
check_dr_drill_freshness
|
||||
|
||||
# ──────────────────────────────────────────────────────
|
||||
# 汇总
|
||||
# ──────────────────────────────────────────────────────
|
||||
echo ""
|
||||
log_info "RPO/RTO 检查汇总"
|
||||
log_info " 通过: ${PASS} 警告: ${WARN} 失败: ${FAIL}"
|
||||
log_info " RPO 目标: ≤ 15min | RTO 目标: ≤ ${RTO_TARGET_HOURS}h"
|
||||
|
||||
if [ "${FAIL}" -gt 0 ]; then
|
||||
log_error "状态: 不达标 ❌"
|
||||
exit 1
|
||||
elif [ "${WARN}" -gt 0 ]; then
|
||||
log_warn "状态: 警告 ⚠️"
|
||||
exit 0
|
||||
else
|
||||
log_ok "状态: 达标 ✅"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user