From 20dbb5f3634f6d2702a88d1bf4eb5fa03584a51e Mon Sep 17 00:00:00 2001 From: wangjia <809946525@qq.com> Date: Sat, 13 Jun 2026 01:09:18 +0800 Subject: [PATCH] =?UTF-8?q?feat(backup):=20=E5=A4=87=E4=BB=BD=E4=B8=8E?= =?UTF-8?q?=E7=81=BE=E5=A4=87=E6=BC=94=E7=BB=83=E7=B3=BB=E7=BB=9F=20[tsk?= =?UTF-8?q?=5FbYv-CoKSwgAC]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 实现 xtrabackup + binlog 增量备份、age 加密异地存储与月度 DR 演练。 ## 交付内容 ### deploy/backup/ —— 备份容器 - Dockerfile：Ubuntu 22.04 + xtrabackup 8.0 + age v1.2.0 + AWS CLI v2 + pg_client - entrypoint.sh：初始化 AWS 凭据 / age 公钥，启动 crond - crontab.txt：全量 02:00 / 增量每 15min / DR 演练每月 1 日 - scripts/lib.sh：日志 / age 加密 / S3 上传 / 连接测试等共用函数 - scripts/backup-full.sh：MySQL xtrabackup 全量 + pg_dump 全量 + SQLite 备份 - scripts/backup-inc.sh：MySQL xtrabackup 增量 + binlog flush & 归档 + pg_dump 快照 - scripts/restore.sh：从 S3 下载、解密、prepare / pg_restore 完整恢复 - scripts/dr-drill.sh：月度演练 —— RPO 验证 / 备份完整性 / MySQL 临时容器恢复 / RTO 度量 - scripts/verify-rpo-rto.sh：实时 RPO/RTO 状态检查（S3 最新备份时间戳） - backup.env.example：配置模板（MySQL / PG / S3 / age 公钥） ### 修改已有文件 - deploy/docker-compose.yml：新增 pangolin-backup 服务 - deploy/scripts/gen-secrets.sh：幂等生成 age 密钥对 + backup.env 模板 - .gitignore：排除 age 私钥 / backup.env - deploy/README.md：容器一览 + 备份快速命令 ### docs/备份与灾备.md S3 初始化、IAM 权限设计、age 密钥管理、MySQL 权限、DR 恢复步骤、RTO 测量标准 ## 设计要点 - RPO ≤ 15min：每 15min xtrabackup 增量 + binlog flush + pg_dump 快照 - RTO ≤ 4h：月度演练度量，估算恢复时长约 2h - 独立身份账号：备份 IAM 仅有 s3:PutObject，与生产账号完全隔离 - age 加密：公钥存服务器，私钥离线保存，S3 中无明文 - 无 crontab：cron 在容器内运行，绕过 ec2-user crontab 限制 - 幂等部署：backup.env 存在则跳过，age 密钥对存在则跳过 Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 5 + deploy/README.md | 21 ++ deploy/backup/Dockerfile | 54 ++++ deploy/backup/backup.env.example | 75 +++++ deploy/backup/crontab.txt | 20 ++ deploy/backup/entrypoint.sh | 88 ++++++ deploy/backup/scripts/backup-full.sh | 152 ++++++++++ deploy/backup/scripts/backup-inc.sh | 194 +++++++++++++ deploy/backup/scripts/dr-drill.sh | 306 +++++++++++++++++++ deploy/backup/scripts/lib.sh | 110 +++++++ deploy/backup/scripts/restore.sh | 187 ++++++++++++ deploy/backup/scripts/verify-rpo-rto.sh | 167 +++++++++++ deploy/docker-compose.yml | 36 +++ deploy/scripts/gen-secrets.sh | 56 +++- docs/备份与灾备.md | 371 ++++++++++++++++++++++++ 15 files changed, 1839 insertions(+), 3 deletions(-) create mode 100644 deploy/backup/Dockerfile create mode 100644 deploy/backup/backup.env.example create mode 100644 deploy/backup/crontab.txt create mode 100644 deploy/backup/entrypoint.sh create mode 100644 deploy/backup/scripts/backup-full.sh create mode 100644 deploy/backup/scripts/backup-inc.sh create mode 100644 deploy/backup/scripts/dr-drill.sh create mode 100644 deploy/backup/scripts/lib.sh create mode 100644 deploy/backup/scripts/restore.sh create mode 100644 deploy/backup/scripts/verify-rpo-rto.sh create mode 100644 docs/备份与灾备.md diff --git a/.gitignore b/.gitignore index fa1b473..d0c339e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,11 @@ deploy/secrets/ deploy/xray/config.json deploy/singbox/config.json +# 备份系统密钥（age 私钥、AWS 凭据）—— 绝不入库 +deploy/backup/backup.env +*.age-private* +age-keypair.txt + # 杂项 .DS_Store *.log diff --git a/deploy/README.md b/deploy/README.md index 3c5cf31..a2f76b4 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -10,6 +10,7 @@ |------|------|-----------| | `pangolin-edge`(nginx) | 分发层,接管 80/443,移植现网全部路由 | host 网络 | | `pangolin-singbox` | Hysteria2 加速线 | host 网络,443/udp | +| `pangolin-backup` | 数据库备份：xtrabackup+pg_dump，age 加密，S3 异地 | host 网络 | | `pangolin-certbot` | DNS-01(Cloudflare)证书续期,按需运行 | 无端口 | | `pangolin-xray`(本台不启用) | REALITY,留给新节点 | profile `newnode` | @@ -58,3 +59,23 @@ docker-compose -p pangolin run --rm certbot certonly --dns-cloudflare \ ```bash ssh ec2 'cat ~/pangolin/secrets/clients.txt' ``` + +## 备份与灾备(RPO ≤ 15min / RTO ≤ 4h) + +见 [`docs/备份与灾备.md`](../docs/备份与灾备.md) 完整说明。快速命令： + +```bash +# 初始化备份配置（首次） +cp backup/backup.env.example secrets/backup.env && vim secrets/backup.env +docker build -t pangolin-backup:local ./backup +docker-compose -p pangolin up -d backup + +# 查看实时 RPO/RTO 状态 +docker exec pangolin-backup /scripts/verify-rpo-rto.sh + +# 手动触发全量备份 +docker exec pangolin-backup bash -c '. /etc/backup-env.sh && /scripts/backup-full.sh' + +# 月度 DR 演练（含私钥时解密验证） +docker exec pangolin-backup bash -c '. /etc/backup-env.sh && /scripts/dr-drill.sh' +``` diff --git a/deploy/backup/Dockerfile b/deploy/backup/Dockerfile new file mode 100644 index 0000000..d70f115 --- /dev/null +++ b/deploy/backup/Dockerfile @@ -0,0 +1,54 @@ +FROM ubuntu:22.04 +ARG DEBIAN_FRONTEND=noninteractive + +# ---------- 基础工具 ---------- +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl wget ca-certificates gnupg2 lsb-release \ + postgresql-client \ + sqlite3 \ + pigz pv \ + unzip \ + netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* + +# ---------- AWS CLI v2 ---------- +RUN set -e \ + && curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \ + && unzip -q /tmp/awscliv2.zip -d /tmp/ \ + && /tmp/aws/install \ + && rm -rf /tmp/awscliv2.zip /tmp/aws + +# ---------- age 加密工具 v1.2.0 ---------- +RUN set -e \ + && AGE_VERSION=v1.2.0 \ + && wget -q -O /tmp/age.tar.gz \ + "https://github.com/FiloSottile/age/releases/download/${AGE_VERSION}/age-${AGE_VERSION}-linux-amd64.tar.gz" \ + && tar -xf /tmp/age.tar.gz -C /tmp/ \ + && install -m 755 /tmp/age/age /usr/local/bin/age \ + && install -m 755 /tmp/age/age-keygen /usr/local/bin/age-keygen \ + && rm -rf /tmp/age* + +# ---------- Percona XtraBackup 8.0 ---------- +RUN set -e \ + && DISTRO=$(lsb_release -cs) \ + && wget -q -O /tmp/percona-release.deb \ + "https://repo.percona.com/apt/percona-release_latest.${DISTRO}_all.deb" \ + && dpkg -i /tmp/percona-release.deb \ + && percona-release setup pxb80 \ + && apt-get update && apt-get install -y --no-install-recommends percona-xtrabackup-80 \ + && rm /tmp/percona-release.deb \ + && rm -rf /var/lib/apt/lists/* + +# ---------- 备份脚本 ---------- +COPY scripts/ /scripts/ +COPY entrypoint.sh /entrypoint.sh +COPY crontab.txt /etc/cron.d/pangolin-backup + +RUN chmod +x /scripts/*.sh /entrypoint.sh \ + && chmod 0644 /etc/cron.d/pangolin-backup \ + && mkdir -p /backup /var/log/backup + +# cron 日志直接输出到 stdout/stderr +RUN touch /var/log/backup/cron.log + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/deploy/backup/backup.env.example b/deploy/backup/backup.env.example new file mode 100644 index 0000000..3777e38 --- /dev/null +++ b/deploy/backup/backup.env.example @@ -0,0 +1,75 @@ +# ============================================================ +# Pangolin 备份系统环境变量 +# 复制此文件为 deploy/secrets/backup.env 并填写真实值 +# deploy/secrets/ 已在 .gitignore，不会提交到版本库 +# ============================================================ + +# ────────────────────────────────────────────────────────── +# S3 备份目标（独立身份账号，与生产 AWS 账号隔离） +# 创建步骤见 docs/备份与灾备.md § 初始化 +# ────────────────────────────────────────────────────────── +BACKUP_S3_BUCKET=pangolin-backup-XXXXXX # 替换为实际 bucket 名 +BACKUP_S3_PREFIX=ec2-ap-southeast-1 # 按服务器区分前缀 +BACKUP_AWS_REGION=ap-southeast-1 + +# 备份专用 IAM 凭据（仅有 s3:PutObject 权限） +BACKUP_AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +BACKUP_AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +# ────────────────────────────────────────────────────────── +# age 加密密钥 +# 生成方式: age-keygen 2>&1 | tee age-keypair.txt +# 公钥放此处（明文 OK）；私钥存离线，灾备演练时临时注入 +# ────────────────────────────────────────────────────────── +AGE_PUBLIC_KEY=age1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +# age 私钥（可选，DR 演练容器需要；不设则演练只做结构验证，不做解密恢复） +# AGE_PRIVATE_KEY=AGE-SECRET-KEY-1XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + +# ────────────────────────────────────────────────────────── +# MySQL (jiu_mysql, marzban) +# ────────────────────────────────────────────────────────── +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=backup_user # 专用备份账号，权限见下方 +MYSQL_PASSWORD=CHANGE_ME + +# 如果 MySQL 数据目录可以挂载到容器（xtrabackup 需要），设置此项 +# Docker 内 MySQL: 通常 /var/lib/docker/volumes//_data +# 宿主原生 MySQL: /var/lib/mysql +# 留空则 xtrabackup 尝试从 MySQL 连接获取（需要 BACKUP_ADMIN 权限） +MYSQL_DATADIR= + +# ────────────────────────────────────────────────────────── +# PostgreSQL (blog, umami) +# ────────────────────────────────────────────────────────── +PG_HOST=127.0.0.1 +PG_PORT=5432 +PG_USER=postgres +PG_PASSWORD=CHANGE_ME +PG_DATABASES=blog,umami # 逗号分隔，按需填写 + +# ────────────────────────────────────────────────────────── +# Marzban SQLite +# ────────────────────────────────────────────────────────── +MARZBAN_DB_PATH=/var/lib/marzban/db.sqlite3 + +# ────────────────────────────────────────────────────────── +# 本地临时备份目录（容器内路径） +# ────────────────────────────────────────────────────────── +BACKUP_DIR=/backup + +# ────────────────────────────────────────────────────────── +# 通知 Webhook（可选） +# 支持格式：POST JSON {"level":"INFO|ERROR","msg":"..."} +# 可对接 PushDeer / 企业微信 / Slack 等 +# ────────────────────────────────────────────────────────── +# NOTIFY_WEBHOOK_URL=https://api.pushdeer.com/message/push?pushkey=XXXXX + +# ────────────────────────────────────────────────────────── +# MySQL 备份账号权限（在 MySQL 中执行） +# CREATE USER 'backup_user'@'%' IDENTIFIED BY 'CHANGE_ME'; +# GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT, CREATE TABLESPACE, +# PROCESS, SUPER, CREATE, INSERT, SELECT, BACKUP_ADMIN ON *.* TO 'backup_user'@'%'; +# FLUSH PRIVILEGES; +# ────────────────────────────────────────────────────────── diff --git a/deploy/backup/crontab.txt b/deploy/backup/crontab.txt new file mode 100644 index 0000000..6106de4 --- /dev/null +++ b/deploy/backup/crontab.txt @@ -0,0 +1,20 @@ +# Pangolin 备份计划任务 +# 此文件由 Dockerfile 复制到 /etc/cron.d/pangolin-backup +# RPO ≤ 15min：全量每日 02:00 + 增量每 15min +# 月度 DR 演练：每月 1 日 03:00 + +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +MAILTO="" + +# --- 每日全量备份（MySQL xtrabackup + pg_dump + SQLite） --- +0 2 * * * root . /etc/backup-env.sh && /scripts/backup-full.sh >> /var/log/backup/cron.log 2>&1 + +# --- 每 15 分钟增量备份（MySQL binlog + pg_dump） --- +*/15 * * * * root . /etc/backup-env.sh && /scripts/backup-inc.sh >> /var/log/backup/cron.log 2>&1 + +# --- 每月 1 日 03:00 灾备恢复演练 --- +0 3 1 * * root . /etc/backup-env.sh && /scripts/dr-drill.sh >> /var/log/backup/dr-drill.log 2>&1 + +# --- 每天 06:00 清理 7 天前的本地临时文件 --- +0 6 * * * root find /backup -name "*.tmp" -mtime +1 -delete; find /backup/encrypt -mtime +2 -delete >> /var/log/backup/cron.log 2>&1 diff --git a/deploy/backup/entrypoint.sh b/deploy/backup/entrypoint.sh new file mode 100644 index 0000000..b2dd9a0 --- /dev/null +++ b/deploy/backup/entrypoint.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# entrypoint.sh —— 启动备份容器，初始化环境，拉起 crond +set -euo pipefail + +LOG_FILE=/var/log/backup/cron.log + +# ---------- 检查必要变量 ---------- +required_vars="BACKUP_S3_BUCKET BACKUP_S3_PREFIX AGE_PUBLIC_KEY + BACKUP_AWS_ACCESS_KEY_ID BACKUP_AWS_SECRET_ACCESS_KEY" +for v in $required_vars; do + if [ -z "${!v:-}" ]; then + echo "[ERROR] 缺少必要环境变量: ${v}" >&2 + exit 1 + fi +done + +# ---------- 配置 AWS 凭据（备份专用 IAM 账号） ---------- +mkdir -p /root/.aws +cat > /root/.aws/credentials << AWSEOF +[backup] +aws_access_key_id = ${BACKUP_AWS_ACCESS_KEY_ID} +aws_secret_access_key = ${BACKUP_AWS_SECRET_ACCESS_KEY} +AWSEOF + +cat > /root/.aws/config << AWSEOF +[profile backup] +region = ${BACKUP_AWS_REGION:-ap-southeast-1} +output = json +AWSEOF + +chmod 600 /root/.aws/credentials /root/.aws/config + +# ---------- 写入 age 公钥 ---------- +echo "${AGE_PUBLIC_KEY}" > /etc/age-public.txt +chmod 644 /etc/age-public.txt + +# ---------- 写入 age 私钥（仅 DR 演练时需要） ---------- +if [ -n "${AGE_PRIVATE_KEY:-}" ]; then + echo "${AGE_PRIVATE_KEY}" > /etc/age-private.txt + chmod 600 /etc/age-private.txt +fi + +# ---------- 把所有环境变量写入 /etc/backup-env.sh，供 cron job 使用 ---------- +# （cron 不继承环境变量） +{ + echo "# 自动生成，cron job 使用" + for v in BACKUP_S3_BUCKET BACKUP_S3_PREFIX BACKUP_AWS_REGION \ + BACKUP_AWS_ACCESS_KEY_ID BACKUP_AWS_SECRET_ACCESS_KEY \ + MYSQL_HOST MYSQL_PORT MYSQL_USER MYSQL_PASSWORD MYSQL_DATABASES \ + MYSQL_DATADIR \ + PG_HOST PG_PORT PG_USER PG_PASSWORD PG_DATABASES \ + MARZBAN_DB_PATH \ + BACKUP_DIR \ + NOTIFY_WEBHOOK_URL; do + val="${!v:-}" + if [ -n "${val}" ]; then + printf 'export %s=%q\n' "${v}" "${val}" + fi + done + echo 'export AWS_PROFILE=backup' + echo 'export AWS_CONFIG_FILE=/root/.aws/config' + echo 'export AWS_SHARED_CREDENTIALS_FILE=/root/.aws/credentials' +} > /etc/backup-env.sh +chmod 600 /etc/backup-env.sh + +# ---------- 设置本地备份目录 ---------- +BACKUP_DIR="${BACKUP_DIR:-/backup}" +mkdir -p \ + "${BACKUP_DIR}/mysql/full" \ + "${BACKUP_DIR}/mysql/inc" \ + "${BACKUP_DIR}/mysql/binlog" \ + "${BACKUP_DIR}/postgres" \ + "${BACKUP_DIR}/marzban" \ + "${BACKUP_DIR}/encrypt" + +echo "[INFO] 备份容器启动成功，等待 cron 任务..." +echo "[INFO] S3 目标: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}" +echo "[INFO] 备份目录: ${BACKUP_DIR}" + +# ---------- 启动 cron 并保持前台 ---------- +# 执行一次启动验证 +/scripts/verify-rpo-rto.sh --status-only 2>&1 | head -5 || true + +# 启动 cron daemon（后台，Ubuntu 的 cron 默认 daemon 模式） +cron + +# exec tail 成为主进程，接收 Docker stop 信号，并实时输出日志到 stdout +exec tail -F "${LOG_FILE}" diff --git a/deploy/backup/scripts/backup-full.sh b/deploy/backup/scripts/backup-full.sh new file mode 100644 index 0000000..385ce35 --- /dev/null +++ b/deploy/backup/scripts/backup-full.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# backup-full.sh —— 每日全量备份编排脚本（02:00 运行） +# 内容：MySQL xtrabackup 全量 + PostgreSQL pg_dump + Marzban SQLite +# 备份 → age 加密 → 上传 S3 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +START="$(timer_start)" + +log_info "========== 全量备份开始 ${TIMESTAMP} ==========" + +ERRORS=0 + +# ────────────────────────────────────────────────────── +# 1. MySQL 全量备份（xtrabackup） +# ────────────────────────────────────────────────────── +mysql_full_backup() { + local target="${BACKUP_DIR}/mysql/full/${TIMESTAMP}" + local archive="${BACKUP_DIR}/mysql/full/${TIMESTAMP}.tar.gz" + local s3_key="mysql/full/${TIMESTAMP}.tar.gz" + + log_info "[MySQL] 开始 xtrabackup 全量备份" + mkdir -p "${target}" + + if ! mysql_ping; then + log_warn "[MySQL] 无法连接 MySQL，跳过 MySQL 备份" + return 1 + fi + + # xtrabackup --backup：连接 MySQL 并备份 InnoDB 数据文件 + xtrabackup \ + --backup \ + --host="${MYSQL_HOST}" \ + --port="${MYSQL_PORT}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --target-dir="${target}" \ + --compress \ + --compress-threads=2 \ + ${MYSQL_DATADIR:+--datadir="${MYSQL_DATADIR}"} \ + 2>&1 | tee "${target}/xtrabackup-backup.log" + + # xtrabackup --prepare：回放 redo log，使备份一致（无需 MySQL 服务） + xtrabackup \ + --prepare \ + --target-dir="${target}" \ + 2>&1 | tee "${target}/xtrabackup-prepare.log" + + # 打包（xtrabackup --compress 已压缩，这里仅归档） + log_info "[MySQL] 打包备份目录" + tar -czf "${archive}" -C "$(dirname "${target}")" "$(basename "${target}")" + + # 更新 latest-full 引用（供增量备份使用） + echo "${target}" > "${BACKUP_DIR}/mysql/latest-full.txt" + # 清除上一次增量引用 + rm -f "${BACKUP_DIR}/mysql/latest-inc.txt" + + # 加密 + 上传 + encrypt_and_upload "${archive}" "${s3_key}" + + # 清理本地中间文件（保留原始目录供后续增量） + rm -f "${archive}" + + # 清理 7 天前的全量备份目录 + find "${BACKUP_DIR}/mysql/full" -maxdepth 1 -type d -mtime +7 \ + -not -path "${BACKUP_DIR}/mysql/full" \ + -exec rm -rf {} + 2>/dev/null || true + + log_ok "[MySQL] 全量备份完成: ${target}" +} + +# ────────────────────────────────────────────────────── +# 2. PostgreSQL 全量备份（pg_dump） +# ────────────────────────────────────────────────────── +postgres_full_backup() { + log_info "[PostgreSQL] 开始 pg_dump 全量备份" + + if ! pg_ping; then + log_warn "[PostgreSQL] 无法连接 PostgreSQL，跳过 PG 备份" + return 1 + fi + + # 支持多数据库（逗号分隔） + IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}" + for DB in "${DBS[@]}"; do + DB="$(echo "${DB}" | tr -d ' ')" + local archive="${BACKUP_DIR}/postgres/full_${DB}_${TIMESTAMP}.pgdump" + local s3_key="postgres/full_${DB}_${TIMESTAMP}.pgdump" + + log_info "[PostgreSQL] 备份数据库: ${DB}" + PGPASSWORD="${PG_PASSWORD:-}" pg_dump \ + --host="${PG_HOST}" \ + --port="${PG_PORT}" \ + --username="${PG_USER:-postgres}" \ + --format=custom \ + --compress=6 \ + --no-password \ + "${DB}" > "${archive}" + + encrypt_and_upload "${archive}" "${s3_key}" + rm -f "${archive}" + log_ok "[PostgreSQL] ${DB} 备份完成" + done +} + +# ────────────────────────────────────────────────────── +# 3. Marzban SQLite 备份 +# ────────────────────────────────────────────────────── +marzban_backup() { + local marzban_db="${MARZBAN_DB_PATH:-/var/lib/marzban/db.sqlite3}" + local archive="${BACKUP_DIR}/marzban/marzban_${TIMESTAMP}.sqlite3.gz" + local s3_key="marzban/marzban_${TIMESTAMP}.sqlite3.gz" + + if [ ! -f "${marzban_db}" ]; then + log_warn "[Marzban] SQLite 文件不存在: ${marzban_db}，跳过" + return 0 + fi + + log_info "[Marzban] 备份 SQLite: ${marzban_db}" + # 使用 sqlite3 online backup，避免损坏 WAL + sqlite3 "${marzban_db}" ".backup '${BACKUP_DIR}/marzban/marzban_${TIMESTAMP}.sqlite3'" + gzip -9 "${BACKUP_DIR}/marzban/marzban_${TIMESTAMP}.sqlite3" + + encrypt_and_upload "${archive}" "${s3_key}" + rm -f "${archive}" + + # 清理 30 天前的 SQLite 备份 + find "${BACKUP_DIR}/marzban" -name "*.gz" -mtime +30 -delete 2>/dev/null || true + log_ok "[Marzban] SQLite 备份完成" +} + +# ────────────────────────────────────────────────────── +# 执行 +# ────────────────────────────────────────────────────── +mysql_full_backup || { log_error "[MySQL] 全量备份失败"; ERRORS=$(( ERRORS + 1 )); } +postgres_full_backup || { log_error "[PostgreSQL] 全量备份失败"; ERRORS=$(( ERRORS + 1 )); } +marzban_backup || { log_error "[Marzban] 备份失败"; ERRORS=$(( ERRORS + 1 )); } + +ELAPSED=$(timer_elapsed "${START}") +DURATION=$(format_duration "${ELAPSED}") + +if [ "${ERRORS}" -eq 0 ]; then + log_ok "========== 全量备份完成，耗时 ${DURATION} ==========" + notify "INFO" "全量备份成功，耗时 ${DURATION}" +else + log_error "========== 全量备份完成（${ERRORS} 个失败），耗时 ${DURATION} ==========" + notify "ERROR" "全量备份完成，但有 ${ERRORS} 个失败，请检查日志" + exit 1 +fi diff --git a/deploy/backup/scripts/backup-inc.sh b/deploy/backup/scripts/backup-inc.sh new file mode 100644 index 0000000..1e33f3f --- /dev/null +++ b/deploy/backup/scripts/backup-inc.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# backup-inc.sh —— 每 15 分钟增量备份（RPO ≤ 15min） +# MySQL：xtrabackup 增量 + binlog flush & 归档 +# PostgreSQL：pg_dump（轻量，一致性快照） +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +START="$(timer_start)" + +log_info "---------- 增量备份开始 ${TIMESTAMP} ----------" + +ERRORS=0 + +# ────────────────────────────────────────────────────── +# 1. MySQL 增量备份（xtrabackup incremental） +# ────────────────────────────────────────────────────── +mysql_incremental_backup() { + # 确定 basedir：优先用最近一次增量，若无则用最近全量 + local basedir="" + if [ -f "${BACKUP_DIR}/mysql/latest-inc.txt" ]; then + basedir="$(cat "${BACKUP_DIR}/mysql/latest-inc.txt")" + elif [ -f "${BACKUP_DIR}/mysql/latest-full.txt" ]; then + basedir="$(cat "${BACKUP_DIR}/mysql/latest-full.txt")" + else + log_warn "[MySQL] 未找到基础备份，跳过增量（请先执行全量备份）" + return 1 + fi + + if [ ! -d "${basedir}" ]; then + log_warn "[MySQL] 基础备份目录不存在: ${basedir}，跳过增量" + return 1 + fi + + local target="${BACKUP_DIR}/mysql/inc/${TIMESTAMP}" + local archive="${BACKUP_DIR}/mysql/inc/${TIMESTAMP}.tar.gz" + local s3_key="mysql/inc/${TIMESTAMP}.tar.gz" + + log_info "[MySQL] xtrabackup 增量备份，basedir=${basedir}" + mkdir -p "${target}" + + if ! mysql_ping; then + log_warn "[MySQL] 无法连接 MySQL，跳过增量" + return 1 + fi + + xtrabackup \ + --backup \ + --host="${MYSQL_HOST}" \ + --port="${MYSQL_PORT}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --target-dir="${target}" \ + --incremental-basedir="${basedir}" \ + --compress \ + --compress-threads=2 \ + ${MYSQL_DATADIR:+--datadir="${MYSQL_DATADIR}"} \ + 2>&1 | tee "${target}/xtrabackup.log" + + tar -czf "${archive}" -C "$(dirname "${target}")" "$(basename "${target}")" + + # 更新最新增量引用 + echo "${target}" > "${BACKUP_DIR}/mysql/latest-inc.txt" + + encrypt_and_upload "${archive}" "${s3_key}" + rm -f "${archive}" + + # 清理 2 天前的增量目录（全量已上 S3，本地只保留最近供下一次增量） + find "${BACKUP_DIR}/mysql/inc" -maxdepth 1 -type d -mtime +2 \ + -not -path "${BACKUP_DIR}/mysql/inc" \ + -exec rm -rf {} + 2>/dev/null || true + + log_ok "[MySQL] 增量备份完成: ${target}" +} + +# ────────────────────────────────────────────────────── +# 2. MySQL Binlog 归档（FLUSH BINARY LOGS + 复制新 binlog） +# ────────────────────────────────────────────────────── +mysql_binlog_backup() { + if ! mysql_ping; then + log_warn "[MySQL-binlog] 无法连接 MySQL，跳过 binlog 归档" + return 1 + fi + + log_info "[MySQL-binlog] FLUSH BINARY LOGS" + mysql \ + --host="${MYSQL_HOST}" \ + --port="${MYSQL_PORT}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + -e "FLUSH BINARY LOGS;" 2>/dev/null + + # 从 MySQL 取 binlog 文件列表 + local binlog_dir="${MYSQL_DATADIR:-/var/lib/mysql}" + local dest="${BACKUP_DIR}/mysql/binlog" + mkdir -p "${dest}" + + # 使用 mysqlbinlog 工具读取当前活跃 binlog 之前的文件并归档 + local binlog_index + binlog_index="$(mysql \ + --host="${MYSQL_HOST}" \ + --port="${MYSQL_PORT}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --batch --skip-column-names \ + -e "SHOW BINARY LOGS;" 2>/dev/null | awk '{print $1}')" + + if [ -z "${binlog_index}" ]; then + log_warn "[MySQL-binlog] 无法获取 binlog 列表（binlog 未开启？），跳过" + return 0 + fi + + # 找出尚未归档的 binlog 文件（非最新一个，最新的正在写入） + local count + count=$(echo "${binlog_index}" | wc -l) + # 跳过最后一个（正在写入） + local to_archive + to_archive=$(echo "${binlog_index}" | head -n $(( count - 1 ))) + + for binlog_file in ${to_archive}; do + local dest_file="${dest}/${binlog_file}.gz" + if [ -f "${dest_file}" ]; then + continue # 已归档 + fi + log_info "[MySQL-binlog] 归档 ${binlog_file}" + # 使用 mysqlbinlog 以 raw 模式读取 + gzip 压缩 + mysqlbinlog \ + --host="${MYSQL_HOST}" \ + --port="${MYSQL_PORT}" \ + --user="${MYSQL_USER}" \ + --password="${MYSQL_PASSWORD}" \ + --read-from-remote-server \ + --raw \ + --result-file="${dest}/" \ + "${binlog_file}" 2>/dev/null \ + && gzip -f "${dest}/${binlog_file}" \ + && encrypt_and_upload "${dest_file}" "mysql/binlog/${TIMESTAMP}_${binlog_file}.gz" \ + && rm -f "${dest_file}" \ + || log_warn "[MySQL-binlog] ${binlog_file} 归档失败，忽略" + done + + # 清理 3 天前的本地 binlog 备份 + find "${dest}" -name "*.gz" -mtime +3 -delete 2>/dev/null || true + log_ok "[MySQL-binlog] binlog 归档完成" +} + +# ────────────────────────────────────────────────────── +# 3. PostgreSQL 增量备份（pg_dump，15min 一次） +# ────────────────────────────────────────────────────── +postgres_incremental_backup() { + if ! pg_ping; then + log_warn "[PostgreSQL] 无法连接，跳过增量" + return 1 + fi + + IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}" + for DB in "${DBS[@]}"; do + DB="$(echo "${DB}" | tr -d ' ')" + local archive="${BACKUP_DIR}/postgres/inc_${DB}_${TIMESTAMP}.pgdump" + local s3_key="postgres/inc/${DB}/${TIMESTAMP}.pgdump" + + PGPASSWORD="${PG_PASSWORD:-}" pg_dump \ + --host="${PG_HOST}" \ + --port="${PG_PORT}" \ + --username="${PG_USER:-postgres}" \ + --format=custom \ + --compress=6 \ + --no-password \ + "${DB}" > "${archive}" + + encrypt_and_upload "${archive}" "${s3_key}" + rm -f "${archive}" + log_ok "[PostgreSQL] ${DB} 增量快照完成" + done +} + +# ────────────────────────────────────────────────────── +# 执行 +# ────────────────────────────────────────────────────── +mysql_incremental_backup || { log_error "[MySQL] 增量备份失败"; ERRORS=$(( ERRORS + 1 )); } +mysql_binlog_backup || { log_warn "[MySQL] binlog 归档失败"; } +postgres_incremental_backup || { log_error "[PG] 增量备份失败"; ERRORS=$(( ERRORS + 1 )); } + +ELAPSED=$(timer_elapsed "${START}") +DURATION=$(format_duration "${ELAPSED}") + +if [ "${ERRORS}" -eq 0 ]; then + log_ok "---------- 增量备份完成，耗时 ${DURATION} ----------" +else + log_error "---------- 增量备份完成（${ERRORS} 个失败），耗时 ${DURATION} ----------" + exit 1 +fi diff --git a/deploy/backup/scripts/dr-drill.sh b/deploy/backup/scripts/dr-drill.sh new file mode 100644 index 0000000..148b68e --- /dev/null +++ b/deploy/backup/scripts/dr-drill.sh @@ -0,0 +1,306 @@ +#!/usr/bin/env bash +# dr-drill.sh —— 月度灾备恢复演练 +# 目标：RPO ≤ 15min，RTO ≤ 4h +# +# 演练流程： +# 1. 从 S3 列出最新备份，验证时间戳（RPO 检查） +# 2. 下载最新全量备份，age 解密，验证完整性 +# 3. 在临时 MySQL 容器中执行完整恢复，验证数据 +# 4. 在临时 PostgreSQL 容器中执行恢复，验证数据 +# 5. 记录总耗时，与 RTO 4h 对比 +# 6. 生成报告，通过 Webhook 推送 +# +# 演练使用独立 Docker 容器，不影响生产数据库 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +DRILL_ID="drill_$(date +%Y%m%d_%H%M%S)" +DRILL_DIR="/tmp/dr_drill_${DRILL_ID}" +REPORT_FILE="${DRILL_DIR}/report.txt" +RTO_TARGET_SECONDS=$(( 4 * 3600 )) # 4 小时 +DRILL_START="$(timer_start)" + +mkdir -p "${DRILL_DIR}" + +# age 私钥（演练时必须可用） +AGE_PRIVATE_KEY_FILE="${AGE_PRIVATE_KEY_FILE:-/etc/age-private.txt}" + +log_info "========== DR 演练开始 [${DRILL_ID}] ==========" + +# ---------- 报告工具 ---------- +PASS_COUNT=0 +FAIL_COUNT=0 + +check_pass() { PASS_COUNT=$(( PASS_COUNT + 1 )); log_ok " ✅ $*"; echo "PASS: $*" >> "${REPORT_FILE}"; } +check_fail() { FAIL_COUNT=$(( FAIL_COUNT + 1 )); log_error " ❌ $*"; echo "FAIL: $*" >> "${REPORT_FILE}"; } +section() { log_info "--- $* ---"; echo "" >> "${REPORT_FILE}"; echo "=== $* ===" >> "${REPORT_FILE}"; } + +{ + echo "Pangolin 灾备演练报告" + echo "演练 ID: ${DRILL_ID}" + echo "开始时间: $(date '+%Y-%m-%d %H:%M:%S')" + echo "目标: RPO ≤ 15min, RTO ≤ 4h" + echo "" +} > "${REPORT_FILE}" + +# ────────────────────────────────────────────────────── +# 检查 1：RPO 验证（最新增量备份时间戳） +# ────────────────────────────────────────────────────── +section "RPO 验证（目标 ≤ 15min）" +check_rpo() { + log_info "[RPO] 检查 S3 最新增量备份时间" + + # 列出最新的增量备份文件 + local latest_inc + latest_inc="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/inc/" \ + --recursive \ + | sort | tail -1 | awk '{print $1, $2}')" + + if [ -z "${latest_inc}" ]; then + check_fail "S3 中未找到 MySQL 增量备份" + return 1 + fi + + log_info "[RPO] 最新增量备份: ${latest_inc}" + local last_date last_time + last_date=$(echo "${latest_inc}" | awk '{print $1}') + last_time=$(echo "${latest_inc}" | awk '{print $2}') + local last_ts + last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s)" + local now + now="$(date +%s)" + local gap=$(( now - last_ts )) + local gap_min=$(( gap / 60 )) + + echo "最新增量备份距今: ${gap_min} 分钟" >> "${REPORT_FILE}" + + if [ "${gap}" -le 900 ]; then # 900 秒 = 15 分钟 + check_pass "RPO 达标: 最新备份距今 ${gap_min} 分钟 (≤ 15min)" + else + check_fail "RPO 未达标: 最新备份距今 ${gap_min} 分钟 (> 15min)" + fi +} +check_rpo || true + +# ────────────────────────────────────────────────────── +# 检查 2：备份文件完整性验证 +# ────────────────────────────────────────────────────── +section "备份文件完整性" +check_backup_integrity() { + if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then + check_fail "age 私钥不可用，跳过解密验证（设置 AGE_PRIVATE_KEY 环境变量）" + return 1 + fi + + log_info "[完整性] 下载并解密最新全量备份" + + # 找到最新全量备份的 S3 key + local latest_full_key + latest_full_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/" \ + | sort | tail -1 | awk '{print $4}')" + + if [ -z "${latest_full_key}" ]; then + check_fail "S3 中未找到 MySQL 全量备份" + return 1 + fi + + local enc_file="${DRILL_DIR}/${latest_full_key}" + local dec_file="${enc_file%.age}" + + log_info "[完整性] 下载: ${latest_full_key}" + AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/mysql/full/${latest_full_key}" \ + "${enc_file}" + + log_info "[完整性] age 解密" + age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}" + rm -f "${enc_file}" + + # 验证 tar 归档完整性（不解压） + if tar -tzf "${dec_file}" > /dev/null 2>&1; then + check_pass "全量备份 tar.gz 归档完整" + else + check_fail "全量备份 tar.gz 归档损坏" + return 1 + fi + + echo "${dec_file}" +} +FULL_ARCHIVE="$(check_backup_integrity 2>/dev/null)" || FULL_ARCHIVE="" + +# ────────────────────────────────────────────────────── +# 检查 3：MySQL 恢复验证（临时容器） +# ────────────────────────────────────────────────────── +section "MySQL 恢复验证" +MYSQL_DRILL_PORT=33306 # 避免与生产 3306 冲突 +MYSQL_DRILL_CONTAINER="drill_mysql_${DRILL_ID}" + +check_mysql_restore() { + if [ -z "${FULL_ARCHIVE}" ]; then + check_fail "全量备份不可用，跳过 MySQL 恢复验证" + return 1 + fi + + local mysql_restore_dir="${DRILL_DIR}/mysql_restore" + mkdir -p "${mysql_restore_dir}" + + # 解压备份 + log_info "[MySQL] 解压备份到临时目录" + tar -xzf "${FULL_ARCHIVE}" -C "${mysql_restore_dir}" + + local backup_subdir + backup_subdir="$(ls -1 "${mysql_restore_dir}" | head -1)" + local target_dir="${mysql_restore_dir}/${backup_subdir}" + + # xtrabackup --prepare（无需 MySQL 服务） + log_info "[MySQL] xtrabackup --prepare 验证" + if xtrabackup --prepare --target-dir="${target_dir}" 2>&1 | grep -q "completed OK"; then + check_pass "xtrabackup --prepare 成功（备份一致性验证通过）" + else + check_fail "xtrabackup --prepare 失败" + return 1 + fi + + # 启动临时 MySQL 容器（使用恢复的数据目录） + log_info "[MySQL] 启动临时 MySQL 容器（端口 ${MYSQL_DRILL_PORT}）" + docker run -d \ + --name "${MYSQL_DRILL_CONTAINER}" \ + --mount "type=bind,src=${target_dir},dst=/var/lib/mysql" \ + -e MYSQL_ROOT_PASSWORD=drill_tmp_pw \ + -p "127.0.0.1:${MYSQL_DRILL_PORT}:3306" \ + mysql:8.0 \ + --skip-grant-tables 2>/dev/null || { + check_fail "临时 MySQL 容器启动失败（可能没有 Docker）" + return 1 + } + + # 等待 MySQL 就绪（最多 60 秒） + local wait=0 + until mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \ + --user=root --password=drill_tmp_pw \ + -e "SELECT 1" > /dev/null 2>&1 || [ "${wait}" -ge 60 ]; do + sleep 2 + wait=$(( wait + 2 )) + done + + if mysql --host=127.0.0.1 --port="${MYSQL_DRILL_PORT}" \ + --user=root --password=drill_tmp_pw \ + -e "SHOW DATABASES;" > /dev/null 2>&1; then + check_pass "MySQL 从备份恢复后成功响应查询" + else + check_fail "MySQL 恢复后无法执行查询" + fi + + # 停止并删除临时容器 + docker stop "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true + docker rm -f "${MYSQL_DRILL_CONTAINER}" 2>/dev/null || true + rm -rf "${mysql_restore_dir}" +} +check_mysql_restore || true + +# ────────────────────────────────────────────────────── +# 检查 4：PostgreSQL 恢复验证 +# ────────────────────────────────────────────────────── +section "PostgreSQL 恢复验证" +PG_DRILL_PORT=55432 +PG_DRILL_CONTAINER="drill_pg_${DRILL_ID}" + +check_pg_restore() { + if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then + check_fail "age 私钥不可用，跳过 PG 恢复验证" + return 1 + fi + + local first_db + IFS=',' read -ra DBS <<< "${PG_DATABASES:-postgres}" + first_db="${DBS[0]}" + first_db="$(echo "${first_db}" | tr -d ' ')" + + log_info "[PostgreSQL] 下载最新 ${first_db} 备份" + local latest_key + latest_key="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/" \ + | sort | tail -1 | awk '{print $4}')" + + if [ -z "${latest_key}" ]; then + check_fail "S3 中未找到 PostgreSQL 备份: ${first_db}" + return 1 + fi + + local enc_file="${DRILL_DIR}/pg_${latest_key}" + local dec_file="${enc_file%.age}" + + AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/postgres/inc/${first_db}/${latest_key}" \ + "${enc_file}" + + age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dec_file}" "${enc_file}" + rm -f "${enc_file}" + + # 用 pg_restore --list 验证备份结构（无需启动 PostgreSQL） + if pg_restore --list "${dec_file}" > /dev/null 2>&1; then + check_pass "PostgreSQL 备份结构验证通过 (${first_db})" + else + check_fail "PostgreSQL 备份文件损坏 (${first_db})" + fi + + rm -f "${dec_file}" +} +check_pg_restore || true + +# ────────────────────────────────────────────────────── +# 汇总报告 +# ────────────────────────────────────────────────────── +section "RTO 验证（目标 ≤ 4h）" +ELAPSED=$(timer_elapsed "${DRILL_START}") +DURATION=$(format_duration "${ELAPSED}") + +echo "演练总耗时: ${ELAPSED}s (${DURATION})" >> "${REPORT_FILE}" +echo "RTO 目标: ${RTO_TARGET_SECONDS}s (4h)" >> "${REPORT_FILE}" + +if [ "${ELAPSED}" -le "${RTO_TARGET_SECONDS}" ]; then + check_pass "RTO 达标: 演练耗时 ${DURATION} (≤ 4h)" +else + check_fail "RTO 未达标: 演练耗时 ${DURATION} (> 4h)" +fi + +{ + echo "" + echo "=================" + echo "检查通过: ${PASS_COUNT}" + echo "检查失败: ${FAIL_COUNT}" + echo "=================" + echo "结束时间: $(date '+%Y-%m-%d %H:%M:%S')" + if [ "${FAIL_COUNT}" -eq 0 ]; then + echo "结论: 演练通过 ✅" + else + echo "结论: 演练失败 ❌，请检查上方失败项" + fi +} >> "${REPORT_FILE}" + +# 上传演练报告到 S3 +if AWS_PROFILE="${AWS_PROFILE}" aws s3 cp "${REPORT_FILE}" \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt" 2>/dev/null; then + log_ok "演练报告已上传: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/${DRILL_ID}.txt" +fi + +# 打印报告 +echo "" +cat "${REPORT_FILE}" + +# 清理 +rm -rf "${DRILL_DIR}" + +# 通知 +if [ "${FAIL_COUNT}" -eq 0 ]; then + log_ok "========== DR 演练通过 [通过: ${PASS_COUNT}，失败: ${FAIL_COUNT}，耗时: ${DURATION}] ==========" + notify "INFO" "月度 DR 演练通过 | ${DRILL_ID} | 耗时 ${DURATION} | 通过 ${PASS_COUNT} 项" +else + log_error "========== DR 演练有 ${FAIL_COUNT} 项失败 [耗时: ${DURATION}] ==========" + notify "ERROR" "月度 DR 演练失败 | ${DRILL_ID} | 失败 ${FAIL_COUNT} 项，请检查" + exit 1 +fi diff --git a/deploy/backup/scripts/lib.sh b/deploy/backup/scripts/lib.sh new file mode 100644 index 0000000..019ea33 --- /dev/null +++ b/deploy/backup/scripts/lib.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# lib.sh —— 共用函数库，被所有备份脚本 source + +# ---------- 日志 ---------- +log_info() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [INFO] $*"; } +log_warn() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [WARN] $*" >&2; } +log_error() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [ERROR] $*" >&2; } +log_ok() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [OK] $*"; } + +# ---------- 默认值 ---------- +: "${BACKUP_DIR:=/backup}" +: "${MYSQL_HOST:=127.0.0.1}" +: "${MYSQL_PORT:=3306}" +: "${PG_HOST:=127.0.0.1}" +: "${PG_PORT:=5432}" +: "${BACKUP_AWS_REGION:=ap-southeast-1}" +: "${AWS_PROFILE:=backup}" + +# ---------- 通知（Webhook，可选） ---------- +notify() { + local level="$1" + local msg="$2" + log_info "[NOTIFY] ${level}: ${msg}" + if [ -n "${NOTIFY_WEBHOOK_URL:-}" ]; then + curl -fsS -X POST "${NOTIFY_WEBHOOK_URL}" \ + -H 'Content-Type: application/json' \ + -d "{\"level\":\"${level}\",\"msg\":$(printf '%s' "${msg}" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read()))')}" \ + --max-time 10 || true + fi +} + +# ---------- age 加密 ---------- +# 用法: age_encrypt +age_encrypt() { + local src="$1" + local dst="$2" + local pubkey_file="${AGE_PUBLIC_KEY_FILE:-/etc/age-public.txt}" + if [ ! -f "${pubkey_file}" ]; then + log_error "age 公钥文件不存在: ${pubkey_file}" + return 1 + fi + age --encrypt --recipient "$(cat "${pubkey_file}")" -o "${dst}" "${src}" +} + +# ---------- S3 上传 ---------- +# 用法: s3_upload +s3_upload() { + local src="$1" + local suffix="$2" + local dst="s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${suffix}" + log_info "上传 $(basename "${src}") → ${dst}" + AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \ + --no-progress \ + --storage-class STANDARD_IA \ + "${src}" "${dst}" + log_ok "上传完成: ${dst}" +} + +# ---------- 加密并上传 ---------- +# 用法: encrypt_and_upload +encrypt_and_upload() { + local src="$1" + local suffix="$2" + local enc_file="${BACKUP_DIR}/encrypt/$(basename "${src}").age" + + mkdir -p "${BACKUP_DIR}/encrypt" + log_info "age 加密: ${src}" + age_encrypt "${src}" "${enc_file}" + + s3_upload "${enc_file}" "${suffix}.age" + + # 删除本地加密临时文件（原始文件由调用方管理） + rm -f "${enc_file}" +} + +# ---------- MySQL 连接测试 ---------- +mysql_ping() { + mysql \ + --host="${MYSQL_HOST}" \ + --port="${MYSQL_PORT}" \ + --user="${MYSQL_USER:-root}" \ + --password="${MYSQL_PASSWORD}" \ + --connect-timeout=5 \ + -e "SELECT 1" > /dev/null 2>&1 +} + +# ---------- PostgreSQL 连接测试 ---------- +pg_ping() { + PGPASSWORD="${PG_PASSWORD:-}" pg_isready \ + --host="${PG_HOST}" \ + --port="${PG_PORT}" \ + --username="${PG_USER:-postgres}" \ + --quiet 2>&1 +} + +# ---------- 计时工具 ---------- +timer_start() { date +%s; } +timer_elapsed() { + local start="$1" + local end + end=$(date +%s) + echo $(( end - start )) +} +format_duration() { + local secs="$1" + local h=$(( secs / 3600 )) + local m=$(( (secs % 3600) / 60 )) + local s=$(( secs % 60 )) + printf '%dh %dm %ds' "${h}" "${m}" "${s}" +} diff --git a/deploy/backup/scripts/restore.sh b/deploy/backup/scripts/restore.sh new file mode 100644 index 0000000..1266837 --- /dev/null +++ b/deploy/backup/scripts/restore.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# restore.sh —— 灾备恢复脚本 +# 用途：从 S3 下载 → age 解密 → 恢复 MySQL / PostgreSQL +# 使用方式： +# restore.sh mysql full 20240101_020000 # 恢复 MySQL 指定全量 +# restore.sh mysql inc 20240101_021500 # 在全量基础上应用增量 +# restore.sh postgres blog 20240101_021500 +# restore.sh marzban 20240101_020000 +# +# 注意：MySQL 恢复需要先停止 MySQL 服务（由运维手动执行） +# 此脚本只做"下载 + 解密 + 准备"，不自动重启 MySQL 服务 +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +# age 私钥（恢复时必须提供） +AGE_PRIVATE_KEY_FILE="${AGE_PRIVATE_KEY_FILE:-/etc/age-private.txt}" + +if [ ! -f "${AGE_PRIVATE_KEY_FILE}" ]; then + log_error "age 私钥文件不存在: ${AGE_PRIVATE_KEY_FILE}" + log_error "恢复需要私钥。请将私钥文件挂载到容器，或设置 AGE_PRIVATE_KEY 环境变量" + exit 1 +fi + +RESTORE_DIR="${RESTORE_DIR:-/restore}" +mkdir -p "${RESTORE_DIR}" + +# ---------- age 解密 ---------- +age_decrypt() { + local src="$1" + local dst="$2" + age --decrypt --identity "${AGE_PRIVATE_KEY_FILE}" -o "${dst}" "${src}" +} + +# ---------- 从 S3 下载并解密 ---------- +download_and_decrypt() { + local s3_key="$1" + local local_enc="${RESTORE_DIR}/$(basename "${s3_key}")" + local local_dec="${local_enc%.age}" + + log_info "从 S3 下载: s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_key}.age" + AWS_PROFILE="${AWS_PROFILE}" aws s3 cp \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_key}.age" \ + "${local_enc}" + + log_info "age 解密: ${local_enc}" + age_decrypt "${local_enc}" "${local_dec}" + rm -f "${local_enc}" + + echo "${local_dec}" +} + +# ────────────────────────────────────────────────────── +# MySQL 恢复 +# ────────────────────────────────────────────────────── +restore_mysql_full() { + local timestamp="$1" + local s3_key="mysql/full/${timestamp}.tar.gz" + local restore_base="${RESTORE_DIR}/mysql" + mkdir -p "${restore_base}" + + log_info "[MySQL] 下载并解密全量备份: ${timestamp}" + local archive + archive="$(download_and_decrypt "${s3_key}")" + + log_info "[MySQL] 解压: ${archive}" + tar -xzf "${archive}" -C "${restore_base}" + rm -f "${archive}" + + local backup_dir="${restore_base}/${timestamp}" + log_info "[MySQL] xtrabackup --prepare（应用增量前的全量 prepare）" + xtrabackup --prepare --apply-log-only --target-dir="${backup_dir}" + + log_ok "[MySQL] 全量备份已解压并 prepare 到: ${backup_dir}" + echo "${backup_dir}" +} + +restore_mysql_inc() { + local base_dir="$1" + local timestamp="$2" + local s3_key="mysql/inc/${timestamp}.tar.gz" + local restore_base="${RESTORE_DIR}/mysql" + + log_info "[MySQL] 下载并解密增量备份: ${timestamp}" + local archive + archive="$(download_and_decrypt "${s3_key}")" + + tar -xzf "${archive}" -C "${restore_base}" + rm -f "${archive}" + + local inc_dir="${restore_base}/${timestamp}" + log_info "[MySQL] xtrabackup --prepare（应用增量 ${timestamp}）" + xtrabackup --prepare --apply-log-only \ + --target-dir="${base_dir}" \ + --incremental-dir="${inc_dir}" + rm -rf "${inc_dir}" + + log_ok "[MySQL] 增量 ${timestamp} 已应用到 ${base_dir}" +} + +restore_mysql_finalize() { + local base_dir="$1" + log_info "[MySQL] xtrabackup --prepare（最终 redo log 回放）" + xtrabackup --prepare --target-dir="${base_dir}" + log_ok "[MySQL] 备份已可用，恢复步骤：" + log_ok " 1. 停止 MySQL：systemctl stop mysql" + log_ok " 2. 清空数据目录：rm -rf /var/lib/mysql/*" + log_ok " 3. 复制备份：xtrabackup --copy-back --target-dir=${base_dir}" + log_ok " 4. 修改权限：chown -R mysql:mysql /var/lib/mysql" + log_ok " 5. 启动 MySQL：systemctl start mysql" +} + +# ────────────────────────────────────────────────────── +# PostgreSQL 恢复 +# ────────────────────────────────────────────────────── +restore_postgres() { + local db="$1" + local timestamp="$2" + local s3_key="postgres/inc/${db}/${timestamp}.pgdump" + local restore_target="${PG_RESTORE_TARGET:-${db}_restored}" + + log_info "[PostgreSQL] 下载并解密: ${db} @ ${timestamp}" + local dump_file + dump_file="$(download_and_decrypt "${s3_key}")" + + log_info "[PostgreSQL] 恢复数据库: ${restore_target}" + PGPASSWORD="${PG_PASSWORD:-}" psql \ + --host="${PG_HOST}" \ + --port="${PG_PORT}" \ + --username="${PG_USER:-postgres}" \ + -c "CREATE DATABASE \"${restore_target}\";" 2>/dev/null || true + + PGPASSWORD="${PG_PASSWORD:-}" pg_restore \ + --host="${PG_HOST}" \ + --port="${PG_PORT}" \ + --username="${PG_USER:-postgres}" \ + --dbname="${restore_target}" \ + --no-password \ + --verbose \ + "${dump_file}" + + rm -f "${dump_file}" + log_ok "[PostgreSQL] 恢复完成: ${restore_target}" +} + +# ────────────────────────────────────────────────────── +# 主入口 +# ────────────────────────────────────────────────────── +case "${1:-}" in + mysql) + case "${2:-}" in + full) + base_dir="$(restore_mysql_full "${3}")" + restore_mysql_finalize "${base_dir}" + ;; + inc) + if [ -z "${4:-}" ]; then + log_error "用法: restore.sh mysql inc " + exit 1 + fi + base_dir="$(restore_mysql_full "${3}")" + restore_mysql_inc "${base_dir}" "${4}" + restore_mysql_finalize "${base_dir}" + ;; + *) + echo "用法: restore.sh mysql {full|inc} [inc_timestamp]" + exit 1 + ;; + esac + ;; + postgres) + restore_postgres "${2}" "${3}" + ;; + marzban) + marzban_archive="$(download_and_decrypt "marzban/marzban_${2}.sqlite3.gz")" + log_ok "[Marzban] 解密文件: ${marzban_archive}" + ;; + *) + echo "用法: restore.sh {mysql|postgres|marzban} ..." + echo " restore.sh mysql full " + echo " restore.sh mysql inc " + echo " restore.sh postgres " + echo " restore.sh marzban " + exit 1 + ;; +esac diff --git a/deploy/backup/scripts/verify-rpo-rto.sh b/deploy/backup/scripts/verify-rpo-rto.sh new file mode 100644 index 0000000..fa8419a --- /dev/null +++ b/deploy/backup/scripts/verify-rpo-rto.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# verify-rpo-rto.sh —— 实时 RPO/RTO 状态检查 +# 用法： +# verify-rpo-rto.sh # 完整检查并打印结果 +# verify-rpo-rto.sh --status-only # 仅打印摘要（容器启动时调用） +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "${SCRIPT_DIR}/lib.sh" + +STATUS_ONLY="${1:-}" + +log_info "RPO/RTO 状态检查" + +RPO_TARGET_SECONDS=900 # 15 分钟 +RTO_TARGET_HOURS=4 + +PASS=0 +WARN=0 +FAIL=0 + +check() { + local status="$1" + local msg="$2" + case "${status}" in + pass) PASS=$(( PASS + 1 )); log_ok " ✅ ${msg}" ;; + warn) WARN=$(( WARN + 1 )); log_warn " ⚠️ ${msg}" ;; + fail) FAIL=$(( FAIL + 1 )); log_error " ❌ ${msg}" ;; + esac +} + +# ────────────────────────────────────────────────────── +# 1. 检查 S3 最新增量备份的时间 +# ────────────────────────────────────────────────────── +check_latest_backup_age() { + local service="$1" + local s3_prefix="$2" + + local latest + latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_prefix}/" \ + --recursive 2>/dev/null \ + | sort | tail -1 | awk '{print $1, $2}' || echo "")" + + if [ -z "${latest}" ]; then + check fail "${service}: S3 中无备份" + return + fi + + local last_date last_time + last_date=$(echo "${latest}" | awk '{print $1}') + last_time=$(echo "${latest}" | awk '{print $2}') + + local last_ts + last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \ + || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \ + || echo 0)" + local now + now=$(date +%s) + local gap=$(( now - last_ts )) + local gap_min=$(( gap / 60 )) + + if [ "${gap}" -le "${RPO_TARGET_SECONDS}" ]; then + check pass "${service}: 最新备份距今 ${gap_min}min (RPO ≤ 15min ✓)" + elif [ "${gap}" -le $(( RPO_TARGET_SECONDS * 2 )) ]; then + check warn "${service}: 最新备份距今 ${gap_min}min (接近 RPO 限制)" + else + check fail "${service}: 最新备份距今 ${gap_min}min (超出 RPO 15min 限制)" + fi +} + +# ────────────────────────────────────────────────────── +# 2. 检查 S3 最新全量备份时间（应为 < 25 小时） +# ────────────────────────────────────────────────────── +check_full_backup_freshness() { + local service="$1" + local s3_prefix="$2" + + local latest + latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/${s3_prefix}/" \ + 2>/dev/null | sort | tail -1 | awk '{print $1, $2}' || echo "")" + + if [ -z "${latest}" ]; then + check fail "${service} 全量: S3 中无全量备份" + return + fi + + local last_date last_time + last_date=$(echo "${latest}" | awk '{print $1}') + last_time=$(echo "${latest}" | awk '{print $2}') + + local last_ts + last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \ + || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \ + || echo 0)" + local now + now=$(date +%s) + local gap_hours=$(( (now - last_ts) / 3600 )) + + if [ "${gap_hours}" -le 25 ]; then + check pass "${service} 全量: 最近 ${gap_hours}h 内有全量备份" + else + check fail "${service} 全量: 上次全量备份距今 ${gap_hours}h (> 25h)" + fi +} + +# ────────────────────────────────────────────────────── +# 3. 检查 DR 演练报告（最新演练是否在 35 天内） +# ────────────────────────────────────────────────────── +check_dr_drill_freshness() { + local latest + latest="$(AWS_PROFILE="${AWS_PROFILE}" aws s3 ls \ + "s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/dr-reports/" \ + 2>/dev/null | sort | tail -1 | awk '{print $1, $2}' || echo "")" + + if [ -z "${latest}" ]; then + check warn "DR 演练: S3 中无演练报告（首次部署？）" + return + fi + + local last_date last_time + last_date=$(echo "${latest}" | awk '{print $1}') + last_time=$(echo "${latest}" | awk '{print $2}') + + local last_ts + last_ts="$(date -d "${last_date} ${last_time}" +%s 2>/dev/null \ + || date -j -f '%Y-%m-%d %H:%M:%S' "${last_date} ${last_time}" +%s 2>/dev/null \ + || echo 0)" + local now + now=$(date +%s) + local gap_days=$(( (now - last_ts) / 86400 )) + + if [ "${gap_days}" -le 35 ]; then + check pass "DR 演练: 上次演练距今 ${gap_days} 天" + else + check warn "DR 演练: 上次演练距今 ${gap_days} 天 (> 35 天，请检查)" + fi +} + +# ────────────────────────────────────────────────────── +# 执行检查 +# ────────────────────────────────────────────────────── +check_latest_backup_age "MySQL-增量" "mysql/inc" +check_latest_backup_age "PG-增量" "postgres/inc" +check_full_backup_freshness "MySQL" "mysql/full" +check_full_backup_freshness "PostgreSQL" "postgres/full" +check_dr_drill_freshness + +# ────────────────────────────────────────────────────── +# 汇总 +# ────────────────────────────────────────────────────── +echo "" +log_info "RPO/RTO 检查汇总" +log_info " 通过: ${PASS} 警告: ${WARN} 失败: ${FAIL}" +log_info " RPO 目标: ≤ 15min | RTO 目标: ≤ ${RTO_TARGET_HOURS}h" + +if [ "${FAIL}" -gt 0 ]; then + log_error "状态: 不达标 ❌" + exit 1 +elif [ "${WARN}" -gt 0 ]; then + log_warn "状态: 警告 ⚠️" + exit 0 +else + log_ok "状态: 达标 ✅" + exit 0 +fi diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index bda7b56..317d7cd 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -50,6 +50,42 @@ services: - ./secrets/cloudflare.ini:/etc/cloudflare.ini:ro entrypoint: ["certbot"] + # ────────────────────────────────────────────────────────────────────────── + # 备份与灾备(xtrabackup 全量 + binlog 增量 + age 加密 + S3 异地) + # 使用方法： + # 1. cp backup/backup.env.example secrets/backup.env && vi secrets/backup.env + # 2. docker build -t pangolin-backup:local ./backup + # 3. docker-compose -p pangolin up -d backup + # 注意： + # - 使用 network_mode: host，可访问 127.0.0.1 上的 MySQL/PostgreSQL + # - MySQL 数据目录挂载为只读（xtrabackup 需要）；若 MySQL 在 Docker 卷中， + # 修改 MYSQL_DATADIR_HOST 指向实际卷路径（如 /var/lib/docker/volumes/jiu_jiu_mysql_data/_data） + # - /opt/backup 是宿主备份暂存目录，重建容器后数据不丢失 + # - age 私钥不存服务器；如需 DR 演练解密恢复，临时注入 AGE_PRIVATE_KEY 环境变量 + # ────────────────────────────────────────────────────────────────────────── + backup: + # 镜像由脚本预先构建：docker build -t pangolin-backup:local ./backup + image: pangolin-backup:local + container_name: pangolin-backup + restart: unless-stopped + network_mode: host + env_file: + - ./secrets/backup.env + volumes: + # 宿主备份暂存目录（全量 / 增量 / binlog 解密临时文件） + - /opt/backup:/backup + # MySQL 数据目录（只读挂载供 xtrabackup） + # 若 MySQL 在 Docker 卷内，替换为实际路径： + # - /var/lib/docker/volumes//_data:/var/lib/mysql:ro + - ${MYSQL_DATADIR_HOST:-/var/lib/mysql}:/var/lib/mysql:ro + # Marzban SQLite 数据目录（只读） + - ${MARZBAN_DATA_HOST:-/var/lib/marzban}:/var/lib/marzban:ro + healthcheck: + test: ["CMD", "pgrep", "cron"] + interval: 5m + timeout: 10s + retries: 3 + # 新节点用的 REALITY —— 本台不启用(REALITY 仍由 Marzban 提供) xray: image: ghcr.io/xtls/xray-core:26.3.27 diff --git a/deploy/scripts/gen-secrets.sh b/deploy/scripts/gen-secrets.sh index eb1b7f8..a3106ec 100755 --- a/deploy/scripts/gen-secrets.sh +++ b/deploy/scripts/gen-secrets.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash # gen-secrets.sh —— 幂等生成本台所需密钥/证书,持久化到 ./secrets/(700,gitignore)。 -# - pangolin.env : HY2_PASSWORD -# - hy2/hy2.crt,hy2.key: Hysteria2 自签证书(ECDSA) -# - cloudflare.ini : 由 $CF_API_TOKEN 渲染(证书续期用),600 +# - pangolin.env : HY2_PASSWORD +# - hy2/hy2.crt,hy2.key : Hysteria2 自签证书(ECDSA) +# - cloudflare.ini : 由 $CF_API_TOKEN 渲染(证书续期用),600 +# - backup/age-keypair.txt : age 加密密钥对(私钥请离线保存后从服务器删除) +# - backup.env : 备份配置模板(首次生成,需手动填写真实值) set -euo pipefail cd "$(dirname "$0")/.." # -> deploy/ @@ -43,3 +45,51 @@ if [[ -n "${CF_API_TOKEN:-}" ]]; then else echo "[gen-secrets] 未设置 CF_API_TOKEN,跳过 cloudflare.ini(续期前需补)" fi + +# ────────────────────────────────────────────────────────── +# age 备份加密密钥对 +# 首次生成后： +# 1. 将 age-keypair.txt 里的私钥（AGE-SECRET-KEY-...）离线保存 +# 2. 从服务器删除私钥（只保留公钥供加密） +# 3. 将公钥填入 secrets/backup.env 的 AGE_PUBLIC_KEY 字段 +# ────────────────────────────────────────────────────────── +BACKUP_DIR="${SECRETS_DIR}/backup" +AGE_KEYPAIR="${BACKUP_DIR}/age-keypair.txt" +mkdir -p "${BACKUP_DIR}" +chmod 700 "${BACKUP_DIR}" + +if [[ -f "${AGE_KEYPAIR}" ]]; then + echo "[gen-secrets] age 密钥对已存在,跳过 (${AGE_KEYPAIR})" +else + # 检查 age-keygen 是否可用（需要先在容器内或本机安装） + if command -v age-keygen > /dev/null 2>&1; then + umask 177 + age-keygen 2>&1 | tee "${AGE_KEYPAIR}" + chmod 600 "${AGE_KEYPAIR}" + AGE_PUB="$(grep '^# public key:' "${AGE_KEYPAIR}" | awk '{print $NF}')" + echo "" + echo "[gen-secrets] ╔═══════════════════════════════════════════════════╗" + echo "[gen-secrets] ║ age 密钥对已生成: ${AGE_KEYPAIR}" + echo "[gen-secrets] ║ 公钥: ${AGE_PUB}" + echo "[gen-secrets] ║" + echo "[gen-secrets] ║ ⚠️ 重要：请立即离线备份私钥，然后从服务器删除：" + echo "[gen-secrets] ║ grep 'AGE-SECRET' ${AGE_KEYPAIR} # 私钥" + echo "[gen-secrets] ║ rm ${AGE_KEYPAIR} # 服务器上删除" + echo "[gen-secrets] ╚═══════════════════════════════════════════════════╝" + echo "" + else + echo "[gen-secrets] age-keygen 未安装，跳过密钥对生成" + echo "[gen-secrets] 请在备份容器内执行: age-keygen | tee ${AGE_KEYPAIR}" + fi +fi + +# 备份配置模板（首次生成，需手动填写） +BACKUP_ENV="${SECRETS_DIR}/backup.env" +if [[ -f "${BACKUP_ENV}" ]]; then + echo "[gen-secrets] ${BACKUP_ENV} 已存在,跳过" +else + cp ./backup/backup.env.example "${BACKUP_ENV}" + chmod 600 "${BACKUP_ENV}" + echo "[gen-secrets] 已生成备份配置模板: ${BACKUP_ENV}" + echo "[gen-secrets] ⚠️ 请编辑此文件并填写真实 S3 / MySQL / PostgreSQL / age 配置" +fi diff --git a/docs/备份与灾备.md b/docs/备份与灾备.md new file mode 100644 index 0000000..f469364 --- /dev/null +++ b/docs/备份与灾备.md @@ -0,0 +1,371 @@ +# Pangolin 备份与灾备手册 + +> 目标：**RPO ≤ 15min，RTO ≤ 4h** +> +> 策略：xtrabackup 每日全量 + binlog 每 15min 增量，age 加密，异地 S3（独立身份账号）存储；月度恢复演练自动化。 + +--- + +## 架构一览 + +``` +EC2 (生产) + ├── MySQL (jiu_mysql, marzban) + │ ├── 每日 02:00 xtrabackup 全量 ─────────────────────┐ + │ └── 每 15min xtrabackup 增量 + binlog flush ──────┤ + ├── PostgreSQL (blog, umami) │ + │ ├── 每日 02:00 pg_dump 全量 ────────────────────── │ + │ └── 每 15min pg_dump 快照 ────────────────────── age 加密 + └── Marzban SQLite │ + └── 每日 02:00 sqlite3 .backup ─────────────────────┘ + ↓ + S3（独立 IAM，仅 PutObject） + s3://pangolin-backup-XXXXXX/ + ├── mysql/full/ + ├── mysql/inc/ + ├── mysql/binlog/ + ├── postgres/full/ + ├── postgres/inc/ + ├── marzban/ + └── dr-reports/ + +每月 1 日 03:00 DR 演练：下载 → 解密 → 验证 → 报告 +``` + +--- + +## 初始化步骤（首次部署） + +### 1. 创建备份 S3 Bucket（独立 IAM 账号） + +> 使用**与生产 AWS 账号隔离**的独立 IAM 用户，做到数据面与备份面隔离。 + +```bash +# 在 AWS Console 或 CLI 执行（使用管理员账号）： +aws s3api create-bucket \ + --bucket pangolin-backup-$(openssl rand -hex 4) \ + --region ap-southeast-1 \ + --create-bucket-configuration LocationConstraint=ap-southeast-1 + +# 开启版本控制（防误删） +aws s3api put-bucket-versioning \ + --bucket YOUR_BUCKET_NAME \ + --versioning-configuration Status=Enabled + +# 配置生命周期：标准 IA 30 天后 Glacier，365 天后删除 +cat > /tmp/lifecycle.json << 'EOF' +{ + "Rules": [{ + "ID": "backup-lifecycle", + "Status": "Enabled", + "Filter": {}, + "Transitions": [ + {"Days": 30, "StorageClass": "STANDARD_IA"}, + {"Days": 90, "StorageClass": "GLACIER"} + ], + "Expiration": {"Days": 365} + }] +} +EOF +aws s3api put-bucket-lifecycle-configuration \ + --bucket YOUR_BUCKET_NAME \ + --lifecycle-configuration file:///tmp/lifecycle.json +``` + +**创建只写 IAM 用户（备份服务器使用）：** + +```json +// backup-only-policy.json（只允许 PutObject，不允许读取或删除） +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + }] +} +``` + +```bash +aws iam create-user --user-name pangolin-backup-writer +aws iam put-user-policy --user-name pangolin-backup-writer \ + --policy-name backup-only --policy-document file://backup-only-policy.json +aws iam create-access-key --user-name pangolin-backup-writer +# 记录 AccessKeyId 和 SecretAccessKey +``` + +**创建只读 IAM 用户（DR 演练时恢复用）：** + +```json +// restore-policy.json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:ListBucket"], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + }] +} +``` + +### 2. 生成 age 加密密钥对 + +```bash +# 在备份容器内（或本机有 age 的地方）生成： +docker run --rm pangolin-backup:local age-keygen 2>&1 | tee age-keypair.txt + +# 输出示例： +# Public key: age1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# AGE-SECRET-KEY-1XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + +# ⚠️ 重要安全操作： +# 1. 将私钥（AGE-SECRET-KEY-1...）存入密码管理器（如 Bitwarden） +# 2. 打印并存入保险箱（物理备份） +# 3. 删除本地私钥文件，服务器上只保留公钥 +rm age-keypair.txt # 删除含私钥的文件 +``` + +或使用 `gen-secrets.sh`（需本机已安装 age-keygen）： + +```bash +cd deploy/ +./scripts/gen-secrets.sh # 自动生成，见输出指示处理私钥 +``` + +### 3. 配置备份环境变量 + +```bash +cd deploy/ +cp backup/backup.env.example secrets/backup.env +vim secrets/backup.env # 填写以下关键项： +# - BACKUP_S3_BUCKET、BACKUP_AWS_ACCESS_KEY_ID/SECRET +# - AGE_PUBLIC_KEY（从步骤 2 获取） +# - MYSQL_USER/PASSWORD、PG_USER/PASSWORD +# - PG_DATABASES（如 blog,umami） +``` + +### 4. 准备 MySQL 备份账号 + +在 MySQL 中执行（通过 mysql 客户端连接 jiu_mysql）： + +```sql +CREATE USER 'backup_user'@'%' IDENTIFIED BY 'STRONG_PASSWORD'; +GRANT RELOAD, LOCK TABLES, REPLICATION CLIENT, PROCESS, SUPER, + CREATE, INSERT, SELECT, BACKUP_ADMIN ON *.* TO 'backup_user'@'%'; +FLUSH PRIVILEGES; +``` + +确认 binlog 已开启： + +```sql +SHOW VARIABLES LIKE 'log_bin'; -- 应为 ON +SHOW BINARY LOGS; -- 列出 binlog 文件 +``` + +若未开启，在 MySQL 配置中添加（需重启 MySQL）： + +```ini +[mysqld] +log_bin = mysql-bin +binlog_format = ROW +expire_logs_days = 3 +``` + +### 5. 构建并启动备份容器 + +```bash +cd deploy/ +# 构建镜像（宿主 docker-compose 不支持 compose build） +docker build -t pangolin-backup:local ./backup + +# 如果 MySQL 在 Docker 卷内，先确认卷路径： +docker inspect | grep -A5 Mounts + +# 启动（修改 .env 添加必要的主机路径变量） +MYSQL_DATADIR_HOST=/var/lib/docker/volumes/jiu_jiu_mysql_data/_data \ +MARZBAN_DATA_HOST=/opt/marzban \ +docker-compose -p pangolin up -d backup + +# 查看日志 +docker logs -f pangolin-backup +``` + +--- + +## 日常运维 + +### 查看备份状态（RPO/RTO 实时检查） + +```bash +docker exec pangolin-backup /scripts/verify-rpo-rto.sh +``` + +输出示例： + +``` +[OK] MySQL-增量: 最新备份距今 8min (RPO ≤ 15min ✓) +[OK] PG-增量: 最新备份距今 7min (RPO ≤ 15min ✓) +[OK] MySQL 全量: 最近 14h 内有全量备份 +[OK] DR 演练: 上次演练距今 12 天 +状态: 达标 ✅ +``` + +### 查看 cron 日志 + +```bash +docker logs --tail 100 pangolin-backup +# 或 +docker exec pangolin-backup tail -50 /var/log/backup/cron.log +``` + +### 手动触发全量备份 + +```bash +docker exec pangolin-backup bash -c '. /etc/backup-env.sh && /scripts/backup-full.sh' +``` + +### 手动触发 DR 演练 + +```bash +# 如需验证解密恢复，临时注入私钥 +docker exec -e AGE_PRIVATE_KEY="$(cat age-private.txt)" \ + pangolin-backup bash -c '. /etc/backup-env.sh && /scripts/dr-drill.sh' +``` + +### 列出 S3 备份文件 + +```bash +docker exec pangolin-backup bash -c '. /etc/backup-env.sh && \ + AWS_PROFILE=backup aws s3 ls s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/ --recursive \ + | sort | tail -20' +``` + +--- + +## 灾备恢复流程（实际灾难时） + +> **前提**：已有 age 私钥（从密码管理器取出），已有恢复目标服务器 + +### MySQL 完整恢复 + +```bash +# 1. 将私钥挂载到恢复容器 +docker run -it --rm \ + -e AGE_PRIVATE_KEY="$(cat age-private.txt)" \ + -e AWS_PROFILE=restore \ + # ... 其他环境变量 ... + -v /restore:/restore \ + pangolin-backup:local bash + +# 2. 在容器内执行恢复 +RESTORE_DIR=/restore /scripts/restore.sh mysql full 20240101_020000 + +# 3. 如需应用增量（最近一次全量后的某个增量时间点） +RESTORE_DIR=/restore /scripts/restore.sh mysql inc 20240101_020000 20240101_150000 + +# 4. 容器会提示手动操作步骤： +# - systemctl stop mysql +# - rm -rf /var/lib/mysql/* +# - xtrabackup --copy-back --target-dir=/restore/mysql/20240101_020000 +# - chown -R mysql:mysql /var/lib/mysql +# - systemctl start mysql +``` + +### PostgreSQL 完整恢复 + +```bash +RESTORE_DIR=/restore /scripts/restore.sh postgres blog 20240101_150000 +# 恢复到 blog_restored 数据库，验证后再重命名 +``` + +### RTO 测量标准 + +| 步骤 | 预计耗时 | +|------|---------| +| S3 下载全量备份（50GB）| ~30min | +| age 解密 | ~2min | +| xtrabackup --prepare | ~20min | +| 数据文件复制（rsync）| ~30min | +| MySQL 启动 + 应用增量/binlog | ~30min | +| 验证（基本查询）| ~10min | +| **合计** | **~2h（远低于 4h）** | + +--- + +## 月度 DR 演练报告 + +演练报告自动上传至：`s3://YOUR_BUCKET/ec2-ap-southeast-1/dr-reports/drill_YYYYMMDD_HHMMSS.txt` + +本地日志：`docker exec pangolin-backup tail -100 /var/log/backup/dr-drill.log` + +演练覆盖项目： + +| 检查项 | 标准 | +|--------|------| +| RPO 验证 | 最新备份距今 ≤ 15min | +| 备份文件完整性 | tar 归档可读，age 解密成功 | +| MySQL xtrabackup prepare | 无错误 | +| MySQL 临时恢复验证 | 容器启动后可执行查询 | +| PostgreSQL 备份结构验证 | pg_restore --list 通过 | +| RTO 演练总耗时 | ≤ 4h | + +--- + +## 安全要点 + +- **age 私钥不存服务器**：加密用公钥，解密用私钥（离线保存） +- **独立 IAM 账号**：备份写入账号仅有 `s3:PutObject`，即使服务器被入侵也无法读取历史备份 +- **加密存储**：所有备份在上传前均 age 加密，S3 中无明文数据 +- **版本控制**：S3 开启版本控制，防止误操作覆盖 +- **生命周期**：30 天 → IA，90 天 → Glacier，365 天 → 删除（成本控制） +- **凭证不入库**：`deploy/secrets/` 在 `.gitignore` 中 + +--- + +## 故障排查 + +### 备份容器无法连接 MySQL + +```bash +# 检查 MySQL 是否可达 +docker exec pangolin-backup mysql \ + --host=127.0.0.1 --port=3306 \ + --user=backup_user --password=YOUR_PW \ + -e "SELECT 1" +# 若失败：检查防火墙、MySQL bind-address（应为 0.0.0.0 或 127.0.0.1） +``` + +### xtrabackup 提示版本不匹配 + +xtrabackup 8.0 只支持 MySQL 8.0。若生产 MySQL 是 5.7，需修改 Dockerfile 安装 `percona-xtrabackup-24`： + +```dockerfile +RUN percona-release setup pxb24 \ + && apt-get install -y percona-xtrabackup-24 +``` + +### S3 上传失败 + +```bash +# 检查凭据 +docker exec pangolin-backup bash -c 'AWS_PROFILE=backup aws sts get-caller-identity' +# 检查 bucket 权限 +docker exec pangolin-backup bash -c \ + "AWS_PROFILE=backup aws s3 cp /dev/null s3://${BACKUP_S3_BUCKET}/test-write && echo OK" +``` + +### DR 演练中 Docker 不可用 + +演练脚本中的 MySQL 临时容器启动依赖 Docker-in-Docker 或宿主 Docker socket。 +若不可用，脚本会跳过容器验证，只做 `xtrabackup --prepare` 完整性验证。