diff --git a/.github/workflows/mysql-backup-deploy.yml b/.github/workflows/mysql-backup-deploy.yml new file mode 100644 index 0000000..d4be1df --- /dev/null +++ b/.github/workflows/mysql-backup-deploy.yml @@ -0,0 +1,230 @@ +name: MySQL Backup Deploy + +on: + workflow_dispatch: + inputs: + mode: + description: validate는 상태만 확인하고 install은 백업 구성을 설치합니다. + required: true + default: validate + type: choice + options: + - validate + - install + +permissions: + id-token: write + contents: read + +concurrency: + group: prod-mysql-backup-deploy + cancel-in-progress: false + +env: + AWS_REGION: ap-northeast-2 + LOCAL_SSH_PORT: "2222" + SESSION_MANAGER_PLUGIN_VERSION: "1.2.835.0" + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Install Session Manager plugin + run: | + set -Eeuo pipefail + plugin_dir="$(mktemp -d "$RUNNER_TEMP/session-manager-plugin.XXXXXX")" + trap 'rm -rf -- "$plugin_dir"' EXIT + plugin_base_url="https://s3.amazonaws.com/session-manager-downloads/plugin/${SESSION_MANAGER_PLUGIN_VERSION}/ubuntu_64bit" + + curl -fsSL "$plugin_base_url/session-manager-plugin.deb" \ + -o "$plugin_dir/session-manager-plugin.deb" + curl -fsSL "$plugin_base_url/session-manager-plugin.deb.sig" \ + -o "$plugin_dir/session-manager-plugin.deb.sig" + + cat >"$plugin_dir/session-manager-plugin.gpg" <<'EOF' + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mFIEZ5ERQxMIKoZIzj0DAQcCAwQjuZy+IjFoYg57sLTGhF3aZLBaGpzB+gY6j7Ix + P7NqbpXyjVj8a+dy79gSd64OEaMxUb7vw/jug+CfRXwVGRMNtIBBV1MgU1NNIFNl + c3Npb24gTWFuYWdlciA8c2Vzc2lvbi1tYW5hZ2VyLXBsdWdpbi1zaWduZXJAYW1h + em9uLmNvbT4gKEFXUyBTeXN0ZW1zIE1hbmFnZXIgU2Vzc2lvbiBNYW5hZ2VyIFBs + dWdpbiBMaW51eCBTaWduZXIgS2V5KYkBAAQQEwgAqAUCZ5ERQ4EcQVdTIFNTTSBT + ZXNzaW9uIE1hbmFnZXIgPHNlc3Npb24tbWFuYWdlci1wbHVnaW4tc2lnbmVyQGFt + YXpvbi5jb20+IChBV1MgU3lzdGVtcyBNYW5hZ2VyIFNlc3Npb24gTWFuYWdlciBQ + bHVnaW4gTGludXggU2lnbmVyIEtleSkWIQR5WWNxJM4JOtUB1HosTUr/b2dX7gIe + AwIbAwIVCAAKCRAsTUr/b2dX7rO1AQCa1kig3lQ78W/QHGU76uHx3XAyv0tfpE9U + oQBCIwFLSgEA3PDHt3lZ+s6m9JLGJsy+Cp5ZFzpiF6RgluR/2gA861M= + =2DQm + -----END PGP PUBLIC KEY BLOCK----- + EOF + + export GNUPGHOME="$plugin_dir/gnupg" + install -d -m 700 "$GNUPGHOME" + gpg --batch --import "$plugin_dir/session-manager-plugin.gpg" + signing_key_fingerprint="$(gpg --batch --with-colons --fingerprint 2C4D4AFF6F6757EE | awk -F: '$1 == "fpr" {print $10; exit}')" + if [[ "$signing_key_fingerprint" != "7959637124CE093AD501D47A2C4D4AFF6F6757EE" ]]; then + echo "::error::Unexpected Session Manager plugin signing key fingerprint" + exit 1 + fi + gpg --batch --verify \ + "$plugin_dir/session-manager-plugin.deb.sig" \ + "$plugin_dir/session-manager-plugin.deb" + + package_version="$(dpkg-deb --field "$plugin_dir/session-manager-plugin.deb" Version)" + if [[ "$package_version" != "${SESSION_MANAGER_PLUGIN_VERSION}-1" ]]; then + echo "::error::Unexpected Session Manager plugin package version: $package_version" + exit 1 + fi + sudo dpkg -i "$plugin_dir/session-manager-plugin.deb" + echo "/usr/local/sessionmanagerplugin/bin" >> "$GITHUB_PATH" + + - name: Validate or install MySQL backup + env: + DEPLOY_MODE: ${{ inputs.mode }} + MYSQL_BACKUP_BUCKET: ${{ vars.MYSQL_BACKUP_BUCKET_NAME }} + MYSQL_DATABASE: ${{ vars.MYSQL_BACKUP_DATABASE_NAME }} + DB_HOST_FINGERPRINT: ${{ vars.PROD_DB_SSH_HOST_KEY_ED25519 }} + run: | + set -Eeuo pipefail + umask 077 + + : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET_NAME repository variable is required}" + : "${MYSQL_DATABASE:?MYSQL_BACKUP_DATABASE_NAME repository variable is required}" + : "${DB_HOST_FINGERPRINT:?PROD_DB_SSH_HOST_KEY_ED25519 repository variable is required}" + if [[ "$DEPLOY_MODE" != "validate" && "$DEPLOY_MODE" != "install" ]]; then + echo "::error::Invalid deployment mode" + exit 1 + fi + if [[ ! "$MYSQL_BACKUP_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]]; then + echo "::error::Invalid S3 bucket name" + exit 1 + fi + if [[ ! "$MYSQL_DATABASE" =~ ^[A-Za-z0-9_]+$ ]]; then + echo "::error::Invalid database name" + exit 1 + fi + if [[ ! "$DB_HOST_FINGERPRINT" =~ ^SHA256:[A-Za-z0-9+/]{43}$ ]]; then + echo "::error::Invalid DB EC2 SSH host key fingerprint" + exit 1 + fi + KEY_DIR="$(mktemp -d "$RUNNER_TEMP/mysql-backup-eic.XXXXXX")" + SSM_PID="" + cleanup() { + if [[ -n "$SSM_PID" ]]; then + kill "$SSM_PID" 2>/dev/null || true + wait "$SSM_PID" 2>/dev/null || true + fi + rm -rf "$KEY_DIR" + } + trap cleanup EXIT + + ssh-keygen -q -t ed25519 -N '' -f "$KEY_DIR/id_ed25519" + + API_INSTANCE_ID="$(aws ec2 describe-instances \ + --filters \ + 'Name=tag:Name,Values=solid-connection-server-prod' \ + 'Name=instance-state-name,Values=running' \ + --query 'Reservations[].Instances[].InstanceId' \ + --output text)" + DB_INSTANCE_ID="$(aws ec2 describe-instances \ + --filters \ + 'Name=tag:Name,Values=solid-connection-db-mysql-prod' \ + 'Name=instance-state-name,Values=running' \ + --query 'Reservations[].Instances[].InstanceId' \ + --output text)" + + if [[ "$API_INSTANCE_ID" == *$'\t'* || -z "$API_INSTANCE_ID" || "$API_INSTANCE_ID" == "None" ]]; then + echo "::error::Exactly one running Prod API EC2 instance is required" + exit 1 + fi + if [[ "$DB_INSTANCE_ID" == *$'\t'* || -z "$DB_INSTANCE_ID" || "$DB_INSTANCE_ID" == "None" ]]; then + echo "::error::Exactly one running Prod DB EC2 instance is required" + exit 1 + fi + + DB_PRIVATE_IP="$(aws ec2 describe-instances \ + --instance-ids "$DB_INSTANCE_ID" \ + --query 'Reservations[0].Instances[0].PrivateIpAddress' \ + --output text)" + if [[ -z "$DB_PRIVATE_IP" || "$DB_PRIVATE_IP" == "None" ]]; then + echo "::error::Prod DB EC2 private IP was not found" + exit 1 + fi + + aws ssm start-session \ + --target "$API_INSTANCE_ID" \ + --document-name AWS-StartPortForwardingSessionToRemoteHost \ + --parameters "{\"host\":[\"$DB_PRIVATE_IP\"],\"portNumber\":[\"22\"],\"localPortNumber\":[\"$LOCAL_SSH_PORT\"]}" \ + >"$KEY_DIR/ssm-tunnel.log" 2>&1 & + SSM_PID=$! + + for _ in $(seq 1 60); do + kill -0 "$SSM_PID" 2>/dev/null || { + cat "$KEY_DIR/ssm-tunnel.log" >&2 + exit 1 + } + nc -z 127.0.0.1 "$LOCAL_SSH_PORT" 2>/dev/null && break + sleep 1 + done + nc -z 127.0.0.1 "$LOCAL_SSH_PORT" + + KNOWN_HOSTS_FILE="$KEY_DIR/known_hosts" + ssh-keyscan -t ed25519 -p "$LOCAL_SSH_PORT" 127.0.0.1 >"$KNOWN_HOSTS_FILE" 2>/dev/null + ACTUAL_FINGERPRINT="$(ssh-keygen -lf "$KNOWN_HOSTS_FILE" -E sha256 | awk '{print $2}')" + if [[ "$ACTUAL_FINGERPRINT" != "$DB_HOST_FINGERPRINT" ]]; then + echo "::error::DB EC2 SSH host key fingerprint mismatch" + exit 1 + fi + + SSH_OPTIONS=( + -p "$LOCAL_SSH_PORT" + -i "$KEY_DIR/id_ed25519" + -o IdentitiesOnly=yes + -o StrictHostKeyChecking=yes + -o UserKnownHostsFile="$KNOWN_HOSTS_FILE" + -o ConnectTimeout=10 + ) + + if [[ "$DEPLOY_MODE" == "validate" ]]; then + aws ec2-instance-connect send-ssh-public-key \ + --region "$AWS_REGION" \ + --instance-id "$DB_INSTANCE_ID" \ + --instance-os-user ubuntu \ + --ssh-public-key "file://$KEY_DIR/id_ed25519.pub" >/dev/null + REMOTE_VALIDATE_COMMAND="env MYSQL_BACKUP_BUCKET=$(printf '%q' "$MYSQL_BACKUP_BUCKET") MYSQL_DATABASE=$(printf '%q' "$MYSQL_DATABASE") AWS_REGION=$(printf '%q' "$AWS_REGION") bash -s" + ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 \ + "sudo bash -c $(printf '%q' "$REMOTE_VALIDATE_COMMAND")" \ + < scripts/mysql_backup/validate-remote.sh + exit 0 + fi + + CONFIG_FILE="$KEY_DIR/mysql-backup.env" + { + printf 'MYSQL_BACKUP_BUCKET=%s\n' "$MYSQL_BACKUP_BUCKET" + printf 'MYSQL_DATABASE=%s\n' "$MYSQL_DATABASE" + printf 'AWS_REGION=%s\n' "$AWS_REGION" + } >"$CONFIG_FILE" + + cp -R scripts/mysql_backup "$KEY_DIR/bundle" + cp "$CONFIG_FILE" "$KEY_DIR/bundle/mysql-backup.env" + tar -C "$KEY_DIR/bundle" -czf "$KEY_DIR/mysql-backup.tar.gz" . + + aws ec2-instance-connect send-ssh-public-key \ + --region "$AWS_REGION" \ + --instance-id "$DB_INSTANCE_ID" \ + --instance-os-user ubuntu \ + --ssh-public-key "file://$KEY_DIR/id_ed25519.pub" >/dev/null + + REMOTE_INSTALL_COMMAND='set -Eeuo pipefail; install_dir="$(mktemp -d /tmp/mysql-backup-install.XXXXXX)"; trap '\''rm -rf -- "$install_dir"'\'' EXIT; tar -xzf - -C "$install_dir"; "$install_dir/install.sh" "$install_dir/mysql-backup.env"' + ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 \ + "sudo bash -c $(printf '%q' "$REMOTE_INSTALL_COMMAND")" \ + <"$KEY_DIR/mysql-backup.tar.gz" diff --git a/modules/app_stack/scripts/mysql_setup.sh.tftpl b/modules/app_stack/scripts/mysql_setup.sh.tftpl index 2c1c21e..de4557a 100644 --- a/modules/app_stack/scripts/mysql_setup.sh.tftpl +++ b/modules/app_stack/scripts/mysql_setup.sh.tftpl @@ -50,6 +50,7 @@ DATA_VOLUME_SERIAL="$(printf '%s' "$DATA_VOLUME_ID" | tr -d '-')" DATA_VOLUME_DEVICE="/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_$DATA_VOLUME_SERIAL" DATA_VOLUME_MOUNT="/mnt/mysql-data" MYSQL_DATA_DIR="$DATA_VOLUME_MOUNT/mysql" +MYSQL_BACKUP_DIR="$DATA_VOLUME_MOUNT/mysql-backup" for i in $(seq 1 120); do if [ -e "$DATA_VOLUME_DEVICE" ]; then @@ -95,6 +96,11 @@ mkdir -p "$MYSQL_DATA_DIR" chown -R 999:999 "$MYSQL_DATA_DIR" chmod 750 "$MYSQL_DATA_DIR" +# dump 압축 파일과 업로드 상태는 용량이 작은 루트 볼륨이 아닌 데이터 EBS에 둡니다. +mkdir -p "$MYSQL_BACKUP_DIR/staging" "$MYSQL_BACKUP_DIR/state" +chown -R root:root "$MYSQL_BACKUP_DIR" +chmod 700 "$MYSQL_BACKUP_DIR" "$MYSQL_BACKUP_DIR/staging" "$MYSQL_BACKUP_DIR/state" + mkdir -p /etc/mysql/conf.d cat > /etc/mysql/conf.d/tuning.cnf <<'CNFEOF' ${mysql_config_content} diff --git a/scripts/mysql_backup/README.md b/scripts/mysql_backup/README.md new file mode 100644 index 0000000..0a7f3e9 --- /dev/null +++ b/scripts/mysql_backup/README.md @@ -0,0 +1,22 @@ +# Prod MySQL S3 백업 + +DB EC2에서 다음 systemd 작업을 실행합니다. + +- `mysql-backup-dump.timer`: 매일 03:00 KST에 일관된 전체 dump, 체크섬, manifest를 업로드합니다. +- `mysql-backup-binlog.timer`: 매 5분 경계마다 binlog를 회전하고 닫힌 파일과 manifest를 업로드합니다. + +백업 파일은 `/mnt/mysql-data/mysql-backup`에서만 임시 생성합니다. S3에는 manifest를 마지막에 업로드하며, 같은 key가 이미 존재하면 체크섬이 같은 경우에만 업로드를 생략합니다. 이 방식으로 재시도 시 Object Lock 버킷에 불필요한 객체 버전이 생기는 것을 방지합니다. + +## 배포 전 GitHub 설정 + +Repository Secrets: + +- `AWS_ROLE_ARN`: 배포 워크플로우가 AssumeRole할 IAM 역할 ARN + +Repository Variables: + +- `MYSQL_BACKUP_BUCKET_NAME`: 백업 버킷 이름 +- `MYSQL_BACKUP_DATABASE_NAME`: 백업할 DB 이름. 필수값이며 공개 코드에 기본값을 두지 않습니다. +- `PROD_DB_SSH_HOST_KEY_ED25519`: DB EC2의 ED25519 host key SHA-256 fingerprint + +> `MySQL Backup Deploy` 워크플로우는 기본값인 `validate`에서 변경 없이 사전 조건 또는 설치 상태만 검사합니다. 실제 설치는 `install`을 명시적으로 선택해야 합니다. diff --git a/scripts/mysql_backup/bin/mysql-backup-binlog b/scripts/mysql_backup/bin/mysql-backup-binlog new file mode 100644 index 0000000..0317003 --- /dev/null +++ b/scripts/mysql_backup/bin/mysql-backup-binlog @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql-backup}" +# shellcheck source=../lib/backup-common.sh +source "$LIB_DIR/backup-common.sh" + +require_backup_environment +require_commands aws docker flock sha256sum + +exec 9>"$STATE_DIR/binlog.lock" +if ! flock -n 9; then + echo "Another binlog backup is already running." >&2 + exit 0 +fi + +readonly BINLOG_STATE_FILE="$STATE_DIR/binlog-last-uploaded" +readonly BINLOG_STATE_VERSION="1" + +state_bucket="" +state_uuid="" +last_uploaded="" +rotation_slot="-1" +rotation_from="-" +rotation_to="-" + +write_binlog_state() { + local persisted_last_uploaded="${last_uploaded:--}" + + printf '%s %s %s %s %s %s %s\n' \ + "$BINLOG_STATE_VERSION" \ + "$MYSQL_BACKUP_BUCKET" \ + "$database_server_uuid" \ + "$persisted_last_uploaded" \ + "$rotation_slot" \ + "$rotation_from" \ + "$rotation_to" >"$BINLOG_STATE_FILE.tmp" + mv "$BINLOG_STATE_FILE.tmp" "$BINLOG_STATE_FILE" +} + +read_active_binlog() { + mysql_query 'SHOW BINARY LOG STATUS;' | awk 'NR == 1 {print $1}' +} + +validate_binlog_name() { + [[ "$1" =~ ^binlog\.[0-9]{6}$ ]] +} + +database_server_uuid="$(server_uuid)" +active_binlog="$(read_active_binlog)" +binlog_basename="$(mysql_query 'SELECT @@log_bin_basename;')" +host_binlog_index="$MYSQL_DATA_DIR/$(basename "$binlog_basename").index" + +if ! validate_binlog_name "$active_binlog" || \ + [[ ! "$database_server_uuid" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then + echo "Unexpected MySQL binary log metadata." >&2 + exit 1 +fi +if [[ ! -r "$host_binlog_index" ]]; then + echo "MySQL binary log index is not readable: $host_binlog_index" >&2 + exit 1 +fi + +state_is_compatible=false +if [[ -s "$BINLOG_STATE_FILE" ]]; then + state_version="" + persisted_last_uploaded="" + read -r \ + state_version \ + state_bucket \ + state_uuid \ + persisted_last_uploaded \ + rotation_slot \ + rotation_from \ + rotation_to <"$BINLOG_STATE_FILE" || true + + if [[ "$persisted_last_uploaded" == "-" ]]; then + last_uploaded="" + else + last_uploaded="$persisted_last_uploaded" + fi + + if [[ "$state_version" == "$BINLOG_STATE_VERSION" && \ + "$state_bucket" == "$MYSQL_BACKUP_BUCKET" && \ + "$state_uuid" == "$database_server_uuid" && \ + "$rotation_slot" =~ ^-?[0-9]+$ && \ + ( -z "$last_uploaded" || "$last_uploaded" =~ ^binlog\.[0-9]{6}$ ) && \ + ( "$rotation_from" == "-" || "$rotation_from" =~ ^binlog\.[0-9]{6}$ ) && \ + ( "$rotation_to" == "-" || "$rotation_to" =~ ^binlog\.[0-9]{6}$ ) ]]; then + state_is_compatible=true + fi +fi + +if [[ "$state_is_compatible" != "true" ]]; then + # 상태 형식이나 목적지가 달라지면 현재 S3 버킷을 기준으로 체인을 다시 구성합니다. + uploaded_keys="$(aws s3api list-objects-v2 \ + --bucket "$MYSQL_BACKUP_BUCKET" \ + --prefix 'binlog/' \ + --region "$AWS_REGION" \ + --query "Contents[?contains(Key, '$database_server_uuid') && ends_with(Key, '.manifest.json')].Key" \ + --output text)" + last_uploaded="" + previous_uploaded="" + while IFS= read -r uploaded_binlog; do + if [[ -n "$previous_uploaded" ]]; then + previous_uploaded_number=$((10#${previous_uploaded##*.})) + uploaded_number=$((10#${uploaded_binlog##*.})) + if ((uploaded_number != previous_uploaded_number + 1)); then + echo "S3 binlog manifest gap detected between $previous_uploaded and $uploaded_binlog; recovering from the last contiguous manifest." >&2 + break + fi + fi + last_uploaded="$uploaded_binlog" + previous_uploaded="$uploaded_binlog" + done < <(printf '%s' "$uploaded_keys" | \ + tr '\t' '\n' | \ + sed -n 's/.*-\(binlog\.[0-9]\{6\}\)\.manifest\.json$/\1/p' | \ + sort -u) + if [[ -n "$last_uploaded" ]]; then + echo "Recovered the last uploaded binlog from S3: $last_uploaded" + fi + state_bucket="$MYSQL_BACKUP_BUCKET" + state_uuid="$database_server_uuid" + rotation_slot="-1" + rotation_from="-" + rotation_to="-" + write_binlog_state +fi + +if [[ -n "$last_uploaded" ]]; then + active_number=$((10#${active_binlog##*.})) + last_uploaded_number=$((10#${last_uploaded##*.})) + if ((active_number <= last_uploaded_number)); then + echo "Binary log numbering moved backwards. Clear the state only after starting a new verified backup chain." >&2 + exit 1 + fi +fi + +current_rotation_slot=$(( $(date -u +%s) / 300 )) +if [[ "$rotation_slot" != "$current_rotation_slot" ]]; then + # 회전 의도를 먼저 기록하여 FLUSH 성공 직후 프로세스가 종료되어도 재시도에서 중복 회전하지 않습니다. + rotation_slot="$current_rotation_slot" + rotation_from="$active_binlog" + rotation_to="-" + write_binlog_state +fi + +if [[ "$rotation_to" == "-" ]]; then + observed_active_binlog="$(read_active_binlog)" + if [[ "$observed_active_binlog" == "$rotation_from" ]]; then + mysql_query 'FLUSH BINARY LOGS;' + observed_active_binlog="$(read_active_binlog)" + fi + if ! validate_binlog_name "$observed_active_binlog" || [[ "$observed_active_binlog" == "$rotation_from" ]]; then + echo "MySQL binary log rotation did not advance from $rotation_from." >&2 + exit 1 + fi + rotation_to="$observed_active_binlog" + write_binlog_state +fi + +# 동일한 5분 구간의 재시도는 다시 FLUSH하지 않고 현재 active log를 기준으로 미업로드 파일만 처리합니다. +active_binlog="$(read_active_binlog)" +if ! validate_binlog_name "$active_binlog"; then + echo "Unexpected active binary log after rotation." >&2 + exit 1 +fi + +previous_binlog="$last_uploaded" +uploaded_count=0 +active_found=false + +while IFS= read -r indexed_path; do + binlog_name="$(basename "$indexed_path")" + if [[ "$binlog_name" == "$active_binlog" ]]; then + active_found=true + break + fi + if ! validate_binlog_name "$binlog_name"; then + echo "Unexpected binary log file name: $binlog_name" >&2 + exit 1 + fi + if [[ -n "$last_uploaded" && ("$binlog_name" < "$last_uploaded" || "$binlog_name" == "$last_uploaded") ]]; then + continue + fi + + if [[ -n "$previous_binlog" ]]; then + previous_number=$((10#${previous_binlog##*.})) + current_number=$((10#${binlog_name##*.})) + if ((current_number != previous_number + 1)); then + echo "Binary log gap detected between $previous_binlog and $binlog_name." >&2 + exit 1 + fi + fi + + binlog_file="$MYSQL_DATA_DIR/$binlog_name" + if [[ ! -s "$binlog_file" ]]; then + echo "Closed binary log file is missing or empty: $binlog_file" >&2 + exit 1 + fi + + closed_epoch="$(stat -c %Y "$binlog_file")" + closed_at="$(date -u -d "@$closed_epoch" +%Y-%m-%dT%H:%M:%SZ)" + key_date="$(date -u -d "@$closed_epoch" +%Y/%m/%d)" + key_time="$(date -u -d "@$closed_epoch" +%H%M%S)" + binlog_checksum="$(sha256_file "$binlog_file")" + binlog_size="$(stat -c %s "$binlog_file")" + manifest_file="$STAGING_DIR/${database_server_uuid}-${binlog_name}.manifest.json" + + cat >"$manifest_file" <&2 + exit 1 +fi +if [[ -n "$previous_binlog" ]]; then + previous_number=$((10#${previous_binlog##*.})) + active_number=$((10#${active_binlog##*.})) + if ((active_number != previous_number + 1)); then + echo "Binary log gap detected between $previous_binlog and active log $active_binlog." >&2 + exit 1 + fi +fi + +success_epoch="$(date -u +%s)" +printf '%s\n' "$success_epoch" >"$STATE_DIR/last-binlog-success.tmp" +mv "$STATE_DIR/last-binlog-success.tmp" "$STATE_DIR/last-binlog-success" +echo "binlog backup completed: uploaded=$uploaded_count active=$active_binlog" diff --git a/scripts/mysql_backup/bin/mysql-backup-dump b/scripts/mysql_backup/bin/mysql-backup-dump new file mode 100755 index 0000000..7f891a6 --- /dev/null +++ b/scripts/mysql_backup/bin/mysql-backup-dump @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql-backup}" +# shellcheck source=../lib/backup-common.sh +source "$LIB_DIR/backup-common.sh" + +require_backup_environment +require_commands aws docker flock gzip sha256sum + +exec 9>"$STATE_DIR/dump.lock" +if ! flock -n 9; then + echo "Another mysqldump backup is already running." >&2 + exit 0 +fi + +readonly CURRENT_JOB_FILE="$STATE_DIR/dump-current" +if [[ -s "$CURRENT_JOB_FILE" ]]; then + job_id="$(<"$CURRENT_JOB_FILE")" +else + job_id="$(date -u +%Y%m%dT%H%M%SZ)" + printf '%s\n' "$job_id" >"$CURRENT_JOB_FILE.tmp" + mv "$CURRENT_JOB_FILE.tmp" "$CURRENT_JOB_FILE" +fi + +if [[ ! "$job_id" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then + echo "Invalid persisted dump job id." >&2 + exit 1 +fi + +readonly JOB_DIR="$STAGING_DIR/dump-$job_id" +readonly DUMP_FILE="$JOB_DIR/${MYSQL_DATABASE}-${job_id}.sql.gz" +readonly CHECKSUM_FILE="$DUMP_FILE.sha256" +readonly MANIFEST_FILE="$JOB_DIR/manifest.json" + +# 완료 포인터 제거 후 프로세스가 종료되어 남은 이전 staging은 다음 실행에서 정리합니다. +for stale_job_dir in "$STAGING_DIR"/dump-*; do + if [[ -d "$stale_job_dir" && "$stale_job_dir" != "$JOB_DIR" ]]; then + rm -rf -- "$stale_job_dir" + fi +done +install -d -m 700 "$JOB_DIR" + +if [[ ! -s "$DUMP_FILE" ]]; then + partial_dump="$DUMP_FILE.partial" + rm -f "$partial_dump" + + docker exec "$MYSQL_CONTAINER" sh -lc \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysqldump -uroot --single-transaction --quick --source-data=2 --routines --events --triggers --hex-blob --set-gtid-purged=OFF --no-tablespaces "$1"' \ + sh "$MYSQL_DATABASE" | gzip -1 >"$partial_dump" + + test -s "$partial_dump" + mv "$partial_dump" "$DUMP_FILE" +fi + +dump_checksum="$(sha256_file "$DUMP_FILE")" +printf '%s %s\n' "$dump_checksum" "$(basename "$DUMP_FILE")" >"$CHECKSUM_FILE" + +source_status="$(gzip -dc "$DUMP_FILE" | grep -m1 '^-- CHANGE REPLICATION SOURCE TO' || true)" +if [[ -z "$source_status" ]]; then + echo "The dump does not contain a binary log recovery position." >&2 + exit 1 +fi + +source_file="$(sed -n "s/.*SOURCE_LOG_FILE='\([^']*\)'.*/\1/p" <<<"$source_status")" +source_position="$(sed -n 's/.*SOURCE_LOG_POS=\([0-9]*\).*/\1/p' <<<"$source_status")" +database_server_uuid="$(server_uuid)" +dump_size="$(stat -c %s "$DUMP_FILE")" +created_at="${job_id:0:4}-${job_id:4:2}-${job_id:6:2}T${job_id:9:2}:${job_id:11:2}:${job_id:13:2}Z" + +if [[ ! "$source_file" =~ ^binlog\.[0-9]{6}$ || ! "$source_position" =~ ^[0-9]+$ || ! "$database_server_uuid" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then + echo "Failed to parse dump recovery metadata." >&2 + exit 1 +fi + +cat >"$MANIFEST_FILE" <"$STATE_DIR/last-dump-success.tmp" +mv "$STATE_DIR/last-dump-success.tmp" "$STATE_DIR/last-dump-success" +rm -f "$CURRENT_JOB_FILE" +rm -rf "$JOB_DIR" +echo "mysqldump backup completed: $OBJECT_PREFIX" diff --git a/scripts/mysql_backup/bin/mysql-backup-validate b/scripts/mysql_backup/bin/mysql-backup-validate new file mode 100755 index 0000000..729fa76 --- /dev/null +++ b/scripts/mysql_backup/bin/mysql-backup-validate @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly LIB_DIR="${MYSQL_BACKUP_LIB_DIR:-/usr/local/lib/solid-connection/mysql-backup}" +# shellcheck source=../lib/backup-common.sh +source "$LIB_DIR/backup-common.sh" + +require_backup_environment +require_commands aws docker flock gzip sha256sum + +mountpoint -q /mnt/mysql-data +docker inspect "$MYSQL_CONTAINER" >/dev/null + +mysql_settings="$(mysql_query ' +SELECT @@log_bin, @@binlog_format, @@server_id, @@sync_binlog, @@innodb_flush_log_at_trx_commit; +')" +read -r log_bin binlog_format server_id sync_binlog flush_policy <<<"$mysql_settings" + +if [[ "$log_bin" != "1" || "$binlog_format" != "ROW" || "$server_id" == "0" ]]; then + echo "MySQL binlog configuration is not suitable for backup." >&2 + exit 1 +fi +if [[ "$sync_binlog" != "1" || "$flush_policy" != "1" ]]; then + echo "MySQL durability configuration is not sync_binlog=1 and innodb_flush_log_at_trx_commit=1." >&2 + exit 1 +fi + +schema_exists="$(mysql_query " +SELECT COUNT(*) +FROM information_schema.schemata +WHERE schema_name = '$MYSQL_DATABASE'; +")" +if [[ "$schema_exists" != "1" ]]; then + echo "MySQL backup database does not exist: $MYSQL_DATABASE" >&2 + exit 1 +fi + +database_bytes="$(mysql_query " +SELECT COALESCE(SUM(data_length + index_length), 0) +FROM information_schema.tables +WHERE table_schema = '$MYSQL_DATABASE'; +")" +available_bytes="$(df --output=avail -B1 "$BACKUP_ROOT" | tail -1 | tr -d ' ')" + +if [[ ! "$database_bytes" =~ ^[0-9]+$ || ! "$available_bytes" =~ ^[0-9]+$ ]]; then + echo "Failed to calculate database size or available backup space." >&2 + exit 1 +fi +required_bytes=$((database_bytes * 2 + 268435456)) + +if ((available_bytes < required_bytes)); then + echo "Insufficient backup staging space: available=$available_bytes required=$required_bytes" >&2 + exit 1 +fi + +aws s3api head-bucket \ + --bucket "$MYSQL_BACKUP_BUCKET" \ + --region "$AWS_REGION" >/dev/null + +echo "MySQL backup validation succeeded." +echo "database_bytes=$database_bytes available_bytes=$available_bytes" diff --git a/scripts/mysql_backup/install.sh b/scripts/mysql_backup/install.sh new file mode 100644 index 0000000..db79395 --- /dev/null +++ b/scripts/mysql_backup/install.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SOURCE_DIR +readonly CONFIG_SOURCE="${1:-$SOURCE_DIR/mysql-backup.env}" +readonly INSTALL_LIB_DIR="/usr/local/lib/solid-connection/mysql-backup" +readonly INSTALL_BIN_DIR="/usr/local/libexec/solid-connection" +readonly CONFIG_DIR="/etc/solid-connection" +readonly CONFIG_FILE="$CONFIG_DIR/mysql-backup.env" +readonly INSTALL_LOCK_FILE="/run/lock/solid-connection-mysql-backup-install.lock" +readonly -a TIMER_UNITS=( + mysql-backup-binlog.timer + mysql-backup-dump.timer +) + +if ((EUID != 0)); then + echo "The installer must run as root." >&2 + exit 1 +fi +if [[ ! -f "$CONFIG_SOURCE" ]]; then + echo "Backup environment file is missing." >&2 + exit 1 +fi + +for command_name in aws bash cp docker flock gzip install mountpoint mv sha256sum systemctl systemd-analyze; do + command -v "$command_name" >/dev/null || { + echo "Required command is not installed: $command_name" >&2 + exit 1 + } +done + +if ! mountpoint -q /mnt/mysql-data; then + echo "/mnt/mysql-data is not mounted; refusing to create backup files on the root volume." >&2 + exit 1 +fi + +# GitHub Actions와 수동 실행이 겹쳐도 하나의 설치 트랜잭션만 진행합니다. +exec 200>"$INSTALL_LOCK_FILE" +flock 200 + +install -d -m 755 -o root -g root "$INSTALL_LIB_DIR" "$INSTALL_BIN_DIR" "$CONFIG_DIR" +install -d -m 700 -o root -g root \ + /mnt/mysql-data/mysql-backup \ + /mnt/mysql-data/mysql-backup/staging \ + /mnt/mysql-data/mysql-backup/state + +TRANSACTION_DIR="$(mktemp -d /tmp/mysql-backup-install-transaction.XXXXXX)" +readonly TRANSACTION_DIR +readonly CANDIDATE_DIR="$TRANSACTION_DIR/candidate" +readonly ROLLBACK_DIR="$TRANSACTION_DIR/rollback" +install -d -m 700 "$CANDIDATE_DIR/lib" "$CANDIDATE_DIR/bin" "$CANDIDATE_DIR/systemd" "$ROLLBACK_DIR" +trap 'rm -rf -- "$TRANSACTION_DIR"' EXIT + +install -m 644 "$SOURCE_DIR/lib/backup-common.sh" "$CANDIDATE_DIR/lib/backup-common.sh" +for script in "$SOURCE_DIR"/bin/*; do + install -m 755 "$script" "$CANDIDATE_DIR/bin/$(basename "$script")" +done +install -m 600 "$CONFIG_SOURCE" "$CANDIDATE_DIR/mysql-backup.env" +install -m 644 "$SOURCE_DIR"/systemd/*.service "$SOURCE_DIR"/systemd/*.timer "$CANDIDATE_DIR/systemd/" + +load_candidate_config() { + local key + local value + + while IFS='=' read -r key value || [[ -n "$key" ]]; do + [[ -z "$key" || "$key" == \#* ]] && continue + case "$key" in + MYSQL_BACKUP_BUCKET|MYSQL_DATABASE|AWS_REGION) + printf -v "$key" '%s' "$value" + export "$key" + ;; + *) + echo "Unsupported backup configuration key: $key" >&2 + return 1 + ;; + esac + done <"$CANDIDATE_DIR/mysql-backup.env" +} + +# 현재 설치를 건드리기 전에 후보 스크립트와 설정을 검증합니다. +for script in "$CANDIDATE_DIR"/bin/* "$CANDIDATE_DIR/lib/backup-common.sh"; do + bash -n "$script" +done +load_candidate_config +MYSQL_BACKUP_LIB_DIR="$CANDIDATE_DIR/lib" \ + "$CANDIDATE_DIR/bin/mysql-backup-validate" + +declare -a TARGETS=() +declare -A TIMER_WAS_ENABLED=() +declare -A TIMER_WAS_ACTIVE=() +transaction_started=false +transaction_committed=false + +register_target() { + local target="$1" + local backup="$ROLLBACK_DIR${target}" + + TARGETS+=("$target") + if [[ -e "$target" || -L "$target" ]]; then + install -d -m 700 "$(dirname "$backup")" + cp -a "$target" "$backup" + fi +} + +atomic_install() { + local source="$1" + local target="$2" + local mode="$3" + local temporary="${target}.new" + + rm -f -- "$temporary" + install -m "$mode" -o root -g root "$source" "$temporary" + mv -Tf "$temporary" "$target" +} + +rollback_installation() { + local target + local backup + local timer + + for target in "${TARGETS[@]}"; do + backup="$ROLLBACK_DIR${target}" + if [[ -e "$backup" || -L "$backup" ]]; then + rm -f -- "$target" + cp -a "$backup" "$target" + else + rm -f -- "$target" + fi + done + + systemctl daemon-reload || true + for timer in "${TIMER_UNITS[@]}"; do + if [[ "${TIMER_WAS_ENABLED[$timer]}" == "true" ]]; then + systemctl enable "$timer" >/dev/null 2>&1 || true + else + systemctl disable "$timer" >/dev/null 2>&1 || true + fi + if [[ "${TIMER_WAS_ACTIVE[$timer]}" == "true" ]]; then + systemctl start "$timer" >/dev/null 2>&1 || true + else + systemctl stop "$timer" >/dev/null 2>&1 || true + fi + done +} + +cleanup() { + local exit_code=$? + + trap - EXIT + if [[ "$transaction_started" == "true" && "$transaction_committed" != "true" ]]; then + echo "Installation failed; restoring the previous backup pipeline." >&2 + set +e + rollback_installation + fi + rm -rf -- "$TRANSACTION_DIR" + exit "$exit_code" +} +trap cleanup EXIT + +for timer in "${TIMER_UNITS[@]}"; do + if systemctl is-enabled --quiet "$timer" 2>/dev/null; then + TIMER_WAS_ENABLED["$timer"]="true" + else + TIMER_WAS_ENABLED["$timer"]="false" + fi + if systemctl is-active --quiet "$timer" 2>/dev/null; then + TIMER_WAS_ACTIVE["$timer"]="true" + else + TIMER_WAS_ACTIVE["$timer"]="false" + fi + register_target "/etc/systemd/system/timers.target.wants/$timer" +done + +register_target "$INSTALL_LIB_DIR/backup-common.sh" +for script in "$CANDIDATE_DIR"/bin/*; do + register_target "$INSTALL_BIN_DIR/$(basename "$script")" +done +register_target "$CONFIG_FILE" +for unit in "$CANDIDATE_DIR"/systemd/*; do + register_target "/etc/systemd/system/$(basename "$unit")" +done +transaction_started=true + +# 실행 중인 dump/binlog가 끝난 뒤 교체하여 한 작업에서 서로 다른 버전이 섞이지 않게 합니다. +exec 198>/mnt/mysql-data/mysql-backup/state/dump.lock +exec 199>/mnt/mysql-data/mysql-backup/state/binlog.lock +flock 198 +flock 199 + +atomic_install "$CANDIDATE_DIR/lib/backup-common.sh" "$INSTALL_LIB_DIR/backup-common.sh" 644 +for script in "$CANDIDATE_DIR"/bin/*; do + atomic_install "$script" "$INSTALL_BIN_DIR/$(basename "$script")" 755 +done +atomic_install "$CANDIDATE_DIR/mysql-backup.env" "$CONFIG_FILE" 600 +declare -a INSTALLED_UNIT_TARGETS=() +for unit in "$CANDIDATE_DIR"/systemd/*; do + unit_target="/etc/systemd/system/$(basename "$unit")" + atomic_install "$unit" "$unit_target" 644 + INSTALLED_UNIT_TARGETS+=("$unit_target") +done + +systemd-analyze verify "${INSTALLED_UNIT_TARGETS[@]}" +systemctl daemon-reload +systemctl enable --now "${TIMER_UNITS[@]}" +transaction_committed=true + +systemctl --no-pager --full status "${TIMER_UNITS[@]}" diff --git a/scripts/mysql_backup/lib/backup-common.sh b/scripts/mysql_backup/lib/backup-common.sh new file mode 100644 index 0000000..7bacf44 --- /dev/null +++ b/scripts/mysql_backup/lib/backup-common.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly BACKUP_ROOT="${MYSQL_BACKUP_ROOT:-/mnt/mysql-data/mysql-backup}" +readonly STAGING_DIR="$BACKUP_ROOT/staging" +readonly STATE_DIR="$BACKUP_ROOT/state" +readonly MYSQL_DATA_DIR="${MYSQL_DATA_DIR:-/mnt/mysql-data/mysql}" +readonly MYSQL_CONTAINER="${MYSQL_CONTAINER:-mysql-server}" + +require_backup_environment() { + : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET is required}" + : "${MYSQL_DATABASE:?MYSQL_DATABASE is required}" + : "${AWS_REGION:?AWS_REGION is required}" + + if [[ ! "$MYSQL_BACKUP_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]]; then + echo "Invalid S3 bucket name." >&2 + return 1 + fi + if [[ ! "$MYSQL_DATABASE" =~ ^[A-Za-z0-9_]+$ ]]; then + echo "Invalid MySQL database name." >&2 + return 1 + fi + +} + +require_commands() { + local command_name + for command_name in "$@"; do + command -v "$command_name" >/dev/null || { + echo "Required command is not installed: $command_name" >&2 + return 1 + } + done +} + +mysql_query() { + local query="$1" + docker exec "$MYSQL_CONTAINER" sh -lc \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" exec mysql -uroot --batch --skip-column-names -e "$1"' \ + sh "$query" +} + +server_uuid() { + mysql_query 'SELECT @@server_uuid;' +} + +sha256_file() { + sha256sum "$1" | awk '{print $1}' +} + +head_object_checksum() { + local object_key="$1" + local error_file + local checksum + + error_file="$(mktemp)" + if checksum="$(aws s3api head-object \ + --bucket "$MYSQL_BACKUP_BUCKET" \ + --key "$object_key" \ + --region "$AWS_REGION" \ + --query 'Metadata.sha256' \ + --output text 2>"$error_file")"; then + rm -f "$error_file" + printf '%s' "$checksum" + return 0 + fi + + if grep -Eq '(404|Not Found|NoSuchKey)' "$error_file"; then + rm -f "$error_file" + return 1 + fi + + cat "$error_file" >&2 + rm -f "$error_file" + return 2 +} + +upload_file_once() { + local local_file="$1" + local object_key="$2" + local checksum + local remote_checksum + local head_status + + checksum="$(sha256_file "$local_file")" + if remote_checksum="$(head_object_checksum "$object_key")"; then + if [[ "$remote_checksum" != "$checksum" ]]; then + echo "S3 object already exists with a different checksum: $object_key" >&2 + return 1 + fi + echo "Skipping an object that is already uploaded: $object_key" + return 0 + else + head_status=$? + if ((head_status != 1)); then + return "$head_status" + fi + fi + + aws s3 cp "$local_file" "s3://$MYSQL_BACKUP_BUCKET/$object_key" \ + --region "$AWS_REGION" \ + --only-show-errors \ + --no-progress \ + --metadata "sha256=$checksum" +} diff --git a/scripts/mysql_backup/systemd/mysql-backup-binlog.service b/scripts/mysql_backup/systemd/mysql-backup-binlog.service new file mode 100644 index 0000000..3f89b67 --- /dev/null +++ b/scripts/mysql_backup/systemd/mysql-backup-binlog.service @@ -0,0 +1,23 @@ +[Unit] +Description=Rotate and upload closed MySQL binary logs to S3 +Requires=docker.service mnt-mysql\x2ddata.mount +After=docker.service mnt-mysql\x2ddata.mount network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=/etc/solid-connection/mysql-backup.env +ExecStart=/usr/local/libexec/solid-connection/mysql-backup-binlog +TimeoutStartSec=4min +UMask=0077 +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/mnt/mysql-data/mysql-backup + +[Install] +WantedBy=multi-user.target diff --git a/scripts/mysql_backup/systemd/mysql-backup-binlog.timer b/scripts/mysql_backup/systemd/mysql-backup-binlog.timer new file mode 100644 index 0000000..7294ac1 --- /dev/null +++ b/scripts/mysql_backup/systemd/mysql-backup-binlog.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Archive MySQL binary logs every five minutes + +[Timer] +OnCalendar=*-*-* *:00/5:00 +Persistent=true +AccuracySec=1s +RandomizedDelaySec=0 +Unit=mysql-backup-binlog.service + +[Install] +WantedBy=timers.target diff --git a/scripts/mysql_backup/systemd/mysql-backup-dump.service b/scripts/mysql_backup/systemd/mysql-backup-dump.service new file mode 100644 index 0000000..7b46207 --- /dev/null +++ b/scripts/mysql_backup/systemd/mysql-backup-dump.service @@ -0,0 +1,23 @@ +[Unit] +Description=Upload a consistent MySQL dump to S3 +Requires=docker.service mnt-mysql\x2ddata.mount +After=docker.service mnt-mysql\x2ddata.mount network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +EnvironmentFile=/etc/solid-connection/mysql-backup.env +ExecStart=/usr/local/libexec/solid-connection/mysql-backup-dump +TimeoutStartSec=2h +UMask=0077 +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/mnt/mysql-data/mysql-backup + +[Install] +WantedBy=multi-user.target diff --git a/scripts/mysql_backup/systemd/mysql-backup-dump.timer b/scripts/mysql_backup/systemd/mysql-backup-dump.timer new file mode 100644 index 0000000..749758a --- /dev/null +++ b/scripts/mysql_backup/systemd/mysql-backup-dump.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run the MySQL dump backup daily at 03:00 KST + +[Timer] +OnCalendar=*-*-* 03:00:00 Asia/Seoul +Persistent=true +AccuracySec=1s +RandomizedDelaySec=0 +Unit=mysql-backup-dump.service + +[Install] +WantedBy=timers.target diff --git a/scripts/mysql_backup/tests/run.sh b/scripts/mysql_backup/tests/run.sh new file mode 100755 index 0000000..ffb66d8 --- /dev/null +++ b/scripts/mysql_backup/tests/run.sh @@ -0,0 +1,382 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +readonly PROJECT_DIR +TEST_ROOT="$(mktemp -d)" +readonly TEST_ROOT +trap 'rm -rf "$TEST_ROOT"' EXIT + +assert_equals() { + local expected="$1" + local actual="$2" + local message="$3" + + if [[ "$expected" != "$actual" ]]; then + echo "FAIL: $message (expected=$expected actual=$actual)" >&2 + exit 1 + fi +} + +test_upload_idempotency() { + ( + export MYSQL_BACKUP_BUCKET="test-bucket" + export MYSQL_DATABASE="test_database" + export AWS_REGION="ap-northeast-2" + # shellcheck source=../lib/backup-common.sh + source "$PROJECT_DIR/scripts/mysql_backup/lib/backup-common.sh" + + local expected_checksum + expected_checksum="$(sha256_file "$PROJECT_DIR/scripts/mysql_backup/README.md")" + head_object_checksum() { printf '%s' "$expected_checksum"; } + aws() { echo "Unexpected upload for an existing object." >&2; return 99; } + upload_file_once "$PROJECT_DIR/scripts/mysql_backup/README.md" "test/key" + + head_object_checksum() { printf '%s' 'different-checksum'; } + if upload_file_once "$PROJECT_DIR/scripts/mysql_backup/README.md" "test/key" 2>/dev/null; then + echo "An object with a different checksum must not be overwritten." >&2 + exit 1 + fi + ) +} + +test_validate_requires_schema() { + local fixture_dir="$TEST_ROOT/validate" + local backup_dir="$fixture_dir/backup" + + mkdir -p "$fixture_dir/lib" "$backup_dir" + cat >"$fixture_dir/lib/backup-common.sh" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly BACKUP_ROOT="$TEST_BACKUP_ROOT" +readonly MYSQL_CONTAINER="mysql-server" +require_backup_environment() { :; } +require_commands() { :; } +mountpoint() { :; } +docker() { :; } +df() { printf 'Avail\n9999999999\n'; } +aws() { :; } +mysql_query() { + if [[ "$1" == *'@@log_bin'* ]]; then + printf '%s\n' '1 ROW 1 1 1' + elif [[ "$1" == *'information_schema.schemata'* ]]; then + printf '%s\n' "$TEST_SCHEMA_EXISTS" + elif [[ "$1" == *'information_schema.tables'* ]]; then + printf '%s\n' '1024' + else + echo "Unexpected validation query: $1" >&2 + return 1 + fi +} +EOF + + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_SCHEMA_EXISTS=1 \ + MYSQL_BACKUP_BUCKET="test-bucket" \ + MYSQL_DATABASE="test_database" \ + AWS_REGION="ap-northeast-2" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-validate" >/dev/null + + if TEST_BACKUP_ROOT="$backup_dir" \ + TEST_SCHEMA_EXISTS=0 \ + MYSQL_BACKUP_BUCKET="test-bucket" \ + MYSQL_DATABASE="missing_database" \ + AWS_REGION="ap-northeast-2" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-validate" >/dev/null 2>&1; then + echo "Validation must reject a missing backup database." >&2 + exit 1 + fi +} + +write_fake_common() { + local fixture_dir="$1" + + mkdir -p "$fixture_dir/lib" + cat >"$fixture_dir/lib/backup-common.sh" <<'EOF' +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly BACKUP_ROOT="$TEST_BACKUP_ROOT" +readonly STAGING_DIR="$BACKUP_ROOT/staging" +readonly STATE_DIR="$BACKUP_ROOT/state" +readonly MYSQL_DATA_DIR="$TEST_MYSQL_DATA_DIR" +readonly MYSQL_CONTAINER="mysql-server" +readonly MYSQL_BACKUP_BUCKET="${MYSQL_BACKUP_BUCKET:-test-bucket}" +readonly AWS_REGION="${AWS_REGION:-ap-northeast-2}" + +require_backup_environment() { :; } +require_commands() { :; } +flock() { return 0; } +aws() { printf '%s' "${TEST_S3_KEYS:-}"; } +date() { + case "${*: -1}" in + +%Y%m%dT%H%M%SZ) printf '%s\n' '20260716T030000Z' ;; + +%Y-%m-%dT%H:%M:%SZ) printf '%s\n' '2026-07-16T03:00:00Z' ;; + +%Y/%m/%d) printf '%s\n' '2026/07/16' ;; + +%H%M%S) printf '%s\n' '030000' ;; + +%s) printf '%s\n' '1784170800' ;; + *) command date "$@" ;; + esac +} +stat() { + if [[ "$1 $2" == '-c %Y' ]]; then + printf '%s\n' '1784170800' + elif [[ "$1 $2" == '-c %s' ]]; then + wc -c <"$3" | tr -d ' ' + else + command stat "$@" + fi +} +sha256_file() { sha256sum "$1" | awk '{print $1}'; } +server_uuid() { printf '%s\n' '11111111-2222-3333-4444-555555555555'; } +upload_file_once() { + printf '%s\n' "$2" >>"$TEST_UPLOAD_LOG" + if [[ "$2" == */manifest.json || "$2" == *.manifest.json ]]; then + cp "$1" "$TEST_MANIFEST_CAPTURE" + fi + if [[ -n "${TEST_FAIL_MANIFEST_ONCE:-}" && "$2" == */manifest.json && ! -e "$TEST_FAIL_MARKER" ]]; then + touch "$TEST_FAIL_MARKER" + return 1 + fi +} +EOF +} + +test_binlog_chain() { + local fixture_dir="$TEST_ROOT/binlog" + local data_dir="$fixture_dir/mysql" + local backup_dir="$fixture_dir/backup" + local upload_log="$fixture_dir/uploads" + local manifest_capture="$fixture_dir/manifest.json" + local active_file="$fixture_dir/active-binlog" + local flush_log="$fixture_dir/flushes" + local rotation_slot=$((1784170800 / 300)) + + write_fake_common "$fixture_dir" + mkdir -p "$data_dir" "$backup_dir/staging" "$backup_dir/state" + printf '%s\n' \ + '/var/lib/mysql/binlog.000001' \ + '/var/lib/mysql/binlog.000002' \ + '/var/lib/mysql/binlog.000003' \ + '/var/lib/mysql/binlog.000004' >"$data_dir/binlog.index" + printf 'one' >"$data_dir/binlog.000001" + printf 'two' >"$data_dir/binlog.000002" + printf 'three' >"$data_dir/binlog.000003" + printf 'four' >"$data_dir/binlog.000004" + printf '%s\n' 'binlog.000004' >"$active_file" + + cat >>"$fixture_dir/lib/backup-common.sh" <<'EOF' +mysql_query() { + case "$1" in + 'FLUSH BINARY LOGS;') + current="$(<"$TEST_ACTIVE_FILE")" + current_number=$((10#${current##*.})) + next_binlog="$(printf 'binlog.%06d' "$((current_number + 1))")" + printf 'closed-%s' "$current" >"$TEST_MYSQL_DATA_DIR/$current" + printf '/var/lib/mysql/%s\n' "$next_binlog" >>"$TEST_MYSQL_DATA_DIR/binlog.index" + printf '%s\n' "$next_binlog" >"$TEST_ACTIVE_FILE" + printf '%s\n' "$next_binlog" >>"$TEST_FLUSH_LOG" + ;; + 'SHOW BINARY LOG STATUS;') printf '%s 157\n' "$(<"$TEST_ACTIVE_FILE")" ;; + 'SELECT @@server_uuid;') server_uuid ;; + 'SELECT @@log_bin_basename;') printf '%s\n' '/var/lib/mysql/binlog' ;; + *) echo "Unexpected query: $1" >&2; return 1 ;; + esac +} +EOF + + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" + + assert_equals "8" "$(wc -l <"$upload_log" | tr -d ' ')" "four closed binlogs and manifests must be uploaded" + uploaded_keys=() + while IFS= read -r uploaded_key; do + uploaded_keys+=("$uploaded_key") + done <"$upload_log" + for ((i = 0; i < ${#uploaded_keys[@]}; i += 2)); do + assert_equals \ + "${uploaded_keys[i]}.manifest.json" \ + "${uploaded_keys[i + 1]}" \ + "each binlog must be uploaded immediately before its manifest" + done + assert_equals \ + "1 test-bucket 11111111-2222-3333-4444-555555555555 binlog.000004 $rotation_slot binlog.000004 binlog.000005" \ + "$(<"$backup_dir/state/binlog-last-uploaded")" \ + "the destination, rotation slot, and last closed binlog must be persisted" + assert_equals "1" "$(wc -l <"$flush_log" | tr -d ' ')" "the first run must rotate once" + + : >"$upload_log" + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" >/dev/null + assert_equals "1" "$(wc -l <"$flush_log" | tr -d ' ')" "a retry in the same five-minute slot must not rotate again" + assert_equals "0" "$(wc -l <"$upload_log" | tr -d ' ')" "a completed slot retry must not upload another object" + + # FLUSH 직후 상태 완료 기록 전에 종료된 경우에도 active log 변화로 회전 성공을 복구합니다. + printf '1 test-bucket 11111111-2222-3333-4444-555555555555 binlog.000003 %s binlog.000004 -\n' \ + "$rotation_slot" >"$backup_dir/state/binlog-last-uploaded" + : >"$upload_log" + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" >/dev/null + assert_equals "1" "$(wc -l <"$flush_log" | tr -d ' ')" "a pending state with an advanced active log must not rotate again" + assert_equals \ + "1 test-bucket 11111111-2222-3333-4444-555555555555 binlog.000004 $rotation_slot binlog.000004 binlog.000005" \ + "$(<"$backup_dir/state/binlog-last-uploaded")" \ + "the pending rotation state must converge after retry" + + rm -f "$backup_dir/state/binlog-last-uploaded" + : >"$upload_log" + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + TEST_S3_KEYS='binlog/2026/07/16/025500-11111111-2222-3333-4444-555555555555-binlog.000002.manifest.json' \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" >/dev/null + assert_equals "6" "$(wc -l <"$upload_log" | tr -d ' ')" "S3 state recovery must skip older binlogs and upload the remaining closed chain" + + rm -f "$backup_dir/state/binlog-last-uploaded" + : >"$upload_log" + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + TEST_S3_KEYS=$'binlog/2026/07/16/025000-11111111-2222-3333-4444-555555555555-binlog.000001.manifest.json\tbinlog/2026/07/16/025500-11111111-2222-3333-4444-555555555555-binlog.000003.manifest.json' \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" >/dev/null + assert_equals "10" "$(wc -l <"$upload_log" | tr -d ' ')" "S3 recovery must resume before a manifest gap and repair the local closed chain" + + # 동일 서버에서 목적지가 바뀌면 이전 버킷의 last_uploaded를 신뢰하지 않습니다. + printf '%s\n' 'binlog.000006' >"$active_file" + printf '%s\n' \ + '/var/lib/mysql/binlog.000001' \ + '/var/lib/mysql/binlog.000002' \ + '/var/lib/mysql/binlog.000003' \ + '/var/lib/mysql/binlog.000004' \ + '/var/lib/mysql/binlog.000005' \ + '/var/lib/mysql/binlog.000006' >"$data_dir/binlog.index" + printf 'five' >"$data_dir/binlog.000005" + printf '1 old-bucket 11111111-2222-3333-4444-555555555555 binlog.000005 %s binlog.000005 binlog.000006\n' \ + "$rotation_slot" >"$backup_dir/state/binlog-last-uploaded" + : >"$upload_log" + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + MYSQL_BACKUP_BUCKET="new-bucket" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" >/dev/null + [[ "$(<"$backup_dir/state/binlog-last-uploaded")" == "1 new-bucket "* ]] + assert_equals "12" "$(wc -l <"$upload_log" | tr -d ' ')" "a destination change must rebuild the available closed chain in the new bucket" + + printf '%s\n' \ + '/var/lib/mysql/binlog.000001' \ + '/var/lib/mysql/binlog.000003' \ + '/var/lib/mysql/binlog.000004' >"$data_dir/binlog.index" + printf '%s\n' 'binlog.000004' >"$active_file" + printf '1 test-bucket 11111111-2222-3333-4444-555555555555 binlog.000001 %s binlog.000003 binlog.000004\n' \ + "$rotation_slot" >"$backup_dir/state/binlog-last-uploaded" + + if TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$data_dir" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + TEST_ACTIVE_FILE="$active_file" \ + TEST_FLUSH_LOG="$flush_log" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-binlog" >/dev/null 2>&1; then + echo "A missing binlog in the chain must fail the backup." >&2 + exit 1 + fi +} + +test_dump_retry_manifest() { + local fixture_dir="$TEST_ROOT/dump" + local backup_dir="$fixture_dir/backup" + local upload_log="$fixture_dir/uploads" + local manifest_capture="$fixture_dir/manifest.json" + local first_manifest="$fixture_dir/first-manifest.json" + local first_attempt_log="$fixture_dir/first-attempt.log" + + write_fake_common "$fixture_dir" + mkdir -p "$backup_dir/staging" "$backup_dir/state" + mkdir -p "$backup_dir/staging/dump-20260715T030000Z" + cat >>"$fixture_dir/lib/backup-common.sh" <<'EOF' +docker() { + printf '%s\n' "-- CHANGE REPLICATION SOURCE TO SOURCE_LOG_FILE='binlog.000003', SOURCE_LOG_POS=157;" + printf '%s\n' 'CREATE TABLE example (id bigint);' +} +EOF + + if TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$fixture_dir/mysql" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + MYSQL_DATABASE="test_database" \ + TEST_FAIL_MANIFEST_ONCE=1 \ + TEST_FAIL_MARKER="$fixture_dir/failed-once" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-dump" >"$first_attempt_log" 2>&1; then + echo "The first dump attempt must simulate a manifest upload failure." >&2 + exit 1 + fi + if [[ ! -f "$manifest_capture" ]]; then + cat "$first_attempt_log" >&2 + echo "The failed attempt did not reach the manifest upload." >&2 + exit 1 + fi + [[ ! -e "$backup_dir/staging/dump-20260715T030000Z" ]] + assert_equals \ + "20260716T030000Z" \ + "$(<"$backup_dir/state/dump-current")" \ + "a failed dump attempt must retain a valid retry job id" + [[ ! -e "$backup_dir/state/dump-current.tmp" ]] + cp "$manifest_capture" "$first_manifest" + rm -f "$manifest_capture" + sleep 1 + + TEST_BACKUP_ROOT="$backup_dir" \ + TEST_MYSQL_DATA_DIR="$fixture_dir/mysql" \ + TEST_UPLOAD_LOG="$upload_log" \ + TEST_MANIFEST_CAPTURE="$manifest_capture" \ + MYSQL_DATABASE="test_database" \ + TEST_FAIL_MANIFEST_ONCE=1 \ + TEST_FAIL_MARKER="$fixture_dir/failed-once" \ + MYSQL_BACKUP_LIB_DIR="$fixture_dir/lib" \ + bash "$PROJECT_DIR/scripts/mysql_backup/bin/mysql-backup-dump" >/dev/null + + cmp "$first_manifest" "$manifest_capture" + [[ ! -e "$backup_dir/state/dump-current" ]] +} + +test_upload_idempotency +test_validate_requires_schema +test_binlog_chain +test_dump_retry_manifest +echo "All MySQL backup tests passed." diff --git a/scripts/mysql_backup/validate-remote.sh b/scripts/mysql_backup/validate-remote.sh new file mode 100755 index 0000000..7d2f850 --- /dev/null +++ b/scripts/mysql_backup/validate-remote.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly CONFIG_FILE="/etc/solid-connection/mysql-backup.env" +readonly VALIDATE_BIN="/usr/local/libexec/solid-connection/mysql-backup-validate" + +if ((EUID != 0)); then + echo "Remote validation must run as root." >&2 + exit 1 +fi + +if [[ -x "$VALIDATE_BIN" && -f "$CONFIG_FILE" ]]; then + unset MYSQL_BACKUP_BUCKET MYSQL_DATABASE AWS_REGION + while IFS='=' read -r key value || [[ -n "$key" ]]; do + [[ -z "$key" || "$key" == \#* ]] && continue + case "$key" in + MYSQL_BACKUP_BUCKET|MYSQL_DATABASE|AWS_REGION) + printf -v "$key" '%s' "$value" + export "$key" + ;; + *) + echo "Unsupported backup configuration key: $key" >&2 + exit 1 + ;; + esac + done <"$CONFIG_FILE" + "$VALIDATE_BIN" + systemctl is-enabled --quiet mysql-backup-binlog.timer mysql-backup-dump.timer + systemctl is-active --quiet mysql-backup-binlog.timer mysql-backup-dump.timer + systemctl list-timers --no-pager --all 'mysql-backup-*' + exit 0 +fi + +for command_name in aws docker flock gzip sha256sum; do + command -v "$command_name" >/dev/null || { + echo "Required command is not installed: $command_name" >&2 + exit 1 + } +done +: "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET is required for pre-installation validation}" +: "${MYSQL_DATABASE:?MYSQL_DATABASE is required for pre-installation validation}" +: "${AWS_REGION:?AWS_REGION is required for pre-installation validation}" +if [[ ! "$MYSQL_BACKUP_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]]; then + echo "Invalid S3 bucket name." >&2 + exit 1 +fi +if [[ ! "$MYSQL_DATABASE" =~ ^[A-Za-z0-9_]+$ ]]; then + echo "Invalid MySQL database name." >&2 + exit 1 +fi +mountpoint -q /mnt/mysql-data +docker inspect mysql-server >/dev/null +mysql_settings="$(docker exec mysql-server sh -lc ' + MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --batch --skip-column-names -e " + SELECT @@log_bin, @@binlog_format, @@server_id, + @@sync_binlog, @@innodb_flush_log_at_trx_commit; + " +')" +read -r log_bin binlog_format server_id sync_binlog flush_policy <<<"$mysql_settings" +if [[ "$log_bin" != "1" || "$binlog_format" != "ROW" || "$server_id" == "0" ]]; then + echo "MySQL binlog configuration is not suitable for backup." >&2 + exit 1 +fi +if [[ "$sync_binlog" != "1" || "$flush_policy" != "1" ]]; then + echo "MySQL durability configuration is not sync_binlog=1 and innodb_flush_log_at_trx_commit=1." >&2 + exit 1 +fi +schema_exists="$(docker exec mysql-server sh -lc \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --batch --skip-column-names -e "$1"' \ + sh "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = '$MYSQL_DATABASE';")" +if [[ "$schema_exists" != "1" ]]; then + echo "MySQL backup database does not exist: $MYSQL_DATABASE" >&2 + exit 1 +fi +database_bytes="$(docker exec mysql-server sh -lc \ + 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot --batch --skip-column-names -e "$1"' \ + sh "SELECT COALESCE(SUM(data_length + index_length), 0) FROM information_schema.tables WHERE table_schema = '$MYSQL_DATABASE';")" +available_bytes="$(df --output=avail -B1 /mnt/mysql-data | tail -1 | tr -d ' ')" +if [[ ! "$database_bytes" =~ ^[0-9]+$ || ! "$available_bytes" =~ ^[0-9]+$ ]]; then + echo "Failed to calculate database size or available backup space." >&2 + exit 1 +fi +required_bytes=$((database_bytes * 2 + 268435456)) +if ((available_bytes < required_bytes)); then + echo "Insufficient backup staging space: available=$available_bytes required=$required_bytes" >&2 + exit 1 +fi +aws s3api head-bucket --bucket "$MYSQL_BACKUP_BUCKET" --region "$AWS_REGION" >/dev/null +df -h / /mnt/mysql-data +echo "Pre-installation validation succeeded."