feat: MySQL S3 백업 systemd 파이프라인 구성

.github/workflows/mysql-backup-deploy.yml

@@ -0,0 +1,230 @@
+name: MySQL Backup Deploy
+
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: validate는 상태만 확인하고 install은 백업 구성을 설치합니다.
+        required: true
+        default: validate
+        type: choice
+        options:
+          - validate
+          - install
+
+permissions:
+  id-token: write
+  contents: read
+
+concurrency:
+  group: prod-mysql-backup-deploy
+  cancel-in-progress: false
+
+env:
+  AWS_REGION: ap-northeast-2
+  LOCAL_SSH_PORT: "2222"
+  SESSION_MANAGER_PLUGIN_VERSION: "1.2.835.0"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Install Session Manager plugin
+        run: |
+          set -Eeuo pipefail
+          plugin_dir="$(mktemp -d "$RUNNER_TEMP/session-manager-plugin.XXXXXX")"
+          trap 'rm -rf -- "$plugin_dir"' EXIT
+          plugin_base_url="https://s3.amazonaws.com/session-manager-downloads/plugin/${SESSION_MANAGER_PLUGIN_VERSION}/ubuntu_64bit"
+
+          curl -fsSL "$plugin_base_url/session-manager-plugin.deb" \
+            -o "$plugin_dir/session-manager-plugin.deb"
+          curl -fsSL "$plugin_base_url/session-manager-plugin.deb.sig" \
+            -o "$plugin_dir/session-manager-plugin.deb.sig"
+
+          cat >"$plugin_dir/session-manager-plugin.gpg" <<'EOF'
+          -----BEGIN PGP PUBLIC KEY BLOCK-----
+
+          mFIEZ5ERQxMIKoZIzj0DAQcCAwQjuZy+IjFoYg57sLTGhF3aZLBaGpzB+gY6j7Ix
+          P7NqbpXyjVj8a+dy79gSd64OEaMxUb7vw/jug+CfRXwVGRMNtIBBV1MgU1NNIFNl
+          c3Npb24gTWFuYWdlciA8c2Vzc2lvbi1tYW5hZ2VyLXBsdWdpbi1zaWduZXJAYW1h
+          em9uLmNvbT4gKEFXUyBTeXN0ZW1zIE1hbmFnZXIgU2Vzc2lvbiBNYW5hZ2VyIFBs
+          dWdpbiBMaW51eCBTaWduZXIgS2V5KYkBAAQQEwgAqAUCZ5ERQ4EcQVdTIFNTTSBT
+          ZXNzaW9uIE1hbmFnZXIgPHNlc3Npb24tbWFuYWdlci1wbHVnaW4tc2lnbmVyQGFt
+          YXpvbi5jb20+IChBV1MgU3lzdGVtcyBNYW5hZ2VyIFNlc3Npb24gTWFuYWdlciBQ
+          bHVnaW4gTGludXggU2lnbmVyIEtleSkWIQR5WWNxJM4JOtUB1HosTUr/b2dX7gIe
+          AwIbAwIVCAAKCRAsTUr/b2dX7rO1AQCa1kig3lQ78W/QHGU76uHx3XAyv0tfpE9U
+          oQBCIwFLSgEA3PDHt3lZ+s6m9JLGJsy+Cp5ZFzpiF6RgluR/2gA861M=
+          =2DQm
+          -----END PGP PUBLIC KEY BLOCK-----
+          EOF
+
+          export GNUPGHOME="$plugin_dir/gnupg"
+          install -d -m 700 "$GNUPGHOME"
+          gpg --batch --import "$plugin_dir/session-manager-plugin.gpg"
+          signing_key_fingerprint="$(gpg --batch --with-colons --fingerprint 2C4D4AFF6F6757EE | awk -F: '$1 == "fpr" {print $10; exit}')"
+          if [[ "$signing_key_fingerprint" != "7959637124CE093AD501D47A2C4D4AFF6F6757EE" ]]; then
+            echo "::error::Unexpected Session Manager plugin signing key fingerprint"
+            exit 1
+          fi
+          gpg --batch --verify \
+            "$plugin_dir/session-manager-plugin.deb.sig" \
+            "$plugin_dir/session-manager-plugin.deb"
+
+          package_version="$(dpkg-deb --field "$plugin_dir/session-manager-plugin.deb" Version)"
+          if [[ "$package_version" != "${SESSION_MANAGER_PLUGIN_VERSION}-1" ]]; then
+            echo "::error::Unexpected Session Manager plugin package version: $package_version"
+            exit 1
+          fi
+          sudo dpkg -i "$plugin_dir/session-manager-plugin.deb"
+          echo "/usr/local/sessionmanagerplugin/bin" >> "$GITHUB_PATH"
+
+      - name: Validate or install MySQL backup
+        env:
+          DEPLOY_MODE: ${{ inputs.mode }}
+          MYSQL_BACKUP_BUCKET: ${{ vars.MYSQL_BACKUP_BUCKET_NAME }}
+          MYSQL_DATABASE: ${{ vars.MYSQL_BACKUP_DATABASE_NAME }}
+          DB_HOST_FINGERPRINT: ${{ vars.PROD_DB_SSH_HOST_KEY_ED25519 }}
+        run: |
+          set -Eeuo pipefail
+          umask 077
+
+          : "${MYSQL_BACKUP_BUCKET:?MYSQL_BACKUP_BUCKET_NAME repository variable is required}"
+          : "${MYSQL_DATABASE:?MYSQL_BACKUP_DATABASE_NAME repository variable is required}"
+          : "${DB_HOST_FINGERPRINT:?PROD_DB_SSH_HOST_KEY_ED25519 repository variable is required}"
+          if [[ "$DEPLOY_MODE" != "validate" && "$DEPLOY_MODE" != "install" ]]; then
+            echo "::error::Invalid deployment mode"
+            exit 1
+          fi
+          if [[ ! "$MYSQL_BACKUP_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]]; then
+            echo "::error::Invalid S3 bucket name"
+            exit 1
+          fi
+          if [[ ! "$MYSQL_DATABASE" =~ ^[A-Za-z0-9_]+$ ]]; then
+            echo "::error::Invalid database name"
+            exit 1
+          fi
+          if [[ ! "$DB_HOST_FINGERPRINT" =~ ^SHA256:[A-Za-z0-9+/]{43}$ ]]; then
+            echo "::error::Invalid DB EC2 SSH host key fingerprint"
+            exit 1
+          fi
+          KEY_DIR="$(mktemp -d "$RUNNER_TEMP/mysql-backup-eic.XXXXXX")"
+          SSM_PID=""
+          cleanup() {
+            if [[ -n "$SSM_PID" ]]; then
+              kill "$SSM_PID" 2>/dev/null || true
+              wait "$SSM_PID" 2>/dev/null || true
+            fi
+            rm -rf "$KEY_DIR"
+          }
+          trap cleanup EXIT
+
+          ssh-keygen -q -t ed25519 -N '' -f "$KEY_DIR/id_ed25519"
+
+          API_INSTANCE_ID="$(aws ec2 describe-instances \
+            --filters \
+              'Name=tag:Name,Values=solid-connection-server-prod' \
+              'Name=instance-state-name,Values=running' \
+            --query 'Reservations[].Instances[].InstanceId' \
+            --output text)"
+          DB_INSTANCE_ID="$(aws ec2 describe-instances \
+            --filters \
+              'Name=tag:Name,Values=solid-connection-db-mysql-prod' \
+              'Name=instance-state-name,Values=running' \
+            --query 'Reservations[].Instances[].InstanceId' \
+            --output text)"
+
+          if [[ "$API_INSTANCE_ID" == *$'\t'* || -z "$API_INSTANCE_ID" || "$API_INSTANCE_ID" == "None" ]]; then
+            echo "::error::Exactly one running Prod API EC2 instance is required"
+            exit 1
+          fi
+          if [[ "$DB_INSTANCE_ID" == *$'\t'* || -z "$DB_INSTANCE_ID" || "$DB_INSTANCE_ID" == "None" ]]; then
+            echo "::error::Exactly one running Prod DB EC2 instance is required"
+            exit 1
+          fi
+
+          DB_PRIVATE_IP="$(aws ec2 describe-instances \
+            --instance-ids "$DB_INSTANCE_ID" \
+            --query 'Reservations[0].Instances[0].PrivateIpAddress' \
+            --output text)"
+          if [[ -z "$DB_PRIVATE_IP" || "$DB_PRIVATE_IP" == "None" ]]; then
+            echo "::error::Prod DB EC2 private IP was not found"
+            exit 1
+          fi
+
+          aws ssm start-session \
+            --target "$API_INSTANCE_ID" \
+            --document-name AWS-StartPortForwardingSessionToRemoteHost \
+            --parameters "{\"host\":[\"$DB_PRIVATE_IP\"],\"portNumber\":[\"22\"],\"localPortNumber\":[\"$LOCAL_SSH_PORT\"]}" \
+            >"$KEY_DIR/ssm-tunnel.log" 2>&1 &
+          SSM_PID=$!
+
+          for _ in $(seq 1 60); do
+            kill -0 "$SSM_PID" 2>/dev/null || {
+              cat "$KEY_DIR/ssm-tunnel.log" >&2
+              exit 1
+            }
+            nc -z 127.0.0.1 "$LOCAL_SSH_PORT" 2>/dev/null && break
+            sleep 1
+          done
+          nc -z 127.0.0.1 "$LOCAL_SSH_PORT"
+
+          KNOWN_HOSTS_FILE="$KEY_DIR/known_hosts"
+          ssh-keyscan -t ed25519 -p "$LOCAL_SSH_PORT" 127.0.0.1 >"$KNOWN_HOSTS_FILE" 2>/dev/null
+          ACTUAL_FINGERPRINT="$(ssh-keygen -lf "$KNOWN_HOSTS_FILE" -E sha256 | awk '{print $2}')"
+          if [[ "$ACTUAL_FINGERPRINT" != "$DB_HOST_FINGERPRINT" ]]; then
+            echo "::error::DB EC2 SSH host key fingerprint mismatch"
+            exit 1
+          fi
+
+          SSH_OPTIONS=(
+            -p "$LOCAL_SSH_PORT"
+            -i "$KEY_DIR/id_ed25519"
+            -o IdentitiesOnly=yes
+            -o StrictHostKeyChecking=yes
+            -o UserKnownHostsFile="$KNOWN_HOSTS_FILE"
+            -o ConnectTimeout=10
+          )
+
+          if [[ "$DEPLOY_MODE" == "validate" ]]; then
+            aws ec2-instance-connect send-ssh-public-key \
+              --region "$AWS_REGION" \
+              --instance-id "$DB_INSTANCE_ID" \
+              --instance-os-user ubuntu \
+              --ssh-public-key "file://$KEY_DIR/id_ed25519.pub" >/dev/null
+            REMOTE_VALIDATE_COMMAND="env MYSQL_BACKUP_BUCKET=$(printf '%q' "$MYSQL_BACKUP_BUCKET") MYSQL_DATABASE=$(printf '%q' "$MYSQL_DATABASE") AWS_REGION=$(printf '%q' "$AWS_REGION") bash -s"
+            ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 \
+              "sudo bash -c $(printf '%q' "$REMOTE_VALIDATE_COMMAND")" \
+              < scripts/mysql_backup/validate-remote.sh
+            exit 0
+          fi
+
+          CONFIG_FILE="$KEY_DIR/mysql-backup.env"
+          {
+            printf 'MYSQL_BACKUP_BUCKET=%s\n' "$MYSQL_BACKUP_BUCKET"
+            printf 'MYSQL_DATABASE=%s\n' "$MYSQL_DATABASE"
+            printf 'AWS_REGION=%s\n' "$AWS_REGION"
+          } >"$CONFIG_FILE"
+
+          cp -R scripts/mysql_backup "$KEY_DIR/bundle"
+          cp "$CONFIG_FILE" "$KEY_DIR/bundle/mysql-backup.env"
+          tar -C "$KEY_DIR/bundle" -czf "$KEY_DIR/mysql-backup.tar.gz" .
+
+          aws ec2-instance-connect send-ssh-public-key \
+            --region "$AWS_REGION" \
+            --instance-id "$DB_INSTANCE_ID" \
+            --instance-os-user ubuntu \
+            --ssh-public-key "file://$KEY_DIR/id_ed25519.pub" >/dev/null
+
+          REMOTE_INSTALL_COMMAND='set -Eeuo pipefail; install_dir="$(mktemp -d /tmp/mysql-backup-install.XXXXXX)"; trap '\''rm -rf -- "$install_dir"'\'' EXIT; tar -xzf - -C "$install_dir"; "$install_dir/install.sh" "$install_dir/mysql-backup.env"'
+          ssh "${SSH_OPTIONS[@]}" ubuntu@127.0.0.1 \
+            "sudo bash -c $(printf '%q' "$REMOTE_INSTALL_COMMAND")" \
+            <"$KEY_DIR/mysql-backup.tar.gz"

modules/app_stack/scripts/mysql_setup.sh.tftpl

@@ -50,6 +50,7 @@ DATA_VOLUME_SERIAL="$(printf '%s' "$DATA_VOLUME_ID" | tr -d '-')"
 DATA_VOLUME_DEVICE="/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_$DATA_VOLUME_SERIAL"
 DATA_VOLUME_MOUNT="/mnt/mysql-data"
 MYSQL_DATA_DIR="$DATA_VOLUME_MOUNT/mysql"
+MYSQL_BACKUP_DIR="$DATA_VOLUME_MOUNT/mysql-backup"
 
 for i in $(seq 1 120); do
   if [ -e "$DATA_VOLUME_DEVICE" ]; then
@@ -95,6 +96,11 @@ mkdir -p "$MYSQL_DATA_DIR"
 chown -R 999:999 "$MYSQL_DATA_DIR"
 chmod 750 "$MYSQL_DATA_DIR"
 
+# dump 압축 파일과 업로드 상태는 용량이 작은 루트 볼륨이 아닌 데이터 EBS에 둡니다.
+mkdir -p "$MYSQL_BACKUP_DIR/staging" "$MYSQL_BACKUP_DIR/state"
+chown -R root:root "$MYSQL_BACKUP_DIR"
+chmod 700 "$MYSQL_BACKUP_DIR" "$MYSQL_BACKUP_DIR/staging" "$MYSQL_BACKUP_DIR/state"
+
 mkdir -p /etc/mysql/conf.d
 cat > /etc/mysql/conf.d/tuning.cnf <<'CNFEOF'
 ${mysql_config_content}

scripts/mysql_backup/README.md

@@ -0,0 +1,22 @@
+# Prod MySQL S3 백업
+
+DB EC2에서 다음 systemd 작업을 실행합니다.
+
+- `mysql-backup-dump.timer`: 매일 03:00 KST에 일관된 전체 dump, 체크섬, manifest를 업로드합니다.
+- `mysql-backup-binlog.timer`: 매 5분 경계마다 binlog를 회전하고 닫힌 파일과 manifest를 업로드합니다.
+
+백업 파일은 `/mnt/mysql-data/mysql-backup`에서만 임시 생성합니다. S3에는 manifest를 마지막에 업로드하며, 같은 key가 이미 존재하면 체크섬이 같은 경우에만 업로드를 생략합니다. 이 방식으로 재시도 시 Object Lock 버킷에 불필요한 객체 버전이 생기는 것을 방지합니다.
+
+## 배포 전 GitHub 설정
+
+Repository Secrets:
+
+- `AWS_ROLE_ARN`: 배포 워크플로우가 AssumeRole할 IAM 역할 ARN
+
+Repository Variables:
+
+- `MYSQL_BACKUP_BUCKET_NAME`: 백업 버킷 이름
+- `MYSQL_BACKUP_DATABASE_NAME`: 백업할 DB 이름. 필수값이며 공개 코드에 기본값을 두지 않습니다.
+- `PROD_DB_SSH_HOST_KEY_ED25519`: DB EC2의 ED25519 host key SHA-256 fingerprint
+
+> `MySQL Backup Deploy` 워크플로우는 기본값인 `validate`에서 변경 없이 사전 조건 또는 설치 상태만 검사합니다. 실제 설치는 `install`을 명시적으로 선택해야 합니다.