-
Notifications
You must be signed in to change notification settings - Fork 0
feat(db): 一貫したスナップショットを書き出す backup_to を追加 #44
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -704,6 +704,38 @@ impl Database { | |
| Ok(()) | ||
| } | ||
|
|
||
| /// DB の一貫したスナップショットを `dest` に書き出す。 | ||
| /// | ||
| /// 単純なファイルコピーでは WAL に未反映のトランザクションが取り残され、 | ||
| /// 古い、または壊れた複製ができる。`VACUUM INTO` は writer 接続上で | ||
| /// 単一ファイルの完結したコピーを作るため、-wal / -shm を伴わない。 | ||
| /// | ||
| /// `strip_tokens` を立てると、複製側の `accounts.token` を空にしてから | ||
| /// 返す。キーチェーンが永続しない環境では DB に API トークンが平文で | ||
| /// 残るため、持ち出す成果物からは落とせるようにする。 | ||
| pub fn backup_to(&self, dest: &Path, strip_tokens: bool) -> Result<(), NoteDeckError> { | ||
| if dest.exists() { | ||
| // VACUUM INTO は既存ファイルへの書き出しを拒否する | ||
| std::fs::remove_file(dest).map_err(|e| { | ||
| NoteDeckError::Database(rusqlite::Error::SqliteFailure( | ||
| rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_CANTOPEN), | ||
| Some(format!("failed to replace existing backup: {e}")), | ||
| )) | ||
| })?; | ||
| } | ||
| { | ||
| let conn = self.lock_write()?; | ||
| conn.execute("VACUUM INTO ?1", params![dest.to_string_lossy()])?; | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
# Map the file structure and inspect the relevant area in src/db.rs
git ls-files 'src/db.rs' 'src/*.rs' | sed -n '1,20p'
wc -l src/db.rs
sed -n '680,760p' src/db.rsRepository: notedeck-dev/notecli Length of output: 3457 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Show the exact function around the VACUUM INTO call and any path validation nearby
sed -n '700,750p' src/db.rs
printf '\n---\n'
rg -n "to_string_lossy|VACUUM INTO|PathBuf|canonicalize|metadata|permissions" src/db.rsRepository: notedeck-dev/notecli Length of output: 2905 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Inspect permission-hardening logic and any path validation around backup callers
sed -n '120,210p' src/db.rs
printf '\n--- callers ---\n'
rg -n "backup_to\(" -S .
printf '\n--- path validation ---\n'
rg -n "to_str\(|to_string_lossy\(|is_utf8|utf-8|UTF-8|non-UTF|Path::new|OsStr|OsString" src/db.rs src/*.rsRepository: notedeck-dev/notecli Length of output: 4491 Reject non-UTF-8 backup paths. 🤖 Prompt for AI Agents |
||
| } | ||
| Self::restrict_permissions(dest); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔒 Security & Privacy | 🟠 Major | ⚡ Quick win Propagate permission-hardening failures for backups.
🤖 Prompt for AI Agents |
||
|
|
||
| if strip_tokens { | ||
| let copy = Connection::open(dest)?; | ||
| copy.execute("UPDATE accounts SET token = ''", [])?; | ||
| } | ||
|
Comment on lines
+732
to
+735
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔒 Security & Privacy | 🔴 Critical | 🏗️ Heavy lift 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
# Map the file and inspect the relevant area around the reported lines.
ast-grep outline src/db.rs --view expanded > /tmp/db_outline.txt
sed -n '700,780p' src/db.rs | cat -n
# Find the strip_tokens implementation and any related backup/publish logic.
rg -n "strip_tokens|VACUUM|secure_delete|Connection::open\\(dest\\)|UPDATE accounts SET token" src/db.rsRepository: notedeck-dev/notecli Length of output: 5002 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Inspect DB initialization pragmas and any backup-related tests/docs.
sed -n '1,260p' src/db.rs | cat -n
printf '\n--- backup tests ---\n'
sed -n '1460,1525p' src/db.rs | cat -n
printf '\n--- search for secure_delete / temp backup handling ---\n'
rg -n "secure_delete|temp file|tempfile|rename\\(|VACUUM INTO|backup_to\\(" srcRepository: notedeck-dev/notecli Length of output: 14773 Make 🤖 Prompt for AI Agents |
||
| Ok(()) | ||
| } | ||
|
|
||
| /// Delete a single note from the cache (e.g. when a deletion event is received). | ||
| pub fn delete_cached_note(&self, note_id: &str) -> Result<(), NoteDeckError> { | ||
| let conn = self.lock_write()?; | ||
|
|
@@ -1433,6 +1465,51 @@ mod tests { | |
| } | ||
| } | ||
|
|
||
| #[test] | ||
| fn backup_to_produces_a_readable_copy_with_data() { | ||
| let (dir, db) = temp_db(); | ||
| db.upsert_account(&sample_account()).unwrap(); | ||
| let dest = dir.path().join("backup.db"); | ||
|
|
||
| db.backup_to(&dest, false).unwrap(); | ||
|
|
||
| // VACUUM INTO は単一ファイルで完結する (-wal を伴わない) | ||
| assert!(dest.exists()); | ||
| assert!(!dir.path().join("backup.db-wal").exists()); | ||
|
|
||
| let restored = Database::open(&dest).unwrap(); | ||
| let accounts = restored.load_accounts().unwrap(); | ||
| assert_eq!(accounts.len(), 1); | ||
| assert_eq!(accounts[0].token, "test-token"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn backup_to_can_strip_tokens_from_the_copy() { | ||
| let (dir, db) = temp_db(); | ||
| db.upsert_account(&sample_account()).unwrap(); | ||
| let dest = dir.path().join("backup.db"); | ||
|
|
||
| db.backup_to(&dest, true).unwrap(); | ||
|
|
||
| let restored = Database::open(&dest).unwrap(); | ||
| assert_eq!(restored.load_accounts().unwrap()[0].token, ""); | ||
| // 元の DB は触らない | ||
| assert_eq!(db.load_accounts().unwrap()[0].token, "test-token"); | ||
| } | ||
|
|
||
| #[test] | ||
| fn backup_to_overwrites_an_existing_file() { | ||
| let (dir, db) = temp_db(); | ||
| db.upsert_account(&sample_account()).unwrap(); | ||
| let dest = dir.path().join("backup.db"); | ||
| std::fs::write(&dest, b"stale").unwrap(); | ||
|
|
||
| db.backup_to(&dest, false).unwrap(); | ||
|
|
||
| let restored = Database::open(&dest).unwrap(); | ||
| assert_eq!(restored.load_accounts().unwrap().len(), 1); | ||
| } | ||
|
|
||
| #[test] | ||
| fn account_upsert_and_load() { | ||
| let (_dir, db) = temp_db(); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🗄️ Data Integrity & Integration | 🔴 Critical | ⚡ Quick win
🧩 Analysis chain
🏁 Script executed:
Repository: notedeck-dev/notecli
Length of output: 13512
🏁 Script executed:
Repository: notedeck-dev/notecli
Length of output: 6083
🏁 Script executed:
Repository: notedeck-dev/notecli
Length of output: 4278
Reject the live database path here. If
destpoints to the open database file,remove_file(dest)unlinks the live DB beforeVACUUM INTO, so existing connections keep using the unlinked inode while the path is recreated as the backup. Comparedestagainst the current database filename before deletion and add a same-path regression test.🤖 Prompt for AI Agents
🔒 Security & Privacy | 🟠 Major | ⚡ Quick win
🧩 Analysis chain
🏁 Script executed:
Repository: notedeck-dev/notecli
Length of output: 12915
🏁 Script executed:
Repository: notedeck-dev/notecli
Length of output: 3286
🌐 Web query:
SQLite VACUUM INTO existing destination WAL SHM sidecar files strip_tokens backup overwrite💡 Result:
The SQLite VACUUM INTO command requires that the destination file does not exist [1][2]. If the destination file already exists, the command will fail with an error [1][2]. It cannot be used to overwrite an existing database file directly. Regarding WAL (Write-Ahead Logging) and SHM (shared-memory) sidecar files: 1. VACUUM INTO creates a new, compacted database file [1][2]. It does not create WAL or SHM files for the destination at the moment of creation [3][4]. 2. When restoring a database, existing WAL (-wal) and SHM (-shm) sidecar files associated with the destination must be manually deleted before the new database file is put in place [5]. These sidecar files are intrinsically linked to a specific database file state and are not compatible with a different or restored version of the database [5][6]. 3. If the destination file exists (and for some reason you are replacing it), you must remove the old database file along with its associated *-wal and *-shm sidecar files to prevent corruption or invalid state [5]. In summary, VACUUM INTO is a safe, non-destructive operation because it enforces the absence of the destination file [1]. If you intend to use it as part of an update or restoration process, you must manually clear the destination directory of the old database and its sidecar files before executing the command [5].
Citations:
🌐 Web query:
SQLite stale WAL file replaced database file ignored salt checksum same path💡 Result:
When you replace an SQLite database file while leaving an old Write-Ahead Log (-wal) and shared memory (-shm) file in place, SQLite will attempt to process the stale files during the next connection [1][2][3]. Because the WAL file contains metadata including "salt" values that must match the current database file header, a mismatch will occur when they do not align [4][5][6]. Here is how SQLite handles this situation: 1. Salt Mismatch: The WAL header contains two "salt" values [7][5]. When SQLite opens a database, it checks if the salt values in the -wal file match the database file [4][6]. If you have replaced the database file, the old WAL file will contain salts from the previous database [1][3]. SQLite sees this mismatch and correctly identifies the WAL as "stale" or belonging to a different database instance [3]. 2. Checksum Verification: Beyond salt checks, SQLite verifies the integrity of each "frame" (transaction) in the WAL file by computing a checksum that includes the header and frame data [8][4][6]. If the WAL is stale, or if there is a mismatch during the recovery/checkpointing process, the checksum verification will fail [8][4][6]. When a frame fails checksum validation, SQLite stops processing the WAL at that point [8][9][10]. 3. Outcome: If the WAL and SHM files do not match the new database file, the database can appear "corrupt" or "malformed," potentially leading to error messages such as "database disk image is malformed" [1][3]. SQLite does not automatically "ignore" these files in a way that preserves data; instead, it may attempt to roll back or checkpoint them, which can lead to data loss or corruption of the newly placed database file [3][9][11]. To safely replace a database file in WAL mode, you must delete the accompanying -wal and -shm files alongside the main database file [1][2][3]. The only safe way to remove these files without manual deletion is to ensure all connections are closed properly, allowing SQLite to clean them up, or by opening the database and closing it immediately via the SQLite API [11][10]. Manually replacing only the.db file while leaving stale auxiliary files in the same path is a frequent cause of database corruption [1][3].
Citations:
Remove SQLite sidecars when overwriting backups. Deleting only
destleaves any existingdest-wal/dest-shmfiles behind, so an overwritten backup can still expose old WAL contents and break the strip-tokens guarantee. Remove the full SQLite file set or write to a fresh temp path and rename it into place.🤖 Prompt for AI Agents