Mega Beta5

Cargo.toml

@@ -18,6 +18,10 @@ categories = ["database-implementations", "data-structures", "caching"]
 default = ["wti-predictable-search"]
 perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"]
 s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"]
+# Moves unique WorkTablesIndex structural CDC work out of the table mutation
+# path and into the background persistence worker. The persisted page format
+# is unchanged, so stores remain readable with or without this feature.
+logical-index-persistence = ["worktable_codegen/logical-index-persistence"]
 wti-hybrid-search = ["indexset/wt-slice-binary-search"]
 wti-predictable-search = ["indexset/custom-binary-search"]
 wti-std-search = ["indexset/std-binary-search"]
@@ -54,6 +58,7 @@ prettytable-rs = "^0.10"
 psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] }
 rkyv = { version = "0.8.17", features = ["uuid-1"] }
 reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] }
+rustc-hash = "2.1.1"
 rusty-s3 = { version = "0.10.2", optional = true }
 smart-default = "0.7.1"
 tokio = { version = "1", features = ["full"] }

README.md

@@ -40,8 +40,10 @@ cargo add worktable@1.0.0-beta.4
 > not mean the change is on stable storage. `wait_for_ops()` and `close()` flush the
 > persistence pipeline, but the current disk format has no transaction journal and
 > does not `fsync` every batch. Process or power loss can therefore lose acknowledged
-> changes. A torn store is refused with `PersistenceLoadError` rather than opened as
-> plausible-but-invented rows. See the [durability and recovery contract](docs/persistence-durability.md).
+> changes. Normal loads refuse a torn store with `PersistenceLoadError` rather than
+> opening plausible-but-invented rows. A deliberately explicit recovery mode can read
+> individually validated rows from a private scratch copy through a surviving index;
+> see the [durability and recovery contract](docs/persistence-durability.md).
 
 Persistence is implemented, not planned. `PersistedWorkTable` and `PersistenceConfig` are
 exported from the crate root; the prelude carries `DiskPersistenceEngine`,
@@ -96,7 +98,10 @@ need an external snapshot/rebuild strategy. A graceful persistence error is
 terminal and surfaced consistently, but abrupt termination can currently leave
 a partial multi-file batch. Loading audits archived rows plus primary and
 secondary index consistency before exposing the table; torn state is refused as
-`PersistenceLoadError` and must be restored or rebuilt as documented above.
+`PersistenceLoadError`. Offline recovery tools may opt into `LoadMode::Recovery` to
+copy individually validated rows through a surviving index into a clean table, which
+must then pass a normal strict load before publication. This mode is not an in-place
+repair and must never serve live traffic.
 
 Generated persisted tables store row-schema, primary-key, and secondary-index
 metadata in `SpaceInfo`. Existing legacy files whose schema metadata is

codegen/Cargo.toml

@@ -8,6 +8,7 @@ repository = "https://github.com/pathscale/WorkTable"
 
 [features]
 s3-support = []
+logical-index-persistence = []
 # Compatibility no-op retained for downstream manifests.
 versioned-row-publication = []
 

codegen/src/generators/in_memory/queries/select.rs

@@ -35,10 +35,29 @@ impl InMemoryGenerator {
                 let iter = std::iter::once_with(move || {
                     let read_guard = self.0.data.read_guard();
                     self.0.primary_index.pk_map
-                        .iter_links()
-                        .filter_map(move |link| {
+                        .iter_values()
+                        .filter_map(move |(primary_key, link)| {
                             let _read_guard = &read_guard;
-                            self.0.data.select_non_ghosted(link.0).ok()
+                            let mut current_link = link.0;
+                            for _ in 0..64 {
+                                if let Ok(row) = self.0.data.select_non_ghosted(current_link) {
+                                    return Some(row);
+                                }
+
+                                // A reinsert publishes the replacement link
+                                // before retiring the captured one. Follow that
+                                // replacement instead of silently omitting the
+                                // row from a concurrent full-table scan.
+                                let replacement: Link = self.0.primary_index.pk_map
+                                    .lookup_for_select(&primary_key)
+                                    .map(Into::into)?;
+                                if replacement == current_link {
+                                    return None;
+                                }
+                                current_link = replacement;
+                                std::hint::spin_loop();
+                            }
+                            None
                         })
                 }).flatten();