From ba06b82e4cbab2f5addbedee3f10b5b8da53ec51 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 07:17:15 +0700 Subject: [PATCH 01/23] feat: parse per-index backend selection --- codegen/src/common/model/column.rs | 22 ++- codegen/src/common/model/index.rs | 30 ++++ codegen/src/common/model/mod.rs | 4 +- codegen/src/common/model/persistence.rs | 18 +++ codegen/src/common/parser/attribute.rs | 37 +++-- codegen/src/common/parser/columns.rs | 25 ++++ codegen/src/common/parser/index.rs | 82 ++++++++++- codegen/src/generators/in_memory/mod.rs | 2 +- codegen/src/worktable/mod.rs | 185 +++++++++++++++++++++++- 9 files changed, 390 insertions(+), 15 deletions(-) create mode 100644 codegen/src/common/model/persistence.rs diff --git a/codegen/src/common/model/column.rs b/codegen/src/common/model/column.rs index a9fa4b1f..611c9d73 100644 --- a/codegen/src/common/model/column.rs +++ b/codegen/src/common/model/column.rs @@ -1,8 +1,8 @@ use indexmap::IndexMap; use std::collections::HashMap; -use crate::common::model::GeneratorType; use crate::common::model::index::Index; +use crate::common::model::{GeneratorType, IndexBackend}; use proc_macro2::{Ident, TokenStream}; use quote::quote; use syn::spanned::Spanned; @@ -18,6 +18,7 @@ pub struct Columns { pub field_positions: HashMap, pub indexes: IndexMap, pub primary_keys: Vec, + pub primary_index_backend: IndexBackend, pub generator_type: GeneratorType, } @@ -28,6 +29,7 @@ pub struct Row { pub is_primary_key: bool, pub gen_type: GeneratorType, pub optional: bool, + pub index_backend: Option, } impl Columns { @@ -37,6 +39,7 @@ impl Columns { let mut sized = true; let mut pk = vec![]; let mut gen_type = None; + let mut primary_index_backend = None; for (pos, row) in rows.into_iter().enumerate() { let type_ = &row.type_; @@ -59,7 +62,23 @@ impl Columns { } else { gen_type = Some(row.gen_type) } + let backend = row.index_backend.unwrap_or_default(); + if let Some(existing) = primary_index_backend { + if existing != backend { + return Err(syn::Error::new( + row.name.span(), + "all columns in a composite primary key must use the same index backend", + )); + } + } else { + primary_index_backend = Some(backend); + } pk.push(row.name); + } else if row.index_backend.is_some() { + return Err(syn::Error::new( + row.name.span(), + "`using` on a column is only valid after `primary_key`; select secondary index backends in `indexes`", + )); } } @@ -72,6 +91,7 @@ impl Columns { columns_map, indexes: Default::default(), primary_keys: pk, + primary_index_backend: primary_index_backend.unwrap_or_default(), generator_type: gen_type.expect("set"), field_positions, }) diff --git a/codegen/src/common/model/index.rs b/codegen/src/common/model/index.rs index 07135a84..f3297324 100644 --- a/codegen/src/common/model/index.rs +++ b/codegen/src/common/model/index.rs @@ -1,8 +1,38 @@ use proc_macro2::Ident; +/// Physical implementation selected for a generated index. +/// +/// `WorktablesIndex` is deliberately the default so existing declarations keep +/// their current implementation and persistence semantics when `using` is +/// absent. Vanilla upstream IndexSet is an explicit, parallel backend. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum IndexBackend { + #[default] + WorktablesIndex, + Indexset, + Congee, + Arctic, +} + +impl IndexBackend { + pub fn is_memory_only(self) -> bool { + matches!(self, Self::Congee | Self::Arctic) + } + + pub fn name(self) -> &'static str { + match self { + Self::WorktablesIndex => "worktables_index", + Self::Indexset => "indexset", + Self::Congee => "congee", + Self::Arctic => "arctic", + } + } +} + #[derive(Debug, Clone, PartialEq)] pub struct Index { pub name: Ident, pub field: Ident, pub is_unique: bool, + pub backend: IndexBackend, } diff --git a/codegen/src/common/model/mod.rs b/codegen/src/common/model/mod.rs index 48967812..93604295 100644 --- a/codegen/src/common/model/mod.rs +++ b/codegen/src/common/model/mod.rs @@ -2,12 +2,14 @@ mod column; mod config; mod index; pub mod operation; +mod persistence; mod primary_key; mod queries; pub use column::{Columns, Row}; pub use config::Config; -pub use index::Index; +pub use index::{Index, IndexBackend}; pub use operation::Operation; +pub use persistence::Persistence; pub use primary_key::{GeneratorType, PrimaryKey}; pub use queries::Queries; diff --git a/codegen/src/common/model/persistence.rs b/codegen/src/common/model/persistence.rs new file mode 100644 index 00000000..bef59fa5 --- /dev/null +++ b/codegen/src/common/model/persistence.rs @@ -0,0 +1,18 @@ +/// Whether the table declaration explicitly selected persistence. +/// +/// Keeping `Omitted` distinct from `MemoryOnly` lets the macro require an +/// explicit `persist: false` acknowledgement before selecting an index backend +/// that cannot participate in disk or S3 persistence. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Persistence { + #[default] + Omitted, + MemoryOnly, + Persisted, +} + +impl Persistence { + pub fn is_persisted(self) -> bool { + matches!(self, Self::Persisted) + } +} diff --git a/codegen/src/common/parser/attribute.rs b/codegen/src/common/parser/attribute.rs index 5036bcb8..e1412594 100644 --- a/codegen/src/common/parser/attribute.rs +++ b/codegen/src/common/parser/attribute.rs @@ -1,13 +1,14 @@ use proc_macro2::TokenTree; use syn::spanned::Spanned as _; +use crate::common::model::Persistence; use crate::common::parser::Parser; // TODO: Move this to separate attributes section because now it only parses persist. impl Parser { - pub fn parse_persist(&mut self) -> syn::Result { + pub fn parse_persist(&mut self) -> syn::Result { let Some(ident) = self.input_iter.peek().cloned() else { - return Ok(false); + return Ok(Persistence::Omitted); }; let TokenTree::Ident(ident) = ident else { return Err(syn::Error::new(ident.span(), "Expected field name identifier.")); @@ -22,9 +23,11 @@ impl Parser { .ok_or(syn::Error::new(self.input.span(), "Expected token."))?; let res = if let TokenTree::Ident(bool) = bool { if bool.to_string().as_str() == "true" { - Ok(true) + Ok(Persistence::Persisted) + } else if bool.to_string().as_str() == "false" { + Ok(Persistence::MemoryOnly) } else { - Ok(false) + Err(syn::Error::new(bool.span(), "expected `true` or `false`")) } } else { Err(syn::Error::new(bool.span(), "Expected identifier.")) @@ -33,7 +36,7 @@ impl Parser { res } else { - Ok(false) + Ok(Persistence::Omitted) } } } @@ -43,6 +46,7 @@ mod tests { use quote::quote; use crate::common::Parser; + use crate::common::model::Persistence; #[test] fn test_empty() { @@ -50,7 +54,7 @@ mod tests { let mut parser = Parser::new(tokens); let empty = parser.parse_persist(); assert!(empty.is_ok()); - assert!(!empty.unwrap()) + assert_eq!(empty.unwrap(), Persistence::Omitted) } #[test] @@ -67,7 +71,22 @@ mod tests { let mut parser = Parser::new(tokens); let name = parser.parse_persist(); assert!(name.is_ok()); - assert!(name.unwrap()); + assert_eq!(name.unwrap(), Persistence::Persisted); + } + + #[test] + fn test_explicit_memory_only() { + let tokens = quote! {persist: false,}; + let mut parser = Parser::new(tokens); + let persistence = parser.parse_persist().unwrap(); + assert_eq!(persistence, Persistence::MemoryOnly); + } + + #[test] + fn test_invalid_boolean() { + let tokens = quote! {persist: maybe,}; + let mut parser = Parser::new(tokens); + assert!(parser.parse_persist().is_err()); } #[test] @@ -76,7 +95,7 @@ mod tests { let mut parser = Parser::new(tokens); let name = parser.parse_persist(); assert!(name.is_ok()); - assert!(!name.unwrap()); + assert_eq!(name.unwrap(), Persistence::Omitted); } #[test] @@ -85,6 +104,6 @@ mod tests { let mut parser = Parser::new(tokens); let name = parser.parse_persist(); assert!(name.is_ok()); - assert!(!name.unwrap()); + assert_eq!(name.unwrap(), Persistence::Omitted); } } diff --git a/codegen/src/common/parser/columns.rs b/codegen/src/common/parser/columns.rs index 7d45741c..65126f64 100644 --- a/codegen/src/common/parser/columns.rs +++ b/codegen/src/common/parser/columns.rs @@ -108,6 +108,8 @@ impl Parser { false }; + let index_backend = self.try_parse_index_backend()?; + self.try_parse_comma()?; Ok(Row { @@ -116,6 +118,7 @@ impl Parser { is_primary_key, gen_type, optional, + index_backend, }) } } @@ -310,5 +313,27 @@ mod tests { assert!(row.optional); assert!(!row.is_primary_key) } + + #[test] + fn test_primary_backend_parse() { + let row_tokens = quote! {id: u64 primary_key autoincrement using congee,}; + + let mut parser = Parser::new(row_tokens); + let row = parser.parse_row().unwrap(); + + assert_eq!(row.index_backend, Some(crate::common::model::IndexBackend::Congee)); + } + + #[test] + fn test_using_rejected_on_plain_column() { + let tokens = quote! {columns: { + id: u64 primary_key, + value: u64 using arctic, + }}; + + let mut parser = Parser::new(tokens); + let error = parser.parse_columns().unwrap_err(); + assert!(error.to_string().contains("only valid after `primary_key`")); + } } } diff --git a/codegen/src/common/parser/index.rs b/codegen/src/common/parser/index.rs index ee172b6d..1f41bd0f 100644 --- a/codegen/src/common/parser/index.rs +++ b/codegen/src/common/parser/index.rs @@ -1,10 +1,45 @@ use crate::common::Parser; -use crate::common::model::Index; +use crate::common::model::{Index, IndexBackend}; use indexmap::IndexMap; use proc_macro2::{Delimiter, Ident, TokenTree}; use syn::spanned::Spanned; impl Parser { + pub fn try_parse_index_backend(&mut self) -> syn::Result> { + let Some(TokenTree::Ident(using)) = self.input_iter.peek() else { + return Ok(None); + }; + if using != "using" { + return Ok(None); + } + + let using_span = using.span(); + self.input_iter.next(); + let backend = self.input_iter.next().ok_or_else(|| { + syn::Error::new( + using_span, + "expected an index backend after `using`: `worktables_index`, `indexset`, `congee`, or `arctic`", + ) + })?; + let TokenTree::Ident(backend) = backend else { + return Err(syn::Error::new( + backend.span(), + "expected an index backend identifier after `using`", + )); + }; + + match backend.to_string().as_str() { + "worktables_index" => Ok(Some(IndexBackend::WorktablesIndex)), + "indexset" => Ok(Some(IndexBackend::Indexset)), + "congee" => Ok(Some(IndexBackend::Congee)), + "arctic" => Ok(Some(IndexBackend::Arctic)), + _ => Err(syn::Error::new( + backend.span(), + "unknown index backend; expected `worktables_index`, `indexset`, `congee`, or `arctic`", + )), + } + } + pub fn parse_indexes(&mut self) -> syn::Result> { let ident = self.input_iter.next().ok_or(syn::Error::new( self.input.span(), @@ -86,6 +121,8 @@ impl Parser { false }; + let backend = self.try_parse_index_backend()?.unwrap_or_default(); + self.try_parse_comma()?; Ok(( @@ -94,7 +131,50 @@ impl Parser { name: ident, field: row_name, is_unique, + backend, }, )) } } + +#[cfg(test)] +mod tests { + use quote::quote; + + use crate::common::Parser; + use crate::common::model::IndexBackend; + + #[test] + fn absent_using_defaults_to_worktables_index() { + let mut parser = Parser::new(quote! { value_idx: value unique, }); + let (_, index) = parser.parse_index().unwrap(); + assert_eq!(index.backend, IndexBackend::WorktablesIndex); + } + + #[test] + fn parses_all_backends() { + for (tokens, expected) in [ + ( + quote! { value_idx: value unique using worktables_index, }, + IndexBackend::WorktablesIndex, + ), + ( + quote! { value_idx: value unique using indexset, }, + IndexBackend::Indexset, + ), + (quote! { value_idx: value unique using congee, }, IndexBackend::Congee), + (quote! { value_idx: value unique using arctic, }, IndexBackend::Arctic), + ] { + let mut parser = Parser::new(tokens); + let (_, index) = parser.parse_index().unwrap(); + assert_eq!(index.backend, expected); + } + } + + #[test] + fn rejects_unknown_backend() { + let mut parser = Parser::new(quote! { value_idx: value unique using unknown, }); + let error = parser.parse_index().unwrap_err(); + assert!(error.to_string().contains("unknown index backend")); + } +} diff --git a/codegen/src/generators/in_memory/mod.rs b/codegen/src/generators/in_memory/mod.rs index 5dcc3e05..114ce2d1 100644 --- a/codegen/src/generators/in_memory/mod.rs +++ b/codegen/src/generators/in_memory/mod.rs @@ -68,7 +68,7 @@ pub fn expand(input: TokenStream) -> syn::Result { } "persist" => { // Skip persist flag for in_memory - it's always false - parser.parse_persist()?; + let _ = parser.parse_persist()?; } _ => return Err(syn::Error::new(ident.span(), "Unexpected identifier")), } diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index eac9d5fb..4a053612 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -1,6 +1,7 @@ use proc_macro2::TokenStream; use crate::common::Parser; +use crate::common::model::{Columns, Persistence}; pub fn expand(input: TokenStream) -> syn::Result { let mut parser = Parser::new(input); @@ -11,7 +12,7 @@ pub fn expand(input: TokenStream) -> syn::Result { let name = parser.parse_name()?; let version = parser.parse_version()?.unwrap_or(1); - let is_persist = parser.parse_persist()?; + let persistence = parser.parse_persist()?; while let Some(ident) = parser.peek_next() { match ident.to_string().as_str() { "columns" => { @@ -45,9 +46,189 @@ pub fn expand(input: TokenStream) -> syn::Result { columns.indexes = i } - if is_persist { + validate_index_backends(&columns, persistence)?; + + if persistence.is_persisted() { crate::generators::persist::expand(name, columns, queries, config, version) } else { crate::generators::in_memory::expand_from_parsed(name, columns, queries, config) } } + +fn validate_index_backends(columns: &Columns, persistence: Persistence) -> syn::Result<()> { + let memory_only = if columns.primary_index_backend.is_memory_only() { + Some(( + columns.primary_index_backend, + columns.primary_keys.first().expect("primary key exists"), + true, + )) + } else { + columns + .indexes + .values() + .find(|index| index.backend.is_memory_only()) + .map(|index| (index.backend, &index.name, false)) + }; + + if let Some((backend, ident, is_primary)) = memory_only { + let kind = if is_primary { "primary index" } else { "index" }; + match persistence { + Persistence::MemoryOnly => {} + Persistence::Omitted => { + return Err(syn::Error::new( + ident.span(), + format!( + "{kind} `{ident}` uses `{}`, which requires an explicitly written `persist: false`", + backend.name() + ), + )); + } + Persistence::Persisted => { + return Err(syn::Error::new( + ident.span(), + format!( + "{kind} `{ident}` uses `{}`, but persisted and S3-backed tables require `worktables_index` or `indexset`", + backend.name() + ), + )); + } + } + } + + if let Some(index) = columns + .indexes + .values() + .find(|index| !index.is_unique && index.backend.is_memory_only()) + { + return Err(syn::Error::new( + index.name.span(), + format!( + "non-unique index `{}` cannot use `{}`; non-unique indexes currently require `worktables_index` or `indexset`", + index.name, + index.backend.name() + ), + )); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use quote::quote; + + use super::expand; + + #[test] + fn absent_using_keeps_worktables_index_default() { + let output = expand(quote! { + name: DefaultBackend, + persist: true, + columns: { + id: u64 primary_key autoincrement, + value: u64, + }, + indexes: { + value_idx: value unique, + }, + }); + + assert!(output.is_ok()); + } + + #[test] + fn explicit_indexset_is_persistence_compatible() { + let output = expand(quote! { + name: ExplicitIndexset, + persist: true, + columns: { + id: u64 primary_key autoincrement using indexset, + value: u64, + }, + indexes: { + value_idx: value unique using indexset, + }, + }); + + assert!(output.is_ok()); + } + + #[test] + fn explicit_worktables_index_is_persistence_compatible() { + let output = expand(quote! { + name: ExplicitWorktablesIndex, + persist: true, + columns: { + id: u64 primary_key autoincrement using worktables_index, + value: u64, + }, + indexes: { + value_idx: value unique using worktables_index, + }, + }); + + assert!(output.is_ok()); + } + + #[test] + fn memory_backend_requires_explicit_false() { + let error = expand(quote! { + name: MissingAcknowledgement, + columns: { + id: u64 primary_key using congee, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("explicitly written `persist: false`")); + } + + #[test] + fn memory_backend_rejects_persistence() { + let error = expand(quote! { + name: PersistentArctic, + persist: true, + columns: { + id: u64 primary_key, + value: u64, + }, + indexes: { + value_idx: value unique using arctic, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("persisted and S3-backed tables")); + } + + #[test] + fn memory_backend_accepts_explicit_false() { + let output = expand(quote! { + name: ExplicitMemory, + persist: false, + columns: { + id: u64 primary_key using congee, + }, + }); + + assert!(output.is_ok()); + } + + #[test] + fn memory_backend_rejects_non_unique_indexes() { + let error = expand(quote! { + name: NonUniqueArctic, + persist: false, + columns: { + id: u64 primary_key, + value: u64, + }, + indexes: { + value_idx: value using arctic, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("non-unique indexes currently require")); + } +} From 6583e174cd106aed4acf99cd6e66f4bfaa1d8702 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 07:28:09 +0700 Subject: [PATCH 02/23] feat: add parallel upstream indexset adapter --- Cargo.toml | 1 + src/index/mod.rs | 2 + src/index/unique.rs | 168 ++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- 4 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 src/index/unique.rs diff --git a/Cargo.toml b/Cargo.toml index 93bf7359..d317a1b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ eyre = "0.6.12" fastrand = "2.3.0" futures = "0.3.30" indexset = { package = "WorkTablesIndex", version = "=0.0.1", features = ["concurrent", "cdc", "multimap"] } +vanilla_indexset = { package = "indexset", version = "=0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] } log = "0.4.29" diff --git a/src/index/mod.rs b/src/index/mod.rs index 1da0dd2e..ca4af0b4 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -3,6 +3,7 @@ mod multipair; mod primary_index; mod table_index; mod table_secondary_index; +mod unique; mod unsized_node; pub use available_index::AvailableIndex; @@ -14,6 +15,7 @@ pub use table_index::{TableIndex, TableIndexCdc, convert_change_events}; pub use table_secondary_index::{ IndexError, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, }; +pub use unique::{UniqueIndex, UpstreamIndexMap}; pub use unsized_node::UnsizedNode; #[derive(Clone, Debug)] diff --git a/src/index/unique.rs b/src/index/unique.rs new file mode 100644 index 00000000..6326dd1c --- /dev/null +++ b/src/index/unique.rs @@ -0,0 +1,168 @@ +//! Backend-neutral operations required by a unique WorkTable index. +//! +//! The trait deliberately returns copied/cloned values instead of exposing a +//! backend's guard type. That keeps generated code independent from the +//! concurrency and reclamation strategy used by each index implementation. + +use std::fmt::Debug; +use std::hash::Hash; +use std::ops::RangeBounds; + +use crate::IndexMap; +use indexset::core::node::NodeLike; +use indexset::core::pair::Pair; +use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; +use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +use vanilla_indexset::core::pair::Pair as VanillaPair; + +/// Point, mutation, and ordered-scan operations used by generated unique +/// indexes. Implementations are statically dispatched; this adds no virtual +/// call to the lookup path. +pub trait UniqueIndex: Debug + Default + Send + Sync +where + K: Clone + Ord, + V: Clone, +{ + fn get_value(&self, key: &K) -> Option; + fn insert_value(&self, key: K, value: V) -> Option; + fn insert_value_checked(&self, key: K, value: V) -> Option<()>; + fn remove_value(&self, key: &K) -> Option<(K, V)>; + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn iter_values(&self) -> impl DoubleEndedIterator + '_; + + fn range_values(&self, range: R) -> impl DoubleEndedIterator + '_ + where + R: RangeBounds; +} + +impl UniqueIndex for IndexMap +where + K: Debug + Eq + Hash + Clone + Send + Sync + Ord + 'static, + V: Debug + Clone + Send + Sync + Ord + 'static, + Node: NodeLike> + Debug + Send + Sync + 'static, +{ + #[inline] + fn get_value(&self, key: &K) -> Option { + self.get(key).map(|entry| entry.get().value.clone()) + } + + #[inline] + fn insert_value(&self, key: K, value: V) -> Option { + self.insert(key, value) + } + + #[inline] + fn insert_value_checked(&self, key: K, value: V) -> Option<()> { + self.checked_insert(key, value) + } + + #[inline] + fn remove_value(&self, key: &K) -> Option<(K, V)> { + self.remove(key) + } + + #[inline] + fn len(&self) -> usize { + self.len() + } + + #[inline] + fn iter_values(&self) -> impl DoubleEndedIterator + '_ { + self.iter().map(|(key, value)| (key.clone(), value.clone())) + } + + #[inline] + fn range_values(&self, range: R) -> impl DoubleEndedIterator + '_ + where + R: RangeBounds, + { + self.range(range).map(|(key, value)| (key.clone(), value.clone())) + } +} + +impl UniqueIndex for VanillaIndexMap +where + K: Debug + Eq + Hash + Clone + Send + Sync + Ord + 'static, + V: Debug + Clone + Send + Sync + Ord + 'static, + Node: VanillaNodeLike> + Debug + Send + Sync + 'static, +{ + #[inline] + fn get_value(&self, key: &K) -> Option { + self.get(key).map(|entry| entry.get().value.clone()) + } + + #[inline] + fn insert_value(&self, key: K, value: V) -> Option { + self.insert(key, value) + } + + #[inline] + fn insert_value_checked(&self, key: K, value: V) -> Option<()> { + self.checked_insert(key, value) + } + + #[inline] + fn remove_value(&self, key: &K) -> Option<(K, V)> { + self.remove(key) + } + + #[inline] + fn len(&self) -> usize { + self.len() + } + + #[inline] + fn iter_values(&self) -> impl DoubleEndedIterator + '_ { + self.iter().map(|(key, value)| (key.clone(), value.clone())) + } + + #[inline] + fn range_values(&self, range: R) -> impl DoubleEndedIterator + '_ + where + R: RangeBounds, + { + self.range(range).map(|(key, value)| (key.clone(), value.clone())) + } +} + +/// Vanilla upstream IndexSet map, kept distinct from WorkTable's default +/// WorkTablesIndex alias so both implementations may coexist in one binary. +pub type UpstreamIndexMap>> = VanillaIndexMap; + +#[cfg(test)] +mod tests { + use super::{UniqueIndex, UpstreamIndexMap}; + use crate::IndexMap; + + fn assert_unique_index_contract() + where + I: UniqueIndex, + { + let index = I::default(); + assert!(index.is_empty()); + assert_eq!(index.insert_value_checked(2, 20), Some(())); + assert_eq!(index.insert_value_checked(1, 10), Some(())); + assert_eq!(index.insert_value_checked(2, 99), None); + assert_eq!(index.get_value(&2), Some(20)); + assert_eq!(index.insert_value(2, 22), Some(20)); + assert_eq!(index.iter_values().collect::>(), vec![(1, 10), (2, 22)]); + assert_eq!(index.range_values(2..=2).collect::>(), vec![(2, 22)]); + assert_eq!(index.remove_value(&1), Some((1, 10))); + assert_eq!(index.len(), 1); + } + + #[test] + fn worktables_index_implements_contract() { + assert_unique_index_contract::>(); + } + + #[test] + fn upstream_indexset_implements_contract() { + assert_unique_index_contract::>(); + } +} diff --git a/src/lib.rs b/src/lib.rs index 4cd58ae0..d1a40e5c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,8 +45,8 @@ pub mod prelude { pub use crate::{ AvailableIndex, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, - TableSecondaryIndexInfo, UnsizedNode, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, - vacuum::VacuumPersistence, vacuum::WorkTableVacuum, + TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, UpstreamIndexMap, WorkTable, WorkTableError, + vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum, }; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, From c889db77c58385c5011eb52b45c2f3687dfdb11a Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 07:33:26 +0700 Subject: [PATCH 03/23] feat: add congee unique-index adapter --- Cargo.toml | 1 + src/index/congee.rs | 244 ++++++++++++++++++++++++++++++++++++++++++++ src/index/mod.rs | 2 + src/index/unique.rs | 12 +-- src/lib.rs | 8 +- 5 files changed, 257 insertions(+), 10 deletions(-) create mode 100644 src/index/congee.rs diff --git a/Cargo.toml b/Cargo.toml index d317a1b2..09184f48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktabl [dependencies] async-trait = "0.1.89" +congee = "=0.4.1" convert_case = "0.6.0" data_bucket = "=0.4.1" # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" } diff --git a/src/index/congee.rs b/src/index/congee.rs new file mode 100644 index 00000000..86a2b668 --- /dev/null +++ b/src/index/congee.rs @@ -0,0 +1,244 @@ +//! Congee adapter for memory-only unique WorkTable indexes. + +use std::fmt::{self, Debug}; +use std::ops::RangeBounds; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use congee::{Congee, DefaultAllocator}; + +use super::UniqueIndex; + +/// Lossless conversion between a WorkTable key and Congee's machine-word key. +/// +/// Congee 0.4 stores keys as one `usize`, so this backend intentionally accepts +/// only integer keys that fit without truncation. NanoID and composite keys +/// should use WorkTablesIndex, IndexSet, or Arctic. +pub trait CongeeKey: Copy + Debug + Ord + Send + Sync + 'static { + fn into_congee(self) -> usize; + fn from_congee(value: usize) -> Self; +} + +macro_rules! impl_congee_key { + ($($ty:ty),* $(,)?) => { + $( + impl CongeeKey for $ty { + #[inline] + fn into_congee(self) -> usize { self as usize } + + #[inline] + fn from_congee(value: usize) -> Self { value as Self } + } + )* + }; +} + +impl_congee_key!(u8, u16, u32, usize); + +#[cfg(target_pointer_width = "64")] +impl_congee_key!(u64); + +/// A Congee adaptive radix tree with WorkTable's unique-index contract. +/// +/// Values are held through `Arc` pointers because Congee's payload is one +/// machine word while a WorkTable `Link` is wider. The adapter follows the +/// reclamation pattern used by Congee's own `CongeeArc` implementation. +pub struct CongeeIndex { + inner: Congee, + len: AtomicUsize, + marker: std::marker::PhantomData<(K, V)>, +} + +impl Debug for CongeeIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CongeeIndex") + .field("len", &self.len.load(Ordering::Relaxed)) + .finish_non_exhaustive() + } +} + +impl Default for CongeeIndex +where + K: CongeeKey, + V: Clone + Debug + Send + Sync + 'static, +{ + fn default() -> Self { + let drainer = |_key: usize, pointer: usize| { + // SAFETY: every payload inserted below originates from + // `Arc::into_raw` with exactly one tree-owned strong reference. + drop(unsafe { Arc::from_raw(pointer as *const V) }); + }; + Self { + inner: Congee::new_with_drainer(DefaultAllocator {}, drainer), + len: AtomicUsize::new(0), + marker: std::marker::PhantomData, + } + } +} + +impl CongeeIndex +where + K: CongeeKey, + V: Clone + Debug + Send + Sync + 'static, +{ + #[inline] + fn clone_pointer(pointer: usize) -> Arc { + // SAFETY: the caller holds a Congee epoch guard, so the tree-owned + // strong reference cannot be reclaimed while it is cloned. + let owned = unsafe { Arc::from_raw(pointer as *const V) }; + let cloned = Arc::clone(&owned); + let _ = Arc::into_raw(owned); + cloned + } + + #[inline] + fn retire_old(pointer: usize, guard: &congee::epoch::Guard) -> Arc { + // SAFETY: a successful replacement/removal transfers the tree-owned + // strong reference to this call. + let owned = unsafe { Arc::from_raw(pointer as *const V) }; + let delayed = Arc::clone(&owned); + guard.defer(move || drop(delayed)); + owned + } + + #[cold] + fn allocation_failure() -> ! { + panic!("Congee failed to allocate an index node") + } +} + +impl UniqueIndex for CongeeIndex +where + K: CongeeKey, + V: Clone + Debug + Send + Sync + 'static, +{ + #[inline] + fn get_value(&self, key: &K) -> Option { + let guard = self.inner.pin(); + let pointer = self.inner.get(&key.into_congee(), &guard)?; + Some(Self::clone_pointer(pointer).as_ref().clone()) + } + + #[inline] + fn insert_value(&self, key: K, value: V) -> Option { + let guard = self.inner.pin(); + let pointer = Arc::into_raw(Arc::new(value)) as usize; + match self.inner.insert(key.into_congee(), pointer, &guard) { + Ok(Some(old)) => Some(Self::retire_old(old, &guard).as_ref().clone()), + Ok(None) => { + self.len.fetch_add(1, Ordering::Relaxed); + None + } + Err(_) => { + // SAFETY: insertion failed, so ownership never transferred. + drop(unsafe { Arc::from_raw(pointer as *const V) }); + Self::allocation_failure() + } + } + } + + #[inline] + fn insert_value_checked(&self, key: K, value: V) -> Option<()> { + let guard = self.inner.pin(); + let pointer = Arc::into_raw(Arc::new(value)) as usize; + let result = self + .inner + .compute_or_insert(key.into_congee(), |old| old.unwrap_or(pointer), &guard); + + match result { + Ok(Some(_)) => { + // The closure returned the existing pointer, so the new value + // was never installed. + drop(unsafe { Arc::from_raw(pointer as *const V) }); + None + } + Ok(None) => { + self.len.fetch_add(1, Ordering::Relaxed); + Some(()) + } + Err(_) => { + // SAFETY: insertion failed, so ownership never transferred. + drop(unsafe { Arc::from_raw(pointer as *const V) }); + Self::allocation_failure() + } + } + } + + #[inline] + fn remove_value(&self, key: &K) -> Option<(K, V)> { + let guard = self.inner.pin(); + let pointer = self.inner.remove(&key.into_congee(), &guard)?; + self.len.fetch_sub(1, Ordering::Relaxed); + Some((*key, Self::retire_old(pointer, &guard).as_ref().clone())) + } + + #[inline] + fn len(&self) -> usize { + self.len.load(Ordering::Relaxed) + } + + fn iter_values(&self) -> impl DoubleEndedIterator + '_ { + let mut values = self + .inner + .keys() + .into_iter() + .filter_map(|key| { + let key = K::from_congee(key); + self.get_value(&key).map(|value| (key, value)) + }) + .collect::>(); + values.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + values.into_iter() + } + + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.iter_values().filter(move |(key, _)| range.contains(key)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Barrier}; + + use super::{CongeeIndex, UniqueIndex}; + + #[test] + fn implements_unique_index_contract() { + let index = CongeeIndex::::default(); + assert_eq!(index.insert_value_checked(1, 10), Some(())); + assert_eq!(index.insert_value_checked(1, 11), None); + assert_eq!(index.get_value(&1), Some(10)); + assert_eq!(index.insert_value(1, 12), Some(10)); + assert_eq!(index.range_values(1..=1).collect::>(), vec![(1, 12)]); + assert_eq!(index.remove_value(&1), Some((1, 12))); + assert!(index.is_empty()); + } + + #[test] + fn checked_insert_has_one_winner_under_contention() { + let index = Arc::new(CongeeIndex::::default()); + let barrier = Arc::new(Barrier::new(9)); + let mut threads = Vec::new(); + + for value in 0..8 { + let index = Arc::clone(&index); + let barrier = Arc::clone(&barrier); + threads.push(std::thread::spawn(move || { + barrier.wait(); + index.insert_value_checked(7, value).is_some() + })); + } + + barrier.wait(); + let winners = threads + .into_iter() + .map(|thread| thread.join().unwrap()) + .filter(|won| *won) + .count(); + assert_eq!(winners, 1); + assert_eq!(index.len(), 1); + } +} diff --git a/src/index/mod.rs b/src/index/mod.rs index ca4af0b4..e961d875 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -1,4 +1,5 @@ mod available_index; +mod congee; mod multipair; mod primary_index; mod table_index; @@ -7,6 +8,7 @@ mod unique; mod unsized_node; pub use available_index::AvailableIndex; +pub use congee::{CongeeIndex, CongeeKey}; pub use indexset::concurrent::map::BTreeMap as IndexMap; pub use indexset::concurrent::multimap::BTreeMultiMap as IndexMultiMap; pub use multipair::MultiPairRecreate; diff --git a/src/index/unique.rs b/src/index/unique.rs index 6326dd1c..3e6ad37a 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -35,9 +35,9 @@ where fn iter_values(&self) -> impl DoubleEndedIterator + '_; - fn range_values(&self, range: R) -> impl DoubleEndedIterator + '_ + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where - R: RangeBounds; + R: RangeBounds + 'a; } impl UniqueIndex for IndexMap @@ -77,9 +77,9 @@ where } #[inline] - fn range_values(&self, range: R) -> impl DoubleEndedIterator + '_ + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where - R: RangeBounds, + R: RangeBounds + 'a, { self.range(range).map(|(key, value)| (key.clone(), value.clone())) } @@ -122,9 +122,9 @@ where } #[inline] - fn range_values(&self, range: R) -> impl DoubleEndedIterator + '_ + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where - R: RangeBounds, + R: RangeBounds + 'a, { self.range(range).map(|(key, value)| (key.clone(), value.clone())) } diff --git a/src/lib.rs b/src/lib.rs index d1a40e5c..ea61f690 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,10 +43,10 @@ pub mod prelude { pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; pub use crate::{ - AvailableIndex, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, PrimaryIndex, TableIndex, - TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, - TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, UpstreamIndexMap, WorkTable, WorkTableError, - vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum, + AvailableIndex, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, + PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, + TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, UpstreamIndexMap, WorkTable, + WorkTableError, vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum, }; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, From a7b83e93acb32fba5d6083eaa6ddc227339240be Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 07:41:39 +0700 Subject: [PATCH 04/23] feat: add arctic unique-index adapter --- Cargo.toml | 1 + src/index/arctic.rs | 148 ++++++++++++++++++++++++++++++++++++++++++++ src/index/mod.rs | 2 + src/lib.rs | 9 +-- 4 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 src/index/arctic.rs diff --git a/Cargo.toml b/Cargo.toml index 09184f48..5b0d76d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktabl [dependencies] async-trait = "0.1.89" +arctic-map = "=0.1.4" congee = "=0.4.1" convert_case = "0.6.0" data_bucket = "=0.4.1" diff --git a/src/index/arctic.rs b/src/index/arctic.rs new file mode 100644 index 00000000..a53ee84a --- /dev/null +++ b/src/index/arctic.rs @@ -0,0 +1,148 @@ +//! Arctic adapter for memory-only unique WorkTable indexes. + +use std::fmt::{self, Debug}; +use std::ops::RangeBounds; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arctic::{ConcurrentMap, Key, Order}; + +use super::UniqueIndex; + +/// Arctic's lock-free adaptive radix tree with WorkTable's unique-index +/// contract. +/// +/// WorkTable links are stored as boxed values because Arctic's inline value +/// representation is limited to 64 bits. Point operations remain directly +/// backed by Arctic; ordered scans are collected into a stable snapshot to +/// satisfy WorkTable's double-ended query interface. +pub struct ArcticIndex { + inner: ConcurrentMap>, + len: AtomicUsize, +} + +impl Debug for ArcticIndex { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ArcticIndex") + .field("len", &self.len.load(Ordering::Relaxed)) + .finish_non_exhaustive() + } +} + +impl Default for ArcticIndex +where + K: Key, +{ + fn default() -> Self { + Self { + inner: ConcurrentMap::default(), + len: AtomicUsize::new(0), + } + } +} + +impl UniqueIndex for ArcticIndex +where + K: Key + Clone + Debug + Ord + Send + Sync + 'static, + V: Clone + Debug + Send + Sync + 'static, +{ + #[inline] + fn get_value(&self, key: &K) -> Option { + self.inner.get(key.borrow()).map(|value| (*value).clone()) + } + + #[inline] + fn insert_value(&self, key: K, value: V) -> Option { + let updated = self.inner.upsert(key.as_insert(), Box::new(value)); + let old = updated.old().cloned(); + if old.is_none() { + self.len.fetch_add(1, Ordering::Relaxed); + } + old + } + + #[inline] + fn insert_value_checked(&self, key: K, value: V) -> Option<()> { + match self.inner.insert(key.as_insert(), Box::new(value)) { + Ok(_) => { + self.len.fetch_add(1, Ordering::Relaxed); + Some(()) + } + Err((_old, new)) => { + drop(new); + None + } + } + } + + #[inline] + fn remove_value(&self, key: &K) -> Option<(K, V)> { + let old = self.inner.remove(key.borrow())?; + self.len.fetch_sub(1, Ordering::Relaxed); + Some((key.clone(), (*old).clone())) + } + + #[inline] + fn len(&self) -> usize { + self.len.load(Ordering::Relaxed) + } + + fn iter_values(&self) -> impl DoubleEndedIterator + '_ { + let shard = self.inner.all(); + shard + .entries(Order::Ascend) + .map(|(key, value)| (key, value.clone())) + .collect::>() + .into_iter() + } + + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.iter_values().filter(move |(key, _)| range.contains(key)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Barrier}; + + use super::{ArcticIndex, UniqueIndex}; + + #[test] + fn implements_unique_index_contract() { + let index = ArcticIndex::::default(); + assert_eq!(index.insert_value_checked(1, 10), Some(())); + assert_eq!(index.insert_value_checked(1, 11), None); + assert_eq!(index.get_value(&1), Some(10)); + assert_eq!(index.insert_value(1, 12), Some(10)); + assert_eq!(index.range_values(1..=1).collect::>(), vec![(1, 12)]); + assert_eq!(index.remove_value(&1), Some((1, 12))); + assert!(index.is_empty()); + } + + #[test] + fn checked_insert_has_one_winner_under_contention() { + let index = Arc::new(ArcticIndex::::default()); + let barrier = Arc::new(Barrier::new(9)); + let mut threads = Vec::new(); + + for value in 0..8 { + let index = Arc::clone(&index); + let barrier = Arc::clone(&barrier); + threads.push(std::thread::spawn(move || { + barrier.wait(); + index.insert_value_checked(7, value).is_some() + })); + } + + barrier.wait(); + let winners = threads + .into_iter() + .map(|thread| thread.join().unwrap()) + .filter(|won| *won) + .count(); + assert_eq!(winners, 1); + assert_eq!(index.len(), 1); + } +} diff --git a/src/index/mod.rs b/src/index/mod.rs index e961d875..ea512e59 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -1,3 +1,4 @@ +mod arctic; mod available_index; mod congee; mod multipair; @@ -7,6 +8,7 @@ mod table_secondary_index; mod unique; mod unsized_node; +pub use arctic::ArcticIndex; pub use available_index::AvailableIndex; pub use congee::{CongeeIndex, CongeeKey}; pub use indexset::concurrent::map::BTreeMap as IndexMap; diff --git a/src/lib.rs b/src/lib.rs index ea61f690..56d8c9f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,10 +43,11 @@ pub mod prelude { pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; pub use crate::{ - AvailableIndex, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, MultiPairRecreate, - PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, - TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, UpstreamIndexMap, WorkTable, - WorkTableError, vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, vacuum::WorkTableVacuum, + ArcticIndex, AvailableIndex, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, + MultiPairRecreate, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, + TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, + UpstreamIndexMap, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, + vacuum::WorkTableVacuum, }; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, From 65b6c98f4157cbcd11209eddaed3778599cfbd39 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 08:23:56 +0700 Subject: [PATCH 05/23] feat: generate selected unique-index backends --- .../src/generators/in_memory/index/info.rs | 17 ++- codegen/src/generators/in_memory/index/mod.rs | 39 ++++-- .../src/generators/in_memory/index/usual.rs | 4 +- .../src/generators/in_memory/primary_key.rs | 14 +- .../generators/in_memory/queries/delete.rs | 10 +- .../generators/in_memory/queries/in_place.rs | 4 +- .../generators/in_memory/queries/select.rs | 4 +- .../generators/in_memory/queries/update.rs | 20 +-- .../src/generators/in_memory/table/impls.rs | 18 +-- .../generators/in_memory/table/index_fns.rs | 24 +++- codegen/src/generators/in_memory/table/mod.rs | 20 ++- codegen/src/generators/index_backend.rs | 120 ++++++++++++++++++ codegen/src/generators/mod.rs | 1 + codegen/src/generators/persist/index/mod.rs | 39 ++++-- codegen/src/generators/persist/index/usual.rs | 4 +- codegen/src/generators/persist/primary_key.rs | 14 +- .../src/generators/persist/queries/delete.rs | 10 +- .../generators/persist/queries/in_place.rs | 4 +- .../src/generators/persist/queries/select.rs | 4 +- .../src/generators/persist/queries/update.rs | 20 +-- codegen/src/generators/persist/table/impls.rs | 24 ++-- .../src/generators/persist/table/index_fns.rs | 24 +++- codegen/src/generators/persist/table/mod.rs | 20 ++- codegen/src/generators/read_only/index/mod.rs | 39 ++++-- .../src/generators/read_only/index/usual.rs | 4 +- .../src/generators/read_only/primary_key.rs | 14 +- .../generators/read_only/queries/select.rs | 4 +- .../src/generators/read_only/table/impls.rs | 22 ++-- .../generators/read_only/table/index_fns.rs | 24 +++- codegen/src/generators/read_only/table/mod.rs | 20 ++- codegen/src/worktable/mod.rs | 82 +++++++++++- src/index/arctic.rs | 59 ++++++++- src/index/congee.rs | 13 +- src/index/mod.rs | 6 +- src/index/primary_index.rs | 74 +++++------ src/index/table_index/cdc.rs | 59 ++++++++- src/index/table_index/mod.rs | 83 +++++++++++- src/index/table_index/util.rs | 65 ++++++++++ src/index/unique.rs | 49 ++++++- src/lib.rs | 8 +- src/mem_stat/mod.rs | 53 +++++++- src/table/mod.rs | 45 +++---- src/table/system_info.rs | 10 +- src/table/vacuum/vacuum.rs | 28 ++-- 44 files changed, 937 insertions(+), 282 deletions(-) create mode 100644 codegen/src/generators/index_backend.rs diff --git a/codegen/src/generators/in_memory/index/info.rs b/codegen/src/generators/in_memory/index/info.rs index eda1a254..1ce5113b 100644 --- a/codegen/src/generators/in_memory/index/info.rs +++ b/codegen/src/generators/in_memory/index/info.rs @@ -26,15 +26,28 @@ impl InMemoryGenerator { let index_name_str = index_field_name.to_string(); if idx.is_unique { + let (capacity, node_count) = match idx.backend { + crate::common::model::IndexBackend::WorktablesIndex + | crate::common::model::IndexBackend::Indexset => ( + quote! { self.#index_field_name.capacity() }, + quote! { self.#index_field_name.node_count() }, + ), + crate::common::model::IndexBackend::Congee | crate::common::model::IndexBackend::Arctic => ( + // Neither ART exposes allocator capacity or internal + // node counts through its stable public API. + quote! { self.#index_field_name.len() }, + quote! { 0 }, + ), + }; quote! { info.push(IndexInfo { name: #index_name_str.to_string(), index_type: IndexKind::Unique, key_count: self.#index_field_name.len(), - capacity: self.#index_field_name.capacity(), + capacity: #capacity, heap_size: self.#index_field_name.heap_size(), used_size: self.#index_field_name.used_size(), - node_count: self.#index_field_name.node_count(), + node_count: #node_count, }); } } else { diff --git a/codegen/src/generators/in_memory/index/mod.rs b/codegen/src/generators/in_memory/index/mod.rs index 7d67de24..d912985f 100644 --- a/codegen/src/generators/in_memory/index/mod.rs +++ b/codegen/src/generators/in_memory/index/mod.rs @@ -4,6 +4,7 @@ mod usual; use crate::common::name_generator::{WorktableNameGenerator, is_float, is_unsized}; use crate::generators::in_memory::InMemoryGenerator; +use crate::generators::index_backend::unique_index_type; use convert_case::{Case, Casing}; use proc_macro2::TokenStream; use quote::quote; @@ -58,13 +59,14 @@ impl InMemoryGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { - if is_unsized(&t.to_string()) { - quote! { - #i: IndexMap<#t, OffsetEqLink, UnsizedNode>> - } + let value_type = quote! { OffsetEqLink }; + let worktables_node = if is_unsized(&t.to_string()) { + Some(quote! { UnsizedNode> }) } else { - quote! {#i: IndexMap<#t, OffsetEqLink>} - } + None + }; + let index_type = unique_index_type(idx.backend, &t, &value_type, worktables_node)?; + quote! { #i: #index_type } } else { if is_unsized(&t.to_string()) { quote! {#i: IndexMultiMap<#t, OffsetEqLink, UnsizedNode>>} @@ -126,12 +128,27 @@ impl InMemoryGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { - if is_unsized(&t.to_string()) { - quote! { - #i: IndexMap::with_maximum_node_size(#const_name), + match idx.backend { + crate::common::model::IndexBackend::WorktablesIndex => { + if is_unsized(&t.to_string()) { + quote! { #i: IndexMap::with_maximum_node_size(#const_name), } + } else { + quote! { + #i: IndexMap::with_maximum_node_size( + get_index_page_size_from_data_length::<#t>(#const_name) + ), + } + } + } + crate::common::model::IndexBackend::Indexset => quote! { + #i: UpstreamIndexMap::with_maximum_node_size( + get_index_page_size_from_data_length::<#t>(#const_name) + ), + }, + crate::common::model::IndexBackend::Congee + | crate::common::model::IndexBackend::Arctic => { + quote! { #i: Default::default(), } } - } else { - quote! {#i: IndexMap::with_maximum_node_size(get_index_page_size_from_data_length::<#t>(#const_name)),} } } else { if is_unsized(&t.to_string()) { diff --git a/codegen/src/generators/in_memory/index/usual.rs b/codegen/src/generators/in_memory/index/usual.rs index 69ed4bb1..2c90abc7 100644 --- a/codegen/src/generators/in_memory/index/usual.rs +++ b/codegen/src/generators/in_memory/index/usual.rs @@ -63,7 +63,7 @@ impl InMemoryGenerator { } }; quote! { - if self.#index_field_name.insert_checked(#row.clone(), link).is_none() { + if TableIndex::insert_checked(&self.#index_field_name, #row.clone(), link).is_none() { return Err(IndexError::AlreadyExists { at: #available_index_ident::#index_variant, inserted_already: inserted_indexes.clone(), @@ -129,7 +129,7 @@ impl InMemoryGenerator { let row = &row_old; let val_old = #row.clone(); if val_new != val_old { - if self.#index_field_name.insert_checked(val_new.clone(), link_new).is_none() { + if TableIndex::insert_checked(&self.#index_field_name, val_new.clone(), link_new).is_none() { return Err(IndexError::AlreadyExists { at: #available_index_ident::#index_variant, inserted_already: inserted_indexes.clone(), diff --git a/codegen/src/generators/in_memory/primary_key.rs b/codegen/src/generators/in_memory/primary_key.rs index 1789a6ba..31de463a 100644 --- a/codegen/src/generators/in_memory/primary_key.rs +++ b/codegen/src/generators/in_memory/primary_key.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use crate::common::model::{GeneratorType, PrimaryKey}; use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec}; use crate::generators::in_memory::InMemoryGenerator; +use crate::generators::index_backend::primary_key_backend_impl; use proc_macro2::{Ident, TokenStream}; use quote::quote; @@ -28,7 +29,7 @@ impl InMemoryGenerator { }) .collect::>(); - let def = self.gen_primary_key_type(); + let def = self.gen_primary_key_type()?; let impl_ = self.gen_table_primary_key_impl()?; self.pk = Some(PrimaryKey { ident, values }); @@ -41,7 +42,7 @@ impl InMemoryGenerator { /// Generates table's primary key struct definition. It's newtype for type that was chosen as primary key column in /// definition. - fn gen_primary_key_type(&self) -> TokenStream { + fn gen_primary_key_type(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let ident = name_generator.get_primary_key_type_ident(); @@ -64,10 +65,13 @@ impl InMemoryGenerator { } else { quote! {} }; + let (backend_derive, backend_impl) = + primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; - quote! { + Ok(quote! { #[derive( Clone, + #backend_derive rkyv::Archive, Debug, Default, @@ -86,7 +90,9 @@ impl InMemoryGenerator { )] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] pub struct #ident(#(#types),*); - } + + #backend_impl + }) } /// Generates `TablePrimaryKey` trait implementation for primary key. It depends on generator type. diff --git a/codegen/src/generators/in_memory/queries/delete.rs b/codegen/src/generators/in_memory/queries/delete.rs index 9f08eb41..fad4375f 100644 --- a/codegen/src/generators/in_memory/queries/delete.rs +++ b/codegen/src/generators/in_memory/queries/delete.rs @@ -110,8 +110,8 @@ impl InMemoryGenerator { let link = match self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound) { Ok(l) => l, Err(e) => { @@ -126,8 +126,8 @@ impl InMemoryGenerator { let link = self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let row = self.0.select(pk.clone()).unwrap(); #process @@ -253,7 +253,7 @@ impl InMemoryGenerator { }; quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { - let row_to_update = self.0.indexes.#index.get(#by).map(|v| v.get().value.into()); + let row_to_update = self.0.indexes.#index.get_value(#by).map(Into::into); if let Some(link) = row_to_update { let row = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; self.delete(row.get_primary_key()).await?; diff --git a/codegen/src/generators/in_memory/queries/in_place.rs b/codegen/src/generators/in_memory/queries/in_place.rs index 2fcbbb83..0ed20b22 100644 --- a/codegen/src/generators/in_memory/queries/in_place.rs +++ b/codegen/src/generators/in_memory/queries/in_place.rs @@ -114,8 +114,8 @@ impl InMemoryGenerator { let link = self .0 .primary_index.pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; unsafe { self.0 diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index a86ddea1..e5999881 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -30,8 +30,8 @@ impl InMemoryGenerator { #row_fields_ident> { let iter = self.0.primary_index.pk_map - .iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .iter_links() + .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/in_memory/queries/update.rs b/codegen/src/generators/in_memory/queries/update.rs index 15503b92..8b2ecc45 100644 --- a/codegen/src/generators/in_memory/queries/update.rs +++ b/codegen/src/generators/in_memory/queries/update.rs @@ -92,8 +92,8 @@ impl InMemoryGenerator { let mut link: Link = self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let row_old = self.0.data.select_non_ghosted(link)?; @@ -461,8 +461,8 @@ impl InMemoryGenerator { let mut link: Link = self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let mut bytes = rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; @@ -618,8 +618,8 @@ impl InMemoryGenerator { // matched range before their lock was acquired are // skipped; rows that joined the range after the snapshot // are not touched. - let link: Link = match self.0.primary_index.pk_map.get(&pk) { - Some(v) => v.get().value.into(), + let link: Link = match self.0.primary_index.pk_map.get_value(&pk) { + Some(v) => v.into(), None => continue, }; if self.0.data.select_non_ghosted(link)?.#by_field != by { @@ -705,8 +705,8 @@ impl InMemoryGenerator { let mut link: Link = self.0.indexes .#index - .get(#by) - .map(|v| v.get().value.into()) + .get_value(#by) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let pk = self.0.data.select_non_ghosted(link)?.get_primary_key().clone(); @@ -720,8 +720,8 @@ impl InMemoryGenerator { let link = loop { let link = self.0.indexes.#index - .get(#by) - .map(|v| v.get().value.into()) + .get_value(#by) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; if let Err(e) = self.0.data.select_non_vacuumed(link) { diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index b647c68c..fa69a2cf 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -97,22 +97,22 @@ impl InMemoryGenerator { }; quote! { - pub fn select_by_pk_range(&self, range: R) -> SelectQueryBuilder<#row_type, - impl DoubleEndedIterator + '_, + pub fn select_by_pk_range<'a, R, Pk>(&'a self, range: R) -> SelectQueryBuilder<#row_type, + impl DoubleEndedIterator + 'a, #column_range_type, #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds, - Pk: Clone, + R: std::ops::RangeBounds + 'a, + Pk: Clone + 'a, { let converted_range = ( range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); let rows = self.0.primary_index.pk_map - .range(converted_range) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .range_links(converted_range) + .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); #pk_sorted_by } @@ -165,7 +165,7 @@ impl InMemoryGenerator { pub async fn upsert(&self, row: #row_type) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); loop { - let need_to_update = self.0.primary_index.pk_map.get(&pk).is_some(); + let need_to_update = self.0.primary_index.pk_map.get_value(&pk).is_some(); if need_to_update { match self.update(row.clone()).await { core::result::Result::Ok(_) => return core::result::Result::Ok(()), @@ -262,7 +262,7 @@ impl InMemoryGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { - let first = self.0.primary_index.pk_map.iter().next().map(|(k, v)| (k.clone(), v.0)); + let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k.clone(), v.0)); let Some((mut k, link)) = first else { return Ok(()) }; @@ -273,7 +273,7 @@ impl InMemoryGenerator { let mut ind = false; while !ind { let next = { - let mut iter = self.0.primary_index.pk_map.range(k.clone()..); + let mut iter = self.0.primary_index.pk_map.range_values(k.clone()..); let next = iter.next().map(|(k, v)| (k.clone(), v.0)).filter(|(key, _)| key != &k); if next.is_some() { next diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index 0545d63b..c71448d5 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -75,7 +75,7 @@ impl InMemoryGenerator { Ok(quote! { pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { - let link: Link = self.0.indexes.#field_ident.get(#by).map(|kv| kv.get().value.into())?; + let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; self.0.data.select_non_ghosted(link).ok() } }) @@ -146,18 +146,28 @@ impl InMemoryGenerator { } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; + let (index_range, select_row) = if idx.is_unique { + ( + quote! { self.0.indexes.#field_ident.range_links(#range_arg) }, + quote! { |link| self.0.data.select_non_ghosted(link.0).ok() }, + ) + } else { + ( + quote! { self.0.indexes.#field_ident.range(#range_arg) }, + quote! { |(_, link)| self.0.data.select_non_ghosted(link.0).ok() }, + ) + }; Ok(quote! { - pub fn #fn_name(&self, range: R) -> SelectQueryBuilder<#row_ident, - impl DoubleEndedIterator + '_, + pub fn #fn_name<'a, R>(&'a self, range: R) -> SelectQueryBuilder<#row_ident, + impl DoubleEndedIterator + 'a, #column_range_type, #row_fields_ident> where - R: #range_bounds + R: #range_bounds + 'a { - let rows = self.0.indexes.#field_ident - .range(#range_arg) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + let rows = #index_range + .filter_map(#select_row); SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/in_memory/table/mod.rs b/codegen/src/generators/in_memory/table/mod.rs index ac013e4a..cd75ec87 100644 --- a/codegen/src/generators/in_memory/table/mod.rs +++ b/codegen/src/generators/in_memory/table/mod.rs @@ -3,6 +3,7 @@ use quote::quote; use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec}; use crate::generators::in_memory::InMemoryGenerator; +use crate::generators::index_backend::unique_index_type; mod impls; mod index_fns; @@ -74,15 +75,22 @@ impl InMemoryGenerator { #[derive(Debug)] }; - let node_type = if pk_types_unsized { - quote! { + let key_type = quote! { #primary_key_type }; + let value_type = quote! { OffsetEqLink<#inner_const_name> }; + let worktables_node = if pk_types_unsized { + Some(quote! { UnsizedNode>> - } + }) } else { - quote! { - Vec>> - } + None }; + let node_type = unique_index_type( + self.columns.primary_index_backend, + &key_type, + &value_type, + worktables_node, + ) + .unwrap_or_else(|error| error.into_compile_error()); if self.config.as_ref().and_then(|c| c.page_size).is_some() { quote! { diff --git a/codegen/src/generators/index_backend.rs b/codegen/src/generators/index_backend.rs new file mode 100644 index 00000000..0cf6f7d3 --- /dev/null +++ b/codegen/src/generators/index_backend.rs @@ -0,0 +1,120 @@ +use proc_macro2::{Ident, TokenStream}; +use quote::quote; + +use crate::common::model::IndexBackend; + +/// Generates the concrete unique-map type selected by the DSL while keeping +/// WorkTablesIndex's custom node type available for persisted/unsized indexes. +pub(crate) fn unique_index_type( + backend: IndexBackend, + key: &TokenStream, + value: &TokenStream, + worktables_node: Option, +) -> syn::Result { + match backend { + IndexBackend::WorktablesIndex => Ok(match worktables_node { + Some(node) => quote! { IndexMap<#key, #value, #node> }, + None => quote! { IndexMap<#key, #value> }, + }), + IndexBackend::Indexset => { + if worktables_node.is_some() { + Err(syn::Error::new_spanned( + key, + "`using indexset` does not yet support variable-sized keys; use `worktables_index` for this index", + )) + } else { + Ok(quote! { UpstreamIndexMap<#key, #value> }) + } + } + IndexBackend::Congee => Ok(quote! { CongeeIndex<#key, #value> }), + IndexBackend::Arctic => Ok(quote! { ArcticIndex<#key, #value> }), + } +} + +/// Generates the small codec needed when an ART is used for WorkTable's +/// generated primary-key newtype. ART backends intentionally accept only the +/// lossless, native integer shapes supported by their public APIs. +pub(crate) fn primary_key_backend_impl( + backend: IndexBackend, + primary_key: &Ident, + fields: &[&TokenStream], +) -> syn::Result<(TokenStream, TokenStream)> { + match backend { + IndexBackend::WorktablesIndex | IndexBackend::Indexset => Ok((quote! {}, quote! {})), + IndexBackend::Congee => { + let field = single_supported_field(backend, fields, &["u8", "u16", "u32", "u64", "usize"])?; + let width_guard = if field.to_string() == "u64" { + quote! { + #[cfg(not(target_pointer_width = "64"))] + compile_error!("`using congee` with a `u64` primary key requires a 64-bit target"); + } + } else { + quote! {} + }; + + Ok(( + quote! { Copy, }, + quote! { + #width_guard + impl CongeeKey for #primary_key { + #[inline] + fn into_congee(self) -> usize { + self.0 as usize + } + + #[inline] + fn from_congee(value: usize) -> Self { + Self(value as #field) + } + } + }, + )) + } + IndexBackend::Arctic => { + let field = single_supported_field(backend, fields, &["u16", "u32", "u64", "u128"])?; + Ok(( + quote! { Copy, }, + quote! { + impl ArcticKey for #primary_key { + type Raw = #field; + + #[inline] + fn to_arctic(&self) -> Self::Raw { + self.0 + } + + #[inline] + fn from_arctic(value: Self::Raw) -> Self { + Self(value) + } + } + }, + )) + } + } +} + +fn single_supported_field<'a>( + backend: IndexBackend, + fields: &'a [&TokenStream], + supported: &[&str], +) -> syn::Result<&'a TokenStream> { + let [field] = fields else { + return Err(syn::Error::new_spanned( + fields.first().copied().cloned().unwrap_or_default(), + format!("`using {}` requires a single-column primary key", backend.name()), + )); + }; + if !supported.contains(&field.to_string().as_str()) { + return Err(syn::Error::new_spanned( + *field, + format!( + "`using {}` does not support primary-key type `{}`; supported types: {}", + backend.name(), + field, + supported.join(", ") + ), + )); + } + Ok(field) +} diff --git a/codegen/src/generators/mod.rs b/codegen/src/generators/mod.rs index 22a1b0db..6eed7863 100644 --- a/codegen/src/generators/mod.rs +++ b/codegen/src/generators/mod.rs @@ -1,3 +1,4 @@ pub mod in_memory; +pub(crate) mod index_backend; pub mod persist; pub mod read_only; diff --git a/codegen/src/generators/persist/index/mod.rs b/codegen/src/generators/persist/index/mod.rs index 6fdaa67a..0b86b239 100644 --- a/codegen/src/generators/persist/index/mod.rs +++ b/codegen/src/generators/persist/index/mod.rs @@ -3,6 +3,7 @@ mod info; mod usual; use crate::common::name_generator::{WorktableNameGenerator, is_float, is_unsized}; +use crate::generators::index_backend::unique_index_type; use crate::generators::persist::PersistGenerator; use convert_case::{Case, Casing}; use proc_macro2::TokenStream; @@ -50,13 +51,14 @@ impl PersistGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { - if is_unsized(&t.to_string()) { - quote! { - #i: IndexMap<#t, OffsetEqLink, UnsizedNode>> - } + let value_type = quote! { OffsetEqLink }; + let worktables_node = if is_unsized(&t.to_string()) { + Some(quote! { UnsizedNode> }) } else { - quote! {#i: IndexMap<#t, OffsetEqLink>} - } + None + }; + let index_type = unique_index_type(idx.backend, &t, &value_type, worktables_node)?; + quote! { #i: #index_type } } else { if is_unsized(&t.to_string()) { quote! {#i: IndexMultiMap<#t, OffsetEqLink, UnsizedNode>>} @@ -105,12 +107,27 @@ impl PersistGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { - if is_unsized(&t.to_string()) { - quote! { - #i: IndexMap::with_maximum_node_size(#const_name), + match idx.backend { + crate::common::model::IndexBackend::WorktablesIndex => { + if is_unsized(&t.to_string()) { + quote! { #i: IndexMap::with_maximum_node_size(#const_name), } + } else { + quote! { + #i: IndexMap::with_maximum_node_size( + get_index_page_size_from_data_length::<#t>(#const_name) + ), + } + } + } + crate::common::model::IndexBackend::Indexset => quote! { + #i: UpstreamIndexMap::with_maximum_node_size( + get_index_page_size_from_data_length::<#t>(#const_name) + ), + }, + crate::common::model::IndexBackend::Congee + | crate::common::model::IndexBackend::Arctic => { + quote! { #i: Default::default(), } } - } else { - quote! {#i: IndexMap::with_maximum_node_size(get_index_page_size_from_data_length::<#t>(#const_name)),} } } else { if is_unsized(&t.to_string()) { diff --git a/codegen/src/generators/persist/index/usual.rs b/codegen/src/generators/persist/index/usual.rs index fb006be1..e8629cc8 100644 --- a/codegen/src/generators/persist/index/usual.rs +++ b/codegen/src/generators/persist/index/usual.rs @@ -59,7 +59,7 @@ impl PersistGenerator { } }; quote! { - if self.#index_field_name.insert_checked(#row.clone(), link).is_none() { + if TableIndex::insert_checked(&self.#index_field_name, #row.clone(), link).is_none() { return Err(IndexError::AlreadyExists { at: #available_index_ident::#index_variant, inserted_already: inserted_indexes.clone(), @@ -125,7 +125,7 @@ impl PersistGenerator { let row = &row_old; let val_old = #row.clone(); if val_new != val_old { - if self.#index_field_name.insert_checked(val_new.clone(), link_new).is_none() { + if TableIndex::insert_checked(&self.#index_field_name, val_new.clone(), link_new).is_none() { return Err(IndexError::AlreadyExists { at: #available_index_ident::#index_variant, inserted_already: inserted_indexes.clone(), diff --git a/codegen/src/generators/persist/primary_key.rs b/codegen/src/generators/persist/primary_key.rs index 82b59864..68622fcf 100644 --- a/codegen/src/generators/persist/primary_key.rs +++ b/codegen/src/generators/persist/primary_key.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use crate::common::model::{GeneratorType, PrimaryKey}; use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec}; +use crate::generators::index_backend::primary_key_backend_impl; use crate::generators::persist::PersistGenerator; use proc_macro2::{Ident, TokenStream}; @@ -27,7 +28,7 @@ impl PersistGenerator { }) .collect::>(); - let def = self.gen_primary_key_type(); + let def = self.gen_primary_key_type()?; let impl_ = self.gen_table_primary_key_impl()?; self.pk = Some(PrimaryKey { ident, values }); @@ -38,7 +39,7 @@ impl PersistGenerator { }) } - fn gen_primary_key_type(&self) -> TokenStream { + fn gen_primary_key_type(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let ident = name_generator.get_primary_key_type_ident(); @@ -60,10 +61,13 @@ impl PersistGenerator { } else { quote! {} }; + let (backend_derive, backend_impl) = + primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; - quote! { + Ok(quote! { #[derive( Clone, + #backend_derive rkyv::Archive, Debug, Default, @@ -82,7 +86,9 @@ impl PersistGenerator { )] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] pub struct #ident(#(#types),*); - } + + #backend_impl + }) } fn gen_table_primary_key_impl(&self) -> syn::Result { diff --git a/codegen/src/generators/persist/queries/delete.rs b/codegen/src/generators/persist/queries/delete.rs index 1482d3db..e7fd6b58 100644 --- a/codegen/src/generators/persist/queries/delete.rs +++ b/codegen/src/generators/persist/queries/delete.rs @@ -103,8 +103,8 @@ impl PersistGenerator { let link = match self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound) { Ok(l) => l, Err(e) => { @@ -119,8 +119,8 @@ impl PersistGenerator { let link = self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let row = self.0.select(pk.clone()).unwrap(); #process @@ -246,7 +246,7 @@ impl PersistGenerator { }; quote! { pub async fn #name(&self, by: #type_) -> core::result::Result<(), WorkTableError> { - let row_to_update = self.0.indexes.#index.get(#by).map(|v| v.get().value.into()); + let row_to_update = self.0.indexes.#index.get_value(#by).map(Into::into); if let Some(link) = row_to_update { let row = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; self.delete(row.get_primary_key()).await?; diff --git a/codegen/src/generators/persist/queries/in_place.rs b/codegen/src/generators/persist/queries/in_place.rs index e8c4c8c1..6de2d822 100644 --- a/codegen/src/generators/persist/queries/in_place.rs +++ b/codegen/src/generators/persist/queries/in_place.rs @@ -116,8 +116,8 @@ impl PersistGenerator { let link = self .0 .primary_index.pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; unsafe { self.0 diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 5db8e71c..57d1e6bc 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -30,8 +30,8 @@ impl PersistGenerator { #row_fields_ident> { let iter = self.0.primary_index.pk_map - .iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .iter_links() + .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index 99149ff6..368f688e 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -92,8 +92,8 @@ impl PersistGenerator { let mut link: Link = self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let row_old = self.0.data.select_non_ghosted(link)?; @@ -418,8 +418,8 @@ impl PersistGenerator { let mut link: Link = self.0 .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let mut bytes = rkyv::to_bytes::(&row).map_err(|_| WorkTableError::SerializeError)?; @@ -575,8 +575,8 @@ impl PersistGenerator { // matched range before their lock was acquired are // skipped; rows that joined the range after the snapshot // are not touched. - let link: Link = match self.0.primary_index.pk_map.get(&pk) { - Some(v) => v.get().value.into(), + let link: Link = match self.0.primary_index.pk_map.get_value(&pk) { + Some(v) => v.into(), None => continue, }; if self.0.data.select_non_ghosted(link)?.#by_field != by { @@ -662,8 +662,8 @@ impl PersistGenerator { let mut link: Link = self.0.indexes .#index - .get(#by) - .map(|v| v.get().value.into()) + .get_value(#by) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let pk = self.0.data.select_non_ghosted(link)?.get_primary_key().clone(); @@ -677,8 +677,8 @@ impl PersistGenerator { let link = loop { let link = self.0.indexes.#index - .get(#by) - .map(|v| v.get().value.into()) + .get_value(#by) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; if let Err(e) = self.0.data.select_non_vacuumed(link) { diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index eafdc3ac..b1587ad6 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -72,6 +72,10 @@ impl PersistGenerator { }) .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); + let pk_map = match self.columns.primary_index_backend { + crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap }, + _ => quote! { IndexMap }, + }; let index_setup = if pk_types_unsized { quote! { inner.primary_index = std::sync::Arc::new(PrimaryIndex { @@ -83,7 +87,7 @@ impl PersistGenerator { quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: IndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), + pk_map: #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), reverse_pk_map: IndexMap::new(), }); } @@ -183,22 +187,22 @@ impl PersistGenerator { }; quote! { - pub fn select_by_pk_range(&self, range: R) -> SelectQueryBuilder<#row_type, - impl DoubleEndedIterator + '_, + pub fn select_by_pk_range<'a, R, Pk>(&'a self, range: R) -> SelectQueryBuilder<#row_type, + impl DoubleEndedIterator + 'a, #column_range_type, #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds, - Pk: Clone, + R: std::ops::RangeBounds + 'a, + Pk: Clone + 'a, { let converted_range = ( range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); let rows = self.0.primary_index.pk_map - .range(converted_range) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .range_links(converted_range) + .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); #pk_sorted_by } @@ -261,7 +265,7 @@ impl PersistGenerator { pub async fn upsert(&self, row: #row_type) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); loop { - let need_to_update = self.0.primary_index.pk_map.get(&pk).is_some(); + let need_to_update = self.0.primary_index.pk_map.get_value(&pk).is_some(); if need_to_update { match self.update(row.clone()).await { core::result::Result::Ok(_) => return core::result::Result::Ok(()), @@ -369,7 +373,7 @@ impl PersistGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { - let first = self.0.primary_index.pk_map.iter().next().map(|(k, v)| (k.clone(), v.0)); + let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k.clone(), v.0)); let Some((mut k, link)) = first else { return Ok(()) }; @@ -380,7 +384,7 @@ impl PersistGenerator { let mut ind = false; while !ind { let next = { - let mut iter = self.0.primary_index.pk_map.range(k.clone()..); + let mut iter = self.0.primary_index.pk_map.range_values(k.clone()..); let next = iter.next().map(|(k, v)| (k.clone(), v.0)).filter(|(key, _)| key != &k); if next.is_some() { next diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index d682a6ac..6581e779 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -75,7 +75,7 @@ impl PersistGenerator { Ok(quote! { pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { - let link: Link = self.0.indexes.#field_ident.get(#by).map(|kv| kv.get().value.into())?; + let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; self.0.data.select_non_ghosted(link).ok() } }) @@ -146,18 +146,28 @@ impl PersistGenerator { } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; + let (index_range, select_row) = if idx.is_unique { + ( + quote! { self.0.indexes.#field_ident.range_links(#range_arg) }, + quote! { |link| self.0.data.select_non_ghosted(link.0).ok() }, + ) + } else { + ( + quote! { self.0.indexes.#field_ident.range(#range_arg) }, + quote! { |(_, link)| self.0.data.select_non_ghosted(link.0).ok() }, + ) + }; Ok(quote! { - pub fn #fn_name(&self, range: R) -> SelectQueryBuilder<#row_ident, - impl DoubleEndedIterator + '_, + pub fn #fn_name<'a, R>(&'a self, range: R) -> SelectQueryBuilder<#row_ident, + impl DoubleEndedIterator + 'a, #column_range_type, #row_fields_ident> where - R: #range_bounds + R: #range_bounds + 'a { - let rows = self.0.indexes.#field_ident - .range(#range_arg) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + let rows = #index_range + .filter_map(#select_row); SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/persist/table/mod.rs b/codegen/src/generators/persist/table/mod.rs index 51f3658e..f35e2693 100644 --- a/codegen/src/generators/persist/table/mod.rs +++ b/codegen/src/generators/persist/table/mod.rs @@ -6,6 +6,7 @@ use proc_macro2::{Literal, TokenStream}; use quote::quote; use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec}; +use crate::generators::index_backend::unique_index_type; use crate::generators::persist::PersistGenerator; impl PersistGenerator { @@ -95,15 +96,22 @@ impl PersistGenerator { } }; - let node_type = if pk_types_unsized { - quote! { + let key_type = quote! { #primary_key_type }; + let value_type = quote! { OffsetEqLink<#inner_const_name> }; + let worktables_node = if pk_types_unsized { + Some(quote! { UnsizedNode>> - } + }) } else { - quote! { - Vec>> - } + None }; + let node_type = unique_index_type( + self.columns.primary_index_backend, + &key_type, + &value_type, + worktables_node, + ) + .unwrap_or_else(|error| error.into_compile_error()); if self.config.as_ref().and_then(|c| c.page_size).is_some() { quote! { diff --git a/codegen/src/generators/read_only/index/mod.rs b/codegen/src/generators/read_only/index/mod.rs index 5aab17ba..048c679e 100644 --- a/codegen/src/generators/read_only/index/mod.rs +++ b/codegen/src/generators/read_only/index/mod.rs @@ -2,6 +2,7 @@ mod info; mod usual; use crate::common::name_generator::{WorktableNameGenerator, is_float, is_unsized}; +use crate::generators::index_backend::unique_index_type; use crate::generators::read_only::ReadOnlyGenerator; use convert_case::{Case, Casing}; use proc_macro2::TokenStream; @@ -47,13 +48,14 @@ impl ReadOnlyGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { - if is_unsized(&t.to_string()) { - quote! { - #i: IndexMap<#t, OffsetEqLink, UnsizedNode>> - } + let value_type = quote! { OffsetEqLink }; + let worktables_node = if is_unsized(&t.to_string()) { + Some(quote! { UnsizedNode> }) } else { - quote! {#i: IndexMap<#t, OffsetEqLink>} - } + None + }; + let index_type = unique_index_type(idx.backend, &t, &value_type, worktables_node)?; + quote! { #i: #index_type } } else { if is_unsized(&t.to_string()) { quote! {#i: IndexMultiMap<#t, OffsetEqLink, UnsizedNode>>} @@ -103,12 +105,27 @@ impl ReadOnlyGenerator { #[allow(clippy::collapsible_else_if)] let res = if idx.is_unique { - if is_unsized(&t.to_string()) { - quote! { - #i: IndexMap::with_maximum_node_size(#const_name), + match idx.backend { + crate::common::model::IndexBackend::WorktablesIndex => { + if is_unsized(&t.to_string()) { + quote! { #i: IndexMap::with_maximum_node_size(#const_name), } + } else { + quote! { + #i: IndexMap::with_maximum_node_size( + get_index_page_size_from_data_length::<#t>(#const_name) + ), + } + } + } + crate::common::model::IndexBackend::Indexset => quote! { + #i: UpstreamIndexMap::with_maximum_node_size( + get_index_page_size_from_data_length::<#t>(#const_name) + ), + }, + crate::common::model::IndexBackend::Congee + | crate::common::model::IndexBackend::Arctic => { + quote! { #i: Default::default(), } } - } else { - quote! {#i: IndexMap::with_maximum_node_size(get_index_page_size_from_data_length::<#t>(#const_name)),} } } else { if is_unsized(&t.to_string()) { diff --git a/codegen/src/generators/read_only/index/usual.rs b/codegen/src/generators/read_only/index/usual.rs index f0cf34d5..0af30608 100644 --- a/codegen/src/generators/read_only/index/usual.rs +++ b/codegen/src/generators/read_only/index/usual.rs @@ -59,7 +59,7 @@ impl ReadOnlyGenerator { } }; quote! { - if self.#index_field_name.insert_checked(#row.clone(), link).is_none() { + if TableIndex::insert_checked(&self.#index_field_name, #row.clone(), link).is_none() { return Err(IndexError::AlreadyExists { at: #available_index_ident::#index_variant, inserted_already: inserted_indexes.clone(), @@ -125,7 +125,7 @@ impl ReadOnlyGenerator { let row = &row_old; let val_old = #row.clone(); if val_new != val_old { - if self.#index_field_name.insert_checked(val_new.clone(), link_new).is_none() { + if TableIndex::insert_checked(&self.#index_field_name, val_new.clone(), link_new).is_none() { return Err(IndexError::AlreadyExists { at: #available_index_ident::#index_variant, inserted_already: inserted_indexes.clone(), diff --git a/codegen/src/generators/read_only/primary_key.rs b/codegen/src/generators/read_only/primary_key.rs index 1e02ea3e..99a89aa6 100644 --- a/codegen/src/generators/read_only/primary_key.rs +++ b/codegen/src/generators/read_only/primary_key.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use crate::common::model::{GeneratorType, PrimaryKey}; use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec}; +use crate::generators::index_backend::primary_key_backend_impl; use crate::generators::read_only::ReadOnlyGenerator; use proc_macro2::{Ident, TokenStream}; @@ -27,7 +28,7 @@ impl ReadOnlyGenerator { }) .collect::>(); - let def = self.gen_primary_key_type(); + let def = self.gen_primary_key_type()?; let impl_ = self.gen_table_primary_key_impl()?; self.pk = Some(PrimaryKey { ident, values }); @@ -38,7 +39,7 @@ impl ReadOnlyGenerator { }) } - fn gen_primary_key_type(&self) -> TokenStream { + fn gen_primary_key_type(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let ident = name_generator.get_primary_key_type_ident(); @@ -60,10 +61,13 @@ impl ReadOnlyGenerator { } else { quote! {} }; + let (backend_derive, backend_impl) = + primary_key_backend_impl(self.columns.primary_index_backend, &ident, types)?; - quote! { + Ok(quote! { #[derive( Clone, + #backend_derive rkyv::Archive, Debug, Default, @@ -82,7 +86,9 @@ impl ReadOnlyGenerator { )] #[rkyv(derive(PartialEq, Eq, PartialOrd, Ord, Debug))] pub struct #ident(#(#types),*); - } + + #backend_impl + }) } fn gen_table_primary_key_impl(&self) -> syn::Result { diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 9e700ad1..92ec583f 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -30,8 +30,8 @@ impl ReadOnlyGenerator { #row_fields_ident> { let iter = self.0.primary_index.pk_map - .iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .iter_links() + .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index c1ffd3a1..3b6f8b3a 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -70,6 +70,10 @@ impl ReadOnlyGenerator { }) .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); + let pk_map = match self.columns.primary_index_backend { + crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap }, + _ => quote! { IndexMap }, + }; let index_setup = if pk_types_unsized { quote! { @@ -82,7 +86,7 @@ impl ReadOnlyGenerator { quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); inner.primary_index = std::sync::Arc::new(PrimaryIndex { - pk_map: IndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), + pk_map: #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size), reverse_pk_map: IndexMap::new(), }); } @@ -179,22 +183,22 @@ impl ReadOnlyGenerator { }; quote! { - pub fn select_by_pk_range(&self, range: R) -> SelectQueryBuilder<#row_type, - impl DoubleEndedIterator + '_, + pub fn select_by_pk_range<'a, R, Pk>(&'a self, range: R) -> SelectQueryBuilder<#row_type, + impl DoubleEndedIterator + 'a, #column_range_type, #row_fields_ident> where #primary_key_type: From, - R: std::ops::RangeBounds, - Pk: Clone, + R: std::ops::RangeBounds + 'a, + Pk: Clone + 'a, { let converted_range = ( range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); let rows = self.0.primary_index.pk_map - .range(converted_range) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + .range_links(converted_range) + .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); #pk_sorted_by } @@ -277,7 +281,7 @@ impl ReadOnlyGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { - let first = self.0.primary_index.pk_map.iter().next().map(|(k, v)| (k.clone(), v.0)); + let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k.clone(), v.0)); let Some((mut k, link)) = first else { return Ok(()) }; @@ -288,7 +292,7 @@ impl ReadOnlyGenerator { let mut ind = false; while !ind { let next = { - let mut iter = self.0.primary_index.pk_map.range(k.clone()..); + let mut iter = self.0.primary_index.pk_map.range_values(k.clone()..); let next = iter.next().map(|(k, v)| (k.clone(), v.0)).filter(|(key, _)| key != &k); if next.is_some() { next diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index 819490d3..e6dad433 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -75,7 +75,7 @@ impl ReadOnlyGenerator { Ok(quote! { pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { - let link: Link = self.0.indexes.#field_ident.get(#by).map(|kv| kv.get().value.into())?; + let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; self.0.data.select_non_ghosted(link).ok() } }) @@ -146,18 +146,28 @@ impl ReadOnlyGenerator { } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; + let (index_range, select_row) = if idx.is_unique { + ( + quote! { self.0.indexes.#field_ident.range_links(#range_arg) }, + quote! { |link| self.0.data.select_non_ghosted(link.0).ok() }, + ) + } else { + ( + quote! { self.0.indexes.#field_ident.range(#range_arg) }, + quote! { |(_, link)| self.0.data.select_non_ghosted(link.0).ok() }, + ) + }; Ok(quote! { - pub fn #fn_name(&self, range: R) -> SelectQueryBuilder<#row_ident, - impl DoubleEndedIterator + '_, + pub fn #fn_name<'a, R>(&'a self, range: R) -> SelectQueryBuilder<#row_ident, + impl DoubleEndedIterator + 'a, #column_range_type, #row_fields_ident> where - R: #range_bounds + R: #range_bounds + 'a { - let rows = self.0.indexes.#field_ident - .range(#range_arg) - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()); + let rows = #index_range + .filter_map(#select_row); SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/read_only/table/mod.rs b/codegen/src/generators/read_only/table/mod.rs index 2c8c31f2..8b1993f8 100644 --- a/codegen/src/generators/read_only/table/mod.rs +++ b/codegen/src/generators/read_only/table/mod.rs @@ -6,6 +6,7 @@ use proc_macro2::TokenStream; use quote::quote; use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec}; +use crate::generators::index_backend::unique_index_type; use crate::generators::read_only::ReadOnlyGenerator; impl ReadOnlyGenerator { @@ -91,15 +92,22 @@ impl ReadOnlyGenerator { } }; - let node_type = if pk_types_unsized { - quote! { + let key_type = quote! { #primary_key_type }; + let value_type = quote! { OffsetEqLink<#inner_const_name> }; + let worktables_node = if pk_types_unsized { + Some(quote! { UnsizedNode>> - } + }) } else { - quote! { - Vec>> - } + None }; + let node_type = unique_index_type( + self.columns.primary_index_backend, + &key_type, + &value_type, + worktables_node, + ) + .unwrap_or_else(|error| error.into_compile_error()); quote! { #derive diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index 4a053612..bf3c54fd 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -1,7 +1,7 @@ use proc_macro2::TokenStream; use crate::common::Parser; -use crate::common::model::{Columns, Persistence}; +use crate::common::model::{Columns, IndexBackend, Persistence}; pub fn expand(input: TokenStream) -> syn::Result { let mut parser = Parser::new(input); @@ -98,18 +98,44 @@ fn validate_index_backends(columns: &Columns, persistence: Persistence) -> syn:: if let Some(index) = columns .indexes .values() - .find(|index| !index.is_unique && index.backend.is_memory_only()) + .find(|index| !index.is_unique && index.backend != IndexBackend::WorktablesIndex) { return Err(syn::Error::new( index.name.span(), format!( - "non-unique index `{}` cannot use `{}`; non-unique indexes currently require `worktables_index` or `indexset`", + "non-unique index `{}` cannot use `{}`; non-unique indexes currently require `worktables_index`", index.name, index.backend.name() ), )); } + for (column, index) in &columns.indexes { + let key_type = columns + .columns_map + .get(column) + .expect("an index always references a validated column") + .to_string(); + let supported = match index.backend { + IndexBackend::Congee => Some(&["u8", "u16", "u32", "u64", "usize"][..]), + IndexBackend::Arctic => Some(&["u16", "u32", "u64", "u128"][..]), + IndexBackend::WorktablesIndex | IndexBackend::Indexset => None, + }; + if let Some(supported) = supported + && !supported.contains(&key_type.as_str()) + { + return Err(syn::Error::new( + index.name.span(), + format!( + "index `{}` uses `{}`, which does not support key type `{key_type}`; supported types: {}", + index.name, + index.backend.name(), + supported.join(", ") + ), + )); + } + } + Ok(()) } @@ -231,4 +257,54 @@ mod tests { assert!(error.to_string().contains("non-unique indexes currently require")); } + + #[test] + fn congee_rejects_non_machine_word_secondary_keys() { + let error = expand(quote! { + name: StringCongee, + persist: false, + columns: { + id: u64 primary_key, + name: String, + }, + indexes: { + name_idx: name unique using congee, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("does not support key type `String`")); + } + + #[test] + fn arctic_rejects_unsupported_secondary_keys() { + let error = expand(quote! { + name: ByteArctic, + persist: false, + columns: { + id: u64 primary_key, + value: u8, + }, + indexes: { + value_idx: value unique using arctic, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("supported types: u16, u32, u64, u128")); + } + + #[test] + fn congee_rejects_unsupported_primary_keys() { + let error = expand(quote! { + name: StringPrimaryCongee, + persist: false, + columns: { + id: String primary_key using congee, + }, + }) + .unwrap_err(); + + assert!(error.to_string().contains("does not support primary-key type `String`")); + } } diff --git a/src/index/arctic.rs b/src/index/arctic.rs index a53ee84a..1a6f9be9 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -1,5 +1,6 @@ //! Arctic adapter for memory-only unique WorkTable indexes. +use std::borrow::Borrow; use std::fmt::{self, Debug}; use std::ops::RangeBounds; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -8,6 +9,35 @@ use arctic::{ConcurrentMap, Key, Order}; use super::UniqueIndex; +/// Lossless conversion between a WorkTable key and a native Arctic key. +/// +/// Keeping this trait local lets generated primary-key newtypes delegate to +/// their underlying integer without implementing Arctic's low-level key API. +pub trait ArcticKey: Clone + Debug + Ord + Send + Sync + 'static { + type Raw: Key + Clone + Debug + Ord + Send + Sync + 'static; + + fn to_arctic(&self) -> Self::Raw; + fn from_arctic(value: Self::Raw) -> Self; +} + +macro_rules! impl_arctic_key { + ($($ty:ty),* $(,)?) => { + $( + impl ArcticKey for $ty { + type Raw = Self; + + #[inline] + fn to_arctic(&self) -> Self::Raw { *self } + + #[inline] + fn from_arctic(value: Self::Raw) -> Self { value } + } + )* + }; +} + +impl_arctic_key!(u16, u32, u64, u128); + /// Arctic's lock-free adaptive radix tree with WorkTable's unique-index /// contract. /// @@ -15,12 +45,12 @@ use super::UniqueIndex; /// representation is limited to 64 bits. Point operations remain directly /// backed by Arctic; ordered scans are collected into a stable snapshot to /// satisfy WorkTable's double-ended query interface. -pub struct ArcticIndex { - inner: ConcurrentMap>, +pub struct ArcticIndex { + inner: ConcurrentMap>, len: AtomicUsize, } -impl Debug for ArcticIndex { +impl Debug for ArcticIndex { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("ArcticIndex") .field("len", &self.len.load(Ordering::Relaxed)) @@ -30,7 +60,7 @@ impl Debug for ArcticIndex { impl Default for ArcticIndex where - K: Key, + K: ArcticKey, { fn default() -> Self { Self { @@ -42,16 +72,18 @@ where impl UniqueIndex for ArcticIndex where - K: Key + Clone + Debug + Ord + Send + Sync + 'static, + K: ArcticKey, V: Clone + Debug + Send + Sync + 'static, { #[inline] fn get_value(&self, key: &K) -> Option { + let key = key.to_arctic(); self.inner.get(key.borrow()).map(|value| (*value).clone()) } #[inline] fn insert_value(&self, key: K, value: V) -> Option { + let key = key.to_arctic(); let updated = self.inner.upsert(key.as_insert(), Box::new(value)); let old = updated.old().cloned(); if old.is_none() { @@ -62,6 +94,7 @@ where #[inline] fn insert_value_checked(&self, key: K, value: V) -> Option<()> { + let key = key.to_arctic(); match self.inner.insert(key.as_insert(), Box::new(value)) { Ok(_) => { self.len.fetch_add(1, Ordering::Relaxed); @@ -76,7 +109,8 @@ where #[inline] fn remove_value(&self, key: &K) -> Option<(K, V)> { - let old = self.inner.remove(key.borrow())?; + let raw_key = key.to_arctic(); + let old = self.inner.remove(raw_key.borrow())?; self.len.fetch_sub(1, Ordering::Relaxed); Some((key.clone(), (*old).clone())) } @@ -90,17 +124,28 @@ where let shard = self.inner.all(); shard .entries(Order::Ascend) - .map(|(key, value)| (key, value.clone())) + .map(|(key, value)| (K::from_arctic(key), value.clone())) .collect::>() .into_iter() } + fn iter_links(&self) -> impl DoubleEndedIterator + '_ { + self.iter_values().map(|(_, value)| value) + } + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where R: RangeBounds + 'a, { self.iter_values().filter(move |(key, _)| range.contains(key)) } + + fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.range_values(range).map(|(_, value)| value) + } } #[cfg(test)] diff --git a/src/index/congee.rs b/src/index/congee.rs index 86a2b668..dc76dd21 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -187,16 +187,27 @@ where self.get_value(&key).map(|value| (key, value)) }) .collect::>(); - values.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + values.sort_unstable_by_key(|entry| entry.0); values.into_iter() } + fn iter_links(&self) -> impl DoubleEndedIterator + '_ { + self.iter_values().map(|(_, value)| value) + } + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where R: RangeBounds + 'a, { self.iter_values().filter(move |(key, _)| range.contains(key)) } + + fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.range_values(range).map(|(_, value)| value) + } } #[cfg(test)] diff --git a/src/index/mod.rs b/src/index/mod.rs index ea512e59..196f05b4 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -8,18 +8,18 @@ mod table_secondary_index; mod unique; mod unsized_node; -pub use arctic::ArcticIndex; +pub use arctic::{ArcticIndex, ArcticKey}; pub use available_index::AvailableIndex; pub use congee::{CongeeIndex, CongeeKey}; pub use indexset::concurrent::map::BTreeMap as IndexMap; pub use indexset::concurrent::multimap::BTreeMultiMap as IndexMultiMap; pub use multipair::MultiPairRecreate; pub use primary_index::PrimaryIndex; -pub use table_index::{TableIndex, TableIndexCdc, convert_change_events}; +pub use table_index::{TableIndex, TableIndexCdc, convert_change_events, convert_upstream_change_events}; pub use table_secondary_index::{ IndexError, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, }; -pub use unique::{UniqueIndex, UpstreamIndexMap}; +pub use unique::{UniqueIndex, UpstreamIndexMap, UpstreamIndexPair}; pub use unsized_node::UnsizedNode; #[derive(Clone, Debug)] diff --git a/src/index/primary_index.rs b/src/index/primary_index.rs index 3589b2f6..d36e7722 100644 --- a/src/index/primary_index.rs +++ b/src/index/primary_index.rs @@ -8,11 +8,10 @@ use std::hash::Hash; use data_bucket::Link; use indexset::cdc::change::ChangeEvent; -use indexset::core::node::NodeLike; use indexset::core::pair::Pair; use crate::util::OffsetEqLink; -use crate::{IndexMap, TableIndex, TableIndexCdc, convert_change_events}; +use crate::{IndexMap, TableIndex, TableIndexCdc, UniqueIndex}; /// Combined storage for primary and reverse indexes. /// @@ -20,40 +19,37 @@ use crate::{IndexMap, TableIndex, TableIndexCdc, convert_change_events}; /// - **Forward index**: `PrimaryKey` → [`OffsetEqLink`] (primary lookups) /// - **Reverse index**: [`OffsetEqLink`] → `PrimaryKey` (vacuum, position queries) #[derive(Debug)] -pub struct PrimaryIndex< - PrimaryKey, - const DATA_LENGTH: usize, - PkNodeType = Vec>>, -> where +pub struct PrimaryIndex>> +where PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: UniqueIndex>, { - pub pk_map: IndexMap, PkNodeType>, + pub pk_map: PkMap, pub reverse_pk_map: IndexMap, PrimaryKey>, } -impl Default for PrimaryIndex +impl Default for PrimaryIndex where PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: UniqueIndex>, { fn default() -> Self { Self { - pk_map: IndexMap::default(), + pk_map: PkMap::default(), reverse_pk_map: IndexMap::default(), } } } -impl TableIndex - for PrimaryIndex +impl TableIndex + for PrimaryIndex where - PrimaryKey: Debug + Eq + Hash + Clone + Send + Ord, - PkNodeType: NodeLike>> + Send + 'static, + PrimaryKey: Debug + Eq + Hash + Clone + Send + Ord + 'static, + PkMap: UniqueIndex>, { fn insert(&self, value: PrimaryKey, link: Link) -> Option { let offset_link = OffsetEqLink(link); - let old = self.pk_map.insert(value.clone(), offset_link); + let old = self.pk_map.insert_value(value.clone(), offset_link); if let Some(old_link) = old { // Update reverse index self.reverse_pk_map.remove(&old_link); @@ -64,43 +60,41 @@ where fn insert_checked(&self, value: PrimaryKey, link: Link) -> Option<()> { let offset_link = OffsetEqLink(link); - self.pk_map.checked_insert(value.clone(), offset_link)?; + self.pk_map.insert_value_checked(value.clone(), offset_link)?; self.reverse_pk_map.checked_insert(offset_link, value)?; Some(()) } fn remove(&self, value: &PrimaryKey, _: Link) -> Option<(PrimaryKey, Link)> { - let (_, old_link) = self.pk_map.remove(value)?; + let (_, old_link) = self.pk_map.remove_value(value)?; self.reverse_pk_map.remove(&old_link); Some((value.clone(), old_link.0)) } } -impl TableIndexCdc - for PrimaryIndex +impl TableIndexCdc + for PrimaryIndex where - PrimaryKey: Debug + Eq + Hash + Clone + Send + Ord, - PkNodeType: NodeLike>> + Send + 'static, + PrimaryKey: Debug + Eq + Hash + Clone + Send + Ord + 'static, + PkMap: UniqueIndex> + TableIndexCdc, { fn insert_cdc(&self, value: PrimaryKey, link: Link) -> (Option, Vec>>) { let offset_link = OffsetEqLink(link); - let (res, evs) = self.pk_map.insert_cdc(value.clone(), offset_link); - let res_link = res.map(|l| l.0); - if let Some(res) = res { - self.reverse_pk_map.remove(&res); + let (old_link, events) = TableIndexCdc::insert_cdc(&self.pk_map, value.clone(), link); + if let Some(old_link) = old_link { + self.reverse_pk_map.remove(&OffsetEqLink(old_link)); } self.reverse_pk_map.insert(offset_link, value); - (res_link, convert_change_events(evs)) + (old_link, events) } fn insert_checked_cdc(&self, value: PrimaryKey, link: Link) -> Option>>> { let offset_link = OffsetEqLink(link); - let res = self.pk_map.checked_insert_cdc(value.clone(), offset_link); - - if let Some(evs) = res { + let events = TableIndexCdc::insert_checked_cdc(&self.pk_map, value.clone(), link); + if let Some(events) = events { self.reverse_pk_map.insert(offset_link, value); - Some(convert_change_events(evs)) + Some(events) } else { None } @@ -109,16 +103,14 @@ where fn remove_cdc( &self, value: PrimaryKey, - _: Link, + link: Link, ) -> (Option<(PrimaryKey, Link)>, Vec>>) { - let (res, evs) = self.pk_map.remove_cdc(&value); - - if let Some((pk, old_link)) = res { - let offset_link = OffsetEqLink(old_link.0); - self.reverse_pk_map.remove(&offset_link); - (Some((pk, old_link.0)), convert_change_events(evs)) + let (removed, events) = TableIndexCdc::remove_cdc(&self.pk_map, value, link); + if let Some((key, old_link)) = removed { + self.reverse_pk_map.remove(&OffsetEqLink(old_link)); + (Some((key, old_link)), events) } else { - (None, convert_change_events(evs)) + (None, events) } } } @@ -130,7 +122,7 @@ mod tests { const TEST_DATA_LENGTH: usize = 4096; - type TestPrimaryIndex = PrimaryIndex>>>; + type TestPrimaryIndex = PrimaryIndex; #[test] fn test_default_creates_empty_indexes() { diff --git a/src/index/table_index/cdc.rs b/src/index/table_index/cdc.rs index 58c352b1..ea442c3a 100644 --- a/src/index/table_index/cdc.rs +++ b/src/index/table_index/cdc.rs @@ -6,10 +6,12 @@ use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +use vanilla_indexset::core::pair::Pair as VanillaPair; -use crate::index::table_index::util::convert_change_events; +use crate::index::table_index::util::{convert_change_events, convert_upstream_change_events}; use crate::util::OffsetEqLink; -use crate::{IndexMap, IndexMultiMap}; +use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex, UpstreamIndexMap}; pub trait TableIndexCdc { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>); @@ -70,3 +72,56 @@ where (res_pair, convert_change_events(evs)) } } + +impl TableIndexCdc for UpstreamIndexMap, Node> +where + T: Debug + Eq + Hash + Clone + Send + Ord, + Node: VanillaNodeLike>> + Send + 'static, +{ + fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>) { + let (res, events) = self.insert_cdc(value, OffsetEqLink(link)); + (res.map(|value| value.0), convert_upstream_change_events(events)) + } + + fn insert_checked_cdc(&self, value: T, link: Link) -> Option>>> { + self.checked_insert_cdc(value, OffsetEqLink(link)) + .map(convert_upstream_change_events) + } + + fn remove_cdc(&self, value: T, _: Link) -> (Option<(T, Link)>, Vec>>) { + let (res, events) = self.remove_cdc(&value); + ( + res.map(|(key, value)| (key, value.0)), + convert_upstream_change_events(events), + ) + } +} + +/// Memory-only ARTs participate in the common mutation path but emit no +/// durable events. The DSL prevents these implementations from appearing in +/// a persisted table. +macro_rules! impl_memory_only_cdc { + ($index:ty, [$($bound:tt)*]) => { + impl TableIndexCdc for $index + where + T: $($bound)*, + { + fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>) { + let old = self.insert_value(value, OffsetEqLink(link)).map(|value| value.0); + (old, Vec::new()) + } + + fn insert_checked_cdc(&self, value: T, link: Link) -> Option>>> { + self.insert_value_checked(value, OffsetEqLink(link)).map(|()| Vec::new()) + } + + fn remove_cdc(&self, value: T, _: Link) -> (Option<(T, Link)>, Vec>>) { + let removed = self.remove_value(&value).map(|(key, value)| (key, value.0)); + (removed, Vec::new()) + } + } + }; +} + +impl_memory_only_cdc!(CongeeIndex>, [CongeeKey + Eq + Hash]); +impl_memory_only_cdc!(ArcticIndex>, [ArcticKey + Eq + Hash]); diff --git a/src/index/table_index/mod.rs b/src/index/table_index/mod.rs index c44264bb..9eb3543e 100644 --- a/src/index/table_index/mod.rs +++ b/src/index/table_index/mod.rs @@ -5,15 +5,17 @@ use data_bucket::Link; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +use vanilla_indexset::core::pair::Pair as VanillaPair; use crate::util::OffsetEqLink; -use crate::{IndexMap, IndexMultiMap}; +use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMap, IndexMultiMap, UniqueIndex, UpstreamIndexMap}; mod cdc; pub mod util; pub use cdc::TableIndexCdc; -pub use util::convert_change_events; +pub use util::{convert_change_events, convert_upstream_change_events}; pub trait TableIndex { fn insert(&self, value: T, link: Link) -> Option; @@ -43,20 +45,91 @@ where } } +#[inline] +fn unique_insert(index: &I, value: T, link: Link) -> Option +where + T: Debug + Eq + Hash + Clone + Send + Ord, + I: UniqueIndex, +{ + index.insert_value(value, OffsetEqLink(link)).map(|l| l.0) +} + +#[inline] +fn unique_insert_checked(index: &I, value: T, link: Link) -> Option<()> +where + T: Debug + Eq + Hash + Clone + Send + Ord, + I: UniqueIndex, +{ + index.insert_value_checked(value, OffsetEqLink(link)) +} + +#[inline] +fn unique_remove(index: &I, value: &T) -> Option<(T, Link)> +where + T: Debug + Eq + Hash + Clone + Send + Ord, + I: UniqueIndex, +{ + index.remove_value(value).map(|(v, l)| (v, l.0)) +} + +macro_rules! impl_unique_table_index { + ($index:ty, [$($bound:tt)*]) => { + impl TableIndex for $index + where + T: $($bound)*, + { + fn insert(&self, value: T, link: Link) -> Option { + unique_insert(self, value, link) + } + + fn insert_checked(&self, value: T, link: Link) -> Option<()> { + unique_insert_checked(self, value, link) + } + + fn remove(&self, value: &T, _: Link) -> Option<(T, Link)> { + unique_remove(self, value) + } + } + }; +} + impl TableIndex for IndexMap where T: Debug + Eq + Hash + Clone + Send + Ord, Node: NodeLike> + Send + 'static, { fn insert(&self, value: T, link: Link) -> Option { - self.insert(value, OffsetEqLink(link)).map(|l| l.0) + unique_insert(self, value, link) } fn insert_checked(&self, value: T, link: Link) -> Option<()> { - self.checked_insert(value, OffsetEqLink(link)) + unique_insert_checked(self, value, link) } fn remove(&self, value: &T, _: Link) -> Option<(T, Link)> { - self.remove(value).map(|(v, l)| (v, l.0)) + unique_remove(self, value) } } + +impl TableIndex for UpstreamIndexMap +where + T: Debug + Eq + Hash + Clone + Send + Ord, + Node: VanillaNodeLike> + Send + 'static, +{ + fn insert(&self, value: T, link: Link) -> Option { + unique_insert(self, value, link) + } + + fn insert_checked(&self, value: T, link: Link) -> Option<()> { + unique_insert_checked(self, value, link) + } + + fn remove(&self, value: &T, _: Link) -> Option<(T, Link)> { + unique_remove(self, value) + } +} + +impl_unique_table_index!(CongeeIndex, [CongeeKey + Eq + Hash]); +impl_unique_table_index!(ArcticIndex, [ + ArcticKey + Eq + Hash +]); diff --git a/src/index/table_index/util.rs b/src/index/table_index/util.rs index 7346ecb3..c254ae81 100644 --- a/src/index/table_index/util.rs +++ b/src/index/table_index/util.rs @@ -1,5 +1,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::core::pair::Pair; +use vanilla_indexset::cdc::change::ChangeEvent as VanillaChangeEvent; +use vanilla_indexset::core::pair::Pair as VanillaPair; pub fn convert_change_event(ev: ChangeEvent>) -> ChangeEvent> where @@ -75,3 +77,66 @@ where { evs.into_iter().map(convert_change_event).collect() } + +/// Normalizes upstream IndexSet CDC events into WorkTablesIndex's event type, +/// which remains the stable persistence boundary used by DataBucket. +pub fn convert_upstream_change_events( + evs: Vec>>, +) -> Vec>> +where + L1: Into, +{ + evs.into_iter() + .map(|event| match event { + VanillaChangeEvent::InsertAt { + event_id, + max_value, + value, + index, + } => ChangeEvent::InsertAt { + event_id: event_id.inner().into(), + max_value: upstream_pair(max_value), + value: upstream_pair(value), + index, + }, + VanillaChangeEvent::RemoveAt { + event_id, + max_value, + value, + index, + } => ChangeEvent::RemoveAt { + event_id: event_id.inner().into(), + max_value: upstream_pair(max_value), + value: upstream_pair(value), + index, + }, + VanillaChangeEvent::CreateNode { event_id, max_value } => ChangeEvent::CreateNode { + event_id: event_id.inner().into(), + max_value: upstream_pair(max_value), + }, + VanillaChangeEvent::RemoveNode { event_id, max_value } => ChangeEvent::RemoveNode { + event_id: event_id.inner().into(), + max_value: upstream_pair(max_value), + }, + VanillaChangeEvent::SplitNode { + event_id, + max_value, + split_index, + } => ChangeEvent::SplitNode { + event_id: event_id.inner().into(), + max_value: upstream_pair(max_value), + split_index, + }, + }) + .collect() +} + +fn upstream_pair(pair: VanillaPair) -> Pair +where + L1: Into, +{ + Pair { + key: pair.key, + value: pair.value.into(), + } +} diff --git a/src/index/unique.rs b/src/index/unique.rs index 3e6ad37a..fedb6fd5 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -18,7 +18,7 @@ use vanilla_indexset::core::pair::Pair as VanillaPair; /// Point, mutation, and ordered-scan operations used by generated unique /// indexes. Implementations are statically dispatched; this adds no virtual /// call to the lookup path. -pub trait UniqueIndex: Debug + Default + Send + Sync +pub trait UniqueIndex: Default where K: Clone + Ord, V: Clone, @@ -35,16 +35,22 @@ where fn iter_values(&self) -> impl DoubleEndedIterator + '_; + fn iter_links(&self) -> impl DoubleEndedIterator + '_; + fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where R: RangeBounds + 'a; + + fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a; } impl UniqueIndex for IndexMap where - K: Debug + Eq + Hash + Clone + Send + Sync + Ord + 'static, - V: Debug + Clone + Send + Sync + Ord + 'static, - Node: NodeLike> + Debug + Send + Sync + 'static, + K: Debug + Eq + Hash + Clone + Send + Ord + 'static, + V: Debug + Clone + Send + Ord + 'static, + Node: NodeLike> + Send + 'static, { #[inline] fn get_value(&self, key: &K) -> Option { @@ -76,6 +82,11 @@ where self.iter().map(|(key, value)| (key.clone(), value.clone())) } + #[inline] + fn iter_links(&self) -> impl DoubleEndedIterator + '_ { + self.iter().map(|(_, value)| value.clone()) + } + #[inline] fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where @@ -83,13 +94,21 @@ where { self.range(range).map(|(key, value)| (key.clone(), value.clone())) } + + #[inline] + fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.range(range).map(|(_, value)| value.clone()) + } } impl UniqueIndex for VanillaIndexMap where - K: Debug + Eq + Hash + Clone + Send + Sync + Ord + 'static, - V: Debug + Clone + Send + Sync + Ord + 'static, - Node: VanillaNodeLike> + Debug + Send + Sync + 'static, + K: Debug + Eq + Hash + Clone + Send + Ord + 'static, + V: Debug + Clone + Send + Ord + 'static, + Node: VanillaNodeLike> + Send + 'static, { #[inline] fn get_value(&self, key: &K) -> Option { @@ -121,6 +140,11 @@ where self.iter().map(|(key, value)| (key.clone(), value.clone())) } + #[inline] + fn iter_links(&self) -> impl DoubleEndedIterator + '_ { + self.iter().map(|(_, value)| value.clone()) + } + #[inline] fn range_values<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a where @@ -128,11 +152,20 @@ where { self.range(range).map(|(key, value)| (key.clone(), value.clone())) } + + #[inline] + fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a + where + R: RangeBounds + 'a, + { + self.range(range).map(|(_, value)| value.clone()) + } } /// Vanilla upstream IndexSet map, kept distinct from WorkTable's default /// WorkTablesIndex alias so both implementations may coexist in one binary. pub type UpstreamIndexMap>> = VanillaIndexMap; +pub type UpstreamIndexPair = VanillaPair; #[cfg(test)] mod tests { @@ -151,7 +184,9 @@ mod tests { assert_eq!(index.get_value(&2), Some(20)); assert_eq!(index.insert_value(2, 22), Some(20)); assert_eq!(index.iter_values().collect::>(), vec![(1, 10), (2, 22)]); + assert_eq!(index.iter_links().collect::>(), vec![10, 22]); assert_eq!(index.range_values(2..=2).collect::>(), vec![(2, 22)]); + assert_eq!(index.range_links(2..=2).collect::>(), vec![22]); assert_eq!(index.remove_value(&1), Some((1, 10))); assert_eq!(index.len(), 1); } diff --git a/src/lib.rs b/src/lib.rs index 56d8c9f6..bdc87ca1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,11 +43,11 @@ pub mod prelude { pub use crate::table::system_info::{IndexInfo, IndexKind, SystemInfo}; pub use crate::util::{OffsetEqLink, OrderedF32Def, OrderedF64Def}; pub use crate::{ - ArcticIndex, AvailableIndex, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, IndexMultiMap, - MultiPairRecreate, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, + ArcticIndex, ArcticKey, AvailableIndex, CongeeIndex, CongeeKey, Difference, IndexError, IndexMap, + IndexMultiMap, MultiPairRecreate, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, TableSecondaryIndexEventsOps, TableSecondaryIndexInfo, UniqueIndex, UnsizedNode, - UpstreamIndexMap, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, vacuum::VacuumPersistence, - vacuum::WorkTableVacuum, + UpstreamIndexMap, UpstreamIndexPair, WorkTable, WorkTableError, vacuum::EmptyDataVacuum, + vacuum::VacuumPersistence, vacuum::WorkTableVacuum, }; pub use data_bucket::{ DATA_VERSION, DataPage, GENERAL_HEADER_SIZE, GeneralHeader, GeneralPage, INNER_PAGE_SIZE, IndexPage, Interval, diff --git a/src/mem_stat/mod.rs b/src/mem_stat/mod.rs index e55ea3f5..f30d2e36 100644 --- a/src/mem_stat/mod.rs +++ b/src/mem_stat/mod.rs @@ -14,11 +14,13 @@ use ordered_float::OrderedFloat; use psc_nanoid::PackedNanoid; use psc_nanoid::packed::AlphabetPackExt; use uuid::Uuid; +use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; +use vanilla_indexset::core::pair::Pair as VanillaPair; -use crate::IndexMultiMap; use crate::persistence::OperationType; use crate::prelude::OperationId; use crate::util::OffsetEqLink; +use crate::{ArcticIndex, ArcticKey, CongeeIndex, CongeeKey, IndexMultiMap, UniqueIndex, UpstreamIndexMap}; use crate::{IndexMap, impl_memstat_zero}; pub trait MemStat { @@ -78,6 +80,55 @@ where } } +impl MemStat for UpstreamIndexMap +where + K: Debug + Ord + Clone + 'static + MemStat + Send, + V: Debug + Clone + 'static + MemStat + Send, + Node: VanillaNodeLike> + Send + 'static, +{ + fn heap_size(&self) -> usize { + let slot_size = std::mem::size_of::>(); + let base_heap = self.capacity() * slot_size; + let kv_heap: usize = self.iter().map(|(k, v)| k.heap_size() + v.heap_size()).sum(); + base_heap + kv_heap + } + + fn used_size(&self) -> usize { + let pair_size = std::mem::size_of::>(); + let base = self.len() * pair_size; + let used: usize = self.iter().map(|(k, v)| k.used_size() + v.used_size()).sum(); + base + used + } +} + +impl MemStat for CongeeIndex +where + K: CongeeKey, + V: Clone + Debug + Send + Sync + 'static, +{ + fn heap_size(&self) -> usize { + self.len() * std::mem::size_of::<(K, V)>() + } + + fn used_size(&self) -> usize { + self.len() * std::mem::size_of::<(K, V)>() + } +} + +impl MemStat for ArcticIndex +where + K: ArcticKey, + V: Clone + Debug + Send + Sync + 'static, +{ + fn heap_size(&self) -> usize { + self.len() * std::mem::size_of::<(K, V)>() + } + + fn used_size(&self) -> usize { + self.len() * std::mem::size_of::<(K, V)>() + } +} + impl MemStat for IndexMultiMap where K: Debug + Ord + Clone + 'static + MemStat + Send, diff --git a/src/table/mod.rs b/src/table/mod.rs index cb54aaa9..f5a74782 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -13,8 +13,6 @@ use crate::{ }; use data_bucket::INNER_PAGE_SIZE; use derive_more::{Display, Error, From}; -use indexset::core::node::NodeLike; -use indexset::core::pair::Pair; #[cfg(feature = "perf_measurements")] use performance_measurement_codegen::performance_measurement; use rkyv::api::high::HighDeserializer; @@ -39,15 +37,15 @@ pub struct WorkTable< LockType = (), PkGen = ::Generator, const DATA_LENGTH: usize = INNER_PAGE_SIZE, - PkNodeType = Vec>>, + PkMap = IndexMap>, > where PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, Row: StorableRow + Send + Clone + 'static, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: crate::UniqueIndex>, { pub data: Arc>, - pub primary_index: Arc>, + pub primary_index: Arc>, pub indexes: Arc, @@ -72,7 +70,7 @@ impl< LockType, PkGen, const DATA_LENGTH: usize, - PkNodeType, + PkMap, > Default for WorkTable< Row, @@ -83,20 +81,20 @@ impl< LockType, PkGen, DATA_LENGTH, - PkNodeType, + PkMap, > where PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, SecondaryIndexes: Default, PkGen: Default, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: crate::UniqueIndex>, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, { fn default() -> Self { Self { data: Arc::new(DataPages::new()), - primary_index: Arc::new(PrimaryIndex::default()), + primary_index: Arc::new(PrimaryIndex::::default()), indexes: Arc::new(SecondaryIndexes::default()), pk_gen: Default::default(), lock_manager: Default::default(), @@ -116,23 +114,12 @@ impl< LockType, PkGen, const DATA_LENGTH: usize, - PkNodeType, -> - WorkTable< - Row, - PrimaryKey, - AvailableTypes, - AvailableIndexes, - SecondaryIndexes, - LockType, - PkGen, - DATA_LENGTH, - PkNodeType, - > + PkMap, +> WorkTable where Row: TableRow, PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: crate::UniqueIndex>, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, { @@ -152,7 +139,7 @@ where <::WrappedRow as Archive>::Archived: Deserialize<::WrappedRow, HighDeserializer>, { - let link: Option = self.primary_index.pk_map.get(&pk).map(|v| v.get().value.into()); + let link: Option = self.primary_index.pk_map.get_value(&pk).map(Into::into); if let Some(link) = link { self.data.select_non_ghosted(link).ok() } else { @@ -224,6 +211,7 @@ where PkGen: PrimaryKeyGeneratorState, ::State: Debug, AvailableIndexes: Debug + AvailableIndex, + PrimaryIndex: TableIndexCdc, { let pk = row.get_primary_key().clone(); @@ -348,8 +336,8 @@ where let old_link: Link = self .primary_index .pk_map - .get(&pk) - .map(|v| v.get().value.into()) + .get_value(&pk) + .map(Into::into) .ok_or(WorkTableError::NotFound)?; let new_link = self.data.insert(row_new.clone()).map_err(WorkTableError::PagesError)?; unsafe { @@ -398,6 +386,7 @@ where + TableSecondaryIndexCdc, PkGen: PrimaryKeyGeneratorState, AvailableIndexes: Debug + AvailableIndex, + PrimaryIndex: TableIndexCdc, { let pk = row_new.get_primary_key().clone(); if pk != row_old.get_primary_key() { @@ -405,8 +394,8 @@ where } // Get old link - if not found, no events to acknowledge - let old_link = match self.primary_index.pk_map.get(&pk) { - Some(v) => v.get().value.into(), + let old_link = match self.primary_index.pk_map.get_value(&pk) { + Some(v) => v.into(), None => return (None, Err(WorkTableError::NotFound)), }; diff --git a/src/table/system_info.rs b/src/table/system_info.rs index 1f722ed3..65ff1247 100644 --- a/src/table/system_info.rs +++ b/src/table/system_info.rs @@ -1,12 +1,10 @@ -use indexset::core::node::NodeLike; -use indexset::core::pair::Pair; use prettytable::{Table, format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR, row}; use std::fmt::{self, Debug, Display, Formatter}; use crate::in_memory::{RowWrapper, StorableRow}; use crate::mem_stat::MemStat; use crate::util::OffsetEqLink; -use crate::{TableSecondaryIndexInfo, WorkTable}; +use crate::{TableSecondaryIndexInfo, UniqueIndex, WorkTable}; #[derive(Debug)] pub struct SystemInfo { @@ -54,13 +52,13 @@ impl< LockType, PkGen, const DATA_LENGTH: usize, - NodeType, -> WorkTable + PkMap, +> WorkTable where PrimaryKey: Debug + Clone + Ord + Send + 'static + std::hash::Hash, Row: StorableRow + Send + Clone + 'static, ::WrappedRow: RowWrapper, - NodeType: NodeLike>> + Send + 'static, + PkMap: UniqueIndex>, SecondaryIndexes: MemStat + TableSecondaryIndexInfo, { pub fn system_info(&self) -> SystemInfo { diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index 1cb3d8b7..ca03f4dd 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -6,8 +6,6 @@ use std::time::{Duration, Instant}; use data_bucket::Link; use data_bucket::page::PageId; -use indexset::core::node::NodeLike; -use indexset::core::pair::Pair; use rkyv::rancor::Strategy; use rkyv::ser::Serializer; use rkyv::ser::allocator::ArenaHandle; @@ -24,6 +22,7 @@ use crate::vacuum::WorkTableVacuum; use crate::vacuum::fragmentation_info::FragmentationInfo; use crate::{ AvailableIndex, PrimaryIndex, TableIndex, TableIndexCdc, TableRow, TableSecondaryIndex, TableSecondaryIndexCdc, + UniqueIndex, }; use async_trait::async_trait; use ordered_float::OrderedFloat; @@ -33,7 +32,7 @@ use rkyv::api::high::HighDeserializer; pub struct EmptyDataVacuum< Row, PrimaryKey, - PkNodeType, + PkMap, SecondaryIndexes, AvailableTypes, AvailableIndexes, @@ -43,7 +42,7 @@ pub struct EmptyDataVacuum< > where PrimaryKey: Clone + Ord + Send + 'static + std::hash::Hash, Row: StorableRow + Send + Clone + 'static + Debug, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: UniqueIndex>, { table_name: &'static str, @@ -51,7 +50,7 @@ pub struct EmptyDataVacuum< lock_manager: Arc>, - primary_index: Arc>, + primary_index: Arc>, secondary_indexes: Arc, /// Persistence sink for row moves. `None` for in-memory tables; persisted @@ -65,7 +64,7 @@ pub struct EmptyDataVacuum< impl< Row, PrimaryKey, - PkNodeType, + PkMap, SecondaryIndexes, AvailableTypes, AvailableIndexes, @@ -76,7 +75,7 @@ impl< EmptyDataVacuum< Row, PrimaryKey, - PkNodeType, + PkMap, SecondaryIndexes, AvailableTypes, AvailableIndexes, @@ -87,7 +86,7 @@ impl< where Row: TableRow + StorableRow + Send + Clone + 'static, PrimaryKey: Debug + Clone + Ord + Send + TablePrimaryKey + std::hash::Hash, - PkNodeType: NodeLike>> + Send + 'static, + PkMap: UniqueIndex>, ::WrappedRow: RowWrapper, Row: Archive + Clone @@ -101,13 +100,14 @@ where + TableSecondaryIndexCdc, AvailableIndexes: Debug + AvailableIndex, LockType: RowLock, + PrimaryIndex: TableIndexCdc, { /// Creates a new [`EmptyDataVacuum`] from the given [`WorkTable`] components. pub fn new( table_name: &'static str, data_pages: Arc>, lock_manager: Arc>, - primary_index: Arc>, + primary_index: Arc>, secondary_indexes: Arc, ) -> Self { Self { @@ -330,7 +330,7 @@ where impl< Row, PrimaryKey, - PkNodeType, + PkMap, SecondaryIndexes, AvailableTypes, AvailableIndexes, @@ -341,7 +341,7 @@ impl< for EmptyDataVacuum< Row, PrimaryKey, - PkNodeType, + PkMap, SecondaryIndexes, AvailableTypes, AvailableIndexes, @@ -352,7 +352,7 @@ impl< where Row: TableRow + StorableRow + Send + Sync + Clone + 'static, PrimaryKey: Debug + Clone + Ord + Send + Sync + TablePrimaryKey + std::hash::Hash, - PkNodeType: NodeLike>> + Send + Sync + 'static, + PkMap: UniqueIndex> + Send + Sync + 'static, ::WrappedRow: RowWrapper, Row: Archive + Clone @@ -373,6 +373,7 @@ where AvailableTypes: Send + Sync + 'static, AvailableIndexes: Send + Sync + 'static, LockType: RowLock + Send + Sync, + PrimaryIndex: TableIndexCdc, { fn table_name(&self) -> &str { self.table_name @@ -393,7 +394,6 @@ mod tests { use std::collections::HashMap; use std::sync::Arc; - use indexset::core::pair::Pair; use worktable_codegen::{MemStat, worktable}; use crate::in_memory::{ArchivedRowWrapper, RowWrapper, StorableRow}; @@ -422,7 +422,7 @@ mod tests { ) -> EmptyDataVacuum< TestRow, TestPrimaryKey, - Vec>>, + IndexMap>, TestIndex, TestAvaiableTypes, TestAvailableIndexes, From bb118bce8075ff49f22de153a2b6d851c82eedd7 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 08:24:50 +0700 Subject: [PATCH 06/23] feat: preserve persistence across indexset providers --- codegen/src/persist_index/generator.rs | 38 +++++++++++++++---- codegen/src/persist_table/generator/mod.rs | 1 + .../persist_table/generator/space_file/mod.rs | 19 +++++++--- .../generator/space_file/worktable_impls.rs | 26 +++++++++++-- codegen/src/persist_table/mod.rs | 2 + codegen/src/persist_table/parser.rs | 11 ++++++ 6 files changed, 81 insertions(+), 16 deletions(-) diff --git a/codegen/src/persist_index/generator.rs b/codegen/src/persist_index/generator.rs index 84884c30..60b72789 100644 --- a/codegen/src/persist_index/generator.rs +++ b/codegen/src/persist_index/generator.rs @@ -275,16 +275,16 @@ impl Generator { .struct_def .fields .iter() - .map(|f| { - f.ident + .map(|field| { + let i = field + .ident .as_ref() - .expect("index fields should always be named fields") - }) - .map(|i| { + .expect("index fields should always be named fields"); let ty = self .field_types .get(i) .expect("should be available as constructed from same values"); + let uses_upstream = field.ty.to_token_stream().to_string().contains("UpstreamIndexMap"); if is_unsized(&ty.to_string()) { quote! { let mut pages = vec![]; @@ -295,6 +295,24 @@ impl Generator { let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); let #i = (toc.pages, pages); } + } else if uses_upstream { + quote! { + let size = get_index_page_size_from_data_length::<#ty>(#const_name); + let mut pages = vec![]; + for node in self.#i.iter_nodes() { + let node: Vec> = node + .lock_arc() + .iter() + .map(|pair| IndexPair { + key: pair.key.clone(), + value: pair.value, + }) + .collect(); + pages.push(IndexPage::from_node(&node, size)); + } + let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); + let #i = (toc.pages, pages); + } } else { quote! { let size = get_index_page_size_from_data_length::<#ty>(#const_name); @@ -340,6 +358,7 @@ impl Generator { let i = f.ident.as_ref().expect("index fields should always be named fields"); let index_type = f.ty.to_token_stream().to_string(); let is_unique = !index_type.contains("IndexMultiMap"); + let uses_upstream = index_type.contains("UpstreamIndexMap"); let mut split = index_type.split("<"); let t = Ident::new( split.next().expect("index type should always have generics").trim(), @@ -366,13 +385,18 @@ impl Generator { // exactly one node, `get_node()` already yields it in order // with the true maximum last, so it attaches directly. let unique_reconstruct = |attach: TokenStream| { + let pair_type = if uses_upstream { + quote! { UpstreamIndexPair } + } else { + quote! { IndexPair } + }; quote! { for page in persisted.#i.1 { - let inner: Vec> = page + let inner: Vec<#pair_type<#ty, OffsetEqLink>> = page .inner .get_node() .into_iter() - .map(|p| IndexPair { + .map(|p| #pair_type { key: p.key, value: p.value.into(), }) diff --git a/codegen/src/persist_table/generator/mod.rs b/codegen/src/persist_table/generator/mod.rs index 2211b97d..70f5483d 100644 --- a/codegen/src/persist_table/generator/mod.rs +++ b/codegen/src/persist_table/generator/mod.rs @@ -16,6 +16,7 @@ pub struct PersistTableAttributes { pub struct Generator { pub struct_def: ItemStruct, pub pk_ident: Ident, + pub pk_upstream: bool, pub attributes: PersistTableAttributes, } diff --git a/codegen/src/persist_table/generator/space_file/mod.rs b/codegen/src/persist_table/generator/space_file/mod.rs index bd7c8200..5666b1fe 100644 --- a/codegen/src/persist_table/generator/space_file/mod.rs +++ b/codegen/src/persist_table/generator/space_file/mod.rs @@ -142,15 +142,25 @@ impl Generator { let primary_index = PrimaryIndex { pk_map, reverse_pk_map }; } } else { + let map_type = if self.pk_upstream { + quote! { UpstreamIndexMap } + } else { + quote! { IndexMap } + }; + let pair_type = if self.pk_upstream { + quote! { UpstreamIndexPair } + } else { + quote! { IndexPair } + }; quote! { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); - let pk_map = IndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size); + let pk_map = #map_type::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size); for page in self.primary_index.1 { let node = page .inner .get_node() .into_iter() - .map(|p| IndexPair { + .map(|p| #pair_type { key: p.key, value: p.value.into(), }) @@ -159,9 +169,8 @@ impl Generator { } // Reconstruct reverse_pk_map by iterating over pk_map let mut reverse_pk_map = IndexMap::, #pk_type>::new(); - for entry in pk_map.iter() { - let (pk, link) = entry; - reverse_pk_map.insert(*link, pk.clone()); + for (pk, link) in pk_map.iter_values() { + reverse_pk_map.insert(link, pk); } let primary_index = PrimaryIndex { pk_map, reverse_pk_map }; } diff --git a/codegen/src/persist_table/generator/space_file/worktable_impls.rs b/codegen/src/persist_table/generator/space_file/worktable_impls.rs index 3e2e4f03..cacb6792 100644 --- a/codegen/src/persist_table/generator/space_file/worktable_impls.rs +++ b/codegen/src/persist_table/generator/space_file/worktable_impls.rs @@ -87,14 +87,32 @@ impl Generator { } } } else { + let collect_pages = if self.pk_upstream { + quote! { + for node in self.0.primary_index.pk_map.iter_nodes() { + let node: Vec>> = node + .lock_arc() + .iter() + .map(|pair| IndexPair { + key: pair.key.clone(), + value: pair.value, + }) + .collect(); + pages.push(IndexPage::from_node(&node, size)); + } + } + } else { + quote! { + for node in self.0.primary_index.pk_map.iter_nodes() { + pages.push(IndexPage::from_node(node.lock_arc().as_ref(), size)); + } + } + }; quote! { pub fn get_peristed_primary_key_with_toc(&self) -> (Vec>>, Vec>>) { let size = get_index_page_size_from_data_length::<#pk_type>(#const_name); let mut pages = vec![]; - for node in self.0.primary_index.pk_map.iter_nodes() { - let page = IndexPage::from_node(node.lock_arc().as_ref(), size); - pages.push(page); - } + #collect_pages let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); (toc.pages, pages) } diff --git a/codegen/src/persist_table/mod.rs b/codegen/src/persist_table/mod.rs index b7ba5abb..10f43266 100644 --- a/codegen/src/persist_table/mod.rs +++ b/codegen/src/persist_table/mod.rs @@ -12,11 +12,13 @@ pub use generator::WT_INDEX_EXTENSION; pub fn expand(input: TokenStream) -> syn::Result { let input_fn = Parser::parse_struct(input)?; let pk_ident = Parser::parse_pk_ident(&input_fn); + let pk_upstream = Parser::primary_key_uses_upstream(&input_fn); let attributes = Parser::parse_attributes(&input_fn.attrs); let generator = Generator { struct_def: input_fn, pk_ident, + pk_upstream, attributes, }; diff --git a/codegen/src/persist_table/parser.rs b/codegen/src/persist_table/parser.rs index 41961cb8..06281426 100644 --- a/codegen/src/persist_table/parser.rs +++ b/codegen/src/persist_table/parser.rs @@ -25,6 +25,17 @@ impl Parser { Ident::new(pk_type.trim(), Span::mixed_site()) } + pub fn primary_key_uses_upstream(item: &ItemStruct) -> bool { + item.fields + .iter() + .next() + .expect("WorkTable wrapper has one field") + .ty + .to_token_stream() + .to_string() + .contains("UpstreamIndexMap") + } + pub fn parse_attributes(attrs: &Vec) -> PersistTableAttributes { let mut res = PersistTableAttributes { pk_unsized: false, From 2cdad141b9a529a85af34405be9e006a1dd08b97 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 08:25:18 +0700 Subject: [PATCH 07/23] test: cover backend selection and provider switching --- tests/worktable/index_backends.rs | 234 ++++++++++++++++++++++++++++++ tests/worktable/mod.rs | 1 + 2 files changed, 235 insertions(+) create mode 100644 tests/worktable/index_backends.rs diff --git a/tests/worktable/index_backends.rs b/tests/worktable/index_backends.rs new file mode 100644 index 00000000..6330632e --- /dev/null +++ b/tests/worktable/index_backends.rs @@ -0,0 +1,234 @@ +use worktable::prelude::*; +use worktable::worktable; + +use crate::remove_dir_if_exists; + +worktable! { + name: MixedBackend, + persist: false, + columns: { + id: u64 primary_key autoincrement using congee, + wti_key: u64, + upstream_key: u64, + congee_key: u64, + arctic_key: u64, + }, + indexes: { + wti_idx: wti_key unique using worktables_index, + upstream_idx: upstream_key unique using indexset, + congee_idx: congee_key unique using congee, + arctic_idx: arctic_key unique using arctic, + }, +} + +mod provider_switch_wti { + use worktable::prelude::*; + use worktable::worktable; + + worktable! { + name: ProviderSwitch, + persist: true, + columns: { + id: u64 primary_key autoincrement, + unique_key: u64, + }, + indexes: { + unique_idx: unique_key unique, + }, + } +} + +mod provider_switch_upstream { + use worktable::prelude::*; + use worktable::worktable; + + worktable! { + name: ProviderSwitch, + persist: true, + columns: { + id: u64 primary_key autoincrement using indexset, + unique_key: u64, + }, + indexes: { + unique_idx: unique_key unique using indexset, + }, + } +} + +worktable! { + name: UpstreamPrimary, + persist: false, + columns: { + id: u64 primary_key autoincrement using indexset, + value: u64, + }, +} + +worktable! { + name: ArcticPrimary, + persist: false, + columns: { + id: u64 primary_key autoincrement using arctic, + value: u64, + }, +} + +worktable! { + name: PersistedUpstream, + persist: true, + columns: { + id: u64 primary_key autoincrement using indexset, + unique_key: u64, + }, + indexes: { + unique_idx: unique_key unique using indexset, + }, +} + +#[test] +fn all_unique_backends_coexist_in_one_table() { + let table = MixedBackendWorkTable::default(); + let row = MixedBackendRow { + id: table.get_next_pk().into(), + wti_key: 11, + upstream_key: 12, + congee_key: 13, + arctic_key: 14, + }; + + let pk = table.insert(row.clone()).unwrap(); + assert_eq!(table.select(pk), Some(row.clone())); + assert_eq!(table.select_by_wti_key(11), Some(row.clone())); + assert_eq!(table.select_by_upstream_key(12), Some(row.clone())); + assert_eq!(table.select_by_congee_key(13), Some(row.clone())); + assert_eq!(table.select_by_arctic_key(14), Some(row)); +} + +#[test] +fn alternative_primary_backends_support_point_crud() { + let upstream = UpstreamPrimaryWorkTable::default(); + let upstream_row = UpstreamPrimaryRow { + id: upstream.get_next_pk().into(), + value: 1, + }; + let upstream_pk = upstream.insert(upstream_row.clone()).unwrap(); + assert_eq!(upstream.select(upstream_pk), Some(upstream_row)); + + let arctic = ArcticPrimaryWorkTable::default(); + let arctic_row = ArcticPrimaryRow { + id: arctic.get_next_pk().into(), + value: 2, + }; + let arctic_pk = arctic.insert(arctic_row.clone()).unwrap(); + assert_eq!(arctic.select(arctic_pk), Some(arctic_row)); +} + +#[tokio::test] +async fn upstream_indexset_survives_persist_reload_and_more_writes() { + const ROOT: &str = "tests/data/index_backend_upstream_runtime"; + remove_dir_if_exists(ROOT.to_string()).await; + + let config = DiskConfig::new_with_table_name( + ROOT, + PersistedUpstreamWorkTable::name_snake_case(), + PersistedUpstreamWorkTable::version(), + ); + let engine = PersistedUpstreamPersistenceEngine::new(config.clone()).await.unwrap(); + let table = PersistedUpstreamWorkTable::load(engine).await.unwrap(); + + for unique_key in 0..1_024 { + table + .insert(PersistedUpstreamRow { + id: table.get_next_pk().into(), + unique_key, + }) + .unwrap(); + } + table.wait_for_ops().await; + drop(table); + + let engine = PersistedUpstreamPersistenceEngine::new(config.clone()).await.unwrap(); + let table = PersistedUpstreamWorkTable::load(engine).await.unwrap(); + assert_eq!(table.count(), 1_024); + assert_eq!(table.select_by_unique_key(777).unwrap().unique_key, 777); + + let added_pk = table + .insert(PersistedUpstreamRow { + id: table.get_next_pk().into(), + unique_key: 2_000, + }) + .unwrap(); + let added_id: u64 = added_pk.clone().into(); + table.delete(10).await.unwrap(); + table.wait_for_ops().await; + drop(table); + + let engine = PersistedUpstreamPersistenceEngine::new(config).await.unwrap(); + let table = PersistedUpstreamWorkTable::load(engine).await.unwrap(); + assert_eq!(table.count(), 1_024); + assert!(table.select(10).is_none()); + assert_eq!(table.select(added_pk).unwrap().unique_key, 2_000); + assert_eq!(table.select_by_unique_key(2_000).unwrap().id, added_id); + drop(table); + + remove_dir_if_exists(ROOT.to_string()).await; +} + +#[tokio::test] +async fn persisted_tables_can_switch_between_wti_and_upstream_without_rebuild() { + use provider_switch_upstream as upstream; + use provider_switch_wti as wti; + + const ROOT: &str = "tests/data/index_backend_provider_switch_runtime"; + remove_dir_if_exists(ROOT.to_string()).await; + + let wti_config = DiskConfig::new_with_table_name( + ROOT, + wti::ProviderSwitchWorkTable::name_snake_case(), + wti::ProviderSwitchWorkTable::version(), + ); + let engine = wti::ProviderSwitchPersistenceEngine::new(wti_config.clone()) + .await + .unwrap(); + let table = wti::ProviderSwitchWorkTable::load(engine).await.unwrap(); + for unique_key in 0..1_024 { + table + .insert(wti::ProviderSwitchRow { + id: table.get_next_pk().into(), + unique_key, + }) + .unwrap(); + } + table.wait_for_ops().await; + drop(table); + + let upstream_config = DiskConfig::new_with_table_name( + ROOT, + upstream::ProviderSwitchWorkTable::name_snake_case(), + upstream::ProviderSwitchWorkTable::version(), + ); + let engine = upstream::ProviderSwitchPersistenceEngine::new(upstream_config) + .await + .unwrap(); + let table = upstream::ProviderSwitchWorkTable::load(engine).await.unwrap(); + assert_eq!(table.count(), 1_024); + assert_eq!(table.select_by_unique_key(600).unwrap().unique_key, 600); + table.delete(10).await.unwrap(); + table + .insert(upstream::ProviderSwitchRow { + id: table.get_next_pk().into(), + unique_key: 2_000, + }) + .unwrap(); + table.wait_for_ops().await; + drop(table); + + let engine = wti::ProviderSwitchPersistenceEngine::new(wti_config).await.unwrap(); + let table = wti::ProviderSwitchWorkTable::load(engine).await.unwrap(); + assert_eq!(table.count(), 1_024); + assert!(table.select(10).is_none()); + assert_eq!(table.select_by_unique_key(2_000).unwrap().unique_key, 2_000); + drop(table); + + remove_dir_if_exists(ROOT.to_string()).await; +} diff --git a/tests/worktable/mod.rs b/tests/worktable/mod.rs index b4db3042..e0c9c0e4 100644 --- a/tests/worktable/mod.rs +++ b/tests/worktable/mod.rs @@ -8,6 +8,7 @@ mod delete; mod float; mod in_place; mod index; +mod index_backends; mod nid; mod option; mod tuple_primary_key; From 114574f0f005560712a738655857d39818c2c262 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 08:26:10 +0700 Subject: [PATCH 08/23] docs: define using semantics and benchmark handoff --- README.md | 4 +- docs/index-backend-dsl-proposal.md | 232 +++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 docs/index-backend-dsl-proposal.md diff --git a/README.md b/README.md index d2d09f3b..fbcf1cb8 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ cargo add worktable | **Embedded optimized** | Very low overhead, built for resource-sensitive environments. | | **Typed tables from a macro** | `worktable!` generates the table, row and primary-key types. No hand-written boilerplate per table. | | **Primary and secondary indexes** | Autoincrement or supplied primary keys; unique and non-unique secondary indexes, each adding a `select_by_` method. | +| **Per-index physical selection** | An optional `using` clause can statically select WorkTablesIndex, vanilla IndexSet, Congee, or Arctic where their capabilities fit. See [the backend guide](docs/index-backend-dsl-proposal.md). | | **Generated queries** | `select`, `insert`, `upsert`, `update`, `delete` and a `select_all` query builder on every table, plus the custom update/delete queries you declare. | | **Paged in-memory storage** | Records live in `DataPages` with a free list for reuse. `rkyv` gives zero-copy access to archived rows. | | **Concurrency** | Lock-free concurrent indexes with change-data-capture, plus a row-level `LockMap` for ordered access. | @@ -46,6 +47,8 @@ S3 support layers *on top of* the disk engine rather than replacing it. worktable = { version = "0.9", features = ["s3-support"] } # S3 sync, optional ``` +Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic are explicitly memory-only and require `persist: false`. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md). + ## Relationship to `data_bucket` WorkTable is built on [`data_bucket`](https://crates.io/crates/data_bucket), which @@ -396,4 +399,3 @@ enum WorkTableError Check out - [Examples](./examples) - diff --git a/docs/index-backend-dsl-proposal.md b/docs/index-backend-dsl-proposal.md new file mode 100644 index 00000000..1c447bfd --- /dev/null +++ b/docs/index-backend-dsl-proposal.md @@ -0,0 +1,232 @@ +# Per-index backends with `using` + +**Status:** Experimental implementation in draft PR #187 + +**Default:** `worktables_index` + +**Backends in this change:** `worktables_index`, `indexset`, `congee`, `arctic` + +## Why this exists + +WorkTable no longer has to make one physical-index tradeoff for every access path. The optional `using` modifier selects a concrete backend for a primary or unique secondary index while preserving the generated WorkTable API. + +The generated table contains concrete map types. Selection is resolved by the macro; there is no runtime backend enum, trait object, virtual call, or per-lookup selection branch. + +This has two distinct uses: + +- **Production migration:** persisted tables can run WorkTablesIndex and vanilla IndexSet in parallel and select either per index. Both use the existing WorkTablesIndex/DataBucket disk representation, so changing the provider does not require rebuilding table data. +- **Research and measurement:** explicitly memory-only tables can test Congee or Arctic on access paths where their ART layouts may beat a B-tree. + +The useful paper claim is not that WorkTable bundles several maps. It is that a generated table can statically select a physical implementation per access path, keep a stable typed API, and reject incompatible persistence or key semantics at compile time. + +## Syntax + +`using` is optional on a primary-key declaration and on an index declaration: + +```rust +worktable!( + name: Order, + persist: false, + columns: { + id: u64 primary_key autoincrement using congee, + account_id: u64, + sequence: u64, + public_id: PackedNanoid, + }, + indexes: { + account_idx: account_id unique using arctic, + sequence_idx: sequence unique using indexset, + public_id_idx: public_id unique using worktables_index, + }, +); +``` + +There is no separate `config` syntax for this feature. The physical choice stays next to the access path it controls. + +### The absent-`using` default + +Omitting `using` always means `worktables_index`: + +```rust +columns: { + id: u64 primary_key autoincrement, // WorkTablesIndex +}, +indexes: { + account_idx: account_id unique, // WorkTablesIndex +} +``` + +The explicit equivalent is: + +```rust +columns: { + id: u64 primary_key autoincrement using worktables_index, +}, +indexes: { + account_idx: account_id unique using worktables_index, +} +``` + +This default is intentional. Vanilla `indexset` is an explicit fourth backend; it is not the default and does not silently replace WorkTablesIndex. + +## Persistence is controlled by the existing `persist` declaration + +No new persistence keyword is introduced. + +| Declaration | Meaning | Allowed backends | +|---|---|---| +| `persist` omitted | Existing non-persisted table behavior | WorkTablesIndex or vanilla IndexSet; ART use is rejected until `persist: false` is explicit | +| `persist: false` | Explicitly memory-only | All four backends, subject to key and uniqueness constraints | +| `persist: true` | Local durable persistence plus in-memory indexes | WorkTablesIndex and vanilla IndexSet | +| S3 support | Existing S3 sync layered over local persistence | Same restrictions as `persist: true` | + +Congee and Arctic require the literal `persist: false`. Omitting `persist` is not sufficient acknowledgement. This makes a memory-only physical choice visible during review: + +```rust +worktable!( + name: QuoteCache, + persist: false, + columns: { + id: u64 primary_key autoincrement using congee, + symbol_id: u64, + }, + indexes: { + symbol_idx: symbol_id unique using arctic, + }, +); +``` + +The macro rejects the same schema with `persist: true`, and rejects it when `persist` is omitted. + +## Current capability matrix + +| Capability | `worktables_index` | `indexset` | `congee` | `arctic` | +|---|---:|---:|---:|---:| +| Primary index | Yes | Yes | Yes | Yes | +| Unique secondary index | Yes | Yes | Yes | Yes | +| Non-unique secondary index | Yes | No | No | No | +| Persisted local disk | Yes | Yes | No | No | +| Existing S3 persistence path | Yes | Yes | No | No | +| Variable-sized keys | Yes | Not in this change | No | No | +| Ordered point/range API | Yes | Yes | Adapter snapshot for scans | Adapter snapshot for scans | +| Default when `using` is absent | Yes | No | No | No | + +Alternative backends currently require `unique`. A non-unique declaration such as `value_idx: value using arctic` fails at macro expansion and tells the author to use `worktables_index`. + +### Key constraints + +- **Congee:** `u8`, `u16`, `u32`, `usize`, and `u64` on 64-bit targets. Its native key and payload are one machine word. Composite, NanoID, string, signed, and floating-point keys are rejected. +- **Arctic:** `u16`, `u32`, `u64`, and `u128` in this initial adapter. Its crate supports more representations, but WorkTable exposes only the shapes covered by the current contract tests. +- **Vanilla IndexSet:** sized ordered keys in this change. Variable-sized keys remain on WorkTablesIndex. +- **WorkTablesIndex:** retains the existing generic and variable-sized key support. + +WorkTable wraps a declared primary key in a generated newtype. For Congee and Arctic, code generation emits a lossless codec from that newtype to the supported native integer key. Unsupported primary-key shapes fail during macro expansion. + +NanoID is deliberately not claimed for either ART yet. Current NanoID/PackedNanoid indexes must use WorkTablesIndex or vanilla IndexSet until an order-preserving, contract-tested Arctic codec is added. UUID-specific work is outside this feature; downstream schemas can continue the separate UUID-to-NanoID migration. + +## Persistence and provider switching + +WorkTablesIndex remains the persistence format boundary used by DataBucket. The vanilla IndexSet adapter performs two normalizations: + +1. vanilla IndexSet structural CDC events are converted to the equivalent WorkTablesIndex event type before they reach persistence; +2. vanilla IndexSet node pairs are converted to and from the existing WorkTablesIndex/DataBucket page representation during snapshot and reload. + +The selected provider is therefore an in-memory implementation detail, not a new disk format. The test suite covers this sequence with split indexes: + +1. create and persist a table using WorkTablesIndex; +2. reload the same files using vanilla IndexSet; +3. delete and insert rows through vanilla IndexSet and persist the resulting CDC; +4. reload the same files using WorkTablesIndex again. + +It also separately covers vanilla IndexSet persist → reload → mutate → reload. This is the technical basis for deploying the two providers in parallel without a full data rebuild. + +This is still a sensitive storage path. Production rollout should retain backups, verify the exact downstream schema/version, and run crash/torn-write and sustained post-reload mutation tests before changing a live table. + +## Hot-path and performance details + +Backend dispatch itself is static and should compile away. That does **not** mean every adapter operation has identical cost. + +### WorkTablesIndex and vanilla IndexSet + +- Point operations call the selected B-tree directly. +- Ordered iteration and ranges stream from the selected tree. +- `OffsetEqLink` values are copied out of backend guards; no heap allocation is introduced for a point lookup. +- Both can emit persistence-compatible structural CDC. + +### Congee + +- Point lookup and mutation call Congee directly. +- WorkTable links do not fit in Congee's one-word payload. The adapter stores an `Arc` pointer, so inserts allocate and reads clone the `Arc` before copying the link. +- `iter_values` and `range_values` currently collect a full key/value snapshot and sort it. A narrow range therefore scans and allocates for the whole index. + +### Arctic + +- Point lookup and mutation call Arctic directly. +- WorkTable links are stored in `Box` values because Arctic's inline value is limited to 64 bits. Inserts allocate; reads copy the link from the box. +- Ordered scans currently collect all entries into a `Vec` and then filter the requested range. +- Concurrent scan behavior inherits Arctic's non-linearizable traversal contract. + +The ART adapters are consequently candidates for point-heavy, explicitly memory-only paths—not automatic wins for `select_all`, range-heavy access, iteration, or vacuum. Tomorrow's measurements must separate point lookup, write/allocation cost, range width, full iteration, and reclamation rather than reporting one blended throughput number. + +### Memory diagnostics + +WorkTablesIndex and vanilla IndexSet expose node capacity and topology used by existing `system_info` reporting. Congee and Arctic do not expose equivalent stable allocator statistics. For those adapters: + +- reported used/heap bytes are only a payload-size lower bound; +- reported capacity equals logical length; +- reported node count is zero/unknown. + +Use allocator/RSS measurements for comparative memory results; do not treat the ART `system_info` fields as total resident memory. + +## Dependency and fork status + +This implementation uses the published crates directly: + +- `WorkTablesIndex 0.0.1` as the default `indexset` dependency alias already used by WorkTable; +- vanilla `indexset 0.15.0` under the `vanilla_indexset` Cargo name; +- `congee 0.4.1` through its public `Congee`, `compute_or_insert`, and `new_with_drainer` APIs; +- `arctic-map 0.1.4` through its public concurrent map API. + +There is no Congee fork, Cargo patch, or direct `CongeeArc` dependency in this PR. The WorkTable adapter supplies the checked-insert and pointer-lifetime behavior it needs using vanilla Congee's public API. + +## Correctness coverage in this PR + +The implementation includes: + +- parser/default tests for all four names; +- compile-time rejection of persisted ARTs, implicit memory-only ARTs, alternative non-unique indexes, and unsupported key shapes; +- shared unique-index contract tests for WorkTablesIndex and vanilla IndexSet; +- adapter contract and concurrent checked-insert tests for Congee and Arctic; +- a generated table using all four providers simultaneously; +- generated primary-key CRUD tests for vanilla IndexSet, Congee, and Arctic; +- vanilla IndexSet persist/reload/post-reload mutation coverage; +- WorkTablesIndex → vanilla IndexSet → WorkTablesIndex disk-provider switching coverage. + +These are correctness gates, not performance evidence. + +## Benchmark handoff + +Use draft PR #187 and vary only `using` between otherwise identical generated schemas. The first downstream targets are `web3.trading-backend` and `agencyzero`. + +Minimum useful ARM campaign: + +1. sequential autoincrement `u64` primary key; +2. unique `u64` secondary key with the production hit/miss distribution; +3. 1 thread and representative contended thread counts; +4. point-read, insert, delete, and production mixed traces measured separately; +5. range widths 1, 8, 64, and 1,024 plus full iteration; +6. p50, p99, throughput, allocations/op, RSS/entry, and post-churn reclamation; +7. persisted WorkTablesIndex versus persisted vanilla IndexSet, including reload and writes after reload; +8. memory-only Congee and Arctic only where `persist: false` is operationally valid. + +Run release builds on the actual ARM deployment class. SIMD should not be treated as a reason to prefer a backend; any x86 result is a portability check, not the primary HFT decision. + +For the paper, the strongest controlled experiment keeps the WorkTable schema, generated methods, data set, and operation trace fixed and changes one `using` clause. Include this feature only if an end-to-end WorkTable workload shows a material, repeatable gain or memory reduction without weakening the required persistence and scan semantics. + +## Production versus research classification + +- **WorkTablesIndex:** production default. +- **Vanilla IndexSet:** potential production migration backend because it preserves local/S3 persistence through the existing format boundary; still requires downstream stress and performance validation. +- **Congee and Arctic:** research/experimental memory-only backends in this PR. Promotion requires relevant downstream evidence, allocation/reclamation review, and a workload that does not depend on the current allocating scan path. + +That boundary is deliberate: `using` exposes optional physical specialization without quietly weakening WorkTable's in-memory/on-disk coordination contract. From 0f822dd499971f19f52bfd78be79d79216b3c283 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 14:16:54 +0700 Subject: [PATCH 09/23] fix: harden index backend beta --- .github/workflows/rust.yml | 22 ++- Cargo.toml | 10 +- README.md | 5 +- codegen/Cargo.toml | 2 +- codegen/src/common/parser/name.rs | 1 + docs/index-backend-dsl-proposal.md | 13 +- .../codegen/src/performance_measurement.rs | 3 +- src/in_memory/empty_link_registry.rs | 22 ++- src/index/congee.rs | 24 ++-- src/index/unique.rs | 81 ++++++++++- tests/persistence/sync/many_strings.rs | 9 +- tests/worktable/index_backends.rs | 130 +++++++++++++++--- 12 files changed, 265 insertions(+), 57 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4e333751..a9fb5d54 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -9,6 +9,9 @@ on: env: CARGO_TERM_COLOR: always +permissions: + contents: read + jobs: build: @@ -16,15 +19,15 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: cache-on-failure: "true" add-job-id-key: "false" - name: Build - run: cargo build --verbose + run: cargo build --workspace --all-targets --all-features --verbose - name: Run tests - run: cargo test --verbose + run: cargo test --workspace --all-targets --all-features --verbose clippy_check: @@ -33,17 +36,10 @@ jobs: timeout-minutes: 15 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: cache-on-failure: "true" add-job-id-key: "false" - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - components: clippy - override: true - - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-targets --all-features -- -D warnings + - name: Clippy (deny warnings) + run: cargo clippy --workspace --all-targets --all-features -- -D warnings diff --git a/Cargo.toml b/Cargo.toml index 5b0d76d7..76a9dd19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "examples", "performance_measurement", "performance_measur [package] name = "worktable" -version = "0.9.4" +version = "1.0.0-beta.1" edition = "2024" authors = ["Handy-caT"] license = "MIT" @@ -25,14 +25,14 @@ async-trait = "0.1.89" arctic-map = "=0.1.4" congee = "=0.4.1" convert_case = "0.6.0" -data_bucket = "=0.4.1" +data_bucket = "=0.5.0" # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" } # data_bucket = { path = "../DataBucket", version = "0.3.14" } derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] } eyre = "0.6.12" fastrand = "2.3.0" futures = "0.3.30" -indexset = { package = "WorkTablesIndex", version = "=0.0.1", features = ["concurrent", "cdc", "multimap"] } +indexset = { package = "WorkTablesIndex", version = "=0.0.3", features = ["concurrent", "cdc", "multimap"] } vanilla_indexset = { package = "indexset", version = "=0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] } @@ -43,7 +43,7 @@ performance_measurement = { path = "performance_measurement", version = "0.1.0", performance_measurement_codegen = { path = "performance_measurement/codegen", version = "0.1.0", optional = true } prettytable-rs = "^0.10" psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] } -rkyv = { version = "0.8.9", features = ["uuid-1"] } +rkyv = { version = "0.8.17", features = ["uuid-1"] } reqwest = { version = "0.12", optional = true, default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2"] } rusty-s3 = { package = "rusty-s3-temp", version = "0.9.0", optional = true } smart-default = "0.7.1" @@ -52,7 +52,7 @@ tracing = "0.1" url = { version = "2", optional = true } uuid = { version = "1.10.0", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } -worktable_codegen = { path = "codegen", version = "=0.9.4" } +worktable_codegen = { path = "codegen", version = "=1.0.0-beta.1" } [dev-dependencies] chrono = "0.4.43" diff --git a/README.md b/README.md index fbcf1cb8..3e00fabd 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ from a macro, and that persisting it is one feature flag away. ## Install ```sh -cargo add worktable +cargo add worktable@1.0.0-beta.1 ``` ## What you get @@ -44,7 +44,7 @@ S3 support layers *on top of* the disk engine rather than replacing it. ```toml [dependencies] -worktable = { version = "0.9", features = ["s3-support"] } # S3 sync, optional +worktable = { version = "=1.0.0-beta.1", features = ["s3-support"] } # S3 sync, optional ``` Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic are explicitly memory-only and require `persist: false`. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md). @@ -398,4 +398,3 @@ enum WorkTableError ## Examples Check out - [Examples](./examples) - diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 05ace59c..c16dc503 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "worktable_codegen" -version = "0.9.4" +version = "1.0.0-beta.1" edition = "2024" license = "MIT" description = "Proc-macro companion crate for worktable: the worktable! macro and its derives." diff --git a/codegen/src/common/parser/name.rs b/codegen/src/common/parser/name.rs index 73d18b38..783e853d 100644 --- a/codegen/src/common/parser/name.rs +++ b/codegen/src/common/parser/name.rs @@ -148,6 +148,7 @@ mod tests { let mut parser = Parser::new(tokens); let name = parser.parse_name().unwrap(); + assert_eq!(name, "TestName"); let version = parser.parse_version().unwrap(); assert_eq!(version, Some(5)); diff --git a/docs/index-backend-dsl-proposal.md b/docs/index-backend-dsl-proposal.md index 1c447bfd..f900e841 100644 --- a/docs/index-backend-dsl-proposal.md +++ b/docs/index-backend-dsl-proposal.md @@ -153,6 +153,14 @@ Backend dispatch itself is static and should compile away. That does **not** mea - `OffsetEqLink` values are copied out of backend guards; no heap allocation is introduced for a point lookup. - Both can emit persistence-compatible structural CDC. +Direct dispatch does not strengthen either provider's concurrent-read +semantics. During unrelated structural mutations, WorkTablesIndex and vanilla +IndexSet can transiently return a point miss for a key that is present after +the writers quiesce. This PR verifies mutation integrity and quiescent reads; +it does not claim linearizable point reads for those providers. Workloads that +require stable concurrent visibility need the separate row-publication and +stable-miss protocol before their results are publishable. + ### Congee - Point lookup and mutation call Congee directly. @@ -182,7 +190,7 @@ Use allocator/RSS measurements for comparative memory results; do not treat the This implementation uses the published crates directly: -- `WorkTablesIndex 0.0.1` as the default `indexset` dependency alias already used by WorkTable; +- `WorkTablesIndex 0.0.3` as the default `indexset` dependency alias already used by WorkTable; - vanilla `indexset 0.15.0` under the `vanilla_indexset` Cargo name; - `congee 0.4.1` through its public `Congee`, `compute_or_insert`, and `new_with_drainer` APIs; - `arctic-map 0.1.4` through its public concurrent map API. @@ -195,8 +203,9 @@ The implementation includes: - parser/default tests for all four names; - compile-time rejection of persisted ARTs, implicit memory-only ARTs, alternative non-unique indexes, and unsupported key shapes; -- shared unique-index contract tests for WorkTablesIndex and vanilla IndexSet; +- shared unique-index contract and concurrent mutation-integrity tests for all four providers; - adapter contract and concurrent checked-insert tests for Congee and Arctic; +- immediate disjoint insert/read/remove tests for Congee and Arctic; - a generated table using all four providers simultaneously; - generated primary-key CRUD tests for vanilla IndexSet, Congee, and Arctic; - vanilla IndexSet persist/reload/post-reload mutation coverage; diff --git a/performance_measurement/codegen/src/performance_measurement.rs b/performance_measurement/codegen/src/performance_measurement.rs index ca6e1321..16ccd9c7 100644 --- a/performance_measurement/codegen/src/performance_measurement.rs +++ b/performance_measurement/codegen/src/performance_measurement.rs @@ -80,14 +80,13 @@ pub fn parse_name(iter: &mut token_stream::IntoIter, attr: &TokenStream) -> syn: #[cfg(test)] mod tests { - use proc_macro2::TokenStream; use quote::quote; use super::parse_attr; #[test] fn test_attr_parse() { - let tokens = TokenStream::from(quote! {prefix_name = "Test"}); + let tokens = quote! {prefix_name = "Test"}; let attr = parse_attr(tokens).unwrap(); assert_eq!(attr.name, "Test".to_string()) diff --git a/src/in_memory/empty_link_registry.rs b/src/in_memory/empty_link_registry.rs index 3030420f..c750414e 100644 --- a/src/in_memory/empty_link_registry.rs +++ b/src/in_memory/empty_link_registry.rs @@ -173,11 +173,12 @@ impl EmptyLinkRegistry { let mut iter = self.length_ord_links.iter().rev(); let (_, max_length_link) = iter.next()?; + let max_length_link = *max_length_link; drop(iter); - self.remove_link(*max_length_link); + self.remove_link(max_length_link); - Some(*max_length_link) + Some(max_length_link) } pub fn iter(&self) -> impl Iterator + '_ { @@ -388,6 +389,23 @@ mod tests { assert_eq!(registry.pop_max().unwrap().length, 50); } + #[test] + fn test_pop_max_preserves_link_across_repeated_removal() { + let registry = EmptyLinkRegistry::::default(); + + for page_id in 1..=10_000_u32 { + let link = Link { + page_id: page_id.into(), + offset: page_id, + length: page_id % 1_024 + 1, + }; + registry.push(link); + assert_eq!(registry.pop_max(), Some(link)); + } + + assert!(registry.pop_max().is_none()); + } + #[test] fn test_iter_returns_all_links() { let registry = EmptyLinkRegistry::::default(); diff --git a/src/index/congee.rs b/src/index/congee.rs index dc76dd21..946c7d75 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -66,7 +66,7 @@ where let drainer = |_key: usize, pointer: usize| { // SAFETY: every payload inserted below originates from // `Arc::into_raw` with exactly one tree-owned strong reference. - drop(unsafe { Arc::from_raw(pointer as *const V) }); + drop(unsafe { Self::arc_from_pointer(pointer) }); }; Self { inner: Congee::new_with_drainer(DefaultAllocator {}, drainer), @@ -81,11 +81,19 @@ where K: CongeeKey, V: Clone + Debug + Send + Sync + 'static, { + #[inline] + unsafe fn arc_from_pointer(pointer: usize) -> Arc { + // SAFETY: callers guarantee that `pointer` was produced by + // `Arc::into_raw(...).expose_provenance()` for the same `V` and still + // owns one strong reference. + unsafe { Arc::from_raw(std::ptr::with_exposed_provenance(pointer)) } + } + #[inline] fn clone_pointer(pointer: usize) -> Arc { // SAFETY: the caller holds a Congee epoch guard, so the tree-owned // strong reference cannot be reclaimed while it is cloned. - let owned = unsafe { Arc::from_raw(pointer as *const V) }; + let owned = unsafe { Self::arc_from_pointer(pointer) }; let cloned = Arc::clone(&owned); let _ = Arc::into_raw(owned); cloned @@ -95,7 +103,7 @@ where fn retire_old(pointer: usize, guard: &congee::epoch::Guard) -> Arc { // SAFETY: a successful replacement/removal transfers the tree-owned // strong reference to this call. - let owned = unsafe { Arc::from_raw(pointer as *const V) }; + let owned = unsafe { Self::arc_from_pointer(pointer) }; let delayed = Arc::clone(&owned); guard.defer(move || drop(delayed)); owned @@ -122,7 +130,7 @@ where #[inline] fn insert_value(&self, key: K, value: V) -> Option { let guard = self.inner.pin(); - let pointer = Arc::into_raw(Arc::new(value)) as usize; + let pointer = Arc::into_raw(Arc::new(value)).expose_provenance(); match self.inner.insert(key.into_congee(), pointer, &guard) { Ok(Some(old)) => Some(Self::retire_old(old, &guard).as_ref().clone()), Ok(None) => { @@ -131,7 +139,7 @@ where } Err(_) => { // SAFETY: insertion failed, so ownership never transferred. - drop(unsafe { Arc::from_raw(pointer as *const V) }); + drop(unsafe { Self::arc_from_pointer(pointer) }); Self::allocation_failure() } } @@ -140,7 +148,7 @@ where #[inline] fn insert_value_checked(&self, key: K, value: V) -> Option<()> { let guard = self.inner.pin(); - let pointer = Arc::into_raw(Arc::new(value)) as usize; + let pointer = Arc::into_raw(Arc::new(value)).expose_provenance(); let result = self .inner .compute_or_insert(key.into_congee(), |old| old.unwrap_or(pointer), &guard); @@ -149,7 +157,7 @@ where Ok(Some(_)) => { // The closure returned the existing pointer, so the new value // was never installed. - drop(unsafe { Arc::from_raw(pointer as *const V) }); + drop(unsafe { Self::arc_from_pointer(pointer) }); None } Ok(None) => { @@ -158,7 +166,7 @@ where } Err(_) => { // SAFETY: insertion failed, so ownership never transferred. - drop(unsafe { Arc::from_raw(pointer as *const V) }); + drop(unsafe { Self::arc_from_pointer(pointer) }); Self::allocation_failure() } } diff --git a/src/index/unique.rs b/src/index/unique.rs index fedb6fd5..4b15df42 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -169,8 +169,10 @@ pub type UpstreamIndexPair = VanillaPair; #[cfg(test)] mod tests { + use std::sync::Arc; + use super::{UniqueIndex, UpstreamIndexMap}; - use crate::IndexMap; + use crate::{ArcticIndex, CongeeIndex, IndexMap}; fn assert_unique_index_contract() where @@ -191,6 +193,69 @@ mod tests { assert_eq!(index.len(), 1); } + fn assert_disjoint_concurrent_insert_then_remove() + where + I: UniqueIndex + Send + Sync + 'static, + { + let index = Arc::new(I::default()); + let mut threads = Vec::new(); + for worker in 0..8_u64 { + let index = Arc::clone(&index); + threads.push(std::thread::spawn(move || { + for sequence in 0..1_000_u64 { + let key = worker * 1_000 + sequence; + assert_eq!(index.insert_value_checked(key, key + 1), Some(())); + } + })); + } + for thread in threads { + thread.join().unwrap(); + } + + assert_eq!(index.len(), 8_000); + for key in 0..8_000_u64 { + assert_eq!(index.get_value(&key), Some(key + 1)); + } + + let mut threads = Vec::new(); + for worker in 0..8_u64 { + let index = Arc::clone(&index); + threads.push(std::thread::spawn(move || { + for sequence in 0..1_000_u64 { + let key = worker * 1_000 + sequence; + assert_eq!(index.remove_value(&key), Some((key, key + 1))); + } + })); + } + for thread in threads { + thread.join().unwrap(); + } + assert!(index.is_empty()); + } + + fn assert_immediate_disjoint_crud() + where + I: UniqueIndex + Send + Sync + 'static, + { + let index = Arc::new(I::default()); + let mut threads = Vec::new(); + for worker in 0..8_u64 { + let index = Arc::clone(&index); + threads.push(std::thread::spawn(move || { + for sequence in 0..1_000_u64 { + let key = worker * 1_000 + sequence; + assert_eq!(index.insert_value_checked(key, key + 1), Some(())); + assert_eq!(index.get_value(&key), Some(key + 1)); + assert_eq!(index.remove_value(&key), Some((key, key + 1))); + } + })); + } + for thread in threads { + thread.join().unwrap(); + } + assert!(index.is_empty()); + } + #[test] fn worktables_index_implements_contract() { assert_unique_index_contract::>(); @@ -200,4 +265,18 @@ mod tests { fn upstream_indexset_implements_contract() { assert_unique_index_contract::>(); } + + #[test] + fn all_backends_preserve_disjoint_concurrent_mutations() { + assert_disjoint_concurrent_insert_then_remove::>(); + assert_disjoint_concurrent_insert_then_remove::>(); + assert_disjoint_concurrent_insert_then_remove::>(); + assert_disjoint_concurrent_insert_then_remove::>(); + } + + #[test] + fn art_backends_make_disjoint_mutations_immediately_visible() { + assert_immediate_disjoint_crud::>(); + assert_immediate_disjoint_crud::>(); + } } diff --git a/tests/persistence/sync/many_strings.rs b/tests/persistence/sync/many_strings.rs index 28b1b9e3..ad4fece7 100644 --- a/tests/persistence/sync/many_strings.rs +++ b/tests/persistence/sync/many_strings.rs @@ -130,7 +130,14 @@ fn test_space_update_query_pk_many_times_sync() { { let engine = TestSyncPersistenceEngine::new(config.clone()).await.unwrap(); let table = TestSyncWorkTable::load(engine).await.unwrap(); - assert!(table.select(pk.clone()).is_some()); + if table.select(pk.clone()).is_none() { + let direct = table.0.primary_index.pk_map.get_value(&TestSyncPrimaryKey(pk.clone())); + let entries = table.0.primary_index.pk_map.iter_values().collect::>(); + panic!( + "final primary lookup missed: direct={direct:?}, entries={entries:?}, row_count={}", + table.count() + ); + } assert_eq!(table.select(pk.clone()).unwrap().another, 511); assert_eq!(table.select(pk).unwrap().field, "Some field value".to_string()); } diff --git a/tests/worktable/index_backends.rs b/tests/worktable/index_backends.rs index 6330632e..dec823a1 100644 --- a/tests/worktable/index_backends.rs +++ b/tests/worktable/index_backends.rs @@ -73,6 +73,15 @@ worktable! { }, } +worktable! { + name: CongeePrimary, + persist: false, + columns: { + id: u64 primary_key autoincrement using congee, + value: u64, + }, +} + worktable! { name: PersistedUpstream, persist: true, @@ -85,8 +94,8 @@ worktable! { }, } -#[test] -fn all_unique_backends_coexist_in_one_table() { +#[tokio::test] +async fn all_unique_backends_support_crud_ranges_and_conflict_rollback() { let table = MixedBackendWorkTable::default(); let row = MixedBackendRow { id: table.get_next_pk().into(), @@ -97,30 +106,111 @@ fn all_unique_backends_coexist_in_one_table() { }; let pk = table.insert(row.clone()).unwrap(); + let second = MixedBackendRow { + id: table.get_next_pk().into(), + wti_key: 31, + upstream_key: 32, + congee_key: 33, + arctic_key: 34, + }; + table.insert(second.clone()).unwrap(); + assert_eq!(table.select(pk), Some(row.clone())); assert_eq!(table.select_by_wti_key(11), Some(row.clone())); assert_eq!(table.select_by_upstream_key(12), Some(row.clone())); assert_eq!(table.select_by_congee_key(13), Some(row.clone())); - assert_eq!(table.select_by_arctic_key(14), Some(row)); -} + assert_eq!(table.select_by_arctic_key(14), Some(row.clone())); -#[test] -fn alternative_primary_backends_support_point_crud() { - let upstream = UpstreamPrimaryWorkTable::default(); - let upstream_row = UpstreamPrimaryRow { - id: upstream.get_next_pk().into(), - value: 1, - }; - let upstream_pk = upstream.insert(upstream_row.clone()).unwrap(); - assert_eq!(upstream.select(upstream_pk), Some(upstream_row)); + for (attempt, duplicate_backend) in ["wti", "indexset", "congee", "arctic"].into_iter().enumerate() { + let base = 100 + attempt as u64 * 10; + let candidate = MixedBackendRow { + id: table.get_next_pk().into(), + wti_key: if duplicate_backend == "wti" { 11 } else { base + 1 }, + upstream_key: if duplicate_backend == "indexset" { 12 } else { base + 2 }, + congee_key: if duplicate_backend == "congee" { 13 } else { base + 3 }, + arctic_key: if duplicate_backend == "arctic" { 14 } else { base + 4 }, + }; + assert!(table.insert(candidate.clone()).is_err()); + assert!(table.select(candidate.id).is_none()); + if candidate.wti_key != 11 { + assert!(table.select_by_wti_key(candidate.wti_key).is_none()); + } + if candidate.upstream_key != 12 { + assert!(table.select_by_upstream_key(candidate.upstream_key).is_none()); + } + if candidate.congee_key != 13 { + assert!(table.select_by_congee_key(candidate.congee_key).is_none()); + } + if candidate.arctic_key != 14 { + assert!(table.select_by_arctic_key(candidate.arctic_key).is_none()); + } + } + assert_eq!(table.count(), 2); - let arctic = ArcticPrimaryWorkTable::default(); - let arctic_row = ArcticPrimaryRow { - id: arctic.get_next_pk().into(), - value: 2, + let updated = MixedBackendRow { + id: row.id, + wti_key: 21, + upstream_key: 22, + congee_key: 23, + arctic_key: 24, }; - let arctic_pk = arctic.insert(arctic_row.clone()).unwrap(); - assert_eq!(arctic.select(arctic_pk), Some(arctic_row)); + table.update(updated.clone()).await.unwrap(); + assert_eq!(table.select(pk), Some(updated.clone())); + assert!(table.select_by_wti_key(11).is_none()); + assert!(table.select_by_upstream_key(12).is_none()); + assert!(table.select_by_congee_key(13).is_none()); + assert!(table.select_by_arctic_key(14).is_none()); + assert_eq!(table.select_by_wti_key(21), Some(updated.clone())); + assert_eq!(table.select_by_upstream_key(22), Some(updated.clone())); + assert_eq!(table.select_by_congee_key(23), Some(updated.clone())); + assert_eq!(table.select_by_arctic_key(24), Some(updated.clone())); + + for rows in [ + table.select_by_wti_key_range(20..=31).execute().unwrap(), + table.select_by_upstream_key_range(20..=32).execute().unwrap(), + table.select_by_congee_key_range(20..=33).execute().unwrap(), + table.select_by_arctic_key_range(20..=34).execute().unwrap(), + ] { + assert_eq!(rows.len(), 2); + } + + table.delete(row.id).await.unwrap(); + assert!(table.select(pk).is_none()); + assert!(table.select_by_wti_key(21).is_none()); + assert!(table.select_by_upstream_key(22).is_none()); + assert!(table.select_by_congee_key(23).is_none()); + assert!(table.select_by_arctic_key(24).is_none()); + assert_eq!(table.count(), 1); +} + +#[tokio::test] +async fn alternative_primary_backends_support_point_crud() { + macro_rules! assert_point_crud { + ($table:ty, $row:ident) => {{ + let table = <$table>::default(); + let original = $row { + id: table.get_next_pk().into(), + value: 1, + }; + let pk = table.insert(original.clone()).unwrap(); + assert_eq!(table.select(pk.clone()), Some(original.clone())); + + let updated = $row { + id: original.id, + value: 2, + }; + table.update(updated.clone()).await.unwrap(); + assert_eq!(table.select(pk.clone()), Some(updated)); + + table.delete(original.id).await.unwrap(); + assert!(table.select(pk).is_none()); + assert_eq!(table.count(), 0); + }}; + } + + assert_point_crud!(UpstreamPrimaryWorkTable, UpstreamPrimaryRow); + assert_point_crud!(CongeePrimaryWorkTable, CongeePrimaryRow); + assert_point_crud!(ArcticPrimaryWorkTable, ArcticPrimaryRow); } #[tokio::test] @@ -169,6 +259,7 @@ async fn upstream_indexset_survives_persist_reload_and_more_writes() { assert!(table.select(10).is_none()); assert_eq!(table.select(added_pk).unwrap().unique_key, 2_000); assert_eq!(table.select_by_unique_key(2_000).unwrap().id, added_id); + table.wait_for_ops().await; drop(table); remove_dir_if_exists(ROOT.to_string()).await; @@ -228,6 +319,7 @@ async fn persisted_tables_can_switch_between_wti_and_upstream_without_rebuild() assert_eq!(table.count(), 1_024); assert!(table.select(10).is_none()); assert_eq!(table.select_by_unique_key(2_000).unwrap().unique_key, 2_000); + table.wait_for_ops().await; drop(table); remove_dir_if_exists(ROOT.to_string()).await; From ba6a8d0a3c864207201be16fed06bfce3d0629d1 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 14:33:39 +0700 Subject: [PATCH 10/23] fix: coalesce reused persistence slots --- src/persistence/operation/batch.rs | 103 ++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 25 deletions(-) diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index 914d0655..9f06b1cd 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -66,6 +66,43 @@ impl From for BatchInnerRow { } } +/// Coalesces durable row writes by physical storage slot. +/// +/// `Link::length` can change when an unsized row is reinserted into a reused +/// `(page_id, offset)`. Treating the two lengths as different keys leaves +/// overlapping writes in the same batch, whose eventual application order is +/// derived from a hash map. The newest operation must be the only write for a +/// physical slot. +fn latest_data_writes( + ops: &[Operation], +) -> BatchData { + let mut latest: HashMap<(PageId, u32), (OperationId, Link, Vec)> = HashMap::new(); + for op in ops { + let Some(bytes) = op.bytes() else { + continue; + }; + let link = op.link(); + let operation_id = op.operation_id(); + let key = (link.page_id, link.offset); + match latest.entry(key) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + if operation_id > entry.get().0 { + entry.insert((operation_id, link, bytes.to_vec())); + } + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert((operation_id, link, bytes.to_vec())); + } + } + } + + let mut data = HashMap::new(); + for (_, (_, link, bytes)) in latest { + data.entry(link.page_id).or_insert_with(Vec::new).push((link, bytes)); + } + data +} + #[derive(Debug)] pub struct BatchOperation { ops: Vec>, @@ -372,31 +409,47 @@ where } pub fn get_batch_data_op(&self) -> eyre::Result { - let mut data = HashMap::new(); - for link in self.info_wt.iter_links() { - let last_op = self - .info_wt - .select_by_link(link) - .order_on(BatchInnerRowFields::OperationId, Order::Desc) - .limit(1) - .execute()?; - let op_row = last_op - .into_iter() - .next() - .expect("if link is in info_wt at least one row exists"); - let pos = op_row.pos; - let op = self - .ops - .get(pos) - .expect("pos should be correct as was set while batch build"); - if let Some(data_bytes) = op.bytes() { - let link = op.link(); - data.entry(link.page_id) - .and_modify(|v: &mut Vec<_>| v.push((link, data_bytes.to_vec()))) - .or_insert(vec![(link, data_bytes.to_vec())]); - } - } + Ok(latest_data_writes(&self.ops)) + } +} + +#[cfg(test)] +mod tests { + use data_bucket::Link; + use uuid::Uuid; + + use super::latest_data_writes; + use crate::persistence::operation::{InsertOperation, Operation, OperationId}; + + fn insert(id: u128, link: Link, bytes: Vec) -> Operation<(), u64, ()> { + Operation::Insert(InsertOperation { + id: OperationId::Single(Uuid::from_u128(id)), + primary_key_events: vec![], + secondary_keys_events: (), + pk_gen_state: (), + bytes, + link, + }) + } - Ok(data) + #[test] + fn variable_length_link_reuse_keeps_only_the_newest_physical_write() { + let old_link = Link { + page_id: 1.into(), + offset: 128, + length: 4, + }; + let new_link = Link { + page_id: 1.into(), + offset: 128, + length: 6, + }; + + // Deliberately reverse vector order: operation ids, not incidental + // collection order, define which bytes are newest. + let batch = latest_data_writes(&[insert(2, new_link, vec![2; 6]), insert(1, old_link, vec![1; 4])]); + let writes = batch.get(&1.into()).unwrap(); + + assert_eq!(writes, &vec![(new_link, vec![2; 6])]); } } From 5dd57799e2581684e30aeb2db050d445ccb8a91a Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 14:53:35 +0700 Subject: [PATCH 11/23] test: linearize vacuum oracle transitions --- tests/worktable/vacuum.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/worktable/vacuum.rs b/tests/worktable/vacuum.rs index df12d310..2e49cca6 100644 --- a/tests/worktable/vacuum.rs +++ b/tests/worktable/vacuum.rs @@ -168,10 +168,15 @@ async fn vacuum_parallel_with_upserts() { let delete_table = table.clone(); let ids_to_delete: Arc> = Arc::new(rows.iter().step_by(2).map(|p| p.0).collect()); let row_state = Arc::new(Mutex::new(rows.iter().cloned().collect::>())); + let row_locks = Arc::new((0..3000).map(|_| tokio::sync::Mutex::new(())).collect::>()); let task_ids = ids_to_delete.clone(); let task_row_state = Arc::clone(&row_state); + let task_row_locks = Arc::clone(&row_locks); let delete_task = tokio::spawn(async move { for id in task_ids.iter() { + // Keep the table mutation and oracle transition ordered for this + // key. Operations on different keys and vacuum remain concurrent. + let _row_guard = task_row_locks[*id as usize].lock().await; delete_table.delete(*id).await.unwrap(); { let mut g = task_row_state.lock(); @@ -189,6 +194,7 @@ async fn vacuum_parallel_with_upserts() { data: format!("test_data_{}", i), }; let id = row.id; + let _row_guard = row_locks[id as usize].lock().await; table.upsert(row.clone()).await.unwrap(); { let mut g = row_state.lock(); From 2958f5f380eda2ee19748dce371fc3a8fbed66fe Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 16:35:13 +0700 Subject: [PATCH 12/23] perf: preserve backend-neutral lookup latency --- Cargo.toml | 9 ++++-- README.md | 2 ++ .../src/generators/in_memory/table/impls.rs | 2 +- codegen/src/generators/persist/table/impls.rs | 2 +- src/index/arctic.rs | 10 +++++-- src/index/congee.rs | 23 +++++++------- src/index/unique.rs | 30 +++++++++++++++---- src/table/mod.rs | 2 +- 8 files changed, 56 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76a9dd19..4332c179 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,8 +15,13 @@ keywords = ["database", "embedded", "in-memory", "index", "storage"] categories = ["database-implementations", "data-structures", "caching"] [features] +default = ["wti-predictable-search"] perf_measurements = ["dep:performance_measurement", "dep:performance_measurement_codegen"] s3-support = ["dep:rusty-s3", "dep:url", "dep:reqwest", "dep:walkdir", "worktable_codegen/s3-support"] +wti-hybrid-search = ["indexset/wt-slice-binary-search"] +wti-predictable-search = ["indexset/custom-binary-search"] +wti-std-search = ["indexset/std-binary-search"] +wti-superslice-search = ["indexset/superslice-binary-search"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -25,14 +30,14 @@ async-trait = "0.1.89" arctic-map = "=0.1.4" congee = "=0.4.1" convert_case = "0.6.0" -data_bucket = "=0.5.0" +data_bucket = "=0.5.1" # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" } # data_bucket = { path = "../DataBucket", version = "0.3.14" } derive_more = { version = "2.0.1", features = ["from", "error", "display", "debug", "into"] } eyre = "0.6.12" fastrand = "2.3.0" futures = "0.3.30" -indexset = { package = "WorkTablesIndex", version = "=0.0.3", features = ["concurrent", "cdc", "multimap"] } +indexset = { package = "WorkTablesIndex", version = "=0.0.4", default-features = false, features = ["concurrent", "cdc", "multimap"] } vanilla_indexset = { package = "indexset", version = "=0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { path = "../indexset", version = "0.15.0", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "=0.12.12", features = ["concurrent", "cdc", "multimap"] } diff --git a/README.md b/README.md index 3e00fabd..c6c2eaa2 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ worktable = { version = "=1.0.0-beta.1", features = ["s3-support"] } # S3 sync Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic are explicitly memory-only and require `persist: false`. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md). +WorkTablesIndex uses its predictable branch-based node search by default in WorkTable. This avoids a measured regression for sequential numeric-key workloads. Alternative search policies remain compile-time feature gates: disable WorkTable's default features and enable one of `wti-hybrid-search`, `wti-std-search`, or `wti-superslice-search` (plus any other features such as `s3-support`). Enable exactly one `wti-*-search` feature. + ## Relationship to `data_bucket` WorkTable is built on [`data_bucket`](https://crates.io/crates/data_bucket), which diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index fa69a2cf..82b37e6d 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -165,7 +165,7 @@ impl InMemoryGenerator { pub async fn upsert(&self, row: #row_type) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); loop { - let need_to_update = self.0.primary_index.pk_map.get_value(&pk).is_some(); + let need_to_update = self.0.primary_index.pk_map.contains_key(&pk); if need_to_update { match self.update(row.clone()).await { core::result::Result::Ok(_) => return core::result::Result::Ok(()), diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index b1587ad6..5d05f9ca 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -265,7 +265,7 @@ impl PersistGenerator { pub async fn upsert(&self, row: #row_type) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); loop { - let need_to_update = self.0.primary_index.pk_map.get_value(&pk).is_some(); + let need_to_update = self.0.primary_index.pk_map.contains_key(&pk); if need_to_update { match self.update(row.clone()).await { core::result::Result::Ok(_) => return core::result::Result::Ok(()), diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 1a6f9be9..1d187503 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -76,9 +76,15 @@ where V: Clone + Debug + Send + Sync + 'static, { #[inline] - fn get_value(&self, key: &K) -> Option { + fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { let key = key.to_arctic(); - self.inner.get(key.borrow()).map(|value| (*value).clone()) + self.inner.get(key.borrow()).map(|value| read(&value)) + } + + #[inline] + fn contains_key(&self, key: &K) -> bool { + let key = key.to_arctic(); + self.inner.get(key.borrow()).is_some() } #[inline] diff --git a/src/index/congee.rs b/src/index/congee.rs index 946c7d75..d89d1164 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -89,16 +89,6 @@ where unsafe { Arc::from_raw(std::ptr::with_exposed_provenance(pointer)) } } - #[inline] - fn clone_pointer(pointer: usize) -> Arc { - // SAFETY: the caller holds a Congee epoch guard, so the tree-owned - // strong reference cannot be reclaimed while it is cloned. - let owned = unsafe { Self::arc_from_pointer(pointer) }; - let cloned = Arc::clone(&owned); - let _ = Arc::into_raw(owned); - cloned - } - #[inline] fn retire_old(pointer: usize, guard: &congee::epoch::Guard) -> Arc { // SAFETY: a successful replacement/removal transfers the tree-owned @@ -121,10 +111,19 @@ where V: Clone + Debug + Send + Sync + 'static, { #[inline] - fn get_value(&self, key: &K) -> Option { + fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { let guard = self.inner.pin(); let pointer = self.inner.get(&key.into_congee(), &guard)?; - Some(Self::clone_pointer(pointer).as_ref().clone()) + // SAFETY: the epoch guard keeps the tree-owned `Arc` alive for the + // duration of `read`, and the pointer originated from `Arc::into_raw`. + let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + Some(read(value)) + } + + #[inline] + fn contains_key(&self, key: &K) -> bool { + let guard = self.inner.pin(); + self.inner.get(&key.into_congee(), &guard).is_some() } #[inline] diff --git a/src/index/unique.rs b/src/index/unique.rs index 4b15df42..17260e0d 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -23,7 +23,14 @@ where K: Clone + Ord, V: Clone, { - fn get_value(&self, key: &K) -> Option; + fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option; + + #[inline] + fn get_value(&self, key: &K) -> Option { + self.with_value(key, Clone::clone) + } + + fn contains_key(&self, key: &K) -> bool; fn insert_value(&self, key: K, value: V) -> Option; fn insert_value_checked(&self, key: K, value: V) -> Option<()>; fn remove_value(&self, key: &K) -> Option<(K, V)>; @@ -53,8 +60,13 @@ where Node: NodeLike> + Send + 'static, { #[inline] - fn get_value(&self, key: &K) -> Option { - self.get(key).map(|entry| entry.get().value.clone()) + fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { + self.get(key).map(|entry| read(&entry.get().value)) + } + + #[inline] + fn contains_key(&self, key: &K) -> bool { + self.get(key).is_some() } #[inline] @@ -111,8 +123,13 @@ where Node: VanillaNodeLike> + Send + 'static, { #[inline] - fn get_value(&self, key: &K) -> Option { - self.get(key).map(|entry| entry.get().value.clone()) + fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { + self.get(key).map(|entry| read(&entry.get().value)) + } + + #[inline] + fn contains_key(&self, key: &K) -> bool { + self.get(key).is_some() } #[inline] @@ -183,6 +200,9 @@ mod tests { assert_eq!(index.insert_value_checked(2, 20), Some(())); assert_eq!(index.insert_value_checked(1, 10), Some(())); assert_eq!(index.insert_value_checked(2, 99), None); + assert!(index.contains_key(&1)); + assert!(!index.contains_key(&3)); + assert_eq!(index.with_value(&2, |value| value + 1), Some(21)); assert_eq!(index.get_value(&2), Some(20)); assert_eq!(index.insert_value(2, 22), Some(20)); assert_eq!(index.iter_values().collect::>(), vec![(1, 10), (2, 22)]); diff --git a/src/table/mod.rs b/src/table/mod.rs index f5a74782..d070ed9e 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -139,7 +139,7 @@ where <::WrappedRow as Archive>::Archived: Deserialize<::WrappedRow, HighDeserializer>, { - let link: Option = self.primary_index.pk_map.get_value(&pk).map(Into::into); + let link = self.primary_index.pk_map.with_value(&pk, |value| value.0); if let Some(link) = link { self.data.select_non_ghosted(link).ok() } else { From 2af3c4ccaf213dcc2a4a4634e423394167ff7f0a Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 18:33:08 +0700 Subject: [PATCH 13/23] fix: confirm concurrent index misses --- src/index/arctic.rs | 5 +++++ src/index/congee.rs | 5 +++++ src/index/unique.rs | 49 +++++++++++++++++++++++++++++++++++++++++---- src/table/mod.rs | 17 ++++++++++++++-- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 1d187503..077bae6b 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -75,6 +75,11 @@ where K: ArcticKey, V: Clone + Debug + Send + Sync + 'static, { + #[inline] + fn get_value(&self, key: &K) -> Option { + self.with_value(key, Clone::clone) + } + #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { let key = key.to_arctic(); diff --git a/src/index/congee.rs b/src/index/congee.rs index d89d1164..acef21a6 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -110,6 +110,11 @@ where K: CongeeKey, V: Clone + Debug + Send + Sync + 'static, { + #[inline] + fn get_value(&self, key: &K) -> Option { + self.with_value(key, Clone::clone) + } + #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { let guard = self.inner.pin(); diff --git a/src/index/unique.rs b/src/index/unique.rs index 17260e0d..eb75e83d 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -23,11 +23,22 @@ where K: Clone + Ord, V: Clone, { - fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option; + fn get_value(&self, key: &K) -> Option; #[inline] - fn get_value(&self, key: &K) -> Option { - self.with_value(key, Clone::clone) + fn lookup_for_select(&self, key: &K) -> Option { + self.get_value(key) + } + + #[cold] + #[inline(never)] + fn confirm_lookup_for_select(&self, key: &K) -> Option { + self.get_value(key) + } + + #[inline] + fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { + self.get_value(key).as_ref().map(read) } fn contains_key(&self, key: &K) -> bool; @@ -59,6 +70,22 @@ where V: Debug + Clone + Send + Ord + 'static, Node: NodeLike> + Send + 'static, { + #[inline] + fn get_value(&self, key: &K) -> Option { + self.get(key).map(|entry| entry.get().value.clone()) + } + + #[inline] + fn lookup_for_select(&self, key: &K) -> Option { + IndexMap::lookup_for_select(self, key) + } + + #[cold] + #[inline(never)] + fn confirm_lookup_for_select(&self, key: &K) -> Option { + IndexMap::confirm_lookup_for_select(self, key) + } + #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { self.get(key).map(|entry| read(&entry.get().value)) @@ -122,6 +149,11 @@ where V: Debug + Clone + Send + Ord + 'static, Node: VanillaNodeLike> + Send + 'static, { + #[inline] + fn get_value(&self, key: &K) -> Option { + self.get(key).map(|entry| entry.get().value.clone()) + } + #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { self.get(key).map(|entry| read(&entry.get().value)) @@ -202,6 +234,8 @@ mod tests { assert_eq!(index.insert_value_checked(2, 99), None); assert!(index.contains_key(&1)); assert!(!index.contains_key(&3)); + assert_eq!(index.lookup_for_select(&2), Some(20)); + assert_eq!(index.confirm_lookup_for_select(&2), Some(20)); assert_eq!(index.with_value(&2, |value| value + 1), Some(21)); assert_eq!(index.get_value(&2), Some(20)); assert_eq!(index.insert_value(2, 22), Some(20)); @@ -234,7 +268,14 @@ mod tests { assert_eq!(index.len(), 8_000); for key in 0..8_000_u64 { - assert_eq!(index.get_value(&key), Some(key + 1)); + let value = index.get_value(&key); + if value != Some(key + 1) { + let iterated = index.iter_values().find(|(candidate, _)| *candidate == key); + panic!( + "backend={}, key={key}, point={value:?}, iterated={iterated:?}", + std::any::type_name::(), + ); + } } let mut threads = Vec::new(); diff --git a/src/table/mod.rs b/src/table/mod.rs index d070ed9e..45824abf 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -139,14 +139,27 @@ where <::WrappedRow as Archive>::Archived: Deserialize<::WrappedRow, HighDeserializer>, { - let link = self.primary_index.pk_map.with_value(&pk, |value| value.0); + let link = self.primary_index.pk_map.lookup_for_select(&pk).map(|value| value.0); if let Some(link) = link { self.data.select_non_ghosted(link).ok() } else { - None + self.select_after_primary_index_miss(&pk) } } + #[cold] + #[inline(never)] + fn select_after_primary_index_miss(&self, pk: &PrimaryKey) -> Option + where + LockType: 'static, + Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + <::WrappedRow as Archive>::Archived: + Deserialize<::WrappedRow, HighDeserializer>, + { + let link = self.primary_index.pk_map.confirm_lookup_for_select(pk)?.0; + self.data.select_non_ghosted(link).ok() + } + #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "WorkTable"))] pub fn insert(&self, row: Row) -> Result where From 801ba8e8a0b9ae840358d11bd49ca31946171aac Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 10:16:50 +0700 Subject: [PATCH 14/23] test: combine index backends with versioned publication --- Cargo.toml | 1 + README.md | 24 + benches/cases/unique_index.rs | 22 + codegen/Cargo.toml | 1 + .../generators/in_memory/queries/select.rs | 6 +- .../src/generators/in_memory/table/impls.rs | 9 +- .../generators/in_memory/table/index_fns.rs | 102 +++- .../src/generators/persist/queries/select.rs | 6 +- codegen/src/generators/persist/table/impls.rs | 9 +- .../src/generators/persist/table/index_fns.rs | 102 +++- .../generators/read_only/queries/select.rs | 6 +- .../src/generators/read_only/table/impls.rs | 9 +- .../generators/read_only/table/index_fns.rs | 102 +++- docs/versioned-row-publication.md | 86 +++ src/in_memory/mod.rs | 6 +- src/in_memory/pages.rs | 572 ++++++++++++++++-- src/in_memory/publication.rs | 51 ++ src/in_memory/row.rs | 14 +- src/table/mod.rs | 49 +- src/table/vacuum/vacuum.rs | 19 +- tests/worktable/float.rs | 70 +++ tests/worktable/index/insert.rs | 38 ++ tests/worktable/index/range.rs | 33 + 23 files changed, 1237 insertions(+), 100 deletions(-) create mode 100644 docs/versioned-row-publication.md create mode 100644 src/in_memory/publication.rs diff --git a/Cargo.toml b/Cargo.toml index 4332c179..8a54f7a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ wti-hybrid-search = ["indexset/wt-slice-binary-search"] wti-predictable-search = ["indexset/custom-binary-search"] wti-std-search = ["indexset/std-binary-search"] wti-superslice-search = ["indexset/superslice-binary-search"] +versioned-row-publication = ["worktable_codegen/versioned-row-publication"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index c6c2eaa2..311f694a 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,30 @@ Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected e WorkTablesIndex uses its predictable branch-based node search by default in WorkTable. This avoids a measured regression for sequential numeric-key workloads. Alternative search policies remain compile-time feature gates: disable WorkTable's default features and enable one of `wti-hybrid-search`, `wti-std-search`, or `wti-superslice-search` (plus any other features such as `s3-support`). Enable exactly one `wti-*-search` feature. +## Concurrent read/write publication + +The default build preserves the existing lowest-latency page path and requires +applications to exclude reads that overlap page-byte mutation. Applications +that need generated reads to overlap updates, inserts, deletes, and vacuum can +opt into immutable row-version publication: + +```toml +[dependencies] +worktable = { version = "=1.0.0-beta.1", features = ["versioned-row-publication"] } +``` + +In this mode, generated reads acquire an immutable owned row version instead +of borrowing the mutable archived page image. Writers replace a per-row version +only after a complete page mutation, insert visibility is an atomic lifecycle +transition after every index is installed, and deleted or relocated links are +not reused until readers that could have captured them have drained. Page bytes +remain the persistence image and are internally serialized; range queries are +still non-snapshot reads. The mode intentionally trades memory, an atomic +read-side grace-period counter, and publication bookkeeping for this stronger +concurrent-read contract. See +[`docs/versioned-row-publication.md`](docs/versioned-row-publication.md) for the +protocol and its scope. + ## Relationship to `data_bucket` WorkTable is built on [`data_bucket`](https://crates.io/crates/data_bucket), which diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index 7eb69e7c..acf75f5e 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -1,6 +1,7 @@ use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, black_box, criterion_group}; use std::sync::Arc; use tokio::runtime::Runtime; +use worktable::prelude::SelectQueryExecutor; use crate::common::*; @@ -64,6 +65,26 @@ fn select_by_unique_index(c: &mut Criterion) { }); } +fn select_by_unique_index_range(c: &mut Criterion) { + let table = UniqueIndexWorkTable::default(); + + for i in 1..=1000i64 { + let row = UniqueIndexRow { + id: table.get_next_pk().into(), + test: i, + another: i as u64, + }; + table.insert(row).unwrap(); + } + + c.bench_function("unique_index_select_by_test_range", |b| { + b.iter(|| { + let test = fastrand::i64(1..=1000); + black_box(table.select_by_test_range(test..=test).execute()) + }) + }); +} + fn update(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let table = Arc::new(UniqueIndexWorkTable::default()); @@ -224,6 +245,7 @@ criterion_group! { insert, select_by_pk, select_by_unique_index, + select_by_unique_index_range, update, delete, upsert_insert, diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index c16dc503..26b8e18b 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -8,6 +8,7 @@ repository = "https://github.com/pathscale/WorkTable" [features] s3-support = [] +versioned-row-publication = [] [lib] name = "worktable_codegen" diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index e5999881..1f52b25f 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -29,9 +29,13 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let iter = self.0.primary_index.pk_map .iter_links() - .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index 82b37e6d..f714471a 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -110,9 +110,13 @@ impl InMemoryGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); + let read_guard = self.0.data.read_guard(); let rows = self.0.primary_index.pk_map .range_links(converted_range) - .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); #pk_sorted_by } @@ -262,7 +266,8 @@ impl InMemoryGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { - let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k.clone(), v.0)); + let _read_guard = self.0.data.read_guard(); + let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k, v.0)); let Some((mut k, link)) = first else { return Ok(()) }; diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index c71448d5..fbc83785 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -63,7 +63,9 @@ impl InMemoryGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}").as_str(), Span::mixed_site()); let field_ident = &idx.name; - let by = if is_float(type_.to_string().as_str()) { + let row_field_ident = &idx.field; + let is_float = is_float(type_.to_string().as_str()); + let by = if is_float { quote! { &OrderedFloat(by) } @@ -72,11 +74,47 @@ impl InMemoryGenerator { &by } }; + let predicate_matches = if is_float { + quote! { + OrderedFloat(row.#row_field_ident).eq(&OrderedFloat(by)) + } + } else { + quote! { + row.#row_field_ident.eq(&by) + } + }; + let select = if cfg!(feature = "versioned-row-publication") { + quote! { + loop { + let link: Link = self.0.indexes.#field_ident + .get_value(#by) + .map(Into::into)?; + if let Ok(row) = self.0.data.select_non_ghosted(link) { + if #predicate_matches { + return Some(row); + } + } + + let current_link: Option = self.0.indexes.#field_ident + .get_value(#by) + .map(Into::into); + if current_link == Some(link) { + return None; + } + } + } + } else { + quote! { + let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; + let row = self.0.data.select_non_ghosted(link).ok()?; + #predicate_matches.then_some(row) + } + }; Ok(quote! { pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { - let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; - self.0.data.select_non_ghosted(link).ok() + let _read_guard = self.0.data.read_guard(); + #select } }) } @@ -109,10 +147,14 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .get(#by) .into_iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()) + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) .filter(move |r| &r.#row_field_ident == &by); SelectQueryBuilder::new(rows) @@ -131,32 +173,73 @@ impl InMemoryGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}_range").as_str(), Span::mixed_site()); let field_ident = &idx.name; + let row_field_ident = &idx.field; let column_pascal = Ident::new(&i.to_string().to_case(Case::Pascal), Span::mixed_site()); + let revalidate = cfg!(feature = "versioned-row-publication"); let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( quote! { std::ops::RangeBounds<#type_> }, - quote! { + if revalidate { + quote! { + ( + predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), + predicate_range.1.as_ref().map(|v| OrderedFloat(*v)), + ) + } + } else { + quote! { ( range.start_bound().map(|v| OrderedFloat(*v)), range.end_bound().map(|v| OrderedFloat(*v)), ) + } }, ) + } else if revalidate { + ( + quote! { std::ops::RangeBounds<#type_> }, + quote! { predicate_range.clone() }, + ) } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; let (index_range, select_row) = if idx.is_unique { ( quote! { self.0.indexes.#field_ident.range_links(#range_arg) }, - quote! { |link| self.0.data.select_non_ghosted(link.0).ok() }, + quote! { + move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + } + }, ) } else { ( quote! { self.0.indexes.#field_ident.range(#range_arg) }, - quote! { |(_, link)| self.0.data.select_non_ghosted(link.0).ok() }, + quote! { + move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + } + }, ) }; + let predicate_setup = revalidate.then(|| { + quote! { + let predicate_range = ( + range.start_bound().cloned(), + range.end_bound().cloned(), + ); + } + }); + let predicate_filter = revalidate.then(|| { + quote! { + .filter(move |row| { + std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + }) + } + }); Ok(quote! { pub fn #fn_name<'a, R>(&'a self, range: R) -> SelectQueryBuilder<#row_ident, @@ -166,8 +249,11 @@ impl InMemoryGenerator { where R: #range_bounds + 'a { + #predicate_setup + let read_guard = self.0.data.read_guard(); let rows = #index_range - .filter_map(#select_row); + .filter_map(#select_row) + #predicate_filter; SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 57d1e6bc..66fad252 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -29,9 +29,13 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let iter = self.0.primary_index.pk_map .iter_links() - .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index 5d05f9ca..c4152e5b 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -200,9 +200,13 @@ impl PersistGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); + let read_guard = self.0.data.read_guard(); let rows = self.0.primary_index.pk_map .range_links(converted_range) - .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); #pk_sorted_by } @@ -373,7 +377,8 @@ impl PersistGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { - let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k.clone(), v.0)); + let _read_guard = self.0.data.read_guard(); + let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k, v.0)); let Some((mut k, link)) = first else { return Ok(()) }; diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index 6581e779..06a321cf 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -63,7 +63,9 @@ impl PersistGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}").as_str(), Span::mixed_site()); let field_ident = &idx.name; - let by = if is_float(type_.to_string().as_str()) { + let row_field_ident = &idx.field; + let is_float = is_float(type_.to_string().as_str()); + let by = if is_float { quote! { &OrderedFloat(by) } @@ -72,11 +74,47 @@ impl PersistGenerator { &by } }; + let predicate_matches = if is_float { + quote! { + OrderedFloat(row.#row_field_ident).eq(&OrderedFloat(by)) + } + } else { + quote! { + row.#row_field_ident.eq(&by) + } + }; + let select = if cfg!(feature = "versioned-row-publication") { + quote! { + loop { + let link: Link = self.0.indexes.#field_ident + .get_value(#by) + .map(Into::into)?; + if let Ok(row) = self.0.data.select_non_ghosted(link) { + if #predicate_matches { + return Some(row); + } + } + + let current_link: Option = self.0.indexes.#field_ident + .get_value(#by) + .map(Into::into); + if current_link == Some(link) { + return None; + } + } + } + } else { + quote! { + let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; + let row = self.0.data.select_non_ghosted(link).ok()?; + #predicate_matches.then_some(row) + } + }; Ok(quote! { pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { - let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; - self.0.data.select_non_ghosted(link).ok() + let _read_guard = self.0.data.read_guard(); + #select } }) } @@ -109,10 +147,14 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .get(#by) .into_iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()) + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) .filter(move |r| &r.#row_field_ident == &by); SelectQueryBuilder::new(rows) @@ -131,32 +173,73 @@ impl PersistGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}_range").as_str(), Span::mixed_site()); let field_ident = &idx.name; + let row_field_ident = &idx.field; let column_pascal = Ident::new(&i.to_string().to_case(Case::Pascal), Span::mixed_site()); + let revalidate = cfg!(feature = "versioned-row-publication"); let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( quote! { std::ops::RangeBounds<#type_> }, - quote! { + if revalidate { + quote! { + ( + predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), + predicate_range.1.as_ref().map(|v| OrderedFloat(*v)), + ) + } + } else { + quote! { ( range.start_bound().map(|v| OrderedFloat(*v)), range.end_bound().map(|v| OrderedFloat(*v)), ) + } }, ) + } else if revalidate { + ( + quote! { std::ops::RangeBounds<#type_> }, + quote! { predicate_range.clone() }, + ) } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; let (index_range, select_row) = if idx.is_unique { ( quote! { self.0.indexes.#field_ident.range_links(#range_arg) }, - quote! { |link| self.0.data.select_non_ghosted(link.0).ok() }, + quote! { + move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + } + }, ) } else { ( quote! { self.0.indexes.#field_ident.range(#range_arg) }, - quote! { |(_, link)| self.0.data.select_non_ghosted(link.0).ok() }, + quote! { + move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + } + }, ) }; + let predicate_setup = revalidate.then(|| { + quote! { + let predicate_range = ( + range.start_bound().cloned(), + range.end_bound().cloned(), + ); + } + }); + let predicate_filter = revalidate.then(|| { + quote! { + .filter(move |row| { + std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + }) + } + }); Ok(quote! { pub fn #fn_name<'a, R>(&'a self, range: R) -> SelectQueryBuilder<#row_ident, @@ -166,8 +249,11 @@ impl PersistGenerator { where R: #range_bounds + 'a { + #predicate_setup + let read_guard = self.0.data.read_guard(); let rows = #index_range - .filter_map(#select_row); + .filter_map(#select_row) + #predicate_filter; SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 92ec583f..79bffa6f 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -29,9 +29,13 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let iter = self.0.primary_index.pk_map .iter_links() - .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index 3b6f8b3a..ac96c660 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -196,9 +196,13 @@ impl ReadOnlyGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); + let read_guard = self.0.data.read_guard(); let rows = self.0.primary_index.pk_map .range_links(converted_range) - .filter_map(|link| self.0.data.select_non_ghosted(link.0).ok()); + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }); #pk_sorted_by } @@ -281,7 +285,8 @@ impl ReadOnlyGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { - let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k.clone(), v.0)); + let _read_guard = self.0.data.read_guard(); + let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k, v.0)); let Some((mut k, link)) = first else { return Ok(()) }; diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index e6dad433..a2c85fc2 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -63,7 +63,9 @@ impl ReadOnlyGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}").as_str(), Span::mixed_site()); let field_ident = &idx.name; - let by = if is_float(type_.to_string().as_str()) { + let row_field_ident = &idx.field; + let is_float = is_float(type_.to_string().as_str()); + let by = if is_float { quote! { &OrderedFloat(by) } @@ -72,11 +74,47 @@ impl ReadOnlyGenerator { &by } }; + let predicate_matches = if is_float { + quote! { + OrderedFloat(row.#row_field_ident).eq(&OrderedFloat(by)) + } + } else { + quote! { + row.#row_field_ident.eq(&by) + } + }; + let select = if cfg!(feature = "versioned-row-publication") { + quote! { + loop { + let link: Link = self.0.indexes.#field_ident + .get_value(#by) + .map(Into::into)?; + if let Ok(row) = self.0.data.select_non_ghosted(link) { + if #predicate_matches { + return Some(row); + } + } + + let current_link: Option = self.0.indexes.#field_ident + .get_value(#by) + .map(Into::into); + if current_link == Some(link) { + return None; + } + } + } + } else { + quote! { + let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; + let row = self.0.data.select_non_ghosted(link).ok()?; + #predicate_matches.then_some(row) + } + }; Ok(quote! { pub fn #fn_name(&self, by: #type_) -> Option<#row_ident> { - let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; - self.0.data.select_non_ghosted(link).ok() + let _read_guard = self.0.data.read_guard(); + #select } }) } @@ -109,10 +147,14 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { + let read_guard = self.0.data.read_guard(); let rows = self.0.indexes.#field_ident .get(#by) .into_iter() - .filter_map(|(_, link)| self.0.data.select_non_ghosted(link.0).ok()) + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) .filter(move |r| &r.#row_field_ident == &by); SelectQueryBuilder::new(rows) @@ -131,32 +173,73 @@ impl ReadOnlyGenerator { let type_ = columns_map.get(i).ok_or(syn::Error::new(i.span(), "Row not found"))?; let fn_name = Ident::new(format!("select_by_{i}_range").as_str(), Span::mixed_site()); let field_ident = &idx.name; + let row_field_ident = &idx.field; let column_pascal = Ident::new(&i.to_string().to_case(Case::Pascal), Span::mixed_site()); + let revalidate = cfg!(feature = "versioned-row-publication"); let (range_bounds, range_arg) = if is_float(type_.to_string().as_str()) { ( quote! { std::ops::RangeBounds<#type_> }, - quote! { + if revalidate { + quote! { + ( + predicate_range.0.as_ref().map(|v| OrderedFloat(*v)), + predicate_range.1.as_ref().map(|v| OrderedFloat(*v)), + ) + } + } else { + quote! { ( range.start_bound().map(|v| OrderedFloat(*v)), range.end_bound().map(|v| OrderedFloat(*v)), ) + } }, ) + } else if revalidate { + ( + quote! { std::ops::RangeBounds<#type_> }, + quote! { predicate_range.clone() }, + ) } else { (quote! { std::ops::RangeBounds<#type_> }, quote! { range }) }; let (index_range, select_row) = if idx.is_unique { ( quote! { self.0.indexes.#field_ident.range_links(#range_arg) }, - quote! { |link| self.0.data.select_non_ghosted(link.0).ok() }, + quote! { + move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + } + }, ) } else { ( quote! { self.0.indexes.#field_ident.range(#range_arg) }, - quote! { |(_, link)| self.0.data.select_non_ghosted(link.0).ok() }, + quote! { + move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + } + }, ) }; + let predicate_setup = revalidate.then(|| { + quote! { + let predicate_range = ( + range.start_bound().cloned(), + range.end_bound().cloned(), + ); + } + }); + let predicate_filter = revalidate.then(|| { + quote! { + .filter(move |row| { + std::ops::RangeBounds::contains(&predicate_range, &row.#row_field_ident) + }) + } + }); Ok(quote! { pub fn #fn_name<'a, R>(&'a self, range: R) -> SelectQueryBuilder<#row_ident, @@ -166,8 +249,11 @@ impl ReadOnlyGenerator { where R: #range_bounds + 'a { + #predicate_setup + let read_guard = self.0.data.read_guard(); let rows = #index_range - .filter_map(#select_row); + .filter_map(#select_row) + #predicate_filter; SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/docs/versioned-row-publication.md b/docs/versioned-row-publication.md new file mode 100644 index 00000000..fbcc289e --- /dev/null +++ b/docs/versioned-row-publication.md @@ -0,0 +1,86 @@ +# Versioned row publication + +Status: feature-gated prototype behind `versioned-row-publication`. + +## Problem + +The original fast path deserializes directly from archived bytes in an +`UnsafeCell` page while same-size updates mutate those bytes in place. Returning +an owned row prevents references from escaping, but it does not make a read +that overlaps a write data-race-free: the deserializer can still access bytes +while a writer changes them. A sequence counter cannot repair this in Rust, +because detecting a race after it happened does not make the racy byte access +defined. + +Physical links introduce a second issue. A reader can obtain a link from an +index, pause, and resume after delete or vacuum has reused that address for a +different row. Predicate revalidation helps, but reclamation must also cover +the interval from index lookup through acquisition of a stable row version. + +## Protocol + +With the feature enabled, `DataPages` maintains two representations: + +- Archived page bytes are the compact persistence and mutation image. All + accesses that can overlap a mutation are serialized by an internal page + barrier. +- A concurrent link map holds an immutable application-visible row version. + Each slot contains an `Arc` behind a short per-slot pointer lock and + atomic ghost, deleted, and vacuum lifecycle bits. + +The generated API follows these publication rules: + +1. **Read.** Acquire a read-grace guard before consulting an index. Resolve the + link, acquire an `Arc` to its complete published version, check lifecycle and + index predicates, clone the owned row, and release the guard. Unique and + primary-key point reads retry when the mapping swings to a replacement link + while it is being resolved. A reader never accesses mutable archived bytes + after a slot has been hydrated. +2. **Insert.** Serialize the complete row and stage a ghosted version. Install + the primary and secondary indexes. Only after every checked index insert + succeeds does the lifecycle transition publish the version with release + ordering. Failed inserts retire an unpublished version. +3. **Update.** Hold the generated row/field lock, mutate the archived image + under the page barrier, deserialize the completed wrapper, then replace the + immutable version. A concurrent reader can return the complete old version + or the complete new version, never a partially updated row. +4. **Delete.** Remove index reachability, mark the version deleted, and retire + its physical link. The empty-link allocator cannot reuse it while a reader + that could have captured the old index entry remains active. +5. **Vacuum.** Copy a complete row to a staged destination version, swing its + indexes, and retire the source publication and page. Retired links, slots, + and pages become reusable only after a read-side grace period. +6. **Reload.** Persisted tables hydrate immutable slots lazily under the page + barrier. Subsequent generated reads use the published version map. + +The grace period is quiescent-state reclamation: a feature-only atomic counter +tracks generated reads, and retirement queues are drained when that counter is +zero. `Arc` ownership independently keeps a version alive after a reader has +acquired it. + +## Guarantees and non-guarantees + +For generated table APIs in this mode: + +- reads do not race with mutation of archived page bytes; +- a read returns a complete row version; +- ghosted or deleted versions are not returned; +- a retired physical link is not reused while a pre-existing generated read + can still resolve it; and +- unique, non-unique point, and secondary-index range lookups revalidate each + resolved row predicate. + +This is not MVCC and does not add multi-operation transactions or snapshot +range scans. A scan may include or omit a concurrently inserted or updated row. +Point-read retry may starve under perpetual replacement churn. +The guarantee also does not cover callers that bypass generated table methods +and directly invoke low-level `Data` page mutation APIs. + +## Cost model and rollout + +The feature is off by default. It adds one owned row copy plus slot/map +metadata per live physical link, an atomic increment/decrement per generated +read, a concurrent publication-map lookup, and writer-side page serialization. +Those costs are inappropriate to impose silently on latency-sensitive users. +The default path remains unchanged; benchmark results for both modes must be +reported before this feature is proposed for default enablement. diff --git a/src/in_memory/mod.rs b/src/in_memory/mod.rs index 2b02cf0e..0da40382 100644 --- a/src/in_memory/mod.rs +++ b/src/in_memory/mod.rs @@ -1,9 +1,11 @@ mod data; mod empty_link_registry; mod pages; +#[cfg(feature = "versioned-row-publication")] +mod publication; mod row; pub use data::{DATA_INNER_LENGTH, Data, ExecutionError as DataExecutionError}; pub use empty_link_registry::EmptyLinkRegistry; -pub use pages::{DataPages, ExecutionError as PagesExecutionError}; -pub use row::{ArchivedRowWrapper, Query, RowWrapper, StorableRow}; +pub use pages::{DataPages, ExecutionError as PagesExecutionError, ReadGuard as DataPagesReadGuard}; +pub use row::{ArchivedRowWrapper, PublicationSafe, Query, RowWrapper, StorableRow}; diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 08a41fb8..475a3ba3 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -1,5 +1,7 @@ use data_bucket::page::PageId; use derive_more::{Display, Error, From}; +#[cfg(feature = "versioned-row-publication")] +use parking_lot::Mutex; use parking_lot::RwLock; #[cfg(feature = "perf_measurements")] use performance_measurement_codegen::performance_measurement; @@ -10,7 +12,12 @@ use rkyv::{ ser::{Serializer, allocator::ArenaHandle, sharing::Share}, util::AlignedVec, }; +#[cfg(feature = "versioned-row-publication")] +use std::collections::HashMap; use std::collections::VecDeque; +#[cfg(feature = "versioned-row-publication")] +use std::hash::{BuildHasherDefault, Hasher}; +use std::marker::PhantomData; use std::{ fmt::Debug, sync::Arc, @@ -18,7 +25,11 @@ use std::{ }; use crate::in_memory::empty_link_registry::EmptyLinkRegistry; +#[cfg(feature = "versioned-row-publication")] +use crate::in_memory::publication::{DELETED, GHOSTED, PublishedRow, VACUUMED}; use crate::prelude::ArchivedRowWrapper; +#[cfg(feature = "versioned-row-publication")] +use crate::util::OffsetEqLink; use crate::{ in_memory::{ DATA_INNER_LENGTH, Data, DataExecutionError, @@ -31,11 +42,78 @@ fn page_id_mapper(page_id: usize) -> usize { page_id - 1usize } +/// `OffsetEqLink` already reduces publication keys to a trusted internal u64 +/// storage offset. Avoid hashing that offset again on every versioned read. +#[cfg(feature = "versioned-row-publication")] +#[derive(Default)] +struct PublicationHasher(u64); + +#[cfg(feature = "versioned-row-publication")] +impl Hasher for PublicationHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write(&mut self, bytes: &[u8]) { + let mut hash = 0xcbf29ce484222325u64; + for byte in bytes { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + self.0 = hash; + } + + fn write_u64(&mut self, value: u64) { + self.0 = value; + } +} + +#[cfg(feature = "versioned-row-publication")] +type PublicationMap = + HashMap, Arc>, BuildHasherDefault>; + +pub struct ReadGuard<'a> { + #[cfg(feature = "versioned-row-publication")] + active_readers: &'a AtomicU64, + marker: PhantomData<&'a ()>, +} + +impl Drop for ReadGuard<'_> { + fn drop(&mut self) { + #[cfg(feature = "versioned-row-publication")] + self.active_readers.fetch_sub(1, Ordering::SeqCst); + } +} + #[derive(Debug)] pub struct DataPages where Row: StorableRow, { + /// Immutable application-visible row versions. Published readers never + /// borrow the mutable archived page image. + #[cfg(feature = "versioned-row-publication")] + published_rows: RwLock>, + + /// Protects the mutable page image used by writers, vacuum, and + /// persistence. Application reads use `published_rows` after hydration. + #[cfg(feature = "versioned-row-publication")] + page_access: RwLock<()>, + + /// Read-side grace period protecting the interval from index lookup until + /// an immutable row version has been acquired. + #[cfg(feature = "versioned-row-publication")] + active_readers: AtomicU64, + + #[cfg(feature = "versioned-row-publication")] + retired_links: Mutex>, + + #[cfg(feature = "versioned-row-publication")] + retired_pages: Mutex>, + + #[cfg(feature = "versioned-row-publication")] + retired_publications: Mutex>>, + /// Pages vector. Currently, not lock free. pages: RwLock::WrappedRow, DATA_LENGTH>>>>, @@ -66,8 +144,126 @@ where Row: StorableRow, ::WrappedRow: RowWrapper, { + #[cfg(feature = "versioned-row-publication")] + fn publication_flags(row: &::WrappedRow) -> u8 { + let mut flags = 0; + if row.is_ghosted() { + flags |= GHOSTED; + } + if row.is_deleted() { + flags |= DELETED; + } + if row.is_vacuumed() { + flags |= VACUUMED; + } + flags + } + + #[cfg(feature = "versioned-row-publication")] + fn publish_wrapped_row(&self, link: Link, wrapped: ::WrappedRow) { + let flags = Self::publication_flags(&wrapped); + let row = wrapped.get_inner(); + let key = OffsetEqLink(link); + + let mut published_rows = self.published_rows.write(); + if let Some(slot) = published_rows.get(&key).cloned() { + drop(published_rows); + slot.replace(row, flags); + } else { + published_rows.insert(key, Arc::new(PublishedRow::new(row, flags))); + } + } + + #[cfg(feature = "versioned-row-publication")] + fn stage_published_row(&self, link: Link, row: Row) { + let wrapped = ::WrappedRow::from_inner(row); + self.publish_wrapped_row(link, wrapped); + } + + #[cfg(feature = "versioned-row-publication")] + fn published_slot(&self, link: Link) -> Option>> { + self.published_rows.read().get(&OffsetEqLink(link)).cloned() + } + + #[cfg(feature = "versioned-row-publication")] + fn published_slot_or_hydrate(&self, link: Link) -> Result>, ExecutionError> + where + <::WrappedRow as Archive>::Archived: + Deserialize<::WrappedRow, HighDeserializer>, + { + if let Some(slot) = self.published_slot(link) { + return Ok(slot); + } + + let _page_access = self.page_access.read(); + if let Some(slot) = self.published_slot(link) { + return Ok(slot); + } + + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; + let flags = Self::publication_flags(&wrapped); + let slot = Arc::new(PublishedRow::new(wrapped.get_inner(), flags)); + let mut published_rows = self.published_rows.write(); + Ok(published_rows.entry(OffsetEqLink(link)).or_insert(slot).clone()) + } + + pub fn read_guard(&self) -> ReadGuard<'_> { + #[cfg(feature = "versioned-row-publication")] + self.active_readers.fetch_add(1, Ordering::SeqCst); + + ReadGuard { + #[cfg(feature = "versioned-row-publication")] + active_readers: &self.active_readers, + marker: PhantomData, + } + } + + #[cfg(feature = "versioned-row-publication")] + fn reclaim_retired(&self) { + if self.active_readers.load(Ordering::SeqCst) != 0 { + return; + } + + let mut retired_links = self.retired_links.lock(); + let mut retired_pages = self.retired_pages.lock(); + let mut retired_publications = self.retired_publications.lock(); + if self.active_readers.load(Ordering::SeqCst) != 0 { + return; + } + + let mut published_rows = self.published_rows.write(); + for link in retired_links.drain(..) { + published_rows.remove(&OffsetEqLink(link)); + self.empty_links.push(link); + } + for link in retired_publications.drain(..) { + published_rows.remove(&link); + } + drop(published_rows); + if !retired_pages.is_empty() { + let mut empty_pages = self.empty_pages.write(); + empty_pages.extend(retired_pages.drain(..)); + } + } + pub fn new() -> Self { Self { + #[cfg(feature = "versioned-row-publication")] + published_rows: RwLock::new(PublicationMap::default()), + #[cfg(feature = "versioned-row-publication")] + page_access: RwLock::new(()), + #[cfg(feature = "versioned-row-publication")] + active_readers: AtomicU64::new(0), + #[cfg(feature = "versioned-row-publication")] + retired_links: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_pages: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_publications: Mutex::new(Vec::new()), // We are starting ID's from `1` because `0`'s page in file is info page. pages: RwLock::new(vec![Arc::new(Data::new(1.into()))]), empty_links: EmptyLinkRegistry::::default(), @@ -85,6 +281,18 @@ where } else { let last_page_id = vec.len(); Self { + #[cfg(feature = "versioned-row-publication")] + published_rows: RwLock::new(PublicationMap::default()), + #[cfg(feature = "versioned-row-publication")] + page_access: RwLock::new(()), + #[cfg(feature = "versioned-row-publication")] + active_readers: AtomicU64::new(0), + #[cfg(feature = "versioned-row-publication")] + retired_links: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_pages: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + retired_publications: Mutex::new(Vec::new()), pages: RwLock::new(vec), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), @@ -97,13 +305,20 @@ where pub fn insert(&self, row: Row) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, { - let general_row = ::WrappedRow::from_inner(row); + let general_row = ::WrappedRow::from_inner(row.clone()); + + #[cfg(feature = "versioned-row-publication")] + self.reclaim_retired(); if let Some(link) = self.empty_links.pop_max() { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let current_page: usize = page_id_mapper(link.page_id.into()); let page = &pages[current_page]; @@ -113,6 +328,8 @@ where if let Some(l) = left_link { self.empty_links.push(l); } + #[cfg(feature = "versioned-row-publication")] + self.stage_published_row(link, row.clone()); return Ok(link); } Err(e) => match e { @@ -129,11 +346,19 @@ where loop { let (link, tried_page) = { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let current_page = page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize); let page = &pages[current_page]; - (page.save_row(&general_row), current_page) + let link = page.save_row(&general_row); + #[cfg(feature = "versioned-row-publication")] + if let Ok(saved_link) = &link { + self.stage_published_row(*saved_link, row.clone()); + } + + (link, current_page) }; match link { Ok(link) => { @@ -191,12 +416,17 @@ where /// Allocates a new page or reuses a free page from `empty_pages`. /// Does **NOT** set the page as `current`. pub fn allocate_new_or_pop_free(&self) -> Arc::WrappedRow, DATA_LENGTH>> { + #[cfg(feature = "versioned-row-publication")] + self.reclaim_retired(); + let page_id = { let mut empty_pages = self.empty_pages.write(); empty_pages.pop_front() }; if let Some(page_id) = page_id { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let index = page_id_mapper(page_id.into()); let page = pages[index].clone(); @@ -216,54 +446,104 @@ where #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "DataPages"))] pub fn select>(&self, link: L) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { let link = link.into(); - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; - Ok(gen_row.get_inner()) + #[cfg(feature = "versioned-row-publication")] + { + let slot = self.published_slot_or_hydrate(link)?; + Ok(slot.snapshot().as_ref().clone()) + } + + #[cfg(not(feature = "versioned-row-publication"))] + { + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; + Ok(gen_row.get_inner()) + } } pub fn select_non_ghosted(&self, link: Link) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; - if gen_row.is_ghosted() { - return Err(ExecutionError::Ghosted); + #[cfg(feature = "versioned-row-publication")] + { + let slot = self.published_slot_or_hydrate(link)?; + let (row, flags) = slot.load(); + if flags & GHOSTED != 0 { + return Err(ExecutionError::Ghosted); + } + if flags & DELETED != 0 { + return Err(ExecutionError::Deleted); + } + Ok(row.as_ref().clone()) + } + + #[cfg(not(feature = "versioned-row-publication"))] + { + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; + if gen_row.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + Ok(gen_row.get_inner()) } - Ok(gen_row.get_inner()) } pub fn select_non_vacuumed(&self, link: Link) -> Result where - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, + Row: Archive + + Clone + + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable + Deserialize<::WrappedRow, HighDeserializer>, { - let pages = self.pages.read(); - let page = pages - .get(page_id_mapper(link.page_id.into())) - .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; - if gen_row.is_ghosted() { - return Err(ExecutionError::Ghosted); + #[cfg(feature = "versioned-row-publication")] + { + let slot = self.published_slot_or_hydrate(link)?; + let (row, flags) = slot.load(); + if flags & GHOSTED != 0 { + return Err(ExecutionError::Ghosted); + } + if flags & VACUUMED != 0 { + return Err(ExecutionError::Vacuumed); + } + if flags & DELETED != 0 { + return Err(ExecutionError::Deleted); + } + Ok(row.as_ref().clone()) } - if gen_row.is_vacuumed() { - return Err(ExecutionError::Vacuumed); + + #[cfg(not(feature = "versioned-row-publication"))] + { + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(link.page_id))?; + let gen_row = page.get_row(link).map_err(ExecutionError::DataPageError)?; + if gen_row.is_ghosted() { + return Err(ExecutionError::Ghosted); + } + if gen_row.is_vacuumed() { + return Err(ExecutionError::Vacuumed); + } + Ok(gen_row.get_inner()) } - Ok(gen_row.get_inner()) } #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "DataPages"))] @@ -272,6 +552,8 @@ where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, Op: Fn(&<::WrappedRow as Archive>::Archived) -> Res, { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.read(); let pages = self.pages.read(); let page = pages .get::(page_id_mapper(link.page_id.into())) @@ -287,18 +569,31 @@ where where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, <::WrappedRow as Archive>::Archived: Portable, + <::WrappedRow as Archive>::Archived: + Deserialize<::WrappedRow, HighDeserializer>, Op: FnMut(&mut <::WrappedRow as Archive>::Archived) -> Res, { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let page = pages .get(page_id_mapper(link.page_id.into())) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = unsafe { - page.get_mut_row_ref(link) - .map_err(ExecutionError::DataPageError)? - .unseal_unchecked() + let res = { + let gen_row = unsafe { + page.get_mut_row_ref(link) + .map_err(ExecutionError::DataPageError)? + .unseal_unchecked() + }; + op(gen_row) }; - let res = op(gen_row); + + #[cfg(feature = "versioned-row-publication")] + { + let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; + self.publish_wrapped_row(link, wrapped); + } + Ok(res) } @@ -310,19 +605,24 @@ where /// - The operation does not cause data races or memory corruption. pub unsafe fn update(&self, row: Row, link: Link) -> Result where - Row: Archive, + Row: Archive + Clone, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); let pages = self.pages.read(); let page = pages .get(page_id_mapper(link.page_id.into())) .ok_or(ExecutionError::PageNotFound(link.page_id))?; - let gen_row = ::WrappedRow::from_inner(row); - unsafe { + let gen_row = ::WrappedRow::from_inner(row.clone()); + let result = unsafe { page.save_row_by_link(&gen_row, link) .map_err(ExecutionError::DataPageError) - } + }?; + #[cfg(feature = "versioned-row-publication")] + self.stage_published_row(link, row); + Ok(result) } pub fn delete(&self, link: Link) -> Result<(), ExecutionError> @@ -330,15 +630,26 @@ where Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, { unsafe { self.with_mut_ref(link, |r| r.delete())? } + #[cfg(feature = "versioned-row-publication")] + { + self.retired_links.lock().push(link); + self.reclaim_retired(); + } + + #[cfg(not(feature = "versioned-row-publication"))] self.empty_links.push(link); Ok(()) } pub fn select_raw(&self, link: Link) -> Result, ExecutionError> { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.read(); let pages = self.pages.read(); let page = pages .get(page_id_mapper(link.page_id.into())) @@ -348,7 +659,15 @@ where pub fn mark_page_empty(&self, page_id: PageId) { if u32::from(page_id) != self.current_page_id.load(Ordering::Acquire) { + #[cfg(feature = "versioned-row-publication")] + { + self.retired_pages.lock().push(page_id); + self.reclaim_retired(); + } + + #[cfg(not(feature = "versioned-row-publication"))] let mut g = self.empty_pages.write(); + #[cfg(not(feature = "versioned-row-publication"))] g.push_back(page_id); } } @@ -395,6 +714,8 @@ where } pub fn get_bytes(&self) -> Vec<([u8; DATA_LENGTH], u32)> { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.read(); let pages = self.pages.read(); pages .iter() @@ -402,6 +723,75 @@ where .collect() } + pub(crate) fn reset_page(&self, page_id: PageId) -> Result<(), ExecutionError> { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); + let pages = self.pages.read(); + let page = pages + .get(page_id_mapper(page_id.into())) + .ok_or(ExecutionError::PageNotFound(page_id))?; + page.reset(); + + Ok(()) + } + + /// Copies a row to another page without exposing either mutable byte + /// image to application readers. + pub(crate) unsafe fn move_row_for_vacuum( + &self, + from_link: Link, + to_page_id: PageId, + ) -> Result<(Vec, Link), ExecutionError> + where + Row: Clone, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, + { + #[cfg(feature = "versioned-row-publication")] + let _page_access = self.page_access.write(); + let pages = self.pages.read(); + let from_page = pages + .get(page_id_mapper(from_link.page_id.into())) + .ok_or(ExecutionError::PageNotFound(from_link.page_id))?; + let to_page = pages + .get(page_id_mapper(to_page_id.into())) + .ok_or(ExecutionError::PageNotFound(to_page_id))?; + + let raw_data = from_page + .get_raw_row(from_link) + .map_err(ExecutionError::DataPageError)?; + let archived = unsafe { + from_page + .get_mut_row_ref(from_link) + .map_err(ExecutionError::DataPageError)? + .unseal_unchecked() + }; + archived.set_in_vacuum_process(); + let new_link = to_page.save_raw_row(&raw_data).map_err(ExecutionError::DataPageError)?; + + #[cfg(feature = "versioned-row-publication")] + { + let old_wrapped = from_page.get_row(from_link).map_err(ExecutionError::DataPageError)?; + self.publish_wrapped_row(from_link, old_wrapped); + let new_wrapped = to_page.get_row(new_link).map_err(ExecutionError::DataPageError)?; + self.publish_wrapped_row(new_link, new_wrapped); + } + + Ok((raw_data, new_link)) + } + + pub(crate) fn retire_published_link(&self, link: Link) { + #[cfg(feature = "versioned-row-publication")] + { + self.retired_publications.lock().push(OffsetEqLink(link)); + self.reclaim_retired(); + } + + #[cfg(not(feature = "versioned-row-publication"))] + let _ = link; + } + pub fn get_page_count(&self) -> usize { self.pages.read().len() } @@ -466,7 +856,11 @@ mod tests { use std::collections::HashSet; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; + #[cfg(feature = "versioned-row-publication")] + use std::sync::mpsc; use std::thread; + #[cfg(feature = "versioned-row-publication")] + use std::time::Duration; use std::time::Instant; use parking_lot::RwLock; @@ -603,6 +997,104 @@ mod tests { assert_eq!(res.err(), Some(PagesExecutionError::Ghosted)) } + #[cfg(feature = "versioned-row-publication")] + #[test] + fn versioned_insert_stays_hidden_until_unghost() { + let pages = DataPages::::new(); + let row = TestRow { a: 7, b: 9 }; + let link = pages.insert(row).unwrap(); + + assert_eq!(pages.select_non_ghosted(link), Err(ExecutionError::Ghosted)); + unsafe { + pages.with_mut_ref(link, |archived| archived.unghost()).unwrap(); + } + assert_eq!(pages.select_non_ghosted(link), Ok(row)); + } + + #[cfg(feature = "versioned-row-publication")] + #[test] + fn versioned_reader_observes_old_row_while_page_update_is_incomplete() { + let pages = Arc::new(DataPages::::new()); + let link = pages.insert(TestRow { a: 0, b: 0 }).unwrap(); + unsafe { + pages.with_mut_ref(link, |row| row.unghost()).unwrap(); + } + + let (first_field_written_tx, first_field_written_rx) = mpsc::channel(); + let (finish_update_tx, finish_update_rx) = mpsc::channel(); + let writer_pages = pages.clone(); + let writer = thread::spawn(move || unsafe { + writer_pages + .with_mut_ref(link, |archived| { + archived.inner.a = 1.into(); + first_field_written_tx.send(()).unwrap(); + finish_update_rx.recv().unwrap(); + archived.inner.b = 1.into(); + }) + .unwrap(); + }); + + first_field_written_rx.recv().unwrap(); + let (read_tx, read_rx) = mpsc::channel(); + let reader_pages = pages.clone(); + let reader = thread::spawn(move || { + read_tx.send(reader_pages.select_non_ghosted(link)).unwrap(); + }); + + assert_eq!( + read_rx.recv_timeout(Duration::from_secs(1)).unwrap(), + Ok(TestRow { a: 0, b: 0 }), + "reader must use the old immutable version instead of page bytes" + ); + + finish_update_tx.send(()).unwrap(); + writer.join().unwrap(); + reader.join().unwrap(); + assert_eq!(pages.select_non_ghosted(link), Ok(TestRow { a: 1, b: 1 })); + } + + #[cfg(feature = "versioned-row-publication")] + #[test] + fn retired_version_survives_link_reuse_for_in_flight_reader() { + let pages = DataPages::::new(); + let link = pages.insert(TestRow { a: 1, b: 1 }).unwrap(); + unsafe { + pages.with_mut_ref(link, |row| row.unghost()).unwrap(); + } + let old_slot = pages.published_slot(link).unwrap(); + let old_version = old_slot.snapshot(); + + pages.delete(link).unwrap(); + let reused_link = pages.insert(TestRow { a: 2, b: 2 }).unwrap(); + assert_eq!(reused_link, link); + assert_eq!(pages.select_non_ghosted(reused_link), Err(ExecutionError::Ghosted)); + unsafe { + pages.with_mut_ref(reused_link, |row| row.unghost()).unwrap(); + } + + assert_eq!(old_version.as_ref(), &TestRow { a: 1, b: 1 }); + assert_eq!(pages.select_non_ghosted(reused_link), Ok(TestRow { a: 2, b: 2 })); + } + + #[cfg(feature = "versioned-row-publication")] + #[test] + fn read_grace_period_prevents_link_aba() { + let pages = DataPages::::new(); + let old_link = pages.insert(TestRow { a: 1, b: 1 }).unwrap(); + unsafe { + pages.with_mut_ref(old_link, |row| row.unghost()).unwrap(); + } + + let read_guard = pages.read_guard(); + pages.delete(old_link).unwrap(); + let new_link = pages.insert(TestRow { a: 2, b: 2 }).unwrap(); + assert_ne!(new_link, old_link, "retired link was reused by an active reader"); + + drop(read_guard); + pages.reclaim_retired(); + assert!(pages.get_empty_links().contains(&old_link)); + } + #[test] fn select_non_vacuumed_returns_row_when_valid() { let pages = DataPages::::new(); diff --git a/src/in_memory/publication.rs b/src/in_memory/publication.rs new file mode 100644 index 00000000..a0f0907b --- /dev/null +++ b/src/in_memory/publication.rs @@ -0,0 +1,51 @@ +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU8, Ordering}; + +use parking_lot::RwLock; + +pub(super) const GHOSTED: u8 = 1 << 0; +pub(super) const DELETED: u8 = 1 << 1; +pub(super) const VACUUMED: u8 = 1 << 2; + +/// One immutable application-visible row version plus atomic lifecycle bits. +/// +/// Readers hold an `Arc` to a complete version, so replacing or retiring a +/// version cannot invalidate an in-flight read. The short per-row lock only +/// protects the `Arc` pointer; readers never access mutable archived bytes. +pub(super) struct PublishedRow { + row: RwLock>, + flags: AtomicU8, +} + +impl PublishedRow { + pub(super) fn new(row: Row, flags: u8) -> Self { + Self { + row: RwLock::new(Arc::new(row)), + flags: AtomicU8::new(flags), + } + } + + pub(super) fn replace(&self, row: Row, flags: u8) { + *self.row.write() = Arc::new(row); + self.flags.store(flags, Ordering::Release); + } + + pub(super) fn load(&self) -> (Arc, u8) { + let flags = self.flags.load(Ordering::Acquire); + let row = self.row.read().clone(); + (row, flags) + } + + pub(super) fn snapshot(&self) -> Arc { + self.row.read().clone() + } +} + +impl Debug for PublishedRow { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PublishedRow") + .field("flags", &self.flags.load(Ordering::Relaxed)) + .finish_non_exhaustive() + } +} diff --git a/src/in_memory/row.rs b/src/in_memory/row.rs index 2d75a8b3..115b9b73 100644 --- a/src/in_memory/row.rs +++ b/src/in_memory/row.rs @@ -2,10 +2,22 @@ use std::fmt::Debug; use rkyv::Archive; +#[cfg(feature = "versioned-row-publication")] +pub trait PublicationSafe: Send + Sync + 'static {} + +#[cfg(feature = "versioned-row-publication")] +impl PublicationSafe for T {} + +#[cfg(not(feature = "versioned-row-publication"))] +pub trait PublicationSafe {} + +#[cfg(not(feature = "versioned-row-publication"))] +impl PublicationSafe for T {} + /// Common trait for the `Row`s that can be stored on the [`Data`] page. /// /// [`Data`]: crate::in_memory::data::Data -pub trait StorableRow { +pub trait StorableRow: PublicationSafe { type WrappedRow: Archive + Debug; } diff --git a/src/table/mod.rs b/src/table/mod.rs index 45824abf..4fad318c 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -21,7 +21,7 @@ use rkyv::ser::Serializer; use rkyv::ser::allocator::ArenaHandle; use rkyv::ser::sharing::Share; use rkyv::util::AlignedVec; -use rkyv::{Archive, Deserialize, Serialize}; +use rkyv::{Archive, Deserialize, Portable, Serialize}; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; @@ -139,11 +139,30 @@ where <::WrappedRow as Archive>::Archived: Deserialize<::WrappedRow, HighDeserializer>, { - let link = self.primary_index.pk_map.lookup_for_select(&pk).map(|value| value.0); - if let Some(link) = link { - self.data.select_non_ghosted(link).ok() - } else { - self.select_after_primary_index_miss(&pk) + let _read_guard = self.data.read_guard(); + #[cfg(feature = "versioned-row-publication")] + { + loop { + let link: Link = self.primary_index.pk_map.get_value(&pk).map(Into::into)?; + if let Ok(row) = self.data.select_non_ghosted(link) { + return Some(row); + } + + let current_link: Option = self.primary_index.pk_map.get_value(&pk).map(Into::into); + if current_link == Some(link) { + return None; + } + } + } + + #[cfg(not(feature = "versioned-row-publication"))] + { + let link = self.primary_index.pk_map.lookup_for_select(&pk).map(|value| value.0); + if let Some(link) = link { + self.data.select_non_ghosted(link).ok() + } else { + self.select_after_primary_index_miss(&pk) + } } } @@ -168,7 +187,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, AvailableTypes: 'static, AvailableIndexes: AvailableIndex, @@ -184,9 +205,9 @@ where if let Err(e) = self.indexes.save_row(row.clone(), link) { return match e { IndexError::AlreadyExists { at, inserted_already } => { - self.data.delete(link).map_err(WorkTableError::PagesError)?; self.primary_index.remove(&pk, link); self.indexes.delete_from_indexes(row, link, inserted_already)?; + self.data.delete(link).map_err(WorkTableError::PagesError)?; Err(WorkTableError::AlreadyExists(at.to_string_value())) } @@ -216,7 +237,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, SecondaryEvents: Debug + Default + Clone + TableSecondaryIndexEventsOps, SecondaryIndexes: TableSecondaryIndex @@ -335,7 +358,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, AvailableTypes: 'static, AvailableIndexes: Debug + AvailableIndex, @@ -392,7 +417,9 @@ where + for<'a> Serialize, Share>, rkyv::rancor::Error>>, ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: ArchivedRowWrapper, + <::WrappedRow as Archive>::Archived: ArchivedRowWrapper + + Portable + + Deserialize<::WrappedRow, HighDeserializer>, PrimaryKey: Clone, SecondaryEvents: Debug + Default + Clone + TableSecondaryIndexEventsOps, SecondaryIndexes: TableSecondaryIndex diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index ca03f4dd..e81344ff 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -207,13 +207,11 @@ where } fn free_page(&self, page_id: PageId) { - let p = self.data_pages.get_page(page_id).expect("should exist as called"); - p.reset() + self.data_pages.reset_page(page_id).expect("should exist as called") } async fn move_data_from(&self, from: PageId, to: PageId) -> (bool, bool) { let to_page = self.data_pages.get_page(to).expect("should exist as link exists"); - let from_page = self.data_pages.get_page(from).expect("should exist as link exists"); let to_free_space = to_page.free_space(); let page_start = OffsetEqLink::<_>(Link { @@ -268,18 +266,13 @@ where self.lock_manager.remove_with_lock_check(&pk); continue; } - let raw_data = from_page - .get_raw_row(from_link.0) - .expect("link is not bigger than free offset"); - unsafe { + let (raw_data, new_link) = unsafe { self.data_pages - .with_mut_ref(from_link.0, |r| r.set_in_vacuum_process()) - .expect("link should be valid") - } - let new_link = to_page - .save_raw_row(&raw_data) - .expect("page is not full as checked on links collection"); + .move_row_for_vacuum(from_link.0, to) + .expect("links and destination capacity were checked") + }; self.update_index_after_move(pk.clone(), from_link.0, new_link, raw_data); + self.data_pages.retire_published_link(from_link.0); lock.unlock(); self.lock_manager.remove_with_lock_check(&pk); diff --git a/tests/worktable/float.rs b/tests/worktable/float.rs index 02f751c9..576c707b 100644 --- a/tests/worktable/float.rs +++ b/tests/worktable/float.rs @@ -16,6 +16,76 @@ worktable! ( } ); +worktable! ( + name: TestUniqueFloat, + columns: { + id: u64 primary_key autoincrement, + value: f64, + }, + indexes: { + value_idx: value unique, + } +); + +#[test] +fn unique_float_point_read_revalidates_the_returned_row() { + let table = TestUniqueFloatWorkTable::default(); + let first = TestUniqueFloatRow { + id: table.get_next_pk().into(), + value: 1.5, + }; + let second = TestUniqueFloatRow { + id: table.get_next_pk().into(), + value: 2.5, + }; + table.insert(first.clone()).unwrap(); + table.insert(second.clone()).unwrap(); + + let second_link = table + .0 + .primary_index + .pk_map + .get(&TestUniqueFloatPrimaryKey(second.id)) + .map(|entry| entry.get().value.0) + .unwrap(); + TableIndex::insert(&table.0.indexes.value_idx, OrderedFloat(first.value), second_link); + + assert!(table.select_by_value(first.value).is_none()); + assert_eq!(table.select_by_value(second.value), Some(second)); +} + +#[cfg(feature = "versioned-row-publication")] +#[test] +fn float_range_read_revalidates_each_resolved_row() { + let table = TestFloatWorkTable::default(); + let inside = TestFloatRow { + id: table.get_next_pk().into(), + test: 1, + another: 10.0, + exchange: "inside".to_string(), + }; + let outside = TestFloatRow { + id: table.get_next_pk().into(), + test: 2, + another: 100.0, + exchange: "outside".to_string(), + }; + table.insert(inside.clone()).unwrap(); + table.insert(outside.clone()).unwrap(); + + let outside_link = table + .0 + .primary_index + .pk_map + .get(&TestFloatPrimaryKey(outside.id)) + .map(|entry| entry.get().value.0) + .unwrap(); + TableIndex::insert(&table.0.indexes.another_idx, OrderedFloat(15.0), outside_link); + + let rows = table.select_by_another_range(0.0..20.0).execute().unwrap(); + assert_eq!(rows, vec![inside]); +} + #[test] fn select_all_range_float_test() { let table = TestFloatWorkTable::default(); diff --git a/tests/worktable/index/insert.rs b/tests/worktable/index/insert.rs index ec447335..910e1067 100644 --- a/tests/worktable/index/insert.rs +++ b/tests/worktable/index/insert.rs @@ -38,6 +38,44 @@ async fn insert() { assert!(table.select(2).is_none()) } +#[test] +fn unique_point_read_revalidates_the_returned_row() { + let table = TestWorkTable::default(); + let first = TestRow { + id: table.get_next_pk().into(), + val: 13, + attr1: "first".to_string(), + attr2: -128, + attr3: 1, + attr4: "first-unique".to_string(), + }; + let second = TestRow { + id: table.get_next_pk().into(), + val: 14, + attr1: "second".to_string(), + attr2: 128, + attr3: 2, + attr4: "second-unique".to_string(), + }; + table.insert(first.clone()).unwrap(); + table.insert(second.clone()).unwrap(); + + let second_link = table + .0 + .primary_index + .pk_map + .get(&TestPrimaryKey(second.id)) + .map(|entry| entry.get().value.0) + .unwrap(); + + // Model the transient state where a unique-index entry still names a row + // whose indexed field has already changed. + TableIndex::insert(&table.0.indexes.attr2_idx, first.attr2, second_link); + + assert!(table.select_by_attr2(first.attr2).is_none()); + assert_eq!(table.select_by_attr2(second.attr2), Some(second)); +} + #[tokio::test] async fn insert_when_pk_exists() { let table = TestWorkTable::default(); diff --git a/tests/worktable/index/range.rs b/tests/worktable/index/range.rs index 99591ce6..39f98ddc 100644 --- a/tests/worktable/index/range.rs +++ b/tests/worktable/index/range.rs @@ -32,6 +32,39 @@ worktable!( } ); +#[cfg(feature = "versioned-row-publication")] +#[test] +fn range_read_revalidates_each_resolved_row() { + let table = RangeTestWorkTable::default(); + let inside = RangeTestRow { + id: table.get_next_pk().into(), + value: 10, + name: "inside".to_string(), + }; + let outside = RangeTestRow { + id: table.get_next_pk().into(), + value: 100, + name: "outside".to_string(), + }; + table.insert(inside.clone()).unwrap(); + table.insert(outside.clone()).unwrap(); + + let outside_link = table + .0 + .primary_index + .pk_map + .get(&RangeTestPrimaryKey(outside.id)) + .map(|entry| entry.get().value.0) + .unwrap(); + + // Model the transient state where an index entry still falls inside the + // requested range but its completed row version has already moved out. + TableIndex::insert(&table.0.indexes.val_idx, 15, outside_link); + + let rows = table.select_by_value_range(0..20).execute().unwrap(); + assert_eq!(rows, vec![inside]); +} + #[test] fn test_range_select_basic() { let table = RangeTestWorkTable::default(); From 8a64759da75ee99415c24e071620bb74e81b1cac Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 10:40:33 +0700 Subject: [PATCH 15/23] fix: gate bounded retry for transient index misses --- Cargo.toml | 1 + README.md | 10 ++++++++++ codegen/Cargo.toml | 1 + .../src/generators/in_memory/table/index_fns.rs | 17 +++++++++++++++-- .../src/generators/persist/table/index_fns.rs | 17 +++++++++++++++-- .../src/generators/read_only/table/index_fns.rs | 17 +++++++++++++++-- docs/versioned-row-publication.md | 13 ++++++++++--- src/table/mod.rs | 13 ++++++++++++- 8 files changed, 79 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8a54f7a4..c0138b61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ wti-predictable-search = ["indexset/custom-binary-search"] wti-std-search = ["indexset/std-binary-search"] wti-superslice-search = ["indexset/superslice-binary-search"] versioned-row-publication = ["worktable_codegen/versioned-row-publication"] +stable-index-read-retry = ["versioned-row-publication", "worktable_codegen/stable-index-read-retry"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index 311f694a..5ff9517f 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,16 @@ opt into immutable row-version publication: worktable = { version = "=1.0.0-beta.1", features = ["versioned-row-publication"] } ``` +WorkTablesIndex and IndexSet users who need an acknowledged concurrent insert +to be immediately visible to a point lookup can additionally enable the +bounded stable-miss confirmation. It implies versioned row publication and is +separately gated so the default latency-sensitive path is unchanged: + +```toml +[dependencies] +worktable = { version = "0.9", features = ["stable-index-read-retry"] } +``` + In this mode, generated reads acquire an immutable owned row version instead of borrowing the mutable archived page image. Writers replace a per-row version only after a complete page mutation, insert visibility is an atomic lifecycle diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 26b8e18b..bdb4f442 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -9,6 +9,7 @@ repository = "https://github.com/pathscale/WorkTable" [features] s3-support = [] versioned-row-publication = [] +stable-index-read-retry = ["versioned-row-publication"] [lib] name = "worktable_codegen" diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index fbc83785..a1a8dd00 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -83,12 +83,21 @@ impl InMemoryGenerator { row.#row_field_ident.eq(&by) } }; + let retry_stable_miss = cfg!(feature = "stable-index-read-retry"); let select = if cfg!(feature = "versioned-row-publication") { quote! { + let mut retry_stable_miss = #retry_stable_miss; loop { - let link: Link = self.0.indexes.#field_ident + let Some(link) = self.0.indexes.#field_ident .get_value(#by) - .map(Into::into)?; + .map(Into::into) + else { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } + return None; + }; if let Ok(row) = self.0.data.select_non_ghosted(link) { if #predicate_matches { return Some(row); @@ -99,6 +108,10 @@ impl InMemoryGenerator { .get_value(#by) .map(Into::into); if current_link == Some(link) { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } return None; } } diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index 06a321cf..5250e031 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -83,12 +83,21 @@ impl PersistGenerator { row.#row_field_ident.eq(&by) } }; + let retry_stable_miss = cfg!(feature = "stable-index-read-retry"); let select = if cfg!(feature = "versioned-row-publication") { quote! { + let mut retry_stable_miss = #retry_stable_miss; loop { - let link: Link = self.0.indexes.#field_ident + let Some(link) = self.0.indexes.#field_ident .get_value(#by) - .map(Into::into)?; + .map(Into::into) + else { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } + return None; + }; if let Ok(row) = self.0.data.select_non_ghosted(link) { if #predicate_matches { return Some(row); @@ -99,6 +108,10 @@ impl PersistGenerator { .get_value(#by) .map(Into::into); if current_link == Some(link) { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } return None; } } diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index a2c85fc2..47d62838 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -83,12 +83,21 @@ impl ReadOnlyGenerator { row.#row_field_ident.eq(&by) } }; + let retry_stable_miss = cfg!(feature = "stable-index-read-retry"); let select = if cfg!(feature = "versioned-row-publication") { quote! { + let mut retry_stable_miss = #retry_stable_miss; loop { - let link: Link = self.0.indexes.#field_ident + let Some(link) = self.0.indexes.#field_ident .get_value(#by) - .map(Into::into)?; + .map(Into::into) + else { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } + return None; + }; if let Ok(row) = self.0.data.select_non_ghosted(link) { if #predicate_matches { return Some(row); @@ -99,6 +108,10 @@ impl ReadOnlyGenerator { .get_value(#by) .map(Into::into); if current_link == Some(link) { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } return None; } } diff --git a/docs/versioned-row-publication.md b/docs/versioned-row-publication.md index fbcc289e..65b9f575 100644 --- a/docs/versioned-row-publication.md +++ b/docs/versioned-row-publication.md @@ -34,8 +34,12 @@ The generated API follows these publication rules: link, acquire an `Arc` to its complete published version, check lifecycle and index predicates, clone the owned row, and release the guard. Unique and primary-key point reads retry when the mapping swings to a replacement link - while it is being resolved. A reader never accesses mutable archived bytes - after a slot has been hydrated. + while it is being resolved. With the additional + `stable-index-read-retry` feature, they also retry one apparently stable miss + once: the B-tree-family providers can transiently report no mapping during a + concurrent structural insertion, while the ART providers do not exhibit + that behavior. A reader never accesses mutable archived bytes after a slot + has been hydrated. 2. **Insert.** Serialize the complete row and stage a ghosted version. Install the primary and secondary indexes. Only after every checked index insert succeeds does the lifecycle transition publish the version with release @@ -72,7 +76,8 @@ For generated table APIs in this mode: This is not MVCC and does not add multi-operation transactions or snapshot range scans. A scan may include or omit a concurrently inserted or updated row. -Point-read retry may starve under perpetual replacement churn. +Point-read replacement retry may starve under perpetual replacement churn. The +feature-gated stable-miss retry is bounded to one additional probe. The guarantee also does not cover callers that bypass generated table methods and directly invoke low-level `Data` page mutation APIs. @@ -81,6 +86,8 @@ and directly invoke low-level `Data` page mutation APIs. The feature is off by default. It adds one owned row copy plus slot/map metadata per live physical link, an atomic increment/decrement per generated read, a concurrent publication-map lookup, and writer-side page serialization. +`stable-index-read-retry` does not add an index probe to successful point hits, +but a true point miss performs one bounded confirmation probe. Those costs are inappropriate to impose silently on latency-sensitive users. The default path remains unchanged; benchmark results for both modes must be reported before this feature is proposed for default enablement. diff --git a/src/table/mod.rs b/src/table/mod.rs index 4fad318c..44a2d46f 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -142,14 +142,25 @@ where let _read_guard = self.data.read_guard(); #[cfg(feature = "versioned-row-publication")] { + let mut retry_stable_miss = cfg!(feature = "stable-index-read-retry"); loop { - let link: Link = self.primary_index.pk_map.get_value(&pk).map(Into::into)?; + let Some(link) = self.primary_index.pk_map.get_value(&pk).map(Into::into) else { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } + return None; + }; if let Ok(row) = self.data.select_non_ghosted(link) { return Some(row); } let current_link: Option = self.primary_index.pk_map.get_value(&pk).map(Into::into); if current_link == Some(link) { + if std::mem::take(&mut retry_stable_miss) { + std::hint::spin_loop(); + continue; + } return None; } } From 5e40afd4ce0c9fb729742decf9a8540b2d37c0cb Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 18:38:41 +0700 Subject: [PATCH 16/23] fix: validate combined publication campaign --- README.md | 2 +- src/table/mod.rs | 6 +++--- tests/persistence/sync/many_strings.rs | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5ff9517f..a64d5dbc 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ separately gated so the default latency-sensitive path is unchanged: ```toml [dependencies] -worktable = { version = "0.9", features = ["stable-index-read-retry"] } +worktable = { version = "=1.0.0-beta.1", features = ["stable-index-read-retry"] } ``` In this mode, generated reads acquire an immutable owned row version instead diff --git a/src/table/mod.rs b/src/table/mod.rs index 44a2d46f..5d41f473 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -149,10 +149,10 @@ where std::hint::spin_loop(); continue; } - return None; + break None; }; if let Ok(row) = self.data.select_non_ghosted(link) { - return Some(row); + break Some(row); } let current_link: Option = self.primary_index.pk_map.get_value(&pk).map(Into::into); @@ -161,7 +161,7 @@ where std::hint::spin_loop(); continue; } - return None; + break None; } } } diff --git a/tests/persistence/sync/many_strings.rs b/tests/persistence/sync/many_strings.rs index ad4fece7..9b2281dd 100644 --- a/tests/persistence/sync/many_strings.rs +++ b/tests/persistence/sync/many_strings.rs @@ -133,8 +133,9 @@ fn test_space_update_query_pk_many_times_sync() { if table.select(pk.clone()).is_none() { let direct = table.0.primary_index.pk_map.get_value(&TestSyncPrimaryKey(pk.clone())); let entries = table.0.primary_index.pk_map.iter_values().collect::>(); + let data = direct.map(|link| table.0.data.select_non_ghosted(link.into())); panic!( - "final primary lookup missed: direct={direct:?}, entries={entries:?}, row_count={}", + "final primary lookup missed: direct={direct:?}, data={data:?}, entries={entries:?}, row_count={}", table.count() ); } From ba1c5c0daa568fab940e70a4560bc132688755db Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 18:44:25 +0700 Subject: [PATCH 17/23] fix: gate redundant publication fallback --- src/table/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/table/mod.rs b/src/table/mod.rs index 5d41f473..de800764 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -177,6 +177,7 @@ where } } + #[cfg(not(feature = "versioned-row-publication"))] #[cold] #[inline(never)] fn select_after_primary_index_miss(&self, pk: &PrimaryKey) -> Option From 3b74428be8d4d9a09c7bdf543df3b1eaf62075a3 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 19:16:18 +0700 Subject: [PATCH 18/23] fix: linearize same-key upserts --- .../generators/in_memory/queries/update.rs | 15 +++- .../src/generators/in_memory/table/impls.rs | 76 ++++++++----------- .../src/generators/persist/queries/update.rs | 15 +++- codegen/src/generators/persist/table/impls.rs | 76 ++++++++----------- tests/worktable/upsert.rs | 25 ++---- 5 files changed, 99 insertions(+), 108 deletions(-) diff --git a/codegen/src/generators/in_memory/queries/update.rs b/codegen/src/generators/in_memory/queries/update.rs index 8b2ecc45..59421a69 100644 --- a/codegen/src/generators/in_memory/queries/update.rs +++ b/codegen/src/generators/in_memory/queries/update.rs @@ -34,6 +34,8 @@ impl InMemoryGenerator { fn gen_full_row_update(&mut self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_ident = name_generator.get_row_type_ident(); + let lock_ident = name_generator.get_lock_type_ident(); + let pk_ident = name_generator.get_primary_key_type_ident(); let row_updates = self .columns @@ -83,12 +85,23 @@ impl InMemoryGenerator { pub async fn update(&self, row: #row_ident) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); let op_lock = { #full_row_lock }; - let _guard = LockGuard::new( + let guard = LockGuard::new( op_lock, self.0.lock_manager.clone(), pk.clone(), ); + self.update_with_guard(row, guard).await + } + + #[inline] + async fn update_with_guard( + &self, + row: #row_ident, + _guard: LockGuard<#lock_ident, #pk_ident>, + ) -> core::result::Result<(), WorkTableError> { + let pk = row.get_primary_key(); + let mut link: Link = self.0 .primary_index .pk_map diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index f714471a..1b79c3b2 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -150,64 +150,52 @@ impl InMemoryGenerator { fn gen_table_upsert_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_type = name_generator.get_row_type_ident(); + let full_row_lock = self.gen_full_lock_for_update(); quote! { /// Inserts the row if its primary key is absent, updates it /// otherwise. /// - /// Concurrency: **system-wide lock-free, not wait-free per call.** - /// A retry is only taken when a concurrent delete or insert - /// flipped this key's existence between the existence check and - /// the operation, so some operation on this key completes on - /// every iteration -- but under sustained adversarial churn on - /// the same key an individual call can retry indefinitely. There - /// is deliberately no retry limit: upsert is semantically - /// infallible for primary-key conflicts, and a limit would trade - /// theoretical starvation for real spurious errors. Each - /// conflicting round yields to the scheduler before retrying so - /// the interfering task can complete. + /// A definitely absent key takes the same optimistic lock-free + /// insert path as `insert`. Existing keys and insertion collisions + /// acquire one full-row lock across the repeated existence check + /// and selected mutation, so upserts, updates, and deletes on the + /// same key cannot invalidate that decision. pub async fn upsert(&self, row: #row_type) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); + if !self.0.primary_index.pk_map.contains_key(&pk) { + match self.insert(row.clone()) { + core::result::Result::Ok(_) => return core::result::Result::Ok(()), + core::result::Result::Err(WorkTableError::PrimaryAlreadyExists) => {} + core::result::Result::Err(e) => return core::result::Result::Err(e), + } + } loop { - let need_to_update = self.0.primary_index.pk_map.contains_key(&pk); - if need_to_update { - match self.update(row.clone()).await { - core::result::Result::Ok(_) => return core::result::Result::Ok(()), - // Row was deleted concurrently between the check and the - // update; retry as an insert. - core::result::Result::Err(WorkTableError::NotFound) => { - tokio::task::yield_now().await; - continue; - } - // Row is mid-flight: a concurrent insert publishes - // the primary-key entry before unghosting the row - // data (and insert takes no row lock), and a - // concurrent delete ghosts data it is about to - // unindex. Both are transient; retry. - core::result::Result::Err(WorkTableError::PagesError(e)) if e.is_row_absent() => { - tokio::task::yield_now().await; - continue; - } - core::result::Result::Err(e) => return core::result::Result::Err(e), - } + let op_lock = { #full_row_lock }; + let guard = LockGuard::new( + op_lock, + self.0.lock_manager.clone(), + pk.clone(), + ); + + let result = if self.0.primary_index.pk_map.contains_key(&pk) { + self.update_with_guard(row.clone(), guard).await } else { match self.insert(row.clone()) { - core::result::Result::Ok(_) => return core::result::Result::Ok(()), - // Row was inserted concurrently between the check and the - // insert; retry as an update. Secondary-index conflicts are - // real errors and are propagated. Progress is lock-free, - // not wait-free: a retry is only taken when a concurrent - // delete/insert flipped this key's existence between the - // check and the operation, so the system as a whole makes - // progress on every retry, but this call can in principle - // retry unboundedly under sustained same-key churn. + core::result::Result::Ok(_) => core::result::Result::Ok(()), core::result::Result::Err(WorkTableError::PrimaryAlreadyExists) => { - tokio::task::yield_now().await; - continue; + self.update_with_guard(row.clone(), guard).await } - core::result::Result::Err(e) => return core::result::Result::Err(e), + core::result::Result::Err(e) => core::result::Result::Err(e), } + }; + + match result { + core::result::Result::Err(WorkTableError::NotFound) => {} + core::result::Result::Err(WorkTableError::PagesError(e)) if e.is_row_absent() => {} + other => return other, } + tokio::task::yield_now().await; } } } diff --git a/codegen/src/generators/persist/queries/update.rs b/codegen/src/generators/persist/queries/update.rs index 368f688e..cce93e09 100644 --- a/codegen/src/generators/persist/queries/update.rs +++ b/codegen/src/generators/persist/queries/update.rs @@ -34,6 +34,8 @@ impl PersistGenerator { fn gen_full_row_update(&mut self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_ident = name_generator.get_row_type_ident(); + let lock_ident = name_generator.get_lock_type_ident(); + let pk_ident = name_generator.get_primary_key_type_ident(); let row_updates = self .columns @@ -83,12 +85,23 @@ impl PersistGenerator { pub async fn update(&self, row: #row_ident) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); let op_lock = { #full_row_lock }; - let _guard = LockGuard::new( + let guard = LockGuard::new( op_lock, self.0.lock_manager.clone(), pk.clone(), ); + self.update_with_guard(row, guard).await + } + + #[inline] + async fn update_with_guard( + &self, + row: #row_ident, + _guard: LockGuard<#lock_ident, #pk_ident>, + ) -> core::result::Result<(), WorkTableError> { + let pk = row.get_primary_key(); + let mut link: Link = self.0 .primary_index .pk_map diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index c4152e5b..9bac16b2 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -250,64 +250,52 @@ impl PersistGenerator { fn gen_table_upsert_fn(&self) -> TokenStream { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let row_type = name_generator.get_row_type_ident(); + let full_row_lock = self.gen_full_lock_for_update(); quote! { /// Inserts the row if its primary key is absent, updates it /// otherwise. /// - /// Concurrency: **system-wide lock-free, not wait-free per call.** - /// A retry is only taken when a concurrent delete or insert - /// flipped this key's existence between the existence check and - /// the operation, so some operation on this key completes on - /// every iteration -- but under sustained adversarial churn on - /// the same key an individual call can retry indefinitely. There - /// is deliberately no retry limit: upsert is semantically - /// infallible for primary-key conflicts, and a limit would trade - /// theoretical starvation for real spurious errors. Each - /// conflicting round yields to the scheduler before retrying so - /// the interfering task can complete. + /// A definitely absent key takes the same optimistic lock-free + /// insert path as `insert`. Existing keys and insertion collisions + /// acquire one full-row lock across the repeated existence check + /// and selected mutation, so upserts, updates, and deletes on the + /// same key cannot invalidate that decision. pub async fn upsert(&self, row: #row_type) -> core::result::Result<(), WorkTableError> { let pk = row.get_primary_key(); + if !self.0.primary_index.pk_map.contains_key(&pk) { + match self.insert(row.clone()) { + core::result::Result::Ok(_) => return core::result::Result::Ok(()), + core::result::Result::Err(WorkTableError::PrimaryAlreadyExists) => {} + core::result::Result::Err(e) => return core::result::Result::Err(e), + } + } loop { - let need_to_update = self.0.primary_index.pk_map.contains_key(&pk); - if need_to_update { - match self.update(row.clone()).await { - core::result::Result::Ok(_) => return core::result::Result::Ok(()), - // Row was deleted concurrently between the check and the - // update; retry as an insert. - core::result::Result::Err(WorkTableError::NotFound) => { - tokio::task::yield_now().await; - continue; - } - // Row is mid-flight: a concurrent insert publishes - // the primary-key entry before unghosting the row - // data (and insert takes no row lock), and a - // concurrent delete ghosts data it is about to - // unindex. Both are transient; retry. - core::result::Result::Err(WorkTableError::PagesError(e)) if e.is_row_absent() => { - tokio::task::yield_now().await; - continue; - } - core::result::Result::Err(e) => return core::result::Result::Err(e), - } + let op_lock = { #full_row_lock }; + let guard = LockGuard::new( + op_lock, + self.0.lock_manager.clone(), + pk.clone(), + ); + + let result = if self.0.primary_index.pk_map.contains_key(&pk) { + self.update_with_guard(row.clone(), guard).await } else { match self.insert(row.clone()) { - core::result::Result::Ok(_) => return core::result::Result::Ok(()), - // Row was inserted concurrently between the check and the - // insert; retry as an update. Secondary-index conflicts are - // real errors and are propagated. Progress is lock-free, - // not wait-free: a retry is only taken when a concurrent - // delete/insert flipped this key's existence between the - // check and the operation, so the system as a whole makes - // progress on every retry, but this call can in principle - // retry unboundedly under sustained same-key churn. + core::result::Result::Ok(_) => core::result::Result::Ok(()), core::result::Result::Err(WorkTableError::PrimaryAlreadyExists) => { - tokio::task::yield_now().await; - continue; + self.update_with_guard(row.clone(), guard).await } - core::result::Result::Err(e) => return core::result::Result::Err(e), + core::result::Result::Err(e) => core::result::Result::Err(e), } + }; + + match result { + core::result::Result::Err(WorkTableError::NotFound) => {} + core::result::Result::Err(WorkTableError::PagesError(e)) if e.is_row_absent() => {} + other => return other, } + tokio::task::yield_now().await; } } } diff --git a/tests/worktable/upsert.rs b/tests/worktable/upsert.rs index 47b87cb2..183fc6f6 100644 --- a/tests/worktable/upsert.rs +++ b/tests/worktable/upsert.rs @@ -13,30 +13,19 @@ worktable!( }, ); -/// Upsert is system-wide lock-free but not wait-free per call: a retry is -/// taken exactly when a concurrent delete/insert flips the key's existence -/// between the check and the operation. This test drives sustained -/// adversarial churn on ONE key while several tasks upsert it and asserts -/// that every upsert completes without surfacing a spurious conflict error, -/// under a timeout that turns pathological starvation into a failure -/// instead of a hang. -/// Moderate tier: 40/40 stable when run solo, but under full-suite parallel -/// load the pre-existing #169 stall still fires (~1/30 suite runs), so both -/// tiers stay ignored until that lands. Run with `-- --ignored` to validate -/// the upsert retry behavior. +/// After its optimistic absent-key fast path, upsert holds the generated +/// full-row lock across its repeated existence check and mutation. This test +/// drives sustained adversarial churn on one key while several tasks upsert +/// it, asserting that every operation completes without a spurious conflict +/// error. The timeout turns a lock-order regression into a failure instead of +/// a hang. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "exposes the pre-existing #169 lock-free-insert stall under suite load"] async fn upsert_completes_under_same_key_churn() { churn_run(100, 200).await; } -/// Intense variant: reliably exposes pre-existing engine races under extreme -/// same-key churn that are unrelated to the upsert retry loop (raw `insert` -/// takes no row lock and publishes the pk entry before unghosting the data; -/// under saturation a churn round can stall past the timeout). Tracked in the -/// lock-free-insert issue; un-ignore when that lands. +/// Intense variant for the same-key upsert/delete linearization protocol. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -#[ignore = "exposes pre-existing lock-free-insert races under extreme same-key churn"] async fn upsert_completes_under_extreme_same_key_churn() { churn_run(5_000, 2_000).await; } From 92ae190cf32b2b90cafe318693bf26db966fbcf8 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 19:21:26 +0700 Subject: [PATCH 19/23] test: cover raw insert mutation churn --- tests/worktable/upsert.rs | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/worktable/upsert.rs b/tests/worktable/upsert.rs index 183fc6f6..7ba61a3b 100644 --- a/tests/worktable/upsert.rs +++ b/tests/worktable/upsert.rs @@ -30,6 +30,52 @@ async fn upsert_completes_under_extreme_same_key_churn() { churn_run(5_000, 2_000).await; } +/// A synchronous insert does not participate in the generated async row lock. +/// Its collision and ghost-publication windows must still return typed errors, +/// never panic a concurrent locked delete or strand an upsert waiter. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn raw_insert_delete_churn_never_panics_or_stalls() { + let table = Arc::new(UpsertChurnWorkTable::default()); + const KEY: u64 = 7; + + let churn = { + let table = table.clone(); + tokio::spawn(async move { + for i in 0..5_000u64 { + let _ = table.insert(UpsertChurnRow { id: KEY, val: i }); + let _ = table.delete(KEY).await; + } + }) + }; + + let mut upserters = Vec::new(); + for worker in 0..4u64 { + let table = table.clone(); + upserters.push(tokio::spawn(async move { + for i in 0..2_000u64 { + table + .upsert(UpsertChurnRow { + id: KEY, + val: worker * 10_000 + i, + }) + .await + .expect("upsert must never surface a primary-key conflict"); + } + })); + } + + timeout(Duration::from_secs(60), churn) + .await + .expect("raw insert/delete churn starved") + .unwrap(); + for handle in upserters { + timeout(Duration::from_secs(60), handle) + .await + .expect("upserter starved during raw insert/delete churn") + .unwrap(); + } +} + async fn churn_run(churn_flips: u64, upserts_per_task: u64) { #[allow(non_snake_case)] let CHURN_FLIPS = churn_flips; From 0218d40d2e667989e8f0848eee65f488b305ec57 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 19:29:24 +0700 Subject: [PATCH 20/23] docs: fix public API links --- src/in_memory/data.rs | 5 +---- src/lock/mod.rs | 2 +- src/table/vacuum/fragmentation_info.rs | 2 +- src/table/vacuum/vacuum.rs | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/in_memory/data.rs b/src/in_memory/data.rs index 42bafbee..0202c015 100644 --- a/src/in_memory/data.rs +++ b/src/in_memory/data.rs @@ -48,10 +48,7 @@ impl DerefMut for AlignedBytes { #[derive(Archive, Deserialize, Debug, Serialize)] pub struct Data { - /// [`Id`] of the [`General`] page of this [`Data`]. - /// - /// [`Id]: PageId - /// [`General`]: page::General + /// [`PageId`] of the general page represented by this [`Data`] block. #[rkyv(with = Skip)] pub id: PageId, diff --git a/src/lock/mod.rs b/src/lock/mod.rs index 90432eed..51c1ec0d 100644 --- a/src/lock/mod.rs +++ b/src/lock/mod.rs @@ -108,7 +108,7 @@ impl Lock { /// A lock born in the released state (`is_locked() == false`, waiting on /// it returns immediately). Used for placeholder state, e.g. a fresh - /// [`FullRowLock`](crate::lock::FullRowLock) that no operation holds yet. + /// [`FullRowLock`] that no operation holds yet. /// "Released" refers to the acquisition flag, distinguishing it from /// [`Lock::new`], which starts held by the creating operation. pub fn new_released(id: u16) -> Self { diff --git a/src/table/vacuum/fragmentation_info.rs b/src/table/vacuum/fragmentation_info.rs index 2f1440bd..843b560c 100644 --- a/src/table/vacuum/fragmentation_info.rs +++ b/src/table/vacuum/fragmentation_info.rs @@ -68,7 +68,7 @@ impl EmptyLinkRegistry { self.page_links_map.get(&page_id).map(|(_, link)| *link).collect() } - /// Calculates [`PageFragmentationInfo`] information for all pages with + /// Calculates `PageFragmentationInfo` information for all pages with /// empty [`Link`]s. pub fn get_per_page_info(&self) -> Vec { let mut page_empty_data: HashMap)> = HashMap::new(); diff --git a/src/table/vacuum/vacuum.rs b/src/table/vacuum/vacuum.rs index e81344ff..0e62bcb3 100644 --- a/src/table/vacuum/vacuum.rs +++ b/src/table/vacuum/vacuum.rs @@ -102,7 +102,7 @@ where LockType: RowLock, PrimaryIndex: TableIndexCdc, { - /// Creates a new [`EmptyDataVacuum`] from the given [`WorkTable`] components. + /// Creates a new [`EmptyDataVacuum`] from the given `WorkTable` components. pub fn new( table_name: &'static str, data_pages: Arc>, From f7feda35604881935098fd86700023e1d6c69300 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 21:00:18 +0700 Subject: [PATCH 21/23] fix: use specialized stable index reads by default --- Cargo.toml | 1 - README.md | 17 +++++------ codegen/Cargo.toml | 1 - .../generators/in_memory/table/index_fns.rs | 25 +++++----------- .../src/generators/persist/table/index_fns.rs | 25 +++++----------- .../generators/read_only/table/index_fns.rs | 25 +++++----------- docs/index-backend-dsl-proposal.md | 20 +++++++------ docs/versioned-row-publication.md | 20 ++++++------- src/index/unique.rs | 19 ++---------- src/table/mod.rs | 29 ++----------------- 10 files changed, 56 insertions(+), 126 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c0138b61..8a54f7a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,6 @@ wti-predictable-search = ["indexset/custom-binary-search"] wti-std-search = ["indexset/std-binary-search"] wti-superslice-search = ["indexset/superslice-binary-search"] versioned-row-publication = ["worktable_codegen/versioned-row-publication"] -stable-index-read-retry = ["versioned-row-publication", "worktable_codegen/stable-index-read-retry"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/README.md b/README.md index a64d5dbc..bbb23cb9 100644 --- a/README.md +++ b/README.md @@ -63,15 +63,14 @@ opt into immutable row-version publication: worktable = { version = "=1.0.0-beta.1", features = ["versioned-row-publication"] } ``` -WorkTablesIndex and IndexSet users who need an acknowledged concurrent insert -to be immediately visible to a point lookup can additionally enable the -bounded stable-miss confirmation. It implies versioned row publication and is -separately gated so the default latency-sensitive path is unchanged: - -```toml -[dependencies] -worktable = { version = "=1.0.0-beta.1", features = ["stable-index-read-retry"] } -``` +Generated point lookups use a strict backend-specific visibility contract by +default. WorkTablesIndex keeps its successful-hit fast path and confirms only +an apparent miss against the selected node and adjacent structural boundaries. +Congee and Arctic use their native concurrent point lookups. The explicit +vanilla `using indexset` backend remains experimental and is excluded from the +stable concurrent-read contract because upstream IndexSet does not expose an +equivalent validation primitive. This index-visibility contract is independent +of the optional row publication mode above. In this mode, generated reads acquire an immutable owned row version instead of borrowing the mutable archived page image. Writers replace a per-row version diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index bdb4f442..26b8e18b 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -9,7 +9,6 @@ repository = "https://github.com/pathscale/WorkTable" [features] s3-support = [] versioned-row-publication = [] -stable-index-read-retry = ["versioned-row-publication"] [lib] name = "worktable_codegen" diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index a1a8dd00..846c01bd 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -83,21 +83,12 @@ impl InMemoryGenerator { row.#row_field_ident.eq(&by) } }; - let retry_stable_miss = cfg!(feature = "stable-index-read-retry"); let select = if cfg!(feature = "versioned-row-publication") { quote! { - let mut retry_stable_miss = #retry_stable_miss; loop { - let Some(link) = self.0.indexes.#field_ident - .get_value(#by) - .map(Into::into) - else { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } - return None; - }; + let link: Link = self.0.indexes.#field_ident + .lookup_for_select(#by) + .map(Into::into)?; if let Ok(row) = self.0.data.select_non_ghosted(link) { if #predicate_matches { return Some(row); @@ -105,20 +96,18 @@ impl InMemoryGenerator { } let current_link: Option = self.0.indexes.#field_ident - .get_value(#by) + .lookup_for_select(#by) .map(Into::into); if current_link == Some(link) { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } return None; } } } } else { quote! { - let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; + let link: Link = self.0.indexes.#field_ident + .lookup_for_select(#by) + .map(Into::into)?; let row = self.0.data.select_non_ghosted(link).ok()?; #predicate_matches.then_some(row) } diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index 5250e031..bd159c8d 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -83,21 +83,12 @@ impl PersistGenerator { row.#row_field_ident.eq(&by) } }; - let retry_stable_miss = cfg!(feature = "stable-index-read-retry"); let select = if cfg!(feature = "versioned-row-publication") { quote! { - let mut retry_stable_miss = #retry_stable_miss; loop { - let Some(link) = self.0.indexes.#field_ident - .get_value(#by) - .map(Into::into) - else { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } - return None; - }; + let link: Link = self.0.indexes.#field_ident + .lookup_for_select(#by) + .map(Into::into)?; if let Ok(row) = self.0.data.select_non_ghosted(link) { if #predicate_matches { return Some(row); @@ -105,20 +96,18 @@ impl PersistGenerator { } let current_link: Option = self.0.indexes.#field_ident - .get_value(#by) + .lookup_for_select(#by) .map(Into::into); if current_link == Some(link) { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } return None; } } } } else { quote! { - let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; + let link: Link = self.0.indexes.#field_ident + .lookup_for_select(#by) + .map(Into::into)?; let row = self.0.data.select_non_ghosted(link).ok()?; #predicate_matches.then_some(row) } diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index 47d62838..d606600f 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -83,21 +83,12 @@ impl ReadOnlyGenerator { row.#row_field_ident.eq(&by) } }; - let retry_stable_miss = cfg!(feature = "stable-index-read-retry"); let select = if cfg!(feature = "versioned-row-publication") { quote! { - let mut retry_stable_miss = #retry_stable_miss; loop { - let Some(link) = self.0.indexes.#field_ident - .get_value(#by) - .map(Into::into) - else { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } - return None; - }; + let link: Link = self.0.indexes.#field_ident + .lookup_for_select(#by) + .map(Into::into)?; if let Ok(row) = self.0.data.select_non_ghosted(link) { if #predicate_matches { return Some(row); @@ -105,20 +96,18 @@ impl ReadOnlyGenerator { } let current_link: Option = self.0.indexes.#field_ident - .get_value(#by) + .lookup_for_select(#by) .map(Into::into); if current_link == Some(link) { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } return None; } } } } else { quote! { - let link: Link = self.0.indexes.#field_ident.get_value(#by).map(Into::into)?; + let link: Link = self.0.indexes.#field_ident + .lookup_for_select(#by) + .map(Into::into)?; let row = self.0.data.select_non_ghosted(link).ok()?; #predicate_matches.then_some(row) } diff --git a/docs/index-backend-dsl-proposal.md b/docs/index-backend-dsl-proposal.md index f900e841..947b73af 100644 --- a/docs/index-backend-dsl-proposal.md +++ b/docs/index-backend-dsl-proposal.md @@ -153,13 +153,15 @@ Backend dispatch itself is static and should compile away. That does **not** mea - `OffsetEqLink` values are copied out of backend guards; no heap allocation is introduced for a point lookup. - Both can emit persistence-compatible structural CDC. -Direct dispatch does not strengthen either provider's concurrent-read -semantics. During unrelated structural mutations, WorkTablesIndex and vanilla -IndexSet can transiently return a point miss for a key that is present after -the writers quiesce. This PR verifies mutation integrity and quiescent reads; -it does not claim linearizable point reads for those providers. Workloads that -require stable concurrent visibility need the separate row-publication and -stable-miss protocol before their results are publishable. +Direct dispatch preserves a strict generated point-read contract with a +provider-specific implementation. WorkTablesIndex keeps the normal successful +hit path and sends only an apparent miss through a cold three-node boundary +confirmation. Vanilla IndexSet does not expose a comparable structural +validation primitive; `using indexset` therefore remains experimental and is +excluded from concurrent correctness and published performance claims. This +WorkTablesIndex visibility guarantee is independent of +`versioned-row-publication`, which addresses concurrent page bytes, ghost +publication, and reclamation rather than index routing. ### Congee @@ -190,7 +192,7 @@ Use allocator/RSS measurements for comparative memory results; do not treat the This implementation uses the published crates directly: -- `WorkTablesIndex 0.0.3` as the default `indexset` dependency alias already used by WorkTable; +- `WorkTablesIndex 0.0.4` as the default `indexset` dependency alias already used by WorkTable; - vanilla `indexset 0.15.0` under the `vanilla_indexset` Cargo name; - `congee 0.4.1` through its public `Congee`, `compute_or_insert`, and `new_with_drainer` APIs; - `arctic-map 0.1.4` through its public concurrent map API. @@ -235,7 +237,7 @@ For the paper, the strongest controlled experiment keeps the WorkTable schema, g ## Production versus research classification - **WorkTablesIndex:** production default. -- **Vanilla IndexSet:** potential production migration backend because it preserves local/S3 persistence through the existing format boundary; still requires downstream stress and performance validation. +- **Vanilla IndexSet:** experimental provider. It preserves local/S3 persistence through the existing format boundary, but is excluded from concurrent correctness and published performance claims until upstream offers a stable structural-read primitive or the adapter gains a low-cost algorithm. - **Congee and Arctic:** research/experimental memory-only backends in this PR. Promotion requires relevant downstream evidence, allocation/reclamation review, and a workload that does not depend on the current allocating scan path. That boundary is deliberate: `using` exposes optional physical specialization without quietly weakening WorkTable's in-memory/on-disk coordination contract. diff --git a/docs/versioned-row-publication.md b/docs/versioned-row-publication.md index 65b9f575..2c581338 100644 --- a/docs/versioned-row-publication.md +++ b/docs/versioned-row-publication.md @@ -34,12 +34,12 @@ The generated API follows these publication rules: link, acquire an `Arc` to its complete published version, check lifecycle and index predicates, clone the owned row, and release the guard. Unique and primary-key point reads retry when the mapping swings to a replacement link - while it is being resolved. With the additional - `stable-index-read-retry` feature, they also retry one apparently stable miss - once: the B-tree-family providers can transiently report no mapping during a - concurrent structural insertion, while the ART providers do not exhibit - that behavior. A reader never accesses mutable archived bytes after a slot - has been hydrated. + while it is being resolved. Point lookup itself uses each provider's strict + visibility path: WorkTablesIndex confirms an apparent miss against a stable + three-node structural window and the ART providers retain their native + concurrent point algorithms. Vanilla `using indexset` is experimental and + excluded from this concurrent-read guarantee. A reader never accesses + mutable archived bytes after a slot has been hydrated. 2. **Insert.** Serialize the complete row and stage a ghosted version. Install the primary and secondary indexes. Only after every checked index insert succeeds does the lifecycle transition publish the version with release @@ -76,8 +76,7 @@ For generated table APIs in this mode: This is not MVCC and does not add multi-operation transactions or snapshot range scans. A scan may include or omit a concurrently inserted or updated row. -Point-read replacement retry may starve under perpetual replacement churn. The -feature-gated stable-miss retry is bounded to one additional probe. +Point-read replacement retry may starve under perpetual replacement churn. The guarantee also does not cover callers that bypass generated table methods and directly invoke low-level `Data` page mutation APIs. @@ -86,8 +85,9 @@ and directly invoke low-level `Data` page mutation APIs. The feature is off by default. It adds one owned row copy plus slot/map metadata per live physical link, an atomic increment/decrement per generated read, a concurrent publication-map lookup, and writer-side page serialization. -`stable-index-read-retry` does not add an index probe to successful point hits, -but a true point miss performs one bounded confirmation probe. +The index-visibility algorithm is always active and separate from row +publication: WorkTablesIndex adds no second probe to successful point hits, but +an apparent miss enters a cold three-node confirmation path. Those costs are inappropriate to impose silently on latency-sensitive users. The default path remains unchanged; benchmark results for both modes must be reported before this feature is proposed for default enablement. diff --git a/src/index/unique.rs b/src/index/unique.rs index eb75e83d..d4ba8a01 100644 --- a/src/index/unique.rs +++ b/src/index/unique.rs @@ -30,12 +30,6 @@ where self.get_value(key) } - #[cold] - #[inline(never)] - fn confirm_lookup_for_select(&self, key: &K) -> Option { - self.get_value(key) - } - #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { self.get_value(key).as_ref().map(read) @@ -72,7 +66,7 @@ where { #[inline] fn get_value(&self, key: &K) -> Option { - self.get(key).map(|entry| entry.get().value.clone()) + IndexMap::lookup_for_select(self, key) } #[inline] @@ -80,20 +74,14 @@ where IndexMap::lookup_for_select(self, key) } - #[cold] - #[inline(never)] - fn confirm_lookup_for_select(&self, key: &K) -> Option { - IndexMap::confirm_lookup_for_select(self, key) - } - #[inline] fn with_value(&self, key: &K, read: impl FnOnce(&V) -> R) -> Option { - self.get(key).map(|entry| read(&entry.get().value)) + IndexMap::lookup_for_select(self, key).as_ref().map(read) } #[inline] fn contains_key(&self, key: &K) -> bool { - self.get(key).is_some() + IndexMap::contains_key(self, key) } #[inline] @@ -235,7 +223,6 @@ mod tests { assert!(index.contains_key(&1)); assert!(!index.contains_key(&3)); assert_eq!(index.lookup_for_select(&2), Some(20)); - assert_eq!(index.confirm_lookup_for_select(&2), Some(20)); assert_eq!(index.with_value(&2, |value| value + 1), Some(21)); assert_eq!(index.get_value(&2), Some(20)); assert_eq!(index.insert_value(2, 22), Some(20)); diff --git a/src/table/mod.rs b/src/table/mod.rs index de800764..d0bae122 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -142,25 +142,16 @@ where let _read_guard = self.data.read_guard(); #[cfg(feature = "versioned-row-publication")] { - let mut retry_stable_miss = cfg!(feature = "stable-index-read-retry"); loop { - let Some(link) = self.primary_index.pk_map.get_value(&pk).map(Into::into) else { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } + let Some(link) = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into) else { break None; }; if let Ok(row) = self.data.select_non_ghosted(link) { break Some(row); } - let current_link: Option = self.primary_index.pk_map.get_value(&pk).map(Into::into); + let current_link: Option = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into); if current_link == Some(link) { - if std::mem::take(&mut retry_stable_miss) { - std::hint::spin_loop(); - continue; - } break None; } } @@ -172,25 +163,11 @@ where if let Some(link) = link { self.data.select_non_ghosted(link).ok() } else { - self.select_after_primary_index_miss(&pk) + None } } } - #[cfg(not(feature = "versioned-row-publication"))] - #[cold] - #[inline(never)] - fn select_after_primary_index_miss(&self, pk: &PrimaryKey) -> Option - where - LockType: 'static, - Row: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, - <::WrappedRow as Archive>::Archived: - Deserialize<::WrappedRow, HighDeserializer>, - { - let link = self.primary_index.pk_map.confirm_lookup_for_select(pk)?.0; - self.data.select_non_ghosted(link).ok() - } - #[cfg_attr(feature = "perf_measurements", performance_measurement(prefix_name = "WorkTable"))] pub fn insert(&self, row: Row) -> Result where From 9889752932b8d329ef00c6c22aa08a8328655dd4 Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 23:29:16 +0700 Subject: [PATCH 22/23] fix: address beta hardening review findings --- .github/workflows/rust.yml | 28 ++- Cargo.toml | 2 +- README.md | 7 +- benches/cases/unique_index.rs | 48 +++- .../generators/in_memory/queries/select.rs | 19 +- .../src/generators/in_memory/table/impls.rs | 56 ++--- .../generators/in_memory/table/index_fns.rs | 36 +-- codegen/src/generators/in_memory/table/mod.rs | 11 +- codegen/src/generators/index_backend.rs | 14 +- .../src/generators/persist/queries/select.rs | 19 +- codegen/src/generators/persist/table/impls.rs | 56 ++--- .../src/generators/persist/table/index_fns.rs | 36 +-- codegen/src/generators/persist/table/mod.rs | 34 ++- .../generators/read_only/queries/select.rs | 19 +- .../src/generators/read_only/table/impls.rs | 56 ++--- .../generators/read_only/table/index_fns.rs | 36 +-- codegen/src/generators/read_only/table/mod.rs | 34 ++- codegen/src/persist_index/generator.rs | 111 ++++++--- codegen/src/persist_table/generator/mod.rs | 1 + codegen/src/persist_table/mod.rs | 15 +- codegen/src/persist_table/parser.rs | 16 +- codegen/src/worktable/mod.rs | 6 +- docs/index-backend-dsl-proposal.md | 25 +- docs/versioned-row-publication.md | 54 ++-- response.md | 232 ++++++++++++++++++ src/in_memory/pages.rs | 144 ++++++++--- src/in_memory/publication.rs | 74 ++++-- src/index/arctic.rs | 86 ++++++- src/index/congee.rs | 89 ++++++- src/index/table_index/cdc.rs | 14 +- src/persistence/operation/batch.rs | 5 +- src/table/mod.rs | 12 +- tests/persistence/sync/many_strings.rs | 10 +- tests/worktable/index/range.rs | 30 +++ 34 files changed, 1079 insertions(+), 356 deletions(-) create mode 100644 response.md diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a9fb5d54..e2cc6b81 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -14,9 +14,19 @@ permissions: jobs: build: - + name: Build and test (${{ matrix.name }}) runs-on: ubicloud-standard-2 timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - name: default + args: "" + - name: versioned-publication + args: "--features versioned-row-publication" + - name: all-features + args: "--all-features" steps: - uses: actions/checkout@v4 @@ -25,15 +35,23 @@ jobs: cache-on-failure: "true" add-job-id-key: "false" - name: Build - run: cargo build --workspace --all-targets --all-features --verbose + run: cargo build --workspace --all-targets ${{ matrix.args }} --verbose - name: Run tests - run: cargo test --workspace --all-targets --all-features --verbose + run: cargo test --workspace --all-targets ${{ matrix.args }} --verbose clippy_check: - + name: Clippy (${{ matrix.name }}) runs-on: ubicloud-standard-2 timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - name: default + args: "" + - name: all-features + args: "--all-features" steps: - uses: actions/checkout@v4 @@ -42,4 +60,4 @@ jobs: cache-on-failure: "true" add-job-id-key: "false" - name: Clippy (deny warnings) - run: cargo clippy --workspace --all-targets --all-features -- -D warnings + run: cargo clippy --workspace --all-targets ${{ matrix.args }} -- -D warnings diff --git a/Cargo.toml b/Cargo.toml index 8a54f7a4..300ba940 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,7 @@ smart-default = "0.7.1" tokio = { version = "1", features = ["full"] } tracing = "0.1" url = { version = "2", optional = true } -uuid = { version = "1.10.0", features = ["v4", "v7"] } +uuid = { version = "1.24.0", features = ["v4", "v7"] } walkdir = { version = "2", optional = true } worktable_codegen = { path = "codegen", version = "=1.0.0-beta.1" } diff --git a/README.md b/README.md index bbb23cb9..430957d3 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ worktable = { version = "=1.0.0-beta.1", features = ["s3-support"] } # S3 sync Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic are explicitly memory-only and require `persist: false`. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md). -WorkTablesIndex uses its predictable branch-based node search by default in WorkTable. This avoids a measured regression for sequential numeric-key workloads. Alternative search policies remain compile-time feature gates: disable WorkTable's default features and enable one of `wti-hybrid-search`, `wti-std-search`, or `wti-superslice-search` (plus any other features such as `s3-support`). Enable exactly one `wti-*-search` feature. +WorkTablesIndex uses its predictable branch-based node search by default in WorkTable. This avoids a measured regression for sequential numeric-key workloads. Alternative search policies remain compile-time feature gates: disable WorkTable's default features and enable one of `wti-hybrid-search`, `wti-std-search`, or `wti-superslice-search` (plus any other features such as `s3-support`). Prefer one search feature for an unambiguous build. If Cargo feature unification enables several, WorkTablesIndex applies the documented deterministic precedence rather than rejecting the graph. ## Concurrent read/write publication @@ -64,8 +64,9 @@ worktable = { version = "=1.0.0-beta.1", features = ["versioned-row-publication" ``` Generated point lookups use a strict backend-specific visibility contract by -default. WorkTablesIndex keeps its successful-hit fast path and confirms only -an apparent miss against the selected node and adjacent structural boundaries. +default. WorkTablesIndex 0.0.4 keeps the structural mapping pinned until its +selected node is locked, making both hits and misses definitive; contended +lookups release the structural guard before waiting and retry the mapping. Congee and Arctic use their native concurrent point lookups. The explicit vanilla `using indexset` backend remains experimental and is excluded from the stable concurrent-read contract because upstream IndexSet does not expose an diff --git a/benches/cases/unique_index.rs b/benches/cases/unique_index.rs index acf75f5e..7a4b2c42 100644 --- a/benches/cases/unique_index.rs +++ b/benches/cases/unique_index.rs @@ -1,10 +1,29 @@ use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, black_box, criterion_group}; use std::sync::Arc; use tokio::runtime::Runtime; -use worktable::prelude::SelectQueryExecutor; +use worktable::prelude::*; +use worktable::worktable; use crate::common::*; +worktable! { + name: CongeeRangeBenchmark, + persist: false, + columns: { + id: u64 primary_key using congee, + value: u64, + }, +} + +worktable! { + name: ArcticRangeBenchmark, + persist: false, + columns: { + id: u64 primary_key using arctic, + value: u64, + }, +} + fn insert(c: &mut Criterion) { let table = UniqueIndexWorkTable::default(); @@ -85,6 +104,32 @@ fn select_by_unique_index_range(c: &mut Criterion) { }); } +fn art_primary_key_ranges(c: &mut Criterion) { + const ROWS: u64 = 10_000; + let congee = CongeeRangeBenchmarkWorkTable::default(); + let arctic = ArcticRangeBenchmarkWorkTable::default(); + for id in 0..ROWS { + congee.insert(CongeeRangeBenchmarkRow { id, value: id }).unwrap(); + arctic.insert(ArcticRangeBenchmarkRow { id, value: id }).unwrap(); + } + + let mut group = c.benchmark_group("art_primary_key_single_row_range"); + group.throughput(Throughput::Elements(1)); + group.bench_function("congee", |b| { + b.iter(|| { + let id = black_box(fastrand::u64(0..ROWS)); + black_box(congee.select_by_pk_range(id..=id).execute().unwrap()) + }) + }); + group.bench_function("arctic", |b| { + b.iter(|| { + let id = black_box(fastrand::u64(0..ROWS)); + black_box(arctic.select_by_pk_range(id..=id).execute().unwrap()) + }) + }); + group.finish(); +} + fn update(c: &mut Criterion) { let rt = Runtime::new().unwrap(); let table = Arc::new(UniqueIndexWorkTable::default()); @@ -246,6 +291,7 @@ criterion_group! { select_by_pk, select_by_unique_index, select_by_unique_index_range, + art_primary_key_ranges, update, delete, upsert_insert, diff --git a/codegen/src/generators/in_memory/queries/select.rs b/codegen/src/generators/in_memory/queries/select.rs index 1f52b25f..489758e3 100644 --- a/codegen/src/generators/in_memory/queries/select.rs +++ b/codegen/src/generators/in_memory/queries/select.rs @@ -29,13 +29,18 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { - let read_guard = self.0.data.read_guard(); - let iter = self.0.primary_index.pk_map - .iter_links() - .filter_map(move |link| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }); + // Acquire the grace-period guard only when iteration starts. + // Merely constructing and retaining a query builder must not + // stall retired-link reclamation. + let iter = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map + .iter_links() + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + }).flatten(); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/in_memory/table/impls.rs b/codegen/src/generators/in_memory/table/impls.rs index 1b79c3b2..60e2718e 100644 --- a/codegen/src/generators/in_memory/table/impls.rs +++ b/codegen/src/generators/in_memory/table/impls.rs @@ -110,13 +110,17 @@ impl InMemoryGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); - let read_guard = self.0.data.read_guard(); - let rows = self.0.primary_index.pk_map - .range_links(converted_range) - .filter_map(move |link| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }); + // Delay the grace-period guard until the returned iterator is + // consumed so an idle query builder cannot pin reclamation. + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map + .range_links(converted_range) + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + }).flatten(); #pk_sorted_by } @@ -255,32 +259,18 @@ impl InMemoryGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { let _read_guard = self.0.data.read_guard(); - let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k, v.0)); - let Some((mut k, link)) = first else { - return Ok(()) - }; - - let data = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; - #func - - let mut ind = false; - while !ind { - let next = { - let mut iter = self.0.primary_index.pk_map.range_values(k.clone()..); - let next = iter.next().map(|(k, v)| (k.clone(), v.0)).filter(|(key, _)| key != &k); - if next.is_some() { - next - } else { - iter.next().map(|(k, v)| (k.clone(), v.0)) - } - }; - if let Some((key, link)) = next { - let data = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; - #func - k = key - } else { - ind = true; - }; + // Snapshot the ordered links once. Re-starting a range at every + // key turns materializing backends into quadratic full scans and + // can retain backend node guards across an async callback. + let links = self.0.primary_index.pk_map + .iter_values() + .map(|(_, link)| link.0) + .collect::>(); + for link in links { + let data = self.0.data + .select_non_ghosted(link) + .map_err(WorkTableError::PagesError)?; + #func } core::result::Result::Ok(()) diff --git a/codegen/src/generators/in_memory/table/index_fns.rs b/codegen/src/generators/in_memory/table/index_fns.rs index 846c01bd..68670b69 100644 --- a/codegen/src/generators/in_memory/table/index_fns.rs +++ b/codegen/src/generators/in_memory/table/index_fns.rs @@ -85,7 +85,7 @@ impl InMemoryGenerator { }; let select = if cfg!(feature = "versioned-row-publication") { quote! { - loop { + for _ in 0..64 { let link: Link = self.0.indexes.#field_ident .lookup_for_select(#by) .map(Into::into)?; @@ -101,7 +101,9 @@ impl InMemoryGenerator { if current_link == Some(link) { return None; } + std::hint::spin_loop(); } + None } } else { quote! { @@ -149,15 +151,17 @@ impl InMemoryGenerator { #column_range_type, #row_fields_ident> { - let read_guard = self.0.data.read_guard(); - let rows = self.0.indexes.#field_ident - .get(#by) - .into_iter() - .filter_map(move |(_, link)| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }) - .filter(move |r| &r.#row_field_ident == &by); + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.indexes.#field_ident + .get(#by) + .into_iter() + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + .filter(move |r| &r.#row_field_ident == &by) + }).flatten(); SelectQueryBuilder::new(rows) } @@ -252,10 +256,14 @@ impl InMemoryGenerator { R: #range_bounds + 'a { #predicate_setup - let read_guard = self.0.data.read_guard(); - let rows = #index_range - .filter_map(#select_row) - #predicate_filter; + // Query construction is not an active read. Pin the grace + // period on the first row lookup instead. + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + #index_range + .filter_map(#select_row) + #predicate_filter + }).flatten(); SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/in_memory/table/mod.rs b/codegen/src/generators/in_memory/table/mod.rs index cd75ec87..ee1765ab 100644 --- a/codegen/src/generators/in_memory/table/mod.rs +++ b/codegen/src/generators/in_memory/table/mod.rs @@ -12,7 +12,7 @@ mod select_executor; impl InMemoryGenerator { pub fn gen_table_def(&mut self) -> syn::Result { let page_size_consts = self.gen_page_size_consts(); - let type_ = self.gen_table_type(); + let type_ = self.gen_table_type()?; let default = self.gen_table_default(); let impl_ = self.gen_table_impl(); let index_fns = self.gen_table_index_fns()?; @@ -46,7 +46,7 @@ impl InMemoryGenerator { } } - fn gen_table_type(&self) -> TokenStream { + fn gen_table_type(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let ident = name_generator.get_work_table_ident(); let row_type = name_generator.get_row_type_ident(); @@ -89,10 +89,9 @@ impl InMemoryGenerator { &key_type, &value_type, worktables_node, - ) - .unwrap_or_else(|error| error.into_compile_error()); + )?; - if self.config.as_ref().and_then(|c| c.page_size).is_some() { + Ok(if self.config.as_ref().and_then(|c| c.page_size).is_some() { quote! { #derive pub struct #ident( @@ -126,7 +125,7 @@ impl InMemoryGenerator { > ); } - } + }) } fn gen_page_size_consts(&self) -> TokenStream { diff --git a/codegen/src/generators/index_backend.rs b/codegen/src/generators/index_backend.rs index 0cf6f7d3..70c56db6 100644 --- a/codegen/src/generators/index_backend.rs +++ b/codegen/src/generators/index_backend.rs @@ -43,7 +43,7 @@ pub(crate) fn primary_key_backend_impl( IndexBackend::WorktablesIndex | IndexBackend::Indexset => Ok((quote! {}, quote! {})), IndexBackend::Congee => { let field = single_supported_field(backend, fields, &["u8", "u16", "u32", "u64", "usize"])?; - let width_guard = if field.to_string() == "u64" { + let width_guard = if primitive_name(field).as_deref() == Some("u64") { quote! { #[cfg(not(target_pointer_width = "64"))] compile_error!("`using congee` with a `u64` primary key requires a 64-bit target"); @@ -105,11 +105,12 @@ fn single_supported_field<'a>( format!("`using {}` requires a single-column primary key", backend.name()), )); }; - if !supported.contains(&field.to_string().as_str()) { + let primitive = primitive_name(field); + if !primitive.as_deref().is_some_and(|name| supported.contains(&name)) { return Err(syn::Error::new_spanned( *field, format!( - "`using {}` does not support primary-key type `{}`; supported types: {}", + "`using {}` requires a directly named primitive primary-key type; found `{}`; supported types: {} (type aliases cannot be resolved by the macro)", backend.name(), field, supported.join(", ") @@ -118,3 +119,10 @@ fn single_supported_field<'a>( } Ok(field) } + +fn primitive_name(field: &TokenStream) -> Option { + let syn::Type::Path(type_path) = syn::parse2::(field.clone()).ok()? else { + return None; + }; + type_path.path.segments.last().map(|segment| segment.ident.to_string()) +} diff --git a/codegen/src/generators/persist/queries/select.rs b/codegen/src/generators/persist/queries/select.rs index 66fad252..64247950 100644 --- a/codegen/src/generators/persist/queries/select.rs +++ b/codegen/src/generators/persist/queries/select.rs @@ -29,13 +29,18 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { - let read_guard = self.0.data.read_guard(); - let iter = self.0.primary_index.pk_map - .iter_links() - .filter_map(move |link| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }); + // Acquire the grace-period guard only when iteration starts. + // Merely constructing and retaining a query builder must not + // stall retired-link reclamation. + let iter = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map + .iter_links() + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + }).flatten(); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/persist/table/impls.rs b/codegen/src/generators/persist/table/impls.rs index 9bac16b2..4f23eb84 100644 --- a/codegen/src/generators/persist/table/impls.rs +++ b/codegen/src/generators/persist/table/impls.rs @@ -200,13 +200,17 @@ impl PersistGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); - let read_guard = self.0.data.read_guard(); - let rows = self.0.primary_index.pk_map - .range_links(converted_range) - .filter_map(move |link| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }); + // Delay the grace-period guard until the returned iterator is + // consumed so an idle query builder cannot pin reclamation. + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map + .range_links(converted_range) + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + }).flatten(); #pk_sorted_by } @@ -366,32 +370,18 @@ impl PersistGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { let _read_guard = self.0.data.read_guard(); - let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k, v.0)); - let Some((mut k, link)) = first else { - return Ok(()) - }; - - let data = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; - #func - - let mut ind = false; - while !ind { - let next = { - let mut iter = self.0.primary_index.pk_map.range_values(k.clone()..); - let next = iter.next().map(|(k, v)| (k.clone(), v.0)).filter(|(key, _)| key != &k); - if next.is_some() { - next - } else { - iter.next().map(|(k, v)| (k.clone(), v.0)) - } - }; - if let Some((key, link)) = next { - let data = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; - #func - k = key - } else { - ind = true; - }; + // Snapshot the ordered links once. Re-starting a range at every + // key turns materializing backends into quadratic full scans and + // can retain backend node guards across an async callback. + let links = self.0.primary_index.pk_map + .iter_values() + .map(|(_, link)| link.0) + .collect::>(); + for link in links { + let data = self.0.data + .select_non_ghosted(link) + .map_err(WorkTableError::PagesError)?; + #func } core::result::Result::Ok(()) diff --git a/codegen/src/generators/persist/table/index_fns.rs b/codegen/src/generators/persist/table/index_fns.rs index bd159c8d..bf99fbe6 100644 --- a/codegen/src/generators/persist/table/index_fns.rs +++ b/codegen/src/generators/persist/table/index_fns.rs @@ -85,7 +85,7 @@ impl PersistGenerator { }; let select = if cfg!(feature = "versioned-row-publication") { quote! { - loop { + for _ in 0..64 { let link: Link = self.0.indexes.#field_ident .lookup_for_select(#by) .map(Into::into)?; @@ -101,7 +101,9 @@ impl PersistGenerator { if current_link == Some(link) { return None; } + std::hint::spin_loop(); } + None } } else { quote! { @@ -149,15 +151,17 @@ impl PersistGenerator { #column_range_type, #row_fields_ident> { - let read_guard = self.0.data.read_guard(); - let rows = self.0.indexes.#field_ident - .get(#by) - .into_iter() - .filter_map(move |(_, link)| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }) - .filter(move |r| &r.#row_field_ident == &by); + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.indexes.#field_ident + .get(#by) + .into_iter() + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + .filter(move |r| &r.#row_field_ident == &by) + }).flatten(); SelectQueryBuilder::new(rows) } @@ -252,10 +256,14 @@ impl PersistGenerator { R: #range_bounds + 'a { #predicate_setup - let read_guard = self.0.data.read_guard(); - let rows = #index_range - .filter_map(#select_row) - #predicate_filter; + // Query construction is not an active read. Pin the grace + // period on the first row lookup instead. + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + #index_range + .filter_map(#select_row) + #predicate_filter + }).flatten(); SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/persist/table/mod.rs b/codegen/src/generators/persist/table/mod.rs index f35e2693..b8930c92 100644 --- a/codegen/src/generators/persist/table/mod.rs +++ b/codegen/src/generators/persist/table/mod.rs @@ -13,7 +13,7 @@ impl PersistGenerator { pub fn gen_table_def(&mut self) -> syn::Result { let page_size_consts = self.gen_page_size_consts(); let version_const = self.gen_version_const(); - let type_ = self.gen_table_type(); + let type_ = self.gen_table_type()?; let impl_ = self.gen_table_impl(); let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); @@ -59,7 +59,7 @@ impl PersistGenerator { } } - fn gen_table_type(&self) -> TokenStream { + fn gen_table_type(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let ident = name_generator.get_work_table_ident(); let row_type = name_generator.get_row_type_ident(); @@ -84,16 +84,27 @@ impl PersistGenerator { }) .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); + let pk_upstream = matches!( + self.columns.primary_index_backend, + crate::common::model::IndexBackend::Indexset + ); - let derive = if pk_types_unsized { - quote! { + let derive = match (pk_types_unsized, pk_upstream) { + (true, true) => quote! { + #[derive(Debug, PersistTable)] + #[table(pk_unsized, pk_upstream)] + }, + (true, false) => quote! { #[derive(Debug, PersistTable)] #[table(pk_unsized)] - } - } else { - quote! { + }, + (false, true) => quote! { #[derive(Debug, PersistTable)] - } + #[table(pk_upstream)] + }, + (false, false) => quote! { + #[derive(Debug, PersistTable)] + }, }; let key_type = quote! { #primary_key_type }; @@ -110,10 +121,9 @@ impl PersistGenerator { &key_type, &value_type, worktables_node, - ) - .unwrap_or_else(|error| error.into_compile_error()); + )?; - if self.config.as_ref().and_then(|c| c.page_size).is_some() { + Ok(if self.config.as_ref().and_then(|c| c.page_size).is_some() { quote! { #derive pub struct #ident( @@ -149,6 +159,6 @@ impl PersistGenerator { , #persistence_task ); } - } + }) } } diff --git a/codegen/src/generators/read_only/queries/select.rs b/codegen/src/generators/read_only/queries/select.rs index 79bffa6f..11dd8309 100644 --- a/codegen/src/generators/read_only/queries/select.rs +++ b/codegen/src/generators/read_only/queries/select.rs @@ -29,13 +29,18 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { - let read_guard = self.0.data.read_guard(); - let iter = self.0.primary_index.pk_map - .iter_links() - .filter_map(move |link| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }); + // Acquire the grace-period guard only when iteration starts. + // Merely constructing and retaining a query builder must not + // stall retired-link reclamation. + let iter = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map + .iter_links() + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + }).flatten(); SelectQueryBuilder::new(iter) } diff --git a/codegen/src/generators/read_only/table/impls.rs b/codegen/src/generators/read_only/table/impls.rs index ac96c660..08c19fd7 100644 --- a/codegen/src/generators/read_only/table/impls.rs +++ b/codegen/src/generators/read_only/table/impls.rs @@ -196,13 +196,17 @@ impl ReadOnlyGenerator { range.start_bound().map(|v| #primary_key_type::from(v.clone())), range.end_bound().map(|v| #primary_key_type::from(v.clone())), ); - let read_guard = self.0.data.read_guard(); - let rows = self.0.primary_index.pk_map - .range_links(converted_range) - .filter_map(move |link| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }); + // Delay the grace-period guard until the returned iterator is + // consumed so an idle query builder cannot pin reclamation. + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.primary_index.pk_map + .range_links(converted_range) + .filter_map(move |link| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + }).flatten(); #pk_sorted_by } @@ -286,32 +290,18 @@ impl ReadOnlyGenerator { fn gen_table_iter_inner(&self, func: TokenStream) -> TokenStream { quote! { let _read_guard = self.0.data.read_guard(); - let first = self.0.primary_index.pk_map.iter_values().next().map(|(k, v)| (k, v.0)); - let Some((mut k, link)) = first else { - return Ok(()) - }; - - let data = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; - #func - - let mut ind = false; - while !ind { - let next = { - let mut iter = self.0.primary_index.pk_map.range_values(k.clone()..); - let next = iter.next().map(|(k, v)| (k.clone(), v.0)).filter(|(key, _)| key != &k); - if next.is_some() { - next - } else { - iter.next().map(|(k, v)| (k.clone(), v.0)) - } - }; - if let Some((key, link)) = next { - let data = self.0.data.select_non_ghosted(link).map_err(WorkTableError::PagesError)?; - #func - k = key - } else { - ind = true; - }; + // Snapshot the ordered links once. Re-starting a range at every + // key turns materializing backends into quadratic full scans and + // can retain backend node guards across an async callback. + let links = self.0.primary_index.pk_map + .iter_values() + .map(|(_, link)| link.0) + .collect::>(); + for link in links { + let data = self.0.data + .select_non_ghosted(link) + .map_err(WorkTableError::PagesError)?; + #func } core::result::Result::Ok(()) diff --git a/codegen/src/generators/read_only/table/index_fns.rs b/codegen/src/generators/read_only/table/index_fns.rs index d606600f..42f9ff92 100644 --- a/codegen/src/generators/read_only/table/index_fns.rs +++ b/codegen/src/generators/read_only/table/index_fns.rs @@ -85,7 +85,7 @@ impl ReadOnlyGenerator { }; let select = if cfg!(feature = "versioned-row-publication") { quote! { - loop { + for _ in 0..64 { let link: Link = self.0.indexes.#field_ident .lookup_for_select(#by) .map(Into::into)?; @@ -101,7 +101,9 @@ impl ReadOnlyGenerator { if current_link == Some(link) { return None; } + std::hint::spin_loop(); } + None } } else { quote! { @@ -149,15 +151,17 @@ impl ReadOnlyGenerator { #column_range_type, #row_fields_ident> { - let read_guard = self.0.data.read_guard(); - let rows = self.0.indexes.#field_ident - .get(#by) - .into_iter() - .filter_map(move |(_, link)| { - let _read_guard = &read_guard; - self.0.data.select_non_ghosted(link.0).ok() - }) - .filter(move |r| &r.#row_field_ident == &by); + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + self.0.indexes.#field_ident + .get(#by) + .into_iter() + .filter_map(move |(_, link)| { + let _read_guard = &read_guard; + self.0.data.select_non_ghosted(link.0).ok() + }) + .filter(move |r| &r.#row_field_ident == &by) + }).flatten(); SelectQueryBuilder::new(rows) } @@ -252,10 +256,14 @@ impl ReadOnlyGenerator { R: #range_bounds + 'a { #predicate_setup - let read_guard = self.0.data.read_guard(); - let rows = #index_range - .filter_map(#select_row) - #predicate_filter; + // Query construction is not an active read. Pin the grace + // period on the first row lookup instead. + let rows = std::iter::once_with(move || { + let read_guard = self.0.data.read_guard(); + #index_range + .filter_map(#select_row) + #predicate_filter + }).flatten(); SelectQueryBuilder::new_sorted(rows, #row_fields_ident::#column_pascal) } diff --git a/codegen/src/generators/read_only/table/mod.rs b/codegen/src/generators/read_only/table/mod.rs index 8b1993f8..72bc8c0d 100644 --- a/codegen/src/generators/read_only/table/mod.rs +++ b/codegen/src/generators/read_only/table/mod.rs @@ -13,7 +13,7 @@ impl ReadOnlyGenerator { pub fn gen_table_def(&mut self) -> syn::Result { let page_size_consts = self.gen_page_size_consts(); let version_const = self.gen_version_const(); - let type_ = self.gen_table_type(); + let type_ = self.gen_table_type()?; let impl_ = self.gen_table_impl(); let index_fns = self.gen_table_index_fns()?; let select_query_executor_impl = self.gen_table_select_query_executor_impl(); @@ -51,7 +51,7 @@ impl ReadOnlyGenerator { } } - fn gen_table_type(&self) -> TokenStream { + fn gen_table_type(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_table_name(self.name.to_string()); let ident = name_generator.get_work_table_ident(); let row_type = name_generator.get_row_type_ident(); @@ -75,21 +75,32 @@ impl ReadOnlyGenerator { }) .collect::>(); let pk_types_unsized = is_unsized_vec(pk_types); + let pk_upstream = matches!( + self.columns.primary_index_backend, + crate::common::model::IndexBackend::Indexset + ); // `read_only` and `pk_unsized` are independent: the first selects the read-only // shape of the table (no persistence engine or task, sync `into_worktable`), the // second selects the unsized primary index. A read-only table with an unsized key // needs both, so `read_only` is unconditional here. - let derive = if pk_types_unsized { - quote! { + let derive = match (pk_types_unsized, pk_upstream) { + (true, true) => quote! { + #[derive(Debug, PersistTable)] + #[table(read_only, pk_unsized, pk_upstream)] + }, + (true, false) => quote! { #[derive(Debug, PersistTable)] #[table(read_only, pk_unsized)] - } - } else { - quote! { + }, + (false, true) => quote! { + #[derive(Debug, PersistTable)] + #[table(read_only, pk_upstream)] + }, + (false, false) => quote! { #[derive(Debug, PersistTable)] #[table(read_only)] - } + }, }; let key_type = quote! { #primary_key_type }; @@ -106,10 +117,9 @@ impl ReadOnlyGenerator { &key_type, &value_type, worktables_node, - ) - .unwrap_or_else(|error| error.into_compile_error()); + )?; - quote! { + Ok(quote! { #derive pub struct #ident( WorkTable< @@ -124,6 +134,6 @@ impl ReadOnlyGenerator { #node_type > ); - } + }) } } diff --git a/codegen/src/persist_index/generator.rs b/codegen/src/persist_index/generator.rs index 60b72789..66e63646 100644 --- a/codegen/src/persist_index/generator.rs +++ b/codegen/src/persist_index/generator.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use proc_macro2::{Ident, Literal, TokenStream}; use quote::__private::Span; use quote::{ToTokens, quote}; -use syn::ItemStruct; +use syn::{Field, ItemStruct}; use crate::common::name_generator::{WorktableNameGenerator, is_unsized}; use crate::persist_table::WT_INDEX_EXTENSION; @@ -19,6 +19,44 @@ pub struct Generator { pub attributes: PersistIndexAttributes, } +struct IndexLayout { + type_ident: Ident, + is_unique: bool, + uses_upstream: bool, +} + +fn index_layout(field: &Field) -> syn::Result { + let syn::Type::Path(type_path) = &field.ty else { + return Err(syn::Error::new_spanned( + &field.ty, + "index field must be a concrete index type", + )); + }; + let type_ident = type_path + .path + .segments + .last() + .ok_or_else(|| syn::Error::new_spanned(&field.ty, "index type path cannot be empty"))? + .ident + .clone(); + let (is_unique, uses_upstream) = match type_ident.to_string().as_str() { + "IndexMap" | "TreeIndex" => (true, false), + "UpstreamIndexMap" => (true, true), + "IndexMultiMap" | "TreeMultiIndex" => (false, false), + _ => { + return Err(syn::Error::new_spanned( + &field.ty, + "unsupported persisted index type; use IndexMap, UpstreamIndexMap, or IndexMultiMap directly", + )); + } + }; + Ok(IndexLayout { + type_ident, + is_unique, + uses_upstream, + }) +} + impl WorktableNameGenerator { pub fn from_index_ident(index_ident: &Ident) -> Self { Self { @@ -245,7 +283,7 @@ impl Generator { } } } else { - self.gen_get_persisted_index_fn() + self.gen_get_persisted_index_fn()? }; let from_persisted_fn = self.gen_from_persisted_fn()?; @@ -261,7 +299,7 @@ impl Generator { /// Generates `get_persisted_index` function of `PersistableIndex` trait for persisted index. It maps every /// `TreeIndex` into `Vec` of `IndexPage`s using `IndexPage::from_nod` function. - fn gen_get_persisted_index_fn(&self) -> TokenStream { + fn gen_get_persisted_index_fn(&self) -> syn::Result { let name_generator = WorktableNameGenerator::from_index_ident(&self.struct_def.ident); let const_name = name_generator.get_page_inner_size_const_ident(); @@ -276,6 +314,7 @@ impl Generator { .fields .iter() .map(|field| { + let layout = index_layout(field)?; let i = field .ident .as_ref() @@ -284,9 +323,8 @@ impl Generator { .field_types .get(i) .expect("should be available as constructed from same values"); - let uses_upstream = field.ty.to_token_stream().to_string().contains("UpstreamIndexMap"); if is_unsized(&ty.to_string()) { - quote! { + Ok(quote! { let mut pages = vec![]; for node in self.#i.iter_nodes() { let page = UnsizedIndexPage::from_node(node.lock_arc().as_ref()); @@ -294,9 +332,9 @@ impl Generator { } let (toc, pages) = map_unsized_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); let #i = (toc.pages, pages); - } - } else if uses_upstream { - quote! { + }) + } else if layout.uses_upstream { + Ok(quote! { let size = get_index_page_size_from_data_length::<#ty>(#const_name); let mut pages = vec![]; for node in self.#i.iter_nodes() { @@ -312,9 +350,9 @@ impl Generator { } let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); let #i = (toc.pages, pages); - } + }) } else { - quote! { + Ok(quote! { let size = get_index_page_size_from_data_length::<#ty>(#const_name); let mut pages = vec![]; for node in self.#i.iter_nodes() { @@ -323,19 +361,19 @@ impl Generator { } let (toc, pages) = map_index_pages_to_toc_and_general::<_, { #const_name as u32 }>(pages); let #i = (toc.pages, pages); - } + }) } }) - .collect(); + .collect::>>()?; - quote! { + Ok(quote! { fn get_persisted_index(&self) -> Self::PersistedIndex { #(#field_names_init)* Self::PersistedIndex { #(#idents,)* } } - } + }) } /// Generates `from_persisted` function of `PersistableIndex` trait for persisted index. It maps every page in @@ -355,15 +393,11 @@ impl Generator { .fields .iter() .map(|f| { + let layout = index_layout(f)?; let i = f.ident.as_ref().expect("index fields should always be named fields"); - let index_type = f.ty.to_token_stream().to_string(); - let is_unique = !index_type.contains("IndexMultiMap"); - let uses_upstream = index_type.contains("UpstreamIndexMap"); - let mut split = index_type.split("<"); - let t = Ident::new( - split.next().expect("index type should always have generics").trim(), - Span::mixed_site(), - ); + let is_unique = layout.is_unique; + let uses_upstream = layout.uses_upstream; + let t = layout.type_ident; let ty = self .field_types .get(i) @@ -448,41 +482,41 @@ impl Generator { let node = UnsizedNode::from_inner(inner, #const_name); #i.attach_node(node); }); - quote! { + Ok(quote! { let #i: #t<_, OffsetEqLink, UnsizedNode<_>> = #t::with_maximum_node_size(#const_name); #body - } + }) } else { let body = multi_reconstruct(quote! { let node = UnsizedNode::from_inner(sorted, #const_name); #i.attach_multi_node(node); }); - quote! { + Ok(quote! { let #i: #t<_, OffsetEqLink, UnsizedNode<_>> = #t::with_maximum_node_size(#const_name); #body - } + }) } } else if is_unique { let body = unique_reconstruct(quote! { #i.attach_node(inner); }); - quote! { + Ok(quote! { let size = get_index_page_size_from_data_length::<#ty>(#const_name); let #i: #t<_, OffsetEqLink> = #t::with_maximum_node_size(size); #body - } + }) } else { let body = multi_reconstruct(quote! { #i.attach_multi_node(sorted); }); - quote! { + Ok(quote! { let size = get_index_page_size_from_data_length::<#ty>(#const_name); let #i: #t<_, OffsetEqLink> = #t::with_maximum_node_size(size); #body - } + }) } }) - .collect::>(); + .collect::>>()?; Ok(quote! { fn from_persisted(persisted: Self::PersistedIndex) -> Self { @@ -570,4 +604,19 @@ mod tests { assert!(!attrs.read_only); } + + #[test] + fn rejects_aliases_instead_of_guessing_the_persisted_layout() { + let input = quote! { + #[derive(Debug, Default, Clone)] + pub struct TestIndex { + test_idx: MyIndexAlias, + } + }; + let struct_ = Parser::parse_struct(input).unwrap(); + let generator = Generator::with_attributes(struct_, PersistIndexAttributes::default()); + + let error = generator.gen_persistable_impl().unwrap_err(); + assert!(error.to_string().contains("unsupported persisted index type")); + } } diff --git a/codegen/src/persist_table/generator/mod.rs b/codegen/src/persist_table/generator/mod.rs index 70f5483d..767a36c9 100644 --- a/codegen/src/persist_table/generator/mod.rs +++ b/codegen/src/persist_table/generator/mod.rs @@ -11,6 +11,7 @@ mod space_file; pub struct PersistTableAttributes { pub pk_unsized: bool, pub read_only: bool, + pub pk_upstream: bool, } pub struct Generator { diff --git a/codegen/src/persist_table/mod.rs b/codegen/src/persist_table/mod.rs index 10f43266..530e6e3f 100644 --- a/codegen/src/persist_table/mod.rs +++ b/codegen/src/persist_table/mod.rs @@ -12,8 +12,8 @@ pub use generator::WT_INDEX_EXTENSION; pub fn expand(input: TokenStream) -> syn::Result { let input_fn = Parser::parse_struct(input)?; let pk_ident = Parser::parse_pk_ident(&input_fn); - let pk_upstream = Parser::primary_key_uses_upstream(&input_fn); let attributes = Parser::parse_attributes(&input_fn.attrs); + let pk_upstream = attributes.pk_upstream; let generator = Generator { struct_def: input_fn, @@ -99,4 +99,17 @@ mod tests { "normal into_worktable should be async" ); } + + #[test] + fn pk_upstream_uses_the_upstream_persistence_adapter() { + let input = quote! { + #[derive(Debug)] + #[table(pk_upstream)] + pub struct TestWorkTable(WorkTable); + }; + + let output = expand(input).unwrap().to_string(); + assert!(output.contains("UpstreamIndexMap")); + assert!(output.contains("UpstreamIndexPair")); + } } diff --git a/codegen/src/persist_table/parser.rs b/codegen/src/persist_table/parser.rs index 06281426..b997c0f2 100644 --- a/codegen/src/persist_table/parser.rs +++ b/codegen/src/persist_table/parser.rs @@ -25,21 +25,11 @@ impl Parser { Ident::new(pk_type.trim(), Span::mixed_site()) } - pub fn primary_key_uses_upstream(item: &ItemStruct) -> bool { - item.fields - .iter() - .next() - .expect("WorkTable wrapper has one field") - .ty - .to_token_stream() - .to_string() - .contains("UpstreamIndexMap") - } - pub fn parse_attributes(attrs: &Vec) -> PersistTableAttributes { let mut res = PersistTableAttributes { pk_unsized: false, read_only: false, + pk_upstream: false, }; for attr in attrs { @@ -53,6 +43,10 @@ impl Parser { res.read_only = true; return Ok(()); } + if meta.path.is_ident("pk_upstream") { + res.pk_upstream = true; + return Ok(()); + } Ok(()) }) .expect("always ok even on unrecognized attrs"); diff --git a/codegen/src/worktable/mod.rs b/codegen/src/worktable/mod.rs index bf3c54fd..30876ef9 100644 --- a/codegen/src/worktable/mod.rs +++ b/codegen/src/worktable/mod.rs @@ -305,6 +305,10 @@ mod tests { }) .unwrap_err(); - assert!(error.to_string().contains("does not support primary-key type `String`")); + assert!( + error + .to_string() + .contains("requires a directly named primitive primary-key type; found `String`") + ); } } diff --git a/docs/index-backend-dsl-proposal.md b/docs/index-backend-dsl-proposal.md index 947b73af..13b074ea 100644 --- a/docs/index-backend-dsl-proposal.md +++ b/docs/index-backend-dsl-proposal.md @@ -154,12 +154,13 @@ Backend dispatch itself is static and should compile away. That does **not** mea - Both can emit persistence-compatible structural CDC. Direct dispatch preserves a strict generated point-read contract with a -provider-specific implementation. WorkTablesIndex keeps the normal successful -hit path and sends only an apparent miss through a cold three-node boundary -confirmation. Vanilla IndexSet does not expose a comparable structural -validation primitive; `using indexset` therefore remains experimental and is -excluded from concurrent correctness and published performance claims. This -WorkTablesIndex visibility guarantee is independent of +provider-specific implementation. WorkTablesIndex 0.0.4 holds its structural +mapping stable until the selected node is locked, so both hits and misses are +definitive; a contended lookup drops the structural guard before waiting and +then retries the mapping. Vanilla IndexSet does not expose a comparable +structural validation primitive; `using indexset` therefore remains +experimental and is excluded from concurrent correctness and published +performance claims. This WorkTablesIndex visibility guarantee is independent of `versioned-row-publication`, which addresses concurrent page bytes, ghost publication, and reclamation rather than index routing. @@ -167,16 +168,22 @@ publication, and reclamation rather than index routing. - Point lookup and mutation call Congee directly. - WorkTable links do not fit in Congee's one-word payload. The adapter stores an `Arc` pointer, so inserts allocate and reads clone the `Arc` before copying the link. -- `iter_values` and `range_values` currently collect a full key/value snapshot and sort it. A narrow range therefore scans and allocates for the whole index. +- Ordered reads use Congee's native range scan and materialize only the requested key interval into a `Vec`; a full iteration is therefore O(n) with one result allocation, while a narrow range no longer dumps, re-probes, and sorts the whole tree. ### Arctic - Point lookup and mutation call Arctic directly. - WorkTable links are stored in `Box` values because Arctic's inline value is limited to 64 bits. Inserts allocate; reads copy the link from the box. -- Ordered scans currently collect all entries into a `Vec` and then filter the requested range. +- Ordered reads use Arctic's native bounded traversal and materialize the requested interval into a `Vec`. - Concurrent scan behavior inherits Arctic's non-linearizable traversal contract. -The ART adapters are consequently candidates for point-heavy, explicitly memory-only paths—not automatic wins for `select_all`, range-heavy access, iteration, or vacuum. Tomorrow's measurements must separate point lookup, write/allocation cost, range width, full iteration, and reclamation rather than reporting one blended throughput number. +Generated table traversal snapshots its ordered link list once instead of +restarting a range at every row. The ART adapters are still candidates for +point-heavy, explicitly memory-only paths—not automatic wins for `select_all`, +wide ranges, iteration, or vacuum, because ordered results are materialized +rather than streamed. Measurements must separate point lookup, +write/allocation cost, range width, full iteration, and reclamation rather than +reporting one blended throughput number. ### Memory diagnostics diff --git a/docs/versioned-row-publication.md b/docs/versioned-row-publication.md index 2c581338..9d79dd2b 100644 --- a/docs/versioned-row-publication.md +++ b/docs/versioned-row-publication.md @@ -25,8 +25,9 @@ With the feature enabled, `DataPages` maintains two representations: accesses that can overlap a mutation are serialized by an internal page barrier. - A concurrent link map holds an immutable application-visible row version. - Each slot contains an `Arc` behind a short per-slot pointer lock and - atomic ghost, deleted, and vacuum lifecycle bits. + Each slot contains one `Arc` and its ghost, deleted, and vacuum lifecycle + bits in a single version protected by a short per-slot lock. A reader cannot + observe a row from one publication together with flags from another. The generated API follows these publication rules: @@ -35,11 +36,12 @@ The generated API follows these publication rules: index predicates, clone the owned row, and release the guard. Unique and primary-key point reads retry when the mapping swings to a replacement link while it is being resolved. Point lookup itself uses each provider's strict - visibility path: WorkTablesIndex confirms an apparent miss against a stable - three-node structural window and the ART providers retain their native - concurrent point algorithms. Vanilla `using indexset` is experimental and - excluded from this concurrent-read guarantee. A reader never accesses - mutable archived bytes after a slot has been hydrated. + visibility path: WorkTablesIndex 0.0.4 holds its structural mapping stable + until the selected node is locked, making hits and misses definitive, while + the ART providers retain their native concurrent point algorithms. Vanilla + `using indexset` is experimental and excluded from this concurrent-read + guarantee. A reader never accesses mutable archived bytes after a slot has + been hydrated. 2. **Insert.** Serialize the complete row and stage a ghosted version. Install the primary and secondary indexes. Only after every checked index insert succeeds does the lifecycle transition publish the version with release @@ -55,13 +57,31 @@ The generated API follows these publication rules: indexes, and retire the source publication and page. Retired links, slots, and pages become reusable only after a read-side grace period. 6. **Reload.** Persisted tables hydrate immutable slots lazily under the page - barrier. Subsequent generated reads use the published version map. + barrier. The first read of a row holds the barrier's shared side across + deserialization and publication, so it temporarily excludes writers. + Latency-sensitive applications can warm the table with a scan before + admitting write traffic. Subsequent generated reads use the published + version map. The grace period is quiescent-state reclamation: a feature-only atomic counter tracks generated reads, and retirement queues are drained when that counter is zero. `Arc` ownership independently keeps a version alive after a reader has acquired it. +Creating a lazy `SelectQueryBuilder` does not enter the grace period. The guard +is acquired when iteration first starts, before the backend can yield its first +link, and is released when that iterator is consumed or dropped. A partially +consumed iterator is still an active read: retaining one intentionally delays +link, publication, and page reuse. Retirement backlogs emit progressively +spaced warnings after 1,024 entries so an abandoned or unusually long scan is +observable. + +Retirement follows a strict unlink-before-retire rule. Delete and vacuum remove +or replace every index reference before queueing the old physical link. A +reader that could still resolve the old reference therefore entered the grace +period before reclamation observed quiescence; a later reader cannot acquire +that retired reference. + ## Guarantees and non-guarantees For generated table APIs in this mode: @@ -76,18 +96,22 @@ For generated table APIs in this mode: This is not MVCC and does not add multi-operation transactions or snapshot range scans. A scan may include or omit a concurrently inserted or updated row. -Point-read replacement retry may starve under perpetual replacement churn. -The guarantee also does not cover callers that bypass generated table methods -and directly invoke low-level `Data` page mutation APIs. +A point read retries a mapping that changes while its row version is resolved, +but returns `None` after 64 consecutive replacement races rather +than spinning without a bound. The guarantee also does not cover callers that +bypass generated table methods and directly invoke low-level `Data` page +mutation APIs. ## Cost model and rollout -The feature is off by default. It adds one owned row copy plus slot/map +The feature is off by default. Cargo features unify across a dependency graph, +so any dependency enabling it enables it for every WorkTable consumer in that +build. It adds one owned row copy plus slot/map metadata per live physical link, an atomic increment/decrement per generated -read, a concurrent publication-map lookup, and writer-side page serialization. +read, a sharded publication-map lookup, and writer-side page serialization. The index-visibility algorithm is always active and separate from row -publication: WorkTablesIndex adds no second probe to successful point hits, but -an apparent miss enters a cold three-node confirmation path. +publication: WorkTablesIndex acquires the selected node while its structural +mapping is pinned on the uncontended path, and may retry after node contention. Those costs are inappropriate to impose silently on latency-sensitive users. The default path remains unchanged; benchmark results for both modes must be reported before this feature is proposed for default enablement. diff --git a/response.md b/response.md new file mode 100644 index 00000000..8bf50269 --- /dev/null +++ b/response.md @@ -0,0 +1,232 @@ +# Response to `review.md` + +Thank you for the detailed review. The review was made against `0218d40`; the +branch has moved since then, including the stable WorkTablesIndex read contract +from WorkTablesIndex PR #5 and the follow-up changes described here. + +The four blocking hot-path/CI findings and the publication tearing issue have +been addressed. The counter-based reclamation design has been made materially +safer and observable, but it has not been replaced with a full epoch/QSBR +implementation. That remaining boundary is called out explicitly below. + +## Blocking findings + +### B1: default-build row clones — fixed + +- `insert` and `update` now move the row into its wrapper when versioned + publication is disabled. +- The clone exists only in the versioned configuration, where the owned row is + also needed for publication. +- Successful insert paths move the publication row rather than cloning it a + second time. + +The separate `insert_cdc` reserialization optimization remains a follow-up; it +is not mixed into this correctness patch. + +### B2: grace period and retained query builders — contained, not converted to epochs + +The counter scheme remains, but the structural idle-builder leak is fixed: + +- Lazy `SelectQueryBuilder`s do not acquire a read guard at construction. +- The guard is acquired when iteration starts, before the backend can yield its + first link, and lives only as long as the active iterator. +- A regression test holds an unconsumed builder across delete/insert and proves + that it does not prevent physical-link reuse. +- Delete and vacuum follow a documented unlink-before-retire invariant. A new + reader cannot resolve a retired link after all index references have been + removed; readers that could have resolved it entered the grace period first. +- Retirement backlogs warn at powers of two starting at 1,024 entries. +- An atomic pending count avoids taking the retirement queue locks when there + is no reclamation work. + +A partially consumed or abandoned active iterator still delays reuse. That is +now documented and observable, but it is not the same isolation property as a +real epoch implementation. A future epoch/QSBR conversion remains worthwhile. +A hard queue cap cannot safely discard retirement records; it would need to +apply backpressure or move reclamation to an epoch collector. + +### B3: ART ordered scans — fixed + +- Arctic range operations translate Rust bounds to Arctic's native range API. +- Congee range operations use Congee's native range traversal under a pinned + epoch instead of enumerating all keys and doing one lookup per key. +- Generated table iteration snapshots the ordered links once. It no longer + restarts a range for every row, removing the quadratic behavior. +- Inclusive, exclusive, unbounded, empty, and maximum-key bounds have coverage. +- Criterion now includes single-row primary-key ranges over 10,000-row Congee + and Arctic tables. + +The adapters still materialize the requested interval because the common API +returns an owned double-ended iterator. The cost is now proportional to the +requested interval rather than the entire index per result row. + +### B4: default CI coverage — fixed + +CI now builds and tests a matrix containing: + +- default features; +- `versioned-row-publication`; +- all features, to retain an additive-feature-unification check. + +Strict Clippy runs for both default and all-feature configurations. +WorkTablesIndex PR #5 defines deterministic precedence when multiple search +features are unified. The README now documents that behavior, so WorkTable does +not add a contradictory `compile_error!` for a valid unified Cargo graph. + +There is no longer a separate `stable-index-read-retry` WorkTable feature. The +updated WorkTablesIndex `lookup_for_select` contract is definitive for both +hits and misses. + +## Correctness and concurrency findings + +### C1: torn `(row, flags)` publication — fixed + +The row `Arc` and lifecycle flags now live in one `PublishedVersion` protected +by one short per-slot lock. `load()` reads both under the same lock. A concurrent +test alternates paired row/flag states for 100,000 writes and rejects mixed +versions. + +### C2: global publication-map contention — fixed + +- The single map is replaced with 64 publication shards. +- Shards are selected from a mixed physical offset. +- Reclamation has a pending-work fast path and no longer takes publication + locks when all retirement queues are empty. + +### C3: identity publication hash — fixed + +The physical offset is passed through a SplitMix-style avalanche before it is +used by the hash table, distributing both bucket bits and hashbrown control +tags. + +### C4: unbounded point-read retry — fixed + +Primary-key and generated unique-index point reads retry at most 64 mapping +replacements and issue `spin_loop()` on the retry edge. Perpetual churn can now +produce a bounded `None`, not an indefinitely spinning read. + +### C5: CDC method resolution — fixed + +Both WorkTablesIndex and upstream IndexSet CDC calls now use explicit inherent +type qualification. They cannot silently turn into recursive trait calls. + +### C6: UUID ordering — clarified and dependency floor raised + +The resolved `uuid` implementation does use a shared monotonic v7 context. +Its `Uuid::now_v7()` contract states that UUIDs generated by the same process +are ordered by creation. WorkTable-generated operations use that function. + +The minimum dependency is now `uuid 1.24.0`, matching the documented guarantee, +and `latest_data_writes` documents the constraint. Manually constructed +operations must preserve the ordering contract. A separate `AtomicU64` was not +added because it would duplicate the current operation ID's guaranteed ordering +and expand the persisted/CDC operation representation. + +Reference: + +### C7 and C8: hydration and lock hierarchy — documented + +- Cold hydration's writer-exclusion cost and the warm-up option are documented. +- `DataPages` now states its multi-lock hierarchy, the retirement queue rule, + and the unlink-before-retire obligation. +- `move_row_for_vacuum` now has an explicit `# Safety` contract. + +## Code generation findings + +### D1: token substring backend detection — fixed + +Persisted-index layout is derived from parsed type paths and structured +`pk_upstream` metadata. Unknown types and aliases are rejected instead of being +guessed into an on-disk representation. The primary backend is threaded into +the generated persistence attribute rather than recovered from rendered +tokens. + +### D2: stringified primitive types — improved + +Primitive checks parse `syn::Type::Path` and compare the final path segment. +The diagnostic explicitly says that a directly named primitive is required and +that proc macros cannot resolve Rust type aliases. The current DSL grammar still +accepts identifiers rather than arbitrary qualified type syntax. + +### D3: `compile_error!` in type position — fixed + +The affected table generators now return `syn::Result` and propagate the +original diagnostic with `?`. + +### D4: macro-host feature coupling — documented, not redesigned + +The Cargo feature-unification effect is now explicit in the publication docs. +Replacing the forwarded proc-macro feature with a build-script cfg would be a +larger packaging change and is deferred. + +### D5: upsert rewrite — unchanged in this follow-up + +The single logical row lock and retry rationale remain. Removing redundant +probes and reducing wide-row clones is still a useful measured optimization, +but is separate from these correctness fixes. + +### D6: statement-level cfgs — fixed + +The non-versioned `mark_page_empty` branch is grouped in one cfg block. + +## API and packaging findings + +- **E1:** making Arctic, Congee, and upstream IndexSet optional is deferred. It + changes the feature/API contract and deserves its own binary-size and compile- + time change. +- **E2:** WorkTablesIndex PR #5 deliberately supports additive Cargo feature + unification with deterministic search-policy precedence; this is documented. +- **E3:** the publication docs now state that enabling the feature anywhere in + a dependency graph enables it for all WorkTable consumers in that build. +- **E4:** moving backend types out of the prelude is deferred because it is a + public API decision rather than a correctness patch. +- **E5:** the unsafe vacuum move now documents all caller obligations. + +## Tests and validation + +Added or expanded coverage includes: + +- coherent row/flag publication under concurrent replacement; +- idle query builders and retirement/link reuse; +- native Congee and Arctic range-bound translation; +- generated range-result predicate revalidation; +- persisted index aliases rejected instead of misclassified; +- the ART primary-key range Criterion smoke benchmark. + +The persistence test that formatted the entire primary index on failure has +been restored to a concise assertion. + +Validation against DataBucket 0.5.1 and the updated WorkTablesIndex PR #5 head: + +- strict Clippy, default configuration: pass; +- strict Clippy, all features: pass; +- full default suite: 127 library tests, 333 integration tests (3 ignored), and + 64 codegen tests passed, plus all benchmark smoke targets; +- full all-feature suite: 132 library tests, 336 integration tests (4 ignored), + and 64 codegen tests passed, plus all benchmark smoke targets. + +One initial all-feature run hit an assertion inside `arctic-map 0.1.4` during +its disjoint concurrent mutation test. The unchanged isolated test then passed +20/20 repetitions and the complete all-feature suite passed on rerun. This is +being reported rather than hidden because the dependency is young and pinned. + +No latency claims are made from this host while unrelated benchmarks are +running. The Criterion cases are in place, but publishable before/after numbers +should be collected on a quiet machine. + +## Remaining follow-ups + +The intentional remaining work is: + +1. replace the counter grace period with a true epoch/QSBR collector if active + scans must coexist indefinitely with prompt physical-link reuse; +2. make alternate backend dependencies optional and decide whether their types + belong in the prelude before the stable API; +3. measure and then optimize duplicate insert serialization and redundant + upsert probes/clones; +4. improve ART memory accounting and address the pre-existing row-count reuse + drift separately; +5. run the default/versioned latency matrix on a quiet benchmark host. + +Those are kept explicit so the current patch is not represented as eliminating +every cost or lifecycle caveat raised by the review. diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 475a3ba3..56912271 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -18,6 +18,8 @@ use std::collections::VecDeque; #[cfg(feature = "versioned-row-publication")] use std::hash::{BuildHasherDefault, Hasher}; use std::marker::PhantomData; +#[cfg(feature = "versioned-row-publication")] +use std::sync::atomic::AtomicUsize; use std::{ fmt::Debug, sync::Arc, @@ -42,8 +44,24 @@ fn page_id_mapper(page_id: usize) -> usize { page_id - 1usize } +#[cfg(feature = "versioned-row-publication")] +const PUBLICATION_SHARD_COUNT: usize = 64; +#[cfg(feature = "versioned-row-publication")] +const RETIREMENT_BACKLOG_WARN_AT: usize = 1_024; + +#[cfg(feature = "versioned-row-publication")] +fn mix_publication_offset(mut value: u64) -> u64 { + value ^= value >> 30; + value = value.wrapping_mul(0xbf58_476d_1ce4_e5b9); + value ^= value >> 27; + value = value.wrapping_mul(0x94d0_49bb_1331_11eb); + value ^ (value >> 31) +} + /// `OffsetEqLink` already reduces publication keys to a trusted internal u64 -/// storage offset. Avoid hashing that offset again on every versioned read. +/// storage offset. Avalanche that offset so both hash-table bucket bits and +/// SIMD control bits remain distributed for aligned, monotonically allocated +/// row positions. #[cfg(feature = "versioned-row-publication")] #[derive(Default)] struct PublicationHasher(u64); @@ -64,7 +82,7 @@ impl Hasher for PublicationHasher { } fn write_u64(&mut self, value: u64) { - self.0 = value; + self.0 = mix_publication_offset(value); } } @@ -72,6 +90,30 @@ impl Hasher for PublicationHasher { type PublicationMap = HashMap, Arc>, BuildHasherDefault>; +#[cfg(feature = "versioned-row-publication")] +type PublicationShards = + [RwLock>; PUBLICATION_SHARD_COUNT]; + +#[cfg(feature = "versioned-row-publication")] +fn publication_shard(key: &OffsetEqLink) -> usize { + mix_publication_offset(key.absolute_index()) as usize & (PUBLICATION_SHARD_COUNT - 1) +} + +#[cfg(feature = "versioned-row-publication")] +fn queue_retirement(queue: &Mutex>, pending_retirements: &AtomicUsize, queue_name: &'static str, value: T) { + let mut queue = queue.lock(); + queue.push(value); + let len = queue.len(); + pending_retirements.fetch_add(1, Ordering::Release); + if len >= RETIREMENT_BACKLOG_WARN_AT && len.is_power_of_two() { + tracing::warn!( + queue = queue_name, + len, + "versioned publication retirement backlog is growing" + ); + } +} + pub struct ReadGuard<'a> { #[cfg(feature = "versioned-row-publication")] active_readers: &'a AtomicU64, @@ -85,6 +127,21 @@ impl Drop for ReadGuard<'_> { } } +/// Page storage and, when enabled, immutable row publication. +/// +/// # Versioned-publication synchronization +/// +/// Generated readers enter the grace period before resolving an index link. +/// Writers must remove or replace every index reference before queueing the old +/// link for retirement. That unlink-before-retire invariant is what makes a +/// reader entering after the reclaimer observes zero unable to acquire the old +/// link. +/// +/// Locks are acquired in this order when more than one is needed: +/// `page_access` -> `pages` -> one `published_rows` shard. Reclamation holds the +/// retirement queues, then briefly acquires individual publication shards and +/// the empty-link/page registries. Callers must not invoke reclamation while +/// retaining a retirement-queue guard. #[derive(Debug)] pub struct DataPages where @@ -93,7 +150,7 @@ where /// Immutable application-visible row versions. Published readers never /// borrow the mutable archived page image. #[cfg(feature = "versioned-row-publication")] - published_rows: RwLock>, + published_rows: PublicationShards, /// Protects the mutable page image used by writers, vacuum, and /// persistence. Application reads use `published_rows` after hydration. @@ -114,6 +171,11 @@ where #[cfg(feature = "versioned-row-publication")] retired_publications: Mutex>>, + /// Avoids taking all retirement-queue mutexes on mutations when there is + /// no reclamation work pending. + #[cfg(feature = "versioned-row-publication")] + pending_retirements: AtomicUsize, + /// Pages vector. Currently, not lock free. pages: RwLock::WrappedRow, DATA_LENGTH>>>>, @@ -165,7 +227,7 @@ where let row = wrapped.get_inner(); let key = OffsetEqLink(link); - let mut published_rows = self.published_rows.write(); + let mut published_rows = self.published_rows[publication_shard(&key)].write(); if let Some(slot) = published_rows.get(&key).cloned() { drop(published_rows); slot.replace(row, flags); @@ -182,7 +244,8 @@ where #[cfg(feature = "versioned-row-publication")] fn published_slot(&self, link: Link) -> Option>> { - self.published_rows.read().get(&OffsetEqLink(link)).cloned() + let key = OffsetEqLink(link); + self.published_rows[publication_shard(&key)].read().get(&key).cloned() } #[cfg(feature = "versioned-row-publication")] @@ -207,8 +270,9 @@ where let wrapped = page.get_row(link).map_err(ExecutionError::DataPageError)?; let flags = Self::publication_flags(&wrapped); let slot = Arc::new(PublishedRow::new(wrapped.get_inner(), flags)); - let mut published_rows = self.published_rows.write(); - Ok(published_rows.entry(OffsetEqLink(link)).or_insert(slot).clone()) + let key = OffsetEqLink(link); + let mut published_rows = self.published_rows[publication_shard(&key)].write(); + Ok(published_rows.entry(key).or_insert(slot).clone()) } pub fn read_guard(&self) -> ReadGuard<'_> { @@ -224,6 +288,9 @@ where #[cfg(feature = "versioned-row-publication")] fn reclaim_retired(&self) { + if self.pending_retirements.load(Ordering::Acquire) == 0 { + return; + } if self.active_readers.load(Ordering::SeqCst) != 0 { return; } @@ -235,25 +302,25 @@ where return; } - let mut published_rows = self.published_rows.write(); for link in retired_links.drain(..) { - published_rows.remove(&OffsetEqLink(link)); + let key = OffsetEqLink(link); + self.published_rows[publication_shard(&key)].write().remove(&key); self.empty_links.push(link); } - for link in retired_publications.drain(..) { - published_rows.remove(&link); + for key in retired_publications.drain(..) { + self.published_rows[publication_shard(&key)].write().remove(&key); } - drop(published_rows); if !retired_pages.is_empty() { let mut empty_pages = self.empty_pages.write(); empty_pages.extend(retired_pages.drain(..)); } + self.pending_retirements.store(0, Ordering::Release); } pub fn new() -> Self { Self { #[cfg(feature = "versioned-row-publication")] - published_rows: RwLock::new(PublicationMap::default()), + published_rows: std::array::from_fn(|_| RwLock::new(PublicationMap::default())), #[cfg(feature = "versioned-row-publication")] page_access: RwLock::new(()), #[cfg(feature = "versioned-row-publication")] @@ -264,6 +331,8 @@ where retired_pages: Mutex::new(Vec::new()), #[cfg(feature = "versioned-row-publication")] retired_publications: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + pending_retirements: AtomicUsize::new(0), // We are starting ID's from `1` because `0`'s page in file is info page. pages: RwLock::new(vec![Arc::new(Data::new(1.into()))]), empty_links: EmptyLinkRegistry::::default(), @@ -282,7 +351,7 @@ where let last_page_id = vec.len(); Self { #[cfg(feature = "versioned-row-publication")] - published_rows: RwLock::new(PublicationMap::default()), + published_rows: std::array::from_fn(|_| RwLock::new(PublicationMap::default())), #[cfg(feature = "versioned-row-publication")] page_access: RwLock::new(()), #[cfg(feature = "versioned-row-publication")] @@ -293,6 +362,8 @@ where retired_pages: Mutex::new(Vec::new()), #[cfg(feature = "versioned-row-publication")] retired_publications: Mutex::new(Vec::new()), + #[cfg(feature = "versioned-row-publication")] + pending_retirements: AtomicUsize::new(0), pages: RwLock::new(vec), empty_links: EmptyLinkRegistry::default(), empty_pages: Default::default(), @@ -311,7 +382,10 @@ where ::WrappedRow: Archive + for<'a> Serialize, Share>, rkyv::rancor::Error>>, { + #[cfg(feature = "versioned-row-publication")] let general_row = ::WrappedRow::from_inner(row.clone()); + #[cfg(not(feature = "versioned-row-publication"))] + let general_row = ::WrappedRow::from_inner(row); #[cfg(feature = "versioned-row-publication")] self.reclaim_retired(); @@ -329,7 +403,7 @@ where self.empty_links.push(l); } #[cfg(feature = "versioned-row-publication")] - self.stage_published_row(link, row.clone()); + self.stage_published_row(link, row); return Ok(link); } Err(e) => match e { @@ -352,16 +426,12 @@ where let current_page = page_id_mapper(self.current_page_id.load(Ordering::Acquire) as usize); let page = &pages[current_page]; - let link = page.save_row(&general_row); - #[cfg(feature = "versioned-row-publication")] - if let Ok(saved_link) = &link { - self.stage_published_row(*saved_link, row.clone()); - } - - (link, current_page) + (page.save_row(&general_row), current_page) }; match link { Ok(link) => { + #[cfg(feature = "versioned-row-publication")] + self.stage_published_row(link, row); self.row_count.fetch_add(1, Ordering::Relaxed); return Ok(link); } @@ -615,7 +685,10 @@ where let page = pages .get(page_id_mapper(link.page_id.into())) .ok_or(ExecutionError::PageNotFound(link.page_id))?; + #[cfg(feature = "versioned-row-publication")] let gen_row = ::WrappedRow::from_inner(row.clone()); + #[cfg(not(feature = "versioned-row-publication"))] + let gen_row = ::WrappedRow::from_inner(row); let result = unsafe { page.save_row_by_link(&gen_row, link) .map_err(ExecutionError::DataPageError) @@ -638,7 +711,7 @@ where #[cfg(feature = "versioned-row-publication")] { - self.retired_links.lock().push(link); + queue_retirement(&self.retired_links, &self.pending_retirements, "links", link); self.reclaim_retired(); } @@ -661,14 +734,15 @@ where if u32::from(page_id) != self.current_page_id.load(Ordering::Acquire) { #[cfg(feature = "versioned-row-publication")] { - self.retired_pages.lock().push(page_id); + queue_retirement(&self.retired_pages, &self.pending_retirements, "pages", page_id); self.reclaim_retired(); } #[cfg(not(feature = "versioned-row-publication"))] - let mut g = self.empty_pages.write(); - #[cfg(not(feature = "versioned-row-publication"))] - g.push_back(page_id); + { + let mut g = self.empty_pages.write(); + g.push_back(page_id); + } } } @@ -737,6 +811,15 @@ where /// Copies a row to another page without exposing either mutable byte /// image to application readers. + /// + /// # Safety + /// + /// The caller must hold the row's exclusive logical lock, ensure + /// `from_link` still identifies the indexed source row, and verify that + /// `to_page_id` has enough capacity for the complete serialized row. No + /// concurrent low-level mutation may access either physical row while the + /// move is in progress. After success, the caller must swing every index + /// reference to the returned link before retiring `from_link`. pub(crate) unsafe fn move_row_for_vacuum( &self, from_link: Link, @@ -784,7 +867,12 @@ where pub(crate) fn retire_published_link(&self, link: Link) { #[cfg(feature = "versioned-row-publication")] { - self.retired_publications.lock().push(OffsetEqLink(link)); + queue_retirement( + &self.retired_publications, + &self.pending_retirements, + "publications", + OffsetEqLink(link), + ); self.reclaim_retired(); } diff --git a/src/in_memory/publication.rs b/src/in_memory/publication.rs index a0f0907b..849b33fa 100644 --- a/src/in_memory/publication.rs +++ b/src/in_memory/publication.rs @@ -1,8 +1,6 @@ +use parking_lot::RwLock; use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use std::sync::atomic::{AtomicU8, Ordering}; - -use parking_lot::RwLock; pub(super) const GHOSTED: u8 = 1 << 0; pub(super) const DELETED: u8 = 1 << 1; @@ -11,41 +9,85 @@ pub(super) const VACUUMED: u8 = 1 << 2; /// One immutable application-visible row version plus atomic lifecycle bits. /// /// Readers hold an `Arc` to a complete version, so replacing or retiring a -/// version cannot invalidate an in-flight read. The short per-row lock only -/// protects the `Arc` pointer; readers never access mutable archived bytes. +/// version cannot invalidate an in-flight read. The short per-row lock keeps +/// the `Arc` and its lifecycle flags in one coherent publication; readers +/// never access mutable archived bytes. +struct PublishedVersion { + row: Arc, + flags: u8, +} + pub(super) struct PublishedRow { - row: RwLock>, - flags: AtomicU8, + version: RwLock>, } impl PublishedRow { pub(super) fn new(row: Row, flags: u8) -> Self { Self { - row: RwLock::new(Arc::new(row)), - flags: AtomicU8::new(flags), + version: RwLock::new(PublishedVersion { + row: Arc::new(row), + flags, + }), } } pub(super) fn replace(&self, row: Row, flags: u8) { - *self.row.write() = Arc::new(row); - self.flags.store(flags, Ordering::Release); + *self.version.write() = PublishedVersion { + row: Arc::new(row), + flags, + }; } pub(super) fn load(&self) -> (Arc, u8) { - let flags = self.flags.load(Ordering::Acquire); - let row = self.row.read().clone(); - (row, flags) + let version = self.version.read(); + (version.row.clone(), version.flags) } pub(super) fn snapshot(&self) -> Arc { - self.row.read().clone() + self.version.read().row.clone() } } impl Debug for PublishedRow { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let version = self.version.read(); f.debug_struct("PublishedRow") - .field("flags", &self.flags.load(Ordering::Relaxed)) + .field("flags", &version.flags) .finish_non_exhaustive() } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::thread; + + use super::PublishedRow; + + #[test] + fn row_and_flags_are_loaded_from_one_version() { + const ITERATIONS: usize = 100_000; + + let published = Arc::new(PublishedRow::new(0_u8, 0)); + let done = Arc::new(AtomicBool::new(false)); + let writer = { + let published = published.clone(); + let done = done.clone(); + thread::spawn(move || { + for value in 0..ITERATIONS { + let state = (value & 1) as u8; + published.replace(state, state); + } + done.store(true, Ordering::Release); + }) + }; + + while !done.load(Ordering::Acquire) { + let (row, flags) = published.load(); + assert_eq!(*row, flags); + } + + writer.join().unwrap(); + } +} diff --git a/src/index/arctic.rs b/src/index/arctic.rs index 077bae6b..6f1399df 100644 --- a/src/index/arctic.rs +++ b/src/index/arctic.rs @@ -2,7 +2,7 @@ use std::borrow::Borrow; use std::fmt::{self, Debug}; -use std::ops::RangeBounds; +use std::ops::{Bound, RangeBounds}; use std::sync::atomic::{AtomicUsize, Ordering}; use arctic::{ConcurrentMap, Key, Order}; @@ -14,12 +14,36 @@ use super::UniqueIndex; /// Keeping this trait local lets generated primary-key newtypes delegate to /// their underlying integer without implementing Arctic's low-level key API. pub trait ArcticKey: Clone + Debug + Ord + Send + Sync + 'static { - type Raw: Key + Clone + Debug + Ord + Send + Sync + 'static; + type Raw: ArcticRawKey; fn to_arctic(&self) -> Self::Raw; fn from_arctic(value: Self::Raw) -> Self; } +/// Integer operations needed to translate Rust's inclusive/exclusive bounds +/// into the native range forms accepted by Arctic 0.1. +#[doc(hidden)] +pub trait ArcticRawKey: Key + Copy + Debug + Ord + Send + Sync + 'static { + fn next(self) -> Option; + fn previous(self) -> Option; +} + +macro_rules! impl_arctic_raw_key { + ($($ty:ty),* $(,)?) => { + $( + impl ArcticRawKey for $ty { + #[inline] + fn next(self) -> Option { self.checked_add(1) } + + #[inline] + fn previous(self) -> Option { self.checked_sub(1) } + } + )* + }; +} + +impl_arctic_raw_key!(u16, u32, u64, u128); + macro_rules! impl_arctic_key { ($($ty:ty),* $(,)?) => { $( @@ -148,7 +172,42 @@ where where R: RangeBounds + 'a, { - self.iter_values().filter(move |(key, _)| range.contains(key)) + let lower = match range.start_bound() { + Bound::Included(key) => Some(key.to_arctic()), + Bound::Excluded(key) => key.to_arctic().next(), + Bound::Unbounded => None, + }; + let upper = match range.end_bound() { + Bound::Included(key) => Some(key.to_arctic()), + Bound::Excluded(key) => key.to_arctic().previous(), + Bound::Unbounded => None, + }; + + macro_rules! collect_range { + ($native_range:expr) => {{ + self.inner + .range($native_range) + .entries(Order::Ascend) + .map(|(key, value)| (K::from_arctic(key), value.clone())) + .collect::>() + }}; + } + + let values = match (lower, upper) { + (Some(lower), Some(upper)) if lower <= upper => { + collect_range!(lower.borrow()..=upper.borrow()) + } + (Some(_), Some(_)) => Vec::new(), + (Some(lower), None) => collect_range!(lower.borrow()..), + (None, Some(upper)) => collect_range!(..=upper.borrow()), + (None, None) => self + .inner + .all() + .entries(Order::Ascend) + .map(|(key, value)| (K::from_arctic(key), value.clone())) + .collect(), + }; + values.into_iter() } fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a @@ -161,6 +220,7 @@ where #[cfg(test)] mod tests { + use std::ops::Bound; use std::sync::{Arc, Barrier}; use super::{ArcticIndex, UniqueIndex}; @@ -177,6 +237,26 @@ mod tests { assert!(index.is_empty()); } + #[test] + fn native_ranges_preserve_rust_bounds() { + let index = ArcticIndex::::default(); + for key in 0..10 { + assert_eq!(index.insert_value_checked(key, key * 10), Some(())); + } + + assert_eq!( + index.range_values(3..7).collect::>(), + vec![(3, 30), (4, 40), (5, 50), (6, 60)] + ); + assert_eq!( + index + .range_values((Bound::Excluded(3), Bound::Included(5))) + .collect::>(), + vec![(4, 40), (5, 50)] + ); + assert_eq!(index.range_values(10..).collect::>(), Vec::new()); + } + #[test] fn checked_insert_has_one_winner_under_contention() { let index = Arc::new(ArcticIndex::::default()); diff --git a/src/index/congee.rs b/src/index/congee.rs index acef21a6..ea660679 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -1,7 +1,7 @@ //! Congee adapter for memory-only unique WorkTable indexes. use std::fmt::{self, Debug}; -use std::ops::RangeBounds; +use std::ops::{Bound, RangeBounds}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -103,6 +103,58 @@ where fn allocation_failure() -> ! { panic!("Congee failed to allocate an index node") } + + fn collect_native_range(&self, range: &R) -> Vec<(K, V)> + where + R: RangeBounds, + { + let start = match range.start_bound() { + Bound::Included(key) => key.into_congee(), + Bound::Excluded(key) => match key.into_congee().checked_add(1) { + Some(start) => start, + None => return Vec::new(), + }, + Bound::Unbounded => 0, + }; + let (end, include_max) = match range.end_bound() { + Bound::Included(key) => match key.into_congee().checked_add(1) { + Some(end) => (end, false), + None => (usize::MAX, true), + }, + Bound::Excluded(key) => (key.into_congee(), false), + Bound::Unbounded => (usize::MAX, true), + }; + + let guard = self.inner.pin(); + let mut raw_values = if start < end { + let mut capacity = self.len().max(1); + loop { + let mut values = vec![(0, 0); capacity]; + let scanned = self.inner.range(&start, &end, &mut values, &guard); + if scanned < capacity || capacity > usize::MAX / 2 { + values.truncate(scanned); + break values; + } + capacity *= 2; + } + } else { + Vec::new() + }; + + if include_max && let Some(pointer) = self.inner.get(&usize::MAX, &guard) { + raw_values.push((usize::MAX, pointer)); + } + + raw_values + .into_iter() + .map(|(key, pointer)| { + // SAFETY: the pinned epoch keeps every returned tree-owned + // pointer alive until its value has been cloned. + let value = unsafe { &*std::ptr::with_exposed_provenance::(pointer) }; + (K::from_congee(key), value.clone()) + }) + .collect() + } } impl UniqueIndex for CongeeIndex @@ -190,17 +242,7 @@ where } fn iter_values(&self) -> impl DoubleEndedIterator + '_ { - let mut values = self - .inner - .keys() - .into_iter() - .filter_map(|key| { - let key = K::from_congee(key); - self.get_value(&key).map(|value| (key, value)) - }) - .collect::>(); - values.sort_unstable_by_key(|entry| entry.0); - values.into_iter() + self.collect_native_range(&(..)).into_iter() } fn iter_links(&self) -> impl DoubleEndedIterator + '_ { @@ -211,7 +253,7 @@ where where R: RangeBounds + 'a, { - self.iter_values().filter(move |(key, _)| range.contains(key)) + self.collect_native_range(&range).into_iter() } fn range_links<'a, R>(&'a self, range: R) -> impl DoubleEndedIterator + 'a @@ -224,6 +266,7 @@ where #[cfg(test)] mod tests { + use std::ops::Bound; use std::sync::{Arc, Barrier}; use super::{CongeeIndex, UniqueIndex}; @@ -240,6 +283,26 @@ mod tests { assert!(index.is_empty()); } + #[test] + fn native_ranges_preserve_rust_bounds() { + let index = CongeeIndex::::default(); + for key in 0..10 { + assert_eq!(index.insert_value_checked(key, key * 10), Some(())); + } + + assert_eq!( + index.range_values(3..7).collect::>(), + vec![(3, 30), (4, 40), (5, 50), (6, 60)] + ); + assert_eq!( + index + .range_values((Bound::Excluded(3), Bound::Included(5))) + .collect::>(), + vec![(4, 40), (5, 50)] + ); + assert_eq!(index.range_values(10..).collect::>(), Vec::new()); + } + #[test] fn checked_insert_has_one_winner_under_contention() { let index = Arc::new(CongeeIndex::::default()); diff --git a/src/index/table_index/cdc.rs b/src/index/table_index/cdc.rs index ea442c3a..28fe343a 100644 --- a/src/index/table_index/cdc.rs +++ b/src/index/table_index/cdc.rs @@ -6,6 +6,7 @@ use indexset::cdc::change::ChangeEvent; use indexset::core::multipair::MultiPair; use indexset::core::node::NodeLike; use indexset::core::pair::Pair; +use vanilla_indexset::concurrent::map::BTreeMap as VanillaIndexMap; use vanilla_indexset::core::node::NodeLike as VanillaNodeLike; use vanilla_indexset::core::pair::Pair as VanillaPair; @@ -56,18 +57,18 @@ where Node: NodeLike>> + Send + 'static, { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>) { - let (res, evs) = self.insert_cdc(value, OffsetEqLink(link)); + let (res, evs) = IndexMap::insert_cdc(self, value, OffsetEqLink(link)); let res_link = res.map(|l| l.0); (res_link, convert_change_events(evs)) } fn insert_checked_cdc(&self, value: T, link: Link) -> Option>>> { - let res = self.checked_insert_cdc(value, OffsetEqLink(link)); + let res = IndexMap::checked_insert_cdc(self, value, OffsetEqLink(link)); res.map(|evs| convert_change_events(evs)) } fn remove_cdc(&self, value: T, _: Link) -> (Option<(T, Link)>, Vec>>) { - let (res, evs) = self.remove_cdc(&value); + let (res, evs) = IndexMap::remove_cdc(self, &value); let res_pair = res.map(|(k, v)| (k, v.0)); (res_pair, convert_change_events(evs)) } @@ -79,17 +80,16 @@ where Node: VanillaNodeLike>> + Send + 'static, { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>) { - let (res, events) = self.insert_cdc(value, OffsetEqLink(link)); + let (res, events) = VanillaIndexMap::insert_cdc(self, value, OffsetEqLink(link)); (res.map(|value| value.0), convert_upstream_change_events(events)) } fn insert_checked_cdc(&self, value: T, link: Link) -> Option>>> { - self.checked_insert_cdc(value, OffsetEqLink(link)) - .map(convert_upstream_change_events) + VanillaIndexMap::checked_insert_cdc(self, value, OffsetEqLink(link)).map(convert_upstream_change_events) } fn remove_cdc(&self, value: T, _: Link) -> (Option<(T, Link)>, Vec>>) { - let (res, events) = self.remove_cdc(&value); + let (res, events) = VanillaIndexMap::remove_cdc(self, &value); ( res.map(|(key, value)| (key, value.0)), convert_upstream_change_events(events), diff --git a/src/persistence/operation/batch.rs b/src/persistence/operation/batch.rs index 9f06b1cd..7d6e2e9c 100644 --- a/src/persistence/operation/batch.rs +++ b/src/persistence/operation/batch.rs @@ -72,7 +72,10 @@ impl From for BatchInnerRow { /// `(page_id, offset)`. Treating the two lengths as different keys leaves /// overlapping writes in the same batch, whose eventual application order is /// derived from a hash map. The newest operation must be the only write for a -/// physical slot. +/// physical slot. WorkTable-generated operation IDs use `Uuid::now_v7`, whose +/// shared process context guarantees creation-order sorting even within one +/// millisecond; callers constructing `Operation` values manually must preserve +/// that ordering contract. fn latest_data_writes( ops: &[Operation], ) -> BatchData { diff --git a/src/table/mod.rs b/src/table/mod.rs index d0bae122..265c270e 100644 --- a/src/table/mod.rs +++ b/src/table/mod.rs @@ -142,19 +142,19 @@ where let _read_guard = self.data.read_guard(); #[cfg(feature = "versioned-row-publication")] { - loop { - let Some(link) = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into) else { - break None; - }; + for _ in 0..64 { + let link = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into)?; if let Ok(row) = self.data.select_non_ghosted(link) { - break Some(row); + return Some(row); } let current_link: Option = self.primary_index.pk_map.lookup_for_select(&pk).map(Into::into); if current_link == Some(link) { - break None; + return None; } + std::hint::spin_loop(); } + None } #[cfg(not(feature = "versioned-row-publication"))] diff --git a/tests/persistence/sync/many_strings.rs b/tests/persistence/sync/many_strings.rs index 9b2281dd..88381285 100644 --- a/tests/persistence/sync/many_strings.rs +++ b/tests/persistence/sync/many_strings.rs @@ -130,15 +130,7 @@ fn test_space_update_query_pk_many_times_sync() { { let engine = TestSyncPersistenceEngine::new(config.clone()).await.unwrap(); let table = TestSyncWorkTable::load(engine).await.unwrap(); - if table.select(pk.clone()).is_none() { - let direct = table.0.primary_index.pk_map.get_value(&TestSyncPrimaryKey(pk.clone())); - let entries = table.0.primary_index.pk_map.iter_values().collect::>(); - let data = direct.map(|link| table.0.data.select_non_ghosted(link.into())); - panic!( - "final primary lookup missed: direct={direct:?}, data={data:?}, entries={entries:?}, row_count={}", - table.count() - ); - } + assert!(table.select(pk.clone()).is_some(), "final primary lookup missed"); assert_eq!(table.select(pk.clone()).unwrap().another, 511); assert_eq!(table.select(pk).unwrap().field, "Some field value".to_string()); } diff --git a/tests/worktable/index/range.rs b/tests/worktable/index/range.rs index 39f98ddc..19497bac 100644 --- a/tests/worktable/index/range.rs +++ b/tests/worktable/index/range.rs @@ -32,6 +32,36 @@ worktable!( } ); +#[cfg(feature = "versioned-row-publication")] +#[tokio::test] +async fn idle_select_builder_does_not_pin_retired_links() { + let table = UniqueRangeTestWorkTable::default(); + let first = UniqueRangeTestRow { + id: table.get_next_pk().into(), + num: 1, + }; + let first_pk = table.insert(first).unwrap(); + let first_link = table.0.primary_index.pk_map.get_value(&first_pk).unwrap().0; + + // Construct the lazy query but do not consume it. Query configuration is + // not an active read and must not delay reclamation or slot reuse. + let idle_query = table.select_all(); + table.delete(first_pk).await.unwrap(); + + let second = UniqueRangeTestRow { + id: table.get_next_pk().into(), + num: 2, + }; + let second_pk = table.insert(second).unwrap(); + let second_link = table.0.primary_index.pk_map.get_value(&second_pk).unwrap().0; + + assert_eq!( + OffsetEqLink::<{ worktable::in_memory::DATA_INNER_LENGTH }>(first_link), + OffsetEqLink::<{ worktable::in_memory::DATA_INNER_LENGTH }>(second_link) + ); + drop(idle_query); +} + #[cfg(feature = "versioned-row-publication")] #[test] fn range_read_revalidates_each_resolved_row() { From 355269cde69fc5023ee65547a2b4c537c5d9c88e Mon Sep 17 00:00:00 2001 From: meh Date: Mon, 3 Aug 2026 23:54:40 +0700 Subject: [PATCH 23/23] fix: address follow-up backend review --- response.md | 232 ----------------------------------- src/in_memory/pages.rs | 16 ++- src/index/congee.rs | 7 +- src/index/table_index/cdc.rs | 6 +- 4 files changed, 20 insertions(+), 241 deletions(-) delete mode 100644 response.md diff --git a/response.md b/response.md deleted file mode 100644 index 8bf50269..00000000 --- a/response.md +++ /dev/null @@ -1,232 +0,0 @@ -# Response to `review.md` - -Thank you for the detailed review. The review was made against `0218d40`; the -branch has moved since then, including the stable WorkTablesIndex read contract -from WorkTablesIndex PR #5 and the follow-up changes described here. - -The four blocking hot-path/CI findings and the publication tearing issue have -been addressed. The counter-based reclamation design has been made materially -safer and observable, but it has not been replaced with a full epoch/QSBR -implementation. That remaining boundary is called out explicitly below. - -## Blocking findings - -### B1: default-build row clones — fixed - -- `insert` and `update` now move the row into its wrapper when versioned - publication is disabled. -- The clone exists only in the versioned configuration, where the owned row is - also needed for publication. -- Successful insert paths move the publication row rather than cloning it a - second time. - -The separate `insert_cdc` reserialization optimization remains a follow-up; it -is not mixed into this correctness patch. - -### B2: grace period and retained query builders — contained, not converted to epochs - -The counter scheme remains, but the structural idle-builder leak is fixed: - -- Lazy `SelectQueryBuilder`s do not acquire a read guard at construction. -- The guard is acquired when iteration starts, before the backend can yield its - first link, and lives only as long as the active iterator. -- A regression test holds an unconsumed builder across delete/insert and proves - that it does not prevent physical-link reuse. -- Delete and vacuum follow a documented unlink-before-retire invariant. A new - reader cannot resolve a retired link after all index references have been - removed; readers that could have resolved it entered the grace period first. -- Retirement backlogs warn at powers of two starting at 1,024 entries. -- An atomic pending count avoids taking the retirement queue locks when there - is no reclamation work. - -A partially consumed or abandoned active iterator still delays reuse. That is -now documented and observable, but it is not the same isolation property as a -real epoch implementation. A future epoch/QSBR conversion remains worthwhile. -A hard queue cap cannot safely discard retirement records; it would need to -apply backpressure or move reclamation to an epoch collector. - -### B3: ART ordered scans — fixed - -- Arctic range operations translate Rust bounds to Arctic's native range API. -- Congee range operations use Congee's native range traversal under a pinned - epoch instead of enumerating all keys and doing one lookup per key. -- Generated table iteration snapshots the ordered links once. It no longer - restarts a range for every row, removing the quadratic behavior. -- Inclusive, exclusive, unbounded, empty, and maximum-key bounds have coverage. -- Criterion now includes single-row primary-key ranges over 10,000-row Congee - and Arctic tables. - -The adapters still materialize the requested interval because the common API -returns an owned double-ended iterator. The cost is now proportional to the -requested interval rather than the entire index per result row. - -### B4: default CI coverage — fixed - -CI now builds and tests a matrix containing: - -- default features; -- `versioned-row-publication`; -- all features, to retain an additive-feature-unification check. - -Strict Clippy runs for both default and all-feature configurations. -WorkTablesIndex PR #5 defines deterministic precedence when multiple search -features are unified. The README now documents that behavior, so WorkTable does -not add a contradictory `compile_error!` for a valid unified Cargo graph. - -There is no longer a separate `stable-index-read-retry` WorkTable feature. The -updated WorkTablesIndex `lookup_for_select` contract is definitive for both -hits and misses. - -## Correctness and concurrency findings - -### C1: torn `(row, flags)` publication — fixed - -The row `Arc` and lifecycle flags now live in one `PublishedVersion` protected -by one short per-slot lock. `load()` reads both under the same lock. A concurrent -test alternates paired row/flag states for 100,000 writes and rejects mixed -versions. - -### C2: global publication-map contention — fixed - -- The single map is replaced with 64 publication shards. -- Shards are selected from a mixed physical offset. -- Reclamation has a pending-work fast path and no longer takes publication - locks when all retirement queues are empty. - -### C3: identity publication hash — fixed - -The physical offset is passed through a SplitMix-style avalanche before it is -used by the hash table, distributing both bucket bits and hashbrown control -tags. - -### C4: unbounded point-read retry — fixed - -Primary-key and generated unique-index point reads retry at most 64 mapping -replacements and issue `spin_loop()` on the retry edge. Perpetual churn can now -produce a bounded `None`, not an indefinitely spinning read. - -### C5: CDC method resolution — fixed - -Both WorkTablesIndex and upstream IndexSet CDC calls now use explicit inherent -type qualification. They cannot silently turn into recursive trait calls. - -### C6: UUID ordering — clarified and dependency floor raised - -The resolved `uuid` implementation does use a shared monotonic v7 context. -Its `Uuid::now_v7()` contract states that UUIDs generated by the same process -are ordered by creation. WorkTable-generated operations use that function. - -The minimum dependency is now `uuid 1.24.0`, matching the documented guarantee, -and `latest_data_writes` documents the constraint. Manually constructed -operations must preserve the ordering contract. A separate `AtomicU64` was not -added because it would duplicate the current operation ID's guaranteed ordering -and expand the persisted/CDC operation representation. - -Reference: - -### C7 and C8: hydration and lock hierarchy — documented - -- Cold hydration's writer-exclusion cost and the warm-up option are documented. -- `DataPages` now states its multi-lock hierarchy, the retirement queue rule, - and the unlink-before-retire obligation. -- `move_row_for_vacuum` now has an explicit `# Safety` contract. - -## Code generation findings - -### D1: token substring backend detection — fixed - -Persisted-index layout is derived from parsed type paths and structured -`pk_upstream` metadata. Unknown types and aliases are rejected instead of being -guessed into an on-disk representation. The primary backend is threaded into -the generated persistence attribute rather than recovered from rendered -tokens. - -### D2: stringified primitive types — improved - -Primitive checks parse `syn::Type::Path` and compare the final path segment. -The diagnostic explicitly says that a directly named primitive is required and -that proc macros cannot resolve Rust type aliases. The current DSL grammar still -accepts identifiers rather than arbitrary qualified type syntax. - -### D3: `compile_error!` in type position — fixed - -The affected table generators now return `syn::Result` and propagate the -original diagnostic with `?`. - -### D4: macro-host feature coupling — documented, not redesigned - -The Cargo feature-unification effect is now explicit in the publication docs. -Replacing the forwarded proc-macro feature with a build-script cfg would be a -larger packaging change and is deferred. - -### D5: upsert rewrite — unchanged in this follow-up - -The single logical row lock and retry rationale remain. Removing redundant -probes and reducing wide-row clones is still a useful measured optimization, -but is separate from these correctness fixes. - -### D6: statement-level cfgs — fixed - -The non-versioned `mark_page_empty` branch is grouped in one cfg block. - -## API and packaging findings - -- **E1:** making Arctic, Congee, and upstream IndexSet optional is deferred. It - changes the feature/API contract and deserves its own binary-size and compile- - time change. -- **E2:** WorkTablesIndex PR #5 deliberately supports additive Cargo feature - unification with deterministic search-policy precedence; this is documented. -- **E3:** the publication docs now state that enabling the feature anywhere in - a dependency graph enables it for all WorkTable consumers in that build. -- **E4:** moving backend types out of the prelude is deferred because it is a - public API decision rather than a correctness patch. -- **E5:** the unsafe vacuum move now documents all caller obligations. - -## Tests and validation - -Added or expanded coverage includes: - -- coherent row/flag publication under concurrent replacement; -- idle query builders and retirement/link reuse; -- native Congee and Arctic range-bound translation; -- generated range-result predicate revalidation; -- persisted index aliases rejected instead of misclassified; -- the ART primary-key range Criterion smoke benchmark. - -The persistence test that formatted the entire primary index on failure has -been restored to a concise assertion. - -Validation against DataBucket 0.5.1 and the updated WorkTablesIndex PR #5 head: - -- strict Clippy, default configuration: pass; -- strict Clippy, all features: pass; -- full default suite: 127 library tests, 333 integration tests (3 ignored), and - 64 codegen tests passed, plus all benchmark smoke targets; -- full all-feature suite: 132 library tests, 336 integration tests (4 ignored), - and 64 codegen tests passed, plus all benchmark smoke targets. - -One initial all-feature run hit an assertion inside `arctic-map 0.1.4` during -its disjoint concurrent mutation test. The unchanged isolated test then passed -20/20 repetitions and the complete all-feature suite passed on rerun. This is -being reported rather than hidden because the dependency is young and pinned. - -No latency claims are made from this host while unrelated benchmarks are -running. The Criterion cases are in place, but publishable before/after numbers -should be collected on a quiet machine. - -## Remaining follow-ups - -The intentional remaining work is: - -1. replace the counter grace period with a true epoch/QSBR collector if active - scans must coexist indefinitely with prompt physical-link reuse; -2. make alternate backend dependencies optional and decide whether their types - belong in the prelude before the stable API; -3. measure and then optimize duplicate insert serialization and redundant - upsert probes/clones; -4. improve ART memory accounting and address the pre-existing row-count reuse - drift separately; -5. run the default/versioned latency matrix on a quiet benchmark host. - -Those are kept explicit so the current patch is not represented as eliminating -every cost or lifecycle caveat raised by the review. diff --git a/src/in_memory/pages.rs b/src/in_memory/pages.rs index 56912271..c832d944 100644 --- a/src/in_memory/pages.rs +++ b/src/in_memory/pages.rs @@ -63,9 +63,15 @@ fn mix_publication_offset(mut value: u64) -> u64 { /// SIMD control bits remain distributed for aligned, monotonically allocated /// row positions. #[cfg(feature = "versioned-row-publication")] -#[derive(Default)] struct PublicationHasher(u64); +#[cfg(feature = "versioned-row-publication")] +impl Default for PublicationHasher { + fn default() -> Self { + Self(0xcbf2_9ce4_8422_2325) + } +} + #[cfg(feature = "versioned-row-publication")] impl Hasher for PublicationHasher { fn finish(&self) -> u64 { @@ -73,16 +79,16 @@ impl Hasher for PublicationHasher { } fn write(&mut self, bytes: &[u8]) { - let mut hash = 0xcbf29ce484222325u64; + let mut hash = self.0; for byte in bytes { hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x100000001b3); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); } - self.0 = hash; + self.0 = mix_publication_offset(hash); } fn write_u64(&mut self, value: u64) { - self.0 = mix_publication_offset(value); + self.0 = mix_publication_offset(self.0 ^ value); } } diff --git a/src/index/congee.rs b/src/index/congee.rs index ea660679..62c9887e 100644 --- a/src/index/congee.rs +++ b/src/index/congee.rs @@ -9,6 +9,8 @@ use congee::{Congee, DefaultAllocator}; use super::UniqueIndex; +const INITIAL_RANGE_CAPACITY: usize = 64; + /// Lossless conversion between a WorkTable key and Congee's machine-word key. /// /// Congee 0.4 stores keys as one `usize`, so this backend intentionally accepts @@ -127,7 +129,10 @@ where let guard = self.inner.pin(); let mut raw_values = if start < end { - let mut capacity = self.len().max(1); + // Most generated ranges are narrow. Starting from the full index + // length makes a point range allocate in proportion to the whole + // table before Congee examines its bounds. + let mut capacity = INITIAL_RANGE_CAPACITY; loop { let mut values = vec![(0, 0); capacity]; let scanned = self.inner.range(&start, &end, &mut values, &guard); diff --git a/src/index/table_index/cdc.rs b/src/index/table_index/cdc.rs index 28fe343a..0cb4a588 100644 --- a/src/index/table_index/cdc.rs +++ b/src/index/table_index/cdc.rs @@ -27,14 +27,14 @@ where Node: NodeLike>> + Send + 'static, { fn insert_cdc(&self, value: T, link: Link) -> (Option, Vec>>) { - let (res, evs) = self.insert_cdc(value, OffsetEqLink(link)); + let (res, evs) = IndexMultiMap::insert_cdc(self, value, OffsetEqLink(link)); let pair_evs = evs.into_iter().map(Into::into).collect(); let res_link = res.map(|l| l.0); (res_link, convert_change_events(pair_evs)) } fn insert_checked_cdc(&self, value: T, link: Link) -> Option>>> { - let (res, evs) = self.insert_cdc(value, OffsetEqLink(link)); + let (res, evs) = IndexMultiMap::insert_cdc(self, value, OffsetEqLink(link)); let pair_evs = evs.into_iter().map(Into::into).collect(); if res.is_some() { None @@ -44,7 +44,7 @@ where } fn remove_cdc(&self, value: T, link: Link) -> (Option<(T, Link)>, Vec>>) { - let (res, evs) = self.remove_cdc(&value, &OffsetEqLink(link)); + let (res, evs) = IndexMultiMap::remove_cdc(self, &value, &OffsetEqLink(link)); let pair_evs = evs.into_iter().map(Into::into).collect(); let res_pair = res.map(|(k, v)| (k, v.into())); (res_pair, convert_change_events(pair_evs))