feat: persist native Arctic and Congee indexes

Cargo.toml

@@ -3,7 +3,7 @@ members = ["codegen", "examples", "performance_measurement", "performance_measur
 
 [package]
 name = "worktable"
-version = "1.0.0-beta.1"
+version = "1.0.0-beta.2"
 edition = "2024"
 authors = ["Handy-caT"]
 license = "MIT"
@@ -28,9 +28,10 @@ versioned-row-publication = ["worktable_codegen/versioned-row-publication"]
 
 [dependencies]
 async-trait = "0.1.89"
-arctic-map = "=0.1.4"
-congee = "=0.4.1"
+arctic = { package = "arctic-wt", version = "0.1.4" }
+congee = { package = "congee-wt", version = "0.4.1" }
 convert_case = "0.6.0"
+crc32fast = "1.5.0"
 data_bucket = "=0.5.1"
 # data_bucket = { git = "https://github.com/pathscale/DataBucket", branch = "page_cdc_correction", version = "0.2.7" }
 # data_bucket = { path = "../DataBucket", version = "0.3.14" }
@@ -58,7 +59,7 @@ tracing = "0.1"
 url = { version = "2", optional = true }
 uuid = { version = "1.24.0", features = ["v4", "v7"] }
 walkdir = { version = "2", optional = true }
-worktable_codegen = { path = "codegen", version = "=1.0.0-beta.1" }
+worktable_codegen = { path = "codegen", version = "=1.0.0-beta.2" }
 
 [dev-dependencies]
 chrono = "0.4.43"

README.md

@@ -13,7 +13,7 @@ from a macro, and that persisting it is one feature flag away.
 ## Install
 
 ```sh
-cargo add worktable@1.0.0-beta.1
+cargo add worktable@1.0.0-beta.2
 ```
 
 ## What you get
@@ -44,7 +44,7 @@ S3 support layers *on top of* the disk engine rather than replacing it.
 
 ```toml
 [dependencies]
-worktable = { version = "=1.0.0-beta.1", features = ["s3-support"] }   # S3 sync, optional
+worktable = { version = "=1.0.0-beta.2", features = ["s3-support"] }   # S3 sync, optional
 ```
 
 Persisted indexes default to WorkTablesIndex. Vanilla IndexSet can be selected explicitly with `using indexset` while retaining the existing disk/S3 representation. Congee and Arctic are explicitly memory-only and require `persist: false`. The full syntax and capability matrix are documented in [Per-index backends with `using`](docs/index-backend-dsl-proposal.md).
@@ -60,7 +60,7 @@ opt into immutable row-version publication:
 
 ```toml
 [dependencies]
-worktable = { version = "=1.0.0-beta.1", features = ["versioned-row-publication"] }
+worktable = { version = "=1.0.0-beta.2", features = ["versioned-row-publication"] }
 ```
 
 Generated point lookups use a strict backend-specific visibility contract by

codegen/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "worktable_codegen"
-version = "1.0.0-beta.1"
+version = "1.0.0-beta.2"
 edition = "2024"
 license = "MIT"
 description = "Proc-macro companion crate for worktable: the worktable! macro and its derives."

codegen/src/common/model/index.rs

@@ -15,7 +15,7 @@ pub enum IndexBackend {
 }
 
 impl IndexBackend {
-    pub fn is_memory_only(self) -> bool {
+    pub fn requires_explicit_persistence(self) -> bool {
         matches!(self, Self::Congee | Self::Arctic)
     }
 

codegen/src/generators/index_backend.rs

@@ -31,6 +31,21 @@ pub(crate) fn unique_index_type(
     }
 }
 
+/// Generates the persisted-table variant. ART backends receive a write-side
+/// sequencing wrapper; memory-only tables keep the zero-overhead native type.
+pub(crate) fn persistent_unique_index_type(
+    backend: IndexBackend,
+    key: &TokenStream,
+    value: &TokenStream,
+    worktables_node: Option<TokenStream>,
+) -> syn::Result<TokenStream> {
+    match backend {
+        IndexBackend::Congee => Ok(quote! { PersistentCongeeIndex<#key, #value> }),
+        IndexBackend::Arctic => Ok(quote! { PersistentArcticIndex<#key, #value> }),
+        _ => unique_index_type(backend, key, value, worktables_node),
+    }
+}
+
 /// Generates the small codec needed when an ART is used for WorkTable's
 /// generated primary-key newtype. ART backends intentionally accept only the
 /// lossless, native integer shapes supported by their public APIs.
@@ -67,6 +82,18 @@ pub(crate) fn primary_key_backend_impl(
                             Self(value as #field)
                         }
                     }
+
+                    impl ArtPersistenceKey for #primary_key {
+                        const WIDTH: u8 = <#field as ArtPersistenceKey>::WIDTH;
+
+                        fn encode_art_key(&self, output: &mut Vec<u8>) {
+                            self.0.encode_art_key(output)
+                        }
+
+                        fn decode_art_key(bytes: &[u8]) -> eyre::Result<Self> {
+                            Ok(Self(<#field as ArtPersistenceKey>::decode_art_key(bytes)?))
+                        }
+                    }
                 },
             ))
         }
@@ -88,6 +115,18 @@ pub(crate) fn primary_key_backend_impl(
                             Self(value)
                         }
                     }
+
+                    impl ArtPersistenceKey for #primary_key {
+                        const WIDTH: u8 = <#field as ArtPersistenceKey>::WIDTH;
+
+                        fn encode_art_key(&self, output: &mut Vec<u8>) {
+                            self.0.encode_art_key(output)
+                        }
+
+                        fn decode_art_key(bytes: &[u8]) -> eyre::Result<Self> {
+                            Ok(Self(<#field as ArtPersistenceKey>::decode_art_key(bytes)?))
+                        }
+                    }
                 },
             ))
         }

codegen/src/generators/persist/index/info.rs

@@ -26,15 +26,25 @@ impl PersistGenerator {
             let index_name_str = index_field_name.to_string();
 
             if idx.is_unique {
+                let (capacity, node_count) = match idx.backend {
+                    crate::common::model::IndexBackend::WorktablesIndex
+                    | crate::common::model::IndexBackend::Indexset => (
+                        quote! { self.#index_field_name.capacity() },
+                        quote! { self.#index_field_name.node_count() },
+                    ),
+                    crate::common::model::IndexBackend::Congee | crate::common::model::IndexBackend::Arctic => {
+                        (quote! { self.#index_field_name.len() }, quote! { 0 })
+                    }
+                };
                 quote! {
                     info.push(IndexInfo {
                         name: #index_name_str.to_string(),
                         index_type: IndexKind::Unique,
                         key_count: self.#index_field_name.len(),
-                        capacity: self.#index_field_name.capacity(),
+                        capacity: #capacity,
                         heap_size: self.#index_field_name.heap_size(),
                         used_size: self.#index_field_name.used_size(),
-                        node_count: self.#index_field_name.node_count(),
+                        node_count: #node_count,
                     });
                 }
             } else {

codegen/src/generators/persist/index/mod.rs

@@ -3,7 +3,7 @@ mod info;
 mod usual;
 
 use crate::common::name_generator::{WorktableNameGenerator, is_float, is_unsized};
-use crate::generators::index_backend::unique_index_type;
+use crate::generators::index_backend::persistent_unique_index_type;
 use crate::generators::persist::PersistGenerator;
 use convert_case::{Case, Casing};
 use proc_macro2::TokenStream;
@@ -57,7 +57,7 @@ impl PersistGenerator {
                     } else {
                         None
                     };
-                    let index_type = unique_index_type(idx.backend, &t, &value_type, worktables_node)?;
+                    let index_type = persistent_unique_index_type(idx.backend, &t, &value_type, worktables_node)?;
                     quote! { #i: #index_type }
                 } else {
                     if is_unsized(&t.to_string()) {

codegen/src/generators/persist/table/impls.rs

@@ -72,10 +72,6 @@ impl PersistGenerator {
             })
             .collect::<Vec<_>>();
         let pk_types_unsized = is_unsized_vec(pk_types);
-        let pk_map = match self.columns.primary_index_backend {
-            crate::common::model::IndexBackend::Indexset => quote! { UpstreamIndexMap },
-            _ => quote! { IndexMap },
-        };
         let index_setup = if pk_types_unsized {
             quote! {
                 inner.primary_index = std::sync::Arc::new(PrimaryIndex {
@@ -84,12 +80,33 @@ impl PersistGenerator {
                 });
             }
         } else {
-            quote! {
-                let size = get_index_page_size_from_data_length::<#pk_type>(#const_name);
-                inner.primary_index = std::sync::Arc::new(PrimaryIndex {
-                    pk_map: #pk_map::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size),
-                    reverse_pk_map: IndexMap::new(),
-                });
+            match self.columns.primary_index_backend {
+                crate::common::model::IndexBackend::WorktablesIndex => quote! {
+                    let size = get_index_page_size_from_data_length::<#pk_type>(#const_name);
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
+                        pk_map: IndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size),
+                        reverse_pk_map: IndexMap::new(),
+                    });
+                },
+                crate::common::model::IndexBackend::Indexset => quote! {
+                    let size = get_index_page_size_from_data_length::<#pk_type>(#const_name);
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
+                        pk_map: UpstreamIndexMap::<_, OffsetEqLink<#const_name>>::with_maximum_node_size(size),
+                        reverse_pk_map: IndexMap::new(),
+                    });
+                },
+                crate::common::model::IndexBackend::Arctic => quote! {
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
+                        pk_map: PersistentArcticIndex::<#pk_type, OffsetEqLink<#const_name>>::default(),
+                        reverse_pk_map: IndexMap::new(),
+                    });
+                },
+                crate::common::model::IndexBackend::Congee => quote! {
+                    inner.primary_index = std::sync::Arc::new(PrimaryIndex {
+                        pk_map: PersistentCongeeIndex::<#pk_type, OffsetEqLink<#const_name>>::default(),
+                        reverse_pk_map: IndexMap::new(),
+                    });
+                },
             }
         };
 

codegen/src/generators/persist/table/mod.rs

@@ -6,7 +6,7 @@ use proc_macro2::{Literal, TokenStream};
 use quote::quote;
 
 use crate::common::name_generator::{WorktableNameGenerator, is_unsized_vec};
-use crate::generators::index_backend::unique_index_type;
+use crate::generators::index_backend::persistent_unique_index_type;
 use crate::generators::persist::PersistGenerator;
 
 impl PersistGenerator {
@@ -84,25 +84,28 @@ impl PersistGenerator {
             })
             .collect::<Vec<_>>();
         let pk_types_unsized = is_unsized_vec(pk_types);
-        let pk_upstream = matches!(
-            self.columns.primary_index_backend,
-            crate::common::model::IndexBackend::Indexset
-        );
-
-        let derive = match (pk_types_unsized, pk_upstream) {
-            (true, true) => quote! {
+        let derive = match (pk_types_unsized, self.columns.primary_index_backend) {
+            (true, crate::common::model::IndexBackend::Indexset) => quote! {
                 #[derive(Debug, PersistTable)]
                 #[table(pk_unsized, pk_upstream)]
             },
-            (true, false) => quote! {
+            (true, _) => quote! {
                 #[derive(Debug, PersistTable)]
                 #[table(pk_unsized)]
             },
-            (false, true) => quote! {
+            (false, crate::common::model::IndexBackend::Indexset) => quote! {
                 #[derive(Debug, PersistTable)]
                 #[table(pk_upstream)]
             },
-            (false, false) => quote! {
+            (false, crate::common::model::IndexBackend::Arctic) => quote! {
+                #[derive(Debug, PersistTable)]
+                #[table(pk_arctic)]
+            },
+            (false, crate::common::model::IndexBackend::Congee) => quote! {
+                #[derive(Debug, PersistTable)]
+                #[table(pk_congee)]
+            },
+            (false, crate::common::model::IndexBackend::WorktablesIndex) => quote! {
                 #[derive(Debug, PersistTable)]
             },
         };
@@ -116,7 +119,7 @@ impl PersistGenerator {
         } else {
             None
         };
-        let node_type = unique_index_type(
+        let node_type = persistent_unique_index_type(
             self.columns.primary_index_backend,
             &key_type,
             &value_type,