diff --git a/Cargo.toml b/Cargo.toml index e7b26cb..9850965 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ members = ["codegen", "tools/create-data-file", "tools/dump-data-file"] [package] name = "data_bucket" -version = "0.4.1" +version = "0.5.0" edition = "2021" authors = ["Handy-caT"] license = "MIT" @@ -11,15 +11,15 @@ repository = "https://github.com/pathscale/DataBucket" description = "DataBucket is container for WorkTable's data" [dependencies] -data_bucket_derive = { path = "codegen", version = "=0.3.16" } +data_bucket_derive = { path = "codegen", version = "=0.3.17" } eyre = "0.6.12" derive_more = { version = "1.0.0", features = ["from", "error", "display", "into"] } -rkyv = { version = "0.8.9", features = ["uuid-1"] } +rkyv = { version = "0.8.17", features = ["uuid-1"] } uuid = { version = "1.11.0", features = ["v4"] } psc-nanoid = { version = "3.1.1", features = ["rkyv", "packed"] } ordered-float = "5.0.0" -indexset = { package = "WorkTablesIndex", version = "=0.0.1", features = ["concurrent", "cdc", "multimap"] } +indexset = { package = "WorkTablesIndex", version = "=0.0.3", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", path = "../indexset", version = "0.12.10", features = ["concurrent", "cdc", "multimap"] } # indexset = { package = "wt-indexset", version = "0.12.12", features = ["concurrent", "cdc", "multimap"] } tokio = { version = "1", features = ["full"] } diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 65884f9..b74d3aa 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "data_bucket_derive" -version = "0.3.16" +version = "0.3.17" edition = "2021" authors = ["Handy-caT"] license = "MIT" @@ -13,14 +13,6 @@ path = "src/lib.rs" proc-macro = true [dependencies] -rkyv = { version = "0.7.45" } syn = { version = "2.0.74", features = ["full"] } quote = "1.0.36" proc-macro2 = "1.0.86" -regex = "1.10.6" -convert_case = "0.6.0" - -[dev-dependencies] -derive_more = { version = "1.0.0", features = ["from", "error", "display", "into"] } -rkyv = { version = "0.7.45", features = ["uuid"] } -scc = "2.1.16" diff --git a/codegen/src/persistable/generator/obj_impl.rs b/codegen/src/persistable/generator/obj_impl.rs index a82891e..9f6ee0f 100644 --- a/codegen/src/persistable/generator/obj_impl.rs +++ b/codegen/src/persistable/generator/obj_impl.rs @@ -172,7 +172,7 @@ impl Generator { ); quote! { pub fn #fn_ident() -> usize { - <#ty as Default>::default().aligned_size() + <#ty as SizeMeasurable>::default_aligned_size() } } } @@ -218,7 +218,7 @@ impl Generator { ); let len_in_vec = if is_primitive(&inner_ty_str) { quote! { - align(length * <#inner_ty as Default>::default().aligned_size()) + 8 + align(length * <#inner_ty as SizeMeasurable>::default_aligned_size()) + 8 } } else { quote! { @@ -227,14 +227,14 @@ impl Generator { }; let len_value = if is_primitive(&inner_ty_str) { quote! { - <#inner_ty as Default>::default().aligned_size() + <#inner_ty as SizeMeasurable>::default_aligned_size() } } else { quote! { if <#inner_ty as SizeMeasurable>::align() == Some(8) { - align8(<#inner_ty as Default>::default().aligned_size()) + align8(<#inner_ty as SizeMeasurable>::default_aligned_size()) } else { - <#inner_ty as Default>::default().aligned_size() + <#inner_ty as SizeMeasurable>::default_aligned_size() } } }; diff --git a/codegen/src/persistable/generator/persistable_impl.rs b/codegen/src/persistable/generator/persistable_impl.rs index ba0a259..49fc3dd 100644 --- a/codegen/src/persistable/generator/persistable_impl.rs +++ b/codegen/src/persistable/generator/persistable_impl.rs @@ -212,7 +212,7 @@ impl Generator { let size_type = &f.ty; let size_ident = f.ident.as_ref().unwrap(); quote! { - let size_length = <#size_type as Default>::default().aligned_size(); + let size_length = <#size_type as SizeMeasurable>::default_aligned_size(); let archived = data_bucket::access_archived::<<#size_type as Archive>::Archived>(&bytes[offset..offset + size_length]).expect("torn or corrupt page part: a size field fails validation"); let #size_ident = @@ -237,7 +237,7 @@ impl Generator { fn gen_from_bytes_for_primitive(&self, ty: &Type, ident: &Ident) -> TokenStream { quote! { - let length = <#ty as Default>::default().aligned_size(); + let length = <#ty as SizeMeasurable>::default_aligned_size(); let mut v = rkyv::util::AlignedVec::<4>::new(); v.extend_from_slice(&bytes[offset..offset + length]); let archived = data_bucket::access_archived::<<#ty as Archive>::Archived>(&v[..]).expect("torn or corrupt page part: a field fails validation"); @@ -271,7 +271,7 @@ impl Generator { let value_fn_ident = Ident::new(format!("{ident}_value_size").as_str(), Span::call_site()); let len = if is_primitive(&inner_ty_str) { quote! { - let values_len = align(#size_ident as usize * <#inner_ty as Default>::default().aligned_size()) + 8; + let values_len = align(#size_ident as usize * <#inner_ty as SizeMeasurable>::default_aligned_size()) + 8; } } else { quote! { diff --git a/codegen/src/size_measure/enum_generator.rs b/codegen/src/size_measure/enum_generator.rs new file mode 100644 index 0000000..2e4baa2 --- /dev/null +++ b/codegen/src/size_measure/enum_generator.rs @@ -0,0 +1,62 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::{Fields, ItemEnum}; + +pub struct EnumGenerator { + pub enum_def: ItemEnum, +} + +impl EnumGenerator { + pub fn gen_impl(&self) -> syn::Result { + if !self.enum_def.generics.params.is_empty() { + return Err(syn::Error::new_spanned( + &self.enum_def.generics, + "SizeMeasure does not yet support generic enums", + )); + } + + if let Some(variant) = self + .enum_def + .variants + .iter() + .find(|variant| !matches!(variant.fields, Fields::Unit)) + { + return Err(syn::Error::new_spanned( + variant, + "SizeMeasure supports only fieldless enums; payload sizes may be data-dependent", + )); + } + + let enum_ident = &self.enum_def.ident; + Ok(quote! { + impl SizeMeasurable for #enum_ident + where + #enum_ident: rkyv::Archive, + <#enum_ident as rkyv::Archive>::Archived: Sized, + { + fn aligned_size(&self) -> usize { + std::mem::size_of::<<#enum_ident as rkyv::Archive>::Archived>() + } + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::EnumGenerator; + use syn::parse_quote; + + #[test] + fn rejects_payload_bearing_enums() { + let enum_def = parse_quote! { + enum Payload { + Empty, + Value(u64), + } + }; + + let error = EnumGenerator { enum_def }.gen_impl().unwrap_err(); + assert!(error.to_string().contains("fieldless enums")); + } +} diff --git a/codegen/src/size_measure/generator.rs b/codegen/src/size_measure/generator.rs index 0544975..efcc757 100644 --- a/codegen/src/size_measure/generator.rs +++ b/codegen/src/size_measure/generator.rs @@ -7,7 +7,7 @@ pub struct Generator { } impl Generator { - pub fn gen_impl(&self) -> TokenStream { + pub fn gen_impl(&self) -> syn::Result { let struct_ident = &self.struct_def.ident; let mut num = 0; @@ -37,14 +37,14 @@ impl Generator { .map(|f| { let t = &f.ty; quote! { - if #t::align() == Some(8) { + if <#t as SizeMeasurable>::align() == Some(8) { return Some(8) } } }) .collect::>(); - quote! { + Ok(quote! { impl SizeMeasurable for #struct_ident { fn aligned_size(&self) -> usize { let len = #(#sum+)* 0; @@ -55,6 +55,6 @@ impl Generator { None } } - } + }) } } diff --git a/codegen/src/size_measure/mod.rs b/codegen/src/size_measure/mod.rs index f541a2e..fa50a71 100644 --- a/codegen/src/size_measure/mod.rs +++ b/codegen/src/size_measure/mod.rs @@ -1,20 +1,20 @@ +mod enum_generator; mod generator; mod parser; use proc_macro2::TokenStream; use quote::quote; +use crate::size_measure::enum_generator::EnumGenerator; use crate::size_measure::generator::Generator; -use crate::size_measure::parser::Parser; +use crate::size_measure::parser::{ParsedItem, Parser}; pub fn expand(input: &TokenStream) -> syn::Result { - let input_fn = Parser::parse_struct(input)?; - let gen = Generator { - struct_def: input_fn, + let impl_def = match Parser::parse(input)? { + ParsedItem::Struct(struct_def) => Generator { struct_def }.gen_impl()?, + ParsedItem::Enum(enum_def) => EnumGenerator { enum_def }.gen_impl()?, }; - let impl_def = gen.gen_impl(); - Ok(quote! { #impl_def }) diff --git a/codegen/src/size_measure/parser.rs b/codegen/src/size_measure/parser.rs index 8649504..c013c8d 100644 --- a/codegen/src/size_measure/parser.rs +++ b/codegen/src/size_measure/parser.rs @@ -1,13 +1,23 @@ use proc_macro2::TokenStream; use syn::spanned::Spanned; -use syn::ItemStruct; +use syn::{Item, ItemEnum, ItemStruct}; + +pub enum ParsedItem { + Struct(ItemStruct), + Enum(ItemEnum), +} pub struct Parser; impl Parser { - pub fn parse_struct(input: &TokenStream) -> syn::Result { - match syn::parse2::(input.clone()) { - Ok(data) => Ok(data), + pub fn parse(input: &TokenStream) -> syn::Result { + match syn::parse2::(input.clone()) { + Ok(Item::Struct(data)) => Ok(ParsedItem::Struct(data)), + Ok(Item::Enum(data)) => Ok(ParsedItem::Enum(data)), + Ok(item) => Err(syn::Error::new_spanned( + item, + "SizeMeasure supports structs and fieldless enums", + )), Err(err) => Err(syn::Error::new(input.span(), err.to_string())), } } diff --git a/src/page/index/table_of_contents_page.rs b/src/page/index/table_of_contents_page.rs index 0ee6988..f373f4b 100644 --- a/src/page/index/table_of_contents_page.rs +++ b/src/page/index/table_of_contents_page.rs @@ -21,7 +21,7 @@ where Self { records: BTreeMap::new(), empty_pages: vec![], - estimated_size: usize::default().aligned_size() + 12, + estimated_size: ::default_aligned_size() + 12, } } } diff --git a/src/page/iterators.rs b/src/page/iterators.rs index 8795c23..3e498a0 100644 --- a/src/page/iterators.rs +++ b/src/page/iterators.rs @@ -4,7 +4,7 @@ use rkyv::{de::Pool, rancor::Strategy, Archive, DeserializeUnsized}; use crate::{ page::util::parse_general_header, - persistence::data::{rkyv_data::parse_archived_row, DataTypeValue}, + persistence::data::{rkyv_data::parse_archived_row, DataDecodeError, DataTypeValue}, IndexData, Link, }; @@ -136,7 +136,7 @@ impl DataIterator<'_> { } impl Iterator for DataIterator<'_> { - type Item = Vec; + type Item = Result, DataDecodeError>; fn next(&mut self) -> Option { if self.link_index >= self.links.len() { @@ -210,7 +210,7 @@ mod test { let data_iterator: DataIterator<'_> = DataIterator::new(&mut file, space_info.row_schema, links); assert_eq!( - data_iterator.collect::>(), + data_iterator.collect::, _>>().unwrap(), vec![ vec![ DataTypeValue::I32(1), diff --git a/src/persistence/data/mod.rs b/src/persistence/data/mod.rs index a374587..a247f26 100644 --- a/src/persistence/data/mod.rs +++ b/src/persistence/data/mod.rs @@ -4,10 +4,60 @@ mod util; pub use types::DataTypeValue; +use std::fmt; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum DataDecodeError { + UnsupportedDataType { + data_type: String, + }, + BufferTooShort { + required: usize, + actual: usize, + }, + FieldOutOfBounds { + field_index: usize, + field_end: usize, + actual: usize, + }, + InvalidArchive { + data_type: &'static str, + message: String, + }, +} + +impl fmt::Display for DataDecodeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnsupportedDataType { data_type } => { + write!(f, "unsupported archived row data type `{data_type}`") + } + Self::BufferTooShort { required, actual } => write!( + f, + "archived row buffer is too short: schema requires {required} bytes, buffer has {actual}" + ), + Self::FieldOutOfBounds { + field_index, + field_end, + actual, + } => write!( + f, + "archived row field {field_index} ends at byte {field_end}, beyond buffer length {actual}" + ), + Self::InvalidArchive { data_type, message } => { + write!(f, "invalid archived `{data_type}` field: {message}") + } + } + } +} + +impl std::error::Error for DataDecodeError {} + pub trait DataType { + /// Advances an offset past this type, including its required padding. fn advance_accum(&self, accum: &mut usize); + + /// Validates and decodes a value rooted at the end of `bytes`. #[allow(clippy::wrong_self_convention)] - fn from_pointer(&self, pointer: *const u8, start_pointer: *const u8) -> DataTypeValue; - fn advance_pointer_for_padding(&self, pointer: &mut *const u8, start_pointer: *const u8); - fn advance_pointer(&self, pointer: &mut *const u8); + fn from_archived_bytes(&self, bytes: &[u8]) -> Result; } diff --git a/src/persistence/data/rkyv_data.rs b/src/persistence/data/rkyv_data.rs index 557f707..9ea4a1e 100644 --- a/src/persistence/data/rkyv_data.rs +++ b/src/persistence/data/rkyv_data.rs @@ -1,43 +1,70 @@ -use crate::persistence::data::types::DataTypeValue; +use crate::persistence::data::{DataDecodeError, DataTypeValue}; use std::str::FromStr; +/// Decodes a dynamically described row from an rkyv archive. +/// +/// # Errors +/// +/// Returns [`DataDecodeError`] when the schema names an unsupported type, the +/// fixed-width row root does not fit in `buf`, or any archived field fails +/// rkyv validation. Malformed persisted data never enters an unchecked access +/// path. pub fn parse_archived_row, S2: AsRef>( buf: &[u8], columns: &[(S1, S2)], -) -> Vec { - let mut data_length: usize = { - let mut accum: usize = 0; - for column in columns.iter() { - let value = - DataTypeValue::from_str(column.1.as_ref()).expect("data type should be supported"); - let data_type = value.as_data_type(); - data_type.advance_accum(&mut accum); - } - accum - }; +) -> Result, DataDecodeError> { + if columns.is_empty() { + return Ok(Vec::new()); + } + + let data_types = columns + .iter() + .map(|column| DataTypeValue::from_str(column.1.as_ref())) + .collect::, _>>()?; + let mut data_length = 0usize; + for value in &data_types { + value.as_data_type().advance_accum(&mut data_length); + } if !data_length.is_multiple_of(4) { data_length += 4 - data_length % 4; } - let start_pointer = unsafe { buf.as_ptr().add(buf.len()).sub(data_length) }; - let mut current_pointer = start_pointer; - let mut output: Vec<_> = vec![]; - for column in columns.iter() { - let value = - DataTypeValue::from_str(column.1.as_ref()).expect("data type should be supported"); + let row_start = buf + .len() + .checked_sub(data_length) + .ok_or(DataDecodeError::BufferTooShort { + required: data_length, + actual: buf.len(), + })?; + let mut row_offset = 0usize; + let mut output = Vec::with_capacity(data_types.len()); + for (field_index, value) in data_types.iter().enumerate() { let data_type = value.as_data_type(); - let deserialized = data_type.from_pointer(current_pointer, start_pointer); - data_type.advance_pointer_for_padding(&mut current_pointer, start_pointer); + data_type.advance_accum(&mut row_offset); + let field_end = row_start + .checked_add(row_offset) + .filter(|field_end| *field_end <= buf.len()) + .ok_or(DataDecodeError::FieldOutOfBounds { + field_index, + field_end: row_start.saturating_add(row_offset), + actual: buf.len(), + })?; + + // `rkyv::access` roots the requested archived value at the end of its + // input slice. The slice must therefore end exactly after this field, + // while retaining the complete serialized prefix: an out-of-line + // archived string may point backward into payload bytes before the + // fixed-width row root at `row_start`. + let deserialized = data_type.from_archived_bytes(&buf[..field_end])?; output.push(deserialized); - data_type.advance_pointer(&mut current_pointer); } - output + Ok(output) } #[cfg(test)] mod test { use super::parse_archived_row; - use crate::persistence::data::types::DataTypeValue; + use crate::persistence::data::{DataDecodeError, DataTypeValue}; use rkyv::{Archive, Deserialize, Serialize}; use std::f64::consts::PI; @@ -52,13 +79,54 @@ mod test { string1: "000000000000000".to_string(), }) .unwrap(); - let parsed = parse_archived_row(&buffer, &[("string1", "String")]); + let parsed = parse_archived_row(&buffer, &[("string1", "String")]).unwrap(); assert_eq!( parsed, [DataTypeValue::String("000000000000000".to_string())] ) } + #[test] + fn empty_schema_accepts_empty_buffer() { + let columns: [(String, String); 0] = []; + assert!(parse_archived_row(&[], &columns).unwrap().is_empty()); + } + + #[test] + fn short_buffer_returns_error_before_decoding() { + assert_eq!( + parse_archived_row(&[], &[("col", "String")]), + Err(DataDecodeError::BufferTooShort { + required: 8, + actual: 0, + }) + ); + } + + #[test] + fn malformed_string_archive_is_rejected() { + // An out-of-line string of length 9 whose relative pointer is far + // outside this eight-byte buffer. + let malformed = [0x89, 0, 0, 0, 0x7f, 0x7f, 0x7f, 0x7f]; + assert!(matches!( + parse_archived_row(&malformed, &[("col", "String")]), + Err(DataDecodeError::InvalidArchive { + data_type: "String", + .. + }) + )); + } + + #[test] + fn unsupported_schema_type_is_rejected() { + assert_eq!( + parse_archived_row(&[0; 8], &[("col", "not-a-type")]), + Err(DataDecodeError::UnsupportedDataType { + data_type: "not-a-type".to_owned(), + }) + ); + } + #[derive(Archive, Serialize, Deserialize, Debug)] struct Struct2 { pub int1: i32, @@ -67,10 +135,26 @@ mod test { #[test] fn test_parse_archived_row_int() { let buffer = rkyv::to_bytes::(&Struct2 { int1: 3 }).unwrap(); - let parsed = parse_archived_row(&buffer, &[("int1", "i32")]); + let parsed = parse_archived_row(&buffer, &[("int1", "i32")]).unwrap(); assert_eq!(parsed, [DataTypeValue::I32(3)]) } + #[test] + fn misaligned_primitive_archive_is_rejected() { + let buffer = rkyv::to_bytes::(&Struct2 { int1: 3 }).unwrap(); + let mut storage = Vec::with_capacity(buffer.len() + 1); + storage.push(0); + storage.extend_from_slice(&buffer); + + assert!(matches!( + parse_archived_row(&storage[1..], &[("int1", "i32")]), + Err(DataDecodeError::InvalidArchive { + data_type: "i32", + .. + }) + )); + } + #[derive(Archive, Serialize, Deserialize, Debug)] struct Struct3 { pub float1: f64, @@ -79,10 +163,35 @@ mod test { #[test] fn test_parse_archived_row_float() { let buffer = rkyv::to_bytes::(&Struct3 { float1: PI }).unwrap(); - let parsed = parse_archived_row(&buffer, &[("float1", "f64")]); + let parsed = parse_archived_row(&buffer, &[("float1", "f64")]).unwrap(); assert_eq!(parsed, [DataTypeValue::F64(PI)]) } + #[derive(Archive, Serialize, Deserialize, Debug)] + struct StructWithTrailingLongString { + pub int1: u32, + pub string1: String, + } + + #[test] + fn out_of_line_string_after_primitive_uses_complete_archive_prefix() { + let string1 = "out-of-line string payload after a primitive".to_owned(); + let buffer = rkyv::to_bytes::(&StructWithTrailingLongString { + int1: 42, + string1: string1.clone(), + }) + .unwrap(); + + // The payload precedes the fixed-width archived row root. Keeping the + // full prefix is required for the string's relative pointer to remain + // within the validation subtree. + assert!(buffer.len() > 12); + assert_eq!( + parse_archived_row(&buffer, &[("int1", "u32"), ("string1", "String")]).unwrap(), + [DataTypeValue::U32(42), DataTypeValue::String(string1)] + ); + } + #[derive(Archive, Serialize, Deserialize, Debug)] struct Struct4 { pub string1: String, @@ -129,7 +238,8 @@ mod test { ("int7".to_string(), "i8".to_string()), ("float1".to_string(), "f64".to_string()), ], - ); + ) + .unwrap(); assert_eq!( parsed, [ diff --git a/src/persistence/data/types.rs b/src/persistence/data/types.rs index b30ed33..3490693 100644 --- a/src/persistence/data/types.rs +++ b/src/persistence/data/types.rs @@ -8,8 +8,8 @@ use rkyv::primitive::{ }; use rkyv::string::ArchivedString; -use crate::persistence::data::util::{advance_accum_for_padding, advance_pointer_for_padding}; -use crate::persistence::data::DataType; +use crate::persistence::data::util::advance_accum_for_padding; +use crate::persistence::data::{DataDecodeError, DataType}; #[derive(Debug, Display, From, PartialEq)] pub enum DataTypeValue { @@ -49,10 +49,10 @@ impl DataTypeValue { } impl FromStr for DataTypeValue { - type Err = (); + type Err = DataDecodeError; fn from_str(s: &str) -> Result { - Ok(match s { + let value = match s { "String" => String::default().into(), "i128" => i128::default().into(), "i64" => i64::default().into(), @@ -66,8 +66,13 @@ impl FromStr for DataTypeValue { "u8" => u8::default().into(), "f64" => f64::default().into(), "f32" => f32::default().into(), - _ => unreachable!(), - }) + _ => { + return Err(DataDecodeError::UnsupportedDataType { + data_type: s.to_owned(), + }) + } + }; + Ok(value) } } @@ -77,18 +82,15 @@ impl DataType for String { *accum += size_of::(); } - fn from_pointer(&self, pointer: *const u8, start_pointer: *const u8) -> DataTypeValue { - let current_pointer = advance_pointer_for_padding(pointer, start_pointer, 4); - let archived_ptr: *const ArchivedString = current_pointer.cast(); - unsafe { (*archived_ptr).to_string() }.into() - } - - fn advance_pointer_for_padding(&self, pointer: &mut *const u8, start_pointer: *const u8) { - *pointer = advance_pointer_for_padding(*pointer, start_pointer, 4); - } - - fn advance_pointer(&self, pointer: &mut *const u8) { - *pointer = unsafe { pointer.add(size_of::()) }; + fn from_archived_bytes(&self, bytes: &[u8]) -> Result { + let archived = + rkyv::access::(bytes).map_err(|error| { + DataDecodeError::InvalidArchive { + data_type: "String", + message: error.to_string(), + } + })?; + Ok(archived.as_str().to_owned().into()) } } @@ -100,31 +102,13 @@ macro_rules! impl_datatype { *accum += size_of::<$archived_datatype>(); } - fn from_pointer(&self, pointer: *const u8, start_pointer: *const u8) -> DataTypeValue { - let current_pointer = advance_pointer_for_padding( - pointer, - start_pointer, - size_of::<$archived_datatype>(), - ); - let archived_ptr: *const $archived_datatype = current_pointer.cast(); - - $datatype_value(unsafe { (*archived_ptr) }.into()) - } - - fn advance_pointer_for_padding( - &self, - pointer: &mut *const u8, - start_pointer: *const u8, - ) { - *pointer = advance_pointer_for_padding( - *pointer, - start_pointer, - size_of::<$archived_datatype>(), - ); - } - - fn advance_pointer(&self, pointer: &mut *const u8) { - *pointer = unsafe { pointer.add(size_of::<$archived_datatype>()) }; + fn from_archived_bytes(&self, bytes: &[u8]) -> Result { + let archived = rkyv::access::<$archived_datatype, rkyv::rancor::Error>(bytes) + .map_err(|error| DataDecodeError::InvalidArchive { + data_type: stringify!($datatype), + message: error.to_string(), + })?; + Ok($datatype_value((*archived).into())) } } }; diff --git a/src/persistence/data/util.rs b/src/persistence/data/util.rs index 804b984..a358d01 100644 --- a/src/persistence/data/util.rs +++ b/src/persistence/data/util.rs @@ -4,20 +4,3 @@ pub fn advance_accum_for_padding(mut accum: usize, padding: usize) -> usize { } accum } - -pub fn advance_pointer_for_padding( - mut current_pointer: *const u8, - start_pointer: *const u8, - padding: usize, -) -> *const u8 { - if unsafe { current_pointer.byte_offset_from(start_pointer) % padding as isize != 0 } { - current_pointer = unsafe { - current_pointer.add( - (padding as isize - - current_pointer.byte_offset_from(start_pointer) % padding as isize) - as usize, - ) - }; - } - current_pointer -} diff --git a/src/util/sized.rs b/src/util/sized.rs index a4e2e1f..8a042ae 100644 --- a/src/util/sized.rs +++ b/src/util/sized.rs @@ -38,6 +38,19 @@ pub fn align_vec(mut v: AlignedVec) -> Aligne pub trait SizeMeasurable { /// Returns approximate size of the object archiving via [`rkyv`]. fn aligned_size(&self) -> usize; + + /// Returns the archived size of this type's default value. + /// + /// The default implementation constructs `Self::default()`. Types with an + /// expensive default may override this method in their `SizeMeasurable` + /// implementation and return the size directly. + fn default_aligned_size() -> usize + where + Self: Default, + { + Self::default().aligned_size() + } + fn align() -> Option { None } @@ -60,6 +73,23 @@ macro_rules! size_measurable_for_sized { size_measurable_for_sized! {u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, f32, f64, bool} +macro_rules! size_measurable_for_primitive_arrays { + ($($t:ident),+) => { + $( + impl SizeMeasurable for [$t; N] { + fn aligned_size(&self) -> usize { + mem::size_of::<[$t; N]>() + } + } + )+ + }; +} + +// Deliberately retain the previous `align() == None` behavior of the existing +// `[u8; 20]` and `[u8; 32]` implementations. Changing that value can alter +// persisted tuple and page offsets. +size_measurable_for_primitive_arrays! {u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, f32, f64, bool} + impl SizeMeasurable for Link { fn aligned_size(&self) -> usize { LINK_LENGTH @@ -87,18 +117,6 @@ where } } -impl SizeMeasurable for [u8; 32] { - fn aligned_size(&self) -> usize { - mem::size_of::<[u8; 32]>() - } -} - -impl SizeMeasurable for [u8; 20] { - fn aligned_size(&self) -> usize { - mem::size_of::<[u8; 20]>() - } -} - impl SizeMeasurable for (T1, T2) where T1: SizeMeasurable, @@ -146,10 +164,10 @@ impl SizeMeasurable for String { impl SizeMeasurable for Vec where - T: SizeMeasurable + Default, + T: Default + SizeMeasurable, { fn aligned_size(&self) -> usize { - let val_size = T::default().aligned_size(); + let val_size = ::default_aligned_size(); let vec_content_size = if val_size == 2 { 2 } else if val_size == 4 { @@ -226,19 +244,19 @@ impl VariableSizeMeasurable for String { impl VariableSizeMeasurable for indexset::core::pair::Pair where K: VariableSizeMeasurable, - L: SizeMeasurable + Default, + L: Default + SizeMeasurable, { fn aligned_size(length: usize) -> usize { - align(L::default().aligned_size() + K::aligned_size(length)) + align(::default_aligned_size() + K::aligned_size(length)) } } impl VariableSizeMeasurable for indexset::core::multipair::MultiPair where K: VariableSizeMeasurable, - L: SizeMeasurable + Default, + L: Default + SizeMeasurable, { fn aligned_size(length: usize) -> usize { - align(L::default().aligned_size() + K::aligned_size(length)) + align(::default_aligned_size() + K::aligned_size(length)) } } @@ -249,6 +267,56 @@ mod test { use rkyv::to_bytes; use uuid::Uuid; + #[test] + fn primitive_arrays_preserve_existing_alignment_contract() { + assert_eq!([0u8; 20].aligned_size(), 20); + assert_eq!([0u8; 32].aligned_size(), 32); + assert_eq!(<[u8; 20] as SizeMeasurable>::align(), None); + assert_eq!(<[u8; 32] as SizeMeasurable>::align(), None); + + let values = [1u16, 2, 3]; + assert_eq!( + values.aligned_size(), + rkyv::to_bytes::(&values) + .unwrap() + .len() + ); + } + + #[test] + fn default_size_helper_matches_previous_expression() { + assert_eq!( + ::default_aligned_size(), + u64::default().aligned_size() + ); + } + + #[test] + fn type_can_override_default_size_without_constructing_default() { + struct ExpensiveDefault; + + impl Default for ExpensiveDefault { + fn default() -> Self { + panic!("the sizing override must not construct the default value") + } + } + + impl SizeMeasurable for ExpensiveDefault { + fn aligned_size(&self) -> usize { + 64 + } + + fn default_aligned_size() -> usize { + 64 + } + } + + assert_eq!( + ::default_aligned_size(), + 64 + ); + } + #[test] fn test_uuid() { let u = Uuid::new_v4(); diff --git a/tests/mod.rs b/tests/mod.rs index 4fb49fc..ebf4017 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -7,6 +7,13 @@ struct StringU { pub u: u16, } +#[derive(SizeMeasure, Archive, Serialize)] +enum FixedState { + Idle, + Running, + Complete, +} + #[test] fn test_string_u16() { let s = StringU { @@ -18,3 +25,13 @@ fn test_string_u16() { rkyv::to_bytes::(&s).unwrap().len() ) } + +#[test] +fn test_fieldless_enum_size() { + for state in [FixedState::Idle, FixedState::Running, FixedState::Complete] { + assert_eq!( + state.aligned_size(), + rkyv::to_bytes::(&state).unwrap().len() + ); + } +} diff --git a/tools/create-data-file/Cargo.toml b/tools/create-data-file/Cargo.toml index f873007..e94f28c 100644 --- a/tools/create-data-file/Cargo.toml +++ b/tools/create-data-file/Cargo.toml @@ -7,5 +7,4 @@ edition = "2021" clap = { version = "4.5.21", features = ["derive"] } data_bucket = { path = "../.." } eyre = "0.6.12" -rkyv = { version = "0.8.9", features = ["uuid-1"]} - +rkyv = { version = "0.8.17", features = ["uuid-1"]} diff --git a/tools/dump-data-file/src/main.rs b/tools/dump-data-file/src/main.rs index ec8e7c2..b1c0bde 100644 --- a/tools/dump-data-file/src/main.rs +++ b/tools/dump-data-file/src/main.rs @@ -78,7 +78,7 @@ fn main() -> eyre::Result<()> { for page in pages { let links = LinksIterator::new(&mut file, page, &space_info).collect::>(); for row in DataIterator::new(&mut file, row_schema.clone(), links) { - rows.push(row); + rows.push(row?); } }