diff --git a/crates/iceberg/src/spec/schema/mod.rs b/crates/iceberg/src/spec/schema/mod.rs index 708f91d609..4c13a98327 100644 --- a/crates/iceberg/src/spec/schema/mod.rs +++ b/crates/iceberg/src/spec/schema/mod.rs @@ -36,7 +36,7 @@ use self::_serde::SchemaEnum; use self::id_reassigner::ReassignFieldIds; use self::index::{IndexByName, index_by_id, index_parents}; pub use self::prune_columns::prune_columns; -use super::NestedField; +use super::{Literal, NestedField}; use crate::error::Result; use crate::expr::accessor::StructAccessor; use crate::spec::FormatVersion; @@ -71,7 +71,7 @@ pub struct Schema { id_to_field: HashMap, name_to_id: HashMap, - lowercase_name_to_id: HashMap, + lowercase_name_to_id: HashMap>, id_to_name: HashMap, field_id_to_accessor: HashMap>, @@ -150,10 +150,18 @@ impl SchemaBuilder { index.indexes() }; - let lowercase_name_to_id = name_to_id - .iter() - .map(|(k, v)| (k.to_lowercase(), *v)) - .collect(); + let mut lowercase_name_to_id: HashMap> = + HashMap::with_capacity(name_to_id.len()); + for (name, id) in &name_to_id { + lowercase_name_to_id + .entry(name.to_lowercase()) + .and_modify(|existing| { + if existing.is_some_and(|existing_id| existing_id != *id) { + *existing = None; + } + }) + .or_insert(Some(*id)); + } let highest_field_id = id_to_field.keys().max().cloned().unwrap_or(0); @@ -354,7 +362,9 @@ impl Schema { pub fn field_by_name_case_insensitive(&self, field_name: &str) -> Option<&NestedFieldRef> { self.lowercase_name_to_id .get(&field_name.to_lowercase()) - .and_then(|id| self.field_by_id(*id)) + .copied() + .flatten() + .and_then(|id| self.field_by_id(id)) } /// Get field by alias. @@ -405,8 +415,8 @@ impl Schema { /// Check if this schema is identical to another schema semantically - excluding schema id. pub(crate) fn is_same_schema(&self, other: &SchemaRef) -> bool { - self.as_struct().eq(other.as_struct()) - && self.identifier_field_ids().eq(other.identifier_field_ids()) + same_struct(self.as_struct(), other.as_struct()) + && self.identifier_field_ids == other.identifier_field_ids } /// Change the schema id of this schema. @@ -511,6 +521,48 @@ impl Schema { } } +fn same_struct(left: &StructType, right: &StructType) -> bool { + left.fields().len() == right.fields().len() + && left + .fields() + .iter() + .zip(right.fields()) + .all(|(left, right)| same_field(left, right)) +} + +fn same_field(left: &NestedField, right: &NestedField) -> bool { + left.id == right.id + && left.name == right.name + && left.required == right.required + && left.doc == right.doc + && same_type(&left.field_type, &right.field_type) + && same_default(&left.initial_default, &right.initial_default) + && same_default(&left.write_default, &right.write_default) +} + +fn same_type(left: &Type, right: &Type) -> bool { + match (left, right) { + (Type::Primitive(left), Type::Primitive(right)) => left == right, + (Type::Struct(left), Type::Struct(right)) => same_struct(left, right), + (Type::List(left), Type::List(right)) => { + same_field(&left.element_field, &right.element_field) + } + (Type::Map(left), Type::Map(right)) => { + same_field(&left.key_field, &right.key_field) + && same_field(&left.value_field, &right.value_field) + } + (Type::Variant(left), Type::Variant(right)) => left == right, + _ => false, + } +} + +fn same_default(left: &Option, right: &Option) -> bool { + match (left, right) { + (Some(Literal::Primitive(left)), Some(Literal::Primitive(right))) => left.same_value(right), + _ => left == right, + } +} + impl Display for Schema { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { writeln!(f, "table {{")?; @@ -969,6 +1021,21 @@ table { } } + #[test] + fn test_schema_case_insensitive_lookup_rejects_ambiguous_names() { + let schema = Schema::builder() + .with_fields([ + NestedField::optional(1, "foo", Primitive(PrimitiveType::Int)).into(), + NestedField::optional(2, "FOO", Primitive(PrimitiveType::Long)).into(), + ]) + .build() + .unwrap(); + + assert_eq!(schema.field_by_name("foo").unwrap().id, 1); + assert_eq!(schema.field_by_name("FOO").unwrap().id, 2); + assert!(schema.field_by_name_case_insensitive("fOo").is_none()); + } + #[test] fn test_schema_find_column_name() { let expected_column_name = HashMap::from([ @@ -1307,6 +1374,54 @@ table { assert_eq!(0, schema.highest_field_id()); } + #[test] + fn test_same_schema_ignores_identifier_set_iteration_order() { + let fields = || { + (1..=8) + .map(|id| { + NestedField::required(id, format!("id_{id}"), Primitive(PrimitiveType::Long)) + .into() + }) + .collect::>() + }; + let left = Schema::builder() + .with_fields(fields()) + .with_identifier_field_ids(1..=8) + .build() + .unwrap(); + let right = Schema::builder() + .with_fields(fields()) + .with_identifier_field_ids((1..=8).rev()) + .build() + .unwrap(); + + assert!(left.is_same_schema(&std::sync::Arc::new(right))); + } + + #[test] + fn test_same_schema_uses_java_floating_point_equality_for_defaults() { + let schema_with_default = |default| { + Schema::builder() + .with_fields([ + NestedField::optional(1, "value", Primitive(PrimitiveType::Float)) + .with_initial_default(default) + .into(), + ]) + .build() + .unwrap() + }; + + let negative_zero = schema_with_default(Literal::float(-0.0)); + let positive_zero = std::sync::Arc::new(schema_with_default(Literal::float(0.0))); + assert!(!negative_zero.is_same_schema(&positive_zero)); + + let first_nan = schema_with_default(Literal::float(f32::from_bits(0x7fc0_0001))); + let second_nan = std::sync::Arc::new(schema_with_default(Literal::float(f32::from_bits( + 0x7fc0_0002, + )))); + assert!(first_nan.is_same_schema(&second_nan)); + } + #[test] fn test_field_ids_must_be_unique() { let reassigned_schema = Schema::builder() diff --git a/crates/iceberg/src/spec/values/datum.rs b/crates/iceberg/src/spec/values/datum.rs index 51da5d4e34..2b74abee79 100644 --- a/crates/iceberg/src/spec/values/datum.rs +++ b/crates/iceberg/src/spec/values/datum.rs @@ -299,16 +299,16 @@ impl Display for Datum { (_, PrimitiveLiteral::Float(val)) => write!(f, "{val}"), (_, PrimitiveLiteral::Double(val)) => write!(f, "{val}"), (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { - write!(f, "{}", date::days_to_date(*val)) + write!(f, "{}", date::days_to_iso_date(*val)) } (PrimitiveType::Time, PrimitiveLiteral::Long(val)) => { write!(f, "{}", time::microseconds_to_time(*val)) } (PrimitiveType::Timestamp, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", timestamp::microseconds_to_datetime(*val)) + write!(f, "{}", timestamp::microseconds_to_display(*val)) } (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(val)) => { - write!(f, "{}", timestamptz::microseconds_to_datetimetz(*val)) + write!(f, "{}", timestamptz::microseconds_to_display(*val)) } (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(val)) => { write!(f, "{}", timestamp::nanoseconds_to_datetime(*val)) diff --git a/crates/iceberg/src/spec/values/literal.rs b/crates/iceberg/src/spec/values/literal.rs index 5296eff2a2..5f60e5e719 100644 --- a/crates/iceberg/src/spec/values/literal.rs +++ b/crates/iceberg/src/spec/values/literal.rs @@ -460,11 +460,9 @@ impl Literal { "Failed to convert json number to double", ))?)), ))), - (PrimitiveType::Date, JsonValue::String(s)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( - date::date_to_days(&NaiveDate::parse_from_str(&s, "%Y-%m-%d")?), - )))) - } + (PrimitiveType::Date, JsonValue::String(s)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::Int(date::iso_date_to_days(&s)?), + ))), (PrimitiveType::Date, JsonValue::Number(number)) => { Ok(Some(Literal::Primitive(PrimitiveLiteral::Int( number @@ -482,17 +480,11 @@ impl Literal { )))) } (PrimitiveType::Timestamp, JsonValue::String(s)) => Ok(Some(Literal::Primitive( - PrimitiveLiteral::Long(timestamp::datetime_to_microseconds( - &NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f")?, - )), + PrimitiveLiteral::Long(timestamp::iso_datetime_to_microseconds(&s)?), + ))), + (PrimitiveType::Timestamptz, JsonValue::String(s)) => Ok(Some(Literal::Primitive( + PrimitiveLiteral::Long(timestamptz::iso_datetime_to_microseconds(&s)?), ))), - (PrimitiveType::Timestamptz, JsonValue::String(s)) => { - Ok(Some(Literal::Primitive(PrimitiveLiteral::Long( - timestamptz::datetimetz_to_microseconds(&Utc.from_utc_datetime( - &NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f+00:00")?, - )), - )))) - } (PrimitiveType::TimestampNs, JsonValue::String(s)) => { let ndt = NaiveDateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f")?; let nanos = timestamp::datetime_to_nanoseconds(&ndt).ok_or_else(|| { @@ -663,20 +655,16 @@ impl Literal { } } (PrimitiveType::Date, PrimitiveLiteral::Int(val)) => { - Ok(JsonValue::String(date::days_to_date(val).to_string())) + Ok(JsonValue::String(date::days_to_iso_date(val))) } (PrimitiveType::Time, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( time::microseconds_to_time(val).to_string(), )), (PrimitiveType::Timestamp, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( - timestamp::microseconds_to_datetime(val) - .format("%Y-%m-%dT%H:%M:%S%.f") - .to_string(), + timestamp::microseconds_to_iso_datetime(val), )), (PrimitiveType::Timestamptz, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( - timestamptz::microseconds_to_datetimetz(val) - .format("%Y-%m-%dT%H:%M:%S%.f+00:00") - .to_string(), + timestamptz::microseconds_to_iso_datetime(val), )), (PrimitiveType::TimestampNs, PrimitiveLiteral::Long(val)) => Ok(JsonValue::String( timestamp::nanoseconds_to_datetime(val) diff --git a/crates/iceberg/src/spec/values/primitive.rs b/crates/iceberg/src/spec/values/primitive.rs index 43d5c48c54..a07b7a4fc4 100644 --- a/crates/iceberg/src/spec/values/primitive.rs +++ b/crates/iceberg/src/spec/values/primitive.rs @@ -47,6 +47,18 @@ pub enum PrimitiveLiteral { } impl PrimitiveLiteral { + pub(crate) fn same_value(&self, other: &Self) -> bool { + match (self, other) { + (PrimitiveLiteral::Float(left), PrimitiveLiteral::Float(right)) => { + (left.is_nan() && right.is_nan()) || left.0.to_bits() == right.0.to_bits() + } + (PrimitiveLiteral::Double(left), PrimitiveLiteral::Double(right)) => { + (left.is_nan() && right.is_nan()) || left.0.to_bits() == right.0.to_bits() + } + _ => self == other, + } + } + /// Returns true if the Literal represents a primitive type /// that can be a NaN, and that it's value is NaN pub fn is_nan(&self) -> bool { @@ -57,3 +69,27 @@ impl PrimitiveLiteral { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_same_value_uses_java_floating_point_equality() { + assert!( + !PrimitiveLiteral::Float(OrderedFloat(-0.0)) + .same_value(&PrimitiveLiteral::Float(OrderedFloat(0.0))) + ); + assert!( + !PrimitiveLiteral::Double(OrderedFloat(-0.0)) + .same_value(&PrimitiveLiteral::Double(OrderedFloat(0.0))) + ); + + let first_nan = f32::from_bits(0x7fc0_0001); + let second_nan = f32::from_bits(0x7fc0_0002); + assert!( + PrimitiveLiteral::Float(OrderedFloat(first_nan)) + .same_value(&PrimitiveLiteral::Float(OrderedFloat(second_nan))) + ); + } +} diff --git a/crates/iceberg/src/spec/values/temporal.rs b/crates/iceberg/src/spec/values/temporal.rs index 80aadeb659..e82fec4a4a 100644 --- a/crates/iceberg/src/spec/values/temporal.rs +++ b/crates/iceberg/src/spec/values/temporal.rs @@ -17,24 +17,308 @@ //! Temporal value conversions for dates, times, and timestamps -use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeDelta, TimeZone, Utc}; +use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, TimeZone, Utc}; + +use crate::{Error, ErrorKind, Result}; + +const NANOS_PER_SECOND: i128 = 1_000_000_000; +const NANOS_PER_DAY: i128 = 86_400 * NANOS_PER_SECOND; + +fn invalid_iso_value(kind: &str) -> Error { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid ISO-8601 {kind} value"), + ) +} + +fn civil_from_days(days: i64) -> (i64, u32, u32) { + // Howard Hinnant's proleptic Gregorian calendar conversion. Unlike chrono, + // this covers every date reachable by Iceberg's i32 day and i64 microsecond + // representations. + let days = days + 719_468; + let era = days.div_euclid(146_097); + let day_of_era = days - era * 146_097; + let year_of_era = + (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let mut year = year_of_era + era * 400; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_prime = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_prime + 2) / 5 + 1; + let month = month_prime + if month_prime < 10 { 3 } else { -9 }; + year += i64::from(month <= 2); + (year, month as u32, day as u32) +} + +fn days_from_civil(year: i64, month: u32, day: u32) -> i64 { + let year = year - i64::from(month <= 2); + let era = year.div_euclid(400); + let year_of_era = year - era * 400; + let month_prime = i64::from(month) + if month > 2 { -3 } else { 9 }; + let day_of_year = (153 * month_prime + 2) / 5 + i64::from(day) - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +fn is_leap_year(year: i64) -> bool { + year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) +} + +fn days_in_month(year: i64, month: u32) -> u32 { + match month { + 2 if is_leap_year(year) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +fn format_iso_year(year: i64) -> String { + match year { + 0..=9_999 => format!("{year:04}"), + -9_999..=-1 => format!("-{:04}", -year), + 10_000.. => format!("+{year}"), + _ => year.to_string(), + } +} + +fn format_iso_date(days: i64) -> String { + let (year, month, day) = civil_from_days(days); + format!("{}-{month:02}-{day:02}", format_iso_year(year)) +} + +fn parse_iso_date(value: &str) -> Result { + let (year_and_month, day) = value + .rsplit_once('-') + .ok_or_else(|| invalid_iso_value("date"))?; + let (year, month) = year_and_month + .rsplit_once('-') + .ok_or_else(|| invalid_iso_value("date"))?; + if month.len() != 2 + || day.len() != 2 + || !month.bytes().all(|byte| byte.is_ascii_digit()) + || !day.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(invalid_iso_value("date")); + } + + let (sign, year_digits) = match year.as_bytes().first() { + Some(b'+') => (1_i64, &year[1..]), + Some(b'-') => (-1_i64, &year[1..]), + _ => (1_i64, year), + }; + let signed = year.starts_with(['+', '-']); + if year_digits.len() < 4 + || year_digits.len() > 10 + || (!signed && year_digits.len() != 4) + || (year.starts_with('+') && year_digits.len() == 4) + || !year_digits.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(invalid_iso_value("date")); + } + let magnitude = year_digits + .parse::() + .map_err(|err| invalid_iso_value("date").with_source(err))?; + if year.starts_with('-') && magnitude == 0 { + return Err(invalid_iso_value("date")); + } + let year = sign * magnitude; + let month = month + .parse::() + .map_err(|err| invalid_iso_value("date").with_source(err))?; + let day = day + .parse::() + .map_err(|err| invalid_iso_value("date").with_source(err))?; + if !(1..=12).contains(&month) || day == 0 || day > days_in_month(year, month) { + return Err(invalid_iso_value("date")); + } + Ok(days_from_civil(year, month, day)) +} + +fn parse_two_digits(value: &str, kind: &str) -> Result { + if value.len() != 2 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(invalid_iso_value(kind)); + } + value + .parse::() + .map_err(|err| invalid_iso_value(kind).with_source(err)) +} + +fn parse_iso_time_nanos(value: &str) -> Result { + let mut components = value.split(':'); + let hour = parse_two_digits( + components.next().ok_or_else(|| invalid_iso_value("time"))?, + "time", + )?; + let minute = parse_two_digits( + components.next().ok_or_else(|| invalid_iso_value("time"))?, + "time", + )?; + let second_and_fraction = components.next(); + if components.next().is_some() { + return Err(invalid_iso_value("time")); + } + + let (second, nanos) = if let Some(second_and_fraction) = second_and_fraction { + let (second, fraction) = second_and_fraction + .split_once('.') + .map_or((second_and_fraction, None), |(second, fraction)| { + (second, Some(fraction)) + }); + let second = parse_two_digits(second, "time")?; + let nanos = if let Some(fraction) = fraction { + if fraction.is_empty() + || fraction.len() > 9 + || !fraction.bytes().all(|byte| byte.is_ascii_digit()) + { + return Err(invalid_iso_value("time")); + } + let value = fraction + .parse::() + .map_err(|err| invalid_iso_value("time").with_source(err))?; + value * 10_u32.pow(9 - fraction.len() as u32) + } else { + 0 + }; + (second, nanos) + } else { + (0, 0) + }; + + if hour > 23 || minute > 59 || second > 59 { + return Err(invalid_iso_value("time")); + } + Ok( + (i128::from(hour) * 3_600 + i128::from(minute) * 60 + i128::from(second)) + * NANOS_PER_SECOND + + i128::from(nanos), + ) +} + +fn parse_iso_datetime_nanos(value: &str) -> Result { + let separator = value + .char_indices() + .find_map(|(index, character)| matches!(character, 'T' | 't').then_some(index)) + .ok_or_else(|| invalid_iso_value("timestamp"))?; + let days = parse_iso_date(&value[..separator])?; + let nanos = parse_iso_time_nanos(&value[separator + 1..])?; + Ok(i128::from(days) * NANOS_PER_DAY + nanos) +} + +fn parse_utc_offset(value: &str) -> Result { + if matches!(value, "Z" | "z") { + return Ok(0); + } + let (sign, digits) = match value.as_bytes().first() { + Some(b'+') => (1_i32, &value[1..]), + Some(b'-') => (-1_i32, &value[1..]), + _ => return Err(invalid_iso_value("UTC offset")), + }; + let components = digits.split(':').collect::>(); + if !matches!(components.as_slice(), [_, _] | [_, _, _]) { + return Err(invalid_iso_value("UTC offset")); + } + let hours = parse_two_digits(components[0], "UTC offset")? as i32; + let minutes = parse_two_digits(components[1], "UTC offset")? as i32; + let seconds = components + .get(2) + .map_or(Ok(0), |value| parse_two_digits(value, "UTC offset"))? as i32; + if hours > 18 || minutes > 59 || seconds > 59 || (hours == 18 && (minutes != 0 || seconds != 0)) + { + return Err(invalid_iso_value("UTC offset")); + } + Ok(sign * (hours * 3_600 + minutes * 60 + seconds)) +} + +fn parse_iso_offset_datetime_nanos(value: &str) -> Result<(i128, i32)> { + let time_start = value + .char_indices() + .find_map(|(index, character)| matches!(character, 'T' | 't').then_some(index + 1)) + .ok_or_else(|| invalid_iso_value("timestamptz"))?; + let offset_start = value[time_start..] + .char_indices() + .find_map(|(index, character)| { + matches!(character, 'Z' | 'z' | '+' | '-').then_some(time_start + index) + }) + .ok_or_else(|| invalid_iso_value("timestamptz"))?; + let local_nanos = parse_iso_datetime_nanos(&value[..offset_start])?; + let offset_seconds = parse_utc_offset(&value[offset_start..])?; + Ok(( + local_nanos - i128::from(offset_seconds) * NANOS_PER_SECOND, + offset_seconds, + )) +} + +fn nanos_to_microseconds(nanos: i128) -> Result { + i64::try_from(nanos / 1_000).map_err(|err| { + Error::new( + ErrorKind::DataInvalid, + "Timestamp is outside the representable microsecond range", + ) + .with_source(err) + }) +} + +fn datetime_components(micros: i64) -> (i64, i64, i64, i64, i64) { + const MICROS_PER_DAY: i64 = 86_400_000_000; + let days = micros.div_euclid(MICROS_PER_DAY); + let micros_of_day = micros.rem_euclid(MICROS_PER_DAY); + let seconds_of_day = micros_of_day / 1_000_000; + let micros_of_second = micros_of_day % 1_000_000; + let hour = seconds_of_day / 3_600; + let minute = seconds_of_day % 3_600 / 60; + let second = seconds_of_day % 60; + (days, hour, minute, second, micros_of_second) +} + +fn format_iso_datetime(micros: i64, with_offset: bool) -> String { + let (days, hour, minute, second, micros_of_second) = datetime_components(micros); + let mut result = format!( + "{}T{hour:02}:{minute:02}:{second:02}", + format_iso_date(days) + ); + if micros_of_second != 0 { + let fraction = format!("{micros_of_second:06}"); + result.push('.'); + result.push_str(fraction.trim_end_matches('0')); + } + if with_offset { + result.push_str("+00:00"); + } + result +} + +fn format_display_datetime(micros: i64, with_utc: bool) -> String { + let (days, hour, minute, second, micros_of_second) = datetime_components(micros); + let mut result = format!( + "{} {hour:02}:{minute:02}:{second:02}", + format_iso_date(days) + ); + if micros_of_second % 1_000 != 0 { + result.push_str(&format!(".{micros_of_second:06}")); + } else if micros_of_second != 0 { + result.push_str(&format!(".{:03}", micros_of_second / 1_000)); + } + if with_utc { + result.push_str(" UTC"); + } + result +} pub(crate) mod date { use super::*; - pub(crate) fn date_to_days(date: &NaiveDate) -> i32 { - date.signed_duration_since( - // This is always the same and shouldn't fail - NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(), - ) - .num_days() as i32 + pub(crate) fn days_to_iso_date(days: i32) -> String { + format_iso_date(i64::from(days)) } - pub(crate) fn days_to_date(days: i32) -> NaiveDate { - // This shouldn't fail until the year 262000 - (DateTime::UNIX_EPOCH + TimeDelta::try_days(days as i64).unwrap()) - .naive_utc() - .date() + pub(crate) fn iso_date_to_days(value: &str) -> Result { + i32::try_from(parse_iso_date(value)?).map_err(|err| { + Error::new( + ErrorKind::DataInvalid, + "Date is outside the representable day range", + ) + .with_source(err) + }) } /// Returns unix epoch. @@ -70,13 +354,16 @@ pub(crate) mod time { pub(crate) mod timestamp { use super::*; - pub(crate) fn datetime_to_microseconds(time: &NaiveDateTime) -> i64 { - time.and_utc().timestamp_micros() + pub(crate) fn microseconds_to_iso_datetime(micros: i64) -> String { + format_iso_datetime(micros, false) } - pub(crate) fn microseconds_to_datetime(micros: i64) -> NaiveDateTime { - // This shouldn't fail until the year 262000 - DateTime::from_timestamp_micros(micros).unwrap().naive_utc() + pub(crate) fn microseconds_to_display(micros: i64) -> String { + format_display_datetime(micros, false) + } + + pub(crate) fn iso_datetime_to_microseconds(value: &str) -> Result { + nanos_to_microseconds(parse_iso_datetime_nanos(value)?) } pub(crate) fn nanoseconds_to_datetime(nanos: i64) -> NaiveDateTime { @@ -93,18 +380,27 @@ pub(crate) mod timestamp { pub(crate) mod timestamptz { use super::*; - pub(crate) fn datetimetz_to_microseconds(time: &DateTime) -> i64 { - time.timestamp_micros() + pub(crate) fn microseconds_to_iso_datetime(micros: i64) -> String { + format_iso_datetime(micros, true) } - pub(crate) fn microseconds_to_datetimetz(micros: i64) -> DateTime { - let (secs, rem) = (micros / 1_000_000, micros % 1_000_000); + pub(crate) fn microseconds_to_display(micros: i64) -> String { + format_display_datetime(micros, true) + } - DateTime::from_timestamp(secs, rem as u32 * 1_000).unwrap() + pub(crate) fn iso_datetime_to_microseconds(value: &str) -> Result { + let (nanos, offset_seconds) = parse_iso_offset_datetime_nanos(value)?; + if offset_seconds != 0 { + return Err(invalid_iso_value("UTC timestamptz")); + } + nanos_to_microseconds(nanos) } pub(crate) fn nanoseconds_to_datetimetz(nanos: i64) -> DateTime { - let (secs, rem) = (nanos / 1_000_000_000, nanos % 1_000_000_000); + let (secs, rem) = ( + nanos.div_euclid(1_000_000_000), + nanos.rem_euclid(1_000_000_000), + ); DateTime::from_timestamp(secs, rem as u32).unwrap() } diff --git a/crates/iceberg/src/spec/values/tests.rs b/crates/iceberg/src/spec/values/tests.rs index 8bf3311004..3b3dab64b5 100644 --- a/crates/iceberg/src/spec/values/tests.rs +++ b/crates/iceberg/src/spec/values/tests.rs @@ -213,6 +213,157 @@ fn json_timestamptz() { ); } +#[test] +fn json_pre_epoch_timestamptz() { + check_json_serde( + r#""1969-12-31T23:59:59.999999+00:00""#, + Literal::Primitive(PrimitiveLiteral::Long(-1)), + &Primitive(PrimitiveType::Timestamptz), + ); +} + +#[test] +fn json_date_boundaries() { + for (value, encoded) in [ + (i32::MAX, r#""+5881580-07-11""#), + (i32::MIN, r#""-5877641-06-23""#), + ] { + check_json_serde( + encoded, + Literal::date(value), + &Primitive(PrimitiveType::Date), + ); + } + + for (encoded, value, canonical) in [ + ("+0005881580-07-11", i32::MAX, "+5881580-07-11"), + ("-0005877641-06-23", i32::MIN, "-5877641-06-23"), + ] { + let parsed = Literal::try_from_json( + JsonValue::String(encoded.to_string()), + &Primitive(PrimitiveType::Date), + ) + .unwrap(); + assert_eq!(parsed, Some(Literal::date(value))); + assert_eq!( + parsed + .unwrap() + .try_into_json(&Primitive(PrimitiveType::Date)) + .unwrap(), + JsonValue::String(canonical.to_string()) + ); + } +} + +#[test] +fn json_date_extended_year_transitions() { + let date_type = Primitive(PrimitiveType::Date); + for encoded in ["-10000-01-01", "-0001-12-31", "0000-01-01", "+10000-01-01"] { + let parsed = Literal::try_from_json(JsonValue::String(encoded.to_string()), &date_type) + .unwrap() + .unwrap(); + assert_eq!( + parsed.try_into_json(&date_type).unwrap(), + JsonValue::String(encoded.to_string()) + ); + } + + let positive_zero = + Literal::try_from_json(JsonValue::String("+00000-01-01".to_string()), &date_type) + .unwrap() + .unwrap(); + assert_eq!( + positive_zero.try_into_json(&date_type).unwrap(), + JsonValue::String("0000-01-01".to_string()) + ); + assert!( + Literal::try_from_json(JsonValue::String("-0000-01-01".to_string()), &date_type).is_err() + ); + + let timestamp_type = Primitive(PrimitiveType::Timestamp); + let positive_zero_timestamp = Literal::try_from_json( + JsonValue::String("+00000-01-01T00:00:00".to_string()), + ×tamp_type, + ) + .unwrap() + .unwrap(); + assert_eq!( + positive_zero_timestamp + .try_into_json(×tamp_type) + .unwrap(), + JsonValue::String("0000-01-01T00:00:00".to_string()) + ); +} + +#[test] +fn json_microsecond_timestamp_boundaries() { + for (value, timestamp, timestamptz) in [ + ( + i64::MAX, + r#""+294247-01-10T04:00:54.775807""#, + r#""+294247-01-10T04:00:54.775807+00:00""#, + ), + ( + i64::MIN, + r#""-290308-12-21T19:59:05.224192""#, + r#""-290308-12-21T19:59:05.224192+00:00""#, + ), + ] { + check_json_serde( + timestamp, + Literal::timestamp(value), + &Primitive(PrimitiveType::Timestamp), + ); + check_json_serde( + timestamptz, + Literal::timestamptz(value), + &Primitive(PrimitiveType::Timestamptz), + ); + } +} + +#[test] +fn json_microsecond_timestamp_fraction_rendering() { + for (value, encoded) in [ + (0, r#""1970-01-01T00:00:00""#), + (1, r#""1970-01-01T00:00:00.000001""#), + (10, r#""1970-01-01T00:00:00.00001""#), + (100, r#""1970-01-01T00:00:00.0001""#), + (1_000, r#""1970-01-01T00:00:00.001""#), + (10_000, r#""1970-01-01T00:00:00.01""#), + (100_000, r#""1970-01-01T00:00:00.1""#), + (-1, r#""1969-12-31T23:59:59.999999""#), + ] { + check_json_serde( + encoded, + Literal::timestamp(value), + &Primitive(PrimitiveType::Timestamp), + ); + } +} + +#[test] +fn datum_temporal_boundary_display() { + assert_eq!(Datum::date(i32::MAX).to_string(), "+5881580-07-11"); + assert_eq!(Datum::date(i32::MIN).to_string(), "-5877641-06-23"); + assert_eq!( + Datum::timestamp_micros(i64::MAX).to_string(), + "+294247-01-10 04:00:54.775807" + ); + assert_eq!( + Datum::timestamp_micros(i64::MIN).to_string(), + "-290308-12-21 19:59:05.224192" + ); + assert_eq!( + Datum::timestamptz_micros(i64::MAX).to_string(), + "+294247-01-10 04:00:54.775807 UTC" + ); + assert_eq!( + Datum::timestamptz_micros(i64::MIN).to_string(), + "-290308-12-21 19:59:05.224192 UTC" + ); +} + #[test] fn json_timestamp_ns() { let record = r#""2017-11-16T22:31:08.123456789""#; @@ -235,6 +386,15 @@ fn json_timestamptz_ns() { ); } +#[test] +fn json_pre_epoch_timestamptz_ns() { + check_json_serde( + r#""1969-12-31T23:59:59.999999999+00:00""#, + Literal::timestamptz_nano(-1), + &Primitive(PrimitiveType::TimestamptzNs), + ); +} + #[test] fn json_timestamptz_ns_rejects_non_utc_offset() { // Per the spec, timestamptz_ns single-value serialization must use offset "+00:00"; Java's