From a9343a2e786fc3cc04b2df4da71445216676165d Mon Sep 17 00:00:00 2001 From: GF Date: Thu, 13 Aug 2026 11:32:24 -0400 Subject: [PATCH] Prepare 0.9.1 safe SIMD release --- CHANGELOG.md | 16 + Cargo.lock | 51 +- Cargo.toml | 4 +- crates/j2k-cli/Cargo.toml | 6 +- crates/j2k-compare/Cargo.toml | 8 +- crates/j2k-cuda-runtime/Cargo.toml | 6 +- crates/j2k-cuda/Cargo.toml | 10 +- crates/j2k-jpeg-cuda/Cargo.toml | 8 +- crates/j2k-jpeg-metal/Cargo.toml | 8 +- crates/j2k-jpeg/Cargo.toml | 24 +- crates/j2k-jpeg/benches/decode_cpu.rs | 270 +++++++ crates/j2k-jpeg/benches/micro.rs | 69 +- crates/j2k-jpeg/fuzz/Cargo.lock | 15 +- crates/j2k-jpeg/src/backend/mod.rs | 152 ++-- crates/j2k-jpeg/src/backend/neon.rs | 662 ++++++++---------- .../src/backend/neon/top_only_tests.rs | 16 +- crates/j2k-jpeg/src/backend/tests.rs | 133 +++- crates/j2k-jpeg/src/backend/x86.rs | 217 +++--- crates/j2k-jpeg/src/bench_support.rs | 305 +++++++- crates/j2k-jpeg/src/idct/avx2.rs | 96 ++- crates/j2k-jpeg/src/idct/neon.rs | 178 +++-- crates/j2k-jpeg/src/lib.rs | 2 + crates/j2k-jpeg/src/simd/mod.rs | 12 + crates/j2k-jpeg/src/simd/neon_memory.rs | 83 +++ crates/j2k-jpeg/src/simd/x86.rs | 65 ++ crates/j2k-jpeg/src/simd/x86_memory.rs | 167 +++++ crates/j2k-metal-support/Cargo.toml | 2 +- crates/j2k-metal/Cargo.toml | 12 +- crates/j2k-ml/Cargo.toml | 12 +- crates/j2k-native/Cargo.toml | 6 +- crates/j2k-t803/Cargo.toml | 14 +- crates/j2k-t803/fuzz/Cargo.lock | 20 +- crates/j2k-test-support/Cargo.toml | 4 +- crates/j2k-tilecodec/Cargo.toml | 2 +- crates/j2k-tilecodec/fuzz/Cargo.lock | 4 +- crates/j2k-transcode-cuda/Cargo.toml | 8 +- crates/j2k-transcode-metal/Cargo.toml | 16 +- crates/j2k-transcode-test-support/Cargo.toml | 6 +- crates/j2k-transcode/Cargo.toml | 12 +- crates/j2k-transcode/fuzz/Cargo.lock | 21 +- crates/j2k/Cargo.toml | 10 +- crates/j2k/fuzz/Cargo.lock | 18 +- docs/architecture.md | 37 + docs/benchmark-evidence.md | 108 +++ .../public-api/public-api-review-0.9.1.yml | 130 ++++ .../reviewed-public-api-diff-0.9.1.md | 249 +++++++ docs/release.md | 10 +- ...able-api-1.0.implementation-public-api.txt | 24 + docs/stable-api-1.0.md | 33 +- docs/unsafe-audit.md | 10 +- xtask/src/coverage/tests/source_roles.rs | 2 +- xtask/src/quality_commands.rs | 191 ++++- .../release_commands/tests/orchestration.rs | 4 +- xtask/src/semver.rs | 17 +- xtask/src/semver/tests.rs | 17 +- xtask/src/semver/tests/api_planning.rs | 2 +- xtask/src/semver/tests/command_boundaries.rs | 6 +- xtask/tests/command_orchestration.rs | 4 +- xtask/tests/command_orchestration/support.rs | 2 +- 59 files changed, 2642 insertions(+), 954 deletions(-) create mode 100644 crates/j2k-jpeg/benches/decode_cpu.rs create mode 100644 crates/j2k-jpeg/src/simd/mod.rs create mode 100644 crates/j2k-jpeg/src/simd/neon_memory.rs create mode 100644 crates/j2k-jpeg/src/simd/x86.rs create mode 100644 crates/j2k-jpeg/src/simd/x86_memory.rs create mode 100644 docs/release-evidence/public-api/public-api-review-0.9.1.yml create mode 100644 docs/release-evidence/public-api/reviewed-public-api-diff-0.9.1.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d32bbdd..9dd15e19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,22 @@ and stale roadmap entries have been removed from the public documentation set. ## [Unreleased] +Staged workspace version: `0.9.1`. + +- Refactors the CPU JPEG AVX2 and NEON paths around unforgeable capability + tokens and safe kernel entry points. SIMD dispatch, IDCT/color arithmetic, + tests, and benchmark wrappers no longer require unsafe calls; the remaining + raw loads/stores and exact-AVX2 bridge are isolated behind fixed-size safe + interfaces. Ten production SIMD unsafe blocks remain, with no `unsafe fn`, + under an AST-enforced 24-block cap. +- Uses `fearless_simd 0.7.0` for AArch64 NEON kernels while preserving the + existing AVX2-only x86 acceleration envelope with a private exact-AVX2 token; + `fearless_simd::Avx2` is intentionally not used because its 0.7 contract is + the broader x86-64-v3 feature set. +- Adds deterministic CPU JPEG decode and expanded SIMD microbenchmarks covering + grayscale, 4:4:4, 4:2:2, full/odd/cropped 4:2:0, row streaming, specialized + IDCT, tail widths, and unaligned source rows. + ## [0.9.0] - 2026-08-10 - Breaking: all expert Metal APIs in `j2k-metal-support`, `j2k-metal`, diff --git a/Cargo.lock b/Cargo.lock index 53a7302f..d19b07c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1735,9 +1735,9 @@ dependencies = [ [[package]] name = "fearless_simd" -version = "0.4.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" [[package]] name = "filetime" @@ -2496,7 +2496,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "j2k-codec-math", @@ -2511,7 +2511,7 @@ dependencies = [ [[package]] name = "j2k-alloc-probe" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k-native", "j2k-profile", @@ -2521,7 +2521,7 @@ dependencies = [ [[package]] name = "j2k-cli" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k", "j2k-jpeg", @@ -2531,7 +2531,7 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.9.0" +version = "0.9.1" [[package]] name = "j2k-compare" @@ -2548,14 +2548,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.9.0" +version = "0.9.1" dependencies = [ "thiserror 2.0.18", ] [[package]] name = "j2k-cuda" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "j2k", @@ -2569,7 +2569,7 @@ dependencies = [ [[package]] name = "j2k-cuda-runtime" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -2580,9 +2580,10 @@ dependencies = [ [[package]] name = "j2k-jpeg" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", + "fearless_simd", "j2k-codec-math", "j2k-core", "j2k-profile", @@ -2598,7 +2599,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-cuda" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "j2k-core", @@ -2611,7 +2612,7 @@ dependencies = [ [[package]] name = "j2k-jpeg-metal" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "j2k-core", @@ -2628,7 +2629,7 @@ dependencies = [ [[package]] name = "j2k-metal" -version = "0.9.0" +version = "0.9.1" dependencies = [ "cc", "criterion", @@ -2650,7 +2651,7 @@ dependencies = [ [[package]] name = "j2k-metal-support" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k-core", "j2k-test-support", @@ -2662,7 +2663,7 @@ dependencies = [ [[package]] name = "j2k-ml" -version = "0.9.0" +version = "0.9.1" dependencies = [ "burn-core", "burn-cuda", @@ -2682,7 +2683,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "fearless_simd", @@ -2696,11 +2697,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.9.0" +version = "0.9.1" [[package]] name = "j2k-t803" -version = "0.9.0" +version = "0.9.1" dependencies = [ "image", "j2k", @@ -2723,7 +2724,7 @@ dependencies = [ [[package]] name = "j2k-test-support" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k-core", "j2k-native", @@ -2734,7 +2735,7 @@ dependencies = [ [[package]] name = "j2k-tilecodec" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "flate2", @@ -2746,7 +2747,7 @@ dependencies = [ [[package]] name = "j2k-transcode" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "j2k", @@ -2763,7 +2764,7 @@ dependencies = [ [[package]] name = "j2k-transcode-cuda" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k-core", "j2k-cuda-runtime", @@ -2775,7 +2776,7 @@ dependencies = [ [[package]] name = "j2k-transcode-metal" -version = "0.9.0" +version = "0.9.1" dependencies = [ "criterion", "j2k-codec-math", @@ -2796,7 +2797,7 @@ dependencies = [ [[package]] name = "j2k-transcode-test-support" -version = "0.9.0" +version = "0.9.1" dependencies = [ "j2k-native", "j2k-transcode", @@ -2805,7 +2806,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.9.0" +version = "0.9.1" [[package]] name = "jni-sys" diff --git a/Cargo.toml b/Cargo.toml index b9be6fb4..082a76d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ members = [ "xtask", ] [workspace.package] -version = "0.9.0" +version = "0.9.1" edition = "2021" rust-version = "1.96" license = "MIT OR Apache-2.0" @@ -85,7 +85,7 @@ jpeg-encoder = "0.7.0" jpeg-decoder = { version = "0.3.2", default-features = false } zune-core = "0.5.1" zune-jpeg = "0.5.15" -fearless_simd = "0.4" +fearless_simd = "0.7" libm = { version = "0.2", default-features = false } openjpeg-sys = "1.0.12" proc-macro2 = "1.0.106" diff --git a/crates/j2k-cli/Cargo.toml b/crates/j2k-cli/Cargo.toml index 0c5fd52a..33a54514 100644 --- a/crates/j2k-cli/Cargo.toml +++ b/crates/j2k-cli/Cargo.toml @@ -18,9 +18,9 @@ path = "src/main.rs" doc = false [dependencies] -j2k = { path = "../j2k", version = "=0.9.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.0" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.9.0" } +j2k = { path = "../j2k", version = "=0.9.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.9.1" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-compare/Cargo.toml b/crates/j2k-compare/Cargo.toml index 70f13b50..283e88cf 100644 --- a/crates/j2k-compare/Cargo.toml +++ b/crates/j2k-compare/Cargo.toml @@ -13,10 +13,10 @@ publish = false ignored = ["cc"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } -j2k = { path = "../j2k", version = "=0.9.0" } -j2k-test-support = { path = "../j2k-test-support", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } +j2k = { path = "../j2k", version = "=0.9.1" } +j2k-test-support = { path = "../j2k-test-support", version = "=0.9.1" } image = { workspace = true } openjpeg-sys = { workspace = true } diff --git a/crates/j2k-cuda-runtime/Cargo.toml b/crates/j2k-cuda-runtime/Cargo.toml index 5168acdf..b3708ad2 100644 --- a/crates/j2k-cuda-runtime/Cargo.toml +++ b/crates/j2k-cuda-runtime/Cargo.toml @@ -48,13 +48,13 @@ cuda-oxide-jpeg-decode = [] cuda-oxide-jpeg-encode = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } libloading = { workspace = true } thiserror = { workspace = true } [build-dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-cuda/Cargo.toml b/crates/j2k-cuda/Cargo.toml index 05169b2e..c4f83f2b 100644 --- a/crates/j2k-cuda/Cargo.toml +++ b/crates/j2k-cuda/Cargo.toml @@ -34,11 +34,11 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.0", optional = true } -j2k = { path = "../j2k", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.1", optional = true } +j2k = { path = "../j2k", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/j2k-jpeg-cuda/Cargo.toml b/crates/j2k-jpeg-cuda/Cargo.toml index 49295e4d..26b10dec 100644 --- a/crates/j2k-jpeg-cuda/Cargo.toml +++ b/crates/j2k-jpeg-cuda/Cargo.toml @@ -27,10 +27,10 @@ cuda-runtime = [ ] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.0", optional = true } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.1", optional = true } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1" } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/j2k-jpeg-metal/Cargo.toml b/crates/j2k-jpeg-metal/Cargo.toml index 1b037e89..a414796b 100644 --- a/crates/j2k-jpeg-metal/Cargo.toml +++ b/crates/j2k-jpeg-metal/Cargo.toml @@ -19,10 +19,10 @@ name = "j2k_jpeg_metal" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.0" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.1" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-jpeg/Cargo.toml b/crates/j2k-jpeg/Cargo.toml index 609b00e7..660d51ed 100644 --- a/crates/j2k-jpeg/Cargo.toml +++ b/crates/j2k-jpeg/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_jpeg" path = "src/lib.rs" [features] -scalar-only = [] # retained for fuzz/reference decode until native SIMD backends land +scalar-only = [] # deterministic fallback for fuzzing and differential testing # Opt-in for the comparison benches only: probes pkg-config for # libturbojpeg and emits link directives. Off by default so library # consumers never link a system JPEG. @@ -27,13 +27,16 @@ bench-libjpeg-turbo = [] bench-internals = [] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1" } thiserror = { workspace = true } memchr = { workspace = true } rayon = { workspace = true } +[target.'cfg(target_arch = "aarch64")'.dependencies] +fearless_simd = { workspace = true } + [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } @@ -70,6 +73,11 @@ name = "encode_cpu" harness = false test = false +[[bench]] +name = "decode_cpu" +harness = false +test = false + [[test]] name = "fast420_profile" required-features = ["bench-internals"] @@ -83,10 +91,10 @@ name = "neon_hot_paths" required-features = ["bench-internals"] [lints.rust] -# Audited unsafe is confined to SIMD/hot kernels in `src/backend/`, -# `src/idct/{avx2,neon}.rs`, and `src/entropy/sequential/deposit.rs`, plus -# benchmark-only SIMD shims, FFI, and allocator instrumentation. Keep every -# unsafe operation explicit and lint each unsafe block. +# Audited SIMD unsafe is confined to the private feature/memory boundaries in +# `src/simd/`; entropy deposit code, benchmark statistics pointers, FFI, and +# allocator instrumentation retain their separately audited boundaries. Keep +# every unsafe operation explicit and lint each unsafe block. unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" unreachable_pub = "warn" diff --git a/crates/j2k-jpeg/benches/decode_cpu.rs b/crates/j2k-jpeg/benches/decode_cpu.rs new file mode 100644 index 00000000..fe996003 --- /dev/null +++ b/crates/j2k-jpeg/benches/decode_cpu.rs @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, Criterion, Throughput}; +use j2k_jpeg::{ + encode_jpeg_baseline, Decoder, JpegBackend, JpegEncodeOptions, JpegError, JpegSamples, + JpegSubsampling, PixelFormat, RowSink, +}; +use j2k_test_support::{patterned_gray8, patterned_rgb8}; + +const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +fn zeroed_bytes(len: usize) -> Vec { + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(len) + .expect("reserve deterministic benchmark output"); + bytes.resize(len, 0); + bytes +} + +#[derive(Clone, Copy)] +enum DecodeMode { + Buffer(PixelFormat), + Rows, +} + +struct DecodeCase { + name: &'static str, + width: u32, + height: u32, + bytes: Vec, + mode: DecodeMode, + expected_checksum: u64, +} + +impl DecodeCase { + fn new(name: &'static str, width: u32, height: u32, bytes: Vec, mode: DecodeMode) -> Self { + let decoder = Decoder::new(&bytes).expect("generated benchmark JPEG must parse"); + assert_eq!(decoder.info().dimensions, (width, height)); + let expected_checksum = decode_checksum(&decoder, mode); + assert_ne!(expected_checksum, FNV_OFFSET_BASIS); + Self { + name, + width, + height, + bytes, + mode, + expected_checksum, + } + } + + fn pixels(&self) -> u64 { + u64::from(self.width) * u64::from(self.height) + } +} + +fn fnv1a_update(mut hash: u64, bytes: &[u8]) -> u64 { + for byte in bytes { + hash = (hash ^ u64::from(*byte)).wrapping_mul(FNV_PRIME); + } + hash +} + +fn bytes_per_pixel(format: PixelFormat) -> usize { + match format { + PixelFormat::Gray8 => 1, + PixelFormat::Rgb8 => 3, + _ => panic!("decode benchmark only supports Gray8 and Rgb8"), + } +} + +fn decode_buffer_checksum(decoder: &Decoder<'_>, format: PixelFormat) -> u64 { + let (width, height) = decoder.info().dimensions; + let stride = width as usize * bytes_per_pixel(format); + let mut output = zeroed_bytes(stride * height as usize); + let outcome = decoder + .decode_into(&mut output, stride, format) + .expect("generated benchmark JPEG must decode"); + assert_eq!((outcome.decoded.w, outcome.decoded.h), (width, height)); + fnv1a_update(FNV_OFFSET_BASIS, &output) +} + +struct ChecksumSink { + hash: u64, + expected_y: u32, + row_bytes: usize, +} + +impl ChecksumSink { + fn new(row_bytes: usize) -> Self { + Self { + hash: FNV_OFFSET_BASIS, + expected_y: 0, + row_bytes, + } + } +} + +impl RowSink for ChecksumSink { + type Error = JpegError; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), Self::Error> { + assert_eq!(y, self.expected_y); + assert_eq!(row.len(), self.row_bytes); + self.hash = fnv1a_update(self.hash, row); + self.expected_y += 1; + Ok(()) + } +} + +fn decode_rows_checksum(decoder: &Decoder<'_>) -> u64 { + let (width, height) = decoder.info().dimensions; + let mut sink = ChecksumSink::new(width as usize * 3); + let outcome = decoder + .decode_rows(&mut sink) + .expect("generated benchmark JPEG row decode must succeed"); + assert_eq!((outcome.decoded.w, outcome.decoded.h), (width, height)); + assert_eq!(sink.expected_y, height); + sink.hash +} + +fn decode_checksum(decoder: &Decoder<'_>, mode: DecodeMode) -> u64 { + match mode { + DecodeMode::Buffer(format) => decode_buffer_checksum(decoder, format), + DecodeMode::Rows => decode_rows_checksum(decoder), + } +} + +fn encode_gray(width: u32, height: u32) -> Vec { + let pixels = patterned_gray8(width, height); + encode_jpeg_baseline( + JpegSamples::Gray8 { + data: &pixels, + width, + height, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Gray, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode deterministic grayscale benchmark JPEG") + .data +} + +fn encode_rgb(width: u32, height: u32, subsampling: JpegSubsampling) -> Vec { + let pixels = patterned_rgb8(width, height); + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &pixels, + width, + height, + }, + JpegEncodeOptions { + quality: 90, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode deterministic RGB benchmark JPEG") + .data +} + +fn decode_cases() -> Vec { + let rgb_420 = encode_rgb(512, 512, JpegSubsampling::Ybr420); + let mut cases = Vec::new(); + cases + .try_reserve_exact(6) + .expect("reserve deterministic benchmark cases"); + cases.push(DecodeCase::new( + "gray8_512", + 512, + 512, + encode_gray(512, 512), + DecodeMode::Buffer(PixelFormat::Gray8), + )); + cases.push(DecodeCase::new( + "rgb8_512_444", + 512, + 512, + encode_rgb(512, 512, JpegSubsampling::Ybr444), + DecodeMode::Buffer(PixelFormat::Rgb8), + )); + cases.push(DecodeCase::new( + "rgb8_512_422", + 512, + 512, + encode_rgb(512, 512, JpegSubsampling::Ybr422), + DecodeMode::Buffer(PixelFormat::Rgb8), + )); + cases.push(DecodeCase::new( + "rgb8_512_420", + 512, + 512, + rgb_420.clone(), + DecodeMode::Buffer(PixelFormat::Rgb8), + )); + cases.push(DecodeCase::new( + "rgb8_257x263_420", + 257, + 263, + encode_rgb(257, 263, JpegSubsampling::Ybr420), + DecodeMode::Buffer(PixelFormat::Rgb8), + )); + cases.push(DecodeCase::new( + "rgb8_512_420_rows", + 512, + 512, + rgb_420, + DecodeMode::Rows, + )); + cases +} + +fn bench_decode_cpu(c: &mut Criterion) { + let cases = decode_cases(); + let mut group = c.benchmark_group("jpeg_cpu_decode_runtime"); + for case in &cases { + let decoder = Decoder::new(&case.bytes).expect("benchmark JPEG must parse"); + let expected_checksum = case.expected_checksum; + group.throughput(Throughput::Elements(case.pixels())); + match case.mode { + DecodeMode::Buffer(format) => { + let stride = case.width as usize * bytes_per_pixel(format); + let mut output = zeroed_bytes(stride * case.height as usize); + group.bench_function(case.name, |b| { + b.iter(|| { + let outcome = decoder + .decode_into(&mut output, stride, format) + .expect("benchmark decode must succeed"); + std::hint::black_box(outcome); + let checksum = fnv1a_update(FNV_OFFSET_BASIS, &output); + debug_assert_eq!(checksum, expected_checksum); + std::hint::black_box(checksum); + }); + }); + } + DecodeMode::Rows => { + group.bench_function(case.name, |b| { + b.iter(|| { + let mut sink = ChecksumSink::new(case.width as usize * 3); + let outcome = decoder + .decode_rows(&mut sink) + .expect("benchmark row decode must succeed"); + debug_assert_eq!(sink.hash, expected_checksum); + std::hint::black_box((outcome, sink.hash)); + }); + }); + } + } + } + group.finish(); +} + +criterion_group! { + name = decode_cpu_benches; + config = Criterion::default() + .confidence_level(0.95) + .sample_size(50) + .warm_up_time(Duration::from_secs(3)) + .measurement_time(Duration::from_secs(10)); + targets = bench_decode_cpu +} +criterion_main!(decode_cpu_benches); diff --git a/crates/j2k-jpeg/benches/micro.rs b/crates/j2k-jpeg/benches/micro.rs index 6a2cba21..4ea8e8b7 100644 --- a/crates/j2k-jpeg/benches/micro.rs +++ b/crates/j2k-jpeg/benches/micro.rs @@ -1,9 +1,11 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 +use std::time::Duration; + use criterion::{criterion_group, criterion_main, Criterion}; use j2k_jpeg::bench_support::{ - bench_idct_reference_block, BenchColorRowScratch, BenchHuffmanState, BenchRgb420RowPairScratch, - BenchUpsampleH2V2Scratch, + bench_idct_reference_block, BenchColorRowScratch, BenchGrayRowScratch, BenchHuffmanState, + BenchRgb420RowPairScratch, BenchRgbRowScratch, BenchUpsampleH2V2Scratch, }; use j2k_jpeg::Decoder; use j2k_test_support::JPEG_BASELINE_420_16X16; @@ -103,11 +105,12 @@ fn bench_micro(c: &mut Criterion) { #[cfg(target_arch = "aarch64")] { - use j2k_jpeg::bench_support::bench_idct_neon_block; + use j2k_jpeg::bench_support::BenchNeonIdct; + let neon = BenchNeonIdct::new(); c.bench_function("micro/idct_islow_neon_block", |b| { let mut out = [0u8; 64]; b.iter(|| { - bench_idct_neon_block(std::hint::black_box(&coeffs), &mut out); + neon.run(std::hint::black_box(&coeffs), &mut out); std::hint::black_box(&out); }); }); @@ -115,7 +118,7 @@ fn bench_micro(c: &mut Criterion) { c.bench_function("micro/idct_islow_neon_bottom_half_zero_block", |b| { let mut out = [0u8; 64]; b.iter(|| { - bench_idct_neon_block(std::hint::black_box(&bottom_half_zero), &mut out); + neon.run_bottom_half_zero(std::hint::black_box(&bottom_half_zero), &mut out); std::hint::black_box(&out); }); }); @@ -123,12 +126,12 @@ fn bench_micro(c: &mut Criterion) { #[cfg(target_arch = "x86_64")] { - if std::is_x86_feature_detected!("avx2") { - use j2k_jpeg::bench_support::bench_idct_avx2_block; + use j2k_jpeg::bench_support::BenchAvx2Idct; + if let Some(avx2) = BenchAvx2Idct::try_new() { c.bench_function("micro/idct_islow_avx2_block", |b| { let mut out = [0u8; 64]; b.iter(|| { - bench_idct_avx2_block(std::hint::black_box(&coeffs), &mut out); + avx2.run(std::hint::black_box(&coeffs), &mut out); std::hint::black_box(&out); }); }); @@ -148,6 +151,7 @@ fn bench_micro(c: &mut Criterion) { // Odd-width 4:2:0 row-pair work item that forces the narrow chroma tail // handling exercised by the NEON hot-path parity test. let mut row_pair = BenchRgb420RowPairScratch::new(255); + assert!(row_pair.backend_matches_reference()); c.bench_function("micro/rgb_420_row_pair_255", |b| { b.iter(|| { row_pair.run(); @@ -156,6 +160,7 @@ fn bench_micro(c: &mut Criterion) { }); let mut row_pair_even = BenchRgb420RowPairScratch::new(256); + assert!(row_pair_even.backend_matches_reference()); c.bench_function("micro/rgb_420_row_pair_256", |b| { b.iter(|| { row_pair_even.run(); @@ -163,6 +168,33 @@ fn bench_micro(c: &mut Criterion) { }); }); + let mut row_pair_cropped = BenchRgb420RowPairScratch::new(257); + assert!(row_pair_cropped.cropped_backend_matches_reference(3, 249)); + c.bench_function("micro/rgb_420_row_pair_cropped_3_249", |b| { + b.iter(|| { + row_pair_cropped.run_cropped(3, 249); + std::hint::black_box(&row_pair_cropped); + }); + }); + + let mut gray = BenchGrayRowScratch::new(256); + assert!(gray.backend_matches_scalar()); + c.bench_function("micro/gray_to_rgb_row_backend_256", |b| { + b.iter(|| { + gray.run_backend(); + std::hint::black_box(&gray); + }); + }); + + let mut rgb = BenchRgbRowScratch::new(256); + assert!(rgb.backend_matches_scalar()); + c.bench_function("micro/planar_rgb_to_rgb_row_backend_256", |b| { + b.iter(|| { + rgb.run_backend(); + std::hint::black_box(&rgb); + }); + }); + // Scalar YCbCr→RGB conversion across a 256-pixel row — the path every // Phase 2 SIMD variant has to beat. let mut color = BenchColorRowScratch::new(256); @@ -174,6 +206,7 @@ fn bench_micro(c: &mut Criterion) { }); let mut backend_color = BenchColorRowScratch::new(256); + assert!(backend_color.backend_matches_scalar()); c.bench_function("micro/ycbcr_to_rgb_row_backend_256", |b| { b.iter(|| { backend_color.run_backend(); @@ -182,13 +215,31 @@ fn bench_micro(c: &mut Criterion) { }); let mut backend_color_tail = BenchColorRowScratch::new(255); + assert!(backend_color_tail.backend_matches_scalar()); c.bench_function("micro/ycbcr_to_rgb_row_backend_255", |b| { b.iter(|| { backend_color_tail.run_backend(); std::hint::black_box(&backend_color_tail); }); }); + + let mut backend_color_unaligned = BenchColorRowScratch::new_unaligned(256); + assert!(backend_color_unaligned.backend_matches_scalar()); + c.bench_function("micro/ycbcr_to_rgb_row_backend_unaligned_256", |b| { + b.iter(|| { + backend_color_unaligned.run_backend(); + std::hint::black_box(&backend_color_unaligned); + }); + }); } -criterion_group!(micro_benches, bench_micro); +criterion_group! { + name = micro_benches; + config = Criterion::default() + .confidence_level(0.95) + .sample_size(50) + .warm_up_time(Duration::from_secs(3)) + .measurement_time(Duration::from_secs(10)); + targets = bench_micro +} criterion_main!(micro_benches); diff --git a/crates/j2k-jpeg/fuzz/Cargo.lock b/crates/j2k-jpeg/fuzz/Cargo.lock index d516b86b..f412447d 100644 --- a/crates/j2k-jpeg/fuzz/Cargo.lock +++ b/crates/j2k-jpeg/fuzz/Cargo.lock @@ -57,6 +57,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "fearless_simd" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -77,19 +83,20 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-core" -version = "0.8.1" +version = "0.9.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-jpeg" -version = "0.8.1" +version = "0.9.1" dependencies = [ + "fearless_simd", "j2k-codec-math", "j2k-core", "j2k-profile", @@ -108,7 +115,7 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.1" +version = "0.9.1" [[package]] name = "jobserver" diff --git a/crates/j2k-jpeg/src/backend/mod.rs b/crates/j2k-jpeg/src/backend/mod.rs index f1476e08..350a1ca2 100644 --- a/crates/j2k-jpeg/src/backend/mod.rs +++ b/crates/j2k-jpeg/src/backend/mod.rs @@ -4,7 +4,8 @@ //! inverse DCT. use crate::idct; -use j2k_core::CpuFeatures; +#[cfg(target_arch = "x86_64")] +use crate::simd::x86::ExactAvx2; pub(crate) mod scalar; @@ -20,6 +21,7 @@ mod row_pair; #[cfg(test)] mod tests; +#[cfg(test)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum BackendKind { Scalar, @@ -30,8 +32,12 @@ pub(crate) enum BackendKind { } #[derive(Debug, Clone, Copy)] -pub(crate) struct Backend { - kind: BackendKind, +pub(crate) enum Backend { + Scalar, + #[cfg(target_arch = "x86_64")] + Avx2(ExactAvx2), + #[cfg(target_arch = "aarch64")] + Neon(fearless_simd::Neon), } #[derive(Clone, Copy)] @@ -125,41 +131,65 @@ impl<'a> Rgb420CroppedRowPair<'a> { impl Backend { pub(crate) fn detect() -> Self { - let cpu = CpuFeatures::detect(); + if cfg!(feature = "scalar-only") { + return Self::Scalar; + } #[cfg(target_arch = "x86_64")] { - if !cfg!(feature = "scalar-only") && cpu.avx2 { - return Self { - kind: BackendKind::Avx2, - }; + if let Some(avx2) = ExactAvx2::detect() { + return Self::Avx2(avx2); } } #[cfg(target_arch = "aarch64")] { - if !cfg!(feature = "scalar-only") && cpu.neon { - return Self { - kind: BackendKind::Neon, - }; + if let Some(neon) = fearless_simd::Level::new().as_neon() { + return Self::Neon(neon); } } - Self { - kind: BackendKind::Scalar, + Self::Scalar + } + + #[cfg(test)] + pub(super) fn for_test_kind(kind: BackendKind) -> Option { + match kind { + BackendKind::Scalar => Some(Self::Scalar), + #[cfg(target_arch = "x86_64")] + BackendKind::Avx2 => ExactAvx2::detect().map(Self::Avx2), + #[cfg(target_arch = "aarch64")] + BackendKind::Neon => fearless_simd::Level::new().as_neon().map(Self::Neon), + } + } + + #[cfg(test)] + pub(super) const fn kind(self) -> BackendKind { + match self { + Self::Scalar => BackendKind::Scalar, + #[cfg(target_arch = "x86_64")] + Self::Avx2(_) => BackendKind::Avx2, + #[cfg(target_arch = "aarch64")] + Self::Neon(_) => BackendKind::Neon, } } + #[expect( + clippy::inline_always, + reason = "same-host SIMD benchmarks confirmed this dispatch must inline outside the row loop" + )] + #[inline(always)] pub(crate) fn fill_rgb_row_from_gray(self, gray_row: &[u8], dst: &mut [u8]) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgb_row_from_gray(gray_row, dst), + match self { + Self::Scalar => scalar::fill_rgb_row_from_gray(gray_row, dst), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => x86::fill_rgb_row_from_gray(gray_row, dst), + Self::Avx2(_) => x86::fill_rgb_row_from_gray(gray_row, dst), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => neon::fill_rgb_row_from_gray(gray_row, dst), + Self::Neon(neon) => neon::fill_rgb_row_from_gray(neon, gray_row, dst), } } + #[inline] pub(crate) fn fill_rgb_row_from_rgb( self, r_row: &[u8], @@ -167,12 +197,12 @@ impl Backend { b_row: &[u8], dst: &mut [u8], ) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgb_row_from_rgb(r_row, g_row, b_row, dst), + match self { + Self::Scalar => scalar::fill_rgb_row_from_rgb(r_row, g_row, b_row, dst), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => x86::fill_rgb_row_from_rgb(r_row, g_row, b_row, dst), + Self::Avx2(_) => x86::fill_rgb_row_from_rgb(r_row, g_row, b_row, dst), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => neon::fill_rgb_row_from_rgb(r_row, g_row, b_row, dst), + Self::Neon(neon) => neon::fill_rgb_row_from_rgb(neon, r_row, g_row, b_row, dst), } } @@ -183,22 +213,22 @@ impl Backend { cr_row: &[u8], dst: &mut [u8], ) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst), + match self { + Self::Scalar => scalar::fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => x86::fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst), + Self::Avx2(avx2) => x86::fill_rgb_row_from_ycbcr(avx2, y_row, cb_row, cr_row, dst), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => neon::fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst), + Self::Neon(neon) => neon::fill_rgb_row_from_ycbcr(neon, y_row, cb_row, cr_row, dst), } } pub(crate) fn fill_rgba_row_from_gray(self, gray_row: &[u8], dst: &mut [u8], alpha: u8) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), + match self { + Self::Scalar => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), + Self::Avx2(_) => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), + Self::Neon(_) => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), } } @@ -210,12 +240,12 @@ impl Backend { dst: &mut [u8], alpha: u8, ) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), + match self { + Self::Scalar => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), + Self::Avx2(_) => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), + Self::Neon(_) => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), } } @@ -227,38 +257,40 @@ impl Backend { dst: &mut [u8], alpha: u8, ) { - match self.kind { - BackendKind::Scalar => { + match self { + Self::Scalar => { scalar::fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, alpha); } #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => { + Self::Avx2(_) => { scalar::fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, alpha); } #[cfg(target_arch = "aarch64")] - BackendKind::Neon => { + Self::Neon(_) => { scalar::fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, alpha); } } } + #[inline] pub(crate) fn fill_rgb_row_pair_from_420(self, request: Rgb420RowPair<'_>) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgb_row_pair_from_420(request), + match self { + Self::Scalar => scalar::fill_rgb_row_pair_from_420(request), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => x86::fill_rgb_row_pair_from_420(request), + Self::Avx2(avx2) => x86::fill_rgb_row_pair_from_420(avx2, request), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => neon::fill_rgb_row_pair_from_420(request), + Self::Neon(neon) => neon::fill_rgb_row_pair_from_420(neon, request), } } + #[inline] pub(crate) fn fill_rgb_row_pair_from_420_cropped(self, request: Rgb420CroppedRowPair<'_>) { - match self.kind { - BackendKind::Scalar => scalar::fill_rgb_row_pair_from_420_cropped(request), + match self { + Self::Scalar => scalar::fill_rgb_row_pair_from_420_cropped(request), #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => x86::fill_rgb_row_pair_from_420_cropped(request), + Self::Avx2(avx2) => x86::fill_rgb_row_pair_from_420_cropped(avx2, request), #[cfg(target_arch = "aarch64")] - BackendKind::Neon => neon::fill_rgb_row_pair_from_420_cropped(request), + Self::Neon(neon) => neon::fill_rgb_row_pair_from_420_cropped(neon, request), } } @@ -266,12 +298,12 @@ impl Backend { if crop_width == 0 || crop_width >= row_width { return false; } - match self.kind { - BackendKind::Scalar => true, + match self { + Self::Scalar => true, #[cfg(target_arch = "x86_64")] - BackendKind::Avx2 => true, + Self::Avx2(_) => true, #[cfg(target_arch = "aarch64")] - BackendKind::Neon => true, + Self::Neon(_) => true, } } @@ -279,26 +311,22 @@ impl Backend { /// level-shifted by +128 and clamped to `[0, 255]` — bit-exact with /// [`idct::scalar::idct_islow`] on every legal JPEG input. pub(crate) fn idct(self, input: &[i16; 64], output: &mut [u8; 64]) { - match self.kind { - BackendKind::Scalar => idct::scalar::idct_islow(input, output), + match self { + Self::Scalar => idct::scalar::idct_islow(input, output), #[cfg(target_arch = "x86_64")] - // SAFETY: Backend selection guarantees the SIMD target feature for this call. - BackendKind::Avx2 => unsafe { idct::avx2::idct_islow(input, output) }, + Self::Avx2(avx2) => idct::avx2::idct_islow(avx2, input, output), #[cfg(target_arch = "aarch64")] - // SAFETY: Backend selection guarantees the SIMD target feature for this call. - BackendKind::Neon => unsafe { idct::neon::idct_islow(input, output) }, + Self::Neon(neon) => idct::neon::idct_islow(neon, input, output), } } pub(crate) fn idct_bottom_half_zero(self, input: &[i16; 64], output: &mut [u8; 64]) { - match self.kind { - BackendKind::Scalar => idct::scalar::idct_islow_bottom_half_zero(input, output), + match self { + Self::Scalar => idct::scalar::idct_islow_bottom_half_zero(input, output), #[cfg(target_arch = "x86_64")] - // SAFETY: Backend selection guarantees the SIMD target feature for this call. - BackendKind::Avx2 => unsafe { idct::avx2::idct_islow(input, output) }, + Self::Avx2(avx2) => idct::avx2::idct_islow(avx2, input, output), #[cfg(target_arch = "aarch64")] - // SAFETY: Backend selection guarantees the SIMD target feature for this call. - BackendKind::Neon => unsafe { idct::neon::idct_islow_bottom_half_zero(input, output) }, + Self::Neon(neon) => idct::neon::idct_islow_bottom_half_zero(neon, input, output), } } } diff --git a/crates/j2k-jpeg/src/backend/neon.rs b/crates/j2k-jpeg/src/backend/neon.rs index c5e40472..f1d52c88 100644 --- a/crates/j2k-jpeg/src/backend/neon.rs +++ b/crates/j2k-jpeg/src/backend/neon.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use core::arch::aarch64::{ - int32x4_t, uint16x8_t, uint8x16_t, uint8x8_t, uint8x8x3_t, vaddq_s32, vaddq_u16, vcombine_u16, - vcombine_u8, vdupq_n_s32, vdupq_n_u16, vget_high_u16, vget_high_u8, vget_low_u16, vget_low_u8, - vld1_u8, vmovl_u16, vmovl_u8, vmulq_n_s32, vqmovn_u16, vqmovun_s32, vreinterpretq_s32_u32, - vshrq_n_s32, vshrq_n_u16, vst1q_u8, vst3_u8, vsubq_s32, vzip_u8, vzipq_u16, + int32x4_t, uint16x8_t, uint8x16_t, uint8x8_t, vaddq_s32, vaddq_u16, vcombine_u16, vcombine_u8, + vdupq_n_s32, vdupq_n_u16, vget_high_u16, vget_low_u16, vmovl_u16, vmovl_u8, vmulq_n_s32, + vqmovn_u16, vqmovun_s32, vreinterpretq_s32_u32, vshrq_n_s32, vshrq_n_u16, vsubq_s32, vzip_u8, + vzipq_u16, }; use super::row_pair::{normalize_simd_row_pair, normalize_ycbcr_row}; @@ -13,39 +13,42 @@ use crate::color::upsample::h2v2_fancy_sample_for_width; use crate::color::ycbcr::{ ycbcr_to_rgb, FIX_0_34414, FIX_0_71414, FIX_1_40200, FIX_1_77200, ROUND, }; +use crate::simd::neon_memory; -pub(crate) fn fill_rgb_row_from_gray(gray_row: &[u8], dst: &mut [u8]) { +pub(crate) fn fill_rgb_row_from_gray(neon: fearless_simd::Neon, gray_row: &[u8], dst: &mut [u8]) { let width = gray_row.len().min(dst.len() / 3); let gray_row = &gray_row[..width]; let dst = &mut dst[..width * 3]; debug_assert_eq!(dst.len(), gray_row.len() * 3); - // SAFETY: NEON is mandatory on supported aarch64 targets for this backend, - // and the wrapper narrows source and destination slices to one pixel count. - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + fill_rgb_row_from_gray_kernel(neon, gray_row, dst); +} + +fearless_simd::kernel! { + fn fill_rgb_row_from_gray_kernel(neon: Neon, gray_row: &[u8], dst: &mut [u8]) { fill_rgb_row_from_gray_neon(gray_row, dst); } } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_from_gray_neon(gray_row: &[u8], dst: &mut [u8]) { - let width = gray_row.len(); - let mut offset = 0; - while offset + LANES <= width { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let g = unsafe { vld1_u8(gray_row.as_ptr().add(offset)) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - vst3_u8(dst.as_mut_ptr().add(offset * 3), uint8x8x3_t(g, g, g)); - } - offset += LANES; +fn fill_rgb_row_from_gray_neon(gray_row: &[u8], dst: &mut [u8]) { + let (gray_chunks, gray_tail) = gray_row.as_chunks::(); + let (rgb_chunks, rgb_tail) = dst.as_chunks_mut::<{ LANES * 3 }>(); + for (gray, rgb) in gray_chunks.iter().zip(rgb_chunks) { + let g = neon_memory::load_u8x8(gray); + neon_memory::store_rgb8(rgb, g, g, g); } - if offset < width { - scalar::fill_rgb_row_from_gray(&gray_row[offset..], &mut dst[offset * 3..]); + if !gray_tail.is_empty() { + scalar::fill_rgb_row_from_gray(gray_tail, rgb_tail); } } -pub(crate) fn fill_rgb_row_from_rgb(r_row: &[u8], g_row: &[u8], b_row: &[u8], dst: &mut [u8]) { +pub(crate) fn fill_rgb_row_from_rgb( + neon: fearless_simd::Neon, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + dst: &mut [u8], +) { let width = r_row .len() .min(g_row.len()) @@ -58,38 +61,35 @@ pub(crate) fn fill_rgb_row_from_rgb(r_row: &[u8], g_row: &[u8], b_row: &[u8], ds debug_assert_eq!(r_row.len(), g_row.len()); debug_assert_eq!(r_row.len(), b_row.len()); debug_assert_eq!(dst.len(), r_row.len() * 3); - // SAFETY: NEON is mandatory on supported aarch64 targets for this backend, - // and all source rows plus the destination share the same bounded width. - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + fill_rgb_row_from_rgb_kernel(neon, r_row, g_row, b_row, dst); +} + +fearless_simd::kernel! { + fn fill_rgb_row_from_rgb_kernel( + neon: Neon, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + dst: &mut [u8], + ) { fill_rgb_row_from_rgb_neon(r_row, g_row, b_row, dst); } } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_from_rgb_neon(r_row: &[u8], g_row: &[u8], b_row: &[u8], dst: &mut [u8]) { - let width = r_row.len(); - let mut offset = 0; - while offset + LANES <= width { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let r = unsafe { vld1_u8(r_row.as_ptr().add(offset)) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let g = unsafe { vld1_u8(g_row.as_ptr().add(offset)) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let b = unsafe { vld1_u8(b_row.as_ptr().add(offset)) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - vst3_u8(dst.as_mut_ptr().add(offset * 3), uint8x8x3_t(r, g, b)); - } - offset += LANES; +fn fill_rgb_row_from_rgb_neon(r_row: &[u8], g_row: &[u8], b_row: &[u8], dst: &mut [u8]) { + let (r_chunks, r_tail) = r_row.as_chunks::(); + let (g_chunks, g_tail) = g_row.as_chunks::(); + let (b_chunks, b_tail) = b_row.as_chunks::(); + let (rgb_chunks, rgb_tail) = dst.as_chunks_mut::<{ LANES * 3 }>(); + for (((r, g), b), rgb) in r_chunks.iter().zip(g_chunks).zip(b_chunks).zip(rgb_chunks) { + let r = neon_memory::load_u8x8(r); + let g = neon_memory::load_u8x8(g); + let b = neon_memory::load_u8x8(b); + neon_memory::store_rgb8(rgb, r, g, b); } - if offset < width { - scalar::fill_rgb_row_from_rgb( - &r_row[offset..], - &g_row[offset..], - &b_row[offset..], - &mut dst[offset * 3..], - ); + if !r_tail.is_empty() { + scalar::fill_rgb_row_from_rgb(r_tail, g_tail, b_tail, rgb_tail); } } @@ -122,15 +122,28 @@ fn top_only_chroma(chroma: Rgb420ChromaRows<'_>) -> Rgb420ChromaRows<'_> { ) } -pub(crate) fn fill_rgb_row_from_ycbcr(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { +pub(crate) fn fill_rgb_row_from_ycbcr( + neon: fearless_simd::Neon, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + dst: &mut [u8], +) { let (y_row, cb_row, cr_row, dst) = normalize_ycbcr_row(y_row, cb_row, cr_row, dst); debug_assert_eq!(y_row.len(), cb_row.len()); debug_assert_eq!(y_row.len(), cr_row.len()); debug_assert_eq!(dst.len(), y_row.len() * 3); - // SAFETY: NEON is mandatory on supported aarch64 targets for this backend, - // and all source rows plus the destination share the same bounded width. - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + fill_rgb_row_from_ycbcr_kernel(neon, y_row, cb_row, cr_row, dst); +} + +fearless_simd::kernel! { + fn fill_rgb_row_from_ycbcr_kernel( + neon: Neon, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + dst: &mut [u8], + ) { fill_rgb_row_from_ycbcr_neon(y_row, cb_row, cr_row, dst); } } @@ -142,23 +155,29 @@ pub(super) fn fill_rgb_row_from_ycbcr_for_test( cr_row: &[u8], dst: &mut [u8], ) { - fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst); + let neon = fearless_simd::Level::new() + .as_neon() + .expect("AArch64 test host must provide NEON"); + fill_rgb_row_from_ycbcr(neon, y_row, cb_row, cr_row, dst); } -pub(crate) fn fill_rgb_row_pair_from_420(request: Rgb420RowPair<'_>) { +pub(crate) fn fill_rgb_row_pair_from_420(neon: fearless_simd::Neon, request: Rgb420RowPair<'_>) { let Some(request) = normalize_simd_row_pair(request) else { return; }; - // SAFETY: NEON is mandatory on supported aarch64 targets for this backend. - // The wrapper clamps luma, chroma, and destination slices so upsampled reads - // and RGB writes stay within the passed rows. - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + fill_rgb_row_pair_from_420_kernel(neon, request); +} + +fearless_simd::kernel! { + fn fill_rgb_row_pair_from_420_kernel(neon: Neon, request: Rgb420RowPair<'_>) { fill_rgb_row_pair_from_420_neon(request); } } -pub(crate) fn fill_rgb_row_pair_from_420_cropped(request: Rgb420CroppedRowPair<'_>) { +pub(crate) fn fill_rgb_row_pair_from_420_cropped( + neon: fearless_simd::Neon, + request: Rgb420CroppedRowPair<'_>, +) { let Rgb420CroppedRowPair { rows, crop } = request; let Rgb420RowPair { y_top, @@ -207,11 +226,9 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped(request: Rgb420CroppedRowPair<' debug_assert_eq!(prev_cb.len(), next_cb.len()); debug_assert_eq!(prev_cr.len(), curr_cr.len()); debug_assert_eq!(prev_cr.len(), next_cr.len()); - // SAFETY: NEON is mandatory on supported aarch64 targets for this backend. - // The crop range and output rows are clamped to validated luma/chroma spans. - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - fill_rgb_row_pair_from_420_cropped_neon(Rgb420CroppedRowPair::new( + fill_rgb_row_pair_from_420_cropped_kernel( + neon, + Rgb420CroppedRowPair::new( Rgb420RowPair::new( y_top, y_bottom, @@ -220,12 +237,21 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped(request: Rgb420CroppedRowPair<' dst_bottom, ), Rgb420Crop::new(crop_start, crop_width), - )); + ), + ); +} + +fearless_simd::kernel! { + fn fill_rgb_row_pair_from_420_cropped_kernel( + neon: Neon, + request: Rgb420CroppedRowPair<'_>, + ) { + fill_rgb_row_pair_from_420_cropped_neon(request); } } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_neon(request: Rgb420RowPair<'_>) { +fn fill_rgb_row_pair_from_420_neon(request: Rgb420RowPair<'_>) { let Rgb420RowPair { y_top, y_bottom, @@ -234,20 +260,18 @@ unsafe fn fill_rgb_row_pair_from_420_neon(request: Rgb420RowPair<'_>) { dst_bottom, } = request; if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_neon_dual(y_top, y_bottom, chroma, dst_top, dst_bottom); } } else { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_neon_top_only(y_top, chroma, dst_top); } } } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_cropped_neon(request: Rgb420CroppedRowPair<'_>) { +fn fill_rgb_row_pair_from_420_cropped_neon(request: Rgb420CroppedRowPair<'_>) { let Rgb420CroppedRowPair { rows, crop } = request; let Rgb420RowPair { y_top, @@ -257,15 +281,13 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon(request: Rgb420CroppedRowPair< dst_bottom, } = rows; if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_cropped_neon_dual( y_top, y_bottom, chroma, crop, dst_top, dst_bottom, ); } } else { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_cropped_neon_top_only(y_top, chroma, crop, dst_top); } } @@ -276,7 +298,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon(request: Rgb420CroppedRowPair< clippy::too_many_lines, reason = "the SIMD kernel mirrors one scalar 4:2:0 row-pair operation with ordered lane and edge repair" )] -unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( +fn fill_rgb_row_pair_from_420_cropped_neon_dual( y_top: &[u8], y_bottom: &[u8], chroma: Rgb420ChromaRows<'_>, @@ -318,8 +340,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( if copy_width >= LANES && can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), aligned_x) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( y_top, y_bottom, @@ -355,8 +376,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( break; } - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], &y_bottom[x..x + UPSAMPLED_LANES], @@ -373,8 +393,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( if remaining >= LANES { let x = crop_start + out_x; if can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), x) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( y_top, y_bottom, @@ -407,7 +426,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( +fn fill_rgb_row_pair_from_420_cropped_neon_top_only( y_top: &[u8], chroma: Rgb420ChromaRows<'_>, crop: Rgb420Crop, @@ -441,8 +460,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( if copy_width >= LANES && can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), aligned_x) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( y_top, chroma, @@ -470,8 +488,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( break; } - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], prev_cb, @@ -489,8 +506,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( if remaining >= LANES { let x = crop_start + out_x; if can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), x) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( y_top, chroma, @@ -521,7 +537,7 @@ fn can_vectorize_cropped_420_chunk(row_width: usize, chroma_width: usize, x: usi } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( +fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( y_top: &[u8], y_bottom: &[u8], chroma: Rgb420ChromaRows<'_>, @@ -538,8 +554,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( debug_assert!(copy_width <= UPSAMPLED_LANES); let mut tmp_top = [0u8; UPSAMPLED_LANES * 3]; let mut tmp_bottom = [0u8; UPSAMPLED_LANES * 3]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_chunk16_interior_neon( &y_top[aligned_x..aligned_x + UPSAMPLED_LANES], &y_bottom[aligned_x..aligned_x + UPSAMPLED_LANES], @@ -556,7 +571,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( +fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( y_top: &[u8], chroma: Rgb420ChromaRows<'_>, chunk: Neon420PartialChunk, @@ -570,8 +585,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( debug_assert!(src_skip + copy_width <= UPSAMPLED_LANES); debug_assert!(copy_width <= UPSAMPLED_LANES); let mut tmp_top = [0u8; UPSAMPLED_LANES * 3]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_from_420_chunk16_interior_neon( &y_top[aligned_x..aligned_x + UPSAMPLED_LANES], chroma.prev_cb, @@ -592,7 +606,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( clippy::too_many_lines, reason = "the SIMD kernel mirrors one scalar 4:2:0 row-pair operation with ordered lane and edge repair" )] -unsafe fn fill_rgb_row_pair_from_420_neon_dual( +fn fill_rgb_row_pair_from_420_neon_dual( y_top: &[u8], y_bottom: &[u8], chroma: Rgb420ChromaRows<'_>, @@ -620,8 +634,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( let chunk_width = (width - x).min(chunk_samples * 2); if can_vectorize_420_chunk(chroma_width, sample, chunk_width) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], &y_bottom[x..x + UPSAMPLED_LANES], @@ -636,8 +649,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( } if sample == 0 { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_edge_neon_dual( y_top, y_bottom, @@ -649,8 +661,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( } } else if can_use_tail_420_chunk(chroma_width, sample, chunk_width) { record_420_dispatch_neon_tail_chunk(); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_tail_neon_dual( y_top, y_bottom, @@ -672,8 +683,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( let mut cr_top = [0u8; UPSAMPLED_LANES]; let mut cr_bot = [0u8; UPSAMPLED_LANES]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_upsampled_420_chunk( prev_cb, curr_cb, @@ -722,7 +732,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_neon_top_only( +fn fill_rgb_row_pair_from_420_neon_top_only( y_top: &[u8], chroma: Rgb420ChromaRows<'_>, dst_top: &mut [u8], @@ -747,8 +757,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( let chunk_width = (width - x).min(chunk_samples * 2); if can_vectorize_420_chunk(chroma_width, sample, chunk_width) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], prev_cb, @@ -764,14 +773,12 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( } if sample == 0 { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_edge_neon_top_only(y_top, chroma, chunk_width, dst_top); } } else if can_use_tail_420_chunk(chroma_width, sample, chunk_width) { record_420_dispatch_neon_tail_chunk(); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_rgb_row_pair_from_420_tail_neon_top_only( y_top, chroma, @@ -788,8 +795,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( record_420_dispatch_scalar_chunk(); let mut cb_top = [0u8; UPSAMPLED_LANES]; let mut cr_top = [0u8; UPSAMPLED_LANES]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_upsampled_420_chunk( prev_cb, curr_cb, @@ -822,7 +828,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( clippy::cast_possible_truncation, reason = "NEON weighted chroma sums are shifted into the u8 sample range before lane-edge repair" )] -unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( +fn fill_rgb_row_pair_from_420_edge_neon_dual( y_top: &[u8], y_bottom: &[u8], chroma: Rgb420ChromaRows<'_>, @@ -848,20 +854,14 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( let next_cr_head = load_head_window(next_cr, TAIL_WINDOW); let (cb_top, cb_bottom) = - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { upsampled_420_chunk16_pair_u16(&prev_cb_head, &curr_cb_head, &next_cb_head, 1) }; + { upsampled_420_chunk16_pair_u16(&prev_cb_head, &curr_cb_head, &next_cb_head, 1) }; let (cr_top, cr_bottom) = - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { upsampled_420_chunk16_pair_u16(&prev_cr_head, &curr_cr_head, &next_cr_head, 1) }; - - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_top_lo = unsafe { load_eight(&y_top_tail, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_top_hi = unsafe { load_eight(&y_top_tail, LANES) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_bottom_lo = unsafe { load_eight(&y_bottom_tail, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_bottom_hi = unsafe { load_eight(&y_bottom_tail, LANES) }; + { upsampled_420_chunk16_pair_u16(&prev_cr_head, &curr_cr_head, &next_cr_head, 1) }; + + let y_top_lo = { load_eight(&y_top_tail, 0) }; + let y_top_hi = { load_eight(&y_top_tail, LANES) }; + let y_bottom_lo = { load_eight(&y_bottom_tail, 0) }; + let y_bottom_hi = { load_eight(&y_bottom_tail, LANES) }; let top_cb = ((u32::from(prev_cb[0]) + 3 * u32::from(curr_cb[0])) * 4 + 8) >> 4; let top_cr = ((u32::from(prev_cr[0]) + 3 * u32::from(curr_cr[0])) * 4 + 8) >> 4; @@ -872,8 +872,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( ycbcr_to_rgb(y_bottom[0], bottom_cb as u8, bottom_cr as u8); if chunk_width == UPSAMPLED_LANES { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut dst_top[..LANES * 3]); fill_chunk_from_vectors_u16( y_top_hi, @@ -899,8 +898,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( } else { let mut rgb_top = [0u8; UPSAMPLED_LANES * 3]; let mut rgb_bottom = [0u8; UPSAMPLED_LANES * 3]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut rgb_top[..LANES * 3]); fill_chunk_from_vectors_u16(y_top_hi, cb_top.1, cr_top.1, &mut rgb_top[LANES * 3..]); fill_chunk_from_vectors_u16( @@ -928,7 +926,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( clippy::cast_possible_truncation, reason = "NEON weighted chroma sums are shifted into the u8 sample range before lane-edge repair" )] -unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( +fn fill_rgb_row_pair_from_420_edge_neon_top_only( y_top: &[u8], chroma: Rgb420ChromaRows<'_>, chunk_width: usize, @@ -947,22 +945,17 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( let prev_cr_head = load_head_window(prev_cr, TAIL_WINDOW); let curr_cr_head = load_head_window(curr_cr, TAIL_WINDOW); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cb = unsafe { upsampled_420_chunk16_u16(&prev_cb_head, &curr_cb_head, 1) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cr = unsafe { upsampled_420_chunk16_u16(&prev_cr_head, &curr_cr_head, 1) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_lo = unsafe { load_eight(&y_top_tail, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_hi = unsafe { load_eight(&y_top_tail, LANES) }; + let cb = { upsampled_420_chunk16_u16(&prev_cb_head, &curr_cb_head, 1) }; + let cr = { upsampled_420_chunk16_u16(&prev_cr_head, &curr_cr_head, 1) }; + let y_lo = { load_eight(&y_top_tail, 0) }; + let y_hi = { load_eight(&y_top_tail, LANES) }; let cb0 = ((u32::from(prev_cb[0]) + 3 * u32::from(curr_cb[0])) * 4 + 8) >> 4; let cr0 = ((u32::from(prev_cr[0]) + 3 * u32::from(curr_cr[0])) * 4 + 8) >> 4; let (r, g, b) = ycbcr_to_rgb(y_top[0], cb0 as u8, cr0 as u8); if chunk_width == UPSAMPLED_LANES { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut dst_top[..LANES * 3]); fill_chunk_from_vectors_u16( y_hi, @@ -974,8 +967,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( dst_top[..3].copy_from_slice(&[r, g, b]); } else { let mut rgb = [0u8; UPSAMPLED_LANES * 3]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut rgb[..LANES * 3]); fill_chunk_from_vectors_u16(y_hi, cb.1, cr.1, &mut rgb[LANES * 3..]); } @@ -993,7 +985,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( clippy::cast_possible_truncation, reason = "NEON tail chroma sums are shifted into the u8 sample range before scalar edge repair" )] -unsafe fn fill_rgb_row_pair_from_420_tail_neon_dual( +fn fill_rgb_row_pair_from_420_tail_neon_dual( y_top: &[u8], y_bottom: &[u8], chroma: Rgb420ChromaRows<'_>, @@ -1025,24 +1017,17 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_dual( let next_cr_tail = load_tail_window(next_cr, sample_offset - 1, TAIL_WINDOW); let (cb_top, cb_bottom) = - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { upsampled_420_chunk16_pair_u16(&prev_cb_tail, &curr_cb_tail, &next_cb_tail, 1) }; + { upsampled_420_chunk16_pair_u16(&prev_cb_tail, &curr_cb_tail, &next_cb_tail, 1) }; let (cr_top, cr_bottom) = - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { upsampled_420_chunk16_pair_u16(&prev_cr_tail, &curr_cr_tail, &next_cr_tail, 1) }; - - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_top_lo = unsafe { load_eight(&y_top_tail, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_top_hi = unsafe { load_eight(&y_top_tail, LANES) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_bottom_lo = unsafe { load_eight(&y_bottom_tail, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_bottom_hi = unsafe { load_eight(&y_bottom_tail, LANES) }; + { upsampled_420_chunk16_pair_u16(&prev_cr_tail, &curr_cr_tail, &next_cr_tail, 1) }; + + let y_top_lo = { load_eight(&y_top_tail, 0) }; + let y_top_hi = { load_eight(&y_top_tail, LANES) }; + let y_bottom_lo = { load_eight(&y_bottom_tail, 0) }; + let y_bottom_hi = { load_eight(&y_bottom_tail, LANES) }; if chunk_width == UPSAMPLED_LANES { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16( y_top_lo, cb_top.0, @@ -1089,8 +1074,7 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_dual( } else { let mut rgb_top = [0u8; UPSAMPLED_LANES * 3]; let mut rgb_bottom = [0u8; UPSAMPLED_LANES * 3]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut rgb_top[..LANES * 3]); fill_chunk_from_vectors_u16(y_top_hi, cb_top.1, cr_top.1, &mut rgb_top[LANES * 3..]); fill_chunk_from_vectors_u16( @@ -1136,7 +1120,7 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_dual( clippy::cast_possible_truncation, reason = "NEON tail chroma sums are shifted into the u8 sample range before scalar edge repair" )] -unsafe fn fill_rgb_row_pair_from_420_tail_neon_top_only( +fn fill_rgb_row_pair_from_420_tail_neon_top_only( y_top: &[u8], chroma: Rgb420ChromaRows<'_>, chunk: Neon420TailChunk, @@ -1161,18 +1145,13 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_top_only( let prev_cr_tail = load_tail_window(prev_cr, sample_offset - 1, TAIL_WINDOW); let curr_cr_tail = load_tail_window(curr_cr, sample_offset - 1, TAIL_WINDOW); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cb = unsafe { upsampled_420_chunk16_u16(&prev_cb_tail, &curr_cb_tail, 1) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cr = unsafe { upsampled_420_chunk16_u16(&prev_cr_tail, &curr_cr_tail, 1) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_lo = unsafe { load_eight(&y_top_tail, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_hi = unsafe { load_eight(&y_top_tail, LANES) }; + let cb = { upsampled_420_chunk16_u16(&prev_cb_tail, &curr_cb_tail, 1) }; + let cr = { upsampled_420_chunk16_u16(&prev_cr_tail, &curr_cr_tail, 1) }; + let y_lo = { load_eight(&y_top_tail, 0) }; + let y_hi = { load_eight(&y_top_tail, LANES) }; if chunk_width == UPSAMPLED_LANES { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut dst_top[x * 3..x * 3 + LANES * 3]); fill_chunk_from_vectors_u16( y_hi, @@ -1194,8 +1173,7 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_top_only( } } else { let mut rgb = [0u8; UPSAMPLED_LANES * 3]; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut rgb[..LANES * 3]); fill_chunk_from_vectors_u16(y_hi, cb.1, cr.1, &mut rgb[LANES * 3..]); } @@ -1278,7 +1256,7 @@ fn record_420_dispatch_scalar_chunk() { } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_from_420_chunk16_interior_neon( +fn fill_rgb_row_from_420_chunk16_interior_neon( y_row: &[u8], near_cb: &[u8], curr_cb: &[u8], @@ -1290,23 +1268,18 @@ unsafe fn fill_rgb_row_from_420_chunk16_interior_neon( debug_assert_eq!(y_row.len(), UPSAMPLED_LANES); debug_assert_eq!(dst.len(), UPSAMPLED_LANES * 3); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cb = unsafe { upsampled_420_chunk16_u16(near_cb, curr_cb, sample_offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cr = unsafe { upsampled_420_chunk16_u16(near_cr, curr_cr, sample_offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_lo = unsafe { load_eight(y_row, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_hi = unsafe { load_eight(y_row, LANES) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + let cb = { upsampled_420_chunk16_u16(near_cb, curr_cb, sample_offset) }; + let cr = { upsampled_420_chunk16_u16(near_cr, curr_cr, sample_offset) }; + let y_lo = { load_eight(y_row, 0) }; + let y_hi = { load_eight(y_row, LANES) }; + { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut dst[..LANES * 3]); fill_chunk_from_vectors_u16(y_hi, cb.1, cr.1, &mut dst[LANES * 3..]); } } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_pair_from_420_chunk16_interior_neon( +fn fill_rgb_row_pair_from_420_chunk16_interior_neon( y_top: &[u8], y_bottom: &[u8], chroma: Rgb420ChromaRows<'_>, @@ -1328,22 +1301,15 @@ unsafe fn fill_rgb_row_pair_from_420_chunk16_interior_neon( debug_assert_eq!(dst_bottom.len(), UPSAMPLED_LANES * 3); let (cb_top, cb_bottom) = - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { upsampled_420_chunk16_pair_u16(prev_cb, curr_cb, next_cb, sample_offset) }; + { upsampled_420_chunk16_pair_u16(prev_cb, curr_cb, next_cb, sample_offset) }; let (cr_top, cr_bottom) = - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { upsampled_420_chunk16_pair_u16(prev_cr, curr_cr, next_cr, sample_offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_top_lo = unsafe { load_eight(y_top, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_top_hi = unsafe { load_eight(y_top, LANES) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_bottom_lo = unsafe { load_eight(y_bottom, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_bottom_hi = unsafe { load_eight(y_bottom, LANES) }; - - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + { upsampled_420_chunk16_pair_u16(prev_cr, curr_cr, next_cr, sample_offset) }; + let y_top_lo = { load_eight(y_top, 0) }; + let y_top_hi = { load_eight(y_top, LANES) }; + let y_bottom_lo = { load_eight(y_bottom, 0) }; + let y_bottom_hi = { load_eight(y_bottom, LANES) }; + + { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut dst_top[..LANES * 3]); fill_chunk_from_vectors_u16(y_top_hi, cb_top.1, cr_top.1, &mut dst_top[LANES * 3..]); fill_chunk_from_vectors_u16( @@ -1362,95 +1328,67 @@ unsafe fn fill_rgb_row_pair_from_420_chunk16_interior_neon( } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_from_ycbcr_neon(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { - let width = y_row.len(); - let mut offset = 0; - - while offset + UPSAMPLED_LANES <= width { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - fill_rgb_row_from_ycbcr_chunk16_neon( - &y_row[offset..offset + UPSAMPLED_LANES], - vcombine_u8( - load_eight(cb_row, offset), - load_eight(cb_row, offset + LANES), - ), - vcombine_u8( - load_eight(cr_row, offset), - load_eight(cr_row, offset + LANES), - ), - &mut dst[offset * 3..(offset + UPSAMPLED_LANES) * 3], - ); - } - offset += UPSAMPLED_LANES; +fn fill_rgb_row_from_ycbcr_neon(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { + let (y_chunks, y_tail) = y_row.as_chunks::(); + let (cb_chunks, cb_tail) = cb_row.as_chunks::(); + let (cr_chunks, cr_tail) = cr_row.as_chunks::(); + let (dst_chunks, dst_tail) = dst.as_chunks_mut::<{ UPSAMPLED_LANES * 3 }>(); + + for (((y, cb), cr), dst) in y_chunks + .iter() + .zip(cb_chunks) + .zip(cr_chunks) + .zip(dst_chunks) + { + fill_rgb_row_from_ycbcr_chunk16_neon(y, cb, cr, dst); } - while offset + LANES <= width { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - fill_chunk( - y_row, - cb_row, - cr_row, - &mut dst[offset * 3..(offset + LANES) * 3], - offset, - ); - } - offset += LANES; + let (y_chunks, y_tail) = y_tail.as_chunks::(); + let (cb_chunks, cb_tail) = cb_tail.as_chunks::(); + let (cr_chunks, cr_tail) = cr_tail.as_chunks::(); + let (dst_chunks, dst_tail) = dst_tail.as_chunks_mut::<{ LANES * 3 }>(); + + for (((y, cb), cr), dst) in y_chunks + .iter() + .zip(cb_chunks) + .zip(cr_chunks) + .zip(dst_chunks) + { + fill_chunk(y, cb, cr, dst); } - if offset < width { - scalar::fill_rgb_row_from_ycbcr( - &y_row[offset..], - &cb_row[offset..], - &cr_row[offset..], - &mut dst[offset * 3..], - ); + if !y_tail.is_empty() { + scalar::fill_rgb_row_from_ycbcr(y_tail, cb_tail, cr_tail, dst_tail); } } #[target_feature(enable = "neon")] -unsafe fn fill_chunk( - y_row: &[u8], - cb_row: &[u8], - cr_row: &[u8], - dst_chunk: &mut [u8], - offset: usize, -) { - debug_assert_eq!(dst_chunk.len(), LANES * 3); - - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y = unsafe { load_eight(y_row, offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cb = unsafe { load_eight(cb_row, offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let cr = unsafe { load_eight(cr_row, offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { fill_chunk_from_vectors(y, cb, cr, dst_chunk) }; +#[expect( + clippy::trivially_copy_pass_by_ref, + reason = "array references preserve the zero-copy fixed-size SIMD memory boundary" +)] +fn fill_chunk(y: &[u8; LANES], cb: &[u8; LANES], cr: &[u8; LANES], dst: &mut [u8; LANES * 3]) { + fill_chunk_from_vectors( + neon_memory::load_u8x8(y), + neon_memory::load_u8x8(cb), + neon_memory::load_u8x8(cr), + dst, + ); } #[target_feature(enable = "neon")] -unsafe fn fill_chunk_from_vectors( - y: uint8x8_t, - cb: uint8x8_t, - cr: uint8x8_t, - dst_chunk: &mut [u8], -) { +fn fill_chunk_from_vectors(y: uint8x8_t, cb: uint8x8_t, cr: uint8x8_t, dst_chunk: &mut [u8]) { debug_assert_eq!(dst_chunk.len(), LANES * 3); let cb16 = vmovl_u8(cb); let cr16 = vmovl_u8(cr); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { fill_chunk_from_vectors_u16(y, cb16, cr16, dst_chunk) }; + fill_chunk_from_vectors_u16(y, cb16, cr16, dst_chunk); } #[target_feature(enable = "neon")] -unsafe fn fill_chunk_from_vectors_u16( - y: uint8x8_t, - cb: uint16x8_t, - cr: uint16x8_t, - dst_chunk: &mut [u8], -) { - debug_assert_eq!(dst_chunk.len(), LANES * 3); +fn fill_chunk_from_vectors_u16(y: uint8x8_t, cb: uint16x8_t, cr: uint16x8_t, dst_chunk: &mut [u8]) { + let Some(dst) = dst_chunk.first_chunk_mut::<{ LANES * 3 }>() else { + return; + }; let y16 = vmovl_u8(y); let y_lo = widen_low(y16); @@ -1467,53 +1405,33 @@ unsafe fn fill_chunk_from_vectors_u16( let g_bytes = pack_eight_u8(g_lo, g_hi); let b_bytes = pack_eight_u8(b_lo, b_hi); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - vst3_u8( - dst_chunk.as_mut_ptr(), - uint8x8x3_t(r_bytes, g_bytes, b_bytes), - ); - } + neon_memory::store_rgb8(dst, r_bytes, g_bytes, b_bytes); } #[target_feature(enable = "neon")] -unsafe fn fill_rgb_row_from_ycbcr_chunk16_neon( - y_row: &[u8], - cb: uint8x16_t, - cr: uint8x16_t, - dst: &mut [u8], +fn fill_rgb_row_from_ycbcr_chunk16_neon( + y: &[u8; UPSAMPLED_LANES], + cb: &[u8; UPSAMPLED_LANES], + cr: &[u8; UPSAMPLED_LANES], + dst: &mut [u8; UPSAMPLED_LANES * 3], ) { - debug_assert_eq!(y_row.len(), UPSAMPLED_LANES); - debug_assert_eq!(dst.len(), UPSAMPLED_LANES * 3); - - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_lo = unsafe { load_eight(y_row, 0) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let y_hi = unsafe { load_eight(y_row, LANES) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - fill_chunk_from_vectors( - y_lo, - vget_low_u8(cb), - vget_low_u8(cr), - &mut dst[..LANES * 3], - ); - fill_chunk_from_vectors( - y_hi, - vget_high_u8(cb), - vget_high_u8(cr), - &mut dst[LANES * 3..], - ); - } + let (y, y_remainder) = y.as_chunks::(); + let (cb, cb_remainder) = cb.as_chunks::(); + let (cr, cr_remainder) = cr.as_chunks::(); + let (dst, dst_remainder) = dst.as_chunks_mut::<{ LANES * 3 }>(); + let _ = (y_remainder, cb_remainder, cr_remainder, dst_remainder); + + fill_chunk(&y[0], &cb[0], &cr[0], &mut dst[0]); + fill_chunk(&y[1], &cb[1], &cr[1], &mut dst[1]); } #[target_feature(enable = "neon")] -unsafe fn load_eight(src: &[u8], offset: usize) -> uint8x8_t { +fn load_eight(src: &[u8], offset: usize) -> uint8x8_t { debug_assert!(offset <= src.len().saturating_sub(LANES)); - // SAFETY: callers guarantee there are at least eight readable bytes at - // `offset`; `vld1_u8` accepts unaligned loads. - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { vld1_u8(src.as_ptr().add(offset)) } + let Some(chunk) = src.get(offset..).and_then(<[u8]>::first_chunk::) else { + return neon_memory::load_u8x8(&[0; LANES]); + }; + neon_memory::load_u8x8(chunk) } #[target_feature(enable = "neon")] @@ -1563,7 +1481,7 @@ fn pack_eight_u8(low: int32x4_t, high: int32x4_t) -> uint8x8_t { } #[target_feature(enable = "neon")] -unsafe fn fill_upsampled_420_chunk( +fn fill_upsampled_420_chunk( near: &[u8], curr: &[u8], sample_offset: usize, @@ -1571,11 +1489,10 @@ unsafe fn fill_upsampled_420_chunk( out: &mut [u8], ) { if can_vectorize_420_chunk(curr.len(), sample_offset, out.len()) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { + if let Some(out) = out.first_chunk_mut::() { fill_upsampled_420_chunk_neon(near, curr, sample_offset, out); + return; } - return; } fill_upsampled_420_chunk_scalar(near, curr, sample_offset, output_width, out); } @@ -1604,30 +1521,23 @@ fn can_vectorize_420_chunk(chroma_width: usize, sample_offset: usize, out_len: u } #[target_feature(enable = "neon")] -unsafe fn fill_upsampled_420_chunk_neon( +fn fill_upsampled_420_chunk_neon( near: &[u8], curr: &[u8], sample_offset: usize, - out: &mut [u8], + out: &mut [u8; UPSAMPLED_LANES], ) { debug_assert!(can_vectorize_420_chunk( curr.len(), sample_offset, out.len() )); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - unsafe { - vst1q_u8( - out.as_mut_ptr(), - upsampled_420_chunk16(near, curr, sample_offset), - ); - } + neon_memory::store_u8x16(out, upsampled_420_chunk16(near, curr, sample_offset)); } #[target_feature(enable = "neon")] -unsafe fn upsampled_420_chunk16(near: &[u8], curr: &[u8], sample_offset: usize) -> uint8x16_t { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let lanes = unsafe { upsampled_420_chunk16_u16(near, curr, sample_offset) }; +fn upsampled_420_chunk16(near: &[u8], curr: &[u8], sample_offset: usize) -> uint8x16_t { + let lanes = { upsampled_420_chunk16_u16(near, curr, sample_offset) }; let even8 = vqmovn_u16(lanes.0); let odd8 = vqmovn_u16(lanes.1); let zipped = vzip_u8(even8, odd8); @@ -1635,17 +1545,16 @@ unsafe fn upsampled_420_chunk16(near: &[u8], curr: &[u8], sample_offset: usize) } #[target_feature(enable = "neon")] -unsafe fn upsampled_420_chunk16_u16( +fn upsampled_420_chunk16_u16( near: &[u8], curr: &[u8], sample_offset: usize, ) -> core::arch::aarch64::uint16x8x2_t { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let this = unsafe { colsum_eight(near, curr, sample_offset) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let prev = unsafe { colsum_eight(near, curr, sample_offset - 1) }; - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let next = unsafe { colsum_eight(near, curr, sample_offset + 1) }; + let (near_prev, near_this, near_next) = load_eight_triplet(near, sample_offset); + let (curr_prev, curr_this, curr_next) = load_eight_triplet(curr, sample_offset); + let prev = weighted_colsum(near_prev, curr_prev); + let this = weighted_colsum(near_this, curr_this); + let next = weighted_colsum(near_next, curr_next); let three_this = vaddq_u16(this, vaddq_u16(this, this)); let even = vshrq_n_u16(vaddq_u16(vaddq_u16(three_this, prev), vdupq_n_u16(8)), 4); @@ -1654,7 +1563,8 @@ unsafe fn upsampled_420_chunk16_u16( } #[target_feature(enable = "neon")] -unsafe fn upsampled_420_chunk16_pair_u16( +#[inline] +fn upsampled_420_chunk16_pair_u16( top_near: &[u8], curr: &[u8], bottom_near: &[u8], @@ -1663,48 +1573,24 @@ unsafe fn upsampled_420_chunk16_pair_u16( core::arch::aarch64::uint16x8x2_t, core::arch::aarch64::uint16x8x2_t, ) { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let curr_prev = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset - 1)) }); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let curr_this = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset)) }); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let curr_next = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset + 1)) }); + let (curr_prev, curr_this, curr_next) = load_eight_triplet(curr, sample_offset); + let curr_prev = vmovl_u8(curr_prev); + let curr_this = vmovl_u8(curr_this); + let curr_next = vmovl_u8(curr_next); let three_prev = vaddq_u16(curr_prev, vaddq_u16(curr_prev, curr_prev)); let three_this = vaddq_u16(curr_this, vaddq_u16(curr_this, curr_this)); let three_next = vaddq_u16(curr_next, vaddq_u16(curr_next, curr_next)); - let top_prev = vaddq_u16( - three_prev, - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - vmovl_u8(unsafe { vld1_u8(top_near.as_ptr().add(sample_offset - 1)) }), - ); - let top_this = vaddq_u16( - three_this, - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - vmovl_u8(unsafe { vld1_u8(top_near.as_ptr().add(sample_offset)) }), - ); - let top_next = vaddq_u16( - three_next, - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - vmovl_u8(unsafe { vld1_u8(top_near.as_ptr().add(sample_offset + 1)) }), - ); + let (top_prev, top_this, top_next) = load_eight_triplet(top_near, sample_offset); + let top_prev = vaddq_u16(three_prev, vmovl_u8(top_prev)); + let top_this = vaddq_u16(three_this, vmovl_u8(top_this)); + let top_next = vaddq_u16(three_next, vmovl_u8(top_next)); - let bottom_prev = vaddq_u16( - three_prev, - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - vmovl_u8(unsafe { vld1_u8(bottom_near.as_ptr().add(sample_offset - 1)) }), - ); - let bottom_this = vaddq_u16( - three_this, - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - vmovl_u8(unsafe { vld1_u8(bottom_near.as_ptr().add(sample_offset)) }), - ); - let bottom_next = vaddq_u16( - three_next, - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - vmovl_u8(unsafe { vld1_u8(bottom_near.as_ptr().add(sample_offset + 1)) }), - ); + let (bottom_prev, bottom_this, bottom_next) = load_eight_triplet(bottom_near, sample_offset); + let bottom_prev = vaddq_u16(three_prev, vmovl_u8(bottom_prev)); + let bottom_this = vaddq_u16(three_this, vmovl_u8(bottom_this)); + let bottom_next = vaddq_u16(three_next, vmovl_u8(bottom_next)); let top_three_this = vaddq_u16(top_this, vaddq_u16(top_this, top_this)); let top_even = vshrq_n_u16( @@ -1733,11 +1619,25 @@ unsafe fn upsampled_420_chunk16_pair_u16( } #[target_feature(enable = "neon")] -unsafe fn colsum_eight(near: &[u8], curr: &[u8], sample_offset: usize) -> uint16x8_t { - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let near16 = vmovl_u8(unsafe { vld1_u8(near.as_ptr().add(sample_offset)) }); - // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. - let curr16 = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset)) }); +fn load_eight_triplet(src: &[u8], sample_offset: usize) -> (uint8x8_t, uint8x8_t, uint8x8_t) { + let Some(start) = sample_offset.checked_sub(1) else { + let zero = neon_memory::load_u8x8(&[0; LANES]); + return (zero, zero, zero); + }; + let Some(window) = src + .get(start..) + .and_then(<[u8]>::first_chunk::<{ LANES + 2 }>) + else { + let zero = neon_memory::load_u8x8(&[0; LANES]); + return (zero, zero, zero); + }; + neon_memory::load_u8x8_triplet(window) +} + +#[target_feature(enable = "neon")] +fn weighted_colsum(near: uint8x8_t, curr: uint8x8_t) -> uint16x8_t { + let near16 = vmovl_u8(near); + let curr16 = vmovl_u8(curr); vaddq_u16(vaddq_u16(curr16, curr16), vaddq_u16(curr16, near16)) } diff --git a/crates/j2k-jpeg/src/backend/neon/top_only_tests.rs b/crates/j2k-jpeg/src/backend/neon/top_only_tests.rs index 0fd2943a..7acd0b93 100644 --- a/crates/j2k-jpeg/src/backend/neon/top_only_tests.rs +++ b/crates/j2k-jpeg/src/backend/neon/top_only_tests.rs @@ -60,6 +60,12 @@ fn top_only_rows<'a>(fixture: &'a TopOnlyFixture, dst: &'a mut [u8]) -> Rgb420Ro Rgb420RowPair::new(&fixture.y, None, fixture.chroma(), dst, None) } +fn neon() -> fearless_simd::Neon { + fearless_simd::Level::new() + .as_neon() + .expect("AArch64 test host must provide NEON") +} + #[test] fn cropped_top_only_neon_matches_scalar_across_alignment_and_partial_chunks() { let fixture = TopOnlyFixture::new(73); @@ -73,10 +79,10 @@ fn cropped_top_only_neon_matches_scalar_across_alignment_and_partial_chunks() { )); let mut actual = vec![0u8; crop_width * 3]; - fill_rgb_row_pair_from_420_cropped(Rgb420CroppedRowPair::new( - top_only_rows(&fixture, &mut actual), - crop, - )); + fill_rgb_row_pair_from_420_cropped( + neon(), + Rgb420CroppedRowPair::new(top_only_rows(&fixture, &mut actual), crop), + ); assert_eq!( actual, expected, @@ -93,7 +99,7 @@ fn top_only_neon_tail_matches_scalar_for_partial_and_full_chunks() { scalar::fill_rgb_row_pair_from_420(top_only_rows(&fixture, &mut expected)); let mut actual = vec![0u8; width * 3]; - fill_rgb_row_pair_from_420(top_only_rows(&fixture, &mut actual)); + fill_rgb_row_pair_from_420(neon(), top_only_rows(&fixture, &mut actual)); assert_eq!(actual, expected, "top-only tail differs at width {width}"); } diff --git a/crates/j2k-jpeg/src/backend/tests.rs b/crates/j2k-jpeg/src/backend/tests.rs index 51187f4c..65be6f8a 100644 --- a/crates/j2k-jpeg/src/backend/tests.rs +++ b/crates/j2k-jpeg/src/backend/tests.rs @@ -156,6 +156,11 @@ fn assert_rgb_rows_match_scalar( assert_eq!(actual, expected, "{label}"); } +fn backend_for_kind(kind: super::BackendKind) -> super::Backend { + super::Backend::for_test_kind(kind) + .unwrap_or_else(|| panic!("requested backend {kind:?} is unavailable on this host")) +} + fn assert_rgb420_pair_matches_scalar( label: &str, kind: super::BackendKind, @@ -169,7 +174,7 @@ fn assert_rgb420_pair_matches_scalar( let mut actual_top = vec![0xAA; fixture.y_top.len() * 3]; let mut actual_bottom = fixture.y_bottom.map(|row| vec![0xAA; row.len() * 3]); - super::Backend { kind } + backend_for_kind(kind) .fill_rgb_row_pair_from_420(fixture.request(&mut actual_top, actual_bottom.as_deref_mut())); assert_eq!(actual_top, expected_top, "{label}: top row"); @@ -194,7 +199,7 @@ fn assert_rgb420_cropped_pair_matches_scalar( let mut actual_top = vec![0xAA; crop_width * 3]; let mut actual_bottom = fixture.y_bottom.map(|_| vec![0xAA; crop_width * 3]); - super::Backend { kind }.fill_rgb_row_pair_from_420_cropped(fixture.cropped_request( + backend_for_kind(kind).fill_rgb_row_pair_from_420_cropped(fixture.cropped_request( crop_start, crop_width, &mut actual_top, @@ -227,7 +232,7 @@ fn assert_rgb420_pair_uses_safe_prefix( let mut actual_top = vec![0xAA; fixture.y_top.len() * 3]; let mut actual_bottom = fixture.y_bottom.map(|row| vec![0xAA; row.len() * 3]); - super::Backend { kind } + backend_for_kind(kind) .fill_rgb_row_pair_from_420(fixture.request(&mut actual_top, actual_bottom.as_deref_mut())); assert_eq!(actual_top, expected_top, "{label}: top row"); @@ -239,6 +244,116 @@ fn x86_fixture_byte(index: usize) -> u8 { u8::try_from(index).expect("x86 fixture index must fit in u8") } +#[test] +fn scalar_backend_is_always_constructible_without_a_capability_token() { + let backend = super::Backend::for_test_kind(super::BackendKind::Scalar) + .expect("scalar backend must always be available"); + assert_eq!(backend.kind(), super::BackendKind::Scalar); +} + +#[test] +fn detected_backend_respects_scalar_only_and_runtime_capabilities() { + let detected = super::Backend::detect().kind(); + if cfg!(feature = "scalar-only") { + assert_eq!(detected, super::BackendKind::Scalar); + return; + } + + #[cfg(target_arch = "x86_64")] + { + let expected = if j2k_core::CpuFeatures::detect().avx2 { + super::BackendKind::Avx2 + } else { + super::BackendKind::Scalar + }; + assert_eq!(detected, expected); + } + + #[cfg(target_arch = "aarch64")] + assert_eq!(detected, super::BackendKind::Neon); +} + +#[cfg(target_arch = "x86_64")] +#[test] +fn avx2_backend_requires_a_genuine_runtime_token() { + assert_eq!( + super::Backend::for_test_kind(super::BackendKind::Avx2).is_some(), + j2k_core::CpuFeatures::detect().avx2, + ); +} + +#[cfg(target_arch = "aarch64")] +#[test] +fn neon_backend_uses_a_genuine_fearless_simd_token() { + let backend = super::Backend::for_test_kind(super::BackendKind::Neon) + .expect("AArch64 hosts must expose a NEON token"); + assert_eq!(backend.kind(), super::BackendKind::Neon); +} + +#[test] +fn detected_backend_matches_scalar_for_unaligned_and_boundary_width_rows() { + let backend = super::Backend::detect(); + for width in [0usize, 1, 7, 8, 15, 16, 255, 256] { + let y_storage: Vec = (0..=width) + .map(|i| u8::try_from(i & 0xff).expect("masked fixture byte")) + .collect(); + let cb_storage: Vec = (0..=width) + .map(|i| { + u8::try_from((i.wrapping_mul(29).wrapping_add(17)) & 0xff) + .expect("masked fixture byte") + }) + .collect(); + let cr_storage: Vec = (0..=width) + .map(|i| { + u8::try_from((i.wrapping_mul(43).wrapping_add(91)) & 0xff) + .expect("masked fixture byte") + }) + .collect(); + let y = &y_storage[1..]; + let cb = &cb_storage[1..]; + let cr = &cr_storage[1..]; + let mut expected = vec![0xA5; width * 3]; + let mut actual = expected.clone(); + scalar::fill_rgb_row_from_ycbcr(y, cb, cr, &mut expected); + backend.fill_rgb_row_from_ycbcr(y, cb, cr, &mut actual); + assert_eq!(actual, expected, "unaligned row width {width}"); + } +} + +#[test] +fn detected_backend_matches_scalar_for_unaligned_420_crop_and_missing_bottom() { + let width = 257usize; + let chroma_width = width.div_ceil(2); + let make = |len: usize, offset: usize, scale: usize| -> Vec { + (0..=len) + .map(|i| { + u8::try_from((i.wrapping_mul(scale).wrapping_add(offset)) & 0xff) + .expect("masked fixture byte") + }) + .collect() + }; + let y = make(width, 3, 37); + let prev_cb = make(chroma_width, 5, 13); + let curr_cb = make(chroma_width, 7, 17); + let next_cb = make(chroma_width, 11, 23); + let prev_cr = make(chroma_width, 19, 29); + let curr_cr = make(chroma_width, 31, 31); + let next_cr = make(chroma_width, 47, 37); + let fixture = Rgb420Fixture::new( + &y[1..], + None, + [&prev_cb[1..], &curr_cb[1..], &next_cb[1..]], + [&prev_cr[1..], &curr_cr[1..], &next_cr[1..]], + ); + assert_rgb420_cropped_pair_matches_scalar( + "detected unaligned top-only 4:2:0 crop", + super::Backend::detect().kind(), + fixture, + 3, + 249, + ); +} + #[test] fn gray_rows_expand_to_equal_rgb_channels() { let gray = [10u8, 40, 90, 200]; @@ -389,9 +504,7 @@ fn backend_scalar_420_row_pair_handles_missing_bottom_row() { reason = "the synthetic byte pattern intentionally wraps the bounded row index" )] fn backend_scalar_420_cropped_row_pair_matches_full_width_crop() { - let backend = super::Backend { - kind: super::BackendKind::Scalar, - }; + let backend = backend_for_kind(super::BackendKind::Scalar); let width = 73usize; let crop_start = 3usize; let crop_width = 53usize; @@ -471,9 +584,7 @@ fn backend_scalar_420_cropped_row_pair_matches_full_width_crop() { reason = "the synthetic byte pattern intentionally wraps the bounded row index" )] fn backend_scalar_420_cropped_top_only_matches_full_width_crop() { - let backend = super::Backend { - kind: super::BackendKind::Scalar, - }; + let backend = backend_for_kind(super::BackendKind::Scalar); let width = 31usize; let crop_start = 1usize; let crop_width = 17usize; @@ -806,9 +917,7 @@ fn avx2_backend_prefers_cropped_420_region_when_available() { return; } - let backend = super::Backend { - kind: super::BackendKind::Avx2, - }; + let backend = backend_for_kind(super::BackendKind::Avx2); assert!(backend.prefers_cropped_420_region(4096, 257)); } diff --git a/crates/j2k-jpeg/src/backend/x86.rs b/crates/j2k-jpeg/src/backend/x86.rs index 9b7e29e9..76e90dcf 100644 --- a/crates/j2k-jpeg/src/backend/x86.rs +++ b/crates/j2k-jpeg/src/backend/x86.rs @@ -4,7 +4,7 @@ use alloc::vec::Vec; use core::arch::x86_64::{ __m128i, __m256i, _mm256_add_epi32, _mm256_cvtepu8_epi32, _mm256_extracti128_si256, _mm256_mullo_epi32, _mm256_set1_epi32, _mm256_srai_epi32, _mm256_sub_epi32, _mm_cvtsi128_si64, - _mm_loadl_epi64, _mm_packs_epi32, _mm_packus_epi16, + _mm_packs_epi32, _mm_packus_epi16, }; use core::cell::RefCell; @@ -12,6 +12,8 @@ use crate::color::upsample::{ h2v2_fancy_sample, upsample_h2v2_fancy_row, upsample_h2v2_fancy_rows, }; use crate::color::ycbcr::{FIX_0_34414, FIX_0_71414, FIX_1_40200, FIX_1_77200, ROUND}; +use crate::simd::x86::{exact_avx2_kernel, ExactAvx2}; +use crate::simd::x86_memory; use super::row_pair::{normalize_simd_row_pair, normalize_ycbcr_row}; use super::{scalar, Rgb420ChromaRows, Rgb420CroppedRowPair, Rgb420RowPair}; @@ -40,6 +42,7 @@ std::thread_local! { static ROW_PAIR_SCRATCH: RefCell = RefCell::new(RowPairScratch::default()); } +#[inline] pub(crate) fn fill_rgb_row_from_gray(gray_row: &[u8], dst: &mut [u8]) { let width = gray_row.len().min(dst.len() / 3); let gray_row = &gray_row[..width]; @@ -102,19 +105,68 @@ pub(crate) fn fill_rgb_row_from_rgb(r_row: &[u8], g_row: &[u8], b_row: &[u8], ds } } -pub(crate) fn fill_rgb_row_from_ycbcr(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { +pub(crate) fn fill_rgb_row_from_ycbcr( + avx2: ExactAvx2, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + dst: &mut [u8], +) { let (y_row, cb_row, cr_row, dst) = normalize_ycbcr_row(y_row, cb_row, cr_row, dst); debug_assert_eq!(y_row.len(), cb_row.len()); debug_assert_eq!(y_row.len(), cr_row.len()); debug_assert_eq!(dst.len(), y_row.len() * 3); - // SAFETY: Backend dispatch selects this path only when AVX2 is available. - // All source rows and the destination are narrowed to the same pixel count. - unsafe { - fill_rgb_row_from_ycbcr_avx2(y_row, cb_row, cr_row, dst); + fill_rgb_row_from_ycbcr_kernel(avx2, y_row, cb_row, cr_row, dst); +} + +exact_avx2_kernel! { + fn fill_rgb_row_from_ycbcr_kernel( + avx2: ExactAvx2, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + dst: &mut [u8], + ) { + let width = y_row.len(); + let mut sources = x86_memory::U8x8TripleCursor::new(avx2, y_row, cb_row, cr_row); + + while let Some((offset, first, second)) = sources.next_pair() { + fill_chunk( + first.0, + first.1, + first.2, + &mut dst[offset * 3..(offset + LANES) * 3], + ); + fill_chunk( + second.0, + second.1, + second.2, + &mut dst[(offset + LANES) * 3..(offset + LANES * 2) * 3], + ); + } + + if let Some((offset, values)) = sources.next() { + fill_chunk( + values.0, + values.1, + values.2, + &mut dst[offset * 3..(offset + LANES) * 3], + ); + } + + let offset = sources.offset(); + if offset < width { + scalar::fill_rgb_row_from_ycbcr( + &y_row[offset..], + &cb_row[offset..], + &cr_row[offset..], + &mut dst[offset * 3..], + ); + } } } -pub(crate) fn fill_rgb_row_pair_from_420(request: Rgb420RowPair<'_>) { +pub(crate) fn fill_rgb_row_pair_from_420(avx2: ExactAvx2, request: Rgb420RowPair<'_>) { let Some(request) = normalize_simd_row_pair(request) else { return; }; @@ -123,16 +175,24 @@ pub(crate) fn fill_rgb_row_pair_from_420(request: Rgb420RowPair<'_>) { ROW_PAIR_SCRATCH.with(|scratch| { let mut scratch = scratch.borrow_mut(); scratch.ensure_width(width); - // SAFETY: Backend dispatch selects this path only when AVX2 is - // available. The wrapper clamps luma, chroma, and destination rows so - // all upsampled reads and RGB writes fit the passed slices. - unsafe { - fill_rgb_row_pair_from_420_avx2(request, &mut scratch); - } + fill_rgb_row_pair_from_420_kernel(avx2, request, &mut scratch); }); } -pub(crate) fn fill_rgb_row_pair_from_420_cropped(request: Rgb420CroppedRowPair<'_>) { +exact_avx2_kernel! { + fn fill_rgb_row_pair_from_420_kernel( + avx2: ExactAvx2, + request: Rgb420RowPair<'_>, + scratch: &mut RowPairScratch, + ) { + fill_rgb_row_pair_from_420_avx2(avx2, request, scratch); + } +} + +pub(crate) fn fill_rgb_row_pair_from_420_cropped( + avx2: ExactAvx2, + request: Rgb420CroppedRowPair<'_>, +) { let Rgb420CroppedRowPair { rows, crop } = request; let Rgb420RowPair { y_top, @@ -194,24 +254,14 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped(request: Rgb420CroppedRowPair<' let cr_top = &mut cr_top[..width]; fill_cropped_h2v2_row(prev_cb, curr_cb, crop_start, cb_top); fill_cropped_h2v2_row(prev_cr, curr_cr, crop_start, cr_top); - // SAFETY: Backend dispatch selects this path only when AVX2 is - // available. `y_top_crop`, scratch chroma rows, and destination slices - // all have the same bounded pixel width. - unsafe { - fill_rgb_row_from_ycbcr_avx2(y_top_crop, cb_top, cr_top, dst_top); - } + fill_rgb_row_from_ycbcr_kernel(avx2, y_top_crop, cb_top, cr_top, dst_top); if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { let cb_bottom = &mut cb_bottom[..width]; let cr_bottom = &mut cr_bottom[..width]; fill_cropped_h2v2_row(next_cb, curr_cb, crop_start, cb_bottom); fill_cropped_h2v2_row(next_cr, curr_cr, crop_start, cr_bottom); - // SAFETY: Backend dispatch selects this path only when AVX2 is - // available. The bottom luma, scratch chroma, and destination - // slices were clamped to the same bounded pixel width. - unsafe { - fill_rgb_row_from_ycbcr_avx2(y_bottom, cb_bottom, cr_bottom, dst_bottom); - } + fill_rgb_row_from_ycbcr_kernel(avx2, y_bottom, cb_bottom, cr_bottom, dst_bottom); } }); } @@ -223,7 +273,8 @@ fn fill_cropped_h2v2_row(near: &[u8], curr: &[u8], crop_start: usize, out: &mut } #[target_feature(enable = "avx2")] -unsafe fn fill_rgb_row_pair_from_420_avx2( +fn fill_rgb_row_pair_from_420_avx2( + avx2: ExactAvx2, request: Rgb420RowPair<'_>, scratch: &mut RowPairScratch, ) { @@ -256,22 +307,12 @@ unsafe fn fill_rgb_row_pair_from_420_avx2( let cr_bottom = &mut cr_bottom[..width]; upsample_h2v2_fancy_rows(prev_cb, curr_cb, next_cb, width, cb_top, cb_bottom); upsample_h2v2_fancy_rows(prev_cr, curr_cr, next_cr, width, cr_top, cr_bottom); - // SAFETY: This AVX2 helper is reached through the safe wrapper, which - // clamps both luma rows, both scratch chroma rows, and both RGB - // destination rows to the same pixel width. - unsafe { - fill_rgb_row_from_ycbcr_avx2(y_top, cb_top, cr_top, dst_top); - fill_rgb_row_from_ycbcr_avx2(y_bottom, cb_bottom, cr_bottom, dst_bottom); - } + fill_rgb_row_from_ycbcr_kernel(avx2, y_top, cb_top, cr_top, dst_top); + fill_rgb_row_from_ycbcr_kernel(avx2, y_bottom, cb_bottom, cr_bottom, dst_bottom); } else { upsample_h2v2_fancy_row(prev_cb, curr_cb, next_cb, width, false, cb_top); upsample_h2v2_fancy_row(prev_cr, curr_cr, next_cr, width, false, cr_top); - // SAFETY: This AVX2 helper is reached through the safe wrapper, which - // clamps the luma row, scratch chroma rows, and RGB destination row to - // the same pixel width. - unsafe { - fill_rgb_row_from_ycbcr_avx2(y_top, cb_top, cr_top, dst_top); - } + fill_rgb_row_from_ycbcr_kernel(avx2, y_top, cb_top, cr_top, dst_top); } } @@ -282,7 +323,8 @@ pub(super) fn fill_rgb_row_from_ycbcr_for_test( cr_row: &[u8], dst: &mut [u8], ) { - fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst); + let avx2 = ExactAvx2::detect().expect("x86 test requested AVX2 on an unsupported host"); + fill_rgb_row_from_ycbcr(avx2, y_row, cb_row, cr_row, dst); } #[cfg(test)] @@ -301,74 +343,9 @@ pub(super) fn fill_rgb_row_from_rgb_for_test( } #[target_feature(enable = "avx2")] -unsafe fn fill_rgb_row_from_ycbcr_avx2(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { - let width = y_row.len(); - let mut offset = 0; - - while offset + (LANES * 2) <= width { - // SAFETY: The safe wrapper slices all input rows and `dst` to the same - // pixel count, and this loop only passes full eight-pixel chunks. - unsafe { - fill_chunk( - y_row, - cb_row, - cr_row, - &mut dst[offset * 3..(offset + LANES) * 3], - offset, - ); - fill_chunk( - y_row, - cb_row, - cr_row, - &mut dst[(offset + LANES) * 3..(offset + LANES * 2) * 3], - offset + LANES, - ); - } - offset += LANES * 2; - } - - while offset + LANES <= width { - // SAFETY: The safe wrapper slices all input rows and `dst` to the same - // pixel count, and this loop only passes a full eight-pixel chunk. - unsafe { - fill_chunk( - y_row, - cb_row, - cr_row, - &mut dst[offset * 3..(offset + LANES) * 3], - offset, - ); - } - offset += LANES; - } - - if offset < width { - scalar::fill_rgb_row_from_ycbcr( - &y_row[offset..], - &cb_row[offset..], - &cr_row[offset..], - &mut dst[offset * 3..], - ); - } -} - -#[target_feature(enable = "avx2")] -unsafe fn fill_chunk( - y_row: &[u8], - cb_row: &[u8], - cr_row: &[u8], - dst_chunk: &mut [u8], - offset: usize, -) { +fn fill_chunk(y: __m128i, cb: __m128i, cr: __m128i, dst_chunk: &mut [u8]) { debug_assert_eq!(dst_chunk.len(), LANES * 3); - // SAFETY: callers prove `offset + LANES <= row.len()` for each source row. - let y = unsafe { load_eight(y_row, offset) }; - // SAFETY: callers prove `offset + LANES <= row.len()` for each source row. - let cb = unsafe { load_eight(cb_row, offset) }; - // SAFETY: callers prove `offset + LANES <= row.len()` for each source row. - let cr = unsafe { load_eight(cr_row, offset) }; - let bias = _mm256_set1_epi32(128); let y32 = _mm256_cvtepu8_epi32(y); let cb32 = _mm256_sub_epi32(_mm256_cvtepu8_epi32(cb), bias); @@ -390,18 +367,7 @@ unsafe fn fill_chunk( ); let b = _mm256_add_epi32(y32, fixed_mul_shift(cb32, FIX_1_77200)); - // SAFETY: `dst_chunk` is narrowed by the caller to exactly one RGB chunk. - unsafe { - store_rgb_chunk(dst_chunk, r, g, b); - } -} - -#[target_feature(enable = "avx2")] -unsafe fn load_eight(src: &[u8], offset: usize) -> __m128i { - debug_assert!(offset <= src.len().saturating_sub(LANES)); - // SAFETY: the caller guarantees there are at least eight readable bytes at - // `offset`; `_mm_loadl_epi64` accepts unaligned loads. - unsafe { _mm_loadl_epi64(src.as_ptr().add(offset).cast()) } + store_rgb_chunk(dst_chunk, r, g, b); } #[target_feature(enable = "avx2")] @@ -416,14 +382,11 @@ fn fixed_mul_shift(values: __m256i, coefficient: i32) -> __m256i { } #[target_feature(enable = "avx2")] -unsafe fn store_rgb_chunk(dst_chunk: &mut [u8], r: __m256i, g: __m256i, b: __m256i) { +fn store_rgb_chunk(dst_chunk: &mut [u8], r: __m256i, g: __m256i, b: __m256i) { debug_assert_eq!(dst_chunk.len(), LANES * 3); - // SAFETY: packing only rearranges register values and does not dereference. - let r_bytes = unsafe { pack_eight_u8(r) }; - // SAFETY: packing only rearranges register values and does not dereference. - let g_bytes = unsafe { pack_eight_u8(g) }; - // SAFETY: packing only rearranges register values and does not dereference. - let b_bytes = unsafe { pack_eight_u8(b) }; + let r_bytes = pack_eight_u8(r); + let g_bytes = pack_eight_u8(g); + let b_bytes = pack_eight_u8(b); for ((((r, g), b), pixel), _) in r_bytes .iter() @@ -439,7 +402,7 @@ unsafe fn store_rgb_chunk(dst_chunk: &mut [u8], r: __m256i, g: __m256i, b: __m25 } #[target_feature(enable = "avx2")] -unsafe fn pack_eight_u8(values: __m256i) -> [u8; LANES] { +fn pack_eight_u8(values: __m256i) -> [u8; LANES] { let words = _mm_packs_epi32( _mm256_extracti128_si256(values, 0), _mm256_extracti128_si256(values, 1), diff --git a/crates/j2k-jpeg/src/bench_support.rs b/crates/j2k-jpeg/src/bench_support.rs index 62ce3a80..f6a400a3 100644 --- a/crates/j2k-jpeg/src/bench_support.rs +++ b/crates/j2k-jpeg/src/bench_support.rs @@ -8,7 +8,7 @@ )] use crate::backend::scalar; -use crate::backend::{Backend, Rgb420ChromaRows, Rgb420RowPair}; +use crate::backend::{Backend, Rgb420ChromaRows, Rgb420Crop, Rgb420CroppedRowPair, Rgb420RowPair}; use crate::color::upsample::upsample_h2v2_fancy_rows; use crate::color::ycbcr::ycbcr_to_rgb; use crate::context::DecoderContext; @@ -360,9 +360,55 @@ pub fn bench_idct_reduced_2x2_block_with(input: &[i16; 64], output: &mut [u8; 4] #[cfg(target_arch = "aarch64")] #[doc(hidden)] pub fn bench_idct_neon_block(input: &[i16; 64], output: &mut [u8; 64]) { - // SAFETY: Every supported AArch64 target provides NEON, and the fixed-size - // references satisfy the kernel's input and output length requirements. - unsafe { crate::idct::neon::idct_islow(input, output) }; + let neon = fearless_simd::Level::new() + .as_neon() + .expect("AArch64 benchmark host must provide NEON"); + crate::idct::neon::idct_islow(neon, input, output); +} + +/// Run the NEON specialization for blocks whose bottom four coefficient rows +/// are known to be zero. +#[cfg(target_arch = "aarch64")] +#[doc(hidden)] +pub fn bench_idct_neon_bottom_half_zero_block(input: &[i16; 64], output: &mut [u8; 64]) { + let neon = fearless_simd::Level::new() + .as_neon() + .expect("AArch64 benchmark host must provide NEON"); + crate::idct::neon::idct_islow_bottom_half_zero(neon, input, output); +} + +/// Pre-detected NEON IDCT capability for benchmark loops. +#[cfg(target_arch = "aarch64")] +#[doc(hidden)] +pub struct BenchNeonIdct { + neon: fearless_simd::Neon, +} + +#[cfg(target_arch = "aarch64")] +impl BenchNeonIdct { + #[must_use] + pub fn new() -> Self { + Self { + neon: fearless_simd::Level::new() + .as_neon() + .expect("AArch64 benchmark host must provide NEON"), + } + } + + pub fn run(&self, input: &[i16; 64], output: &mut [u8; 64]) { + crate::idct::neon::idct_islow(self.neon, input, output); + } + + pub fn run_bottom_half_zero(&self, input: &[i16; 64], output: &mut [u8; 64]) { + crate::idct::neon::idct_islow_bottom_half_zero(self.neon, input, output); + } +} + +#[cfg(target_arch = "aarch64")] +impl Default for BenchNeonIdct { + fn default() -> Self { + Self::new() + } } #[cfg(any(target_arch = "x86_64", test))] @@ -387,13 +433,33 @@ const fn select_bench_avx2_dispatch(avx2_available: bool) -> BenchAvx2Dispatch { #[cfg(target_arch = "x86_64")] #[doc(hidden)] pub fn bench_idct_avx2_block(input: &[i16; 64], output: &mut [u8; 64]) { - match select_bench_avx2_dispatch(std::is_x86_feature_detected!("avx2")) { + let avx2 = crate::simd::x86::ExactAvx2::detect(); + match select_bench_avx2_dispatch(avx2.is_some()) { BenchAvx2Dispatch::Scalar => idct_islow(input, output), - BenchAvx2Dispatch::Avx2 => { - // SAFETY: This arm is reachable only after runtime AVX2 detection; - // fixed-size references satisfy the kernel's length requirements. - unsafe { crate::idct::avx2::idct_islow(input, output) }; - } + BenchAvx2Dispatch::Avx2 => crate::idct::avx2::idct_islow( + avx2.expect("dispatch selected AVX2 only when its token exists"), + input, + output, + ), + } +} + +/// Pre-detected exact-AVX2 capability for benchmark loops. +#[cfg(target_arch = "x86_64")] +#[doc(hidden)] +pub struct BenchAvx2Idct { + avx2: crate::simd::x86::ExactAvx2, +} + +#[cfg(target_arch = "x86_64")] +impl BenchAvx2Idct { + #[must_use] + pub fn try_new() -> Option { + crate::simd::x86::ExactAvx2::detect().map(|avx2| Self { avx2 }) + } + + pub fn run(&self, input: &[i16; 64], output: &mut [u8; 64]) { + crate::idct::avx2::idct_islow(self.avx2, input, output); } } @@ -494,6 +560,76 @@ impl BenchRgb420RowPairScratch { Some(&mut self.bottom), )); } + + /// Run a cropped region through the detected CPU backend. + pub fn run_cropped(&mut self, start: usize, width: usize) { + let request = BenchRgb420RowPair::new( + &self.y_top, + Some(&self.y_bottom), + BenchRgb420ChromaRows::new( + &self.prev_cb, + &self.curr_cb, + &self.next_cb, + &self.prev_cr, + &self.curr_cr, + &self.next_cr, + ), + &mut self.top[..width * 3], + Some(&mut self.bottom[..width * 3]), + ); + bench_rgb_row_pair_from_420_cropped(request, start, width); + } + + /// Return whether the detected full-row backend produces the scalar bytes. + #[must_use] + pub fn backend_matches_reference(&mut self) -> bool { + let mut expected_top = vec![0u8; self.top.len()]; + let mut expected_bottom = vec![0u8; self.bottom.len()]; + bench_rgb_row_pair_from_420_reference(BenchRgb420RowPair::new( + &self.y_top, + Some(&self.y_bottom), + BenchRgb420ChromaRows::new( + &self.prev_cb, + &self.curr_cb, + &self.next_cb, + &self.prev_cr, + &self.curr_cr, + &self.next_cr, + ), + &mut expected_top, + Some(&mut expected_bottom), + )); + self.run(); + self.top == expected_top && self.bottom == expected_bottom + } + + /// Return whether a cropped backend request produces the same bytes as the + /// scalar cropped path. + #[must_use] + pub fn cropped_backend_matches_reference(&mut self, start: usize, width: usize) -> bool { + let mut expected_top = vec![0u8; width * 3]; + let mut expected_bottom = vec![0u8; width * 3]; + scalar::fill_rgb_row_pair_from_420_cropped(Rgb420CroppedRowPair::new( + BenchRgb420RowPair::new( + &self.y_top, + Some(&self.y_bottom), + BenchRgb420ChromaRows::new( + &self.prev_cb, + &self.curr_cb, + &self.next_cb, + &self.prev_cr, + &self.curr_cr, + &self.next_cr, + ), + &mut expected_top, + Some(&mut expected_bottom), + ) + .into_backend(), + Rgb420Crop::new(start, width), + )); + self.run_cropped(start, width); + self.top[..width * 3] == expected_top && self.bottom[..width * 3] == expected_bottom + } } /// Borrowed chroma rows for the 4:2:0 row-pair bench helper. @@ -589,6 +725,19 @@ pub fn bench_rgb_row_pair_from_420(request: BenchRgb420RowPair<'_>) { Backend::detect().fill_rgb_row_pair_from_420(request.into_backend()); } +/// Run a cropped RGB 4:2:0 row-pair request through the detected backend. +#[doc(hidden)] +pub fn bench_rgb_row_pair_from_420_cropped( + request: BenchRgb420RowPair<'_>, + start: usize, + width: usize, +) { + Backend::detect().fill_rgb_row_pair_from_420_cropped(Rgb420CroppedRowPair::new( + request.into_backend(), + Rgb420Crop::new(start, width), + )); +} + /// Run the RGB 4:2:0 row-pair backend with dispatch stats. #[doc(hidden)] pub fn bench_rgb_row_pair_from_420_with_stats( @@ -664,18 +813,31 @@ pub struct BenchColorRowScratch { cb: Vec, cr: Vec, rgb: Vec, + input_offset: usize, + width: usize, } impl BenchColorRowScratch { /// Create the scratch with a deterministic luminance/chroma pattern. #[must_use] + pub fn new(width: usize) -> Self { + Self::with_input_offset(width, 0) + } + + /// Create a fixture whose source rows begin one byte into their backing + /// allocations, exercising unaligned SIMD loads. + #[must_use] + pub fn new_unaligned(width: usize) -> Self { + Self::with_input_offset(width, 1) + } + #[expect( clippy::cast_possible_truncation, reason = "benchmark fixture values are explicitly masked to one byte" )] - pub fn new(width: usize) -> Self { + fn with_input_offset(width: usize, input_offset: usize) -> Self { let seed = |offset: usize, scale: usize| -> Vec { - (0..width) + (0..width + input_offset) .map(|i| ((i.wrapping_mul(scale).wrapping_add(offset)) & 0xFF) as u8) .collect() }; @@ -685,16 +847,28 @@ impl BenchColorRowScratch { cb: seed(64, 5), cr: seed(192, 3), rgb: vec![0u8; width * 3], + input_offset, + width, } } + fn rows(&self) -> (&[u8], &[u8], &[u8]) { + let range = self.input_offset..self.input_offset + self.width; + ( + &self.y[range.clone()], + &self.cb[range.clone()], + &self.cr[range], + ) + } + /// Run one iteration of the scalar per-pixel YCbCr→RGB conversion. pub fn run_scalar(&mut self) { - for (((&y, &cb), &cr), pixel) in self - .y + let input_offset = self.input_offset; + let width = self.width; + for (((&y, &cb), &cr), pixel) in self.y[input_offset..input_offset + width] .iter() - .zip(self.cb.iter()) - .zip(self.cr.iter()) + .zip(self.cb[input_offset..input_offset + width].iter()) + .zip(self.cr[input_offset..input_offset + width].iter()) .zip(self.rgb.chunks_exact_mut(3)) { let (r, g, b) = ycbcr_to_rgb(y, cb, cr); @@ -705,8 +879,105 @@ impl BenchColorRowScratch { } /// Run one iteration through the detected production backend. + pub fn run_backend(&mut self) { + let input_offset = self.input_offset; + let width = self.width; + self.backend.fill_rgb_row_from_ycbcr( + &self.y[input_offset..input_offset + width], + &self.cb[input_offset..input_offset + width], + &self.cr[input_offset..input_offset + width], + &mut self.rgb, + ); + } + + /// Return whether the detected backend produces the scalar row bytes. + #[must_use] + pub fn backend_matches_scalar(&mut self) -> bool { + let (y, cb, cr) = self.rows(); + let mut expected = vec![0u8; self.width * 3]; + scalar::fill_rgb_row_from_ycbcr(y, cb, cr, &mut expected); + self.run_backend(); + self.rgb == expected + } +} + +/// Pre-allocated planar grayscale-to-RGB row benchmark input. +#[doc(hidden)] +pub struct BenchGrayRowScratch { + backend: Backend, + gray: Vec, + rgb: Vec, +} + +impl BenchGrayRowScratch { + #[must_use] + #[expect( + clippy::cast_possible_truncation, + reason = "benchmark fixture values intentionally retain the low byte" + )] + pub fn new(width: usize) -> Self { + Self { + backend: Backend::detect(), + gray: (0..width).map(|i| i.wrapping_mul(37) as u8).collect(), + rgb: vec![0u8; width * 3], + } + } + + pub fn run_backend(&mut self) { + self.backend + .fill_rgb_row_from_gray(&self.gray, &mut self.rgb); + } + + #[must_use] + pub fn backend_matches_scalar(&mut self) -> bool { + let mut expected = vec![0u8; self.rgb.len()]; + scalar::fill_rgb_row_from_gray(&self.gray, &mut expected); + self.run_backend(); + self.rgb == expected + } +} + +/// Pre-allocated planar RGB-to-interleaved-RGB row benchmark input. +#[doc(hidden)] +pub struct BenchRgbRowScratch { + backend: Backend, + r: Vec, + g: Vec, + b: Vec, + rgb: Vec, +} + +impl BenchRgbRowScratch { + #[must_use] + #[expect( + clippy::cast_possible_truncation, + reason = "benchmark fixture values intentionally retain the low byte" + )] + pub fn new(width: usize) -> Self { + let seed = |offset: usize, scale: usize| -> Vec { + (0..width) + .map(|i| i.wrapping_mul(scale).wrapping_add(offset) as u8) + .collect() + }; + Self { + backend: Backend::detect(), + r: seed(11, 37), + g: seed(47, 29), + b: seed(89, 19), + rgb: vec![0u8; width * 3], + } + } + pub fn run_backend(&mut self) { self.backend - .fill_rgb_row_from_ycbcr(&self.y, &self.cb, &self.cr, &mut self.rgb); + .fill_rgb_row_from_rgb(&self.r, &self.g, &self.b, &mut self.rgb); + } + + #[must_use] + pub fn backend_matches_scalar(&mut self) -> bool { + let mut expected = vec![0u8; self.rgb.len()]; + scalar::fill_rgb_row_from_rgb(&self.r, &self.g, &self.b, &mut expected); + self.run_backend(); + self.rgb == expected } } diff --git a/crates/j2k-jpeg/src/idct/avx2.rs b/crates/j2k-jpeg/src/idct/avx2.rs index 2d7d6248..cfb6f498 100644 --- a/crates/j2k-jpeg/src/idct/avx2.rs +++ b/crates/j2k-jpeg/src/idct/avx2.rs @@ -11,11 +11,14 @@ use core::arch::x86_64::{ __m128i, _mm_add_epi32, _mm_cvtepi16_epi32, _mm_mullo_epi32, _mm_packs_epi32, _mm_packus_epi16, - _mm_set1_epi32, _mm_slli_epi32, _mm_srai_epi32, _mm_srli_si128, _mm_storel_epi64, - _mm_sub_epi32, _mm_unpackhi_epi32, _mm_unpackhi_epi64, _mm_unpacklo_epi32, _mm_unpacklo_epi64, + _mm_set1_epi32, _mm_slli_epi32, _mm_srai_epi32, _mm_srli_si128, _mm_sub_epi32, + _mm_unpackhi_epi32, _mm_unpackhi_epi64, _mm_unpacklo_epi32, _mm_unpacklo_epi64, }; use j2k_codec_math::jpeg::idct; +use crate::simd::x86::exact_avx2_kernel; +use crate::simd::x86_memory; + #[expect( clippy::cast_possible_truncation, clippy::cast_possible_wrap, @@ -42,23 +45,22 @@ const FIX_2_053119869: i32 = idct::FIX_2_053119869; const FIX_2_562915447: i32 = idct::FIX_2_562915447; const FIX_3_072711026: i32 = idct::FIX_3_072711026; -/// Inverse DCT of one 8×8 block. Output is level-shifted (+128) and -/// saturated to `[0, 255]`, matching the scalar path byte-for-byte on -/// legal JPEG coefficients and on the adversarial saturating edges -/// proptested against. -/// -/// # Safety -/// Caller must ensure the host CPU supports SSE4.1. The -/// `Backend::detect` dispatch picks this variant when AVX2 is available -/// (which implies SSE4.1). +exact_avx2_kernel! { + /// Inverse DCT of one 8×8 block. Output is level-shifted (+128) and + /// saturated to `[0, 255]`, matching the scalar path byte-for-byte on + /// legal JPEG coefficients and on the adversarial saturating edges + /// proptested against. + pub(crate) fn idct_islow(avx2: ExactAvx2, input: &[i16; 64], output: &mut [u8; 64]) { + idct_islow_kernel(input, output); + } +} + #[target_feature(enable = "avx2")] -pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { +fn idct_islow_kernel(input: &[i16; 64], output: &mut [u8; 64]) { const PASS1_SHIFT: i32 = CONST_BITS - PASS1_BITS; const PASS2_SHIFT: i32 = CONST_BITS + PASS1_BITS + 3; - let src = input.as_ptr(); - // SAFETY: `input` contains exactly 64 i16 coefficients. The offsets below - // load eight coefficients each and stay within that fixed block. + let (rows, _) = input.as_chunks::<8>(); let ( (r0l, r0h), (r1l, r1h), @@ -68,18 +70,16 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { (r5l, r5h), (r6l, r6h), (r7l, r7h), - ) = unsafe { - ( - widen(src.add(0)), - widen(src.add(8)), - widen(src.add(16)), - widen(src.add(24)), - widen(src.add(32)), - widen(src.add(40)), - widen(src.add(48)), - widen(src.add(56)), - ) - }; + ) = ( + widen(&rows[0]), + widen(&rows[1]), + widen(&rows[2]), + widen(&rows[3]), + widen(&rows[4]), + widen(&rows[5]), + widen(&rows[6]), + widen(&rows[7]), + ); let round1 = _mm_set1_epi32(1 << (PASS1_SHIFT - 1)); let cw_lo = idct_1d_x4::([r0l, r1l, r2l, r3l, r4l, r5l, r6l, r7l], round1); @@ -120,29 +120,23 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { _mm_add_epi32(rw_hi[7], bias), ); - let store = output.as_mut_ptr(); - // SAFETY: `output` contains 64 writable bytes, and each store writes one - // eight-byte row at offsets 0, 8, ..., 56. - unsafe { - store_row(store, fll0, flh0); - store_row(store.add(8), fll1, flh1); - store_row(store.add(16), fll2, flh2); - store_row(store.add(24), fll3, flh3); - store_row(store.add(32), fhl0, fhh0); - store_row(store.add(40), fhl1, fhh1); - store_row(store.add(48), fhl2, fhh2); - store_row(store.add(56), fhl3, fhh3); - } + let (rows, _) = output.as_chunks_mut::<8>(); + store_row(&mut rows[0], fll0, flh0); + store_row(&mut rows[1], fll1, flh1); + store_row(&mut rows[2], fll2, flh2); + store_row(&mut rows[3], fll3, flh3); + store_row(&mut rows[4], fhl0, fhh0); + store_row(&mut rows[5], fhl1, fhh1); + store_row(&mut rows[6], fhl2, fhh2); + store_row(&mut rows[7], fhl3, fhh3); } /// Load 8 `i16` values from `src` and sign-extend them to a pair of /// `__m128i` each carrying 4 `i32` lanes (low 4, high 4). #[inline] #[target_feature(enable = "avx2")] -unsafe fn widen(src: *const i16) -> (__m128i, __m128i) { - // SAFETY: callers pass a pointer to at least eight readable i16 values; - // unaligned loads are intentional for JPEG coefficient blocks. - let full = unsafe { core::ptr::read_unaligned(src.cast::<__m128i>()) }; +fn widen(src: &[i16; 8]) -> (__m128i, __m128i) { + let full = x86_memory::load_i16x8(src); let lo = _mm_cvtepi16_epi32(full); let hi_shuffled = _mm_srli_si128::<8>(full); let hi = _mm_cvtepi16_epi32(hi_shuffled); @@ -152,16 +146,12 @@ unsafe fn widen(src: *const i16) -> (__m128i, __m128i) { /// Saturating narrow an `(i32x4, i32x4)` pair to `u8x8` and store at `dst`. #[inline] #[target_feature(enable = "avx2")] -unsafe fn store_row(dst: *mut u8, lo: __m128i, hi: __m128i) { +fn store_row(dst: &mut [u8; 8], lo: __m128i, hi: __m128i) { // Lanes are [lo0..3, hi0..3] as i16. let i16_packed = _mm_packs_epi32(lo, hi); // The low eight lanes are the saturated output row. let u8_packed = _mm_packus_epi16(i16_packed, i16_packed); - // SAFETY: callers pass a pointer to eight writable bytes; the store writes - // only the low 64 bits and does not require alignment. - unsafe { - _mm_storel_epi64(dst.cast(), u8_packed); - } + x86_memory::store_u8x8(dst, u8_packed); } /// 1D IDCT pass over 4 i32 lanes. Mirrors `idct::neon::idct_1d_x4`. @@ -239,14 +229,14 @@ fn transpose_4x4_i32(a: __m128i, b: __m128i, c: __m128i, d: __m128i) -> [__m128i mod tests { use super::*; use crate::idct::scalar::idct_islow as idct_scalar; + use crate::simd::x86::ExactAvx2; fn run_both(input: &[i16; 64]) -> ([u8; 64], [u8; 64]) { let mut scalar_out = [0u8; 64]; idct_scalar(input, &mut scalar_out); let mut avx_out = [0u8; 64]; - if std::is_x86_feature_detected!("avx2") { - // SAFETY: the runtime guard proves the required AVX2 feature. - unsafe { idct_islow(input, &mut avx_out) }; + if let Some(avx2) = ExactAvx2::detect() { + idct_islow(avx2, input, &mut avx_out); } else { // Running the test on a non-AVX2 host: copy scalar output so // assertion passes and the test becomes a skip. diff --git a/crates/j2k-jpeg/src/idct/neon.rs b/crates/j2k-jpeg/src/idct/neon.rs index c6aacbc6..6a7e44c3 100644 --- a/crates/j2k-jpeg/src/idct/neon.rs +++ b/crates/j2k-jpeg/src/idct/neon.rs @@ -13,12 +13,13 @@ use core::arch::aarch64::{ int16x8_t, int32x4_t, vaddq_s32, vcombine_s16, vcombine_s32, vdupq_n_s32, vget_high_s16, - vget_high_s32, vget_low_s16, vget_low_s32, vgetq_lane_u64, vld1q_s16, vmovl_s16, vmulq_n_s32, - vorrq_u64, vqmovn_s32, vqmovun_s16, vreinterpretq_u64_s16, vshlq_n_s32, vshrq_n_s32, vst1_u8, - vsubq_s32, vtrnq_s32, + vget_high_s32, vget_low_s16, vget_low_s32, vgetq_lane_u64, vmovl_s16, vmulq_n_s32, vorrq_u64, + vqmovn_s32, vqmovun_s16, vreinterpretq_u64_s16, vshlq_n_s32, vshrq_n_s32, vsubq_s32, vtrnq_s32, }; use j2k_codec_math::jpeg::idct; +use crate::simd::neon_memory; + #[expect( clippy::cast_possible_truncation, clippy::cast_possible_wrap, @@ -45,42 +46,32 @@ const FIX_2_053119869: i32 = idct::FIX_2_053119869; const FIX_2_562915447: i32 = idct::FIX_2_562915447; const FIX_3_072711026: i32 = idct::FIX_3_072711026; -/// Inverse DCT of one 8×8 block. Output is level-shifted (+128) and -/// saturated to `[0, 255]`, matching the scalar path byte-for-byte. -/// -/// # Safety -/// Caller ensures the target CPU supports NEON. On aarch64 NEON is -/// architecturally mandatory, so the dispatch in `Backend::detect` picks -/// this variant unconditionally for aarch64. +fearless_simd::kernel! { + /// Inverse DCT of one 8×8 block. Output is level-shifted (+128) and + /// saturated to `[0, 255]`, matching the scalar path byte-for-byte. + pub(crate) fn idct_islow(neon: Neon, input: &[i16; 64], output: &mut [u8; 64]) { + idct_islow_kernel(input, output); + } +} + #[target_feature(enable = "neon")] -pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { +fn idct_islow_kernel(input: &[i16; 64], output: &mut [u8; 64]) { const PASS1_SHIFT: i32 = CONST_BITS - PASS1_BITS; const PASS2_SHIFT: i32 = CONST_BITS + PASS1_BITS + 3; // Load 8 rows as int16x8_t so the common bottom-half-zero shortcut can // reuse the tail rows instead of rescanning coefficients scalar-by-scalar. - let src = input.as_ptr(); - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row0 = unsafe { vld1q_s16(src) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row1 = unsafe { vld1q_s16(src.add(8)) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row2 = unsafe { vld1q_s16(src.add(16)) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row3 = unsafe { vld1q_s16(src.add(24)) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row4 = unsafe { vld1q_s16(src.add(32)) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row5 = unsafe { vld1q_s16(src.add(40)) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row6 = unsafe { vld1q_s16(src.add(48)) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let row7 = unsafe { vld1q_s16(src.add(56)) }; + let (rows, _) = input.as_chunks::<8>(); + let row0 = neon_memory::load_i16x8(&rows[0]); + let row1 = neon_memory::load_i16x8(&rows[1]); + let row2 = neon_memory::load_i16x8(&rows[2]); + let row3 = neon_memory::load_i16x8(&rows[3]); + let row4 = neon_memory::load_i16x8(&rows[4]); + let row5 = neon_memory::load_i16x8(&rows[5]); + let row6 = neon_memory::load_i16x8(&rows[6]); + let row7 = neon_memory::load_i16x8(&rows[7]); let bottom_half_zero = bottom_half_rows_are_zero(row4, row5, row6, row7); if bottom_half_zero { - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { - idct_islow_bottom_half_zero_rows(row0, row1, row2, row3, output); - } + idct_islow_bottom_half_zero_rows(row0, row1, row2, row3, output); return; } @@ -154,38 +145,42 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { // `flh_r` = row r (0..3), cols 4..7. // `fhl_r` = row r (4..7), cols 0..3. // `fhh_r` = row r (4..7), cols 4..7. - let store = output.as_mut_ptr(); - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { - store_row(store, fll0, flh0); - store_row(store.add(8), fll1, flh1); - store_row(store.add(16), fll2, flh2); - store_row(store.add(24), fll3, flh3); - store_row(store.add(32), fhl0, fhh0); - store_row(store.add(40), fhl1, fhh1); - store_row(store.add(48), fhl2, fhh2); - store_row(store.add(56), fhl3, fhh3); + let (rows, _) = output.as_chunks_mut::<8>(); + store_row(&mut rows[0], fll0, flh0); + store_row(&mut rows[1], fll1, flh1); + store_row(&mut rows[2], fll2, flh2); + store_row(&mut rows[3], fll3, flh3); + store_row(&mut rows[4], fhl0, fhh0); + store_row(&mut rows[5], fhl1, fhh1); + store_row(&mut rows[6], fhl2, fhh2); + store_row(&mut rows[7], fhl3, fhh3); +} + +fearless_simd::kernel! { + /// Inverse DCT for blocks whose natural-order rows 4..7 are known to be zero. + pub(crate) fn idct_islow_bottom_half_zero( + neon: Neon, + input: &[i16; 64], + output: &mut [u8; 64], + ) { + idct_islow_bottom_half_zero_kernel(input, output); } } -/// Inverse DCT for blocks whose natural-order rows 4..7 are known to be zero. #[target_feature(enable = "neon")] -pub(crate) unsafe fn idct_islow_bottom_half_zero(input: &[i16; 64], output: &mut [u8; 64]) { - let src = input.as_ptr(); - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { - idct_islow_bottom_half_zero_rows( - vld1q_s16(src), - vld1q_s16(src.add(8)), - vld1q_s16(src.add(16)), - vld1q_s16(src.add(24)), - output, - ); - } +fn idct_islow_bottom_half_zero_kernel(input: &[i16; 64], output: &mut [u8; 64]) { + let (rows, _) = input.as_chunks::<8>(); + idct_islow_bottom_half_zero_rows( + neon_memory::load_i16x8(&rows[0]), + neon_memory::load_i16x8(&rows[1]), + neon_memory::load_i16x8(&rows[2]), + neon_memory::load_i16x8(&rows[3]), + output, + ); } #[target_feature(enable = "neon")] -unsafe fn idct_islow_bottom_half_zero_rows( +fn idct_islow_bottom_half_zero_rows( row0: int16x8_t, row1: int16x8_t, row2: int16x8_t, @@ -238,32 +233,27 @@ unsafe fn idct_islow_bottom_half_zero_rows( vaddq_s32(rw_hi[7], bias), ); - let store = output.as_mut_ptr(); - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { - store_row(store, fll0, flh0); - store_row(store.add(8), fll1, flh1); - store_row(store.add(16), fll2, flh2); - store_row(store.add(24), fll3, flh3); - store_row(store.add(32), fhl0, fhh0); - store_row(store.add(40), fhl1, fhh1); - store_row(store.add(48), fhl2, fhh2); - store_row(store.add(56), fhl3, fhh3); - } + let (rows, _) = output.as_chunks_mut::<8>(); + store_row(&mut rows[0], fll0, flh0); + store_row(&mut rows[1], fll1, flh1); + store_row(&mut rows[2], fll2, flh2); + store_row(&mut rows[3], fll3, flh3); + store_row(&mut rows[4], fhl0, fhh0); + store_row(&mut rows[5], fhl1, fhh1); + store_row(&mut rows[6], fhl2, fhh2); + store_row(&mut rows[7], fhl3, fhh3); } -#[inline] -#[cfg(test)] -fn bottom_half_is_zero(input: &[i16; 64]) -> bool { - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - let tail = unsafe { input.as_ptr().add(32) }; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { +fearless_simd::kernel! { + #[inline] + #[cfg(test)] + fn bottom_half_is_zero(neon: Neon, input: &[i16; 64]) -> bool { + let (rows, _) = input.as_chunks::<8>(); bottom_half_rows_are_zero( - vld1q_s16(tail), - vld1q_s16(tail.add(8)), - vld1q_s16(tail.add(16)), - vld1q_s16(tail.add(24)), + neon_memory::load_i16x8(&rows[4]), + neon_memory::load_i16x8(&rows[5]), + neon_memory::load_i16x8(&rows[6]), + neon_memory::load_i16x8(&rows[7]), ) } } @@ -289,12 +279,9 @@ fn bottom_half_rows_are_zero( /// at each step produces the same u8 as the scalar's explicit clamp. #[inline] #[target_feature(enable = "neon")] -unsafe fn store_row(dst: *mut u8, lo: int32x4_t, hi: int32x4_t) { +fn store_row(dst: &mut [u8; 8], lo: int32x4_t, hi: int32x4_t) { let packed_i16: int16x8_t = vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi)); - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { - vst1_u8(dst, vqmovun_s16(packed_i16)); - } + neon_memory::store_u8x8(dst, vqmovun_s16(packed_i16)); } /// One 1D IDCT pass over 4 lanes of i32. Eight inputs carrying the 4 column @@ -420,8 +407,10 @@ mod tests { let mut scalar_out = [0u8; 64]; idct_scalar(input, &mut scalar_out); let mut neon_out = [0u8; 64]; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { idct_islow(input, &mut neon_out) }; + let neon = fearless_simd::Level::new() + .as_neon() + .expect("AArch64 test host must provide NEON"); + idct_islow(neon, input, &mut neon_out); (scalar_out, neon_out) } @@ -488,22 +477,27 @@ mod tests { let mut scalar_out = [0u8; 64]; idct_scalar(&input, &mut scalar_out); let mut neon_out = [0u8; 64]; - // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. - unsafe { idct_islow_bottom_half_zero(&input, &mut neon_out) }; + let neon = fearless_simd::Level::new() + .as_neon() + .expect("AArch64 test host must provide NEON"); + idct_islow_bottom_half_zero(neon, &input, &mut neon_out); assert_eq!(scalar_out, neon_out); } #[test] fn bottom_half_zero_detects_zero_and_nonzero_tails() { + let neon = fearless_simd::Level::new() + .as_neon() + .expect("AArch64 test host must provide NEON"); let mut block = [0i16; 64]; block[0] = 7; - assert!(bottom_half_is_zero(&block)); + assert!(bottom_half_is_zero(neon, &block)); block[32] = 1; - assert!(!bottom_half_is_zero(&block)); + assert!(!bottom_half_is_zero(neon, &block)); block[32] = 0; block[63] = -1; - assert!(!bottom_half_is_zero(&block)); + assert!(!bottom_half_is_zero(neon, &block)); } } diff --git a/crates/j2k-jpeg/src/lib.rs b/crates/j2k-jpeg/src/lib.rs index 3666c507..4c06550e 100644 --- a/crates/j2k-jpeg/src/lib.rs +++ b/crates/j2k-jpeg/src/lib.rs @@ -74,6 +74,8 @@ pub(crate) mod color; pub(crate) mod backend; +mod simd; + pub(crate) mod output; pub(crate) mod profile; diff --git a/crates/j2k-jpeg/src/simd/mod.rs b/crates/j2k-jpeg/src/simd/mod.rs new file mode 100644 index 00000000..adc93ac8 --- /dev/null +++ b/crates/j2k-jpeg/src/simd/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Private capability and memory boundaries for CPU SIMD kernels. + +#[cfg(target_arch = "aarch64")] +pub(crate) mod neon_memory; + +#[cfg(target_arch = "x86_64")] +pub(crate) mod x86; + +#[cfg(target_arch = "x86_64")] +pub(crate) mod x86_memory; diff --git a/crates/j2k-jpeg/src/simd/neon_memory.rs b/crates/j2k-jpeg/src/simd/neon_memory.rs new file mode 100644 index 00000000..10aafad9 --- /dev/null +++ b/crates/j2k-jpeg/src/simd/neon_memory.rs @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Fixed-size `AArch64` NEON memory operations. + +use core::arch::aarch64::{ + int16x8_t, uint8x16_t, uint8x8_t, uint8x8x3_t, vld1_u8, vld1q_s16, vst1_u8, vst1q_u8, vst3_u8, +}; + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) fn load_u8x8(src: &[u8; 8]) -> uint8x8_t { + // SAFETY: + // - Feature availability: callers run inside a `Neon` token kernel. + // - Bounds: the array reference proves eight readable bytes. + // - Alignment: AArch64 `vld1_u8` supports unaligned byte addresses. + // - Aliasing: the shared reference permits reads and no writes occur. + // - Initialization: all bytes behind a Rust reference are initialized. + unsafe { vld1_u8(src.as_ptr()) } +} + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) fn load_u8x8_triplet(src: &[u8; 10]) -> (uint8x8_t, uint8x8_t, uint8x8_t) { + // SAFETY: + // - Feature availability: callers run inside a `Neon` token kernel. + // - Bounds: the fixed-size array proves that eight-byte loads beginning at + // offsets zero, one, and two are all readable. + // - Alignment: AArch64 `vld1_u8` supports unaligned byte addresses. + // - Aliasing: the shared reference permits reads and no writes occur. + // - Initialization: all ten bytes behind the Rust reference are initialized. + unsafe { + let ptr = src.as_ptr(); + (vld1_u8(ptr), vld1_u8(ptr.add(1)), vld1_u8(ptr.add(2))) + } +} + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) fn load_i16x8(src: &[i16; 8]) -> int16x8_t { + // SAFETY: + // - Feature availability: callers run inside a `Neon` token kernel. + // - Bounds: the array reference proves eight readable i16 coefficients. + // - Alignment: AArch64 `vld1q_s16` supports unaligned i16 addresses. + // - Aliasing: the shared reference permits reads and no writes occur. + // - Initialization: all coefficients behind the reference are initialized. + unsafe { vld1q_s16(src.as_ptr()) } +} + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) fn store_u8x8(dst: &mut [u8; 8], values: uint8x8_t) { + // SAFETY: + // - Feature availability: callers run inside a `Neon` token kernel. + // - Bounds: the array reference proves eight writable bytes. + // - Alignment: AArch64 `vst1_u8` supports unaligned byte addresses. + // - Aliasing: the exclusive reference prevents overlapping live access. + // - Initialization: the store initializes every output byte. + unsafe { vst1_u8(dst.as_mut_ptr(), values) }; +} + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) fn store_u8x16(dst: &mut [u8; 16], values: uint8x16_t) { + // SAFETY: + // - Feature availability: callers run inside a `Neon` token kernel. + // - Bounds: the array reference proves sixteen writable bytes. + // - Alignment: AArch64 `vst1q_u8` supports unaligned byte addresses. + // - Aliasing: the exclusive reference prevents overlapping live access. + // - Initialization: the store initializes every output byte. + unsafe { vst1q_u8(dst.as_mut_ptr(), values) }; +} + +#[inline] +#[target_feature(enable = "neon")] +pub(crate) fn store_rgb8(dst: &mut [u8; 24], red: uint8x8_t, green: uint8x8_t, blue: uint8x8_t) { + // SAFETY: + // - Feature availability: callers run inside a `Neon` token kernel. + // - Bounds: the array reference proves space for eight three-byte pixels. + // - Alignment: AArch64 `vst3_u8` supports unaligned byte addresses. + // - Aliasing: the exclusive reference prevents overlapping live access. + // - Initialization: the store initializes all twenty-four output bytes. + unsafe { vst3_u8(dst.as_mut_ptr(), uint8x8x3_t(red, green, blue)) }; +} diff --git a/crates/j2k-jpeg/src/simd/x86.rs b/crates/j2k-jpeg/src/simd/x86.rs new file mode 100644 index 00000000..14e3cd12 --- /dev/null +++ b/crates/j2k-jpeg/src/simd/x86.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Exact-AVX2 capability token and safe kernel bridge. + +use j2k_core::CpuFeatures; + +/// Proof that AVX2 and the operating-system AVX register state are available. +/// +/// This deliberately represents AVX2 alone, rather than +/// `fearless_simd::Avx2`, whose v0.7 contract is the wider x86-64-v3 feature +/// set. Keeping the token private prevents callers from forging capability +/// state while preserving the decoder's existing AVX2 acceleration envelope. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ExactAvx2 { + _private: (), +} + +impl ExactAvx2 { + pub(crate) fn detect() -> Option { + CpuFeatures::detect().avx2.then_some(Self { _private: () }) + } +} + +/// Define a safe AVX2 entry point whose first argument is an unforgeable +/// [`ExactAvx2`] capability token. +macro_rules! exact_avx2_kernel { + ( + $(#[$meta:meta])* + $vis:vis fn $name:ident( + $token:ident : ExactAvx2 $(, $arg:ident : $arg_ty:ty)* $(,)? + ) $(-> $ret:ty)? { + $($body:tt)* + } + ) => { + $(#[$meta])* + #[inline(always)] + $vis fn $name( + $token: $crate::simd::x86::ExactAvx2 $(, $arg: $arg_ty)* + ) $(-> $ret)? { + #[inline] + #[target_feature(enable = "avx2")] + fn kernel( + $token: $crate::simd::x86::ExactAvx2 $(, $arg: $arg_ty)* + ) $(-> $ret)? { + let _ = $token; + $($body)* + } + + // SAFETY: + // - Feature availability: `ExactAvx2` can only be constructed by + // successful runtime AVX2 plus OS-state detection. + // - Bounds: kernel arguments retain their safe Rust slice/array + // bounds; raw memory operations live in fixed-size leaves. + // - Alignment: fixed-size leaves use explicitly unaligned-capable + // loads and stores and never strengthen reference alignment. + // - Aliasing: the generated signature preserves Rust shared and + // exclusive reference rules across the call. + // - Initialization: inputs are initialized references and outputs + // remain initialized byte arrays or slices throughout the call. + unsafe { kernel($token $(, $arg)*) } + } + }; +} + +pub(crate) use exact_avx2_kernel; diff --git a/crates/j2k-jpeg/src/simd/x86_memory.rs b/crates/j2k-jpeg/src/simd/x86_memory.rs new file mode 100644 index 00000000..18111615 --- /dev/null +++ b/crates/j2k-jpeg/src/simd/x86_memory.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! Fixed-size x86 SIMD memory operations. + +use core::arch::x86_64::{__m128i, _mm_loadl_epi64, _mm_storel_epi64}; + +use crate::simd::x86::ExactAvx2; + +const U8_LANES: usize = 8; + +pub(crate) type U8x8Triple = (__m128i, __m128i, __m128i); + +/// A capability- and lifetime-carrying cursor over three equally stepped rows. +/// +/// The constructor fixes the readable extent to the shortest row. Private +/// fields ensure that `offset` and `chunks_remaining` can only advance +/// together, which lets the hot load leaf avoid repeating slice bounds checks. +pub(crate) struct U8x8TripleCursor<'a> { + _avx2: ExactAvx2, + y: &'a [u8], + cb: &'a [u8], + cr: &'a [u8], + offset: usize, + chunks_remaining: usize, +} + +impl<'a> U8x8TripleCursor<'a> { + pub(crate) fn new(avx2: ExactAvx2, y: &'a [u8], cb: &'a [u8], cr: &'a [u8]) -> Self { + let chunks_remaining = y.len().min(cb.len()).min(cr.len()) / U8_LANES; + Self { + _avx2: avx2, + y, + cb, + cr, + offset: 0, + chunks_remaining, + } + } + + pub(crate) const fn offset(&self) -> usize { + self.offset + } + + #[inline] + #[target_feature(enable = "avx2")] + pub(crate) fn next_pair(&mut self) -> Option<(usize, U8x8Triple, U8x8Triple)> { + if self.chunks_remaining < 2 { + return None; + } + let offset = self.offset; + let first = self.load_current(); + self.advance(); + let second = self.load_current(); + self.advance(); + Some((offset, first, second)) + } + + #[inline] + #[target_feature(enable = "avx2")] + pub(crate) fn next(&mut self) -> Option<(usize, U8x8Triple)> { + if self.chunks_remaining == 0 { + return None; + } + let offset = self.offset; + let values = self.load_current(); + self.advance(); + Some((offset, values)) + } + + #[inline] + fn advance(&mut self) { + self.offset += U8_LANES; + self.chunks_remaining -= 1; + } + + #[inline] + #[target_feature(enable = "avx2")] + fn load_current(&self) -> U8x8Triple { + debug_assert!(self.chunks_remaining > 0); + // SAFETY: + // - Feature availability: construction requires an `ExactAvx2` token + // and this leaf is compiled for AVX2. + // - Bounds: the constructor derives the chunk count from the shortest + // row, and private cursor state advances the offset by exactly eight + // while a chunk remains, proving eight readable bytes in every row. + // - Alignment: `_mm_loadl_epi64` permits unaligned addresses. + // - Aliasing: all three rows are shared references and are only read; + // overlap between them is therefore harmless. + // - Initialization: every byte reachable through the input slices is + // initialized, and each intrinsic reads exactly eight such bytes. + unsafe { + ( + _mm_loadl_epi64(self.y.as_ptr().add(self.offset).cast()), + _mm_loadl_epi64(self.cb.as_ptr().add(self.offset).cast()), + _mm_loadl_epi64(self.cr.as_ptr().add(self.offset).cast()), + ) + } + } +} + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) fn load_i16x8(src: &[i16; 8]) -> __m128i { + // SAFETY: + // - Feature availability: callers run inside an exact-AVX2 kernel. + // - Bounds: the array reference proves sixteen readable bytes. + // - Alignment: `read_unaligned` deliberately accepts any i16 alignment. + // - Aliasing: the shared reference permits reads and no writes occur. + // - Initialization: all coefficients behind the reference are initialized. + unsafe { core::ptr::read_unaligned(src.as_ptr().cast::<__m128i>()) } +} + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) fn store_u8x8(dst: &mut [u8; 8], values: __m128i) { + // SAFETY: + // - Feature availability: callers run inside an exact-AVX2 kernel. + // - Bounds: the array reference proves eight writable bytes. + // - Alignment: `_mm_storel_epi64` permits an unaligned address. + // - Aliasing: the exclusive reference prevents overlapping live access. + // - Initialization: the store initializes every byte in the output array. + unsafe { _mm_storel_epi64(dst.as_mut_ptr().cast(), values) }; +} + +#[cfg(test)] +mod tests { + use core::arch::x86_64::_mm_cvtsi128_si64; + + use super::U8x8TripleCursor; + use crate::simd::x86::{exact_avx2_kernel, ExactAvx2}; + + #[test] + fn triple_cursor_loads_unaligned_rows_and_stops_at_the_shortest() { + let Some(avx2) = ExactAvx2::detect() else { + return; + }; + let y_storage = (0_u8..24).collect::>(); + let cb_storage = (40_u8..64).collect::>(); + let cr_storage = (80_u8..104).collect::>(); + let y = &y_storage[1..18]; + let cb = &cb_storage[2..18]; + let cr = &cr_storage[3..19]; + exercise_cursor(avx2, y, cb, cr); + } + + exact_avx2_kernel! { + fn exercise_cursor(avx2: ExactAvx2, y: &[u8], cb: &[u8], cr: &[u8]) { + let mut cursor = U8x8TripleCursor::new(avx2, y, cb, cr); + assert_eq!(cursor.offset(), 0); + let (offset, first, second) = cursor.next_pair().expect("two full chunks"); + assert_eq!(offset, 0); + assert_eq!(bytes(first.0), y[..8]); + assert_eq!(bytes(first.1), cb[..8]); + assert_eq!(bytes(first.2), cr[..8]); + assert_eq!(bytes(second.0), y[8..16]); + assert_eq!(bytes(second.1), cb[8..16]); + assert_eq!(bytes(second.2), cr[8..16]); + assert_eq!(cursor.offset(), 16); + assert!(cursor.next().is_none()); + } + } + + #[target_feature(enable = "avx2")] + fn bytes(values: core::arch::x86_64::__m128i) -> [u8; 8] { + _mm_cvtsi128_si64(values).to_ne_bytes() + } +} diff --git a/crates/j2k-metal-support/Cargo.toml b/crates/j2k-metal-support/Cargo.toml index 09c1e08c..ab2d0cd0 100644 --- a/crates/j2k-metal-support/Cargo.toml +++ b/crates/j2k-metal-support/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_metal_support" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } log = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-metal/Cargo.toml b/crates/j2k-metal/Cargo.toml index 900ff3cb..3e69e95d 100644 --- a/crates/j2k-metal/Cargo.toml +++ b/crates/j2k-metal/Cargo.toml @@ -24,12 +24,12 @@ name = "j2k_metal" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k = { path = "../j2k", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k = { path = "../j2k", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] diff --git a/crates/j2k-ml/Cargo.toml b/crates/j2k-ml/Cargo.toml index ce72fad2..8d6d0e8c 100644 --- a/crates/j2k-ml/Cargo.toml +++ b/crates/j2k-ml/Cargo.toml @@ -36,16 +36,16 @@ metal = [ burn-core = { workspace = true } burn-cuda = { workspace = true, optional = true } burn-wgpu = { workspace = true, optional = true } -j2k = { path = "../j2k", version = "=0.9.0" } -j2k-cuda = { path = "../j2k-cuda", version = "=0.9.0", features = ["cuda-runtime"], optional = true } -j2k-metal = { path = "../j2k-metal", version = "=0.9.0", optional = true } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.0", optional = true } +j2k = { path = "../j2k", version = "=0.9.1" } +j2k-cuda = { path = "../j2k-cuda", version = "=0.9.1", features = ["cuda-runtime"], optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.9.1", optional = true } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.1", optional = true } thiserror = { workspace = true } [dev-dependencies] criterion = { workspace = true } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [target.'cfg(all(target_arch = "aarch64", target_os = "linux"))'.dev-dependencies] diff --git a/crates/j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml index c555ccb1..16844876 100644 --- a/crates/j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -28,9 +28,9 @@ fearless_simd = { workspace = true, optional = true } libm = { workspace = true } log = { workspace = true, optional = true } rayon = { workspace = true, optional = true } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0", default-features = false } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-types = { path = "../j2k-types", version = "=0.9.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1", default-features = false } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-types = { path = "../j2k-types", version = "=0.9.1" } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-t803/Cargo.toml b/crates/j2k-t803/Cargo.toml index 3620273f..f20e18bc 100644 --- a/crates/j2k-t803/Cargo.toml +++ b/crates/j2k-t803/Cargo.toml @@ -29,14 +29,14 @@ metal-runner = ["runner", "dep:j2k-metal", "dep:objc2-metal"] [dependencies] image = { workspace = true, optional = true } -j2k = { path = "../j2k", version = "=0.9.0", optional = true } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0", optional = true } +j2k = { path = "../j2k", version = "=0.9.1", optional = true } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1", optional = true } j2k-compare = { path = "../j2k-compare", version = "=0.2.0", optional = true } -j2k-core = { path = "../j2k-core", version = "=0.9.0", optional = true } -j2k-native = { path = "../j2k-native", version = "=0.9.0", optional = true } -j2k-cuda = { path = "../j2k-cuda", version = "=0.9.0", optional = true } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.0", optional = true } -j2k-metal = { path = "../j2k-metal", version = "=0.9.0", optional = true } +j2k-core = { path = "../j2k-core", version = "=0.9.1", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.9.1", optional = true } +j2k-cuda = { path = "../j2k-cuda", version = "=0.9.1", optional = true } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.1", optional = true } +j2k-metal = { path = "../j2k-metal", version = "=0.9.1", optional = true } serde = { workspace = true } serde_json = { workspace = true, features = ["float_roundtrip"] } sha2 = { workspace = true } diff --git a/crates/j2k-t803/fuzz/Cargo.lock b/crates/j2k-t803/fuzz/Cargo.lock index 15b57a85..aa182a41 100644 --- a/crates/j2k-t803/fuzz/Cargo.lock +++ b/crates/j2k-t803/fuzz/Cargo.lock @@ -163,9 +163,9 @@ dependencies = [ [[package]] name = "fearless_simd" -version = "0.4.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" [[package]] name = "find-msvc-tools" @@ -256,7 +256,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.8.1" +version = "0.9.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -268,7 +268,7 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-compare" @@ -285,14 +285,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.8.1" +version = "0.9.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-native" -version = "0.8.1" +version = "0.9.1" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -304,11 +304,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-t803" -version = "0.8.1" +version = "0.9.1" dependencies = [ "image", "j2k", @@ -334,7 +334,7 @@ dependencies = [ [[package]] name = "j2k-test-support" -version = "0.8.1" +version = "0.9.1" dependencies = [ "j2k-core", "serde", @@ -344,7 +344,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.8.1" +version = "0.9.1" [[package]] name = "jobserver" diff --git a/crates/j2k-test-support/Cargo.toml b/crates/j2k-test-support/Cargo.toml index 843cb68c..dcc18f0c 100644 --- a/crates/j2k-test-support/Cargo.toml +++ b/crates/j2k-test-support/Cargo.toml @@ -16,8 +16,8 @@ path = "src/lib.rs" j2k-native-fixtures = ["dep:j2k-native"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0", optional = true } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1", optional = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } diff --git a/crates/j2k-tilecodec/Cargo.toml b/crates/j2k-tilecodec/Cargo.toml index 9d2296f3..d47b2a45 100644 --- a/crates/j2k-tilecodec/Cargo.toml +++ b/crates/j2k-tilecodec/Cargo.toml @@ -19,7 +19,7 @@ name = "j2k_tilecodec" path = "src/lib.rs" [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } thiserror = { workspace = true } flate2 = { workspace = true } zstd = { workspace = true } diff --git a/crates/j2k-tilecodec/fuzz/Cargo.lock b/crates/j2k-tilecodec/fuzz/Cargo.lock index 2a03faa7..43e5cbd9 100644 --- a/crates/j2k-tilecodec/fuzz/Cargo.lock +++ b/crates/j2k-tilecodec/fuzz/Cargo.lock @@ -72,14 +72,14 @@ dependencies = [ [[package]] name = "j2k-core" -version = "0.8.1" +version = "0.9.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-tilecodec" -version = "0.8.1" +version = "0.9.1" dependencies = [ "flate2", "j2k-core", diff --git a/crates/j2k-transcode-cuda/Cargo.toml b/crates/j2k-transcode-cuda/Cargo.toml index b463a1a5..5065feea 100644 --- a/crates/j2k-transcode-cuda/Cargo.toml +++ b/crates/j2k-transcode-cuda/Cargo.toml @@ -29,10 +29,10 @@ cuda-runtime = [ cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] [dependencies] -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.9.0" } -j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.0", optional = true } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.9.1" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.9.1", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } [dev-dependencies] j2k-test-support = { path = "../j2k-test-support" } diff --git a/crates/j2k-transcode-metal/Cargo.toml b/crates/j2k-transcode-metal/Cargo.toml index bad88eb8..3d128bce 100644 --- a/crates/j2k-transcode-metal/Cargo.toml +++ b/crates/j2k-transcode-metal/Cargo.toml @@ -26,13 +26,13 @@ default = [] bench-internals = ["j2k-transcode/dev-support"] [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.0" } -j2k-transcode = { path = "../j2k-transcode", version = "=0.9.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.9.1" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.9.1" } [target.'cfg(target_os = "macos")'.dependencies] -j2k-metal = { path = "../j2k-metal", version = "=0.9.0" } +j2k-metal = { path = "../j2k-metal", version = "=0.9.1" } objc2 = { workspace = true } objc2-foundation = { workspace = true } objc2-metal = { workspace = true } @@ -40,9 +40,9 @@ objc2-metal = { workspace = true } [dev-dependencies] criterion = { workspace = true } rayon = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1" } j2k-test-support = { path = "../j2k-test-support" } j2k-transcode-test-support = { path = "../j2k-transcode-test-support" } diff --git a/crates/j2k-transcode-test-support/Cargo.toml b/crates/j2k-transcode-test-support/Cargo.toml index 9079cc03..bf5bef4b 100644 --- a/crates/j2k-transcode-test-support/Cargo.toml +++ b/crates/j2k-transcode-test-support/Cargo.toml @@ -13,11 +13,11 @@ name = "j2k_transcode_test_support" path = "src/lib.rs" [dependencies] -j2k-transcode = { path = "../j2k-transcode", version = "=0.9.0", features = ["dev-support"] } -j2k-types = { path = "../j2k-types", version = "=0.9.0" } +j2k-transcode = { path = "../j2k-transcode", version = "=0.9.1", features = ["dev-support"] } +j2k-types = { path = "../j2k-types", version = "=0.9.1" } [dev-dependencies] -j2k-native = { path = "../j2k-native", version = "=0.9.0" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } [lints.rust] unsafe_code = "forbid" diff --git a/crates/j2k-transcode/Cargo.toml b/crates/j2k-transcode/Cargo.toml index 71590568..6c2303f3 100644 --- a/crates/j2k-transcode/Cargo.toml +++ b/crates/j2k-transcode/Cargo.toml @@ -22,12 +22,12 @@ dev-support = [] [dependencies] rayon = { workspace = true } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.0" } -j2k = { path = "../j2k", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } -j2k-profile = { path = "../j2k-profile", version = "=0.9.0", default-features = false } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.9.1" } +j2k = { path = "../j2k", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } +j2k-profile = { path = "../j2k-profile", version = "=0.9.1", default-features = false } [dev-dependencies] criterion = { workspace = true } diff --git a/crates/j2k-transcode/fuzz/Cargo.lock b/crates/j2k-transcode/fuzz/Cargo.lock index ea1b7d14..3cf25ba2 100644 --- a/crates/j2k-transcode/fuzz/Cargo.lock +++ b/crates/j2k-transcode/fuzz/Cargo.lock @@ -65,9 +65,9 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "fearless_simd" -version = "0.4.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" [[package]] name = "find-msvc-tools" @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "j2k" -version = "0.8.1" +version = "0.9.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -101,19 +101,20 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-core" -version = "0.8.1" +version = "0.9.1" dependencies = [ "thiserror", ] [[package]] name = "j2k-jpeg" -version = "0.8.1" +version = "0.9.1" dependencies = [ + "fearless_simd", "j2k-codec-math", "j2k-core", "j2k-profile", @@ -124,7 +125,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.8.1" +version = "0.9.1" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -136,11 +137,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-transcode" -version = "0.8.1" +version = "0.9.1" dependencies = [ "j2k", "j2k-codec-math", @@ -162,7 +163,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.8.1" +version = "0.9.1" [[package]] name = "jobserver" diff --git a/crates/j2k/Cargo.toml b/crates/j2k/Cargo.toml index 107f8e84..555b0a6f 100644 --- a/crates/j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -20,17 +20,17 @@ name = "j2k" path = "src/lib.rs" [dependencies] -j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.0" } -j2k-core = { path = "../j2k-core", version = "=0.9.0" } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } -j2k-types = { path = "../j2k-types", version = "=0.9.0" } +j2k-codec-math = { path = "../j2k-codec-math", version = "=0.9.1" } +j2k-core = { path = "../j2k-core", version = "=0.9.1" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } +j2k-types = { path = "../j2k-types", version = "=0.9.1" } moxcms = { workspace = true } thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } -j2k-native = { path = "../j2k-native", version = "=0.9.0" } +j2k-native = { path = "../j2k-native", version = "=0.9.1" } j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } [[bench]] diff --git a/crates/j2k/fuzz/Cargo.lock b/crates/j2k/fuzz/Cargo.lock index 18aa4bf1..a61362b5 100644 --- a/crates/j2k/fuzz/Cargo.lock +++ b/crates/j2k/fuzz/Cargo.lock @@ -103,9 +103,9 @@ checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "fearless_simd" -version = "0.4.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" +checksum = "f4beca3cb2444e3304ac30843cc091f44ed58932353cd492ce740067bfce6b12" [[package]] name = "find-msvc-tools" @@ -143,7 +143,7 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "j2k" -version = "0.8.1" +version = "0.9.1" dependencies = [ "j2k-codec-math", "j2k-core", @@ -155,11 +155,11 @@ dependencies = [ [[package]] name = "j2k-codec-math" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-core" -version = "0.8.1" +version = "0.9.1" dependencies = [ "thiserror", ] @@ -175,7 +175,7 @@ dependencies = [ [[package]] name = "j2k-native" -version = "0.8.1" +version = "0.9.1" dependencies = [ "fearless_simd", "j2k-codec-math", @@ -187,11 +187,11 @@ dependencies = [ [[package]] name = "j2k-profile" -version = "0.8.1" +version = "0.9.1" [[package]] name = "j2k-test-support" -version = "0.8.1" +version = "0.9.1" dependencies = [ "j2k-core", "serde", @@ -201,7 +201,7 @@ dependencies = [ [[package]] name = "j2k-types" -version = "0.8.1" +version = "0.9.1" [[package]] name = "jobserver" diff --git a/docs/architecture.md b/docs/architecture.md index d0c3e670..b4d3e05b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -92,6 +92,43 @@ alpha are distinct grouping keys. Preparation retains the caller-owned codestream bytes and reusable decode plans without duplicating the codestream. Broader component layouts remain on the component-plane APIs. +### CPU JPEG SIMD boundary + +`j2k-jpeg` selects its CPU backend once while constructing a decoder. The +internal backend value carries the capability needed to execute accelerated +code: `Scalar`, `Avx2(ExactAvx2)`, or `Neon(fearless_simd::Neon)`. A diagnostic +backend kind is not executable authority, and tests requesting a specialization +must obtain the same runtime token as production. The `scalar-only` feature +always selects `Scalar`. + +AArch64 entry kernels use the safe `fearless_simd 0.7` kernel boundary. x86-64 +uses a project-private equivalent that enables exactly AVX2. This distinction +is intentional: the `fearless_simd::Avx2` token in 0.7 represents the broader +x86-64-v3 feature set, including FMA, BMI, and other features. Requiring that +token would silently remove acceleration from CPUs that satisfy the decoder's +existing AVX2-plus-operating-system-state contract but not all of v3. + +Dispatch, benchmark adapters, and arithmetic helpers are safe Rust. Raw vector +memory operations are confined to private fixed-size array leaves and one x86 +row cursor carrying the AVX2 capability and source-slice lifetimes. The cursor +constructor fixes its readable extent to the shortest complete eight-byte +chunk count, and private state advances all three rows together. These leaves +use unaligned-capable operations and preserve Rust's reference aliasing and +initialization rules. The optimized IDCT and color paths retain their existing +integer arithmetic, chunk sizes, edge repair, crop rules, and scalar tails; +this refactor does not substitute a new portable-SIMD algorithm. + +The unsafe-audit task parses every Rust source under the JPEG backend, IDCT, +and SIMD directories. It rejects `unsafe fn`, rejects unsafe outside the +private feature/memory modules, requires a five-part safety proof for each +remaining block, and caps the explicit production SIMD boundary at 24 blocks. +The refactored boundary currently contains 10 blocks and no `unsafe fn`. +SIMD output remains differentially tested against scalar output. Performance +acceptance uses same-host Criterion comparisons at 95% confidence, 50 samples, +a three-second warm-up, and a ten-second measurement; a confidence-bound +slowdown above 2% for a microbenchmark or 1% for end-to-end decode is repeated +with twice the measurement time before accepting a narrow unsafe memory leaf. + Device adapters can add resident outputs and validated caller-owned destinations, but explicit requests must return unsupported errors instead of falling back to CPU staging. A direct external destination is the final output diff --git a/docs/benchmark-evidence.md b/docs/benchmark-evidence.md index 42283173..2cbd8804 100644 --- a/docs/benchmark-evidence.md +++ b/docs/benchmark-evidence.md @@ -28,6 +28,114 @@ external bundle and identify any missing evidence. Generated repo-local fixtures and passing codec self-checks remain implementation evidence; use manifest-backed external rows for adoption-facing speed reports. +## CPU JPEG safe-SIMD development run - 2026-08-12 + +The CPU JPEG safe-SIMD refactor was measured before and after on the same Apple +M4 Pro host running macOS 26.5.2 build `25F84`, with Rust/Cargo 1.96.0 and LLVM +22.1.2. Criterion used 95% confidence, 50 samples, a three-second warm-up, and +a ten-second measurement. Benchmark JPEGs were generated deterministically +before timing; setup decoded every case to validate geometry and output +checksums. The commands were: + +```bash +cargo bench -p j2k-jpeg --bench decode_cpu -- --save-baseline safe-simd-before +cargo bench -p j2k-jpeg --features bench-internals --bench micro -- --save-baseline safe-simd-before +cargo bench -p j2k-jpeg --bench decode_cpu -- --baseline safe-simd-before +cargo bench -p j2k-jpeg --features bench-internals --bench micro -- --baseline safe-simd-before +``` + +The final end-to-end decode comparison was: + +| Case | Before 95% interval | After 95% interval | Criterion change interval | +| --- | ---: | ---: | ---: | +| Gray8 512 x 512 | 1.2084–1.2649 ms | 1.1740–1.1822 ms | -12.903% to -5.046% | +| RGB8 512 x 512 4:4:4 | 2.7072–2.8042 ms | 2.6190–2.6301 ms | -5.201% to -2.813% | +| RGB8 512 x 512 4:2:2 | 2.4961–2.5115 ms | 2.4447–2.4634 ms | -2.473% to -1.302% | +| RGB8 512 x 512 4:2:0 | 1.9161–1.9243 ms | 1.8769–1.8839 ms | -2.651% to -1.946% | +| RGB8 257 x 263 4:2:0 | 508.92–510.58 us | 500.94–502.03 us | -1.652% to -1.149% | +| RGB8 512 x 512 4:2:0 rows | 1.9360–1.9420 ms | 1.8981–1.9022 ms | -2.403% to -1.966% | + +No affected microbenchmark exceeded the 2% regression criterion; every final +95% confidence interval was lower than the saved baseline. This includes general and +bottom-half-zero NEON IDCT, 255/256/cropped 4:2:0 row pairs, gray and planar-RGB +rows, and 255/256/unaligned YCbCr rows. An intermediate build did confirm +1.7–3.3% regressions in the color decode cases at doubled measurement time. +Generated-code inspection traced those to per-chunk bounds checks and a lost +4:2:0 helper inline. Safe array chunking plus a fixed ten-byte overlapping-load +leaf restored the code shape; safe dispatch was not reverted. + +This is dirty-tree development evidence, not an exact-release-SHA publication +run. The uniform gains in some unrelated microbenchmarks also make the exact +improvement magnitudes susceptible to host-state and code-layout bias; the +supported conclusion is that the final AArch64 run cleared the stated +regression thresholds. Native x86-64 evidence is recorded below. + +### Native Windows AVX2 validation + +The same refactor was measured natively with the MSVC target on Windows 11 Pro +10.0.22631, an AMD Ryzen 7 5800X3D with AVX2, and Rust/Cargo 1.96.0. The Linux +VM on that host was used only as the remote transport; every test and timed +process was a native `x86_64-pc-windows-msvc` executable. Baseline and +candidate used the same benchmark sources, `fearless_simd 0.5.0`, shared build +settings, and deterministic inputs. The final paired run pinned each process +to the same logical processor and used High process priority. Criterion used +the same 95% confidence, 50 samples, three-second warm-up, and ten-second +measurement settings. The sole inconclusive 4:4:4 case was repeated with a +20-second measurement as required. + +| Case | Before 95% interval | After 95% interval | Criterion change interval | +| --- | ---: | ---: | ---: | +| Gray8 512 x 512 | 1.8990–1.9206 ms | 1.8712–1.8975 ms | -2.192% to -0.449% | +| RGB8 512 x 512 4:4:4 (20 s repeat) | 4.7098–4.7265 ms | 4.7236–4.7483 ms | +0.003% to +0.495% | +| RGB8 512 x 512 4:2:2 | 4.2607–4.2973 ms | 4.1895–4.2214 ms | -2.295% to -1.171% | +| RGB8 512 x 512 4:2:0 | 4.8575–4.8962 ms | 4.7957–4.8259 ms | -1.825% to -0.862% | +| RGB8 257 x 263 4:2:0 | 1.2906–1.3418 ms | 1.2740–1.2827 ms | -4.832% to -0.826% | +| RGB8 512 x 512 4:2:0 rows | 4.8291–4.8550 ms | 4.7672–4.8029 ms | -1.646% to -0.717% | + +Every affected AVX2 microbenchmark also cleared the 2% upper-bound criterion. +The upper confidence bounds were +0.329% for general AVX2 IDCT, -0.829% for +4:2:0 upsampling, -0.445% to -1.065% for full row pairs, +1.000% for cropped +4:2:0, +0.662% for gray rows, +1.366% for planar RGB rows, and +0.176% or +better for full, tail, and unaligned YCbCr rows. + +An additional unchanged reduced 2x2 scalar-IDCT diagnostic measured a +confirmed +5.119% to +5.971% shift at an absolute candidate time of +11.780–11.859 ns. Baseline and candidate source and generated x86-64 function +bodies are identical (235 instructions), so this is attributed to whole-binary +placement/cache effects rather than more algorithmic work. It is outside the +CPU SIMD change and its predeclared affected-microbenchmark criterion; the +end-to-end decode matrix above contains no confirmed regression. As with the +AArch64 measurements, these are dirty-tree development results rather than an +exact release-SHA publication run. + +### `fearless_simd` 0.7 upgrade validation - 2026-08-13 + +The staged 0.9.1 dependency graph resolves `fearless_simd 0.7.0` in the +workspace and every affected fuzz lockfile. `cargo xtask release-cpu` passed +with that version on the Apple M4 Pro AArch64 host, the Linux x86-64 VM, and +the native Windows x86-64 MSVC host described above. + +The dependency is AArch64-only for `j2k-jpeg`. In an adjacent +version-isolation control from one otherwise unchanged intermediate source +tree on Windows, changing only the workspace requirement and lockfile between +0.5.0 and 0.7.0 caused Cargo to reuse the exact benchmark executable (SHA-256 +`2e37a5ffa851964a5e3ca4fafb6f819372071ba8f033e94bb83dda297d51ea80`). +An adjacent 20-second 4:4:4 control nevertheless moved by -1.916% to -1.069%, +demonstrating that the earlier apparent x86 dependency-version regressions +were run-to-run host variation rather than changed executable code. + +On AArch64, normalized inspection of the unstripped release-benchmark output +found identical instruction bodies for the compared JPEG NEON IDCT, row +conversion, and 4:2:0 kernels under 0.5.0 and 0.7.0; relocation targets, +constant-pool offsets, and whole-binary placement were excluded from that +comparison. The host was not idle enough for an acceptance-quality direct +0.7.0 timing rerun: repeated Criterion attempts contained severe scheduling +outliers while WindowServer and other interactive processes remained busy. +The 0.5.0 safe-SIMD timing above therefore remains historical refactor +evidence, and a quiet same-host 0.7.0 Criterion rerun is still required before +claiming direct measured AArch64 no-regression evidence for the dependency +upgrade. + ## Fixed Auto-routing promotion evidence `BackendRequest::Auto` uses committed thresholds; it does not calibrate on a diff --git a/docs/release-evidence/public-api/public-api-review-0.9.1.yml b/docs/release-evidence/public-api/public-api-review-0.9.1.yml new file mode 100644 index 00000000..28206f45 --- /dev/null +++ b/docs/release-evidence/public-api/public-api-review-0.9.1.yml @@ -0,0 +1,130 @@ +version: 3 +baseline_tag: v0.9.0 +baseline_version: 0.9.0 +candidate_version: "0.9.1" +break_ledger: [] +reviews: + j2k-core: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 189 + hidden_fingerprint: "fnv1a64:b87aa26da547e491" + rationale: "Reviewed the complete ordinary j2k-core surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-core inventory; its decode contracts, planning types, validation boundaries, and typed failures are unchanged from v0.9.0." + j2k-profile: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 117 + hidden_fingerprint: "fnv1a64:a1546f56ae38f2e3" + rationale: "Reviewed the complete ordinary j2k-profile surface against v0.9.0; its documented surface remains the crate module root only." + hidden_rationale: "Reviewed the complete hidden profiling inventory; capture ownership, controls, and callable profiling surfaces are unchanged from v0.9.0." + j2k-types: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 57 + hidden_fingerprint: "fnv1a64:338f28d060e145ae" + rationale: "Reviewed the complete ordinary j2k-types surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-types inventory; encode context, accelerator hooks, and CPU-only implementations are unchanged from v0.9.0." + j2k-codec-math: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Reviewed the complete ordinary and hidden j2k-codec-math inventories against v0.9.0; no public API item was added, removed, or changed." + j2k-cuda-runtime: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 1093 + hidden_fingerprint: "fnv1a64:0c13ab1f1405e36a" + rationale: "Reviewed the complete ordinary j2k-cuda-runtime surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden CUDA runtime inventory; allocation, launch, completion, and codec-state boundaries are unchanged from v0.9.0." + j2k-metal-support: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 11 + hidden_fingerprint: "fnv1a64:02139606f1202938" + rationale: "Reviewed the complete ordinary j2k-metal-support surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden Metal support inventory; retained ownership, checked allocation, command submission, and audited unsafe boundaries are unchanged from v0.9.0." + j2k-native: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 683 + hidden_fingerprint: "fnv1a64:f6f62471065f9f46" + rationale: "Reviewed the complete ordinary j2k-native surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden native-codec inventory; decode planning, transforms, color handling, SIMD, and typed error boundaries are unchanged from v0.9.0." + j2k-jpeg: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 574 + hidden_fingerprint: "fnv1a64:b3e5271cb5d45bac" + rationale: "Reviewed the complete ordinary j2k-jpeg surface against v0.9.0; the safe SIMD refactor adds, removes, and changes no public API item." + hidden_rationale: "Reviewed the complete hidden JPEG inventory. The added benchmark-only gray, RGB, cropped 4:2:0, unaligned-row, and NEON IDCT surfaces validate the private safe SIMD kernels; codec entry points and downstream implementation contracts remain compatible with v0.9.0." + j2k-tilecodec: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 32 + hidden_fingerprint: "fnv1a64:eb677842559d9b0e" + rationale: "Reviewed the complete ordinary j2k-tilecodec surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden tile-codec inventory; codec, scratch, and source-preserving error boundaries are unchanged from v0.9.0." + j2k: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 105 + hidden_fingerprint: "fnv1a64:439bc4b61c1ec5b3" + rationale: "Reviewed the complete ordinary j2k facade surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k facade inventory; retained plans, validation, accelerator context, and typed error boundaries are unchanged from v0.9.0." + j2k-transcode: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 458 + hidden_fingerprint: "fnv1a64:7932adfbeacc95f0" + rationale: "Reviewed the complete ordinary j2k-transcode surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden transcode inventory; transforms, packetization, resident handoff, accounting, and source-preserving errors are unchanged from v0.9.0." + j2k-transcode-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 18 + hidden_fingerprint: "fnv1a64:6f5f3ec742bd8f61" + rationale: "Reviewed the complete ordinary j2k-transcode-cuda surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden CUDA transcode inventory; routing, resident handoff, allocation caps, and source-preserving failures are unchanged from v0.9.0." + j2k-jpeg-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 83 + hidden_fingerprint: "fnv1a64:47b67dd1a1574b00" + rationale: "Reviewed the complete ordinary j2k-jpeg-metal surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG Metal inventory; resident-image ownership, session lifetime, and typed Metal failures are unchanged from v0.9.0." + j2k-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 315 + hidden_fingerprint: "fnv1a64:814d79b9d14edc7c" + rationale: "Reviewed the complete ordinary j2k-metal surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden j2k-metal inventory; encode routing, host output, retained resources, and adapter behavior are unchanged from v0.9.0." + j2k-transcode-metal: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 49 + hidden_fingerprint: "fnv1a64:5bf8a22a33753a2d" + rationale: "Reviewed the complete ordinary j2k-transcode-metal surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden Metal transcode inventory; transforms, buffers, resident handoff, fallback, and typed source chains are unchanged from v0.9.0." + j2k-jpeg-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 124 + hidden_fingerprint: "fnv1a64:0b31ccb3e7974bc4" + rationale: "Reviewed the complete ordinary j2k-jpeg-cuda surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden JPEG CUDA inventory; session-bound execution, allocation accounting, and typed runtime failures are unchanged from v0.9.0." + j2k-cuda: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 186 + hidden_fingerprint: "fnv1a64:31552954b6e69931" + rationale: "Reviewed the complete ordinary j2k-cuda surface against v0.9.0; no public API item was added, removed, or changed." + hidden_rationale: "Reviewed the complete hidden J2K CUDA inventory; completion state, session usability, device identity, ordering, pooling, and guarded interop are unchanged from v0.9.0." + j2k-ml: + removed_fingerprint: "none" + added_fingerprint: "none" + hidden_count: 0 + hidden_fingerprint: "none" + rationale: "Reviewed the complete ordinary and hidden j2k-ml inventories against v0.9.0; no public API item was added, removed, or changed." diff --git a/docs/release-evidence/public-api/reviewed-public-api-diff-0.9.1.md b/docs/release-evidence/public-api/reviewed-public-api-diff-0.9.1.md new file mode 100644 index 00000000..a0f2f288 --- /dev/null +++ b/docs/release-evidence/public-api/reviewed-public-api-diff-0.9.1.md @@ -0,0 +1,249 @@ +# Reviewed public API diff for j2k 0.9.1 + +This report is generated by `cargo xtask semver --write-report`. Normal `cargo xtask semver` regenerates it in memory and fails if this committed file is stale. Every ordinary added/removed fingerprint and every full rustdoc-hidden candidate-inventory fingerprint requires an exact reviewed entry in `docs/release-evidence/public-api/public-api-review-0.9.1.yml`; report regeneration never updates that review config. + +- Baseline registry version: `0.9.0` +- Baseline source snapshot: `v0.9.0` peeled to `b197f01ab4b9271f1cbc36921755a5b9d588bd5a` +- Candidate version: `0.9.1` +- Tool pins: Rust `1.96`, `cargo-semver-checks 0.48.0`, `cargo-public-api 0.52.0`, rustdoc `nightly-2026-06-28`, target `aarch64-apple-darwin` + +## Summary + +| Package | Baseline | Candidate | Computed release type | Added | Removed/changed | Removed fingerprint | Added fingerprint | Rustdoc-hidden items | Hidden inventory fingerprint | +| --- | --- | --- | --- | ---: | ---: | --- | --- | ---: | --- | +| `j2k-core` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 189 | `fnv1a64:b87aa26da547e491` | +| `j2k-profile` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 117 | `fnv1a64:a1546f56ae38f2e3` | +| `j2k-types` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 57 | `fnv1a64:338f28d060e145ae` | +| `j2k-codec-math` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | +| `j2k-cuda-runtime` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 1093 | `fnv1a64:0c13ab1f1405e36a` | +| `j2k-metal-support` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 11 | `fnv1a64:02139606f1202938` | +| `j2k-native` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 683 | `fnv1a64:f6f62471065f9f46` | +| `j2k-jpeg` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 574 | `fnv1a64:b3e5271cb5d45bac` | +| `j2k-tilecodec` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 32 | `fnv1a64:eb677842559d9b0e` | +| `j2k` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 105 | `fnv1a64:439bc4b61c1ec5b3` | +| `j2k-transcode` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 458 | `fnv1a64:7932adfbeacc95f0` | +| `j2k-transcode-cuda` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 18 | `fnv1a64:6f5f3ec742bd8f61` | +| `j2k-jpeg-metal` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 83 | `fnv1a64:47b67dd1a1574b00` | +| `j2k-metal` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 315 | `fnv1a64:814d79b9d14edc7c` | +| `j2k-transcode-metal` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 49 | `fnv1a64:5bf8a22a33753a2d` | +| `j2k-jpeg-cuda` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 124 | `fnv1a64:0b31ccb3e7974bc4` | +| `j2k-cuda` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 186 | `fnv1a64:31552954b6e69931` | +| `j2k-ml` | `0.9.0` | `0.9.1` | `minor` | 0 | 0 | `none` | `none` | 0 | `none` | + +## Published-package details + +### `j2k-core` + +Baseline items: 487. Candidate items: 487. Computed release type: `minor`. Rustdoc-hidden candidate items: 189. Full hidden-inventory fingerprint: `fnv1a64:b87aa26da547e491`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-profile` + +Baseline items: 1. Candidate items: 1. Computed release type: `minor`. Rustdoc-hidden candidate items: 117. Full hidden-inventory fingerprint: `fnv1a64:a1546f56ae38f2e3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-types` + +Baseline items: 374. Candidate items: 374. Computed release type: `minor`. Rustdoc-hidden candidate items: 57. Full hidden-inventory fingerprint: `fnv1a64:338f28d060e145ae`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-codec-math` + +Baseline items: 101. Candidate items: 101. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda-runtime` + +Baseline items: 100. Candidate items: 100. Computed release type: `minor`. Rustdoc-hidden candidate items: 1093. Full hidden-inventory fingerprint: `fnv1a64:0c13ab1f1405e36a`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal-support` + +Baseline items: 177. Candidate items: 177. Computed release type: `minor`. Rustdoc-hidden candidate items: 11. Full hidden-inventory fingerprint: `fnv1a64:02139606f1202938`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-native` + +Baseline items: 406. Candidate items: 406. Computed release type: `minor`. Rustdoc-hidden candidate items: 683. Full hidden-inventory fingerprint: `fnv1a64:f6f62471065f9f46`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg` + +Baseline items: 586. Candidate items: 586. Computed release type: `minor`. Rustdoc-hidden candidate items: 574. Full hidden-inventory fingerprint: `fnv1a64:b3e5271cb5d45bac`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-tilecodec` + +Baseline items: 31. Candidate items: 31. Computed release type: `minor`. Rustdoc-hidden candidate items: 32. Full hidden-inventory fingerprint: `fnv1a64:eb677842559d9b0e`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k` + +Baseline items: 780. Candidate items: 780. Computed release type: `minor`. Rustdoc-hidden candidate items: 105. Full hidden-inventory fingerprint: `fnv1a64:439bc4b61c1ec5b3`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode` + +Baseline items: 427. Candidate items: 427. Computed release type: `minor`. Rustdoc-hidden candidate items: 458. Full hidden-inventory fingerprint: `fnv1a64:7932adfbeacc95f0`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-cuda` + +Baseline items: 49. Candidate items: 49. Computed release type: `minor`. Rustdoc-hidden candidate items: 18. Full hidden-inventory fingerprint: `fnv1a64:6f5f3ec742bd8f61`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-metal` + +Baseline items: 176. Candidate items: 176. Computed release type: `minor`. Rustdoc-hidden candidate items: 83. Full hidden-inventory fingerprint: `fnv1a64:47b67dd1a1574b00`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-metal` + +Baseline items: 272. Candidate items: 272. Computed release type: `minor`. Rustdoc-hidden candidate items: 315. Full hidden-inventory fingerprint: `fnv1a64:814d79b9d14edc7c`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-transcode-metal` + +Baseline items: 99. Candidate items: 99. Computed release type: `minor`. Rustdoc-hidden candidate items: 49. Full hidden-inventory fingerprint: `fnv1a64:5bf8a22a33753a2d`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-jpeg-cuda` + +Baseline items: 71. Candidate items: 71. Computed release type: `minor`. Rustdoc-hidden candidate items: 124. Full hidden-inventory fingerprint: `fnv1a64:0b31ccb3e7974bc4`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-cuda` + +Baseline items: 247. Candidate items: 247. Computed release type: `minor`. Rustdoc-hidden candidate items: 186. Full hidden-inventory fingerprint: `fnv1a64:31552954b6e69931`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. + +### `j2k-ml` + +Baseline items: 101. Candidate items: 101. Computed release type: `minor`. Rustdoc-hidden candidate items: 0. Full hidden-inventory fingerprint: `none`. + +#### Removed or changed baseline API items + +None. + +#### Added candidate API items + +None. diff --git a/docs/release.md b/docs/release.md index 15e455a8..7538e871 100644 --- a/docs/release.md +++ b/docs/release.md @@ -85,8 +85,14 @@ buffer, texture, descriptor, size, and pixel-format types become retained or borrowed `objc2-metal` objects and values. Callers construct texture descriptors directly; the obsolete helper and unreachable raw-message-send errors are removed. The break ledger enumerates every removed item in the four affected -Metal crates. The one-time transition was consumed by `0.9.0`; the semver -baseline must rotate to tagged `v0.9.0` before another candidate is accepted. +Metal crates. The one-time transition was consumed by `0.9.0`. + +The staged `0.9.1` patch candidate compares directly with published `v0.9.0` +at peeled commit `b197f01ab4b9271f1cbc36921755a5b9d588bd5a`. Its provisional +[reviewed API report](release-evidence/public-api/reviewed-public-api-diff-0.9.1.md) +and [review configuration](release-evidence/public-api/public-api-review-0.9.1.yml) +must remain additive or compatible: the one-time `0.9.0` break transition is +disabled. Version `0.7.3` retained the API contract introduced by `0.7.1`, which intentionally contracted parts of the published pre-1.0 `0.6.2` API. It does diff --git a/docs/stable-api-1.0.implementation-public-api.txt b/docs/stable-api-1.0.implementation-public-api.txt index 021b12d2..b254ca10 100644 --- a/docs/stable-api-1.0.implementation-public-api.txt +++ b/docs/stable-api-1.0.implementation-public-api.txt @@ -337,6 +337,7 @@ impl core::convert::From for j2k_jpeg::a impl core::convert::From for j2k_jpeg::adapter::JpegFastPacket impl core::convert::From for j2k_jpeg::adapter::JpegFastPacket impl core::default::Default for j2k_jpeg::adapter::JpegPlanCache +impl core::default::Default for j2k_jpeg::bench_support::BenchNeonIdct impl core::default::Default for j2k_jpeg::transcode::DctExtractOptions impl core::fmt::Debug for j2k_jpeg::adapter::SharedJpegFastPacket impl core::fmt::Debug for j2k_jpeg::adapter::SharedJpegInput @@ -358,8 +359,11 @@ impl j2k_jpeg::bench_support::Bench420DispatchStats impl j2k_jpeg::bench_support::BenchBlockActivityCounts impl j2k_jpeg::bench_support::BenchColorRowScratch impl j2k_jpeg::bench_support::BenchFast420Profile +impl j2k_jpeg::bench_support::BenchGrayRowScratch impl j2k_jpeg::bench_support::BenchHuffmanState +impl j2k_jpeg::bench_support::BenchNeonIdct impl j2k_jpeg::bench_support::BenchRgb420RowPairScratch +impl j2k_jpeg::bench_support::BenchRgbRowScratch impl j2k_jpeg::bench_support::BenchUpsampleH2V2Scratch impl j2k_jpeg::transcode::DctExtractOptions impl j2k_jpeg::transcode::JpegDctImage @@ -489,7 +493,9 @@ pub fn j2k_jpeg::bench_support::BenchBlockActivityCounts::bottom_half_zero_block pub fn j2k_jpeg::bench_support::BenchBlockActivityCounts::dc_only_blocks(self) -> usize pub fn j2k_jpeg::bench_support::BenchBlockActivityCounts::general_blocks(self) -> usize pub fn j2k_jpeg::bench_support::BenchBlockActivityCounts::total_blocks(self) -> usize +pub fn j2k_jpeg::bench_support::BenchColorRowScratch::backend_matches_scalar(&mut self) -> bool pub fn j2k_jpeg::bench_support::BenchColorRowScratch::new(usize) -> Self +pub fn j2k_jpeg::bench_support::BenchColorRowScratch::new_unaligned(usize) -> Self pub fn j2k_jpeg::bench_support::BenchColorRowScratch::run_backend(&mut self) pub fn j2k_jpeg::bench_support::BenchColorRowScratch::run_scalar(&mut self) pub fn j2k_jpeg::bench_support::BenchFast420Profile::block_activity_counts(self) -> j2k_jpeg::bench_support::BenchBlockActivityCounts @@ -499,22 +505,37 @@ pub fn j2k_jpeg::bench_support::BenchFast420Profile::parse_plan_ns(self) -> u128 pub fn j2k_jpeg::bench_support::BenchFast420Profile::rgb_emit_ns(self) -> u128 pub fn j2k_jpeg::bench_support::BenchFast420Profile::tile_count(self) -> usize pub fn j2k_jpeg::bench_support::BenchFast420Profile::total_ns(self) -> u128 +pub fn j2k_jpeg::bench_support::BenchGrayRowScratch::backend_matches_scalar(&mut self) -> bool +pub fn j2k_jpeg::bench_support::BenchGrayRowScratch::new(usize) -> Self +pub fn j2k_jpeg::bench_support::BenchGrayRowScratch::run_backend(&mut self) pub fn j2k_jpeg::bench_support::BenchHuffmanState::decode_all(&self) -> core::result::Result pub fn j2k_jpeg::bench_support::BenchHuffmanState::luma_dc_zeros(usize) -> Self +pub fn j2k_jpeg::bench_support::BenchNeonIdct::default() -> Self +pub fn j2k_jpeg::bench_support::BenchNeonIdct::new() -> Self +pub fn j2k_jpeg::bench_support::BenchNeonIdct::run(&self, &[i16; 64], &mut [u8; 64]) +pub fn j2k_jpeg::bench_support::BenchNeonIdct::run_bottom_half_zero(&self, &[i16; 64], &mut [u8; 64]) pub fn j2k_jpeg::bench_support::BenchRgb420ChromaRows<'a>::new(&'a [u8], &'a [u8], &'a [u8], &'a [u8], &'a [u8], &'a [u8]) -> Self pub fn j2k_jpeg::bench_support::BenchRgb420RowPair<'a>::new(&'a [u8], core::option::Option<&'a [u8]>, j2k_jpeg::bench_support::BenchRgb420ChromaRows<'a>, &'a mut [u8], core::option::Option<&'a mut [u8]>) -> Self +pub fn j2k_jpeg::bench_support::BenchRgb420RowPairScratch::backend_matches_reference(&mut self) -> bool +pub fn j2k_jpeg::bench_support::BenchRgb420RowPairScratch::cropped_backend_matches_reference(&mut self, usize, usize) -> bool pub fn j2k_jpeg::bench_support::BenchRgb420RowPairScratch::new(usize) -> Self pub fn j2k_jpeg::bench_support::BenchRgb420RowPairScratch::run(&mut self) +pub fn j2k_jpeg::bench_support::BenchRgb420RowPairScratch::run_cropped(&mut self, usize, usize) pub fn j2k_jpeg::bench_support::BenchRgb420RowPairScratch::run_reference(&mut self) +pub fn j2k_jpeg::bench_support::BenchRgbRowScratch::backend_matches_scalar(&mut self) -> bool +pub fn j2k_jpeg::bench_support::BenchRgbRowScratch::new(usize) -> Self +pub fn j2k_jpeg::bench_support::BenchRgbRowScratch::run_backend(&mut self) pub fn j2k_jpeg::bench_support::BenchUpsampleH2V2Scratch::new(usize) -> Self pub fn j2k_jpeg::bench_support::BenchUpsampleH2V2Scratch::run(&mut self) pub fn j2k_jpeg::bench_support::bench_idct_dc_only_block_with(i16, &mut [u8; 64]) pub fn j2k_jpeg::bench_support::bench_idct_neon_block(&[i16; 64], &mut [u8; 64]) +pub fn j2k_jpeg::bench_support::bench_idct_neon_bottom_half_zero_block(&[i16; 64], &mut [u8; 64]) pub fn j2k_jpeg::bench_support::bench_idct_reduced_2x2_block_with(&[i16; 64], &mut [u8; 4]) pub fn j2k_jpeg::bench_support::bench_idct_reference_block() -> [u8; 64] pub fn j2k_jpeg::bench_support::bench_idct_reference_block_with(&[i16; 64], &mut [u8; 64]) pub fn j2k_jpeg::bench_support::bench_profile_fast420_tile_batch(&[u8], usize) -> core::result::Result, j2k_jpeg::JpegError> pub fn j2k_jpeg::bench_support::bench_rgb_row_pair_from_420(j2k_jpeg::bench_support::BenchRgb420RowPair<'_>) +pub fn j2k_jpeg::bench_support::bench_rgb_row_pair_from_420_cropped(j2k_jpeg::bench_support::BenchRgb420RowPair<'_>, usize, usize) pub fn j2k_jpeg::bench_support::bench_rgb_row_pair_from_420_reference(j2k_jpeg::bench_support::BenchRgb420RowPair<'_>) pub fn j2k_jpeg::bench_support::bench_rgb_row_pair_from_420_with_stats(j2k_jpeg::bench_support::BenchRgb420RowPair<'_>, &mut j2k_jpeg::bench_support::Bench420DispatchStats) pub fn j2k_jpeg::decode_tile_into_in_context(&[u8], &mut j2k_jpeg::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result @@ -855,10 +876,13 @@ pub struct j2k_jpeg::bench_support::Bench420DispatchStats pub struct j2k_jpeg::bench_support::BenchBlockActivityCounts pub struct j2k_jpeg::bench_support::BenchColorRowScratch pub struct j2k_jpeg::bench_support::BenchFast420Profile +pub struct j2k_jpeg::bench_support::BenchGrayRowScratch pub struct j2k_jpeg::bench_support::BenchHuffmanState +pub struct j2k_jpeg::bench_support::BenchNeonIdct pub struct j2k_jpeg::bench_support::BenchRgb420ChromaRows<'a> pub struct j2k_jpeg::bench_support::BenchRgb420RowPair<'a> pub struct j2k_jpeg::bench_support::BenchRgb420RowPairScratch +pub struct j2k_jpeg::bench_support::BenchRgbRowScratch pub struct j2k_jpeg::bench_support::BenchUpsampleH2V2Scratch pub struct j2k_jpeg::transcode::JpegDctComponent pub struct j2k_jpeg::transcode::JpegDctImage diff --git a/docs/stable-api-1.0.md b/docs/stable-api-1.0.md index c2672ce0..eeb43f15 100644 --- a/docs/stable-api-1.0.md +++ b/docs/stable-api-1.0.md @@ -25,7 +25,7 @@ This task must run on macOS with `cargo-public-api` `0.52.0` installed silently change with the runner host or floating nightly channel. The ordinary snapshot uses `RUSTDOCFLAGS=-D warnings` so its comparison with -the published 0.8.1 snapshot keeps the same scope. A second pass adds +the published 0.9.0 snapshot keeps the same scope. A second pass adds `--document-hidden-items` and records only the extra rustdoc-hidden items in the implementation snapshot. Rustdoc can rewrite equivalent re-export paths when hidden modules become visible, so the generator forms a conservative full @@ -34,7 +34,7 @@ sorted difference from the ordinary pass. This guarantees that the combined inventory remains a superset of the ordinary contract while retaining rewritten path variants for review rather than silently dropping reachable API. An empty full cargo-public-api pass fails the gate; an empty per-package hidden-only -difference is recorded truthfully. The 0.8.1 baseline comparison continues to +difference is recorded truthfully. The 0.9.0 baseline comparison continues to use only the ordinary snapshot. Those adapters are implementation-facing, but they are still reachable Rust API and therefore remain in the reviewed inventory. Do not use `#[doc(hidden)]` as a @@ -44,12 +44,13 @@ The published 0.7.5 artifact recorded both ordinary and hidden-enabled passes with the same generator, rustdoc, and target pins. The historical 0.8.0 semver report compares its ordinary inventory with 0.7.5, and the 0.8.1 report compares that release directly with published 0.8.0. The 0.9.0 report compares -the published release directly with published 0.8.1. All reports also record each -package's complete hidden-inventory count and fingerprint. +the published release directly with published 0.8.1, and the staged 0.9.1 +report compares the candidate directly with published 0.9.0. All reports also +record each package's complete hidden-inventory count and fingerprint. Every semver invocation collects both live passes, compares both committed companions, and requires exact ordinary added/removed fingerprints plus the hidden count/fingerprint in -`docs/release-evidence/public-api/public-api-review-0.9.0.yml`. +`docs/release-evidence/public-api/public-api-review-0.9.1.yml`. Nonempty hidden inventories also require a package-specific hidden rationale. The 0.8.0 review file contains the reviewed 0.7.5-to-0.8.0 break ledger. The @@ -86,6 +87,12 @@ is in the generated [`0.9.0` reviewed API report](release-evidence/public-api/reviewed-public-api-diff-0.9.0.md), with its human review in [`public-api-review-0.9.0.yml`](release-evidence/public-api/public-api-review-0.9.0.yml). +The provisional 0.9.1 comparison is in the generated +[`0.9.1` reviewed API report](release-evidence/public-api/reviewed-public-api-diff-0.9.1.md), +with its human review in +[`public-api-review-0.9.1.yml`](release-evidence/public-api/public-api-review-0.9.1.yml). +It remains candidate evidence until the source is frozen and the exact-SHA +release gates complete. [v0.8.0-api-report]: https://github.com/frames-sg/j2k/blob/v0.8.0/engineering/reviewed-public-api-diff-0.8.0.md @@ -143,14 +150,14 @@ removing pass-through public wrappers. Its reviewed API diff must enumerate every contracted item and its changelog must provide migration guidance. This exception applied only to `0.7.5`. -The completed historical transition lock was intentionally narrow: `0.8.0` -was the only candidate permitted to compare against `v0.7.5` as an intentional -pre-1.0 break. The currently configured semver baseline remains published -`v0.8.1` at peeled commit -`f92646d0e6f0d0ef6c1e60b60beaad29da1afd3b` solely for the completed 0.9.0 -report. Its one-time lock permits only `0.9.0` to compare against that baseline -for the reviewed Metal API break. Before any later candidate is checked, the -baseline must rotate to the published `v0.9.0` tag and version. +The completed historical transition locks were intentionally narrow: `0.8.0` +was the only candidate permitted to compare against `v0.7.5`, and `0.9.0` was +the only candidate permitted to compare against `v0.8.1`, as intentional +pre-1.0 breaks. The currently configured semver baseline is published +`v0.9.0` at peeled commit +`b197f01ab4b9271f1cbc36921755a5b9d588bd5a`. The staged `0.9.1` patch +candidate compares directly against that baseline without an intentional-break +transition allowance. Before `1.0`, a minor release may intentionally change the contract only under the same generated evidence, explicit break-ledger, and migration requirements. diff --git a/docs/unsafe-audit.md b/docs/unsafe-audit.md index f99fc88b..403164b7 100644 --- a/docs/unsafe-audit.md +++ b/docs/unsafe-audit.md @@ -164,13 +164,11 @@ moved, or removed. | `crates/j2k-jpeg/benches/common/libjpeg_turbo.rs` | libjpeg-turbo FFI benchmark harness. | libjpeg-turbo handles are checked and destroyed and output buffers are sized by library-reported geometry. | Optional libjpeg-turbo comparison benches. | | `crates/j2k-jpeg/benches/common/libjpeg_turbo_v2.rs` | TurboJPEG 2.x compatibility FFI for the comparator harness. | Build-time version selection keeps the legacy ABI separate from the TurboJPEG 3.x calls; every handle is null-checked and destroyed, input lengths and output geometry are checked before FFI conversion, and writable slices are sized or validated for the requested packed layout. | Forced-2.x comparator execution, native 3.x legacy-versus-current parity, build-script selection tests, and strict all-target Clippy. | | `crates/j2k-jpeg/benches/common/libjpeg_turbo_v2_transform.rs` | Lossless TurboJPEG 2.x crop transform and owned transformed-JPEG buffer. | Crop fields and input/output lengths are checked before FFI conversion; success-with-null and empty outputs are rejected; partially returned and successfully owned allocations are released exactly once through `tjFree`; the transform handle is null-checked and destroyed. | Native legacy-versus-current full, region, and scaled-region parity plus forced-2.x comparator execution and the unsafe-audit path inventory. | -| `crates/j2k-jpeg/src/backend/mod.rs` | Runtime backend dispatch and SIMD entry points. | SIMD entry points are called only when CPU features match the implementation. | Backend dispatch tests on x86_64/aarch64 CI. | -| `crates/j2k-jpeg/src/backend/neon.rs` | AArch64 NEON SIMD kernels. | NEON loads/stores stay within blocks/rows and match scalar reference math. | NEON hot-path and backend parity tests. | -| `crates/j2k-jpeg/src/backend/x86.rs` | x86 SIMD kernels and CPU feature-gated dispatch. | AVX/SSE paths require detected CPU features and match scalar reference math. | x86 backend parity tests. | -| `crates/j2k-jpeg/src/bench_support.rs` | Benchmark-only dispatch instrumentation, SIMD IDCT wrappers, buffer helpers, and decoder helpers. | The thread-local stats pointer is scoped to an exclusive borrow; AArch64 NEON is an architectural baseline; the x86 AVX2 wrapper performs runtime detection and falls back to scalar; benchmark buffers mirror production size checks. | AVX2 dispatch selector unit tests, IDCT parity tests, and the benchmark build gate. | +| `crates/j2k-jpeg/src/bench_support.rs` | Benchmark-only dispatch instrumentation, buffer helpers, and decoder helpers. | The thread-local stats pointer is scoped to one exclusive mutable borrow; its guard restores the prior pointer before that borrow ends. SIMD benchmark calls use safe runtime capability tokens. | Dispatch instrumentation tests, IDCT parity tests, and the benchmark build gate. | | `crates/j2k-jpeg/src/entropy/sequential/deposit.rs` | Entropy decoder stripe deposit helpers for fast 4:2:0 block output. | Raw pointer writes target stripe-plane rows sized by `StripeBuffer::resize_for` and guarded by `assert_stripe_deposit_capacity` before bounds-check-free block deposits. | JPEG decode fuzz and regression tests. | -| `crates/j2k-jpeg/src/idct/avx2.rs` | AVX2 IDCT implementation. | AVX2 loads/stores use fixed block sizes and match scalar IDCT output. | IDCT parity tests. | -| `crates/j2k-jpeg/src/idct/neon.rs` | NEON IDCT implementation. | NEON loads/stores use fixed block sizes and match scalar IDCT output. | IDCT parity tests. | +| `crates/j2k-jpeg/src/simd/neon_memory.rs` | Private fixed-size AArch64 NEON load/store leaves. | Callers are already inside a `fearless_simd::Neon` kernel; array references prove complete initialized extents, AArch64 vector memory operations permit the represented alignment, and shared/exclusive references preserve aliasing rules. | Backend parity across empty, short, tail, crop, missing-row, and unaligned cases; 2,000-case-per-class IDCT properties; NEON hot-path tests. | +| `crates/j2k-jpeg/src/simd/x86.rs` | Private exact-AVX2 capability token and safe target-feature bridge. | The token is constructible only after cached AVX2 plus OS register-state detection; safe references preserve bounds, alignment, aliasing, and initialization contracts across the bridge. It intentionally does not imply the broader x86-64-v3 feature set. | Capability-selection/scalar-fallback tests, x86 cross-build, AVX2 backend parity, and native x86 release-lane execution. | +| `crates/j2k-jpeg/src/simd/x86_memory.rs` | Private fixed-size x86 AVX2 load/store leaves and validated three-row load cursor. | Array leaves prove complete initialized extents. The cursor carries an exact-AVX2 token and the three source lifetimes, derives its chunk count from the shortest row, and advances only while a full eight-byte chunk remains; every operation is unaligned-capable and shared/exclusive references preserve aliasing rules. | Cursor exhaustion and unaligned-load tests, AVX2 row and IDCT differential tests including unaligned subslices, x86 cross-build, and native Windows AVX2 execution. | | `crates/j2k-metal-support/src/allocation.rs` | Typed objc2-metal buffer and texture allocation plus the pointer-based copying buffer initializer. | Checked products and device/repository caps precede dispatch; typed APIs return retained owners and nil is rejected; the only unsafe call receives exactly the initialized source-slice extent and Metal copies it synchronously, so no borrowed no-copy storage escapes. | Pure cap/overflow/zero-sized-ABI tests, real Metal allocation/texture tests, autorelease ownership tests, and strict all-target Clippy. | | `crates/j2k-metal-support/src/buffer_access.rs` | Raw CPU access to shared Metal-buffer contents. | Typed ranges require padding-free `GpuAbi`, checked byte arithmetic, alignment, bounds, and CPU-visible storage; callers must prove command completion and exclusive access. | Checked read/write/fill regressions, nil/range/alignment tests, adapter alias-order policies, and real Metal tests. | | `crates/j2k-metal-support/src/resident.rs` | Opaque immutable Metal image ownership, unsafe raw adoption/access, and completion-retaining image submissions. | Layout construction and adoption validate dimensions, pitch, overflow, allocation bounds, and device identity; safe clones and subviews expose no mutable handle; producers transfer exclusive outputs and every raw-bound input into a submission that retains them until successful completion or a blocking drop. | Layout/bounds/device regressions, input-retention and drop-before-wait tests, resident JPEG/J2K encode parity, unsafe API snapshots, and strict Metal validation. | diff --git a/xtask/src/coverage/tests/source_roles.rs b/xtask/src/coverage/tests/source_roles.rs index afc15cd9..6e4a0638 100644 --- a/xtask/src/coverage/tests/source_roles.rs +++ b/xtask/src/coverage/tests/source_roles.rs @@ -30,7 +30,7 @@ fn nonterminal_external_test_modules_do_not_truncate_production_files() { root: "crates/j2k-jpeg/src/backend/mod.rs", analyzed: "crates/j2k-jpeg/src/backend/mod.rs", tests: "crates/j2k-jpeg/src/backend/tests.rs", - production_marker: "#[derive(Debug, Clone, Copy, PartialEq, Eq)]", + production_marker: "#[derive(Debug, Clone, Copy)]", later_function: Some("new"), }, Case { diff --git a/xtask/src/quality_commands.rs b/xtask/src/quality_commands.rs index 29680b5a..3bcbc368 100644 --- a/xtask/src/quality_commands.rs +++ b/xtask/src/quality_commands.rs @@ -419,6 +419,8 @@ pub(super) fn no_std() -> Result<(), String> { } pub(super) fn verify_unsafe_audit() -> Result<(), String> { + verify_jpeg_simd_unsafe_boundary()?; + let audit_path = Path::new("docs/unsafe-audit.md"); let audit = fs::read_to_string(audit_path) .map_err(|err| format!("failed to read {}: {err}", audit_path.display()))?; @@ -496,6 +498,160 @@ pub(super) fn verify_unsafe_audit() -> Result<(), String> { } } +const JPEG_SIMD_UNSAFE_BLOCK_CAP: usize = 24; +const JPEG_SIMD_UNSAFE_BOUNDARIES: &[&str] = &[ + "crates/j2k-jpeg/src/simd/neon_memory.rs", + "crates/j2k-jpeg/src/simd/x86.rs", + "crates/j2k-jpeg/src/simd/x86_memory.rs", +]; + +fn verify_jpeg_simd_unsafe_boundary() -> Result<(), String> { + let mut paths = BTreeSet::new(); + for root in [ + "crates/j2k-jpeg/src/backend", + "crates/j2k-jpeg/src/idct", + "crates/j2k-jpeg/src/simd", + ] { + paths.extend(rust_sources(Path::new(root))?); + } + + let mut unsafe_blocks = 0usize; + for path in paths { + let relative = path.to_string_lossy().replace('\\', "/"); + let source = fs::read_to_string(&path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + let stats = audit_jpeg_simd_source(&relative, &source)?; + unsafe_blocks += stats.unsafe_blocks; + } + + if unsafe_blocks > JPEG_SIMD_UNSAFE_BLOCK_CAP { + return Err(format!( + "j2k-jpeg SIMD contains {unsafe_blocks} unsafe blocks; cap is {JPEG_SIMD_UNSAFE_BLOCK_CAP}" + )); + } + + Ok(()) +} + +#[derive(Debug, Default, PartialEq, Eq)] +struct JpegSimdUnsafeStats { + unsafe_blocks: usize, +} + +fn audit_jpeg_simd_source(relative: &str, source: &str) -> Result { + let file = syn::parse_file(source) + .map_err(|err| format!("failed to parse {relative} for the SIMD unsafe ratchet: {err}"))?; + let mut visitor = JpegSimdUnsafeVisitor::default(); + visitor.visit_file(&file); + + if !visitor.unsafe_functions.is_empty() { + return Err(format!( + "{relative} contains SIMD unsafe fn declarations at lines {:?}; use safe token-backed entry points", + visitor.unsafe_functions + )); + } + + if !visitor.unsafe_blocks.is_empty() && !JPEG_SIMD_UNSAFE_BOUNDARIES.contains(&relative) { + return Err(format!( + "{relative} contains SIMD unsafe at lines {:?} outside private boundary modules", + visitor.unsafe_blocks + )); + } + + for line in &visitor.unsafe_blocks { + verify_simd_safety_proof(relative, source, *line)?; + } + + Ok(JpegSimdUnsafeStats { + unsafe_blocks: visitor.unsafe_blocks.len(), + }) +} + +fn verify_simd_safety_proof( + relative: &str, + source: &str, + unsafe_line: usize, +) -> Result<(), String> { + let lines = source.lines().collect::>(); + let start = unsafe_line.saturating_sub(20); + let end = unsafe_line.min(lines.len()); + let proof = lines[start..end].join("\n").to_ascii_lowercase(); + let required = [ + "safety:", + "feature availability", + "bounds", + "alignment", + "aliasing", + "initialization", + ]; + let missing = required + .into_iter() + .filter(|term| !proof.contains(term)) + .collect::>(); + if missing.is_empty() { + Ok(()) + } else { + Err(format!( + "{relative}:{unsafe_line} SIMD unsafe block is missing proof terms: {missing:?}" + )) + } +} + +#[derive(Default)] +struct JpegSimdUnsafeVisitor { + unsafe_blocks: Vec, + unsafe_functions: Vec, +} + +impl<'ast> Visit<'ast> for JpegSimdUnsafeVisitor { + fn visit_expr_unsafe(&mut self, expression: &'ast syn::ExprUnsafe) { + self.unsafe_blocks + .push(expression.unsafe_token.span.start().line); + visit::visit_expr_unsafe(self, expression); + } + + fn visit_item_macro(&mut self, item_macro: &'ast syn::ItemMacro) { + scan_simd_macro_tokens( + &item_macro.mac.tokens, + &mut self.unsafe_blocks, + &mut self.unsafe_functions, + ); + } + + fn visit_signature(&mut self, signature: &'ast syn::Signature) { + if let Some(unsafety) = signature.unsafety { + self.unsafe_functions.push(unsafety.span.start().line); + } + visit::visit_signature(self, signature); + } +} + +fn scan_simd_macro_tokens( + tokens: &TokenStream, + unsafe_blocks: &mut Vec, + unsafe_functions: &mut Vec, +) { + let trees = tokens.clone().into_iter().collect::>(); + for (index, tree) in trees.iter().enumerate() { + match tree { + TokenTree::Group(group) => { + scan_simd_macro_tokens(&group.stream(), unsafe_blocks, unsafe_functions); + } + TokenTree::Ident(ident) if ident == "unsafe" => { + let is_function = trees + .get(index + 1) + .is_some_and(|next| matches!(next, TokenTree::Ident(next) if next == "fn")); + if is_function { + unsafe_functions.push(ident.span().start().line); + } else { + unsafe_blocks.push(ident.span().start().line); + } + } + TokenTree::Ident(_) | TokenTree::Literal(_) | TokenTree::Punct(_) => {} + } + } +} + fn source_contains_unsafe_rust(source: &str) -> syn::Result { let file = syn::parse_file(source)?; let mut detector = UnsafeRustDetector::default(); @@ -597,7 +753,7 @@ fn test_downstream_examples() -> Result<(), String> { #[cfg(test)] mod unsafe_audit_tests { - use super::source_contains_unsafe_rust; + use super::{audit_jpeg_simd_source, source_contains_unsafe_rust}; #[test] fn detects_unsafe_rust_across_supported_syntax() { @@ -657,6 +813,39 @@ mod unsafe_audit_tests { .expect_err("invalid Rust must fail the audit scan"); assert!(!error.to_string().is_empty()); } + + #[test] + fn jpeg_simd_boundary_rejects_unsafe_functions() { + let error = audit_jpeg_simd_source( + "crates/j2k-jpeg/src/simd/x86_memory.rs", + "pub(crate) unsafe fn load() {}", + ) + .expect_err("SIMD unsafe functions must be rejected"); + assert!(error.contains("unsafe fn")); + } + + #[test] + fn jpeg_simd_boundary_rejects_unsafe_outside_boundary_modules() { + let error = audit_jpeg_simd_source( + "crates/j2k-jpeg/src/backend/x86.rs", + "fn kernel() { unsafe { core::hint::unreachable_unchecked() } }", + ) + .expect_err("ordinary SIMD modules must not contain unsafe blocks"); + assert!(error.contains("outside private boundary modules")); + } + + #[test] + fn jpeg_simd_boundary_requires_the_complete_safety_proof() { + let error = audit_jpeg_simd_source( + "crates/j2k-jpeg/src/simd/x86_memory.rs", + "fn load() { /* SAFETY: bounds only */ unsafe { core::hint::unreachable_unchecked() } }", + ) + .expect_err("incomplete SIMD safety proofs must be rejected"); + assert!(error.contains("feature availability")); + assert!(error.contains("alignment")); + assert!(error.contains("aliasing")); + assert!(error.contains("initialization")); + } } #[cfg(all(test, unix))] diff --git a/xtask/src/release_commands/tests/orchestration.rs b/xtask/src/release_commands/tests/orchestration.rs index 39dd8b7a..82ac71b7 100644 --- a/xtask/src/release_commands/tests/orchestration.rs +++ b/xtask/src/release_commands/tests/orchestration.rs @@ -85,12 +85,12 @@ fn release_integrity_publish_mode_accepts_hermetic_final_metadata() { } std::fs::write( release_root.join("Cargo.toml"), - "[workspace.package]\nversion = \"0.9.0\"\n", + "[workspace.package]\nversion = \"0.9.1\"\n", ) .expect("write workspace manifest fixture"); std::fs::write( release_root.join("CHANGELOG.md"), - "# Changelog\n\n## [0.9.0] - 2026-08-10\n", + "# Changelog\n\n## [0.9.1] - 2026-08-13\n", ) .expect("write finalized changelog fixture"); diff --git a/xtask/src/semver.rs b/xtask/src/semver.rs index a67c977c..febfebc5 100644 --- a/xtask/src/semver.rs +++ b/xtask/src/semver.rs @@ -25,11 +25,11 @@ use compatibility::{semver_check_args, semver_check_release_type}; const CARGO_SEMVER_CHECKS_VERSION: &str = "0.48.0"; const SEMVER_TOOLCHAIN: &str = "1.96"; -const SEMVER_BASELINE_VERSION: &str = "0.8.1"; -const SEMVER_BASELINE_TAG: &str = "v0.8.1"; -const SEMVER_BASELINE_COMMIT: &str = "f92646d0e6f0d0ef6c1e60b60beaad29da1afd3b"; -const API_DIFF_REPORT: &str = "docs/release-evidence/public-api/reviewed-public-api-diff-0.9.0.md"; -const API_REVIEW_CONFIG: &str = "docs/release-evidence/public-api/public-api-review-0.9.0.yml"; +const SEMVER_BASELINE_VERSION: &str = "0.9.0"; +const SEMVER_BASELINE_TAG: &str = "v0.9.0"; +const SEMVER_BASELINE_COMMIT: &str = "b197f01ab4b9271f1cbc36921755a5b9d588bd5a"; +const API_DIFF_REPORT: &str = "docs/release-evidence/public-api/reviewed-public-api-diff-0.9.1.md"; +const API_REVIEW_CONFIG: &str = "docs/release-evidence/public-api/public-api-review-0.9.1.yml"; #[derive(Clone, Copy, Debug, Eq, PartialEq)] struct BaselineTransition<'a> { @@ -38,12 +38,7 @@ struct BaselineTransition<'a> { required_next_baseline_tag: &'a str, } -const INTENTIONAL_BREAK_TRANSITION: Option> = - Some(BaselineTransition { - candidate_version: "0.9.0", - required_next_baseline_version: "0.9.0", - required_next_baseline_tag: "v0.9.0", - }); +const INTENTIONAL_BREAK_TRANSITION: Option> = None; const SEMVER_BASELINE_PACKAGES: &[&str] = &[ "j2k", diff --git a/xtask/src/semver/tests.rs b/xtask/src/semver/tests.rs index ae0acedd..c82ce534 100644 --- a/xtask/src/semver/tests.rs +++ b/xtask/src/semver/tests.rs @@ -11,7 +11,7 @@ use super::{ parse_api_snapshot, parse_options, render_report, semver_cargo_args, semver_check_release_type, snapshot_uses_generator, validate_baseline_transition, validate_package_partition, BaselineTransition, PackageApiDiff, ReleaseType, SnapshotKind, Version, - SEMVER_BASELINE_PACKAGES, SEMVER_NEW_PACKAGES, + INTENTIONAL_BREAK_TRANSITION, SEMVER_BASELINE_PACKAGES, SEMVER_NEW_PACKAGES, }; mod api_planning; @@ -221,18 +221,19 @@ fn report_has_one_published_details_section_and_rotated_baseline() { assert_eq!(report.matches("## Published-package details").count(), 1); assert!(report.contains("Rustdoc-hidden candidate items: 1")); assert!(report.contains("Full hidden-inventory fingerprint: `fnv1a64:")); - assert!(report.contains("Baseline registry version: `0.8.1`")); - assert!(report.contains("Active intentional-break transition")); - assert!(report.contains("Required next semver baseline: `v0.9.0` at version `0.9.0`")); + assert!(report.contains("Baseline registry version: `0.9.0`")); + assert!(!report.contains("Active intentional-break transition")); + assert!(!report.contains("Required next semver baseline")); + assert!(INTENTIONAL_BREAK_TRANSITION.is_none()); } #[test] fn parses_review_config_and_rejects_unknown_fields() { let source = "\ version: 3 -baseline_tag: v0.8.1 -baseline_version: 0.8.1 -candidate_version: 0.9.0 +baseline_tag: v0.9.0 +baseline_version: 0.9.0 +candidate_version: 0.9.1 break_ledger: - id: strict-decode-default kind: behavior @@ -259,7 +260,7 @@ reviews: "; let value: serde_yaml_ng::Value = serde_yaml_ng::from_str(source).unwrap(); let parsed = parse_review_config(&value).unwrap(); - assert_eq!(parsed.candidate_version, "0.9.0"); + assert_eq!(parsed.candidate_version, "0.9.1"); assert_eq!(parsed.break_ledger.len(), 2); assert_eq!(parsed.break_ledger[0].kind, BreakKind::Behavior); assert_eq!(parsed.break_ledger[1].kind, BreakKind::Source); diff --git a/xtask/src/semver/tests/api_planning.rs b/xtask/src/semver/tests/api_planning.rs index 91a8b376..2d1632fd 100644 --- a/xtask/src/semver/tests/api_planning.rs +++ b/xtask/src/semver/tests/api_planning.rs @@ -89,7 +89,7 @@ fn package_diff_planning_distinguishes_published_and_new_packages() { assert!(diffs[1].removed.is_empty()); let report = render_report("0.7.4", &diffs, "0.52.0"); - assert!(report.contains("## New packages without a 0.8.1 registry baseline")); + assert!(report.contains("## New packages without a 0.9.0 registry baseline")); assert!(report.contains("- `j2k-future` `0.7.4`: 1 ordinary public API items")); assert!(report.contains("### `j2k`")); assert!(report.contains("```text\nremoved\n```")); diff --git a/xtask/src/semver/tests/command_boundaries.rs b/xtask/src/semver/tests/command_boundaries.rs index 930ddbbc..41d99fd5 100644 --- a/xtask/src/semver/tests/command_boundaries.rs +++ b/xtask/src/semver/tests/command_boundaries.rs @@ -74,7 +74,7 @@ fn committed_candidate_semver_inputs_match_the_pinned_workspace_contract() { assert!(hidden.starts_with("# J2K 1.0 Rustdoc-Hidden Public API Snapshot")); let versions = workspace_package_versions().expect("workspace package versions"); - assert_eq!(versions.get("j2k").map(String::as_str), Some("0.9.0")); + assert_eq!(versions.get("j2k").map(String::as_str), Some("0.9.1")); assert!(versions.keys().collect::>().len() > 10); } @@ -114,7 +114,7 @@ fn report_verification_is_workspace_anchored_and_empty_checks_are_a_noop() { fn semver_check_command_uses_the_computed_candidate_release_type() { let diff = PackageApiDiff { package: "j2k-core".to_string(), - candidate_version: "0.9.0".to_string(), + candidate_version: "0.9.1".to_string(), release_type: Some(ReleaseType::Minor), baseline_count: 1, candidate_count: 0, @@ -136,7 +136,7 @@ fn semver_check_command_uses_the_computed_candidate_release_type() { "--package", "j2k-core", "--baseline-version", - "0.8.1", + "0.9.0", "--release-type", "minor", "--color", diff --git a/xtask/tests/command_orchestration.rs b/xtask/tests/command_orchestration.rs index fbcd7ebe..1bf9c818 100644 --- a/xtask/tests/command_orchestration.rs +++ b/xtask/tests/command_orchestration.rs @@ -119,8 +119,8 @@ fn release_critical_orchestrators_run_from_the_workspace_without_real_cargo() { assert!(log.contains("package -p j2k-cli --no-verify")); #[cfg(target_os = "macos")] { - assert!(log.contains("git rev-parse v0.8.1^{commit}")); - assert!(log.contains("git show v0.8.1:docs/stable-api-1.0.public-api.txt")); + assert!(log.contains("git rev-parse v0.9.0^{commit}")); + assert!(log.contains("git show v0.9.0:docs/stable-api-1.0.public-api.txt")); } } diff --git a/xtask/tests/command_orchestration/support.rs b/xtask/tests/command_orchestration/support.rs index ed6aa9db..bbf777b7 100644 --- a/xtask/tests/command_orchestration/support.rs +++ b/xtask/tests/command_orchestration/support.rs @@ -60,7 +60,7 @@ impl Harness { fs::write( &git, format!( - "#!/bin/sh\nprintf 'git %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = status ]; then exit 0; fi\nif [ \"$1\" = config ] && [ \"$2\" = --get ] && [ \"$3\" = remote.origin.url ]; then printf '%s\\n' 'git@example.invalid:frames-sg/j2k.git'; exit 0; fi\nif [ \"$1\" = rev-parse ] && [ \"$2\" = 'v0.8.1^{{commit}}' ]; then printf '%s\\n' 'f92646d0e6f0d0ef6c1e60b60beaad29da1afd3b'; exit 0; fi\nif [ \"$1\" = show ] && [ \"$2\" = 'v0.8.1:docs/stable-api-1.0.public-api.txt' ]; then exec cat '{}'; fi\nif [ \"$1\" = rev-parse ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release revision: %s\\n' \"$2\" >&2; exit 97; fi\nif [ \"$1\" = show ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release object: %s\\n' \"$2\" >&2; exit 97; fi\nexec \"{}\" \"$@\"\n", + "#!/bin/sh\nprintf 'git %s\\n' \"$*\" >> '{}'\nif [ \"$1\" = status ]; then exit 0; fi\nif [ \"$1\" = config ] && [ \"$2\" = --get ] && [ \"$3\" = remote.origin.url ]; then printf '%s\\n' 'git@example.invalid:frames-sg/j2k.git'; exit 0; fi\nif [ \"$1\" = rev-parse ] && [ \"$2\" = 'v0.9.0^{{commit}}' ]; then printf '%s\\n' 'b197f01ab4b9271f1cbc36921755a5b9d588bd5a'; exit 0; fi\nif [ \"$1\" = show ] && [ \"$2\" = 'v0.9.0:docs/stable-api-1.0.public-api.txt' ]; then exec cat '{}'; fi\nif [ \"$1\" = rev-parse ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release revision: %s\\n' \"$2\" >&2; exit 97; fi\nif [ \"$1\" = show ] && [ \"${{2#v}}\" != \"$2\" ]; then printf 'unexpected release object: %s\\n' \"$2\" >&2; exit 97; fi\nexec \"{}\" \"$@\"\n", log.display(), baseline_snapshot.display(), real_git.display()