From 179bf4d70a9cba3ba1c6ac94c19ae6b44a2116e5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Mar 2026 13:54:32 +0000 Subject: [PATCH 1/5] Initial plan From 4e3d695407ddab988df7c77398f247768c0cd983 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:09:20 +0000 Subject: [PATCH 2/5] Initial exploration complete Co-authored-by: PSeitz <1109503+PSeitz@users.noreply.github.com> --- CODE_WALKTHROUGH.md | 531 +++++++++++++++++++++++ EXPLORATION_INDEX.md | 260 ++++++++++++ EXPLORATION_REPORT.md | 965 ++++++++++++++++++++++++++++++++++++++++++ QUICK_REFERENCE.md | 211 +++++++++ 4 files changed, 1967 insertions(+) create mode 100644 CODE_WALKTHROUGH.md create mode 100644 EXPLORATION_INDEX.md create mode 100644 EXPLORATION_REPORT.md create mode 100644 QUICK_REFERENCE.md diff --git a/CODE_WALKTHROUGH.md b/CODE_WALKTHROUGH.md new file mode 100644 index 00000000..385a8393 --- /dev/null +++ b/CODE_WALKTHROUGH.md @@ -0,0 +1,531 @@ +# LZ4_FLEX Code Walkthrough + +This document shows key code snippets and explains the core algorithms. + +## Compression Algorithm Walkthrough + +### Hash Table Initialization + +**File: `src/block/hashtable.rs`** + +```rust +// 4K entry hashtable using 16-bit values (8 KB total) +pub struct HashTable4KU16 { + dict: Box<[u16; 4096]>, +} + +impl HashTable4KU16 { + pub fn new() -> Self { + // Optimized allocation: uses alloc_zeroed instead of alloc + memset + let dict = alloc::vec![0; 4096] + .into_boxed_slice() + .try_into() + .unwrap(); + Self { dict } + } +} + +// Hash function: converts 4-byte sequence to 4K table index +fn hash(sequence: u32) -> u32 { + (sequence.wrapping_mul(2654435761_u32)) >> 16 // FNV-like hash +} +``` + +### Compression Main Loop + +**File: `src/block/compress.rs` (lines 318-489)** + +```rust +pub(crate) fn compress_internal( + input: &[u8], + input_pos: usize, + output: &mut S, + dict: &mut T, + ext_dict: &[u8], + input_stream_offset: usize, +) -> Result { + // ... validation ... + + let mut literal_start = input_pos; + let mut cur = input_pos; + + loop { + let mut non_match_count = 1 << INCREASE_STEPSIZE_BITSHIFT; // 32 + + // Inner loop: search for matches + loop { + let step_size = non_match_count >> INCREASE_STEPSIZE_BITSHIFT; + non_match_count += 1; + + cur = next_cur; + next_cur += step_size; + + if cur > end_pos_check { // Past safe zone + handle_last_literals(output, input, literal_start); + return Ok(output.pos() - output_start_pos); + } + + // Hash the 4-byte sequence at current position + let hash = T::get_hash_at(input, cur); + let candidate = dict.get_at(hash); // Look up in hash table + dict.put_at(hash, cur + input_stream_offset); // Store current position + + // Check if we can reach this candidate (within 64KB window) + if input_stream_offset + cur - candidate > MAX_DISTANCE { + continue; + } + + // Verify the bytes actually match (not just hash collision) + let cand_bytes: u32 = get_batch(candidate_source, candidate); + let curr_bytes: u32 = get_batch(input, cur); + + if cand_bytes == curr_bytes { + break; // Found a match! + } + } + + // Extend match backwards to include preceding literals + backtrack_match( + input, + &mut cur, + literal_start, + candidate_source, + &mut candidate, + ); + + let lit_len = cur - literal_start; + + // Extend match forwards to find complete duplicate length + cur += MINMATCH; // Skip already-matched 4 bytes + candidate += MINMATCH; + let duplicate_length = count_same_bytes(input, &mut cur, candidate_source, candidate); + + // Encode: [token][literal_len?][literals][offset][match_len?] + let token = token_from_literal_and_match_length(lit_len, duplicate_length); + + push_byte(output, token); + if lit_len >= 0xF { + write_integer(output, lit_len - 0xF); + } + copy_literals_wild(output, input, literal_start, lit_len); + push_u16(output, offset); // 16-bit offset in little-endian + if duplicate_length >= 0xF { + write_integer(output, duplicate_length - 0xF); + } + + literal_start = cur; + } +} +``` + +### Byte Matching + +**File: `src/block/compress.rs` (lines 98-145)** + +```rust +#[inline] +#[cfg(feature = "safe-encode")] +fn count_same_bytes( + input: &[u8], + cur: &mut usize, + source: &[u8], + candidate: usize +) -> usize { + const USIZE_SIZE: usize = core::mem::size_of::(); + let cur_slice = &input[*cur..input.len() - END_OFFSET]; + let cand_slice = &source[candidate..]; + + let mut num = 0; + // Compare usize-sized chunks for better performance + for (block1, block2) in cur_slice.chunks_exact(USIZE_SIZE) + .zip(cand_slice.chunks_exact(USIZE_SIZE)) + { + let input_block = usize::from_ne_bytes(block1.try_into().unwrap()); + let match_block = usize::from_ne_bytes(block2.try_into().unwrap()); + + if input_block == match_block { + num += USIZE_SIZE; + } else { + // Found difference - count matching bytes via bit operations + let diff = input_block ^ match_block; + num += (diff.to_le().trailing_zeros() / 8) as usize; + *cur += num; + return num; + } + } + + // Handle remaining bytes (1-7) + num += count_same_bytes_tail(cur_slice, cand_slice, num); + *cur += num; + num +} +``` + +## Decompression Algorithm Walkthrough + +### Fast Path (Hot Loop) + +**File: `src/block/decompress.rs` (lines 259-326)** + +```rust +// Check if we can use optimized fast path +if does_token_fit(token) // both nibbles < 15 + && (input_ptr as usize) <= input_ptr_safe as usize + && output_ptr < safe_output_ptr +{ + // Token fits: literal and match lengths both < 15, no overflow bytes needed + let literal_length = (token >> 4) as usize; + let mut match_length = MINMATCH + (token & 0xF) as usize; + + // Copy literal section - bulk 16-byte copy (may overread safely) + unsafe { + core::ptr::copy_nonoverlapping(input_ptr, output_ptr, 16); + input_ptr = input_ptr.add(literal_length); + output_ptr = output_ptr.add(literal_length); + } + + // Read 16-bit match offset (little-endian) + let offset = read_u16_ptr(&mut input_ptr) as usize; + + let output_len = unsafe { output_ptr.offset_from(output_base) as usize }; + let offset = offset.min(output_len + ext_dict.len()); + + // Calculate source pointer for the match + let start_ptr = unsafe { output_ptr.sub(offset) }; + + // Copy match - handle overlapping regions + if offset >= match_length { + // Non-overlapping: simple copy of 18 bytes + unsafe { + core::ptr::copy(start_ptr, output_ptr, 18); + output_ptr = output_ptr.add(match_length); + } + } else { + // Overlapping: must copy byte-by-byte + unsafe { + duplicate_overlapping(&mut output_ptr, start_ptr, match_length); + } + } + + continue; // Back to fast path +} + +// Slow path for complex tokens (see below...) +``` + +### Variable-Length Integer Decoding + +**File: `src/block/decompress.rs` (lines 131-162)** + +```rust +pub(super) fn read_integer_ptr( + input_ptr: &mut *const u8, + input_ptr_end: *const u8, +) -> Result { + let mut n: usize = 0; + + loop { + // Read next byte + if *input_ptr >= input_ptr_end { + return Err(DecompressError::ExpectedAnotherByte); + } + + let extra = unsafe { input_ptr.read() }; + *input_ptr = unsafe { input_ptr.add(1) }; + n += extra as usize; + + // If byte < 255, we're done. Otherwise, continue. + // Example: 255 + 255 + 10 = 520 + if extra != 0xFF { + break; + } + } + + Ok(n) +} +``` + +### Overlapping Copy (Self-Referential) + +**File: `src/block/decompress.rs` (lines 56-87)** + +```rust +#[inline] +#[cfg_attr(feature = "nightly", optimize(size))] // Prevent unrolling +unsafe fn duplicate_overlapping( + output_ptr: &mut *mut u8, + mut start: *const u8, + match_length: usize, +) { + // Safety: Write zero to handle edge case where output_ptr == start + // This matches the C reference implementation behavior + output_ptr.write(0u8); + let dst_ptr_end = output_ptr.add(match_length); + + // Copy byte-by-byte - allows self-referential copies + // Example: offset=1, match_len=5, data=[A] -> [A,A,A,A,A] + while output_ptr.add(1) < dst_ptr_end { + // Manual unroll (2 iterations) to prevent compiler unrolling + core::ptr::copy(start, *output_ptr, 1); + start = start.add(1); + *output_ptr = output_ptr.add(1); + + core::ptr::copy(start, *output_ptr, 1); + start = start.add(1); + *output_ptr = output_ptr.add(1); + } + + if *output_ptr < dst_ptr_end { + core::ptr::copy(start, *output_ptr, 1); + *output_ptr = output_ptr.add(1); + } +} +``` + +## Frame Format Implementation + +### Frame Encoder + +**File: `src/frame/compress.rs`** + +```rust +pub struct FrameEncoder { + writer: W, + context: CompressionContext, + frame_info: FrameInfo, +} + +impl FrameEncoder { + pub fn new(writer: W) -> Self { + let frame_info = FrameInfo::new(); + Self { + writer, + context: CompressionContext::new(), + frame_info, + } + } + + pub fn with_frame_info(writer: W, frame_info: FrameInfo) -> Self { + Self { + writer, + context: CompressionContext::new(), + frame_info, + } + } +} + +impl Write for FrameEncoder { + fn write(&mut self, buf: &[u8]) -> io::Result { + // 1. Write frame header (if first write) + // 2. Compress input into blocks + // 3. Write block header (size, checksum) + // 4. Write compressed block + // 5. Return bytes written + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } +} + +impl FrameEncoder { + pub fn finish(mut self) -> io::Result> { + // Write end-of-frame marker (4 bytes of zeros) + // Write content checksum if enabled + // Return completed compressed data + Ok(Vec::new()) + } +} +``` + +### Frame Header Format + +**File: `src/frame/header.rs` (lines 27-100)** + +```rust +const LZ4F_MAGIC_NUMBER: u32 = 0x184D2204; // Magic number (4 bytes) + +// Frame descriptor layout: +// Byte 0: FLG (flags) +// - Bits 7-6: Version (01 = v1) +// - Bit 5: Independent blocks (0=linked, 1=independent) +// - Bit 4: Block checksums enabled +// - Bit 3: Content size present +// - Bit 2: Content checksum enabled +// - Bit 1: Reserved (must be 0) +// - Bit 0: Dictionary ID present +// Byte 1: BD (block size info) +// - Bits 7-4: Block size ID (4=64KB, 5=256KB, 6=1MB, 7=4MB, 8=8MB) +// - Bits 3-0: Reserved (must be 0) +// [4-8 bytes]: Content size (optional, if FLG bit 3 = 1) +// [4 bytes]: Dictionary ID (optional, if FLG bit 0 = 1) +// [1 byte]: FLG checksum (CRC32 of FLG and BD bytes) + +#[derive(Clone, Copy, Debug)] +pub enum BlockSize { + Auto = 0, + Max64KB = 4, + Max256KB = 5, + Max1MB = 6, + Max4MB = 7, + Max8MB = 8, +} + +#[derive(Clone, Copy, Debug)] +pub enum BlockMode { + Independent, // Blocks don't reference previous blocks + Linked, // Blocks can reference previous blocks +} +``` + +## Error Handling + +### Decompression Error Handling + +**File: `src/block/mod.rs` (lines 79-96)** + +```rust +pub enum DecompressError { + /// Output buffer too small for decompressed data + OutputTooSmall { + expected: usize, + actual: usize, + }, + /// Literal is out of bounds of the input + LiteralOutOfBounds, + /// Expected another byte, but none found. + ExpectedAnotherByte, + /// Deduplication offset out of bounds (not in buffer). + OffsetOutOfBounds, +} + +// Usage: +match decompress(&compressed, uncompressed_size) { + Ok(data) => println!("Decompressed: {}", String::from_utf8_lossy(&data)), + Err(DecompressError::OutputTooSmall { expected, actual }) => { + eprintln!("Need {} bytes, got {}", expected, actual); + }, + Err(e) => eprintln!("Error: {}", e), +} +``` + +## Performance Optimization Examples + +### 1. Token Fitting Check + +**File: `src/block/decompress.rs` (lines 188-195)** + +```rust +#[inline] +fn does_token_fit(token: u8) -> bool { + // Check if literal length < 15 AND match length < 15 + // If true, no extension bytes needed - saves branch in hot path + !((token & 0xF0) == 0xF0 || (token & 0x0F) == 0x0F) +} +``` + +### 2. Literal Copy with Overread + +**File: `src/block/compress.rs` (lines 527-545)** + +```rust +#[inline] +#[cfg(not(feature = "safe-encode"))] +fn copy_literals_wild(output: &mut impl Sink, input: &[u8], input_start: usize, len: usize) { + unsafe { + // Copy more bytes than needed, but bounds are checked + // This prevents compiler from generating slow memcpy + let start_ptr = input.as_ptr().add(input_start); + match len { + 0..=8 => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), 8), + 9..=16 => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), 16), + 17..=24 => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), 24), + _ => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), len), + } + output.set_pos(output.pos() + len); + } +} +``` + +### 3. Hash Function + +**File: `src/block/hashtable.rs` (lines 19-34)** + +```rust +#[inline] +fn hash(sequence: u32) -> u32 { + // FNV-1 like hash: multiply by prime, shift to get 16-bit index + // This distributes values well across hash table + (sequence.wrapping_mul(2654435761_u32)) >> 16 +} + +#[cfg(target_pointer_width = "64")] +#[inline] +fn hash5(sequence: usize) -> u32 { + // 64-bit hash for better distribution + let primebytes = if cfg!(target_endian = "little") { + 889523592379_usize + } else { + 11400714785074694791_usize + }; + (((sequence << 24).wrapping_mul(primebytes)) >> 48) as u32 +} +``` + +## Safe vs Unsafe Variants + +### Safe Encoding (Default) + +**File: `src/block/compress.rs` (lines 41-43, 492-495)** + +```rust +#[cfg(feature = "safe-encode")] +pub(super) fn get_batch(input: &[u8], n: usize) -> u32 { + u32::from_ne_bytes(input[n..n + 4].try_into().unwrap()) // Bounds checked +} + +#[cfg(feature = "safe-encode")] +fn push_byte(output: &mut impl Sink, el: u8) { + output.push(el); // Uses trait method (no unsafe) +} +``` + +### Unsafe Encoding (Optional) + +**File: `src/block/compress.rs` (lines 34-36, 498-504)** + +```rust +#[cfg(not(feature = "safe-encode"))] +pub(super) fn get_batch(input: &[u8], n: usize) -> u32 { + unsafe { read_u32_ptr(input.as_ptr().add(n)) } // Raw pointer read +} + +#[cfg(not(feature = "safe-encode"))] +fn push_byte(output: &mut impl Sink, el: u8) { + unsafe { + core::ptr::write(output.pos_mut_ptr(), el); // Direct memory write + output.set_pos(output.pos() + 1); + } +} +``` + +## No-std Support + +**File: `src/lib.rs` (lines 71-78)** + +```rust +#![deny(warnings)] +#![deny(missing_docs)] +#![cfg_attr(not(feature = "std"), no_std)] // Conditional no_std +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(feature = "nightly", feature(optimize_attribute))] + +#[cfg_attr(test, macro_use)] +extern crate alloc; // Always require alloc, optionally std +``` + +This allows: +- Block format: works with just `alloc` (no `std`) +- Frame format: requires `std::io::{Read, Write}` + diff --git a/EXPLORATION_INDEX.md b/EXPLORATION_INDEX.md new file mode 100644 index 00000000..d82b3e43 --- /dev/null +++ b/EXPLORATION_INDEX.md @@ -0,0 +1,260 @@ +# LZ4_FLEX Repository Exploration - Complete Index + +This index provides navigation to all exploration documents created during the thorough analysis of the lz4_flex repository. + +## 📋 Documentation Files Created + +### 1. **EXPLORATION_REPORT.md** (33 KB, 965 lines) + - **Most Comprehensive** - Complete technical analysis + - **Contents:** + - Project overview and performance benchmarks + - Complete directory structure (2 levels deep) + - Cargo.toml configuration details + - Main source files purposes and analysis + - Compression algorithm implementation details + - Decompression algorithm walkthrough + - Benchmarking framework overview + - Complete test suite documentation + - Build and test instructions + - Performance-related comments and TODOs in code + - Key architectural observations + + **Read this when:** You need comprehensive technical understanding of the entire project + +--- + +### 2. **QUICK_REFERENCE.md** (6 KB) + - **Fast Navigation** - Quick lookup guide + - **Contents:** + - What is LZ4_FLEX (quick description) + - Quick start examples (block and frame format) + - Build commands + - Testing commands + - Key files summary table + - Feature flags + - Compression/decompression algorithms (brief) + - Performance table + - Error types + - Constraints and features + - CI/CD summary + - Memory usage info + - Testing datasets + + **Read this when:** You need quick reference or examples of how to use the library + +--- + +### 3. **CODE_WALKTHROUGH.md** (15 KB) + - **Detailed Code Examples** - Deep dive into implementations + - **Contents:** + - Compression algorithm walkthrough (with code) + - Hash table initialization + - Main compression loop detailed + - Byte matching algorithm + - Decompression algorithm (with code) + - Fast path optimization (with code) + - Variable-length integer decoding + - Overlapping copy (self-referential) + - Frame format implementation + - Frame header format specification + - Error handling patterns + - Performance optimization examples + - Safe vs unsafe variants + - No-std support details + + **Read this when:** You want to understand the actual code implementation + +--- + +## 🗂️ Repository Structure Summary + +### Source Code (`src/`) +- **lib.rs** (112 lines) - Root module, feature configuration +- **sink.rs** (200+ lines) - Buffer abstraction for compression/decompression +- **fastcpy.rs** & **fastcpy_unsafe.rs** - Fast memory copy implementations + +### Block Format (`src/block/`) +- **mod.rs** (177 lines) - Block format constants and errors +- **compress.rs** (999 lines) - **Compression algorithm** (hash-table based) +- **decompress.rs** (543 lines) - **Unsafe decompression** (pointer-based) +- **decompress_safe.rs** (400 lines) - **Safe decompression** (bounds-checked) +- **hashtable.rs** (248+ lines) - Hash tables (4K, 8K entries, 16/32-bit values) + +### Frame Format (`src/frame/`) +- **mod.rs** (111 lines) - Frame format definitions +- **compress.rs** (471 lines) - Streaming compression with checksums +- **decompress.rs** (448 lines) - Streaming decompression +- **header.rs** (411 lines) - Frame header parsing/generation + +### Testing & Benchmarking +- **benches/binggan_bench.rs** - Performance benchmarks (1000+ lines) +- **tests/tests.rs** - Integration tests (500+ lines) +- **fuzz/fuzz_targets/** - 6 fuzzing harnesses for robustness +- **miri_tests/** - UB detection tests +- **examples/** - 4 usage examples + +## 📊 Project Metrics + +| Metric | Value | +|--------|-------| +| **Version** | 0.12.0 | +| **Rust Edition** | 2021 | +| **Minimum Rust** | 1.81+ | +| **License** | MIT | +| **Total Source Lines** | ~7,000 (excluding tests) | +| **Compression Speed (66KB JSON)** | 1.2-1.6 GiB/s | +| **Decompression Speed (66KB JSON)** | 2.3-6.0 GiB/s | +| **Max Back-reference** | 64 KB | +| **Min Match Length** | 4 bytes | +| **Hash Table Size** | 4K-8K entries | +| **Memory Overhead** | 8-16 KB (hash table) | + +## 🔑 Key Features + +- ✅ Pure Rust implementation +- ✅ Optional safe/unsafe code paths +- ✅ No-std support (block format) +- ✅ Streaming support (frame format) +- ✅ Fast performance (competitive with C) +- ✅ Comprehensive testing (units + fuzz) +- ✅ Cross-validation with C implementation +- ✅ Property-based testing + +## 🎯 Algorithm Overviews + +### Compression (Hash-Table Based) +1. Hash 4-byte sequences +2. Look up previous occurrences +3. Find maximal matches +4. Encode: [token][literals][offset][match_len] +5. Token: [4-bit literal_len][4-bit match_len] + +### Decompression (Pointer-Based) +1. Read token byte +2. Parse literal length +3. Copy literals +4. Read 16-bit offset +5. Parse match length +6. Copy from output[pos-offset] (handles overlaps) +7. Repeat + +## 📈 Performance Characteristics + +### Compression +- JSON (66KB): **1,615 MiB/s** (unsafe) +- Text (10MB): **347 MiB/s** (unsafe) +- Safe variant: ~20-30% slower + +### Decompression +- JSON (66KB): **5,512 MiB/s** (unsafe) +- Text (10MB): **2,734 MiB/s** (unsafe) +- Competitive with C implementation (lzzz) + +## 🔧 Build Variants + +```bash +# Default (safe, with frame) +cargo build + +# Maximum performance (unsafe, no frame) +cargo build --no-default-features + +# Safe block format only +cargo build --no-default-features --features safe-encode,safe-decode + +# Unsafe with frame support +cargo build --no-default-features --features frame +``` + +## ✅ Testing + +- **Unit tests:** Integrated in source files +- **Integration tests:** `tests/tests.rs` +- **Fuzzing:** 6 fuzz targets for robustness +- **Miri:** UB detection on unsafe code +- **Cross-validation:** Against C implementation +- **Property-based:** Random input generation + +## 📚 Important TODOs in Code + +1. **compress.rs:261** - Remove bounds checks in backtrack_match +2. **decompress.rs:106** - Test fastcpy_unsafe in dictionary path +3. **frame/decompress.rs:248,271** - Lazy buffer initialization + +## 🔐 Safety + +- **Default:** Safe Rust (no unsafe) +- **Optional:** Unsafe for performance +- **Fuzzing:** Corrupted input detection +- **Miri:** UB detection +- **Feature gates:** Clear separation of safe/unsafe + +## 📖 How to Navigate + +**I want to understand:** +- **What this project does** → Start with QUICK_REFERENCE.md +- **How compression works** → See CODE_WALKTHROUGH.md compression section +- **How decompression works** → See CODE_WALKTHROUGH.md decompression section +- **Everything in detail** → Read EXPLORATION_REPORT.md +- **Code examples** → See CODE_WALKTHROUGH.md or examples/ directory +- **Performance** → See QUICK_REFERENCE.md performance table or README.md benchmarks + +## 🚀 Quick Start + +### Basic Compression +```rust +use lz4_flex::block::compress_prepend_size; + +let data = b"Hello, world!"; +let compressed = compress_prepend_size(data); +``` + +### Streaming Compression +```rust +use lz4_flex::frame::FrameEncoder; +use std::io::Write; + +let mut encoder = FrameEncoder::new(Vec::new()); +encoder.write_all(b"data")?; +let compressed = encoder.finish()?; +``` + +### Building +```bash +cargo build --release +``` + +### Testing +```bash +cargo test +cargo bench +cargo +nightly fuzz run fuzz_roundtrip +``` + +## 📝 Original Documentation + +- **README.md** - Project overview and performance benchmarks +- **CHANGELOG.md** - Version history +- **SECURITY.md** - Security policy +- **Cargo.toml** - Project configuration and dependencies + +--- + +## Navigation Quick Links + +- [EXPLORATION_REPORT.md](./EXPLORATION_REPORT.md) - Complete technical reference +- [QUICK_REFERENCE.md](./QUICK_REFERENCE.md) - Fast lookup guide +- [CODE_WALKTHROUGH.md](./CODE_WALKTHROUGH.md) - Detailed code examples +- [README.md](./README.md) - Original project documentation +- [src/](./src/) - Source code directory +- [benches/](./benches/) - Benchmark files +- [tests/](./tests/) - Test suite +- [examples/](./examples/) - Usage examples +- [fuzz/](./fuzz/) - Fuzzing harnesses + +--- + +**Last Updated:** March 14, 2024 +**Repository:** https://github.com/pseitz/lz4_flex +**Explored With:** Comprehensive code analysis and documentation generation + diff --git a/EXPLORATION_REPORT.md b/EXPLORATION_REPORT.md new file mode 100644 index 00000000..512af992 --- /dev/null +++ b/EXPLORATION_REPORT.md @@ -0,0 +1,965 @@ +```bash +# Ensure block format works without std +cargo build --no-default-features +# With safe-decode only +cargo build --no-default-features --features safe-decode +# With safe-encode only +cargo build --no-default-features --features safe-encode +# With frame support (requires std) +cargo build --no-default-features --features frame # Will fail - frame needs std +``` +### Miri (Undefined Behavior Detection): +```bash +# Check for UB in unsafe code +MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows" cargo +nightly miri test --no-default-features --features frame +``` +### Examples: +```bash +# Frame format compression (stdin -> stdout) +cargo run --example compress < input.txt > output.lz4 +# Frame format decompression +cargo run --example decompress < output.lz4 > restored.txt +# Block format compression +cargo run --example compress_block < input.txt > output.lz4 +# Block format decompression +cargo run --example decompress_block < output.lz4 > restored.txt +``` +### CI/CD (GitHub Actions): +**File: `.github/workflows/rust.yml`** +Runs on every push and PR: +1. Build with nightly Rust +2. No-std compilation tests (multiple feature combinations) +3. Unit tests (multiple feature combinations) +4. Nightly feature tests +5. Fuzzing (safe and unsafe variants) +6. Semver checking (API compatibility) +--- +## 9. PERFORMANCE-RELATED CODE COMMENTS AND TODOs +### Performance Optimizations in Code: +#### In `/src/block/compress.rs`: +**Line 132:** `#[cold]` attribute on tail counting function +- Marks infrequently used path as cold to optimize hot path +**Line 227:** Comment on byte-at-a-time write_integer +- "simple byte at a time implementation below is faster in most cases" +- Avoids compiler-generated memcpy which is slow for small copies +**Line 261:** TODO on bounds check elimination +```rust +// TODO: It should be possible remove all bounds checks, since we are walking +// backwards +``` +- Opportunity to optimize backtrack_match by proving bounds at compile time +**Line 317:** `#[inline(never)]` on compress_internal +```rust +// Intentionally avoid inlining. +// Empirical tests revealed it to be rarely better but often significantly detrimental. +``` +**Line 533:** Comment on literal copy optimization +```rust +// Note: This used to be a wild copy loop of 8 bytes, but the compiler consistently +// transformed it into a call to memcopy, which hurts performance significantly for +// small copies, which are common. +``` +- Uses match on copy length to prevent compiler auto-vectorization +#### In `/src/block/decompress.rs`: +**Line 39:** Comment on wild copy loop +```rust +// Note: if the compiler auto-vectorizes this it'll hurt performance! +// It's not the case for 16 bytes stepsize, but for 8 bytes. +``` +**Line 56:** `#[cfg_attr(feature = "nightly", optimize(size))]` +- Avoids loop unrolling in duplicate_overlapping to prevent branch explosion +**Line 106:** TODO on fastcpy_unsafe +```rust +// TODO test fastcpy_unsafe +``` +- Opportunity to use unsafe fast copy in dictionary path +#### In `/src/block/decompress_safe.rs`: +**Line 87:** `#[inline(always)]` comment +```rust +#[inline(always)] // (always) necessary to get the best performance in non LTO builds +``` +#### In `/src/block/hashtable.rs`: +**Line 66-67:** Comments on allocation optimization +```rust +// This generates more efficient assembly in contrast to Box::new(slice), because of an +// optimized call alloc_zeroed, vs. alloc + memset +``` +#### In `/src/frame/decompress.rs`: +**Line 248 & 271:** TODO on buffer initialization +```rust +// TODO: Attempt to avoid initialization of read buffer when... +``` +- Opportunity to lazy-initialize frame read buffer +### Feature Gate Performance: +1. **Safe-encode:** Uses bounds-checked slice access instead of unsafe pointer ops +2. **Safe-decode:** Uses bounds-checked indexing instead of unsafe pointer dereferencing +3. **Both safe:** Forbids all unsafe code via `#[forbid(unsafe_code)]` +4. **Neither safe:** Enables raw pointer manipulation for maximum speed +### Hot Path Optimization: +Main optimization in decompression (lines 259-326 of decompress.rs): +- Fast path when token fits in single byte +- Safe distance check ensures we won't overflow buffers +- Uses bulk 16-byte literal copies +- Uses bulk match copies with overlap detection +--- +## 10. KEY OBSERVATIONS +### Architecture Decisions: +1. **Hash Table Approach:** + - Fixed-size 4K-8K tables prevent unbounded memory usage + - Trade-off: More collisions but deterministic memory and cache behavior +2. **Safe/Unsafe Split:** + - Default to safe operations for correctness + - Optional unsafe for performance + - Clear feature gate boundaries +3. **Block vs Frame:** + - Block: Simple, in-memory, good for small data + - Frame: Streaming, supports checksums, good for large data + - Frame uses block compression internally +4. **No Allocator Dependency:** + - Uses `alloc` crate, not `std` + - Enables no_std + alloc support + - Frame format requires std for I/O +### Performance Characteristics: +- **Compression:** Limited by CPU throughput on I/O, memory copies +- **Decompression:** Very fast (3-6 GiB/s), mostly bottleneck-free +- **Memory:** Fixed by window size (64KB) + hash table (0.5-16KB) +- **Fast paths:** Heavily optimized for common case (tokens < 15, safe distance) +### Testing Strategy: +- Unit tests integrated in source +- Separate test file for integration testing +- Fuzzing against corrupted data +- Cross-validation with C implementation +- Property-based testing with random inputs +--- +## Summary of File Organization: +| File | Lines | Purpose | +|------|-------|---------| +| src/lib.rs | 112 | Module root, feature config | +| src/block/mod.rs | 177 | Block format defs, errors | +| src/block/compress.rs | 999 | Compression algorithm | +| src/block/decompress.rs | 543 | Unsafe decompression | +| src/block/decompress_safe.rs | 400 | Safe decompression | +| src/block/hashtable.rs | 248+ | Hash table for dups | +| src/frame/mod.rs | 111 | Frame format defs | +| src/frame/compress.rs | 471 | Frame compression | +| src/frame/decompress.rs | 448 | Frame decompression | +| src/frame/header.rs | 411 | Frame headers | +| src/sink.rs | 200+ | Buffer abstraction | +| src/fastcpy*.rs | 200+ | Fast memory copy | +| benches/binggan_bench.rs | 1000+ | Performance benchmarks | +| tests/tests.rs | 500+ | Integration tests | +| fuzz/fuzz_targets/*.rs | Various | Fuzzing harnesses | +**Total:** ~7000 lines of Rust code (excluding tests/benches) +EOF +cat /tmp/lz4_flex_exploration.md + EC=$? + echo "___BEGIN___COMMAND_DONE_MARKER___$EC" + } +___BEGIN___COMMAND_OUTPUT_MARKER___ +# LZ4_FLEX Repository Exploration Report + +## 1. PROJECT OVERVIEW + +**Project Name:** lz4_flex +**Version:** 0.12.0 +**Description:** Fastest LZ4 implementation in Rust, no unsafe by default +**Authors:** Pascal Seitz, Arthur Silva, ticki +**Repository:** https://github.com/pseitz/lz4_flex +**License:** MIT +**Rust Version Required:** 1.81+ +**Edition:** 2021 + +### Key Characteristics: +- Pure Rust, high performance implementation of LZ4 compression +- Originally based on redox-os' lz4 compression but now a complete rewrite +- Fast clean release build time (1.5s) +- Feature flags to control safe/unsafe code usage +- no-std support with block format +- 32-bit support +- Compared against: lzzzz (C via bindings), lz-fear, and snap (Snappy) + +### Performance Benchmarks (AMD Ryzen 7 5900HX): + +**66KB JSON:** +- lz4_flex unsafe w. unchecked_decode: 1615 MiB/s compress, 5973 MiB/s decompress, 0.2284 ratio +- lz4_flex unsafe: 1615 MiB/s compress, 5512 MiB/s decompress, 0.2284 ratio +- lz4_flex safe: 1272 MiB/s compress, 4540 MiB/s decompress, 0.2284 ratio +- lzzz (lz4 1.9.3): 1469 MiB/s compress, 5313 MiB/s decompress, 0.2283 ratio + +**10 MB dickens:** +- lz4_flex unsafe w. unchecked_decode: 347 MiB/s compress, 3168 MiB/s decompress, 0.6372 ratio +- lz4_flex unsafe: 347 MiB/s compress, 2734 MiB/s decompress, 0.6372 ratio +- lz4_flex safe: 259 MiB/s compress, 2338 MiB/s decompress, 0.6372 ratio +- lzzz: 324 MiB/s compress, 2759 MiB/s decompress, 0.6372 ratio + +--- + +## 2. DIRECTORY STRUCTURE (2 Levels Deep) + +``` +/home/runner/work/lz4_flex/lz4_flex/ +├── .git/ # Git repository +├── .github/ +│ └── workflows/ +│ └── rust.yml # GitHub Actions CI/CD +├── .gitignore +├── .travis.yml # Travis CI configuration (legacy) +├── .vscode/ # VS Code settings +├── benches/ # Benchmark files +│ ├── binggan_bench.rs # Performance benchmarks +│ ├── bench.rs # Alternative benchmark +│ ├── compression_*.txt # Test data files +│ ├── dickens.txt # Large text file for testing +│ └── *.svg # Benchmark result graphs +├── benchmarks/ # Additional benchmarking directory +│ └── src/ +│ └── main.rs +├── examples/ # Example programs +│ ├── compress.rs # Frame format compression example +│ ├── decompress.rs # Frame format decompression example +│ ├── compress_block.rs # Block format compression example +│ └── decompress_block.rs # Block format decompression example +├── fuzz/ # Fuzzing tests +│ └── fuzz_targets/ +│ ├── fuzz_roundtrip.rs +│ ├── fuzz_roundtrip_frame.rs +│ ├── fuzz_roundtrip_cpp_compress.rs +│ ├── fuzz_roundtrip_cpp_decompress.rs +│ ├── fuzz_decomp_corrupt_block.rs +│ └── fuzz_decomp_corrupt_frame.rs +├── logo.jpg # Project logo +├── lz4_bin/ # Binary tool +│ └── src/ +│ └── main.rs +├── miri_tests/ # Miri undefined behavior detection +│ └── src/ +│ └── main.rs +├── src/ # Main source code +│ ├── lib.rs # Library root +│ ├── sink.rs # Sink trait for buffering +│ ├── fastcpy.rs # Safe fast memory copy +│ ├── fastcpy_unsafe.rs # Unsafe fast memory copy +│ ├── block/ # LZ4 Block Format +│ │ ├── mod.rs # Block module definitions +│ │ ├── compress.rs # Block compression algorithm +│ │ ├── decompress.rs # Unsafe decompression +│ │ ├── decompress_safe.rs # Safe decompression +│ │ └── hashtable.rs # Hash table for duplicate detection +│ └── frame/ # LZ4 Frame Format +│ ├── mod.rs # Frame module definitions +│ ├── compress.rs # Frame compression +│ ├── decompress.rs # Frame decompression +│ └── header.rs # Frame header handling +├── tests/ # Integration tests +│ └── tests.rs +├── Cargo.toml # Cargo manifest +├── Cargo.lock # Lock file +├── README.md # Project documentation +├── CHANGELOG.md # Version history +├── SECURITY.md # Security policy +├── LICENSE # MIT License +├── _typos.toml # Typo checking config +└── cliff.toml # Changelog generation config +``` + +--- + +## 3. CARGO.TOML ANALYSIS + +**Key Configuration:** + +```toml +[package] +name = "lz4_flex" +version = "0.12.0" +edition = "2021" +rust-version = "1.81" +description = "Fastest LZ4 implementation in Rust, no unsafe by default." +keywords = ["compression", "lz4", "compress", "decompression", "decompress"] +``` + +**Features:** +- `safe-decode` (default): Uses only safe Rust for decompression +- `safe-encode` (default): Uses only safe Rust for compression +- `frame` (default): Support for LZ4 frame format (requires std) +- `std` (default): Standard library support +- `checked-decode` (default): Adds checks during decoding (important for untrusted input) +- `nightly` (disabled): Uses nightly compiler features + +**Default Features:** `["std", "safe-encode", "safe-decode", "frame", "checked-decode"]` + +**Dependencies:** +- `twox-hash` 2.0.0: Optional, for xxhash32 (frame format) + +**Dev Dependencies:** +- lzzzz 2.0.0: C LZ4 bindings for testing +- lz4-compress 0.1.1: Alternative Rust LZ4 for testing +- snap 1.1.0: Snappy compression for comparison +- serde_json 1.0.91: JSON for tests +- proptest 1.0.0: Property-based testing +- binggan 0.14.0: Benchmarking framework +- jemallocator 0.5.4: Memory allocator for benchmarks +- lz-fear: Git dependency for comparison + +**Build Profiles:** +```toml +[profile.bench] +codegen-units = 1 +lto = true +opt-level = 3 + +[profile.release] +codegen-units = 1 +opt-level = 3 +panic = "unwind" +``` + +--- + +## 4. MAIN SOURCE FILES AND PURPOSES + +### Core Library Files: + +**File: `/src/lib.rs` (112 lines)** +- Root module for the lz4_flex crate +- Exports public API from block and frame modules +- Configures lint rules and features +- Provides no_std support with alloc +- Modules: + - `block`: LZ4 Block Format + - `frame` (optional): LZ4 Frame Format (requires std) + - `sink`: Internal buffering abstraction + - `fastcpy` / `fastcpy_unsafe`: Fast memory copy implementations + +### Block Format Module: + +**File: `/src/block/mod.rs` (177 lines)** +- Defines LZ4 Block Format constants and error types +- Re-exports compression/decompression functions +- Block format constraints: + - `WINDOW_SIZE`: 64 KB (65,536 bytes) + - `MINMATCH`: 4 bytes (minimum duplicate length) + - `MFLIMIT`: 12 bytes (last match start offset) + - `LAST_LITERALS`: 5 bytes (must be literals at end) + - `MAX_DISTANCE`: 65,535 bytes (max back-reference distance) + +**Error Types:** +```rust +pub enum DecompressError { + OutputTooSmall { expected, actual }, + LiteralOutOfBounds, + ExpectedAnotherByte, + OffsetOutOfBounds, +} + +pub enum CompressError { + OutputTooSmall, +} +``` + +**File: `/src/block/compress.rs` (999 lines)** +- Main compression algorithm implementation +- Uses hashtable for fast duplicate detection +- Key functions: + - `compress_internal()`: Main compression loop + - `count_same_bytes()`: Counts matching bytes in streams + - `write_integer()`: Encodes variable-length integers + - `backtrack_match()`: Extends matches backward + - `handle_last_literals()`: Encodes final literal sequence + - `get_batch()`: Reads 4-byte chunks (safe or unsafe variants) + - `copy_literals_wild()`: Copies literal data with optimized variants + +**Algorithm Overview:** +1. Initializes hash table (4K or 8K entries) +2. Scans input for duplicate matches using hashing +3. For each match found: + - Extends backwards to find literal prefix + - Extends forward to find full match length + - Encodes as: [token byte][literal length][literals][match offset][match length] +4. Token byte contains upper 4 bits = literal length, lower 4 bits = match length +5. When lengths exceed 15, additional bytes encode overflow + +**Feature Toggles:** +- `#[cfg(feature = "safe-encode")]`: Forces safe Rust code, no unsafe blocks +- Without flag: Uses pointer operations for performance + +**Performance Optimizations:** +- Uses `#[inline(never)]` for `compress_internal` to prevent function bloat +- Incrementally increases step size when finding matches (exponential backoff) +- Wild copy of up to 24 bytes with intentional overread (safe within bounds) +- Avoids compiler auto-vectorization which hurts small copy performance + +**File: `/src/block/decompress.rs` (543 lines)** +- Unsafe decompression algorithm +- Fast path for common case (token fits in single byte, safe distance from end) +- Functions: + - `decompress_internal()`: Main decompression + - `duplicate()`: Handles self-referential copies + - `duplicate_overlapping()`: Handles overlapping match regions + - `read_integer_ptr()`: Decodes variable-length integers + - `read_u16_ptr()`: Reads 16-bit match offset + +**Key Optimizations:** +- Hot path check: if token fits and safe distance from end, use optimized copy +- Direct pointer manipulation for speed +- Wild 16-byte copies with bounds checking +- Handles dictionary mode for streaming decompression + +**File: `/src/block/decompress_safe.rs` (400 lines)** +- Safe Rust version of decompression +- Used when `safe-decode` feature is enabled +- Same algorithm as decompress.rs but using safe indexing +- Slightly slower but eliminates unsafe code + +**File: `/src/block/hashtable.rs` (248+ lines)** +- Hash table implementations for compression +- Hash function: `(sequence.wrapping_mul(2654435761) >> 16)` +- Two main implementations: + +```rust +// 4K entries, 16-bit values +pub struct HashTable4KU16 { + dict: Box<[u16; 4096]>, +} + +// 4K entries, 32-bit values +pub struct HashTable4K { + dict: Box<[u32; 4096]>, +} + +// 8K entries, 32-bit values (unused/commented) +pub struct HashTable8K { + dict: Box<[u32; 8192]>, +} +``` + +- `HashTable4KU16` used for small data (<64KB), reduces memory +- `HashTable4K` used for larger data, can store larger position values +- Bit shifting ensures hash values fit in table bounds + +### Frame Format Module: + +**File: `/src/frame/mod.rs` (111 lines)** +- Streaming compression/decompression using LZ4 Frame format +- Frame format supports multiple blocks with checksum validation +- Error types: + - CompressionError, DecompressionError + - WrongMagicNumber, UnsupportedBlocksize, UnsupportedVersion + - HeaderChecksumError, BlockChecksumError, ContentChecksumError + - DictionaryNotSupported, SkippableFrame + +**File: `/src/frame/compress.rs` (471 lines)** +- `FrameEncoder`: Implements `Write` trait for streaming compression +- `AutoFinishEncoder`: Auto-finishes on drop +- Methods: + - `new()`: Create encoder with default settings + - `with_frame_info()`: Create with custom frame info + - `write()`: Write and compress data (from Write trait) + - `finish()`: Finalize frame and flush buffers +- Configurable block size (64KB to 8MB) and mode (Independent/Linked) + +**File: `/src/frame/decompress.rs` (448 lines)** +- `FrameDecoder`: Implements `Read` trait for streaming decompression +- Methods: + - `new()`: Create decoder + - `read()`: Read and decompress data (from Read trait) +- Handles: + - Magic number validation (0x184D2204) + - Frame header parsing and checksum verification + - Block-by-block decompression with optional checksums + - Content checksum validation + - Skippable frames + +**File: `/src/frame/header.rs` (411 lines)** +- Frame format specification constants: + - `LZ4F_MAGIC_NUMBER`: 0x184D2204 (4 bytes) + - `MIN_FRAME_INFO_SIZE`: 7 bytes + - `MAX_FRAME_INFO_SIZE`: 19 bytes + - `BLOCK_INFO_SIZE`: 4 bytes +- Frame header flags: + - FLG_INDEPENDENT_BLOCKS: Blocks independent or linked + - FLG_BLOCK_CHECKSUMS: Optional per-block checksums + - FLG_CONTENT_SIZE: Frame size known in advance + - FLG_CONTENT_CHECKSUM: Optional frame checksum + - FLG_DICTIONARY_ID: External dictionary support +- `BlockSize` enum: Auto, 64KB, 256KB, 1MB, 4MB, 8MB +- `BlockMode` enum: Independent, Linked +- `FrameInfo` structure for configuration + +### Utility Files: + +**File: `/src/sink.rs` (200+ lines)** +- Abstraction for buffering output during compression/decompression +- `Sink` trait: Abstract buffer interface +- `SliceSink<'a>`: Preallocated slice-based sink +- Key methods: + - `push()`: Add single byte + - `extend_from_slice()`: Add multiple bytes + - `extend_from_slice_wild()`: Copy with potential overread (for performance) + - `extend_from_within()`: Self-referential copy + - `extend_from_within_overlapping()`: Overlapping copy + - `pos()`: Get current write position + - `capacity()`: Get total capacity + +**File: `/src/fastcpy.rs` (safe version)** +- Safe Rust fast memory copy for slices up to 32 bytes +- Uses "double copy trick" for optimal performance +- Uses usize-aligned reads/writes + +**File: `/src/fastcpy_unsafe.rs` (unsafe version)** +- Unsafe version of fastcpy +- Even more optimized using raw pointers +- Used in performance-critical decompression path + +--- + +## 5. COMPRESSION ALGORITHM DETAILS + +### How Compression Works: + +1. **Initialization:** + - Creates hash table (4K or 8K entries depending on input size) + - Initializes output buffer with capacity = input_len + overhead + +2. **Main Loop:** + - Scans through input looking for matches + - For each position, computes hash of 4-byte sequence + - Looks up hash in table to find previous occurrence + - Checks if previous occurrence actually matches + +3. **Match Finding:** + - Uses rolling hash: `(sequence.wrapping_mul(2654435761)) >> 16` + - Hashtable stores position of each hashed sequence + - Can find matches up to 64KB back (WINDOW_SIZE) + - Matches must be at least 4 bytes (MINMATCH) + +4. **Match Extension:** + - Extends match backward to include preceding literals + - Extends match forward to find full duplicate length + - Counts identical bytes between current and candidate position + +5. **Encoding Format:** + - Token byte: [4-bit literal length][4-bit match length] + - If literal_len >= 15: additional bytes encode overflow (255 + 255 + ... + remainder) + - Raw literal bytes + - 16-bit little-endian match offset + - If match_len >= 15: additional bytes encode overflow + +6. **Restrictions (per LZ4 spec):** + - Last match must start at least 12 bytes before end (MFLIMIT) + - Last 5 bytes must be literals (LAST_LITERALS) + - Blocks < 13 bytes independent cannot be compressed (need at least 1 prior byte) + +### Decompression Process: + +1. **Token Parsing:** + - Read token byte + - Upper 4 bits = literal length + - Lower 4 bits = match length + - If either nibble = 0xF, read additional bytes for overflow + +2. **Fast Path (Hot Loop):** + - If token fits (both nibbles < 15) + - AND input still has 2+ bytes for offset + - AND output has capacity for literals + match + - Use optimized 16-byte literal copy + 18-byte match copy + +3. **Literal Copying:** + - Copy raw bytes directly from input to output + - Variable length from token + extension bytes + +4. **Match Copying:** + - Read 16-bit little-endian offset (distance back in output buffer) + - Copy from output[pos - offset] to output[pos] + - Must handle overlapping regions (self-referential copies) + - Uses 16-byte wild copies when possible for performance + +5. **Error Checking (when safe-decode enabled):** + - Verify output buffer has capacity + - Verify input has sufficient bytes + - Verify offset is within bounds + - Verify no out-of-order reads + +--- + +## 6. BENCHMARKS + +### Benchmark Files: + +**File: `/benches/binggan_bench.rs` (1000+ lines)** +- Primary benchmark using `binggan` framework +- Compares against: lz4_fear, lzzz (C LZ4), snap (Snappy) +- Benchmarks both block and frame formats +- Test datasets: + - compression_1k.txt (1 KB) + - compression_34k.txt (34 KB) + - compression_65k.txt (65 KB) + - compression_66k_JSON.txt (66 KB - JSON) + - dickens.txt (10 MB - text) + - logo.jpg (95 KB - binary) + +- Memory allocator: Uses Jemalloc for accurate measurements +- Metrics: Throughput (MiB/s), peak memory allocation +- Generates SVG graphs of results + +**Benchmark Commands:** +```bash +# Safe mode (default) +cargo bench + +# Unsafe mode +cargo bench --no-default-features +``` + +**Output Graphs:** +- compress_bench.svg: Compression speed comparison +- decompress_bench.svg: Decompression speed comparison +- compress_bench_safe.svg: Safe-mode compression +- decompress_bench_safe.svg: Safe-mode decompression + +### Benchmark Comparisons: + +The benchmarks compare lz4_flex against: +- **lzzz**: C language LZ4 (v1.9.3) via Rust bindings +- **lz-fear**: Pure Rust LZ4 implementation +- **snap**: Snappy compression algorithm (for reference) + +--- + +## 7. TESTS + +### Test File: `/tests/tests.rs` (500+ lines)** + +**Test Categories:** + +1. **Roundtrip Tests:** + - `test_roundtrip()`: Compress then decompress, verify matches original + - Tests with multiple datasets (1K to 10MB) + - Tests both block and frame formats + - Tests both Independent and Linked block modes + +2. **Cross-Implementation Tests:** + - Compress with lz4_flex, decompress with C (lzzzz) + - Compress with C, decompress with lz4_flex + - Verify results match + +3. **Block Format Tests:** + - Small blocks (< 13 bytes) + - Medium blocks (compressed JSON, 66KB) + - Large blocks (10MB text) + - Various size combinations + +4. **Frame Format Tests:** + - Independent blocks + - Linked blocks (referencing previous blocks) + - Different block sizes (64KB, 256KB, 1MB, etc.) + - Content checksum verification + - Block checksum verification + +5. **Edge Case Tests:** + - Empty input + - Single byte input + - Incompressible data + - Highly repetitive data + - Random data + +6. **Dictionary Tests:** + - External dictionary for decompression + - Prefix mode (data continuing from previous block) + +7. **Property-Based Tests:** + - `proptest` for random input generation + - Verifies correctness across random inputs + +### Test Datasets: + +Included in `/benches/`: +- compression_1k.txt +- compression_34k.txt +- compression_65k.txt +- compression_66k_JSON.txt +- dickens.txt (10MB) +- (embedded as byte arrays via `include_bytes!()`) + +### Fuzzing: + +Located in `/fuzz/fuzz_targets/`: + +1. **fuzz_roundtrip.rs**: Roundtrip compression/decompression +2. **fuzz_roundtrip_frame.rs**: Frame format roundtrip +3. **fuzz_roundtrip_cpp_compress.rs**: C compresses, Rust decompresses +4. **fuzz_roundtrip_cpp_decompress.rs**: Rust compresses, C decompresses +5. **fuzz_decomp_corrupt_block.rs**: Decompression of corrupted data +6. **fuzz_decomp_corrupt_frame.rs**: Frame decompression of corrupted data + +**Fuzzing Commands:** +```bash +cargo +nightly fuzz run fuzz_roundtrip -- -max_total_time=30 +cargo +nightly fuzz run fuzz_decomp_corrupt_block -- -max_total_time=30 +``` + +--- + +## 8. BUILD AND TEST INSTRUCTIONS + +### Build: + +```bash +# Debug build +cargo build + +# Release build (optimized) +cargo build --release + +# With all features +cargo build --all-features + +# No default features (maximum performance) +cargo build --no-default-features + +# Specific features +cargo build --no-default-features --features frame +cargo build --features safe-encode --features safe-decode +``` + +### Testing: + +```bash +# Run all tests +cargo test + +# Tests with all features +cargo test --all-features + +# Specific feature tests +cargo test --features safe-encode +cargo test --features safe-decode +cargo test --no-default-features --features frame + +# Tests with nightly compiler +cargo +nightly test --features nightly + +# Run fuzzing (requires nightly) +cargo +nightly fuzz run fuzz_roundtrip -- -max_total_time=30 +``` + +### Benchmarking: + +```bash +# Run benchmarks (default features) +cargo bench + +# Benchmarks with maximum performance (unsafe) +cargo bench --no-default-features + +# Run with specific benchmark +cargo bench --bench binggan_bench +``` + +### No-std Verification: + +```bash +# Ensure block format works without std +cargo build --no-default-features + +# With safe-decode only +cargo build --no-default-features --features safe-decode + +# With safe-encode only +cargo build --no-default-features --features safe-encode + +# With frame support (requires std) +cargo build --no-default-features --features frame # Will fail - frame needs std +``` + +### Miri (Undefined Behavior Detection): + +```bash +# Check for UB in unsafe code +MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows" cargo +nightly miri test --no-default-features --features frame +``` + +### Examples: + +```bash +# Frame format compression (stdin -> stdout) +cargo run --example compress < input.txt > output.lz4 + +# Frame format decompression +cargo run --example decompress < output.lz4 > restored.txt + +# Block format compression +cargo run --example compress_block < input.txt > output.lz4 + +# Block format decompression +cargo run --example decompress_block < output.lz4 > restored.txt +``` + +### CI/CD (GitHub Actions): + +**File: `.github/workflows/rust.yml`** + +Runs on every push and PR: +1. Build with nightly Rust +2. No-std compilation tests (multiple feature combinations) +3. Unit tests (multiple feature combinations) +4. Nightly feature tests +5. Fuzzing (safe and unsafe variants) +6. Semver checking (API compatibility) + +--- + +## 9. PERFORMANCE-RELATED CODE COMMENTS AND TODOs + +### Performance Optimizations in Code: + +#### In `/src/block/compress.rs`: + +**Line 132:** `#[cold]` attribute on tail counting function +- Marks infrequently used path as cold to optimize hot path + +**Line 227:** Comment on byte-at-a-time write_integer +- "simple byte at a time implementation below is faster in most cases" +- Avoids compiler-generated memcpy which is slow for small copies + +**Line 261:** TODO on bounds check elimination +```rust +// TODO: It should be possible remove all bounds checks, since we are walking +// backwards +``` +- Opportunity to optimize backtrack_match by proving bounds at compile time + +**Line 317:** `#[inline(never)]` on compress_internal +```rust +// Intentionally avoid inlining. +// Empirical tests revealed it to be rarely better but often significantly detrimental. +``` + +**Line 533:** Comment on literal copy optimization +```rust +// Note: This used to be a wild copy loop of 8 bytes, but the compiler consistently +// transformed it into a call to memcopy, which hurts performance significantly for +// small copies, which are common. +``` +- Uses match on copy length to prevent compiler auto-vectorization + +#### In `/src/block/decompress.rs`: + +**Line 39:** Comment on wild copy loop +```rust +// Note: if the compiler auto-vectorizes this it'll hurt performance! +// It's not the case for 16 bytes stepsize, but for 8 bytes. +``` + +**Line 56:** `#[cfg_attr(feature = "nightly", optimize(size))]` +- Avoids loop unrolling in duplicate_overlapping to prevent branch explosion + +**Line 106:** TODO on fastcpy_unsafe +```rust +// TODO test fastcpy_unsafe +``` +- Opportunity to use unsafe fast copy in dictionary path + +#### In `/src/block/decompress_safe.rs`: + +**Line 87:** `#[inline(always)]` comment +```rust +#[inline(always)] // (always) necessary to get the best performance in non LTO builds +``` + +#### In `/src/block/hashtable.rs`: + +**Line 66-67:** Comments on allocation optimization +```rust +// This generates more efficient assembly in contrast to Box::new(slice), because of an +// optimized call alloc_zeroed, vs. alloc + memset +``` + +#### In `/src/frame/decompress.rs`: + +**Line 248 & 271:** TODO on buffer initialization +```rust +// TODO: Attempt to avoid initialization of read buffer when... +``` +- Opportunity to lazy-initialize frame read buffer + +### Feature Gate Performance: + +1. **Safe-encode:** Uses bounds-checked slice access instead of unsafe pointer ops +2. **Safe-decode:** Uses bounds-checked indexing instead of unsafe pointer dereferencing +3. **Both safe:** Forbids all unsafe code via `#[forbid(unsafe_code)]` +4. **Neither safe:** Enables raw pointer manipulation for maximum speed + +### Hot Path Optimization: + +Main optimization in decompression (lines 259-326 of decompress.rs): +- Fast path when token fits in single byte +- Safe distance check ensures we won't overflow buffers +- Uses bulk 16-byte literal copies +- Uses bulk match copies with overlap detection + +--- + +## 10. KEY OBSERVATIONS + +### Architecture Decisions: + +1. **Hash Table Approach:** + - Fixed-size 4K-8K tables prevent unbounded memory usage + - Trade-off: More collisions but deterministic memory and cache behavior + +2. **Safe/Unsafe Split:** + - Default to safe operations for correctness + - Optional unsafe for performance + - Clear feature gate boundaries + +3. **Block vs Frame:** + - Block: Simple, in-memory, good for small data + - Frame: Streaming, supports checksums, good for large data + - Frame uses block compression internally + +4. **No Allocator Dependency:** + - Uses `alloc` crate, not `std` + - Enables no_std + alloc support + - Frame format requires std for I/O + +### Performance Characteristics: + +- **Compression:** Limited by CPU throughput on I/O, memory copies +- **Decompression:** Very fast (3-6 GiB/s), mostly bottleneck-free +- **Memory:** Fixed by window size (64KB) + hash table (0.5-16KB) +- **Fast paths:** Heavily optimized for common case (tokens < 15, safe distance) + +### Testing Strategy: + +- Unit tests integrated in source +- Separate test file for integration testing +- Fuzzing against corrupted data +- Cross-validation with C implementation +- Property-based testing with random inputs + +--- + +## Summary of File Organization: + +| File | Lines | Purpose | +|------|-------|---------| +| src/lib.rs | 112 | Module root, feature config | +| src/block/mod.rs | 177 | Block format defs, errors | +| src/block/compress.rs | 999 | Compression algorithm | +| src/block/decompress.rs | 543 | Unsafe decompression | +| src/block/decompress_safe.rs | 400 | Safe decompression | +| src/block/hashtable.rs | 248+ | Hash table for dups | +| src/frame/mod.rs | 111 | Frame format defs | +| src/frame/compress.rs | 471 | Frame compression | +| src/frame/decompress.rs | 448 | Frame decompression | +| src/frame/header.rs | 411 | Frame headers | +| src/sink.rs | 200+ | Buffer abstraction | +| src/fastcpy*.rs | 200+ | Fast memory copy | +| benches/binggan_bench.rs | 1000+ | Performance benchmarks | +| tests/tests.rs | 500+ | Integration tests | +| fuzz/fuzz_targets/*.rs | Various | Fuzzing harnesses | + +**Total:** ~7000 lines of Rust code (excluding tests/benches) + +___BEGIN___COMMAND_DONE_MARKER___0 diff --git a/QUICK_REFERENCE.md b/QUICK_REFERENCE.md new file mode 100644 index 00000000..03d34879 --- /dev/null +++ b/QUICK_REFERENCE.md @@ -0,0 +1,211 @@ +# LZ4_FLEX Quick Reference Guide + +## What is LZ4_FLEX? +A high-performance LZ4 compression library in pure Rust with optional safe/unsafe variants. +- **Compression Speed:** 1.2-1.6 GiB/s (safe/unsafe) +- **Decompression Speed:** 2.3-6.0 GiB/s (safe/unsafe) +- **Features:** Block format (in-memory), Frame format (streaming) + +## Quick Start + +### Basic Usage (Block Format) +```rust +use lz4_flex::block::{compress_prepend_size, decompress_size_prepended}; + +let input = b"Hello, world!"; +let compressed = compress_prepend_size(input); +let decompressed = decompress_size_prepended(&compressed)?; +assert_eq!(input, &decompressed[..]); +``` + +### Streaming (Frame Format) +```rust +use std::io::{Read, Write}; +use lz4_flex::frame::{FrameEncoder, FrameDecoder}; + +// Compress +let mut encoder = FrameEncoder::new(Vec::new()); +encoder.write_all(b"data")?; +let compressed = encoder.finish()?; + +// Decompress +let mut decoder = FrameDecoder::new(&compressed[..]); +let mut decompressed = Vec::new(); +decoder.read_to_end(&mut decompressed)?; +``` + +## Building + +```bash +cargo build --release # Safe+frame (default) +cargo build --no-default-features # Unsafe, no frame +cargo build --no-default-features --features frame # Unsafe + frame +``` + +## Testing + +```bash +cargo test # All tests +cargo test --no-default-features # Unsafe code tests +cargo +nightly fuzz run fuzz_roundtrip # Fuzzing +``` + +## Benchmarking + +```bash +cargo bench # Safe mode +cargo bench --no-default-features # Unsafe mode +``` + +## Key Files + +| File | Purpose | +|------|---------| +| `src/block/compress.rs` (999 lines) | Compression algorithm using hash tables | +| `src/block/decompress.rs` (543 lines) | Fast unsafe decompression | +| `src/block/decompress_safe.rs` (400 lines) | Safe decompression (bounds-checked) | +| `src/frame/compress.rs` (471 lines) | Streaming compression with checksums | +| `src/frame/decompress.rs` (448 lines) | Streaming decompression | +| `src/block/hashtable.rs` | Hash table for duplicate detection | +| `src/sink.rs` | Buffer abstraction for compression | + +## Features + +- `safe-encode` (default): Safe Rust for compression +- `safe-decode` (default): Safe Rust for decompression +- `frame` (default): Streaming LZ4 frame format (requires std) +- `std` (default): Standard library support +- `checked-decode`: Additional decompression safety checks + +## Compression Algorithm + +1. Hash each 4-byte sequence +2. Look up hash to find previous matches +3. Encode as: `[token][literal_length][literals][match_offset][match_length]` +4. Token byte: `[4-bit lit_len][4-bit match_len]` +5. If length >= 15: use extension bytes (255 + 255 + ... + remainder) + +**Constants:** +- Minimum match: 4 bytes +- Maximum back-reference distance: 64 KB +- Window size: 64 KB + +## Decompression Algorithm + +1. Read token byte +2. Parse literal length (upper 4 bits) +3. Copy literals from input +4. Parse match offset (16-bit little-endian) +5. Parse match length (lower 4 bits) +6. Copy from output[pos - offset] (handles overlapping regions) +7. Repeat + +**Fast Path Optimization:** +- If token fits in single byte AND safe distance from end +- Use 16-byte literal copy + 18-byte match copy + +## Performance Characteristics + +| Test | Unsafe | Safe | vs C (lzzz) | +|------|--------|------|------------| +| 66KB JSON compress | 1615 MiB/s | 1272 MiB/s | +10% | +| 66KB JSON decompress | 5512 MiB/s | 4540 MiB/s | +4% | +| 10MB text compress | 347 MiB/s | 259 MiB/s | +7% | +| 10MB text decompress | 2734 MiB/s | 2338 MiB/s | -1% | + +## Common Performance TODOs + +1. **Line 261 of compress.rs:** Remove bounds checks in backtrack_match +2. **Line 106 of decompress.rs:** Test fastcpy_unsafe in dictionary path +3. **Lines 248, 271 of frame/decompress.rs:** Lazy buffer initialization +4. **Line 534 of compress.rs:** Prevent compiler auto-vectorization of literal copy + +## Error Types + +**DecompressError:** +- `OutputTooSmall { expected, actual }` +- `LiteralOutOfBounds` +- `ExpectedAnotherByte` +- `OffsetOutOfBounds` + +**CompressError:** +- `OutputTooSmall` + +## Block Format Constraints + +- Last match must start ≥12 bytes before end +- Last 5 bytes must be literals +- Independent blocks <13 bytes can't be compressed +- Maximum distance: 65,535 bytes + +## Frame Format Features + +- Magic number: `0x184D2204` (4 bytes) +- Optional block checksums (xxhash32) +- Optional content checksum +- Optional content size header +- Independent or linked blocks +- Configurable block sizes: 64KB, 256KB, 1MB, 4MB, 8MB + +## CI/CD + +**GitHub Actions (`.github/workflows/rust.yml`):** +- Tests with nightly Rust +- No-std compilation verification +- Fuzzing tests (safe + unsafe) +- Semver checking + +## Dependencies + +**Runtime:** +- `twox-hash` 2.0.0 (optional, frame format) +- `alloc` (required) + +**Dev:** +- lzzzz (C LZ4 bindings for testing) +- snap (Snappy for comparison) +- proptest (property-based testing) +- binggan (benchmarking) + +## Examples + +Located in `/examples/`: +- `compress.rs` - Frame format compression +- `decompress.rs` - Frame format decompression +- `compress_block.rs` - Block format compression +- `decompress_block.rs` - Block format decompression + +Run: `cargo run --example compress < input.txt > output.lz4` + +## Fuzzing + +```bash +cargo +nightly fuzz run fuzz_roundtrip -- -max_total_time=30 +cargo +nightly fuzz run fuzz_decomp_corrupt_block -- -max_total_time=30 +cargo +nightly fuzz run fuzz_roundtrip_cpp_compress -- -max_total_time=30 +``` + +## Miri (Undefined Behavior Detection) + +```bash +MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows" \ + cargo +nightly miri test --no-default-features --features frame +``` + +## Memory Usage + +- Hash table: 4K entries × 2-4 bytes = 8-16 KB +- Window size: 64 KB (back-reference limit) +- Output buffer: input_size + overhead +- No streaming allocations + +## Testing Datasets + +Included in `/benches/`: +- compression_1k.txt (1 KB) +- compression_34k.txt (34 KB) +- compression_65k.txt (65 KB) +- compression_66k_JSON.txt (66 KB - JSON, highly compressible) +- dickens.txt (10 MB - English text) +- logo.jpg (95 KB - binary/image) + From cd4d647b2f72779f89e69398709a08168027d724 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:09:31 +0000 Subject: [PATCH 3/5] Remove accidentally committed exploration files Co-authored-by: PSeitz <1109503+PSeitz@users.noreply.github.com> --- CODE_WALKTHROUGH.md | 531 ----------------------- EXPLORATION_INDEX.md | 260 ------------ EXPLORATION_REPORT.md | 965 ------------------------------------------ QUICK_REFERENCE.md | 211 --------- 4 files changed, 1967 deletions(-) delete mode 100644 CODE_WALKTHROUGH.md delete mode 100644 EXPLORATION_INDEX.md delete mode 100644 EXPLORATION_REPORT.md delete mode 100644 QUICK_REFERENCE.md diff --git a/CODE_WALKTHROUGH.md b/CODE_WALKTHROUGH.md deleted file mode 100644 index 385a8393..00000000 --- a/CODE_WALKTHROUGH.md +++ /dev/null @@ -1,531 +0,0 @@ -# LZ4_FLEX Code Walkthrough - -This document shows key code snippets and explains the core algorithms. - -## Compression Algorithm Walkthrough - -### Hash Table Initialization - -**File: `src/block/hashtable.rs`** - -```rust -// 4K entry hashtable using 16-bit values (8 KB total) -pub struct HashTable4KU16 { - dict: Box<[u16; 4096]>, -} - -impl HashTable4KU16 { - pub fn new() -> Self { - // Optimized allocation: uses alloc_zeroed instead of alloc + memset - let dict = alloc::vec![0; 4096] - .into_boxed_slice() - .try_into() - .unwrap(); - Self { dict } - } -} - -// Hash function: converts 4-byte sequence to 4K table index -fn hash(sequence: u32) -> u32 { - (sequence.wrapping_mul(2654435761_u32)) >> 16 // FNV-like hash -} -``` - -### Compression Main Loop - -**File: `src/block/compress.rs` (lines 318-489)** - -```rust -pub(crate) fn compress_internal( - input: &[u8], - input_pos: usize, - output: &mut S, - dict: &mut T, - ext_dict: &[u8], - input_stream_offset: usize, -) -> Result { - // ... validation ... - - let mut literal_start = input_pos; - let mut cur = input_pos; - - loop { - let mut non_match_count = 1 << INCREASE_STEPSIZE_BITSHIFT; // 32 - - // Inner loop: search for matches - loop { - let step_size = non_match_count >> INCREASE_STEPSIZE_BITSHIFT; - non_match_count += 1; - - cur = next_cur; - next_cur += step_size; - - if cur > end_pos_check { // Past safe zone - handle_last_literals(output, input, literal_start); - return Ok(output.pos() - output_start_pos); - } - - // Hash the 4-byte sequence at current position - let hash = T::get_hash_at(input, cur); - let candidate = dict.get_at(hash); // Look up in hash table - dict.put_at(hash, cur + input_stream_offset); // Store current position - - // Check if we can reach this candidate (within 64KB window) - if input_stream_offset + cur - candidate > MAX_DISTANCE { - continue; - } - - // Verify the bytes actually match (not just hash collision) - let cand_bytes: u32 = get_batch(candidate_source, candidate); - let curr_bytes: u32 = get_batch(input, cur); - - if cand_bytes == curr_bytes { - break; // Found a match! - } - } - - // Extend match backwards to include preceding literals - backtrack_match( - input, - &mut cur, - literal_start, - candidate_source, - &mut candidate, - ); - - let lit_len = cur - literal_start; - - // Extend match forwards to find complete duplicate length - cur += MINMATCH; // Skip already-matched 4 bytes - candidate += MINMATCH; - let duplicate_length = count_same_bytes(input, &mut cur, candidate_source, candidate); - - // Encode: [token][literal_len?][literals][offset][match_len?] - let token = token_from_literal_and_match_length(lit_len, duplicate_length); - - push_byte(output, token); - if lit_len >= 0xF { - write_integer(output, lit_len - 0xF); - } - copy_literals_wild(output, input, literal_start, lit_len); - push_u16(output, offset); // 16-bit offset in little-endian - if duplicate_length >= 0xF { - write_integer(output, duplicate_length - 0xF); - } - - literal_start = cur; - } -} -``` - -### Byte Matching - -**File: `src/block/compress.rs` (lines 98-145)** - -```rust -#[inline] -#[cfg(feature = "safe-encode")] -fn count_same_bytes( - input: &[u8], - cur: &mut usize, - source: &[u8], - candidate: usize -) -> usize { - const USIZE_SIZE: usize = core::mem::size_of::(); - let cur_slice = &input[*cur..input.len() - END_OFFSET]; - let cand_slice = &source[candidate..]; - - let mut num = 0; - // Compare usize-sized chunks for better performance - for (block1, block2) in cur_slice.chunks_exact(USIZE_SIZE) - .zip(cand_slice.chunks_exact(USIZE_SIZE)) - { - let input_block = usize::from_ne_bytes(block1.try_into().unwrap()); - let match_block = usize::from_ne_bytes(block2.try_into().unwrap()); - - if input_block == match_block { - num += USIZE_SIZE; - } else { - // Found difference - count matching bytes via bit operations - let diff = input_block ^ match_block; - num += (diff.to_le().trailing_zeros() / 8) as usize; - *cur += num; - return num; - } - } - - // Handle remaining bytes (1-7) - num += count_same_bytes_tail(cur_slice, cand_slice, num); - *cur += num; - num -} -``` - -## Decompression Algorithm Walkthrough - -### Fast Path (Hot Loop) - -**File: `src/block/decompress.rs` (lines 259-326)** - -```rust -// Check if we can use optimized fast path -if does_token_fit(token) // both nibbles < 15 - && (input_ptr as usize) <= input_ptr_safe as usize - && output_ptr < safe_output_ptr -{ - // Token fits: literal and match lengths both < 15, no overflow bytes needed - let literal_length = (token >> 4) as usize; - let mut match_length = MINMATCH + (token & 0xF) as usize; - - // Copy literal section - bulk 16-byte copy (may overread safely) - unsafe { - core::ptr::copy_nonoverlapping(input_ptr, output_ptr, 16); - input_ptr = input_ptr.add(literal_length); - output_ptr = output_ptr.add(literal_length); - } - - // Read 16-bit match offset (little-endian) - let offset = read_u16_ptr(&mut input_ptr) as usize; - - let output_len = unsafe { output_ptr.offset_from(output_base) as usize }; - let offset = offset.min(output_len + ext_dict.len()); - - // Calculate source pointer for the match - let start_ptr = unsafe { output_ptr.sub(offset) }; - - // Copy match - handle overlapping regions - if offset >= match_length { - // Non-overlapping: simple copy of 18 bytes - unsafe { - core::ptr::copy(start_ptr, output_ptr, 18); - output_ptr = output_ptr.add(match_length); - } - } else { - // Overlapping: must copy byte-by-byte - unsafe { - duplicate_overlapping(&mut output_ptr, start_ptr, match_length); - } - } - - continue; // Back to fast path -} - -// Slow path for complex tokens (see below...) -``` - -### Variable-Length Integer Decoding - -**File: `src/block/decompress.rs` (lines 131-162)** - -```rust -pub(super) fn read_integer_ptr( - input_ptr: &mut *const u8, - input_ptr_end: *const u8, -) -> Result { - let mut n: usize = 0; - - loop { - // Read next byte - if *input_ptr >= input_ptr_end { - return Err(DecompressError::ExpectedAnotherByte); - } - - let extra = unsafe { input_ptr.read() }; - *input_ptr = unsafe { input_ptr.add(1) }; - n += extra as usize; - - // If byte < 255, we're done. Otherwise, continue. - // Example: 255 + 255 + 10 = 520 - if extra != 0xFF { - break; - } - } - - Ok(n) -} -``` - -### Overlapping Copy (Self-Referential) - -**File: `src/block/decompress.rs` (lines 56-87)** - -```rust -#[inline] -#[cfg_attr(feature = "nightly", optimize(size))] // Prevent unrolling -unsafe fn duplicate_overlapping( - output_ptr: &mut *mut u8, - mut start: *const u8, - match_length: usize, -) { - // Safety: Write zero to handle edge case where output_ptr == start - // This matches the C reference implementation behavior - output_ptr.write(0u8); - let dst_ptr_end = output_ptr.add(match_length); - - // Copy byte-by-byte - allows self-referential copies - // Example: offset=1, match_len=5, data=[A] -> [A,A,A,A,A] - while output_ptr.add(1) < dst_ptr_end { - // Manual unroll (2 iterations) to prevent compiler unrolling - core::ptr::copy(start, *output_ptr, 1); - start = start.add(1); - *output_ptr = output_ptr.add(1); - - core::ptr::copy(start, *output_ptr, 1); - start = start.add(1); - *output_ptr = output_ptr.add(1); - } - - if *output_ptr < dst_ptr_end { - core::ptr::copy(start, *output_ptr, 1); - *output_ptr = output_ptr.add(1); - } -} -``` - -## Frame Format Implementation - -### Frame Encoder - -**File: `src/frame/compress.rs`** - -```rust -pub struct FrameEncoder { - writer: W, - context: CompressionContext, - frame_info: FrameInfo, -} - -impl FrameEncoder { - pub fn new(writer: W) -> Self { - let frame_info = FrameInfo::new(); - Self { - writer, - context: CompressionContext::new(), - frame_info, - } - } - - pub fn with_frame_info(writer: W, frame_info: FrameInfo) -> Self { - Self { - writer, - context: CompressionContext::new(), - frame_info, - } - } -} - -impl Write for FrameEncoder { - fn write(&mut self, buf: &[u8]) -> io::Result { - // 1. Write frame header (if first write) - // 2. Compress input into blocks - // 3. Write block header (size, checksum) - // 4. Write compressed block - // 5. Return bytes written - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - self.writer.flush() - } -} - -impl FrameEncoder { - pub fn finish(mut self) -> io::Result> { - // Write end-of-frame marker (4 bytes of zeros) - // Write content checksum if enabled - // Return completed compressed data - Ok(Vec::new()) - } -} -``` - -### Frame Header Format - -**File: `src/frame/header.rs` (lines 27-100)** - -```rust -const LZ4F_MAGIC_NUMBER: u32 = 0x184D2204; // Magic number (4 bytes) - -// Frame descriptor layout: -// Byte 0: FLG (flags) -// - Bits 7-6: Version (01 = v1) -// - Bit 5: Independent blocks (0=linked, 1=independent) -// - Bit 4: Block checksums enabled -// - Bit 3: Content size present -// - Bit 2: Content checksum enabled -// - Bit 1: Reserved (must be 0) -// - Bit 0: Dictionary ID present -// Byte 1: BD (block size info) -// - Bits 7-4: Block size ID (4=64KB, 5=256KB, 6=1MB, 7=4MB, 8=8MB) -// - Bits 3-0: Reserved (must be 0) -// [4-8 bytes]: Content size (optional, if FLG bit 3 = 1) -// [4 bytes]: Dictionary ID (optional, if FLG bit 0 = 1) -// [1 byte]: FLG checksum (CRC32 of FLG and BD bytes) - -#[derive(Clone, Copy, Debug)] -pub enum BlockSize { - Auto = 0, - Max64KB = 4, - Max256KB = 5, - Max1MB = 6, - Max4MB = 7, - Max8MB = 8, -} - -#[derive(Clone, Copy, Debug)] -pub enum BlockMode { - Independent, // Blocks don't reference previous blocks - Linked, // Blocks can reference previous blocks -} -``` - -## Error Handling - -### Decompression Error Handling - -**File: `src/block/mod.rs` (lines 79-96)** - -```rust -pub enum DecompressError { - /// Output buffer too small for decompressed data - OutputTooSmall { - expected: usize, - actual: usize, - }, - /// Literal is out of bounds of the input - LiteralOutOfBounds, - /// Expected another byte, but none found. - ExpectedAnotherByte, - /// Deduplication offset out of bounds (not in buffer). - OffsetOutOfBounds, -} - -// Usage: -match decompress(&compressed, uncompressed_size) { - Ok(data) => println!("Decompressed: {}", String::from_utf8_lossy(&data)), - Err(DecompressError::OutputTooSmall { expected, actual }) => { - eprintln!("Need {} bytes, got {}", expected, actual); - }, - Err(e) => eprintln!("Error: {}", e), -} -``` - -## Performance Optimization Examples - -### 1. Token Fitting Check - -**File: `src/block/decompress.rs` (lines 188-195)** - -```rust -#[inline] -fn does_token_fit(token: u8) -> bool { - // Check if literal length < 15 AND match length < 15 - // If true, no extension bytes needed - saves branch in hot path - !((token & 0xF0) == 0xF0 || (token & 0x0F) == 0x0F) -} -``` - -### 2. Literal Copy with Overread - -**File: `src/block/compress.rs` (lines 527-545)** - -```rust -#[inline] -#[cfg(not(feature = "safe-encode"))] -fn copy_literals_wild(output: &mut impl Sink, input: &[u8], input_start: usize, len: usize) { - unsafe { - // Copy more bytes than needed, but bounds are checked - // This prevents compiler from generating slow memcpy - let start_ptr = input.as_ptr().add(input_start); - match len { - 0..=8 => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), 8), - 9..=16 => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), 16), - 17..=24 => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), 24), - _ => core::ptr::copy_nonoverlapping(start_ptr, output.pos_mut_ptr(), len), - } - output.set_pos(output.pos() + len); - } -} -``` - -### 3. Hash Function - -**File: `src/block/hashtable.rs` (lines 19-34)** - -```rust -#[inline] -fn hash(sequence: u32) -> u32 { - // FNV-1 like hash: multiply by prime, shift to get 16-bit index - // This distributes values well across hash table - (sequence.wrapping_mul(2654435761_u32)) >> 16 -} - -#[cfg(target_pointer_width = "64")] -#[inline] -fn hash5(sequence: usize) -> u32 { - // 64-bit hash for better distribution - let primebytes = if cfg!(target_endian = "little") { - 889523592379_usize - } else { - 11400714785074694791_usize - }; - (((sequence << 24).wrapping_mul(primebytes)) >> 48) as u32 -} -``` - -## Safe vs Unsafe Variants - -### Safe Encoding (Default) - -**File: `src/block/compress.rs` (lines 41-43, 492-495)** - -```rust -#[cfg(feature = "safe-encode")] -pub(super) fn get_batch(input: &[u8], n: usize) -> u32 { - u32::from_ne_bytes(input[n..n + 4].try_into().unwrap()) // Bounds checked -} - -#[cfg(feature = "safe-encode")] -fn push_byte(output: &mut impl Sink, el: u8) { - output.push(el); // Uses trait method (no unsafe) -} -``` - -### Unsafe Encoding (Optional) - -**File: `src/block/compress.rs` (lines 34-36, 498-504)** - -```rust -#[cfg(not(feature = "safe-encode"))] -pub(super) fn get_batch(input: &[u8], n: usize) -> u32 { - unsafe { read_u32_ptr(input.as_ptr().add(n)) } // Raw pointer read -} - -#[cfg(not(feature = "safe-encode"))] -fn push_byte(output: &mut impl Sink, el: u8) { - unsafe { - core::ptr::write(output.pos_mut_ptr(), el); // Direct memory write - output.set_pos(output.pos() + 1); - } -} -``` - -## No-std Support - -**File: `src/lib.rs` (lines 71-78)** - -```rust -#![deny(warnings)] -#![deny(missing_docs)] -#![cfg_attr(not(feature = "std"), no_std)] // Conditional no_std -#![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(feature = "nightly", feature(optimize_attribute))] - -#[cfg_attr(test, macro_use)] -extern crate alloc; // Always require alloc, optionally std -``` - -This allows: -- Block format: works with just `alloc` (no `std`) -- Frame format: requires `std::io::{Read, Write}` - diff --git a/EXPLORATION_INDEX.md b/EXPLORATION_INDEX.md deleted file mode 100644 index d82b3e43..00000000 --- a/EXPLORATION_INDEX.md +++ /dev/null @@ -1,260 +0,0 @@ -# LZ4_FLEX Repository Exploration - Complete Index - -This index provides navigation to all exploration documents created during the thorough analysis of the lz4_flex repository. - -## 📋 Documentation Files Created - -### 1. **EXPLORATION_REPORT.md** (33 KB, 965 lines) - - **Most Comprehensive** - Complete technical analysis - - **Contents:** - - Project overview and performance benchmarks - - Complete directory structure (2 levels deep) - - Cargo.toml configuration details - - Main source files purposes and analysis - - Compression algorithm implementation details - - Decompression algorithm walkthrough - - Benchmarking framework overview - - Complete test suite documentation - - Build and test instructions - - Performance-related comments and TODOs in code - - Key architectural observations - - **Read this when:** You need comprehensive technical understanding of the entire project - ---- - -### 2. **QUICK_REFERENCE.md** (6 KB) - - **Fast Navigation** - Quick lookup guide - - **Contents:** - - What is LZ4_FLEX (quick description) - - Quick start examples (block and frame format) - - Build commands - - Testing commands - - Key files summary table - - Feature flags - - Compression/decompression algorithms (brief) - - Performance table - - Error types - - Constraints and features - - CI/CD summary - - Memory usage info - - Testing datasets - - **Read this when:** You need quick reference or examples of how to use the library - ---- - -### 3. **CODE_WALKTHROUGH.md** (15 KB) - - **Detailed Code Examples** - Deep dive into implementations - - **Contents:** - - Compression algorithm walkthrough (with code) - - Hash table initialization - - Main compression loop detailed - - Byte matching algorithm - - Decompression algorithm (with code) - - Fast path optimization (with code) - - Variable-length integer decoding - - Overlapping copy (self-referential) - - Frame format implementation - - Frame header format specification - - Error handling patterns - - Performance optimization examples - - Safe vs unsafe variants - - No-std support details - - **Read this when:** You want to understand the actual code implementation - ---- - -## 🗂️ Repository Structure Summary - -### Source Code (`src/`) -- **lib.rs** (112 lines) - Root module, feature configuration -- **sink.rs** (200+ lines) - Buffer abstraction for compression/decompression -- **fastcpy.rs** & **fastcpy_unsafe.rs** - Fast memory copy implementations - -### Block Format (`src/block/`) -- **mod.rs** (177 lines) - Block format constants and errors -- **compress.rs** (999 lines) - **Compression algorithm** (hash-table based) -- **decompress.rs** (543 lines) - **Unsafe decompression** (pointer-based) -- **decompress_safe.rs** (400 lines) - **Safe decompression** (bounds-checked) -- **hashtable.rs** (248+ lines) - Hash tables (4K, 8K entries, 16/32-bit values) - -### Frame Format (`src/frame/`) -- **mod.rs** (111 lines) - Frame format definitions -- **compress.rs** (471 lines) - Streaming compression with checksums -- **decompress.rs** (448 lines) - Streaming decompression -- **header.rs** (411 lines) - Frame header parsing/generation - -### Testing & Benchmarking -- **benches/binggan_bench.rs** - Performance benchmarks (1000+ lines) -- **tests/tests.rs** - Integration tests (500+ lines) -- **fuzz/fuzz_targets/** - 6 fuzzing harnesses for robustness -- **miri_tests/** - UB detection tests -- **examples/** - 4 usage examples - -## 📊 Project Metrics - -| Metric | Value | -|--------|-------| -| **Version** | 0.12.0 | -| **Rust Edition** | 2021 | -| **Minimum Rust** | 1.81+ | -| **License** | MIT | -| **Total Source Lines** | ~7,000 (excluding tests) | -| **Compression Speed (66KB JSON)** | 1.2-1.6 GiB/s | -| **Decompression Speed (66KB JSON)** | 2.3-6.0 GiB/s | -| **Max Back-reference** | 64 KB | -| **Min Match Length** | 4 bytes | -| **Hash Table Size** | 4K-8K entries | -| **Memory Overhead** | 8-16 KB (hash table) | - -## 🔑 Key Features - -- ✅ Pure Rust implementation -- ✅ Optional safe/unsafe code paths -- ✅ No-std support (block format) -- ✅ Streaming support (frame format) -- ✅ Fast performance (competitive with C) -- ✅ Comprehensive testing (units + fuzz) -- ✅ Cross-validation with C implementation -- ✅ Property-based testing - -## 🎯 Algorithm Overviews - -### Compression (Hash-Table Based) -1. Hash 4-byte sequences -2. Look up previous occurrences -3. Find maximal matches -4. Encode: [token][literals][offset][match_len] -5. Token: [4-bit literal_len][4-bit match_len] - -### Decompression (Pointer-Based) -1. Read token byte -2. Parse literal length -3. Copy literals -4. Read 16-bit offset -5. Parse match length -6. Copy from output[pos-offset] (handles overlaps) -7. Repeat - -## 📈 Performance Characteristics - -### Compression -- JSON (66KB): **1,615 MiB/s** (unsafe) -- Text (10MB): **347 MiB/s** (unsafe) -- Safe variant: ~20-30% slower - -### Decompression -- JSON (66KB): **5,512 MiB/s** (unsafe) -- Text (10MB): **2,734 MiB/s** (unsafe) -- Competitive with C implementation (lzzz) - -## 🔧 Build Variants - -```bash -# Default (safe, with frame) -cargo build - -# Maximum performance (unsafe, no frame) -cargo build --no-default-features - -# Safe block format only -cargo build --no-default-features --features safe-encode,safe-decode - -# Unsafe with frame support -cargo build --no-default-features --features frame -``` - -## ✅ Testing - -- **Unit tests:** Integrated in source files -- **Integration tests:** `tests/tests.rs` -- **Fuzzing:** 6 fuzz targets for robustness -- **Miri:** UB detection on unsafe code -- **Cross-validation:** Against C implementation -- **Property-based:** Random input generation - -## 📚 Important TODOs in Code - -1. **compress.rs:261** - Remove bounds checks in backtrack_match -2. **decompress.rs:106** - Test fastcpy_unsafe in dictionary path -3. **frame/decompress.rs:248,271** - Lazy buffer initialization - -## 🔐 Safety - -- **Default:** Safe Rust (no unsafe) -- **Optional:** Unsafe for performance -- **Fuzzing:** Corrupted input detection -- **Miri:** UB detection -- **Feature gates:** Clear separation of safe/unsafe - -## 📖 How to Navigate - -**I want to understand:** -- **What this project does** → Start with QUICK_REFERENCE.md -- **How compression works** → See CODE_WALKTHROUGH.md compression section -- **How decompression works** → See CODE_WALKTHROUGH.md decompression section -- **Everything in detail** → Read EXPLORATION_REPORT.md -- **Code examples** → See CODE_WALKTHROUGH.md or examples/ directory -- **Performance** → See QUICK_REFERENCE.md performance table or README.md benchmarks - -## 🚀 Quick Start - -### Basic Compression -```rust -use lz4_flex::block::compress_prepend_size; - -let data = b"Hello, world!"; -let compressed = compress_prepend_size(data); -``` - -### Streaming Compression -```rust -use lz4_flex::frame::FrameEncoder; -use std::io::Write; - -let mut encoder = FrameEncoder::new(Vec::new()); -encoder.write_all(b"data")?; -let compressed = encoder.finish()?; -``` - -### Building -```bash -cargo build --release -``` - -### Testing -```bash -cargo test -cargo bench -cargo +nightly fuzz run fuzz_roundtrip -``` - -## 📝 Original Documentation - -- **README.md** - Project overview and performance benchmarks -- **CHANGELOG.md** - Version history -- **SECURITY.md** - Security policy -- **Cargo.toml** - Project configuration and dependencies - ---- - -## Navigation Quick Links - -- [EXPLORATION_REPORT.md](./EXPLORATION_REPORT.md) - Complete technical reference -- [QUICK_REFERENCE.md](./QUICK_REFERENCE.md) - Fast lookup guide -- [CODE_WALKTHROUGH.md](./CODE_WALKTHROUGH.md) - Detailed code examples -- [README.md](./README.md) - Original project documentation -- [src/](./src/) - Source code directory -- [benches/](./benches/) - Benchmark files -- [tests/](./tests/) - Test suite -- [examples/](./examples/) - Usage examples -- [fuzz/](./fuzz/) - Fuzzing harnesses - ---- - -**Last Updated:** March 14, 2024 -**Repository:** https://github.com/pseitz/lz4_flex -**Explored With:** Comprehensive code analysis and documentation generation - diff --git a/EXPLORATION_REPORT.md b/EXPLORATION_REPORT.md deleted file mode 100644 index 512af992..00000000 --- a/EXPLORATION_REPORT.md +++ /dev/null @@ -1,965 +0,0 @@ -```bash -# Ensure block format works without std -cargo build --no-default-features -# With safe-decode only -cargo build --no-default-features --features safe-decode -# With safe-encode only -cargo build --no-default-features --features safe-encode -# With frame support (requires std) -cargo build --no-default-features --features frame # Will fail - frame needs std -``` -### Miri (Undefined Behavior Detection): -```bash -# Check for UB in unsafe code -MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows" cargo +nightly miri test --no-default-features --features frame -``` -### Examples: -```bash -# Frame format compression (stdin -> stdout) -cargo run --example compress < input.txt > output.lz4 -# Frame format decompression -cargo run --example decompress < output.lz4 > restored.txt -# Block format compression -cargo run --example compress_block < input.txt > output.lz4 -# Block format decompression -cargo run --example decompress_block < output.lz4 > restored.txt -``` -### CI/CD (GitHub Actions): -**File: `.github/workflows/rust.yml`** -Runs on every push and PR: -1. Build with nightly Rust -2. No-std compilation tests (multiple feature combinations) -3. Unit tests (multiple feature combinations) -4. Nightly feature tests -5. Fuzzing (safe and unsafe variants) -6. Semver checking (API compatibility) ---- -## 9. PERFORMANCE-RELATED CODE COMMENTS AND TODOs -### Performance Optimizations in Code: -#### In `/src/block/compress.rs`: -**Line 132:** `#[cold]` attribute on tail counting function -- Marks infrequently used path as cold to optimize hot path -**Line 227:** Comment on byte-at-a-time write_integer -- "simple byte at a time implementation below is faster in most cases" -- Avoids compiler-generated memcpy which is slow for small copies -**Line 261:** TODO on bounds check elimination -```rust -// TODO: It should be possible remove all bounds checks, since we are walking -// backwards -``` -- Opportunity to optimize backtrack_match by proving bounds at compile time -**Line 317:** `#[inline(never)]` on compress_internal -```rust -// Intentionally avoid inlining. -// Empirical tests revealed it to be rarely better but often significantly detrimental. -``` -**Line 533:** Comment on literal copy optimization -```rust -// Note: This used to be a wild copy loop of 8 bytes, but the compiler consistently -// transformed it into a call to memcopy, which hurts performance significantly for -// small copies, which are common. -``` -- Uses match on copy length to prevent compiler auto-vectorization -#### In `/src/block/decompress.rs`: -**Line 39:** Comment on wild copy loop -```rust -// Note: if the compiler auto-vectorizes this it'll hurt performance! -// It's not the case for 16 bytes stepsize, but for 8 bytes. -``` -**Line 56:** `#[cfg_attr(feature = "nightly", optimize(size))]` -- Avoids loop unrolling in duplicate_overlapping to prevent branch explosion -**Line 106:** TODO on fastcpy_unsafe -```rust -// TODO test fastcpy_unsafe -``` -- Opportunity to use unsafe fast copy in dictionary path -#### In `/src/block/decompress_safe.rs`: -**Line 87:** `#[inline(always)]` comment -```rust -#[inline(always)] // (always) necessary to get the best performance in non LTO builds -``` -#### In `/src/block/hashtable.rs`: -**Line 66-67:** Comments on allocation optimization -```rust -// This generates more efficient assembly in contrast to Box::new(slice), because of an -// optimized call alloc_zeroed, vs. alloc + memset -``` -#### In `/src/frame/decompress.rs`: -**Line 248 & 271:** TODO on buffer initialization -```rust -// TODO: Attempt to avoid initialization of read buffer when... -``` -- Opportunity to lazy-initialize frame read buffer -### Feature Gate Performance: -1. **Safe-encode:** Uses bounds-checked slice access instead of unsafe pointer ops -2. **Safe-decode:** Uses bounds-checked indexing instead of unsafe pointer dereferencing -3. **Both safe:** Forbids all unsafe code via `#[forbid(unsafe_code)]` -4. **Neither safe:** Enables raw pointer manipulation for maximum speed -### Hot Path Optimization: -Main optimization in decompression (lines 259-326 of decompress.rs): -- Fast path when token fits in single byte -- Safe distance check ensures we won't overflow buffers -- Uses bulk 16-byte literal copies -- Uses bulk match copies with overlap detection ---- -## 10. KEY OBSERVATIONS -### Architecture Decisions: -1. **Hash Table Approach:** - - Fixed-size 4K-8K tables prevent unbounded memory usage - - Trade-off: More collisions but deterministic memory and cache behavior -2. **Safe/Unsafe Split:** - - Default to safe operations for correctness - - Optional unsafe for performance - - Clear feature gate boundaries -3. **Block vs Frame:** - - Block: Simple, in-memory, good for small data - - Frame: Streaming, supports checksums, good for large data - - Frame uses block compression internally -4. **No Allocator Dependency:** - - Uses `alloc` crate, not `std` - - Enables no_std + alloc support - - Frame format requires std for I/O -### Performance Characteristics: -- **Compression:** Limited by CPU throughput on I/O, memory copies -- **Decompression:** Very fast (3-6 GiB/s), mostly bottleneck-free -- **Memory:** Fixed by window size (64KB) + hash table (0.5-16KB) -- **Fast paths:** Heavily optimized for common case (tokens < 15, safe distance) -### Testing Strategy: -- Unit tests integrated in source -- Separate test file for integration testing -- Fuzzing against corrupted data -- Cross-validation with C implementation -- Property-based testing with random inputs ---- -## Summary of File Organization: -| File | Lines | Purpose | -|------|-------|---------| -| src/lib.rs | 112 | Module root, feature config | -| src/block/mod.rs | 177 | Block format defs, errors | -| src/block/compress.rs | 999 | Compression algorithm | -| src/block/decompress.rs | 543 | Unsafe decompression | -| src/block/decompress_safe.rs | 400 | Safe decompression | -| src/block/hashtable.rs | 248+ | Hash table for dups | -| src/frame/mod.rs | 111 | Frame format defs | -| src/frame/compress.rs | 471 | Frame compression | -| src/frame/decompress.rs | 448 | Frame decompression | -| src/frame/header.rs | 411 | Frame headers | -| src/sink.rs | 200+ | Buffer abstraction | -| src/fastcpy*.rs | 200+ | Fast memory copy | -| benches/binggan_bench.rs | 1000+ | Performance benchmarks | -| tests/tests.rs | 500+ | Integration tests | -| fuzz/fuzz_targets/*.rs | Various | Fuzzing harnesses | -**Total:** ~7000 lines of Rust code (excluding tests/benches) -EOF -cat /tmp/lz4_flex_exploration.md - EC=$? - echo "___BEGIN___COMMAND_DONE_MARKER___$EC" - } -___BEGIN___COMMAND_OUTPUT_MARKER___ -# LZ4_FLEX Repository Exploration Report - -## 1. PROJECT OVERVIEW - -**Project Name:** lz4_flex -**Version:** 0.12.0 -**Description:** Fastest LZ4 implementation in Rust, no unsafe by default -**Authors:** Pascal Seitz, Arthur Silva, ticki -**Repository:** https://github.com/pseitz/lz4_flex -**License:** MIT -**Rust Version Required:** 1.81+ -**Edition:** 2021 - -### Key Characteristics: -- Pure Rust, high performance implementation of LZ4 compression -- Originally based on redox-os' lz4 compression but now a complete rewrite -- Fast clean release build time (1.5s) -- Feature flags to control safe/unsafe code usage -- no-std support with block format -- 32-bit support -- Compared against: lzzzz (C via bindings), lz-fear, and snap (Snappy) - -### Performance Benchmarks (AMD Ryzen 7 5900HX): - -**66KB JSON:** -- lz4_flex unsafe w. unchecked_decode: 1615 MiB/s compress, 5973 MiB/s decompress, 0.2284 ratio -- lz4_flex unsafe: 1615 MiB/s compress, 5512 MiB/s decompress, 0.2284 ratio -- lz4_flex safe: 1272 MiB/s compress, 4540 MiB/s decompress, 0.2284 ratio -- lzzz (lz4 1.9.3): 1469 MiB/s compress, 5313 MiB/s decompress, 0.2283 ratio - -**10 MB dickens:** -- lz4_flex unsafe w. unchecked_decode: 347 MiB/s compress, 3168 MiB/s decompress, 0.6372 ratio -- lz4_flex unsafe: 347 MiB/s compress, 2734 MiB/s decompress, 0.6372 ratio -- lz4_flex safe: 259 MiB/s compress, 2338 MiB/s decompress, 0.6372 ratio -- lzzz: 324 MiB/s compress, 2759 MiB/s decompress, 0.6372 ratio - ---- - -## 2. DIRECTORY STRUCTURE (2 Levels Deep) - -``` -/home/runner/work/lz4_flex/lz4_flex/ -├── .git/ # Git repository -├── .github/ -│ └── workflows/ -│ └── rust.yml # GitHub Actions CI/CD -├── .gitignore -├── .travis.yml # Travis CI configuration (legacy) -├── .vscode/ # VS Code settings -├── benches/ # Benchmark files -│ ├── binggan_bench.rs # Performance benchmarks -│ ├── bench.rs # Alternative benchmark -│ ├── compression_*.txt # Test data files -│ ├── dickens.txt # Large text file for testing -│ └── *.svg # Benchmark result graphs -├── benchmarks/ # Additional benchmarking directory -│ └── src/ -│ └── main.rs -├── examples/ # Example programs -│ ├── compress.rs # Frame format compression example -│ ├── decompress.rs # Frame format decompression example -│ ├── compress_block.rs # Block format compression example -│ └── decompress_block.rs # Block format decompression example -├── fuzz/ # Fuzzing tests -│ └── fuzz_targets/ -│ ├── fuzz_roundtrip.rs -│ ├── fuzz_roundtrip_frame.rs -│ ├── fuzz_roundtrip_cpp_compress.rs -│ ├── fuzz_roundtrip_cpp_decompress.rs -│ ├── fuzz_decomp_corrupt_block.rs -│ └── fuzz_decomp_corrupt_frame.rs -├── logo.jpg # Project logo -├── lz4_bin/ # Binary tool -│ └── src/ -│ └── main.rs -├── miri_tests/ # Miri undefined behavior detection -│ └── src/ -│ └── main.rs -├── src/ # Main source code -│ ├── lib.rs # Library root -│ ├── sink.rs # Sink trait for buffering -│ ├── fastcpy.rs # Safe fast memory copy -│ ├── fastcpy_unsafe.rs # Unsafe fast memory copy -│ ├── block/ # LZ4 Block Format -│ │ ├── mod.rs # Block module definitions -│ │ ├── compress.rs # Block compression algorithm -│ │ ├── decompress.rs # Unsafe decompression -│ │ ├── decompress_safe.rs # Safe decompression -│ │ └── hashtable.rs # Hash table for duplicate detection -│ └── frame/ # LZ4 Frame Format -│ ├── mod.rs # Frame module definitions -│ ├── compress.rs # Frame compression -│ ├── decompress.rs # Frame decompression -│ └── header.rs # Frame header handling -├── tests/ # Integration tests -│ └── tests.rs -├── Cargo.toml # Cargo manifest -├── Cargo.lock # Lock file -├── README.md # Project documentation -├── CHANGELOG.md # Version history -├── SECURITY.md # Security policy -├── LICENSE # MIT License -├── _typos.toml # Typo checking config -└── cliff.toml # Changelog generation config -``` - ---- - -## 3. CARGO.TOML ANALYSIS - -**Key Configuration:** - -```toml -[package] -name = "lz4_flex" -version = "0.12.0" -edition = "2021" -rust-version = "1.81" -description = "Fastest LZ4 implementation in Rust, no unsafe by default." -keywords = ["compression", "lz4", "compress", "decompression", "decompress"] -``` - -**Features:** -- `safe-decode` (default): Uses only safe Rust for decompression -- `safe-encode` (default): Uses only safe Rust for compression -- `frame` (default): Support for LZ4 frame format (requires std) -- `std` (default): Standard library support -- `checked-decode` (default): Adds checks during decoding (important for untrusted input) -- `nightly` (disabled): Uses nightly compiler features - -**Default Features:** `["std", "safe-encode", "safe-decode", "frame", "checked-decode"]` - -**Dependencies:** -- `twox-hash` 2.0.0: Optional, for xxhash32 (frame format) - -**Dev Dependencies:** -- lzzzz 2.0.0: C LZ4 bindings for testing -- lz4-compress 0.1.1: Alternative Rust LZ4 for testing -- snap 1.1.0: Snappy compression for comparison -- serde_json 1.0.91: JSON for tests -- proptest 1.0.0: Property-based testing -- binggan 0.14.0: Benchmarking framework -- jemallocator 0.5.4: Memory allocator for benchmarks -- lz-fear: Git dependency for comparison - -**Build Profiles:** -```toml -[profile.bench] -codegen-units = 1 -lto = true -opt-level = 3 - -[profile.release] -codegen-units = 1 -opt-level = 3 -panic = "unwind" -``` - ---- - -## 4. MAIN SOURCE FILES AND PURPOSES - -### Core Library Files: - -**File: `/src/lib.rs` (112 lines)** -- Root module for the lz4_flex crate -- Exports public API from block and frame modules -- Configures lint rules and features -- Provides no_std support with alloc -- Modules: - - `block`: LZ4 Block Format - - `frame` (optional): LZ4 Frame Format (requires std) - - `sink`: Internal buffering abstraction - - `fastcpy` / `fastcpy_unsafe`: Fast memory copy implementations - -### Block Format Module: - -**File: `/src/block/mod.rs` (177 lines)** -- Defines LZ4 Block Format constants and error types -- Re-exports compression/decompression functions -- Block format constraints: - - `WINDOW_SIZE`: 64 KB (65,536 bytes) - - `MINMATCH`: 4 bytes (minimum duplicate length) - - `MFLIMIT`: 12 bytes (last match start offset) - - `LAST_LITERALS`: 5 bytes (must be literals at end) - - `MAX_DISTANCE`: 65,535 bytes (max back-reference distance) - -**Error Types:** -```rust -pub enum DecompressError { - OutputTooSmall { expected, actual }, - LiteralOutOfBounds, - ExpectedAnotherByte, - OffsetOutOfBounds, -} - -pub enum CompressError { - OutputTooSmall, -} -``` - -**File: `/src/block/compress.rs` (999 lines)** -- Main compression algorithm implementation -- Uses hashtable for fast duplicate detection -- Key functions: - - `compress_internal()`: Main compression loop - - `count_same_bytes()`: Counts matching bytes in streams - - `write_integer()`: Encodes variable-length integers - - `backtrack_match()`: Extends matches backward - - `handle_last_literals()`: Encodes final literal sequence - - `get_batch()`: Reads 4-byte chunks (safe or unsafe variants) - - `copy_literals_wild()`: Copies literal data with optimized variants - -**Algorithm Overview:** -1. Initializes hash table (4K or 8K entries) -2. Scans input for duplicate matches using hashing -3. For each match found: - - Extends backwards to find literal prefix - - Extends forward to find full match length - - Encodes as: [token byte][literal length][literals][match offset][match length] -4. Token byte contains upper 4 bits = literal length, lower 4 bits = match length -5. When lengths exceed 15, additional bytes encode overflow - -**Feature Toggles:** -- `#[cfg(feature = "safe-encode")]`: Forces safe Rust code, no unsafe blocks -- Without flag: Uses pointer operations for performance - -**Performance Optimizations:** -- Uses `#[inline(never)]` for `compress_internal` to prevent function bloat -- Incrementally increases step size when finding matches (exponential backoff) -- Wild copy of up to 24 bytes with intentional overread (safe within bounds) -- Avoids compiler auto-vectorization which hurts small copy performance - -**File: `/src/block/decompress.rs` (543 lines)** -- Unsafe decompression algorithm -- Fast path for common case (token fits in single byte, safe distance from end) -- Functions: - - `decompress_internal()`: Main decompression - - `duplicate()`: Handles self-referential copies - - `duplicate_overlapping()`: Handles overlapping match regions - - `read_integer_ptr()`: Decodes variable-length integers - - `read_u16_ptr()`: Reads 16-bit match offset - -**Key Optimizations:** -- Hot path check: if token fits and safe distance from end, use optimized copy -- Direct pointer manipulation for speed -- Wild 16-byte copies with bounds checking -- Handles dictionary mode for streaming decompression - -**File: `/src/block/decompress_safe.rs` (400 lines)** -- Safe Rust version of decompression -- Used when `safe-decode` feature is enabled -- Same algorithm as decompress.rs but using safe indexing -- Slightly slower but eliminates unsafe code - -**File: `/src/block/hashtable.rs` (248+ lines)** -- Hash table implementations for compression -- Hash function: `(sequence.wrapping_mul(2654435761) >> 16)` -- Two main implementations: - -```rust -// 4K entries, 16-bit values -pub struct HashTable4KU16 { - dict: Box<[u16; 4096]>, -} - -// 4K entries, 32-bit values -pub struct HashTable4K { - dict: Box<[u32; 4096]>, -} - -// 8K entries, 32-bit values (unused/commented) -pub struct HashTable8K { - dict: Box<[u32; 8192]>, -} -``` - -- `HashTable4KU16` used for small data (<64KB), reduces memory -- `HashTable4K` used for larger data, can store larger position values -- Bit shifting ensures hash values fit in table bounds - -### Frame Format Module: - -**File: `/src/frame/mod.rs` (111 lines)** -- Streaming compression/decompression using LZ4 Frame format -- Frame format supports multiple blocks with checksum validation -- Error types: - - CompressionError, DecompressionError - - WrongMagicNumber, UnsupportedBlocksize, UnsupportedVersion - - HeaderChecksumError, BlockChecksumError, ContentChecksumError - - DictionaryNotSupported, SkippableFrame - -**File: `/src/frame/compress.rs` (471 lines)** -- `FrameEncoder`: Implements `Write` trait for streaming compression -- `AutoFinishEncoder`: Auto-finishes on drop -- Methods: - - `new()`: Create encoder with default settings - - `with_frame_info()`: Create with custom frame info - - `write()`: Write and compress data (from Write trait) - - `finish()`: Finalize frame and flush buffers -- Configurable block size (64KB to 8MB) and mode (Independent/Linked) - -**File: `/src/frame/decompress.rs` (448 lines)** -- `FrameDecoder`: Implements `Read` trait for streaming decompression -- Methods: - - `new()`: Create decoder - - `read()`: Read and decompress data (from Read trait) -- Handles: - - Magic number validation (0x184D2204) - - Frame header parsing and checksum verification - - Block-by-block decompression with optional checksums - - Content checksum validation - - Skippable frames - -**File: `/src/frame/header.rs` (411 lines)** -- Frame format specification constants: - - `LZ4F_MAGIC_NUMBER`: 0x184D2204 (4 bytes) - - `MIN_FRAME_INFO_SIZE`: 7 bytes - - `MAX_FRAME_INFO_SIZE`: 19 bytes - - `BLOCK_INFO_SIZE`: 4 bytes -- Frame header flags: - - FLG_INDEPENDENT_BLOCKS: Blocks independent or linked - - FLG_BLOCK_CHECKSUMS: Optional per-block checksums - - FLG_CONTENT_SIZE: Frame size known in advance - - FLG_CONTENT_CHECKSUM: Optional frame checksum - - FLG_DICTIONARY_ID: External dictionary support -- `BlockSize` enum: Auto, 64KB, 256KB, 1MB, 4MB, 8MB -- `BlockMode` enum: Independent, Linked -- `FrameInfo` structure for configuration - -### Utility Files: - -**File: `/src/sink.rs` (200+ lines)** -- Abstraction for buffering output during compression/decompression -- `Sink` trait: Abstract buffer interface -- `SliceSink<'a>`: Preallocated slice-based sink -- Key methods: - - `push()`: Add single byte - - `extend_from_slice()`: Add multiple bytes - - `extend_from_slice_wild()`: Copy with potential overread (for performance) - - `extend_from_within()`: Self-referential copy - - `extend_from_within_overlapping()`: Overlapping copy - - `pos()`: Get current write position - - `capacity()`: Get total capacity - -**File: `/src/fastcpy.rs` (safe version)** -- Safe Rust fast memory copy for slices up to 32 bytes -- Uses "double copy trick" for optimal performance -- Uses usize-aligned reads/writes - -**File: `/src/fastcpy_unsafe.rs` (unsafe version)** -- Unsafe version of fastcpy -- Even more optimized using raw pointers -- Used in performance-critical decompression path - ---- - -## 5. COMPRESSION ALGORITHM DETAILS - -### How Compression Works: - -1. **Initialization:** - - Creates hash table (4K or 8K entries depending on input size) - - Initializes output buffer with capacity = input_len + overhead - -2. **Main Loop:** - - Scans through input looking for matches - - For each position, computes hash of 4-byte sequence - - Looks up hash in table to find previous occurrence - - Checks if previous occurrence actually matches - -3. **Match Finding:** - - Uses rolling hash: `(sequence.wrapping_mul(2654435761)) >> 16` - - Hashtable stores position of each hashed sequence - - Can find matches up to 64KB back (WINDOW_SIZE) - - Matches must be at least 4 bytes (MINMATCH) - -4. **Match Extension:** - - Extends match backward to include preceding literals - - Extends match forward to find full duplicate length - - Counts identical bytes between current and candidate position - -5. **Encoding Format:** - - Token byte: [4-bit literal length][4-bit match length] - - If literal_len >= 15: additional bytes encode overflow (255 + 255 + ... + remainder) - - Raw literal bytes - - 16-bit little-endian match offset - - If match_len >= 15: additional bytes encode overflow - -6. **Restrictions (per LZ4 spec):** - - Last match must start at least 12 bytes before end (MFLIMIT) - - Last 5 bytes must be literals (LAST_LITERALS) - - Blocks < 13 bytes independent cannot be compressed (need at least 1 prior byte) - -### Decompression Process: - -1. **Token Parsing:** - - Read token byte - - Upper 4 bits = literal length - - Lower 4 bits = match length - - If either nibble = 0xF, read additional bytes for overflow - -2. **Fast Path (Hot Loop):** - - If token fits (both nibbles < 15) - - AND input still has 2+ bytes for offset - - AND output has capacity for literals + match - - Use optimized 16-byte literal copy + 18-byte match copy - -3. **Literal Copying:** - - Copy raw bytes directly from input to output - - Variable length from token + extension bytes - -4. **Match Copying:** - - Read 16-bit little-endian offset (distance back in output buffer) - - Copy from output[pos - offset] to output[pos] - - Must handle overlapping regions (self-referential copies) - - Uses 16-byte wild copies when possible for performance - -5. **Error Checking (when safe-decode enabled):** - - Verify output buffer has capacity - - Verify input has sufficient bytes - - Verify offset is within bounds - - Verify no out-of-order reads - ---- - -## 6. BENCHMARKS - -### Benchmark Files: - -**File: `/benches/binggan_bench.rs` (1000+ lines)** -- Primary benchmark using `binggan` framework -- Compares against: lz4_fear, lzzz (C LZ4), snap (Snappy) -- Benchmarks both block and frame formats -- Test datasets: - - compression_1k.txt (1 KB) - - compression_34k.txt (34 KB) - - compression_65k.txt (65 KB) - - compression_66k_JSON.txt (66 KB - JSON) - - dickens.txt (10 MB - text) - - logo.jpg (95 KB - binary) - -- Memory allocator: Uses Jemalloc for accurate measurements -- Metrics: Throughput (MiB/s), peak memory allocation -- Generates SVG graphs of results - -**Benchmark Commands:** -```bash -# Safe mode (default) -cargo bench - -# Unsafe mode -cargo bench --no-default-features -``` - -**Output Graphs:** -- compress_bench.svg: Compression speed comparison -- decompress_bench.svg: Decompression speed comparison -- compress_bench_safe.svg: Safe-mode compression -- decompress_bench_safe.svg: Safe-mode decompression - -### Benchmark Comparisons: - -The benchmarks compare lz4_flex against: -- **lzzz**: C language LZ4 (v1.9.3) via Rust bindings -- **lz-fear**: Pure Rust LZ4 implementation -- **snap**: Snappy compression algorithm (for reference) - ---- - -## 7. TESTS - -### Test File: `/tests/tests.rs` (500+ lines)** - -**Test Categories:** - -1. **Roundtrip Tests:** - - `test_roundtrip()`: Compress then decompress, verify matches original - - Tests with multiple datasets (1K to 10MB) - - Tests both block and frame formats - - Tests both Independent and Linked block modes - -2. **Cross-Implementation Tests:** - - Compress with lz4_flex, decompress with C (lzzzz) - - Compress with C, decompress with lz4_flex - - Verify results match - -3. **Block Format Tests:** - - Small blocks (< 13 bytes) - - Medium blocks (compressed JSON, 66KB) - - Large blocks (10MB text) - - Various size combinations - -4. **Frame Format Tests:** - - Independent blocks - - Linked blocks (referencing previous blocks) - - Different block sizes (64KB, 256KB, 1MB, etc.) - - Content checksum verification - - Block checksum verification - -5. **Edge Case Tests:** - - Empty input - - Single byte input - - Incompressible data - - Highly repetitive data - - Random data - -6. **Dictionary Tests:** - - External dictionary for decompression - - Prefix mode (data continuing from previous block) - -7. **Property-Based Tests:** - - `proptest` for random input generation - - Verifies correctness across random inputs - -### Test Datasets: - -Included in `/benches/`: -- compression_1k.txt -- compression_34k.txt -- compression_65k.txt -- compression_66k_JSON.txt -- dickens.txt (10MB) -- (embedded as byte arrays via `include_bytes!()`) - -### Fuzzing: - -Located in `/fuzz/fuzz_targets/`: - -1. **fuzz_roundtrip.rs**: Roundtrip compression/decompression -2. **fuzz_roundtrip_frame.rs**: Frame format roundtrip -3. **fuzz_roundtrip_cpp_compress.rs**: C compresses, Rust decompresses -4. **fuzz_roundtrip_cpp_decompress.rs**: Rust compresses, C decompresses -5. **fuzz_decomp_corrupt_block.rs**: Decompression of corrupted data -6. **fuzz_decomp_corrupt_frame.rs**: Frame decompression of corrupted data - -**Fuzzing Commands:** -```bash -cargo +nightly fuzz run fuzz_roundtrip -- -max_total_time=30 -cargo +nightly fuzz run fuzz_decomp_corrupt_block -- -max_total_time=30 -``` - ---- - -## 8. BUILD AND TEST INSTRUCTIONS - -### Build: - -```bash -# Debug build -cargo build - -# Release build (optimized) -cargo build --release - -# With all features -cargo build --all-features - -# No default features (maximum performance) -cargo build --no-default-features - -# Specific features -cargo build --no-default-features --features frame -cargo build --features safe-encode --features safe-decode -``` - -### Testing: - -```bash -# Run all tests -cargo test - -# Tests with all features -cargo test --all-features - -# Specific feature tests -cargo test --features safe-encode -cargo test --features safe-decode -cargo test --no-default-features --features frame - -# Tests with nightly compiler -cargo +nightly test --features nightly - -# Run fuzzing (requires nightly) -cargo +nightly fuzz run fuzz_roundtrip -- -max_total_time=30 -``` - -### Benchmarking: - -```bash -# Run benchmarks (default features) -cargo bench - -# Benchmarks with maximum performance (unsafe) -cargo bench --no-default-features - -# Run with specific benchmark -cargo bench --bench binggan_bench -``` - -### No-std Verification: - -```bash -# Ensure block format works without std -cargo build --no-default-features - -# With safe-decode only -cargo build --no-default-features --features safe-decode - -# With safe-encode only -cargo build --no-default-features --features safe-encode - -# With frame support (requires std) -cargo build --no-default-features --features frame # Will fail - frame needs std -``` - -### Miri (Undefined Behavior Detection): - -```bash -# Check for UB in unsafe code -MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows" cargo +nightly miri test --no-default-features --features frame -``` - -### Examples: - -```bash -# Frame format compression (stdin -> stdout) -cargo run --example compress < input.txt > output.lz4 - -# Frame format decompression -cargo run --example decompress < output.lz4 > restored.txt - -# Block format compression -cargo run --example compress_block < input.txt > output.lz4 - -# Block format decompression -cargo run --example decompress_block < output.lz4 > restored.txt -``` - -### CI/CD (GitHub Actions): - -**File: `.github/workflows/rust.yml`** - -Runs on every push and PR: -1. Build with nightly Rust -2. No-std compilation tests (multiple feature combinations) -3. Unit tests (multiple feature combinations) -4. Nightly feature tests -5. Fuzzing (safe and unsafe variants) -6. Semver checking (API compatibility) - ---- - -## 9. PERFORMANCE-RELATED CODE COMMENTS AND TODOs - -### Performance Optimizations in Code: - -#### In `/src/block/compress.rs`: - -**Line 132:** `#[cold]` attribute on tail counting function -- Marks infrequently used path as cold to optimize hot path - -**Line 227:** Comment on byte-at-a-time write_integer -- "simple byte at a time implementation below is faster in most cases" -- Avoids compiler-generated memcpy which is slow for small copies - -**Line 261:** TODO on bounds check elimination -```rust -// TODO: It should be possible remove all bounds checks, since we are walking -// backwards -``` -- Opportunity to optimize backtrack_match by proving bounds at compile time - -**Line 317:** `#[inline(never)]` on compress_internal -```rust -// Intentionally avoid inlining. -// Empirical tests revealed it to be rarely better but often significantly detrimental. -``` - -**Line 533:** Comment on literal copy optimization -```rust -// Note: This used to be a wild copy loop of 8 bytes, but the compiler consistently -// transformed it into a call to memcopy, which hurts performance significantly for -// small copies, which are common. -``` -- Uses match on copy length to prevent compiler auto-vectorization - -#### In `/src/block/decompress.rs`: - -**Line 39:** Comment on wild copy loop -```rust -// Note: if the compiler auto-vectorizes this it'll hurt performance! -// It's not the case for 16 bytes stepsize, but for 8 bytes. -``` - -**Line 56:** `#[cfg_attr(feature = "nightly", optimize(size))]` -- Avoids loop unrolling in duplicate_overlapping to prevent branch explosion - -**Line 106:** TODO on fastcpy_unsafe -```rust -// TODO test fastcpy_unsafe -``` -- Opportunity to use unsafe fast copy in dictionary path - -#### In `/src/block/decompress_safe.rs`: - -**Line 87:** `#[inline(always)]` comment -```rust -#[inline(always)] // (always) necessary to get the best performance in non LTO builds -``` - -#### In `/src/block/hashtable.rs`: - -**Line 66-67:** Comments on allocation optimization -```rust -// This generates more efficient assembly in contrast to Box::new(slice), because of an -// optimized call alloc_zeroed, vs. alloc + memset -``` - -#### In `/src/frame/decompress.rs`: - -**Line 248 & 271:** TODO on buffer initialization -```rust -// TODO: Attempt to avoid initialization of read buffer when... -``` -- Opportunity to lazy-initialize frame read buffer - -### Feature Gate Performance: - -1. **Safe-encode:** Uses bounds-checked slice access instead of unsafe pointer ops -2. **Safe-decode:** Uses bounds-checked indexing instead of unsafe pointer dereferencing -3. **Both safe:** Forbids all unsafe code via `#[forbid(unsafe_code)]` -4. **Neither safe:** Enables raw pointer manipulation for maximum speed - -### Hot Path Optimization: - -Main optimization in decompression (lines 259-326 of decompress.rs): -- Fast path when token fits in single byte -- Safe distance check ensures we won't overflow buffers -- Uses bulk 16-byte literal copies -- Uses bulk match copies with overlap detection - ---- - -## 10. KEY OBSERVATIONS - -### Architecture Decisions: - -1. **Hash Table Approach:** - - Fixed-size 4K-8K tables prevent unbounded memory usage - - Trade-off: More collisions but deterministic memory and cache behavior - -2. **Safe/Unsafe Split:** - - Default to safe operations for correctness - - Optional unsafe for performance - - Clear feature gate boundaries - -3. **Block vs Frame:** - - Block: Simple, in-memory, good for small data - - Frame: Streaming, supports checksums, good for large data - - Frame uses block compression internally - -4. **No Allocator Dependency:** - - Uses `alloc` crate, not `std` - - Enables no_std + alloc support - - Frame format requires std for I/O - -### Performance Characteristics: - -- **Compression:** Limited by CPU throughput on I/O, memory copies -- **Decompression:** Very fast (3-6 GiB/s), mostly bottleneck-free -- **Memory:** Fixed by window size (64KB) + hash table (0.5-16KB) -- **Fast paths:** Heavily optimized for common case (tokens < 15, safe distance) - -### Testing Strategy: - -- Unit tests integrated in source -- Separate test file for integration testing -- Fuzzing against corrupted data -- Cross-validation with C implementation -- Property-based testing with random inputs - ---- - -## Summary of File Organization: - -| File | Lines | Purpose | -|------|-------|---------| -| src/lib.rs | 112 | Module root, feature config | -| src/block/mod.rs | 177 | Block format defs, errors | -| src/block/compress.rs | 999 | Compression algorithm | -| src/block/decompress.rs | 543 | Unsafe decompression | -| src/block/decompress_safe.rs | 400 | Safe decompression | -| src/block/hashtable.rs | 248+ | Hash table for dups | -| src/frame/mod.rs | 111 | Frame format defs | -| src/frame/compress.rs | 471 | Frame compression | -| src/frame/decompress.rs | 448 | Frame decompression | -| src/frame/header.rs | 411 | Frame headers | -| src/sink.rs | 200+ | Buffer abstraction | -| src/fastcpy*.rs | 200+ | Fast memory copy | -| benches/binggan_bench.rs | 1000+ | Performance benchmarks | -| tests/tests.rs | 500+ | Integration tests | -| fuzz/fuzz_targets/*.rs | Various | Fuzzing harnesses | - -**Total:** ~7000 lines of Rust code (excluding tests/benches) - -___BEGIN___COMMAND_DONE_MARKER___0 diff --git a/QUICK_REFERENCE.md b/QUICK_REFERENCE.md deleted file mode 100644 index 03d34879..00000000 --- a/QUICK_REFERENCE.md +++ /dev/null @@ -1,211 +0,0 @@ -# LZ4_FLEX Quick Reference Guide - -## What is LZ4_FLEX? -A high-performance LZ4 compression library in pure Rust with optional safe/unsafe variants. -- **Compression Speed:** 1.2-1.6 GiB/s (safe/unsafe) -- **Decompression Speed:** 2.3-6.0 GiB/s (safe/unsafe) -- **Features:** Block format (in-memory), Frame format (streaming) - -## Quick Start - -### Basic Usage (Block Format) -```rust -use lz4_flex::block::{compress_prepend_size, decompress_size_prepended}; - -let input = b"Hello, world!"; -let compressed = compress_prepend_size(input); -let decompressed = decompress_size_prepended(&compressed)?; -assert_eq!(input, &decompressed[..]); -``` - -### Streaming (Frame Format) -```rust -use std::io::{Read, Write}; -use lz4_flex::frame::{FrameEncoder, FrameDecoder}; - -// Compress -let mut encoder = FrameEncoder::new(Vec::new()); -encoder.write_all(b"data")?; -let compressed = encoder.finish()?; - -// Decompress -let mut decoder = FrameDecoder::new(&compressed[..]); -let mut decompressed = Vec::new(); -decoder.read_to_end(&mut decompressed)?; -``` - -## Building - -```bash -cargo build --release # Safe+frame (default) -cargo build --no-default-features # Unsafe, no frame -cargo build --no-default-features --features frame # Unsafe + frame -``` - -## Testing - -```bash -cargo test # All tests -cargo test --no-default-features # Unsafe code tests -cargo +nightly fuzz run fuzz_roundtrip # Fuzzing -``` - -## Benchmarking - -```bash -cargo bench # Safe mode -cargo bench --no-default-features # Unsafe mode -``` - -## Key Files - -| File | Purpose | -|------|---------| -| `src/block/compress.rs` (999 lines) | Compression algorithm using hash tables | -| `src/block/decompress.rs` (543 lines) | Fast unsafe decompression | -| `src/block/decompress_safe.rs` (400 lines) | Safe decompression (bounds-checked) | -| `src/frame/compress.rs` (471 lines) | Streaming compression with checksums | -| `src/frame/decompress.rs` (448 lines) | Streaming decompression | -| `src/block/hashtable.rs` | Hash table for duplicate detection | -| `src/sink.rs` | Buffer abstraction for compression | - -## Features - -- `safe-encode` (default): Safe Rust for compression -- `safe-decode` (default): Safe Rust for decompression -- `frame` (default): Streaming LZ4 frame format (requires std) -- `std` (default): Standard library support -- `checked-decode`: Additional decompression safety checks - -## Compression Algorithm - -1. Hash each 4-byte sequence -2. Look up hash to find previous matches -3. Encode as: `[token][literal_length][literals][match_offset][match_length]` -4. Token byte: `[4-bit lit_len][4-bit match_len]` -5. If length >= 15: use extension bytes (255 + 255 + ... + remainder) - -**Constants:** -- Minimum match: 4 bytes -- Maximum back-reference distance: 64 KB -- Window size: 64 KB - -## Decompression Algorithm - -1. Read token byte -2. Parse literal length (upper 4 bits) -3. Copy literals from input -4. Parse match offset (16-bit little-endian) -5. Parse match length (lower 4 bits) -6. Copy from output[pos - offset] (handles overlapping regions) -7. Repeat - -**Fast Path Optimization:** -- If token fits in single byte AND safe distance from end -- Use 16-byte literal copy + 18-byte match copy - -## Performance Characteristics - -| Test | Unsafe | Safe | vs C (lzzz) | -|------|--------|------|------------| -| 66KB JSON compress | 1615 MiB/s | 1272 MiB/s | +10% | -| 66KB JSON decompress | 5512 MiB/s | 4540 MiB/s | +4% | -| 10MB text compress | 347 MiB/s | 259 MiB/s | +7% | -| 10MB text decompress | 2734 MiB/s | 2338 MiB/s | -1% | - -## Common Performance TODOs - -1. **Line 261 of compress.rs:** Remove bounds checks in backtrack_match -2. **Line 106 of decompress.rs:** Test fastcpy_unsafe in dictionary path -3. **Lines 248, 271 of frame/decompress.rs:** Lazy buffer initialization -4. **Line 534 of compress.rs:** Prevent compiler auto-vectorization of literal copy - -## Error Types - -**DecompressError:** -- `OutputTooSmall { expected, actual }` -- `LiteralOutOfBounds` -- `ExpectedAnotherByte` -- `OffsetOutOfBounds` - -**CompressError:** -- `OutputTooSmall` - -## Block Format Constraints - -- Last match must start ≥12 bytes before end -- Last 5 bytes must be literals -- Independent blocks <13 bytes can't be compressed -- Maximum distance: 65,535 bytes - -## Frame Format Features - -- Magic number: `0x184D2204` (4 bytes) -- Optional block checksums (xxhash32) -- Optional content checksum -- Optional content size header -- Independent or linked blocks -- Configurable block sizes: 64KB, 256KB, 1MB, 4MB, 8MB - -## CI/CD - -**GitHub Actions (`.github/workflows/rust.yml`):** -- Tests with nightly Rust -- No-std compilation verification -- Fuzzing tests (safe + unsafe) -- Semver checking - -## Dependencies - -**Runtime:** -- `twox-hash` 2.0.0 (optional, frame format) -- `alloc` (required) - -**Dev:** -- lzzzz (C LZ4 bindings for testing) -- snap (Snappy for comparison) -- proptest (property-based testing) -- binggan (benchmarking) - -## Examples - -Located in `/examples/`: -- `compress.rs` - Frame format compression -- `decompress.rs` - Frame format decompression -- `compress_block.rs` - Block format compression -- `decompress_block.rs` - Block format decompression - -Run: `cargo run --example compress < input.txt > output.lz4` - -## Fuzzing - -```bash -cargo +nightly fuzz run fuzz_roundtrip -- -max_total_time=30 -cargo +nightly fuzz run fuzz_decomp_corrupt_block -- -max_total_time=30 -cargo +nightly fuzz run fuzz_roundtrip_cpp_compress -- -max_total_time=30 -``` - -## Miri (Undefined Behavior Detection) - -```bash -MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-disable-stacked-borrows" \ - cargo +nightly miri test --no-default-features --features frame -``` - -## Memory Usage - -- Hash table: 4K entries × 2-4 bytes = 8-16 KB -- Window size: 64 KB (back-reference limit) -- Output buffer: input_size + overhead -- No streaming allocations - -## Testing Datasets - -Included in `/benches/`: -- compression_1k.txt (1 KB) -- compression_34k.txt (34 KB) -- compression_65k.txt (65 KB) -- compression_66k_JSON.txt (66 KB - JSON, highly compressible) -- dickens.txt (10 MB - English text) -- logo.jpg (95 KB - binary/image) - From 574f9ff0a5e7d91beae618503818cef2f672303e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:11:29 +0000 Subject: [PATCH 4/5] Optimize safe decompression hot loop offset read and safe compression literal copy 1. decompress_safe.rs: Replace clone + read_u16() call with direct array indexing in the hot loop. Since input is &[u8; 16] and literal_length is 0-14, the indices are provably in bounds. This eliminates a clone, function call, bounds check, and error handling path from the hot loop. 2. compress.rs: Pass a wider slice (+8 bytes) to extend_from_slice_wild in the safe-encode path, enabling efficient fixed-size copy paths in slice_copy for small literals. This matches the unsafe version's behavior of unconditionally copying 8/16/24 bytes. Safe because MFLIMIT guarantees sufficient input bytes and get_maximum_output_size provides sufficient output capacity. Co-authored-by: PSeitz <1109503+PSeitz@users.noreply.github.com> --- src/block/compress.rs | 7 ++++++- src/block/decompress_safe.rs | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/block/compress.rs b/src/block/compress.rs index c60fe017..6d973aa7 100644 --- a/src/block/compress.rs +++ b/src/block/compress.rs @@ -521,7 +521,12 @@ fn push_u16(output: &mut impl Sink, el: u16) { #[inline(always)] // (always) necessary otherwise compiler fails to inline it #[cfg(feature = "safe-encode")] fn copy_literals_wild(output: &mut impl Sink, input: &[u8], input_start: usize, len: usize) { - output.extend_from_slice_wild(&input[input_start..input_start + len], len) + // Pass a wider slice (+8 bytes) so that slice_copy can use efficient fixed-size copy + // paths for small literals, matching the unsafe version's behavior of unconditionally + // copying 8/16/24 bytes. This is safe because: + // - MFLIMIT guarantees at least 12 bytes of input after `input_start + len` + // - get_maximum_output_size provides sufficient output capacity margin + output.extend_from_slice_wild(&input[input_start..input_start + len + 8], len) } #[inline] diff --git a/src/block/decompress_safe.rs b/src/block/decompress_safe.rs index df298f7f..fc8c4398 100644 --- a/src/block/decompress_safe.rs +++ b/src/block/decompress_safe.rs @@ -138,8 +138,11 @@ pub(crate) fn decompress_internal( output.extend_from_slice_wild(input, literal_length); input_pos += literal_length; - // clone as we don't want to mutate - let offset = read_u16(input, &mut literal_length.clone())? as usize; + // Read the offset directly from the input array. + // `input` is &[u8; 16] and `literal_length` is at most 14 (guaranteed by + // does_token_fit), so literal_length + 1 <= 15 is always in bounds. + let offset = + u16::from_le_bytes([input[literal_length], input[literal_length + 1]]) as usize; input_pos += 2; let mut match_length = MINMATCH + (token & 0xF) as usize; From 91d35369a82141569010f38b240e3f3b1f38f1a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 14 Mar 2026 14:12:26 +0000 Subject: [PATCH 5/5] Add debug assertions for wild copy bounds safety Co-authored-by: PSeitz <1109503+PSeitz@users.noreply.github.com> --- src/block/compress.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/block/compress.rs b/src/block/compress.rs index 6d973aa7..04c85e43 100644 --- a/src/block/compress.rs +++ b/src/block/compress.rs @@ -526,6 +526,8 @@ fn copy_literals_wild(output: &mut impl Sink, input: &[u8], input_start: usize, // copying 8/16/24 bytes. This is safe because: // - MFLIMIT guarantees at least 12 bytes of input after `input_start + len` // - get_maximum_output_size provides sufficient output capacity margin + debug_assert!(input_start + len + 8 <= input.len()); + debug_assert!(output.pos() + len + 8 <= output.capacity()); output.extend_from_slice_wild(&input[input_start..input_start + len + 8], len) }