From 6edd6adf57a28639bf0b9857e20912697ca5e107 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 13:47:09 -0500 Subject: [PATCH 01/11] feat(internal): emit RLE blocks, literals and sequence tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decoder has always read all three RLE forms — libzstd emits them — but the encoder never wrote any: a 5000-byte run of one byte still went out as a Compressed_Block with a full sequence stream, and a stream whose every sequence shares one code still paid Predefined FSE bits per symbol. Each RLE form is the degenerate one-symbol case of a block's own entropy coding, so they land together: - RLE_Block (Block_Type 1) when every input byte is identical. All three block types share a 3-byte header, so the smallest block is simply the one with the smallest payload; ties keep the simpler form. - RLE literals (Literals_Block_Type 1) when every literal is the same byte. With a dictionary that happens for real inputs whose only literals are a repeated separator, everything else having matched into the dict content. - RLE sequence tables (Symbol_Compression_Mode 1) per LL/OF/ML stream when that stream's every code is the same: one description byte and not a single bit in the bitstream. Mode choice is now a cost comparison in BITS across every valid table for a stream, using FseEncTable.streamBitCost — an exact count that walks the same path encode() does, not an entropy estimate, so a chosen mode can never turn out bigger than predicted. Candidates are considered Repeat, then Predefined, then RLE, and a later one must be strictly cheaper to win, so the dictionary paths keep every tie. Table descriptions are written after the mode byte in stream order LL, OF, ML — the order the decoder resolves them in, which is NOT the LL/ML/OF order the bitstream's extra bits use. The all-RLE case can leave the sequences bitstream holding nothing but its stop bit; EntropyCodingInteropTest confirms real libzstd accepts that, and the other new tests assert the chosen mode rather than only round-tripping, so a regression to Raw/Predefined cannot pass silently. A constant 5000-byte input now compresses to 10 bytes (was 17); the byte-run sample to 42 bytes (was 208 raw). One pinned frame in ByteIdenticalRegressionTest moves (same size: the match-length stream now picks the dictionary's Predefined table over its Repeat table at strictly lower bit cost, not the offset stream picking RLE) and is refreshed per that test's documented procedure. Signed-off-by: James Rich --- .../org/meshtastic/kzstd/internal/Fse.kt | 14 ++ .../meshtastic/kzstd/internal/FseEncoder.kt | 52 +++- .../meshtastic/kzstd/internal/ZstdDecoder.kt | 13 +- .../meshtastic/kzstd/internal/ZstdEncoder.kt | 227 ++++++++++++------ .../org/meshtastic/kzstd/FrameInspector.kt | 131 ++++++++++ .../org/meshtastic/kzstd/RleEncodingTest.kt | 69 ++++++ .../org/meshtastic/kzstd/TestVectors.kt | 42 ++++ .../kzstd/ByteIdenticalRegressionTest.kt | 2 +- .../kzstd/EntropyCodingInteropTest.kt | 57 +++++ 9 files changed, 509 insertions(+), 98 deletions(-) create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt create mode 100644 src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/Fse.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/Fse.kt index 61028e8..fe9a8f6 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/Fse.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/Fse.kt @@ -107,6 +107,20 @@ internal class FseTable( return FseTable(tableLog, symbolTable, nbBitsTable, newStateTable) } + /** + * The degenerate one-symbol table used by Symbol_Compression_Mode 1 + * (RLE): [symbol] has probability 1, so decoding stays in state 0 + * forever and consumes no bits. Shared by the decoder (which reads the + * mode-1 description byte) and the encoder (which emits it when a + * stream's every code is the same). + */ + fun rle(symbol: Int): FseTable = FseTable( + tableLog = 0, + symbol = intArrayOf(symbol), + nbBits = intArrayOf(0), + newState = intArrayOf(0), + ) + /** Floor(log2(v)) for v >= 1. */ private fun highBit(v: Int): Int { var n = 0 diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseEncoder.kt index 0cce2c1..e6d670e 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseEncoder.kt @@ -53,25 +53,53 @@ internal class FseEncTable private constructor( * initial state via [initialState]. */ fun encode(bw: ReverseBitWriter, state: Int, symbol: Int): Int { - val states = symbolStates[symbol] - // Find the decode-state `ds` emitting `symbol` whose output range - // [base, base + 2^nb) contains the target `state`. Ranges partition - // [0,tableSize), so exactly one matches. The encoder emits `state - base` - // in `nb` bits and moves to `ds`. - for (ds in states) { - val nb = nbBits[ds] + val ds = transition(state, symbol) + bw.writeBits(state - newStateBase[ds], nbBits[ds]) + return ds + } + + /** + * The decode-state `ds` emitting [symbol] whose output range + * `[base, base + 2^nb)` contains [state]. Ranges partition `[0,tableSize)`, + * so exactly one matches; the caller emits `state - base` in `nb` bits and + * moves to `ds`. + */ + private fun transition(state: Int, symbol: Int): Int { + for (ds in symbolStates[symbol]) { val base = newStateBase[ds] - val hi = base + (1 shl nb) - if (state in base until hi) { - bw.writeBits(state - base, nb) - return ds - } + if (state >= base && state < base + (1 shl nbBits[ds])) return ds } // The FSE invariant guarantees a match; reaching here means a corrupt // table or an out-of-range symbol the caller failed to bound. throw ZstdException("FSE encode: no transition for symbol $symbol from state $state") } + /** + * Exactly how many bits encoding [codes] (chronological order) with this + * table would cost — every transition plus the flushed initial state — or + * null when some code has no code point here ([isCovered]). + * + * This walks the SAME path [encode] does (backwards from the last code, + * starting at [initialState]), so it is an exact count and not an entropy + * estimate: the cost model that chooses between Predefined / RLE / + * FSE_Compressed / Repeat can therefore never pick a mode that turns out + * bigger than it predicted. Bits from the three sequence streams simply + * add, so streams can be costed independently even though they interleave + * in the final bitstream. + */ + fun streamBitCost(codes: IntArray): Long? { + if (codes.isEmpty()) return null + for (c in codes) if (!isCovered(c)) return null + var state = initialState(codes[codes.size - 1]) + var bits = 0L + for (i in codes.size - 2 downTo 0) { + val ds = transition(state, codes[i]) + bits += nbBits[ds] + state = ds + } + return bits + tableLog + } + /** * Pick the initial encoder state for [symbol] (the LAST output symbol, which * the encoder processes first). Any decode-state that emits [symbol] is a diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdDecoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdDecoder.kt index 0eefe6a..c6b192b 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdDecoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdDecoder.kt @@ -517,7 +517,7 @@ internal object PureZstdDecoder { 1 -> { // RLE: a single byte is the only symbol (probability 1.0), tableLog 0. val symbol = reader.readByte() - rleTable(symbol) + FseTable.rle(symbol) } 2 -> parseFseTable(reader, kind.maxLog, kind.maxSymbol) @@ -526,17 +526,6 @@ internal object PureZstdDecoder { ?: throw ZstdException("repeat mode for ${kind.name} but no prior/dict table") } - private fun rleTable(symbol: Int): FseTable { - // One-state table: always emits `symbol`, consumes 0 bits, stays in - // state 0. - return FseTable( - tableLog = 0, - symbol = intArrayOf(symbol), - nbBits = intArrayOf(0), - newState = intArrayOf(0), - ) - } - /** * Apply zstd's repeat-offset machinery (RFC 8878 §3.1.1.3.2.1.1). Offset * codes 1..3 reference the three repeat-offset slots (with a literal-length diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt index 0d4e7bb..bae8da2 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt @@ -18,28 +18,27 @@ import org.meshtastic.kzstd.ZstdException * - **Frame header:** single-segment, dictID OFF, contentSize OFF, checksum * OFF — byte-for-byte the descriptor the SDK's decoder (and libzstd) accept, * with a window large enough to cover `dict.content + input`. - * - **One Compressed_Block** with Last_Block set. If the compressed block would - * not be smaller than the input, a Raw_Block is emitted instead (a valid - * fallback; the SDK's own `0xFF` skip-compress also covers incompressible - * payloads above this layer). + * - **One block** with Last_Block set, of whichever type is smallest: a + * Compressed_Block, an RLE_Block when every input byte is identical, or a + * Raw_Block when compression does not pay (a valid fallback; the SDK's own + * `0xFF` skip-compress also covers incompressible payloads above this + * layer). * - **Matching:** a 4-byte hash-chain matcher over `[dict.content ++ input]`, so * matches can back-reference the dictionary content. The dict index is built * once per `ZstdDictionary` instance, so the dict content is hashed once and * reused across calls. Lazy (1-step lookahead) matching for a better ratio. - * - **Literals:** reuses the dictionary's trained Huffman table (Treeless, - * litType 3, single-stream only) when it covers every literal byte in this - * block AND the encoded form is smaller than Raw; otherwise a RAW literals - * block (litType 0) — simplest valid option and a good fit because a - * trained dict turns most bytes into matches, leaving few literals to - * begin with. - * - **Sequences:** FSE-coded, independently per LL/OF/ML stream. Each stream - * reuses the dictionary's trained "Repeat" table (mode 3) when the dict - * has one and it assigns nonzero probability to every code this block's - * sequences actually need; otherwise it falls back to the PREDEFINED table - * (mode 0) — always total over its symbol range, and exactly what - * [PureZstdDecoder] / libzstd build from the spec's default distributions. - * This encoder never emits RLE or a fresh FSE_Compressed table for - * sequences. + * - **Literals:** RLE (litType 1) when every literal is the same byte, the + * dictionary's trained Huffman table (Treeless, litType 3, single-stream + * only) when it covers every literal byte in this block, or RAW (litType 0) + * — whichever is smallest. + * - **Sequences:** FSE-coded, independently per LL/OF/ML stream, each stream + * picking the cheapest of: the dictionary's trained "Repeat" table (mode 3) + * when the dict has one that covers this block's codes; RLE (mode 1) when + * every sequence uses the same code; or the PREDEFINED table (mode 0) — + * always total over its symbol range, and exactly what [PureZstdDecoder] / + * libzstd build from the spec's default distributions. Cost is compared in + * bits and computed exactly ([FseEncTable.streamBitCost]), so the chosen + * mode is genuinely the smallest. * - **Offsets:** a repeat-offset code (1..3) when the match distance equals * one of the three most-recently-used offsets (rotated exactly as * [PureZstdDecoder]'s `applyOffset` does), else an explicit literal offset @@ -105,14 +104,31 @@ internal object PureZstdEncoder { out.add(frameHeaderDescriptor(checksum)) out.add(windowDescriptor(dict.content.size + data.size)) - if (compressedBlock != null && compressedBlock.size < data.size) { - // Block_Header (3 bytes LE): last=1, type=2 (Compressed), size=blockSize - writeBlockHeader(out, lastBlock = true, blockType = 2, blockSize = compressedBlock.size) - compressedBlock.forEach { out.add(it) } - } else { - // Raw_Block fallback: the literal bytes are the block. - writeBlockHeader(out, lastBlock = true, blockType = 0, blockSize = data.size) - data.forEach { out.add(it) } + // All three block types carry the same 3-byte header, so the smallest + // block is simply the one with the smallest payload: 1 byte for RLE + // (only when every input byte is identical), `data.size` for Raw, or + // the compressed body. Ties go to the simpler form. + val constantByte = constantByteOrNull(data) + when { + constantByte != null && + data.size > 1 && + (compressedBlock == null || compressedBlock.size > 1) -> { + // RLE_Block: the single repeated byte IS the block body. + writeBlockHeader(out, lastBlock = true, blockType = 1, blockSize = data.size) + out.add(constantByte) + } + + compressedBlock != null && compressedBlock.size < data.size -> { + // Block_Header (3 bytes LE): last=1, type=2 (Compressed), size=blockSize + writeBlockHeader(out, lastBlock = true, blockType = 2, blockSize = compressedBlock.size) + compressedBlock.forEach { out.add(it) } + } + + else -> { + // Raw_Block fallback: the literal bytes are the block. + writeBlockHeader(out, lastBlock = true, blockType = 0, blockSize = data.size) + data.forEach { out.add(it) } + } } if (checksum) { @@ -126,6 +142,14 @@ internal object PureZstdEncoder { return ByteArray(out.size) { out[it] } } + /** The byte every element of [bytes] equals, or null (including when empty). */ + private fun constantByteOrNull(bytes: ByteArray): Byte? { + if (bytes.isEmpty()) return null + val first = bytes[0] + for (b in bytes) if (b != first) return null + return first + } + // ── Frame header ─────────────────────────────────────────────────────────── /** @@ -394,6 +418,19 @@ internal object PureZstdEncoder { * this comparison is mandatory, not a hint. */ private fun writeLiteralsSection(out: ArrayList, literals: ByteArray, dict: ParsedDictionary) { + val rawCost = rawLiteralsHeaderLen(literals.size) + literals.size + + // RLE literals (litType 1): one stored byte regenerates the whole run. + // Reachable whenever every literal is the same byte -- with a dictionary + // that happens for real inputs whose only literals are a repeated + // separator, every other byte having been matched into the dict. + val constant = constantByteOrNull(literals) + if (constant != null && rawLiteralsHeaderLen(literals.size) + 1 < rawCost) { + writeRawOrRleLiteralsHeader(out, literals.size, litType = 1) + out.add(constant) + return + } + val huffman = dict.literalsHuffman if (huffman != null && literals.isNotEmpty() && literals.size <= MAX_TREELESS_LITERALS) { val encTable = HuffmanEncTable.fromDecodeTable(huffman) @@ -454,25 +491,33 @@ internal object PureZstdEncoder { out.add(b2.toByte()) } - private fun writeRawLiteralsHeader(out: ArrayList, size: Int) { - // litType=0 (Raw). size_format selects the Regenerated_Size width: - // - size < 32 : 1-byte header, 5-bit size (size_format bit0 = 0) - // - size < 4096 : 2-byte header, 12-bit size (size_format = 0b01) - // - else : 3-byte header, 20-bit size (size_format = 0b11) + private fun writeRawLiteralsHeader(out: ArrayList, size: Int) = + writeRawOrRleLiteralsHeader(out, size, litType = 0) + + /** + * Literals_Section_Header for the two uncoded forms, litType=0 (Raw) and + * litType=1 (RLE) -- they share one header layout, differing only in + * whether `Regenerated_Size` bytes or a single byte follow. size_format + * selects the Regenerated_Size width: + * - size < 32 : 1-byte header, 5-bit size (size_format bit0 = 0) + * - size < 4096 : 2-byte header, 12-bit size (size_format = 0b01) + * - else : 3-byte header, 20-bit size (size_format = 0b11) + */ + private fun writeRawOrRleLiteralsHeader(out: ArrayList, size: Int, litType: Int) { when { size < 32 -> { - out.add(((size shl 3) or (0 shl 2) or 0).toByte()) // [size:5][00][00] + out.add(((size shl 3) or (0 shl 2) or litType).toByte()) // [size:5][00][type:2] } size < 4096 -> { - val b0 = (0) or (0b01 shl 2) or ((size and 0xF) shl 4) + val b0 = litType or (0b01 shl 2) or ((size and 0xF) shl 4) val b1 = (size ushr 4) and 0xFF out.add(b0.toByte()) out.add(b1.toByte()) } else -> { - val b0 = (0) or (0b11 shl 2) or ((size and 0xF) shl 4) + val b0 = litType or (0b11 shl 2) or ((size and 0xF) shl 4) val b1 = (size ushr 4) and 0xFF val b2 = (size ushr 12) and 0xFF out.add(b0.toByte()) @@ -517,23 +562,27 @@ internal object PureZstdEncoder { val codes = Array(nbSeq) { computeCodes(sequences[it], rep) } // Symbol_Compression_Modes: 2 bits each for LL, OF, ML (high bits - // first), low 2 bits reserved (0). Each stream independently uses the - // dictionary's trained "Repeat" table (mode 3) when the dict has one - // AND it assigns nonzero probability to every code this block's - // sequences actually need -- otherwise Predefined (mode 0), which is - // always total over its whole symbol range. This encoder never emits - // RLE (1) or a fresh FSE_Compressed table (2) for sequences. - val (llTable, llMode) = - resolveSequenceTable(dict.literalLengthFse, LITERAL_LENGTH_MAX_SYMBOL, predefinedLiteralLengthEnc, nbSeq) { - codes[it].llCode - } - val (ofTable, ofMode) = - resolveSequenceTable(dict.offsetFse, OFFSET_MAX_SYMBOL, predefinedOffsetEnc, nbSeq) { codes[it].ofCode } - val (mlTable, mlMode) = - resolveSequenceTable(dict.matchLengthFse, MATCH_LENGTH_MAX_SYMBOL, predefinedMatchLengthEnc, nbSeq) { - codes[it].mlCode - } - out.add(((llMode shl 6) or (ofMode shl 4) or (mlMode shl 2)).toByte()) + // first), low 2 bits reserved (0). Each stream picks its own cheapest + // valid table (see [chooseSequenceTable]). + val llCodes = IntArray(nbSeq) { codes[it].llCode } + val ofCodes = IntArray(nbSeq) { codes[it].ofCode } + val mlCodes = IntArray(nbSeq) { codes[it].mlCode } + val ll = + chooseSequenceTable(dict.literalLengthFse, predefinedLiteralLengthEnc, LITERAL_LENGTH_MAX_SYMBOL, llCodes) + val of = chooseSequenceTable(dict.offsetFse, predefinedOffsetEnc, OFFSET_MAX_SYMBOL, ofCodes) + val ml = chooseSequenceTable(dict.matchLengthFse, predefinedMatchLengthEnc, MATCH_LENGTH_MAX_SYMBOL, mlCodes) + val llTable = ll.table + val ofTable = of.table + val mlTable = ml.table + out.add(((ll.mode shl 6) or (of.mode shl 4) or (ml.mode shl 2)).toByte()) + + // Any table DESCRIPTIONS (an RLE symbol byte, or a full FSE table + // header) follow the mode byte in stream order LL, OF, ML -- the order + // [PureZstdDecoder.decodeSequences] resolves them in, which is NOT the + // LL/ML/OF order the bitstream's extra bits use below. + ll.description.forEach { out.add(it) } + of.description.forEach { out.add(it) } + ml.description.forEach { out.add(it) } val bw = ReverseBitWriter() @@ -669,33 +718,65 @@ internal object PureZstdEncoder { } /** - * Choose the FSE encode table (and its 2-bit Symbol_Compression_Mode) for - * one sequence symbol stream: the dictionary's trained table (Repeat, - * mode 3) if [dictTable] is non-null AND covers every code in - * `0 until nbSeq` via [codeAt] (checked via [FseEncTable.isCovered] before - * committing -- a dict's training corpus commonly never produced some - * symbol, leaving it zero-probability), else [predefined] (Predefined, - * mode 0), which is always total over its declared symbol range. + * One stream's chosen FSE encode table, its 2-bit Symbol_Compression_Mode, + * and the table description bytes (if any) that must follow the mode byte. */ - private inline fun resolveSequenceTable( + private class SeqTableChoice(val table: FseEncTable, val mode: Int, val description: ByteArray) + + /** + * Choose the cheapest valid table for one sequence symbol stream, costed in + * BITS ([FseEncTable.streamBitCost] is exact, not an entropy estimate) plus + * 8 bits per description byte: + * + * - **Repeat (3)** -- the dictionary's trained table, when it exists and + * assigns nonzero probability to every code this block needs (a dict's + * training corpus commonly never produced some symbol). Costs no + * description bytes. + * - **Predefined (0)** -- the spec's default distribution; also free of + * description bytes, and total over its declared symbol range. + * - **RLE (1)** -- when every sequence uses the SAME code: one description + * byte and not a single bit in the bitstream. + * + * Candidates are considered in that order and a later one must be strictly + * cheaper to win, so ties keep the mode that costs no description bytes and + * leaves the dictionary paths in place. + */ + private fun chooseSequenceTable( dictTable: FseTable?, - maxSymbol: Int, predefined: FseEncTable, - nbSeq: Int, - codeAt: (Int) -> Int, - ): Pair { - if (dictTable != null) { - val candidate = FseEncTable.fromDecodeTable(dictTable, maxSymbol) - var covered = true - for (i in 0 until nbSeq) { - if (!candidate.isCovered(codeAt(i))) { - covered = false - break - } + maxSymbol: Int, + codes: IntArray, + ): SeqTableChoice { + var best: SeqTableChoice? = null + var bestBits = Long.MAX_VALUE + + fun consider(candidate: SeqTableChoice) { + val bits = candidate.table.streamBitCost(codes) ?: return + val total = bits + candidate.description.size * 8L + if (total < bestBits) { + bestBits = total + best = candidate } - if (covered) return candidate to 3 } - return predefined to 0 + + if (dictTable != null) { + consider(SeqTableChoice(FseEncTable.fromDecodeTable(dictTable, maxSymbol), 3, ByteArray(0))) + } + consider(SeqTableChoice(predefined, 0, ByteArray(0))) + val constantCode = codes[0].takeIf { first -> codes.all { it == first } } + if (constantCode != null) { + consider( + SeqTableChoice( + FseEncTable.fromDecodeTable(FseTable.rle(constantCode), maxSymbol), + 1, + byteArrayOf(constantCode.toByte()), + ), + ) + } + + // Predefined covers every code these tables can legally carry, so the + // fallback is unreachable in practice; it keeps the function total. + return best ?: SeqTableChoice(predefined, 0, ByteArray(0)) } /** Map a literal length to its FSE code (largest baseline <= length). */ diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt new file mode 100644 index 0000000..3a426ab --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +/** + * Test-only reader for the structural fields of a kzstd-produced frame: + * Block_Type, Literals_Block_Type and the Sequences_Section's + * Symbol_Compression_Modes byte. + * + * Encoding-choice tests need these because a round-trip proves only that the + * frame decodes -- not WHICH of the several valid encodings the cost model + * picked. Asserting the mode is what stops a silent regression to "always Raw + * literals / always Predefined tables" from passing every other test. + * + * It parses only the forms kzstd's own encoder emits (single-segment-off frame + * header, no dictID/content-size/checksum; Raw/RLE/Huffman-size_format-0 + * literals), and throws otherwise rather than guessing. + */ +internal object FrameInspector { + + /** Block_Type (0 Raw, 1 RLE, 2 Compressed) of the frame's first block. */ + fun blockType(frame: ByteArray): Int { + val p = frameHeaderEnd(frame) + return (blockHeader(frame, p) ushr 1) and 0x3 + } + + /** Block_Size field of the frame's first block header. */ + fun blockSize(frame: ByteArray): Int = blockHeader(frame, frameHeaderEnd(frame)) ushr 3 + + /** + * Literals_Block_Type (0 Raw, 1 RLE, 2 Huffman_Compressed, 3 Treeless) of + * the first block, or -1 when that block is not a Compressed_Block. + */ + fun literalsType(frame: ByteArray): Int { + val p = frameHeaderEnd(frame) + if ((blockHeader(frame, p) ushr 1) and 0x3 != 2) return -1 + return frame[p + 3].toInt() and 0x3 + } + + /** + * (llMode, ofMode, mlMode) from the first block's Symbol_Compression_Modes + * byte, or null when the block carries no sequences. + */ + fun sequenceModes(frame: ByteArray): Triple? { + var p = frameHeaderEnd(frame) + check((blockHeader(frame, p) ushr 1) and 0x3 == 2) { "not a Compressed_Block" } + p += 3 + p = skipLiteralsSection(frame, p) + + val nb0 = frame[p].toInt() and 0xFF + val nbSeq = when { + nb0 < 128 -> nb0 + nb0 < 255 -> ((nb0 - 128) shl 8) + (frame[p + 1].toInt() and 0xFF) + else -> (frame[p + 1].toInt() and 0xFF) + ((frame[p + 2].toInt() and 0xFF) shl 8) + 0x7F00 + } + if (nbSeq == 0) return null + p += when { + nb0 < 128 -> 1 + nb0 < 255 -> 2 + else -> 3 + } + val modes = frame[p].toInt() and 0xFF + return Triple((modes ushr 6) and 0x3, (modes ushr 4) and 0x3, (modes ushr 2) and 0x3) + } + + /** Byte offset of the first Block_Header. */ + private fun frameHeaderEnd(frame: ByteArray): Int { + var p = 4 // frame magic + val fhd = frame[p].toInt() and 0xFF + p++ + val fcsFlag = (fhd ushr 6) and 0x3 + val singleSegment = (fhd ushr 5) and 0x1 + val dictIdFlag = fhd and 0x3 + if (singleSegment == 0) p++ // Window_Descriptor + p += when (dictIdFlag) { + 0 -> 0 + 1 -> 1 + 2 -> 2 + else -> 4 + } + p += when (fcsFlag) { + 0 -> if (singleSegment == 1) 1 else 0 + 1 -> 2 + 2 -> 4 + else -> 8 + } + return p + } + + private fun blockHeader(frame: ByteArray, p: Int): Int = (frame[p].toInt() and 0xFF) or + ((frame[p + 1].toInt() and 0xFF) shl 8) or + ((frame[p + 2].toInt() and 0xFF) shl 16) + + /** + * Advance past a Literals_Section (header + body). For Raw/RLE the + * Size_Format field is read PROGRESSIVELY -- bit 2 alone selects the + * 1-byte/5-bit-size form, in which case bit 3 already belongs to the size + * field -- whereas for Huffman-coded literals it is a flat 2-bit field. + */ + private fun skipLiteralsSection(frame: ByteArray, p: Int): Int { + val b0 = frame[p].toInt() and 0xFF + return when (val litType = b0 and 0x3) { + 0, 1 -> { + val headerLen: Int + val regenSize: Int + if ((b0 ushr 2) and 0x1 == 0) { + headerLen = 1 + regenSize = b0 ushr 3 + } else if ((b0 ushr 3) and 0x1 == 0) { + headerLen = 2 + regenSize = ((b0 ushr 4) and 0xF) or ((frame[p + 1].toInt() and 0xFF) shl 4) + } else { + headerLen = 3 + regenSize = ((b0 ushr 4) and 0xF) or + ((frame[p + 1].toInt() and 0xFF) shl 4) or + ((frame[p + 2].toInt() and 0xFF) shl 12) + } + // RLE literals store ONE byte regardless of Regenerated_Size. + p + headerLen + if (litType == 0) regenSize else 1 + } + + else -> { + val sizeFormat = (b0 ushr 2) and 0x3 + check(sizeFormat == 0) { "Huffman literals size_format $sizeFormat not parsed by this helper" } + val b1 = frame[p + 1].toInt() and 0xFF + val b2 = frame[p + 2].toInt() and 0xFF + val compressedSize = (b1 ushr 6) or (b2 shl 2) + p + 3 + compressedSize + } + } + } +} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt new file mode 100644 index 0000000..3f90343 --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * The encoder's RLE forms (RFC 8878): RLE_Block (Block_Type 1), RLE literals + * (Literals_Block_Type 1) and RLE sequence tables (Symbol_Compression_Mode 1). + * Each is the degenerate one-symbol case of the block's own entropy coding, and + * each is chosen only when it is the smallest valid encoding. + * + * The decoder has always READ all three (libzstd emits them); these tests pin + * that kzstd's encoder now WRITES them, since a round-trip alone would stay + * green if the encoder silently fell back to Raw/Predefined. + */ +class RleEncodingTest { + + private val max = TestVectors.MAX_DECOMPRESSED_SIZE + + @Test + fun constantInput_emitsRleBlock() { + val frame = Zstd.compress(TestVectors.constantBytes) + assertEquals(1, FrameInspector.blockType(frame), "constant input did not produce an RLE_Block") + assertEquals(TestVectors.constantBytes.size, FrameInspector.blockSize(frame), "RLE Block_Size") + // Frame magic (4) + descriptor (1) + window (1) + block header (3) + the byte. + assertEquals(10, frame.size, "RLE_Block frame should be 10 bytes") + assertContentEquals(TestVectors.constantBytes, Zstd.decompress(frame, max)) + } + + @Test + fun shortConstantInputsStillRoundTrip() { + for (n in 0..5) { + val data = ByteArray(n) { 'z'.code.toByte() } + val frame = Zstd.compress(data) + assertContentEquals(data, Zstd.decompress(frame, max), "constant input of $n bytes") + } + } + + @Test + fun constantSequenceCodes_useRleTablesForAllThreeStreams() { + val frame = Zstd.compress(TestVectors.byteRuns) + val (llMode, ofMode, mlMode) = FrameInspector.sequenceModes(frame)!! + assertEquals(1, llMode, "literal-length stream did not use an RLE table") + assertEquals(1, ofMode, "offset stream did not use an RLE table") + assertEquals(1, mlMode, "match-length stream did not use an RLE table") + assertContentEquals(TestVectors.byteRuns, Zstd.decompress(frame, max)) + } + + @Test + fun constantLiterals_useRleLiterals() { + val dict = ZstdDictionary(TestVectors.rawContentDict) + val frame = Zstd.compress(TestVectors.rleLiteralsSample, dict) + assertEquals(1, FrameInspector.literalsType(frame), "constant literals did not produce RLE literals") + assertContentEquals(TestVectors.rleLiteralsSample, Zstd.decompress(frame, dict, max)) + } + + @Test + fun rleFormsShrinkOutput() { + assertTrue( + Zstd.compress(TestVectors.constantBytes).size < 12, + "RLE_Block should compress a constant input to a handful of bytes", + ) + val runs = Zstd.compress(TestVectors.byteRuns).size + assertTrue(runs < 60, "byte-run sample compressed to $runs bytes, expected well under 60") + } +} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt index 0ffc32f..73da810 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt @@ -172,6 +172,48 @@ internal object TestVectors { "28b52ffd230d92503b9435010063010399c65a69e84756accc8c4a8508fc8f87036d206e18b8ca544a6861d98e86a8f857ef0d", ) + /** + * One byte value, repeated: the RLE_Block (Block_Type 1) case — the whole + * block collapses to a single stored byte. + */ + val constantBytes: ByteArray = ByteArray(1500) { 'Q'.code.toByte() } + + /** + * Twenty-six eight-byte single-letter runs. Every run compresses to the SAME + * sequence — one literal, a 7-byte match at distance 1 (which is the initial + * repeat-offset slot, so its offset code is 0) — so each of the three + * LL/OF/ML symbol streams is a single repeated code (RLE, mode 1) and none + * of those codes carries extra bits: the sequence bitstream ends up holding + * nothing but its stop bit. + */ + val byteRuns: ByteArray = buildString { + for (c in 'a'..'z') append(c.toString().repeat(8)) + }.encodeToByteArray() + + /** + * A RAW-content dictionary (no trained-dict magic ⇒ content only, no entropy + * tables), used with [rleLiteralsSample] to reach RLE literals: every phrase + * of the sample is a match into this content, leaving only the separator + * bytes as literals. + */ + val rawContentDict: ByteArray = + "the quick brown fox jumps over the lazy dog while nominal reports stream steadily" + .encodeToByteArray() + + /** + * Phrases from [rawContentDict] separated by a single repeated byte. Each + * phrase becomes a dictionary match and each separator a one-byte literal + * run, so the block's whole literals buffer is one byte value repeated — + * RLE literals (Literals_Block_Type 1) — while the block itself is far from + * constant. + */ + val rleLiteralsSample: ByteArray = ( + "A" + "the quick brown fox " + + "A" + "jumps over the lazy " + + "A" + "dog while nominal " + + "A" + "reports stream steadily" + ).encodeToByteArray() + /** The exact plaintext [treelessDictFrame] must decode to (verified by DictEntropyDecodeTest). */ val treelessDictPlaintext: ByteArray = hexToBytes( "7b2274797065223a22686561727462656174222c22736571223a3634383838362c226e6f6465223a226e6f64652d3233222c227374617465223a226f6b222c226c6174223a2d36382e32303131302c226c6f6e223a2d32352e36383235352c226d7367223a22726567696f6e20616e64206e6f6d696e616c206e6f6d696e616c206d6f6e69746f7265642074686520746865227d", diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/ByteIdenticalRegressionTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/ByteIdenticalRegressionTest.kt index 8cb31f4..50e2e0c 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/ByteIdenticalRegressionTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/ByteIdenticalRegressionTest.kt @@ -40,7 +40,7 @@ class ByteIdenticalRegressionTest { "structured, dict-less", ) assertEquals( - "28b52ffd00203d010013410299c65a6956accc8c1a09fc8f4708340271bbc05542a21a1d58a8d012b3f3ee53c5c60b12", + "28b52ffd00203d010013410299c65a6956accc8c1a09f08fa708340fe2c605ae127a54a9038b20b4c49d37a4b2f18204", hex(Zstd.compress(TestVectors.structured[0], dict)), "structured, trained dict", ) diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt new file mode 100644 index 0000000..8f09880 --- /dev/null +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import com.github.luben.zstd.Zstd as LibZstd + +/** + * Cross-oracle for the encoder's dictionary-FREE entropy coding: every block + * form kzstd can now choose for a plain `Zstd.compress(data)` call is handed to + * REAL libzstd (zstd-jni) to decode. kzstd decoding its own output proves only + * self-consistency; only libzstd accepting these frames proves the wire format + * is conformant. + * + * Each test asserts the block/literals/table MODE first, so it fails loudly if + * the cost model stops choosing the form under test instead of silently + * degrading into a round-trip test of the Raw/Predefined fallback. + */ +class EntropyCodingInteropTest { + + private val max = TestVectors.MAX_DECOMPRESSED_SIZE + + @Test + fun rleBlockDecodesUnderLibzstd() { + val data = TestVectors.constantBytes + val frame = Zstd.compress(data) + assertEquals(1, FrameInspector.blockType(frame), "expected an RLE_Block") + assertContentEquals(data, LibZstd.decompress(frame, max)) + } + + /** + * All three sequence streams RLE (mode 1) AND no sequence code carrying + * extra bits: the sequences bitstream degenerates to a single stop-bit byte. + * That is the sharpest edge in the whole sequences section — a decoder that + * insists on consuming at least one real bit rejects it — so it is checked + * against libzstd rather than reasoned about. + */ + @Test + fun rleSequenceTablesDecodeUnderLibzstd() { + val data = TestVectors.byteRuns + val frame = Zstd.compress(data) + assertEquals(Triple(1, 1, 1), FrameInspector.sequenceModes(frame), "expected RLE tables for LL/OF/ML") + assertContentEquals(data, LibZstd.decompress(frame, max)) + } + + @Test + fun rleLiteralsDecodeUnderLibzstd() { + val data = TestVectors.rleLiteralsSample + val dictBytes = TestVectors.rawContentDict + val frame = Zstd.compress(data, ZstdDictionary(dictBytes)) + assertEquals(1, FrameInspector.literalsType(frame), "expected RLE literals") + // Raw-content dictionary (no trained-dict magic): libzstd treats the + // bytes as pure back-reference content, which is exactly what kzstd did. + assertContentEquals(data, LibZstd.decompress(frame, dictBytes, max)) + } +} From 7344466e90ea3ff07494a78d9f3a5ccb4c977496 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 13:54:36 -0500 Subject: [PATCH 02/11] feat(internal): Huffman-code literals from the block's own histogram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without a dictionary — the default, most common call — the encoder had no way to entropy-code literals at all: `Zstd.compress(data)` could only emit Raw literals, because the one Huffman path it had (Treeless, litType 3) reuses a dictionary's trained table and there is no such table to reuse. Build one from the block's own byte histogram instead (litType 2, Huffman_Compressed), and describe it on the wire so any decoder can rebuild it. The construction deliberately goes the long way round — histogram → code lengths → per-symbol weights → HuffmanTable.fromWeights → HuffmanEncTable.fromDecodeTable — so canonical-code assignment happens once, in the routine that already agrees with libzstd because it is what reads libzstd's own descriptions. The encoder therefore cannot drift from the decoder's idea of which code belongs to which symbol. Code lengths come from a plain Huffman tree whose depth multiset is then repaired to respect the format's 11-bit limit while staying a COMPLETE code (an incomplete one makes the description's implied final weight come out wrong), after which the shortest codes are handed to the most frequent symbols. Reassigning at the end is what makes the repair safe: it can shuffle depths freely without ever pairing a long code with a frequent symbol. The literals section is now chosen by building every candidate in full and taking the smallest — RLE, Treeless, fresh Huffman, Raw — with ties keeping the earlier, simpler form, so the dictionary's Treeless path is never displaced by an equally-sized fresh table. Deliberately not implemented, both falling back to Raw: FSE-compressed weight descriptions (so a literal byte above 128 cannot be described, since the direct 4-bit form's header byte is 127 + Number_of_Weights) and the 4-stream literals layout (so this stays single-stream, capped at 1023 bytes either side — the same cap the Treeless path already had). Measured, dictionary-free: 887 bytes of concatenated structured records 487 -> 431, and a 208-byte prose sample 213 -> 190. Real libzstd decodes frames from four shapes of input including a near-uniform 127-symbol alphabet, which is what proves the tree description conformant rather than merely self-consistent. Signed-off-by: James Rich --- .../kzstd/internal/HuffmanBuilder.kt | 204 ++++++++++++++++++ .../meshtastic/kzstd/internal/ZstdEncoder.kt | 182 ++++++++++------ .../kzstd/HuffmanConstructionTest.kt | 161 ++++++++++++++ .../meshtastic/kzstd/HuffmanLiteralsTest.kt | 87 ++++++++ .../org/meshtastic/kzstd/TestVectors.kt | 32 +++ .../kzstd/EntropyCodingInteropTest.kt | 29 +++ 6 files changed, 624 insertions(+), 71 deletions(-) create mode 100644 src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanLiteralsTest.kt diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt new file mode 100644 index 0000000..42c65c8 --- /dev/null +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd.internal + +// Builds a canonical Huffman table for a block's OWN literals (RFC 8878 §4.2.1, +// Literals_Block_Type 2 "Huffman_Compressed") from that block's byte histogram, +// together with the Huffman_Tree_Description that tells a decoder how to +// rebuild it. +// +// The construction deliberately goes the long way round — histogram → code +// lengths → per-symbol weights → HuffmanTable.fromWeights (the DECODER's own +// table build) → HuffmanEncTable.fromDecodeTable — instead of assigning codes +// directly. Canonical-code assignment then happens exactly once, in the routine +// that already agrees with libzstd (it is what reads libzstd's own +// descriptions), so the encoder cannot drift from the decoder's idea of which +// code belongs to which symbol. + +/** + * Longest literal code this encoder will produce. zstd's own encoder caps + * Huffman depth here (`HUF_TABLELOG_DEFAULT`); the format's ceiling is + * [HuffmanTable.HUF_MAX_TABLELOG], one higher, so staying at 11 keeps every + * description comfortably inside what any decoder accepts. + */ +internal const val MAX_LITERAL_CODE_BITS: Int = 11 + +/** + * The most explicit weights a DIRECT (non-FSE-compressed) tree description can + * carry: its header byte is `127 + Number_of_Weights`, which must fit in a + * byte. Weights are written for symbols `0 until lastSymbol` and the final + * present symbol's weight is implied, so the highest literal byte value this + * encoder can Huffman-code is 128. Blocks containing a higher byte value fall + * back to Raw literals — FSE-compressed weight descriptions, which would lift + * the limit, are not implemented. + */ +private const val MAX_DIRECT_WEIGHTS = 128 + +/** A freshly built literals Huffman table plus its wire description. */ +internal class FreshHuffmanTable(val encoder: HuffmanEncTable, val description: ByteArray) + +/** + * Build a Huffman table for the alphabet [histogram] describes (counts per byte + * value), or null when this block cannot use one: + * - fewer than two distinct byte values (one symbol is the RLE literals case, + * and the format cannot describe a one-symbol tree: the sole symbol's weight + * is the implied one, leaving nothing explicit to imply it from); + * - a byte value above [MAX_DIRECT_WEIGHTS]. + */ +internal fun buildLiteralsHuffman(histogram: IntArray): FreshHuffmanTable? { + val present = ArrayList(16) + for (s in histogram.indices) if (histogram[s] > 0) present.add(s) + if (present.size < 2) return null + val lastSymbol = present[present.size - 1] + if (lastSymbol > MAX_DIRECT_WEIGHTS) return null + + val lengths = huffmanCodeLengths(histogram, present) + var maxBits = 0 + for (s in present) if (lengths[s] > maxBits) maxBits = lengths[s] + + // Weight = Max_Number_of_Bits + 1 - Number_of_Bits (RFC 8878 §4.2.1.3), so + // the longest code has weight 1 and an absent symbol weight 0. + val weights = IntArray(lastSymbol + 1) + for (s in present) weights[s] = maxBits + 1 - lengths[s] + + // The description carries weights for symbols 0 until lastSymbol; the last + // present symbol's weight is whatever makes the total a power of two, which + // fromWeights recomputes. + val explicit = IntArray(lastSymbol) { weights[it] } + val decode = HuffmanTable.fromWeights(explicit, lastSymbol) + return FreshHuffmanTable(HuffmanEncTable.fromDecodeTable(decode), writeDirectWeights(explicit)) +} + +/** + * Huffman_Tree_Description in the DIRECT form (RFC 8878 §4.2.1.2): a header + * byte of `127 + Number_of_Weights`, then the weights packed two per byte, high + * nibble first (an odd count leaves the final low nibble zero). + */ +private fun writeDirectWeights(weights: IntArray): ByteArray { + val out = ByteArray(1 + (weights.size + 1) / 2) + out[0] = (127 + weights.size).toByte() + var i = 0 + while (i < weights.size) { + val high = weights[i] shl 4 + val low = if (i + 1 < weights.size) weights[i + 1] else 0 + out[1 + i / 2] = (high or low).toByte() + i += 2 + } + return out +} + +/** + * Code length per symbol (0 for absent), optimal for [histogram] except that no + * code exceeds [MAX_LITERAL_CODE_BITS]. + * + * Builds the Huffman tree, keeps only the resulting MULTISET of depths, repairs + * that multiset to respect the depth limit while staying a COMPLETE code (Kraft + * sum exactly 1 — an incomplete code would make the description's implied final + * weight come out wrong), then hands the shortest codes to the most frequent + * symbols. Reassigning at the end is what makes the repair safe: it can shuffle + * depths freely without ever pairing a long code with a frequent symbol. + */ +private fun huffmanCodeLengths(histogram: IntArray, present: List): IntArray { + val depths = huffmanTreeDepths(histogram, present) + + // Symbols per code length, with anything past the limit clamped onto it. + val countPerLength = IntArray(MAX_LITERAL_CODE_BITS + 1) + for (d in depths) countPerLength[if (d > MAX_LITERAL_CODE_BITS) MAX_LITERAL_CODE_BITS else d]++ + + balanceKraftSum(countPerLength) + + // Shortest lengths to the highest counts; ties by ascending symbol so the + // result is deterministic on every target. + val ordered = present.sortedWith(compareByDescending { histogram[it] }.thenBy { it }) + val lengths = IntArray(histogram.size) + var i = 0 + for (len in 1..MAX_LITERAL_CODE_BITS) { + repeat(countPerLength[len]) { lengths[ordered[i++]] = len } + } + return lengths +} + +/** + * Make the code lengths in [countPerLength] a complete code — Kraft sum exactly + * `2^MAX_LITERAL_CODE_BITS` in units of `2^-MAX_LITERAL_CODE_BITS` — after + * clamping over-long codes onto the limit. + * + * Clamping only ever makes codes SHORTER, so the sum can overrun; each repair + * step lengthens one code by a bit (the longest that is still below the limit, + * which is the cheapest place to spend a bit). Overshooting leaves an + * INCOMPLETE code, so the second loop shortens the longest codes back until the + * sum lands exactly on the target — always possible, because every term is a + * multiple of the smallest one. + */ +private fun balanceKraftSum(countPerLength: IntArray) { + val target = 1 shl MAX_LITERAL_CODE_BITS + var kraft = 0 + for (len in 1..MAX_LITERAL_CODE_BITS) kraft += countPerLength[len] shl (MAX_LITERAL_CODE_BITS - len) + + while (kraft > target) { + var len = MAX_LITERAL_CODE_BITS - 1 + while (countPerLength[len] == 0) len-- + countPerLength[len]-- + countPerLength[len + 1]++ + kraft -= 1 shl (MAX_LITERAL_CODE_BITS - len - 1) + } + while (kraft < target) { + var len = MAX_LITERAL_CODE_BITS + while (countPerLength[len] == 0) len-- + countPerLength[len]-- + countPerLength[len - 1]++ + kraft += 1 shl (MAX_LITERAL_CODE_BITS - len) + } +} + +/** + * Depth of each leaf in the (unconstrained) Huffman tree over [present]'s + * counts, returned in no particular order — only the multiset of depths is + * used. + * + * Classic construction: repeatedly merge the two lightest live nodes. The + * alphabet is at most 256 symbols, so the O(n^2) scan for those two costs + * nothing measurable and avoids a heap. Children are always created before + * their parent, so one reverse pass over the node array propagates depths from + * the root outward. + */ +private fun huffmanTreeDepths(histogram: IntArray, present: List): IntArray { + val leaves = present.size + val nodes = 2 * leaves - 1 + val weight = LongArray(nodes) + val left = IntArray(nodes) + val right = IntArray(nodes) + val live = BooleanArray(nodes) + for (i in 0 until leaves) { + weight[i] = histogram[present[i]].toLong() + live[i] = true + } + + var next = leaves + while (next < nodes) { + var first = -1 + var second = -1 + for (n in 0 until next) { + if (!live[n]) continue + if (first < 0 || weight[n] < weight[first]) { + second = first + first = n + } else if (second < 0 || weight[n] < weight[second]) { + second = n + } + } + weight[next] = weight[first] + weight[second] + left[next] = first + right[next] = second + live[first] = false + live[second] = false + live[next] = true + next++ + } + + val depth = IntArray(nodes) + for (n in nodes - 1 downTo leaves) { + depth[left[n]] = depth[n] + 1 + depth[right[n]] = depth[n] + 1 + } + return IntArray(leaves) { depth[it] } +} diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt index bae8da2..62992a6 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt @@ -400,95 +400,135 @@ internal object PureZstdEncoder { return ByteArray(out.size) { out[it] } } - // Single-stream Treeless Huffman literals (RFC 8878 3.1.1.3.1.1, - // size_format 0) cap both Regenerated_Size and Compressed_Size at 10 bits - // each. The 4-stream jump-table layout, needed only past ~1 KB of - // literals, is out of scope: this codec's real payloads (CoT/TAK protobuf - // bytes) are comfortably under this cap, and runs over it fall back to Raw. - private const val MAX_TREELESS_LITERALS = 1023 + // Single-stream Huffman literals (RFC 8878 3.1.1.3.1.1, size_format 0) + // cap both Regenerated_Size and Compressed_Size at 10 bits each. The + // 4-stream jump-table layout, needed only past ~1 KB of literals, is out of + // scope: this codec's real payloads (CoT/TAK protobuf bytes) are + // comfortably under this cap, and runs over it fall back to Raw. + private const val MAX_SINGLE_STREAM_LITERALS = 1023 /** - * Literals_Section_Header + body. Prefers reusing the dictionary's trained - * Huffman table (Treeless, litType 3) over Raw (litType 0) when ALL of: - * the dict has a table, every literal byte is [HuffmanEncTable.isCovered] - * by it (a dict's training corpus commonly never produced every byte - * value), the encoded size fits the 10-bit field this encoder supports, - * AND the encoded form is actually smaller than Raw -- Treeless can lose - * for a payload that doesn't match the dict's trained distribution, so - * this comparison is mandatory, not a hint. + * Literals_Section_Header + body, in whichever encoding is smallest: + * + * - **RLE (litType 1)** -- one stored byte regenerates the whole run. + * - **Treeless (litType 3)** -- reuses the dictionary's trained Huffman + * table, costing no tree description at all, when the dict has one that + * covers every literal byte in this block (a dict's training corpus + * commonly never produced every byte value). + * - **Huffman_Compressed (litType 2)** -- a table built from THIS block's + * own histogram, plus the tree description a decoder needs to rebuild + * it. Pays for the description, so it wins only once the literals are + * both numerous and skewed enough. + * - **Raw (litType 0)** -- the fallback, and the winner for short or + * high-entropy literal runs. + * + * The comparison is mandatory rather than a hint: an entropy-coded form can + * lose outright (Treeless for a payload unlike the dict's training corpus, + * Huffman for near-uniform literals), so each candidate is built in full + * and measured. Ties keep the earlier, simpler candidate in the list order + * above -- which is also what keeps the dictionary's Treeless path from + * being displaced by an equally-sized fresh table. */ private fun writeLiteralsSection(out: ArrayList, literals: ByteArray, dict: ParsedDictionary) { val rawCost = rawLiteralsHeaderLen(literals.size) + literals.size - - // RLE literals (litType 1): one stored byte regenerates the whole run. - // Reachable whenever every literal is the same byte -- with a dictionary - // that happens for real inputs whose only literals are a repeated - // separator, every other byte having been matched into the dict. - val constant = constantByteOrNull(literals) - if (constant != null && rawLiteralsHeaderLen(literals.size) + 1 < rawCost) { - writeRawOrRleLiteralsHeader(out, literals.size, litType = 1) - out.add(constant) + val best = listOfNotNull( + buildRleLiterals(literals), + buildTreelessLiterals(literals, dict), + buildHuffmanLiterals(literals), + ).minByOrNull { it.size } + + if (best != null && best.size < rawCost) { + best.forEach { out.add(it) } return } - - val huffman = dict.literalsHuffman - if (huffman != null && literals.isNotEmpty() && literals.size <= MAX_TREELESS_LITERALS) { - val encTable = HuffmanEncTable.fromDecodeTable(huffman) - var covered = true - for (b in literals) { - if (!encTable.isCovered(b.toInt() and 0xFF)) { - covered = false - break - } - } - if (covered) { - // Build the actual bitstream (cheap at this size) rather than - // estimating its byte length from summed bit-lengths -- the - // trailing stop bit (see ReverseBitWriter) can push the real - // length one byte past a naive ceil(totalBits/8) estimate. - val bw = ReverseBitWriter() - for (i in literals.size - 1 downTo 0) { - encTable.encode(bw, literals[i].toInt() and 0xFF) - } - val stream = bw.finish() - if (stream.size <= MAX_TREELESS_LITERALS) { - val rawCost = rawLiteralsHeaderLen(literals.size) + literals.size - val treelessCost = TREELESS_HEADER_LEN + stream.size - if (treelessCost < rawCost) { - writeTreelessHuffmanLiteralsHeader(out, literals.size, stream.size) - stream.forEach { out.add(it) } - return - } - } - } - } - // Literals_Section_Header (Raw, litType 0). Regenerated_Size = literals // length, encoded with the 1/2/3-byte size_format variants. writeRawLiteralsHeader(out, literals.size) literals.forEach { out.add(it) } } - private const val TREELESS_HEADER_LEN = 3 + /** RLE literals (litType 1): header + the single repeated byte, or null. */ + private fun buildRleLiterals(literals: ByteArray): ByteArray? { + val constant = constantByteOrNull(literals) ?: return null + val section = ArrayList(4) + writeRawOrRleLiteralsHeader(section, literals.size, litType = 1) + section.add(constant) + return ByteArray(section.size) { section[it] } + } - private fun rawLiteralsHeaderLen(size: Int): Int = when { - size < 32 -> 1 - size < 4096 -> 2 - else -> 3 + /** + * Treeless literals (litType 3): the dictionary's own Huffman table, so the + * section carries no tree description -- only the header and the stream. + * Null when there is no dict table, it does not cover some literal byte, or + * the result overflows the single-stream size fields. + */ + private fun buildTreelessLiterals(literals: ByteArray, dict: ParsedDictionary): ByteArray? { + val huffman = dict.literalsHuffman ?: return null + if (literals.isEmpty() || literals.size > MAX_SINGLE_STREAM_LITERALS) return null + val encTable = HuffmanEncTable.fromDecodeTable(huffman) + for (b in literals) if (!encTable.isCovered(b.toInt() and 0xFF)) return null + val stream = huffmanLiteralsStream(literals, encTable) + if (stream.size > MAX_SINGLE_STREAM_LITERALS) return null + return literalsSection(litType = 3, regenSize = literals.size, body = stream) } /** - * litType=3 (Treeless), size_format=0 (single stream, 10-bit - * Regenerated_Size / Compressed_Size) -- the inverse of - * [PureZstdDecoder]'s size_format-0 Huffman-literals header read. + * Huffman_Compressed literals (litType 2): a canonical table built from this + * block's own byte histogram, described on the wire ahead of the stream. + * Null when the alphabet cannot be Huffman-coded at all + * ([buildLiteralsHuffman]) or the result overflows the single-stream size + * fields. */ - private fun writeTreelessHuffmanLiteralsHeader(out: ArrayList, regenSize: Int, compressedSize: Int) { - val b0 = 3 or ((regenSize and 0xF) shl 4) - val b1 = ((regenSize ushr 4) and 0x3F) or ((compressedSize and 0x3) shl 6) - val b2 = (compressedSize ushr 2) and 0xFF - out.add(b0.toByte()) - out.add(b1.toByte()) - out.add(b2.toByte()) + private fun buildHuffmanLiterals(literals: ByteArray): ByteArray? { + if (literals.isEmpty() || literals.size > MAX_SINGLE_STREAM_LITERALS) return null + val histogram = IntArray(256) + for (b in literals) histogram[b.toInt() and 0xFF]++ + val table = buildLiteralsHuffman(histogram) ?: return null + val stream = huffmanLiteralsStream(literals, table.encoder) + // Compressed_Size counts the tree description AND the stream. + val body = table.description + stream + if (body.size > MAX_SINGLE_STREAM_LITERALS) return null + return literalsSection(litType = 2, regenSize = literals.size, body = body) + } + + /** + * Huffman-code [literals] into one backward bitstream. Written in reverse so + * the decoder, reading the stream backward from its stop bit, recovers them + * in order. + * + * The bitstream is built in full rather than having its length estimated + * from summed code lengths -- the trailing stop bit (see [ReverseBitWriter]) + * can push the real length one byte past a naive `ceil(totalBits/8)`. + */ + private fun huffmanLiteralsStream(literals: ByteArray, encTable: HuffmanEncTable): ByteArray { + val bw = ReverseBitWriter() + for (i in literals.size - 1 downTo 0) encTable.encode(bw, literals[i].toInt() and 0xFF) + return bw.finish() + } + + /** + * A complete Huffman-coded Literals_Section: the size_format-0 header + * (single stream, 10-bit Regenerated_Size / Compressed_Size) followed by + * [body] -- the inverse of [PureZstdDecoder]'s size_format-0 read. For + * litType 2 the body is the tree description plus the stream; for litType 3 + * it is the stream alone. + */ + private fun literalsSection(litType: Int, regenSize: Int, body: ByteArray): ByteArray { + val b0 = litType or ((regenSize and 0xF) shl 4) + val b1 = ((regenSize ushr 4) and 0x3F) or ((body.size and 0x3) shl 6) + val b2 = (body.size ushr 2) and 0xFF + val out = ByteArray(3 + body.size) + out[0] = b0.toByte() + out[1] = b1.toByte() + out[2] = b2.toByte() + body.copyInto(out, 3) + return out + } + + private fun rawLiteralsHeaderLen(size: Int): Int = when { + size < 32 -> 1 + size < 4096 -> 2 + else -> 3 } private fun writeRawLiteralsHeader(out: ArrayList, size: Int) = diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt new file mode 100644 index 0000000..3a8f549 --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import org.meshtastic.kzstd.internal.ForwardByteReader +import org.meshtastic.kzstd.internal.HuffmanEncTable +import org.meshtastic.kzstd.internal.HuffmanTable +import org.meshtastic.kzstd.internal.MAX_LITERAL_CODE_BITS +import org.meshtastic.kzstd.internal.ReverseBitReader +import org.meshtastic.kzstd.internal.ReverseBitWriter +import org.meshtastic.kzstd.internal.buildLiteralsHuffman +import org.meshtastic.kzstd.internal.parseHuffmanTable +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Builds a canonical Huffman table from a block's own byte histogram (RFC 8878 + * §4.2.1) and proves the result is self-inverse: the Huffman_Tree_Description + * kzstd writes, parsed back by kzstd's own [parseHuffmanTable] (the routine + * that already reads libzstd's descriptions), must rebuild the very table the + * encoder used. + * + * That round-trip is the load-bearing check — a table that is internally + * consistent but described wrongly on the wire produces literals that decode to + * garbage everywhere else, and only the description round-trip localises it. + */ +class HuffmanConstructionTest { + + @Test + fun twoEquallyLikelySymbolsGetOneBitCodes() { + val hist = IntArray(256) + hist['a'.code] = 4 + hist['b'.code] = 4 + val built = assertNotNull(buildLiteralsHuffman(hist)) + assertEquals(1, built.encoder.bitLength('a'.code), "'a' code length") + assertEquals(1, built.encoder.bitLength('b'.code), "'b' code length") + } + + @Test + fun skewedHistogramGivesTheFrequentSymbolTheShortestCode() { + val hist = IntArray(256) + hist['a'.code] = 100 + hist['b'.code] = 10 + hist['c'.code] = 5 + hist['d'.code] = 1 + val built = assertNotNull(buildLiteralsHuffman(hist)) + val a = built.encoder.bitLength('a'.code) + assertEquals(1, a, "the dominant symbol should get a 1-bit code") + assertTrue(a < built.encoder.bitLength('b'.code), "'a' must be shorter than 'b'") + assertTrue(built.encoder.bitLength('b'.code) <= built.encoder.bitLength('c'.code), "'b' <= 'c'") + assertTrue(built.encoder.bitLength('c'.code) <= built.encoder.bitLength('d'.code), "'c' <= 'd'") + } + + @Test + fun fewerThanTwoDistinctSymbolsHaveNoHuffmanTable() { + assertNull(buildLiteralsHuffman(IntArray(256)), "empty histogram") + val one = IntArray(256) + one['x'.code] = 9 + assertNull(buildLiteralsHuffman(one), "a single-symbol alphabet is the RLE case, not a Huffman one") + } + + @Test + fun highByteValuesFallBackToNoTable() { + // The direct 4-bit weight description can carry at most 128 explicit + // weights, so a literal byte above 128 cannot be described this way. + val hist = IntArray(256) + hist[1] = 5 + hist[200] = 5 + assertNull(buildLiteralsHuffman(hist), "byte 200 exceeds the direct-weight description's reach") + } + + /** + * A Fibonacci-shaped histogram is the classic worst case: an unconstrained + * Huffman tree over it is a degenerate chain far deeper than the format's + * limit, so this exercises the length-limiting repair, which must still + * leave a COMPLETE code (an incomplete one makes the implied final weight + * come up wrong and the description unparseable). + */ + @Test + fun deepHistogramIsLengthLimitedAndStillRoundTrips() { + val hist = IntArray(256) + var a = 1L + var b = 1L + for (s in 0 until 40) { + hist[s] = a.toInt() + val next = a + b + a = b + b = next + } + val built = assertNotNull(buildLiteralsHuffman(hist)) + for (s in 0 until 40) { + assertTrue( + built.encoder.bitLength(s) in 1..MAX_LITERAL_CODE_BITS, + "symbol $s has out-of-range code length ${built.encoder.bitLength(s)}", + ) + } + assertDescriptionRoundTrips(hist, built.description, built.encoder) + } + + @Test + fun descriptionRoundTripsForAssortedHistograms() { + for (hist in histograms()) { + val built = assertNotNull(buildLiteralsHuffman(hist)) + assertDescriptionRoundTrips(hist, built.description, built.encoder) + } + } + + private fun histograms(): List = listOf( + // Two symbols at the extremes of the describable range. + IntArray(256).also { + it[0] = 1 + it[128] = 1 + }, + // Dense low alphabet. + IntArray(256).also { for (s in 0 until 96) it[s] = s + 1 }, + // Sparse: long runs of absent symbols between present ones. + IntArray(256).also { for (s in 0 until 128 step 17) it[s] = 100 - s }, + // Printable ASCII with an English-ish skew. + IntArray(256).also { + it[' '.code] = 300 + it['e'.code] = 200 + it['t'.code] = 150 + it['a'.code] = 90 + it['z'.code] = 1 + it['q'.code] = 1 + it['\n'.code] = 7 + }, + ) + + /** + * Parse [description] exactly as the decoder would, then check the rebuilt + * table assigns every symbol the same code as [encoder] — by encoding one + * of each present symbol and decoding it back through the parsed table. + */ + private fun assertDescriptionRoundTrips(hist: IntArray, description: ByteArray, encoder: HuffmanEncTable) { + val reader = ForwardByteReader(description, 0, description.size) + val parsed = parseHuffmanTable(reader) + assertEquals(description.size, reader.pos, "description length consumed") + + val symbols = (0..255).filter { hist[it] > 0 } + val bw = ReverseBitWriter() + // Written in reverse so the decoder reads them in `symbols` order. + for (i in symbols.indices.reversed()) encoder.encode(bw, symbols[i]) + val stream = bw.finish() + + val br = ReverseBitReader(stream, 0, stream.size) + for (s in symbols) { + assertEquals(s, parsed.decode(br), "symbol $s did not survive the description round trip") + } + assertRebuiltTableAgrees(parsed, encoder, symbols) + } + + private fun assertRebuiltTableAgrees(parsed: HuffmanTable, encoder: HuffmanEncTable, symbols: List) { + val rebuilt = HuffmanEncTable.fromDecodeTable(parsed) + for (s in symbols) { + assertEquals(encoder.bitLength(s), rebuilt.bitLength(s), "code length for symbol $s") + } + } +} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanLiteralsTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanLiteralsTest.kt new file mode 100644 index 0000000..6c2833c --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanLiteralsTest.kt @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNotEquals +import kotlin.test.assertTrue + +/** + * Adoption guard for Huffman_Compressed literals (Literals_Block_Type 2) built + * from a block's OWN histogram — the dictionary-free path, which is what a + * plain `Zstd.compress(data)` call uses. + * + * Round-trips alone would stay green if the cost model silently regressed to + * Raw literals, so each test asserts the literals type actually chosen, and one + * pins a ratio ratchet against the pre-entropy-coding baseline. + */ +class HuffmanLiteralsTest { + + private val max = TestVectors.MAX_DECOMPRESSED_SIZE + + /** Text whose literals are numerous and skewed — the case Huffman wins. */ + private val prose: ByteArray = ( + "the quick brown fox jumps over the lazy dog while nominal reports stream " + + "steadily across every monitored link and latency stays within throughput " + + "targets even when the region degrades to offline for a while. " + ).encodeToByteArray() + + @Test + fun skewedLiteralsUseFreshHuffman() { + val frame = Zstd.compress(prose) + assertEquals(2, FrameInspector.literalsType(frame), "prose literals were not Huffman-coded") + assertContentEquals(prose, Zstd.decompress(frame, max)) + } + + /** + * Ratio ratchet against the RAW-LITERALS baseline: the exact sizes the + * encoder produced immediately before fresh Huffman literals existed. These + * are the numbers that prove entropy coding is actually engaged and paying + * — a mode assertion alone would still pass if the table were built badly. + * Refresh (never loosen without a reason) if a later, deliberate change + * legitimately moves them. + */ + @Test + fun huffmanLiteralsShrinkOutput() { + val structuredRun = TestVectors.structured.reduce { a, b -> a + b } + val structuredSize = Zstd.compress(structuredRun).size + assertTrue( + structuredSize < 487, + "concatenated structured records: $structuredSize bytes, raw-literals baseline was 487", + ) + val proseSize = Zstd.compress(prose).size + assertTrue(proseSize < 213, "prose: $proseSize bytes, raw-literals baseline was 213") + } + + @Test + fun incompressibleLiteralsStayRaw() { + // Near-uniform bytes: a Huffman table plus its description costs more + // than it saves, so the cost model must keep Raw literals. + val random = TestVectors.corpus.last() + val frame = Zstd.compress(random) + assertNotEquals(2, FrameInspector.literalsType(frame), "near-random literals should not be Huffman-coded") + assertContentEquals(random, Zstd.decompress(frame, max)) + } + + @Test + fun everyCorpusSampleStillRoundTrips() { + for (sample in TestVectors.corpus) { + val frame = Zstd.compress(sample) + assertContentEquals(sample, Zstd.decompress(frame, max), "size=${sample.size}") + } + } + + /** + * Literal bytes above 128 cannot be described with the direct 4-bit weight + * form this encoder writes, so such a block must fall back rather than + * emit an undescribable table. + */ + @Test + fun highByteLiteralsFallBackAndStillRoundTrip() { + val data = ByteArray(600) { i -> (128 + (i * 7) % 100).toByte() } + val frame = Zstd.compress(data) + assertNotEquals(2, FrameInspector.literalsType(frame), "high-byte literals cannot use a direct-weight tree") + assertContentEquals(data, Zstd.decompress(frame, max)) + } +} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt index 73da810..bfed78a 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt @@ -214,6 +214,25 @@ internal object TestVectors { "A" + "reports stream steadily" ).encodeToByteArray() + /** + * Skewed draws over ~90 byte values, with essentially no repeated + * substrings: nearly every byte stays a literal, so the block's literals + * exercise a wide Huffman alphabet whose counts differ by orders of + * magnitude (long weight descriptions, and code lengths spread far enough + * to reach the length limit). + */ + val skewedAlphabet: ByteArray = mappedPseudoRandom(900) { r -> + val x = r % 120 + x * x / 120 + } + + /** + * Near-uniform draws over 127 byte values — close to the widest alphabet a + * direct (non-FSE-compressed) Huffman_Tree_Description can carry, and the + * least favourable shape for Huffman coding that still pays for itself. + */ + val wideAlphabet: ByteArray = mappedPseudoRandom(900) { r -> r % 127 } + /** The exact plaintext [treelessDictFrame] must decode to (verified by DictEntropyDecodeTest). */ val treelessDictPlaintext: ByteArray = hexToBytes( "7b2274797065223a22686561727462656174222c22736571223a3634383838362c226e6f6465223a226e6f64652d3233222c227374617465223a226f6b222c226c6174223a2d36382e32303131302c226c6f6e223a2d32352e36383235352c226d7367223a22726567696f6e20616e64206e6f6d696e616c206e6f6d696e616c206d6f6e69746f7265642074686520746865227d", @@ -221,6 +240,19 @@ internal object TestVectors { private fun hexToBytes(s: String): ByteArray = ByteArray(s.length / 2) { ((s[it * 2].digitToInt(16) shl 4) or s[it * 2 + 1].digitToInt(16)).toByte() } + /** + * Deterministic pseudo-random bytes drawn through [toSymbol], which maps a + * fresh LCG value to a byte value — the way to build a payload with a + * chosen alphabet size and skew that is identical on every target. + */ + private fun mappedPseudoRandom(n: Int, toSymbol: (Int) -> Int): ByteArray { + var s = 0x12345678 + return ByteArray(n) { + s = (s * 1103515245 + 12345) and 0x7FFFFFFF + toSymbol(s ushr 8).toByte() + } + } + /** Deterministic pseudo-random bytes (a 32-bit LCG) — incompressible-ish input. */ private fun pseudoRandom(n: Int): ByteArray { var s = 0x12345678 diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt index 8f09880..622b757 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt @@ -44,6 +44,35 @@ class EntropyCodingInteropTest { assertContentEquals(data, LibZstd.decompress(frame, max)) } + /** + * The whole point of the fresh-Huffman work: a table built from the block's + * own histogram, described on the wire, and rebuilt by an independent + * decoder. Nothing but a real libzstd decode proves the + * Huffman_Tree_Description and the canonical code assignment agree with the + * spec rather than merely with kzstd's own reader. + */ + @Test + fun freshHuffmanLiteralsDecodeUnderLibzstd() { + for (data in huffmanSamples()) { + val frame = Zstd.compress(data) + assertEquals(2, FrameInspector.literalsType(frame), "expected Huffman literals for ${data.size} bytes") + assertContentEquals(data, LibZstd.decompress(frame, max), "size=${data.size}") + } + } + + private fun huffmanSamples(): List = listOf( + ( + "the quick brown fox jumps over the lazy dog while nominal reports stream " + + "steadily across every monitored link and latency stays within throughput " + + "targets even when the region degrades to offline for a while. " + ).encodeToByteArray(), + TestVectors.structured.reduce { a, b -> a + b }, + // Wide alphabets: long weight descriptions, and (for the skewed one) + // code lengths spread far enough to reach the length limit. + TestVectors.skewedAlphabet, + TestVectors.wideAlphabet, + ) + @Test fun rleLiteralsDecodeUnderLibzstd() { val data = TestVectors.rleLiteralsSample From b8f5456a6faac61e808fac83688c0e6be4d180b1 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 14:03:54 -0500 Subject: [PATCH 03/11] feat(internal): build FSE sequence tables from the block's own counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the dictionary-free entropy coding: with no dictionary the three sequence streams could only use the spec's PREDEFINED distributions, which are a guess at a typical block and often a poor fit for the block in hand. Each stream can now carry an FSE_Compressed table (Symbol_Compression_Mode 2) normalized from its own code counts. The two new pieces are both exact inverses of code the decoder already has, which is what makes them checkable in isolation: - normalizeFseCounts scales the counts to fill the table exactly, keeping every code that occurs at a nonzero probability. It deliberately never emits the "less than 1" probability (-1) the format allows: giving rare codes a full cell costs a little ratio and keeps both the normalizer and the description writer free of negative counts, the fiddliest corner of the encoding. - writeFseTableDescription mirrors parseFseTable field for field — the 4-bit biased Accuracy_Log, the field width that shrinks as the probability budget drains, the extra bit taken only above `max`, and the 2-bit run-length groups for absent codes. Accuracy_Log follows zstd's FSE_optimalTableLog, and the normalized counts are fed through FseTable.build — the decoder's own table build — so the encoder cannot hold a different table from the one its description will produce. FseTableDescriptionTest round-trips writer against the decoder's parser over distributions covering each branch (adjacent symbols, zero runs of exactly 3 and 4, a run spanning most of the alphabet, flat, and heavily skewed) at every Accuracy_Log, comparing the rebuilt table cell by cell, and encodes a whole symbol stream through the result. EntropyCodingInteropTest then hands real libzstd a frame using fresh tables for all three streams at once. Measured, dictionary-free: ~7.8 KB of synthetic JSON telemetry records compresses to 1521 bytes with all three streams on their own tables, and 887 bytes of concatenated structured records 431 -> 425 (the offset stream alone repays a description at that size). Short blocks keep Predefined, as they must: a single-sequence block cannot repay a table description. Signed-off-by: James Rich --- .../kzstd/internal/FseTableWriter.kt | 238 ++++++++++++++++++ .../meshtastic/kzstd/internal/ZstdEncoder.kt | 31 ++- .../kzstd/FreshSequenceTablesTest.kt | 67 +++++ .../kzstd/FseTableDescriptionTest.kt | 197 +++++++++++++++ .../org/meshtastic/kzstd/TestVectors.kt | 32 +++ .../kzstd/EntropyCodingInteropTest.kt | 35 +++ 6 files changed, 595 insertions(+), 5 deletions(-) create mode 100644 src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/FseTableDescriptionTest.kt diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt new file mode 100644 index 0000000..f6277d8 --- /dev/null +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd.internal + +import org.meshtastic.kzstd.ZstdException + +// Builds an FSE table for one sequence symbol stream from THAT STREAM's own code +// counts (RFC 8878 §4.1.1, Symbol_Compression_Mode 2 "FSE_Compressed"), and +// writes the table description a decoder needs to rebuild it. +// +// Everything here is the exact inverse of code the decoder already has: +// normalized counts feed FseTable.build (the decoder's own table build, so +// encoder and decoder cannot disagree about the table), and the description +// writer mirrors parseFseTable field for field, including its shrinking field +// width and its run-length encoding of absent symbols. + +/** + * Smallest Accuracy_Log the description can carry: the header stores + * `Accuracy_Log - 5` in 4 bits. + */ +internal const val FSE_MIN_TABLELOG: Int = 5 + +/** A freshly built sequence-stream FSE table plus its wire description. */ +internal class FreshFseTable(val encoder: FseEncTable, val description: ByteArray) + +/** + * Build an FSE table for [codes] (one sequence stream's symbols, in any order — + * only their counts matter), or null when a fresh table is not applicable: + * fewer than two distinct codes is the RLE case, which is always smaller. + * + * [maxSymbol] and [maxLog] are the stream's format-defined bounds (e.g. + * [OFFSET_MAX_SYMBOL] / [OFFSET_MAX_LOG]); the decoder parses the description + * with the same pair, so the table it rebuilds is identical to the one returned + * here. + */ +internal fun buildFreshFseTable(codes: IntArray, maxSymbol: Int, maxLog: Int): FreshFseTable? { + if (codes.size < 2) return null + val counts = IntArray(maxSymbol + 1) + for (c in codes) counts[c]++ + var distinct = 0 + for (c in counts) if (c > 0) distinct++ + if (distinct < 2) return null + + var highestCode = maxSymbol + while (highestCode > 0 && counts[highestCode] == 0) highestCode-- + val tableLog = fseTableLog(codes.size, highestCode, distinct, maxLog) ?: return null + val normalized = normalizeFseCounts(counts, maxSymbol, tableLog) + val decode = FseTable.build(normalized, maxSymbol, tableLog) + return FreshFseTable( + FseEncTable.fromDecodeTable(decode, maxSymbol), + writeFseTableDescription(normalized, maxSymbol, tableLog), + ) +} + +/** + * Accuracy_Log for a stream of [count] symbols whose highest code is + * [highestCode] and which uses [distinct] distinct codes, bounded by the + * stream's [maxLog]; null when even the largest allowed table could not give + * every code a cell (unreachable for the three sequence streams, whose + * alphabets are far smaller than their tables). + * + * This is zstd's `FSE_optimalTableLog`: precision beyond roughly a quarter of + * the symbol count buys nothing but description bytes, yet a table must also + * stay coarse enough to be worth describing and wide enough to spread the + * alphabet — hence the floor derived from both the symbol count and the + * alphabet size, applied AFTER the count-derived ceiling. + */ +private fun fseTableLog(count: Int, highestCode: Int, distinct: Int, maxLog: Int): Int? { + var log = maxLog + val fromCount = highBit(count - 1) - 2 + if (fromCount < log) log = fromCount + val floor = minOf(highBit(count) + 1, highBit(highestCode) + 2) + if (floor > log) log = floor + log = log.coerceIn(FSE_MIN_TABLELOG, maxLog) + while ((1 shl log) < distinct && log < maxLog) log++ + return if ((1 shl log) < distinct) null else log +} + +/** + * Scale [counts] (0..[maxSymbol]) into a distribution summing to EXACTLY + * `1 shl tableLog`, where every symbol that occurs keeps a nonzero probability + * and every symbol that does not stays at zero. + * + * Deliberately never emits the "less than 1" probability (-1) that the format + * allows and the decoder understands: giving rare symbols a full cell costs a + * little ratio but keeps both this routine and the description writer free of + * negative counts, which is the fiddliest corner of the encoding. The caller + * guarantees `distinct <= 1 shl tableLog`, which is what makes a floor of 1 per + * present symbol achievable. + */ +internal fun normalizeFseCounts(counts: IntArray, maxSymbol: Int, tableLog: Int): IntArray { + val tableSize = 1 shl tableLog + var total = 0L + for (s in 0..maxSymbol) total += counts[s] + if (total <= 0) throw ZstdException("FSE normalize: empty count distribution") + + val normalized = IntArray(maxSymbol + 1) + var assigned = 0 + for (s in 0..maxSymbol) { + val c = counts[s] + if (c <= 0) continue + // Round to nearest, but never below one cell. + val share = ((c.toLong() * tableSize + total / 2) / total).toInt().coerceAtLeast(1) + normalized[s] = share + assigned += share + } + + // Rounding leaves the total off by a little either way; settle it against + // the symbol with the most cells, where a cell is worth the least. + while (assigned != tableSize) { + // Give a cell back / take one from the symbol holding the most, but + // never leave a symbol that occurs with no cell at all. + val floor = if (assigned > tableSize) 1 else 0 + var target = -1 + for (s in 0..maxSymbol) { + val cells = normalized[s] + if (cells > floor && (target < 0 || cells > normalized[target])) target = s + } + if (target < 0) { + // Only reachable if more symbols occur than the table has cells, + // which the caller's tableLog choice rules out. + throw ZstdException("FSE normalize: $assigned cells cannot be reduced to $tableSize") + } + if (assigned > tableSize) { + normalized[target]-- + assigned-- + } else { + normalized[target]++ + assigned++ + } + } + return normalized +} + +/** + * Write the FSE_Table_Description for [normalized] (RFC 8878 §4.1.1, "FSE Table + * Description") — the exact inverse of [parseFseTable]. + * + * The description is a forward, LSB-first bit stream: a 4-bit biased + * Accuracy_Log, then one field per symbol whose width shrinks as the remaining + * probability budget drains, so common small counts cost fewer bits. A count + * fits in `nbBits - 1` bits while it is below `max`; otherwise it takes the + * full width, biased by `max` once it reaches `threshold`, which is exactly how + * the reader decides whether to fetch that extra bit. Absent symbols after a + * zero count are run-length encoded in 2-bit groups, a group of 3 meaning + * "three more, keep reading". + */ +internal fun writeFseTableDescription(normalized: IntArray, maxSymbol: Int, tableLog: Int): ByteArray { + val bits = ForwardBitWriter() + val tableSize = 1 shl tableLog + bits.write(tableLog - FSE_MIN_TABLELOG, 4) + + var remaining = tableSize + 1 // +1 for the reader's extra-accuracy bias + var threshold = tableSize + var nbBits = tableLog + 1 + var symbol = 0 + var previousIsZero = false + + while (remaining > 1 && symbol <= maxSymbol) { + if (previousIsZero) { + // Run of further absent symbols, in groups of up to 3. + val start = symbol + while (symbol <= maxSymbol && normalized[symbol] == 0) symbol++ + var skipped = symbol - start + while (skipped >= 3) { + bits.write(3, 2) + skipped -= 3 + } + bits.write(skipped, 2) + previousIsZero = false + continue + } + + val count = normalized[symbol] + symbol++ + val max = (2 * threshold - 1) - remaining + val encoded = count + 1 // the reader subtracts this bias back off + when { + encoded >= threshold -> bits.write(encoded + max, nbBits) + encoded >= max -> bits.write(encoded, nbBits) + else -> bits.write(encoded, nbBits - 1) + } + remaining -= count + previousIsZero = count == 0 + + // Shrink the field width as the budget drains. + while (remaining < threshold) { + nbBits-- + threshold = threshold shr 1 + } + } + if (remaining != 1) { + throw ZstdException("FSE description: distribution does not sum to $tableSize (remaining=$remaining)") + } + return bits.finish() +} + +/** + * Forward, LSB-first bit writer — the inverse of [ForwardBitReader], and used + * only for FSE table descriptions. Bits fill the low end of the current byte + * first; [finish] flushes any partial final byte zero-padded, which is what + * makes the description byte-aligned at its end (where the reader resumes). + */ +internal class ForwardBitWriter { + private val bytes = ArrayList(16) + private var container: Int = 0 + private var bitsInContainer: Int = 0 + + fun write(value: Int, n: Int) { + if (n == 0) return + container = container or ((value and ((1 shl n) - 1)) shl bitsInContainer) + bitsInContainer += n + while (bitsInContainer >= 8) { + bytes.add((container and 0xFF).toByte()) + container = container ushr 8 + bitsInContainer -= 8 + } + } + + fun finish(): ByteArray { + if (bitsInContainer > 0) { + bytes.add((container and 0xFF).toByte()) + container = 0 + bitsInContainer = 0 + } + return ByteArray(bytes.size) { bytes[it] } + } +} + +/** Floor(log2(v)) for v >= 1; 0 for smaller values. */ +private fun highBit(v: Int): Int { + var n = 0 + var x = v + while (x > 1) { + x = x shr 1 + n++ + } + return n +} diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt index 62992a6..79f39c2 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt @@ -34,7 +34,8 @@ import org.meshtastic.kzstd.ZstdException * - **Sequences:** FSE-coded, independently per LL/OF/ML stream, each stream * picking the cheapest of: the dictionary's trained "Repeat" table (mode 3) * when the dict has one that covers this block's codes; RLE (mode 1) when - * every sequence uses the same code; or the PREDEFINED table (mode 0) — + * every sequence uses the same code; a fresh FSE table built from this + * block's own code counts (mode 2); or the PREDEFINED table (mode 0) — * always total over its symbol range, and exactly what [PureZstdDecoder] / * libzstd build from the spec's default distributions. Cost is compared in * bits and computed exactly ([FseEncTable.streamBitCost]), so the chosen @@ -607,10 +608,21 @@ internal object PureZstdEncoder { val llCodes = IntArray(nbSeq) { codes[it].llCode } val ofCodes = IntArray(nbSeq) { codes[it].ofCode } val mlCodes = IntArray(nbSeq) { codes[it].mlCode } - val ll = - chooseSequenceTable(dict.literalLengthFse, predefinedLiteralLengthEnc, LITERAL_LENGTH_MAX_SYMBOL, llCodes) - val of = chooseSequenceTable(dict.offsetFse, predefinedOffsetEnc, OFFSET_MAX_SYMBOL, ofCodes) - val ml = chooseSequenceTable(dict.matchLengthFse, predefinedMatchLengthEnc, MATCH_LENGTH_MAX_SYMBOL, mlCodes) + val ll = chooseSequenceTable( + dict.literalLengthFse, + predefinedLiteralLengthEnc, + LITERAL_LENGTH_MAX_SYMBOL, + LITERAL_LENGTH_MAX_LOG, + llCodes, + ) + val of = chooseSequenceTable(dict.offsetFse, predefinedOffsetEnc, OFFSET_MAX_SYMBOL, OFFSET_MAX_LOG, ofCodes) + val ml = chooseSequenceTable( + dict.matchLengthFse, + predefinedMatchLengthEnc, + MATCH_LENGTH_MAX_SYMBOL, + MATCH_LENGTH_MAX_LOG, + mlCodes, + ) val llTable = ll.table val ofTable = of.table val mlTable = ml.table @@ -776,6 +788,11 @@ internal object PureZstdEncoder { * description bytes, and total over its declared symbol range. * - **RLE (1)** -- when every sequence uses the SAME code: one description * byte and not a single bit in the bitstream. + * - **FSE_Compressed (2)** -- a table built from THIS block's own code + * counts, plus the description a decoder needs to rebuild it. It fits + * the block's actual distribution rather than the spec's guess at one, + * so it wins whenever there are enough sequences to repay its + * description. * * Candidates are considered in that order and a later one must be strictly * cheaper to win, so ties keep the mode that costs no description bytes and @@ -785,6 +802,7 @@ internal object PureZstdEncoder { dictTable: FseTable?, predefined: FseEncTable, maxSymbol: Int, + maxLog: Int, codes: IntArray, ): SeqTableChoice { var best: SeqTableChoice? = null @@ -814,6 +832,9 @@ internal object PureZstdEncoder { ) } + val fresh = buildFreshFseTable(codes, maxSymbol, maxLog) + if (fresh != null) consider(SeqTableChoice(fresh.encoder, 2, fresh.description)) + // Predefined covers every code these tables can legally carry, so the // fallback is unreachable in practice; it keeps the function total. return best ?: SeqTableChoice(predefined, 0, ByteArray(0)) diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt new file mode 100644 index 0000000..6e5916b --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Adoption guard for FSE_Compressed sequence tables (Symbol_Compression_Mode 2) + * built from a block's OWN code counts — the dictionary-free path. + * + * A block with enough sequences must actually choose them: the cost model + * comparing exact bits is only useful if a fresh table ever wins, and a + * regression to always-Predefined would leave every round-trip test green. + */ +class FreshSequenceTablesTest { + + private val max = TestVectors.MAX_DECOMPRESSED_SIZE + + @Test + fun sequenceHeavyBlockUsesFreshTablesForEveryStream() { + val frame = Zstd.compress(TestVectors.logRecords) + val (llMode, ofMode, mlMode) = FrameInspector.sequenceModes(frame)!! + assertEquals(2, llMode, "literal-length stream did not use a fresh FSE table") + assertEquals(2, ofMode, "offset stream did not use a fresh FSE table") + assertEquals(2, mlMode, "match-length stream did not use a fresh FSE table") + assertContentEquals(TestVectors.logRecords, Zstd.decompress(frame, max)) + } + + @Test + fun freshTablesShrinkOutput() { + // Baselines measured on the encoder as it stood before this work: + // predefined FSE tables and raw literals only. Refresh (never loosen + // without a reason) if a later, deliberate change moves them. + val size = Zstd.compress(TestVectors.logRecords).size + assertTrue(size < 2093, "log records: $size bytes, pre-entropy-coding baseline was 2093") + } + + /** + * A short block cannot repay a table description, so it must keep the + * Predefined tables — the cost model has to work in both directions. + */ + @Test + fun shortBlocksKeepPredefinedTables() { + val frame = Zstd.compress("hello hello hello world".encodeToByteArray()) + val (llMode, ofMode, mlMode) = FrameInspector.sequenceModes(frame)!! + assertTrue( + llMode != 2 && ofMode != 2 && mlMode != 2, + "a single-sequence block should not pay for an FSE table description", + ) + } + + @Test + fun freshTablesRoundTripAcrossVariedInputs() { + val inputs = TestVectors.corpus + listOf( + TestVectors.logRecords, + TestVectors.byteRuns, + TestVectors.skewedAlphabet, + TestVectors.structured.reduce { a, b -> a + b }, + ) + for (sample in inputs) { + val frame = Zstd.compress(sample) + assertContentEquals(sample, Zstd.decompress(frame, max), "size=${sample.size}") + } + } +} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FseTableDescriptionTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FseTableDescriptionTest.kt new file mode 100644 index 0000000..414d8c4 --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FseTableDescriptionTest.kt @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import org.meshtastic.kzstd.internal.FSE_MIN_TABLELOG +import org.meshtastic.kzstd.internal.ForwardByteReader +import org.meshtastic.kzstd.internal.FseState +import org.meshtastic.kzstd.internal.FseTable +import org.meshtastic.kzstd.internal.LITERAL_LENGTH_MAX_LOG +import org.meshtastic.kzstd.internal.LITERAL_LENGTH_MAX_SYMBOL +import org.meshtastic.kzstd.internal.MATCH_LENGTH_MAX_LOG +import org.meshtastic.kzstd.internal.MATCH_LENGTH_MAX_SYMBOL +import org.meshtastic.kzstd.internal.OFFSET_MAX_LOG +import org.meshtastic.kzstd.internal.OFFSET_MAX_SYMBOL +import org.meshtastic.kzstd.internal.ReverseBitReader +import org.meshtastic.kzstd.internal.ReverseBitWriter +import org.meshtastic.kzstd.internal.buildFreshFseTable +import org.meshtastic.kzstd.internal.normalizeFseCounts +import org.meshtastic.kzstd.internal.parseFseTable +import org.meshtastic.kzstd.internal.writeFseTableDescription +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The FSE table description writer (RFC 8878 §4.1.1) — the exact inverse of the + * decoder's [parseFseTable] — plus the count normalization that feeds it. + * + * These are bit-level round trips deliberately kept away from the frame format: + * a description that is off by one bit produces a table that decodes every + * sequence in the block to nonsense, and only checking writer against reader + * localises that. The reader here is the SAME routine that already parses + * libzstd's and trained dictionaries' tables, so agreeing with it is agreeing + * with the spec. + */ +class FseTableDescriptionTest { + + @Test + fun normalizationSumsToTableSizeAndKeepsEverySymbol() { + for (counts in countSets()) { + for (tableLog in FSE_MIN_TABLELOG..9) { + val maxSymbol = counts.size - 1 + val distinct = counts.count { it > 0 } + if (distinct > (1 shl tableLog)) continue + val norm = normalizeFseCounts(counts, maxSymbol, tableLog) + assertEquals(1 shl tableLog, norm.sum(), "normalized counts must fill the table exactly") + for (s in 0..maxSymbol) { + if (counts[s] > 0) { + assertTrue(norm[s] >= 1, "symbol $s occurs but was normalized to ${norm[s]}") + } else { + assertEquals(0, norm[s], "absent symbol $s was given probability") + } + } + } + } + } + + @Test + fun descriptionRoundTripsThroughTheDecodersParser() { + for (counts in countSets()) { + for (tableLog in FSE_MIN_TABLELOG..9) { + val maxSymbol = counts.size - 1 + if (counts.count { it > 0 } > (1 shl tableLog)) continue + val norm = normalizeFseCounts(counts, maxSymbol, tableLog) + val description = writeFseTableDescription(norm, maxSymbol, tableLog) + + val reader = ForwardByteReader(description, 0, description.size) + val parsed = parseFseTable(reader, maxLog = 9, maxSymbol = maxSymbol) + assertEquals( + description.size, + reader.pos, + "the parser consumed ${reader.pos} of ${description.size} description bytes", + ) + assertTablesEqual(FseTable.build(norm, maxSymbol, tableLog), parsed, "tableLog=$tableLog") + } + } + } + + /** + * End-to-end for one symbol stream, in exactly the shape the sequences + * section uses: build a table from the stream's own code counts, write its + * description, encode the stream backwards, then parse the description and + * decode the stream forwards. + */ + @Test + fun freshTablesEncodeStreamsTheDecoderReadsBack() { + for ((codes, maxSymbol, maxLog) in codeStreams()) { + val fresh = assertNotNull(buildFreshFseTable(codes, maxSymbol, maxLog), "no table for ${codes.size} codes") + + val bw = ReverseBitWriter() + var state = fresh.encoder.initialState(codes[codes.size - 1]) + for (i in codes.size - 2 downTo 0) state = fresh.encoder.encode(bw, state, codes[i]) + fresh.encoder.flushState(bw, state) + val stream = bw.finish() + + val reader = ForwardByteReader(fresh.description, 0, fresh.description.size) + val table = parseFseTable(reader, maxLog, maxSymbol) + val br = ReverseBitReader(stream, 0, stream.size) + val fseState = FseState(table) + fseState.init(br) + for (i in codes.indices) { + assertEquals(codes[i], fseState.symbol(), "code $i of ${codes.size}") + if (i < codes.size - 1) fseState.update(br) + } + } + } + + @Test + fun singleCodeStreamsHaveNoFreshTable() { + // One distinct code is the RLE case (mode 1), which is always smaller. + assertNull(buildFreshFseTable(IntArray(20) { 4 }, LITERAL_LENGTH_MAX_SYMBOL, LITERAL_LENGTH_MAX_LOG)) + assertNull(buildFreshFseTable(IntArray(0), LITERAL_LENGTH_MAX_SYMBOL, LITERAL_LENGTH_MAX_LOG)) + } + + /** Count distributions covering the shapes the description encoding branches on. */ + private fun countSets(): List = listOf( + // Two symbols, adjacent. + IntArray(36).also { + it[0] = 7 + it[1] = 3 + }, + // Two symbols at the extremes: one long zero run, longer than the + // 3-at-a-time run encoding's group size. + IntArray(36).also { + it[0] = 1 + it[35] = 1 + }, + // Zero runs of exactly 3 and 4 (the run encoding's boundary). + IntArray(36).also { + it[0] = 5 + it[4] = 5 + it[9] = 2 + }, + // Dense and heavily skewed: one symbol takes nearly the whole table. + IntArray(36).also { + it[0] = 1000 + for (s in 1 until 36) it[s] = 1 + }, + // Dense and flat. + IntArray(36).also { for (s in 0 until 36) it[s] = 4 }, + // Match-length shaped: 53 symbols, sparse at the top. + IntArray(53).also { + it[0] = 40 + it[1] = 20 + it[16] = 3 + it[52] = 1 + }, + // Offset shaped. + IntArray(32).also { + it[0] = 30 + it[3] = 12 + it[7] = 4 + it[31] = 1 + }, + ) + + private fun codeStreams(): List> = listOf( + Triple( + IntArray(200) { i -> (i * 7) % 20 }, + LITERAL_LENGTH_MAX_SYMBOL, + LITERAL_LENGTH_MAX_LOG, + ), + Triple( + IntArray(500) { i -> + if (i % 5 == 0) { + 1 + } else if (i % 3 == 0) { + 9 + } else { + 0 + } + }, + MATCH_LENGTH_MAX_SYMBOL, + MATCH_LENGTH_MAX_LOG, + ), + Triple( + IntArray(64) { i -> if (i % 8 == 0) 31 else i % 4 }, + OFFSET_MAX_SYMBOL, + OFFSET_MAX_LOG, + ), + // Only two distinct codes, wildly unbalanced. + Triple( + IntArray(300) { i -> if (i == 150) 35 else 0 }, + LITERAL_LENGTH_MAX_SYMBOL, + LITERAL_LENGTH_MAX_LOG, + ), + ) + + private fun assertTablesEqual(expected: FseTable, actual: FseTable, message: String) { + assertEquals(expected.tableLog, actual.tableLog, "$message: tableLog") + assertContentEquals(expected.symbol, actual.symbol, "$message: symbols") + assertContentEquals(expected.nbBits, actual.nbBits, "$message: nbBits") + assertContentEquals(expected.newState, actual.newState, "$message: newState") + } +} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt index bfed78a..03650a3 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt @@ -214,6 +214,38 @@ internal object TestVectors { "A" + "reports stream steadily" ).encodeToByteArray() + /** + * ~7.8 KB of synthetic JSON telemetry records: realistic, genuinely + * compressible payload with enough sequences (hundreds) for a block's own + * FSE tables to repay their descriptions, and literals skewed enough for a + * fresh Huffman table. Generated from a fixed LCG, so it is byte-identical + * on every target. + */ + val logRecords: ByteArray = buildRecords(60) + + private fun buildRecords(count: Int): ByteArray { + var seed = 0x2468ACE + fun next(): Int { + seed = (seed * 1103515245 + 12345) and 0x7FFFFFFF + return seed ushr 8 + } + val states = listOf("ok", "warn", "offline", "degraded", "recovering", "unknown") + val words = listOf( + "region", "latency", "throughput", "stable", "nominal", + "monitored", "link", "quick", "fox", "dog", + ) + val sb = StringBuilder() + repeat(count) { + val msg = (0 until 3 + next() % 6).joinToString(" ") { words[next() % words.size] } + sb.append("{\"type\":\"telemetry\",\"seq\":").append(next() % 1000000) + .append(",\"node\":\"node-").append(next() % 64) + .append("\",\"state\":\"").append(states[next() % states.size]) + .append("\",\"lat\":").append(next() % 90).append('.').append(next() % 100000) + .append(",\"msg\":\"").append(msg).append("\"}") + } + return sb.toString().encodeToByteArray() + } + /** * Skewed draws over ~90 byte values, with essentially no repeated * substrings: nearly every byte stays a literal, so the block's literals diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt index 622b757..3986904 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt @@ -73,6 +73,41 @@ class EntropyCodingInteropTest { TestVectors.wideAlphabet, ) + /** + * FSE tables built from the block's own code counts, described on the wire. + * The description encoding (a shrinking-width bit stream with run-length + * encoded gaps) has no margin for error, and libzstd rebuilding the same + * table from it — for all three streams at once — is the proof that it is + * right rather than merely symmetric with kzstd's own parser. + */ + @Test + fun freshSequenceTablesDecodeUnderLibzstd() { + val data = TestVectors.logRecords + val frame = Zstd.compress(data) + assertEquals( + Triple(2, 2, 2), + FrameInspector.sequenceModes(frame), + "expected fresh FSE tables for LL/OF/ML", + ) + assertContentEquals(data, LibZstd.decompress(frame, max)) + } + + /** Every dictionary-free encoding choice, over the whole round-trip corpus. */ + @Test + fun wholeCorpusDecodesUnderLibzstd() { + val samples = TestVectors.corpus + listOf( + TestVectors.logRecords, + TestVectors.byteRuns, + TestVectors.constantBytes, + TestVectors.skewedAlphabet, + TestVectors.wideAlphabet, + ) + for (sample in samples) { + val frame = Zstd.compress(sample) + assertContentEquals(sample, LibZstd.decompress(frame, max), "size=${sample.size}") + } + } + @Test fun rleLiteralsDecodeUnderLibzstd() { val data = TestVectors.rleLiteralsSample From 9d9ec2393d2397717fd5e922c5a133474f14875d Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 14:09:12 -0500 Subject: [PATCH 04/11] docs: record the dictionary-free entropy coding in README and CHANGELOG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Describe what a plain Zstd.compress(data) call now does — Huffman literals and FSE sequence tables built from the block's own data, plus the RLE forms — with the measured before/after sizes, and state the two encoder-side limits that remain: single-stream literals (so at most 1023 bytes of literals per block) and direct 4-bit weight descriptions (so a literal byte above 128 falls back to Raw). Both are encoder-only; the decoder reads the 4-stream layout and FSE-compressed weights that libzstd emits. Also pin the new ratio ratchets to the sizes measured on the encoder as it stood before this work, rather than the round numbers they were drafted with, and scope the level-mapping note in CHANGELOG to the mapping itself now that entropy coding legitimately changes level-19 output. Signed-off-by: James Rich --- CHANGELOG.md | 40 ++++++++++++++++--- README.md | 23 +++++++---- .../kzstd/FreshSequenceTablesTest.kt | 2 +- .../org/meshtastic/kzstd/RleEncodingTest.kt | 14 ++++--- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34566a7..9dbdef3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] Encoder-side parity work closing several gaps against the libzstd/RFC 8878 -spec — real ratio improvements for dictionary-compressed frames, no wire -format or public-API changes (except where noted below, for the -Content_Checksum fix). +spec — real ratio improvements for both dictionary-compressed and +dictionary-free frames, plus dictionary-ID and content-checksum validation. +No wire format or public-API changes, except where noted below. ### Fixed @@ -19,6 +19,9 @@ Content_Checksum fix). content, for ANY conformant frame — not just kzstd's own — so a real `libzstd`-produced frame (checksums are on by default in the `zstd` CLI) is no longer accepted with silently-corrupted content. +- The decoder now validates a frame's declared Dictionary_ID against the + supplied dictionary's own embedded ID (when both are present), throwing a + clear `ZstdException` on mismatch instead of a generic corruption error. ### Added @@ -38,11 +41,30 @@ Content_Checksum fix). needs and doing so is smaller than the previous fallback (predefined FSE tables, raw literals) — a real, measurable size reduction for dictionary-compressed frames, not just a wire-format curiosity (#50, #51). +- Without a dictionary — the plain `Zstd.compress(data)` call — the encoder now + entropy-codes each block from the block's OWN data, where before it could + only emit raw literals and the spec's predefined FSE distributions: Huffman + literals built from the block's byte histogram (`Literals_Block_Type` 2), and + FSE tables for the literal-length / offset / match-length streams normalized + from the block's own code counts (`Symbol_Compression_Mode` 2). Measured: + ~7.8 KB of synthetic JSON telemetry records 2007 → 1521 bytes, 887 bytes of + concatenated structured records 487 → 425, a 208-byte prose sample 213 → 190. +- The encoder now also emits the RLE forms the decoder has always read: + `RLE_Block` for a constant input (a 1500-byte run of one byte, 17 → 10 + bytes), RLE literals when every literal is the same byte, and RLE sequence + tables when a stream's every code is the same (a sample of 26 byte-runs, + 85 → 42 bytes). +- Every per-block encoding choice — the literals section and each of the three + sequence streams independently — is now made by measuring every valid + alternative and taking the smallest, so a form is used only when it actually + wins. Ties keep the previous behaviour, and dictionary-compressed frames come + out the same size as before. - `level` (1–22) now governs match-finding search depth: higher levels search more candidate matches per position, which can shrink output at the cost of - more work. Level 19 (`Zstd.DEFAULT_LEVEL`) is byte-identical to every - earlier release; the encoder still uses one fixed strategy at every level, - not zstd's other per-level parameters (#52). + more work. Level 19 (`Zstd.DEFAULT_LEVEL`) maps to exactly the search depth + the encoder always used, so the mapping itself changes no output; the encoder + still uses one fixed strategy at every level, not zstd's other per-level + parameters (#52). ### Fixed @@ -58,6 +80,12 @@ Content_Checksum fix). ### Notes +- Huffman-coded literals stay single-stream, so they apply to at most 1023 + bytes of literals per block, and their tree description uses the direct + 4-bit weight form, so a block containing a literal byte above 128 falls back + to raw literals. FSE-compressed weight descriptions and the 4-stream literals + layout would lift those limits and are not implemented; neither affects + decoding, which reads both. - A dictionary-compressed frame's correctness now depends on the dictionary's entropy tables matching what the decoder is seeded with, not just its content — decoding with the wrong dictionary was already silently wrong diff --git a/README.md b/README.md index 6b67ddc..8e45093 100644 --- a/README.md +++ b/README.md @@ -65,22 +65,31 @@ val back = Zstd.decompress(small, dict, maxSize = 64 * 1024) single fixed greedy/lazy strategy at every level — it does not implement zstd's other per-level parameters (window log, target length, etc.) — but a higher level does search more candidate matches per position, which can shrink output - at the cost of more work. Level 19 (`Zstd.DEFAULT_LEVEL`) is unchanged from - every earlier release. Frames remain fully libzstd-compatible at every level. + at the cost of more work. Frames remain fully libzstd-compatible at every level. - **Single block per frame (≤ 128 KiB input).** `Zstd.compress` emits one zstd block, so its input is bounded by zstd's 128 KiB `Block_Maximum_Size`; a larger input throws `ZstdException`. (`Zstd.decompress` reads multi-block frames from any encoder.) Multi-block encoding to lift the cap is planned. +- **Huffman-coded literals are single-stream and directly described.** The encoder + builds a Huffman table from a block's own literals, but writes only the + single-stream layout (so it applies to at most 1023 bytes of literals per block) + and only the direct 4-bit weight tree description (so a block containing a literal + byte above 128 falls back to raw literals). Both limits are encoder-side only — + `Zstd.decompress` reads the 4-stream layout and FSE-compressed weight descriptions + that libzstd emits. ## Interoperability kzstd reads frames produced by libzstd (including dictionary-compressed frames that use the dictionary's Huffman/FSE entropy tables), and libzstd reads frames -produced by kzstd — including, now, dictionary-compressed frames kzstd itself -produces using the dictionary's trained entropy tables and repeat-offset codes, -when doing so is smaller than the fallback. The test suite cross-checks both -directions against [zstd-jni](https://github.com/luben/zstd-jni) (a -JVM-test-only oracle, never a runtime dependency). +produced by kzstd — including frames kzstd itself entropy-codes: dictionary +frames using the dictionary's trained tables and repeat-offset codes, and +dictionary-free frames using Huffman/FSE tables built from the block's own data +(or the RLE forms, when a block, its literals or a symbol stream is constant). +Each of those forms is picked only when it is the smallest valid encoding. The +test suite cross-checks both directions against +[zstd-jni](https://github.com/luben/zstd-jni) (a JVM-test-only oracle, never a +runtime dependency). ## Building & testing diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt index 6e5916b..b97af1c 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt @@ -34,7 +34,7 @@ class FreshSequenceTablesTest { // predefined FSE tables and raw literals only. Refresh (never loosen // without a reason) if a later, deliberate change moves them. val size = Zstd.compress(TestVectors.logRecords).size - assertTrue(size < 2093, "log records: $size bytes, pre-entropy-coding baseline was 2093") + assertTrue(size < 2007, "log records: $size bytes, pre-entropy-coding baseline was 2007") } /** diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt index 3f90343..831edbe 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/RleEncodingTest.kt @@ -57,13 +57,17 @@ class RleEncodingTest { assertContentEquals(TestVectors.rleLiteralsSample, Zstd.decompress(frame, dict, max)) } + /** + * Ratio ratchet against the sizes the encoder produced before it could emit + * any RLE form (a Compressed_Block with predefined FSE tables in both + * cases). Refresh (never loosen without a reason) if a later, deliberate + * change moves them. + */ @Test fun rleFormsShrinkOutput() { - assertTrue( - Zstd.compress(TestVectors.constantBytes).size < 12, - "RLE_Block should compress a constant input to a handful of bytes", - ) + val constant = Zstd.compress(TestVectors.constantBytes).size + assertTrue(constant < 17, "constant input: $constant bytes, pre-RLE baseline was 17") val runs = Zstd.compress(TestVectors.byteRuns).size - assertTrue(runs < 60, "byte-run sample compressed to $runs bytes, expected well under 60") + assertTrue(runs < 85, "byte-run sample: $runs bytes, pre-RLE baseline was 85") } } From 9f105f47c1f24bd28d9e951db172c1f93bdc3ffd Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 14:23:30 -0500 Subject: [PATCH 05/11] test: drive the entropy coders at their limits against real libzstd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every cross-oracle case so far sat well inside the encoder's bounds, which left the widest tables — the ones with the most room for a writer/reader disagreement — checked only against kzstd's own parser. Two inputs close that: - ~51 KB of synthetic telemetry produces thousands of sequences, which pushes the literal-length Accuracy_Log to the format's ceiling of 9 (the longest FSE descriptions the encoder can write, and the most field-width shrink steps for a reader to follow), spills Number_of_Sequences into its 2-byte form, and leaves far more than 1023 literals so the literals section falls back to Raw while all three sequence streams still carry fresh tables — a combination no previous test produced. - Literals with Fibonacci counts, the classic worst case for Huffman depth: the unconstrained tree is 13 deep against an 11-bit limit, so the table libzstd rebuilds is one the length-limiting repair reshaped. The unit test for that repair now also asserts the limit actually BINDS, so the case cannot quietly stop testing it. Both also round-trip through kzstd's own decoder on every target. Signed-off-by: James Rich --- .../org/meshtastic/kzstd/FrameInspector.kt | 44 ++++++++++++++++--- .../kzstd/FreshSequenceTablesTest.kt | 5 ++- .../kzstd/HuffmanConstructionTest.kt | 5 +++ .../org/meshtastic/kzstd/TestVectors.kt | 43 ++++++++++++++++++ .../kzstd/EntropyCodingInteropTest.kt | 34 +++++++++++++- 5 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt index 3a426ab..bea2fb6 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt @@ -46,20 +46,54 @@ internal object FrameInspector { p += 3 p = skipLiteralsSection(frame, p) + if (sequenceCountAt(frame, p) == 0) return null + p += sequenceCountFieldLen(frame, p) + val modes = frame[p].toInt() and 0xFF + return Triple((modes ushr 6) and 0x3, (modes ushr 4) and 0x3, (modes ushr 2) and 0x3) + } + + /** Number_of_Sequences of the first block (0 when it carries none). */ + fun sequenceCount(frame: ByteArray): Int { + var p = frameHeaderEnd(frame) + 3 + p = skipLiteralsSection(frame, p) + return sequenceCountAt(frame, p) + } + + /** + * Accuracy_Log of the LITERAL-LENGTH stream's FSE table description, or + * null unless that stream is in FSE_Compressed mode. The description's + * first byte holds `Accuracy_Log - 5` in its low 4 bits. + * + * Only the first stream's log is readable without decoding a whole + * description (they are variable-length), and the literal-length stream + * comes first — which is enough, since it is also the stream whose + * Accuracy_Log reaches the format's ceiling first. + */ + fun literalLengthAccuracyLog(frame: ByteArray): Int? { + var p = frameHeaderEnd(frame) + 3 + p = skipLiteralsSection(frame, p) + p += sequenceCountFieldLen(frame, p) + val modes = frame[p].toInt() and 0xFF + if ((modes ushr 6) and 0x3 != 2) return null + return (frame[p + 1].toInt() and 0xF) + 5 + } + + private fun sequenceCountAt(frame: ByteArray, p: Int): Int { val nb0 = frame[p].toInt() and 0xFF - val nbSeq = when { + return when { nb0 < 128 -> nb0 nb0 < 255 -> ((nb0 - 128) shl 8) + (frame[p + 1].toInt() and 0xFF) else -> (frame[p + 1].toInt() and 0xFF) + ((frame[p + 2].toInt() and 0xFF) shl 8) + 0x7F00 } - if (nbSeq == 0) return null - p += when { + } + + private fun sequenceCountFieldLen(frame: ByteArray, p: Int): Int { + val nb0 = frame[p].toInt() and 0xFF + return when { nb0 < 128 -> 1 nb0 < 255 -> 2 else -> 3 } - val modes = frame[p].toInt() and 0xFF - return Triple((modes ushr 6) and 0x3, (modes ushr 4) and 0x3, (modes ushr 2) and 0x3) } /** Byte offset of the first Block_Header. */ diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt index b97af1c..bd42285 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FreshSequenceTablesTest.kt @@ -55,13 +55,16 @@ class FreshSequenceTablesTest { fun freshTablesRoundTripAcrossVariedInputs() { val inputs = TestVectors.corpus + listOf( TestVectors.logRecords, + TestVectors.largeLogRecords, + TestVectors.deepSkewLiterals, TestVectors.byteRuns, TestVectors.skewedAlphabet, TestVectors.structured.reduce { a, b -> a + b }, ) for (sample in inputs) { val frame = Zstd.compress(sample) - assertContentEquals(sample, Zstd.decompress(frame, max), "size=${sample.size}") + val cap = maxOf(max, sample.size + 1024) + assertContentEquals(sample, Zstd.decompress(frame, cap), "size=${sample.size}") } } } diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt index 3a8f549..0d0e19e 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/HuffmanConstructionTest.kt @@ -96,6 +96,11 @@ class HuffmanConstructionTest { "symbol $s has out-of-range code length ${built.encoder.bitLength(s)}", ) } + assertEquals( + MAX_LITERAL_CODE_BITS, + (0 until 40).maxOf { built.encoder.bitLength(it) }, + "the limit should BIND here — if the longest code is shorter, this input stopped testing the repair", + ) assertDescriptionRoundTrips(hist, built.description, built.encoder) } diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt index 03650a3..32d62ed 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt @@ -223,6 +223,49 @@ internal object TestVectors { */ val logRecords: ByteArray = buildRecords(60) + /** + * ~51 KB of the same synthetic telemetry — enough sequences (a few + * thousand) to push the literal-length and match-length Accuracy_Log to the + * format's ceiling, and far more than 1023 literals, so this is the + * "raw literals + a fresh FSE table on every stream" combination, with the + * longest table descriptions the encoder can write. Sized to stay under the + * 128 KiB single-block limit. + */ + val largeLogRecords: ByteArray = buildRecords(400) + + /** + * Literals with FIBONACCI counts — the classic worst case for Huffman + * depth: an unconstrained tree over them is a chain far deeper than the + * 11-bit limit the encoder allows, so this drives the length-limiting + * repair end to end (and through libzstd) rather than only in unit tests. + */ + val deepSkewLiterals: ByteArray = buildDeepSkew() + + private fun buildDeepSkew(): ByteArray { + val counts = ArrayList() + var a = 1 + var b = 1 + while (counts.sum() + a <= 1000) { + counts.add(a) + val next = a + b + a = b + b = next + } + val pool = ArrayList() + for (s in counts.indices) repeat(counts[s]) { pool.add((s + 1).toByte()) } + // Deterministic shuffle, so few long repeats survive for the matcher and + // most bytes stay literals. + var seed = 0x5EED5EED + for (i in pool.indices.reversed()) { + seed = (seed * 1103515245 + 12345) and 0x7FFFFFFF + val j = (seed ushr 8) % (i + 1) + val t = pool[i] + pool[i] = pool[j] + pool[j] = t + } + return ByteArray(pool.size) { pool[it] } + } + private fun buildRecords(count: Int): ByteArray { var seed = 0x2468ACE fun next(): Int { diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt index 3986904..5802a63 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/EntropyCodingInteropTest.kt @@ -67,10 +67,13 @@ class EntropyCodingInteropTest { "targets even when the region degrades to offline for a while. " ).encodeToByteArray(), TestVectors.structured.reduce { a, b -> a + b }, - // Wide alphabets: long weight descriptions, and (for the skewed one) - // code lengths spread far enough to reach the length limit. + // Wide alphabets: long weight descriptions. TestVectors.skewedAlphabet, TestVectors.wideAlphabet, + // Fibonacci counts: the tree is deeper than the format allows, so the + // table libzstd rebuilds here is one the length-limiting repair + // reshaped. + TestVectors.deepSkewLiterals, ) /** @@ -92,6 +95,33 @@ class EntropyCodingInteropTest { assertContentEquals(data, LibZstd.decompress(frame, max)) } + /** + * The extremes of the FSE description writer, in one frame: ~51 KB of + * telemetry produces thousands of sequences, which pushes the + * literal-length Accuracy_Log to the format's ceiling of 9 (the longest + * descriptions the encoder can write, and the most field-width shrink steps + * for a reader to follow), spills Number_of_Sequences into its 2-byte form, + * and leaves far more than 1023 literals so the literals section falls back + * to Raw while all three sequence streams still carry fresh tables. + * + * Every smaller oracle case sits well below those limits, so without this + * one the writer's widest tables would only ever be checked against kzstd's + * own parser. + */ + @Test + fun maxAccuracyLogTablesDecodeUnderLibzstd() { + val data = TestVectors.largeLogRecords + val frame = Zstd.compress(data) + assertEquals(Triple(2, 2, 2), FrameInspector.sequenceModes(frame), "expected fresh FSE tables") + assertEquals(0, FrameInspector.literalsType(frame), "expected Raw literals past the single-stream cap") + assertEquals( + 9, + FrameInspector.literalLengthAccuracyLog(frame), + "expected the maximum literal-length Accuracy_Log", + ) + assertContentEquals(data, LibZstd.decompress(frame, data.size + 1024)) + } + /** Every dictionary-free encoding choice, over the whole round-trip corpus. */ @Test fun wholeCorpusDecodesUnderLibzstd() { From ee5e75b27ac396c899abc0c65cb945f3c8745373 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 14:46:20 -0500 Subject: [PATCH 06/11] feat(internal): split inputs above 128 KiB into multiple blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A block's regenerated content cannot exceed Block_Maximum_Size (RFC 8878 3.1.1.2) — 128 KiB — and the encoder emitted exactly one block per frame, so Zstd.compress rejected anything larger outright. It now cuts the input into 128 KiB chunks, one block each, with Last_Block set on the final block only. The decoder has always read multi-block frames, so nothing changes on that side. Two pieces of per-frame state make this more than a loop: - The three repeat offsets belong to the FRAME, not the block: the decoder keeps rotating the same slots across a block boundary. They are now carried from block to block, and a block is encoded against a COPY that is adopted only when the Compressed form actually wins — a Raw or RLE block carries no sequences, so it must leave the decoder's slots exactly where they were. Restarting them per block would make every repeat code after the first block resolve to some other distance, which round-tripping against kzstd's own decoder (making the same mistake twice) would not reveal. - "Repeat" entropy tables (Symbol_Compression_Mode 3) and Treeless literals (Literals_Block_Type 3) mean the PREVIOUS block's table, and only mean the dictionary's for the frame's first block. Both are therefore offered to the first block alone for now; carrying tables forward across blocks is a separate change. Dictionary matching keeps working in every block, which is not automatic: the dictionary does not move with the chunk, so from the second block on a dictionary match is `priorBytes` further back and may no longer run past the end of the dictionary content — what follows there in the real frame is the frame's first block, not the current chunk. Matches within a chunk are unaffected; this is deliberately the simple form, where a block never matches into an earlier block's output. A windowed matcher that does is a worthwhile follow-up, not this change. Blocks up to 128 KiB take exactly the path they always did (one chunk, priorBytes 0, dictionary state seeded as before), so every pinned frame in ByteIdenticalRegressionTest and every dictionary size is unchanged. The unreachable "no table could encode this stream" fallback now throws instead of silently returning the predefined table: multi-block dictionary matching is what could in principle produce an offset code above the predefined table's 28 — from a block starting more than 512 MB into a frame — and emitting a stream a decoder cannot read is worse than failing. MultiBlockInteropTest hands the multi-block frames to real libzstd, including a multi-megabyte input, an all-Raw-block incompressible one, and a dictionary frame whose second block compresses to a third of its size only because it still reaches the dictionary. MultiBlockFrameTest walks the block chain and asserts it ends exactly at the end of the frame, which is what catches a mis-sized header that still happens to decode. It replaces EncoderBlockLimitTest, whose subject was the rejection that no longer happens. 3 MB of synthetic JSON telemetry now compresses at all: 3,145,728 -> 557,565 bytes in 24 blocks (libzstd -3 gives 579,374, libzstd -19 362,967). Signed-off-by: James Rich --- .../meshtastic/kzstd/internal/ZstdEncoder.kt | 227 ++++++++++++------ .../meshtastic/kzstd/EncoderBlockLimitTest.kt | 32 --- .../org/meshtastic/kzstd/FrameInspector.kt | 28 +++ .../meshtastic/kzstd/MultiBlockFrameTest.kt | 99 ++++++++ .../meshtastic/kzstd/MultiBlockInteropTest.kt | 129 ++++++++++ 5 files changed, 416 insertions(+), 99 deletions(-) delete mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/EncoderBlockLimitTest.kt create mode 100644 src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt create mode 100644 src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt index 79f39c2..dbac950 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt @@ -18,12 +18,13 @@ import org.meshtastic.kzstd.ZstdException * - **Frame header:** single-segment, dictID OFF, contentSize OFF, checksum * OFF — byte-for-byte the descriptor the SDK's decoder (and libzstd) accept, * with a window large enough to cover `dict.content + input`. - * - **One block** with Last_Block set, of whichever type is smallest: a - * Compressed_Block, an RLE_Block when every input byte is identical, or a - * Raw_Block when compression does not pay (a valid fallback; the SDK's own - * `0xFF` skip-compress also covers incompressible payloads above this - * layer). - * - **Matching:** a 4-byte hash-chain matcher over `[dict.content ++ input]`, so + * - **Blocks:** the input is cut into chunks of at most `Block_Maximum_Size` + * (128 KiB), one block each, with Last_Block set on the final one. Each + * block independently takes whichever type is smallest: a Compressed_Block, + * an RLE_Block when every byte of the chunk is identical, or a Raw_Block + * when compression does not pay (a valid fallback; the SDK's own `0xFF` + * skip-compress also covers incompressible payloads above this layer). + * - **Matching:** a 4-byte hash-chain matcher over `[dict.content ++ chunk]`, so * matches can back-reference the dictionary content. The dict index is built * once per `ZstdDictionary` instance, so the dict content is hashed once and * reused across calls. Lazy (1-step lookahead) matching for a better ratio. @@ -56,10 +57,9 @@ internal object PureZstdEncoder { private const val HASH_SIZE = 1 shl HASH_LOG // zstd's Block_Maximum_Size (RFC 8878 §3.1.1.2): a block's regenerated content - // cannot exceed min(windowSize, 128 KiB). This encoder emits ONE block per frame, - // so the input is bounded by this. A larger input would silently produce a frame - // that neither libzstd nor this decoder accepts, so it is rejected up front - // (multi-block encoding to lift the cap is a planned addition). + // cannot exceed min(windowSize, 128 KiB), so this is the chunk size the input + // is cut into. The window this encoder declares always covers the whole input, + // so 128 KiB is the binding half of that minimum. private const val MAX_BLOCK_SIZE = 1 shl 17 // 128 KiB /** @@ -86,61 +86,100 @@ internal object PureZstdEncoder { level: Int = 19, checksum: Boolean = false, ): ByteArray { - if (data.size > MAX_BLOCK_SIZE) { - throw ZstdException( - "input ${data.size} exceeds the single-block limit of $MAX_BLOCK_SIZE bytes; " + - "kzstd emits one block per frame — multi-block encoding is not yet supported", - ) - } val depth = searchDepthFor(level) - // Build the literal+sequence program by matching `data` against the - // combined [dictContent ++ data] history. - val program = buildSequences(data, dict, index, depth) - - // Encode the single block. If it does not beat raw, fall back to a Raw - // block (still a valid frame). - val compressedBlock = encodeCompressedBlock(program, data, dict) val out = ArrayList(data.size + 20) FRAME_MAGIC.forEach { out.add(it) } out.add(frameHeaderDescriptor(checksum)) + // The window must span the whole frame's back-reference history, not just + // one block's: a later block's dictionary match reaches back past every + // block before it. out.add(windowDescriptor(dict.content.size + data.size)) - // All three block types carry the same 3-byte header, so the smallest - // block is simply the one with the smallest payload: 1 byte for RLE - // (only when every input byte is identical), `data.size` for Raw, or - // the compressed body. Ties go to the simpler form. - val constantByte = constantByteOrNull(data) + // The three repeat-offset slots are per-FRAME state that the decoder + // rotates as it executes sequences, so they must be carried from block to + // block here too -- restarting them per block would make every repeat code + // after the first block mean a different distance. Held in a local: this + // singleton keeps no mutable state. + val repeatOffsets = dict.repeatOffsets.copyOf() + + // Cut the input into Block_Maximum_Size chunks, one block each. `do/while` + // rather than `while`, so an empty input still emits the one (empty, + // Last_Block) block a frame must have. + var start = 0 + do { + val end = minOf(start + MAX_BLOCK_SIZE, data.size) + encodeBlock(out, data, start, end, lastBlock = end == data.size, dict, index, depth, repeatOffsets) + start = end + } while (start < data.size) + + // RFC 8878 §3.1.1: the checksum is a trailing 4-byte field covering the + // FULL frame content, so it goes after every block -- not per block, and + // not until the whole multi-block loop above has finished. + if (checksum) { + val h = Xxh64.hash(data) and 0xFFFFFFFFL + out.add((h and 0xFF).toByte()) + out.add(((h ushr 8) and 0xFF).toByte()) + out.add(((h ushr 16) and 0xFF).toByte()) + out.add(((h ushr 24) and 0xFF).toByte()) + } + + return ByteArray(out.size) { out[it] } + } + + /** + * Encode `data[start until end]` as one block appended to [out], choosing the + * smallest of the three block types. + * + * All three carry the same 3-byte header, so the smallest block is simply the + * one with the smallest payload: 1 byte for RLE (only when every byte of the + * chunk is identical), the chunk length for Raw, or the compressed body. Ties + * go to the simpler form. + * + * [repeatOffsets] is the frame's running repeat-offset state. Sequences move + * it, so the block is encoded against a COPY that is adopted only if the + * Compressed form actually wins -- a Raw or RLE block carries no sequences and + * so leaves the decoder's slots exactly where they were. + */ + @Suppress("LongParameterList") + private fun encodeBlock( + out: ArrayList, + data: ByteArray, + start: Int, + end: Int, + lastBlock: Boolean, + dict: ParsedDictionary, + index: MatchIndex, + depth: SearchDepth, + repeatOffsets: IntArray, + ) { + val chunk = data.copyOfRange(start, end) + val program = buildSequences(chunk, dict, index, depth, priorBytes = start) + val blockOffsets = repeatOffsets.copyOf() + val compressedBlock = encodeCompressedBlock(program, chunk, dict, blockOffsets, firstBlock = start == 0) + + val constantByte = constantByteOrNull(chunk) when { constantByte != null && - data.size > 1 && + chunk.size > 1 && (compressedBlock == null || compressedBlock.size > 1) -> { // RLE_Block: the single repeated byte IS the block body. - writeBlockHeader(out, lastBlock = true, blockType = 1, blockSize = data.size) + writeBlockHeader(out, lastBlock, blockType = 1, blockSize = chunk.size) out.add(constantByte) } - compressedBlock != null && compressedBlock.size < data.size -> { - // Block_Header (3 bytes LE): last=1, type=2 (Compressed), size=blockSize - writeBlockHeader(out, lastBlock = true, blockType = 2, blockSize = compressedBlock.size) + compressedBlock != null && compressedBlock.size < chunk.size -> { + // Block_Header (3 bytes LE): last, type=2 (Compressed), size=blockSize + writeBlockHeader(out, lastBlock, blockType = 2, blockSize = compressedBlock.size) compressedBlock.forEach { out.add(it) } + blockOffsets.copyInto(repeatOffsets) } else -> { // Raw_Block fallback: the literal bytes are the block. - writeBlockHeader(out, lastBlock = true, blockType = 0, blockSize = data.size) - data.forEach { out.add(it) } + writeBlockHeader(out, lastBlock, blockType = 0, blockSize = chunk.size) + chunk.forEach { out.add(it) } } } - - if (checksum) { - val h = Xxh64.hash(data) and 0xFFFFFFFFL - out.add((h and 0xFF).toByte()) - out.add(((h ushr 8) and 0xFF).toByte()) - out.add(((h ushr 16) and 0xFF).toByte()) - out.add(((h ushr 24) and 0xFF).toByte()) - } - - return ByteArray(out.size) { out[it] } } /** The byte every element of [bytes] equals, or null (including when empty). */ @@ -224,16 +263,26 @@ internal object PureZstdEncoder { } /** - * Greedy/lazy LZ over `data`, referencing the dictionary content as history. - * Positions are expressed against the virtual `[dictContent ++ data]` array: - * a match at distance `d` from input position `i` copies bytes that may lie - * in the dict content (when `d > i`) or earlier in `data`. + * Greedy/lazy LZ over one block's `data` chunk, referencing the dictionary + * content as history. Positions are expressed against the virtual + * `[dictContent ++ data]` array: a match at distance `d` from input position + * `i` copies bytes that may lie in the dict content (when `d > i`) or earlier + * in `data`. + * + * [priorBytes] is how many bytes of the frame precede this chunk. Matches + * within the chunk are unaffected by it -- this is deliberately the simple + * form, where a block never matches into an earlier block's output -- but the + * dictionary does not move with the chunk: in the real frame it sits + * [priorBytes] further back, so a dictionary match's distance grows by that + * much, and it may no longer run past the dictionary's end (what follows + * there is the frame's first block, not this chunk). */ private fun buildSequences( data: ByteArray, dict: ParsedDictionary, index: MatchIndex, depth: SearchDepth, + priorBytes: Int, ): Program { val dictContent = dict.content val dictLen = dictContent.size @@ -305,13 +354,20 @@ internal object PureZstdEncoder { } // 2) Dictionary chain (positions inside the dict content sharing the - // 4-byte prefix at `cur`). Distances here are large (cur - dictPos). + // 4-byte prefix at `cur`). Distances here are large (cur - dictPos), + // and grow by `priorBytes` for every block after the first, which is + // also why such a match must stop at the dictionary's end. val key = first4(::histByte, cur) index.forEachCandidate(key, depth.maxCandidates) { dictPos -> - val l = matchLength(dictPos, cur, available) + // What follows the dictionary's last byte is this chunk only when + // the chunk is the frame's first block; after that it is the + // frame's earlier output, so the match has to stop at the end of + // the dictionary content. + val limit = if (priorBytes == 0) available else minOf(available, dictLen - dictPos) + val l = matchLength(dictPos, cur, limit) if (l > bestLen) { bestLen = l - bestDist = cur - dictPos + bestDist = cur - dictPos + priorBytes } } @@ -390,13 +446,19 @@ internal object PureZstdEncoder { * FSE-coded sequences section. Returns null only if it could not be built * (it always can for our inputs). */ - private fun encodeCompressedBlock(program: Program, data: ByteArray, dict: ParsedDictionary): ByteArray? { + private fun encodeCompressedBlock( + program: Program, + data: ByteArray, + dict: ParsedDictionary, + repeatOffsets: IntArray, + firstBlock: Boolean, + ): ByteArray? { val out = ArrayList(data.size + 16) - writeLiteralsSection(out, program.literals, dict) + writeLiteralsSection(out, program.literals, dict, firstBlock) // Sequences_Section. - writeSequences(out, program.sequences, dict) + writeSequences(out, program.sequences, dict, repeatOffsets, firstBlock) return ByteArray(out.size) { out[it] } } @@ -415,7 +477,9 @@ internal object PureZstdEncoder { * - **Treeless (litType 3)** -- reuses the dictionary's trained Huffman * table, costing no tree description at all, when the dict has one that * covers every literal byte in this block (a dict's training corpus - * commonly never produced every byte value). + * commonly never produced every byte value). Only offered for the + * frame's FIRST block: after that, "repeat" means the table the previous + * block described, which the dictionary's is not. * - **Huffman_Compressed (litType 2)** -- a table built from THIS block's * own histogram, plus the tree description a decoder needs to rebuild * it. Pays for the description, so it wins only once the literals are @@ -430,11 +494,16 @@ internal object PureZstdEncoder { * above -- which is also what keeps the dictionary's Treeless path from * being displaced by an equally-sized fresh table. */ - private fun writeLiteralsSection(out: ArrayList, literals: ByteArray, dict: ParsedDictionary) { + private fun writeLiteralsSection( + out: ArrayList, + literals: ByteArray, + dict: ParsedDictionary, + firstBlock: Boolean, + ) { val rawCost = rawLiteralsHeaderLen(literals.size) + literals.size val best = listOfNotNull( buildRleLiterals(literals), - buildTreelessLiterals(literals, dict), + if (firstBlock) buildTreelessLiterals(literals, dict) else null, buildHuffmanLiterals(literals), ).minByOrNull { it.size } @@ -568,7 +637,19 @@ internal object PureZstdEncoder { } } - private fun writeSequences(out: ArrayList, sequences: List, dict: ParsedDictionary) { + /** + * Sequences_Section for one block. [repeatOffsets] is the running per-frame + * repeat-offset state, advanced in place by the sequences written here -- + * except when the block carries none, where the decoder likewise never + * reaches the sequence machinery. + */ + private fun writeSequences( + out: ArrayList, + sequences: List, + dict: ParsedDictionary, + repeatOffsets: IntArray, + firstBlock: Boolean, + ) { val nbSeq = sequences.size // Number_of_Sequences. when { @@ -599,8 +680,7 @@ internal object PureZstdEncoder { // ZstdDecoder.applyOffset rotates `rep` per sequence on decode. This // must happen BEFORE picking LL/OF/ML tables below: which table each // stream needs is a function of the codes actually produced. - val rep = dict.repeatOffsets.copyOf() - val codes = Array(nbSeq) { computeCodes(sequences[it], rep) } + val codes = Array(nbSeq) { computeCodes(sequences[it], repeatOffsets) } // Symbol_Compression_Modes: 2 bits each for LL, OF, ML (high bits // first), low 2 bits reserved (0). Each stream picks its own cheapest @@ -608,16 +688,25 @@ internal object PureZstdEncoder { val llCodes = IntArray(nbSeq) { codes[it].llCode } val ofCodes = IntArray(nbSeq) { codes[it].ofCode } val mlCodes = IntArray(nbSeq) { codes[it].mlCode } + // The dictionary's tables are only what "Repeat" means for the frame's + // FIRST block; after that the decoder's repeat slot holds the previous + // block's table instead. val ll = chooseSequenceTable( - dict.literalLengthFse, + dict.literalLengthFse.takeIf { firstBlock }, predefinedLiteralLengthEnc, LITERAL_LENGTH_MAX_SYMBOL, LITERAL_LENGTH_MAX_LOG, llCodes, ) - val of = chooseSequenceTable(dict.offsetFse, predefinedOffsetEnc, OFFSET_MAX_SYMBOL, OFFSET_MAX_LOG, ofCodes) + val of = chooseSequenceTable( + dict.offsetFse.takeIf { firstBlock }, + predefinedOffsetEnc, + OFFSET_MAX_SYMBOL, + OFFSET_MAX_LOG, + ofCodes, + ) val ml = chooseSequenceTable( - dict.matchLengthFse, + dict.matchLengthFse.takeIf { firstBlock }, predefinedMatchLengthEnc, MATCH_LENGTH_MAX_SYMBOL, MATCH_LENGTH_MAX_LOG, @@ -835,9 +924,13 @@ internal object PureZstdEncoder { val fresh = buildFreshFseTable(codes, maxSymbol, maxLog) if (fresh != null) consider(SeqTableChoice(fresh.encoder, 2, fresh.description)) - // Predefined covers every code these tables can legally carry, so the - // fallback is unreachable in practice; it keeps the function total. - return best ?: SeqTableChoice(predefined, 0, ByteArray(0)) + // Predefined covers every code a block of any reachable size produces, so + // this is unreachable in practice -- but silently falling back to a table + // that cannot represent a code would emit a corrupt stream, so say so + // instead. (The one theoretical route is an offset code above the + // predefined table's 28: a dictionary match from a block that starts more + // than 512 MB into the frame.) + return best ?: throw ZstdException("no FSE table can encode this block's sequence codes") } /** Map a literal length to its FSE code (largest baseline <= length). */ diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/EncoderBlockLimitTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/EncoderBlockLimitTest.kt deleted file mode 100644 index bb3413e..0000000 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/EncoderBlockLimitTest.kt +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-3.0-or-later -package org.meshtastic.kzstd - -import kotlin.test.Test -import kotlin.test.assertContentEquals -import kotlin.test.assertFailsWith - -/** - * The encoder emits one zstd block per frame, so its input is bounded by zstd's - * 128 KiB `Block_Maximum_Size` (RFC 8878 §3.1.1.2). Inputs at the limit round-trip; - * larger inputs must be REJECTED with a typed [ZstdException] rather than silently - * producing a frame that neither libzstd nor kzstd can decode. - */ -class EncoderBlockLimitTest { - - private val blockMax = 1 shl 17 // 128 KiB - - @Test - fun acceptsInputUpToTheBlockLimit() { - // Compressible (26-byte cycle) so the frame stays tiny and the test is fast. - val atLimit = ByteArray(blockMax) { (('a'.code) + (it % 26)).toByte() } - val back = Zstd.decompress(Zstd.compress(atLimit), maxSize = blockMax + 16) - assertContentEquals(atLimit, back) - } - - @Test - fun rejectsInputOverTheBlockLimit() { - val overLimit = ByteArray(blockMax + 1) - assertFailsWith { Zstd.compress(overLimit) } - assertFailsWith { Zstd.compress(overLimit, ZstdDictionary.EMPTY) } - } -} diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt index bea2fb6..0a66cb5 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt @@ -26,6 +26,34 @@ internal object FrameInspector { /** Block_Size field of the frame's first block header. */ fun blockSize(frame: ByteArray): Int = blockHeader(frame, frameHeaderEnd(frame)) ushr 3 + /** One block's header fields, as read off the wire. */ + class Block(val type: Int, val size: Int, val last: Boolean) + + /** + * Every block of the frame, in order, by walking the block chain and + * skipping each body. The walk asserts the chain ends exactly at the end of + * the frame, so a wrong Block_Size or a missing/extra Last_Block flag fails + * here rather than surviving as a frame that happens to still decode. + * + * Note the two meanings of `Block_Size`: for Raw and Compressed blocks it is + * the length of the body that follows, but for an RLE block it is the + * REGENERATED length while the body is a single byte. + */ + fun blocks(frame: ByteArray): List { + var p = frameHeaderEnd(frame) + val blocks = ArrayList() + while (true) { + val header = blockHeader(frame, p) + val block = Block(type = (header ushr 1) and 0x3, size = header ushr 3, last = (header and 1) == 1) + blocks.add(block) + p += 3 + if (block.type == 1) 1 else block.size + if (block.last) break + check(p < frame.size) { "block chain ran off the end of a ${frame.size}-byte frame" } + } + check(p == frame.size) { "block chain ended at $p of a ${frame.size}-byte frame" } + return blocks + } + /** * Literals_Block_Type (0 Raw, 1 RLE, 2 Huffman_Compressed, 3 Treeless) of * the first block, or -1 when that block is not a Compressed_Block. diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt new file mode 100644 index 0000000..c2c6795 --- /dev/null +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * A zstd block's regenerated content cannot exceed `Block_Maximum_Size` + * (RFC 8878 §3.1.1.2) — 128 KiB — so anything larger has to be split across + * SEVERAL blocks of one frame, with `Last_Block` set on the final block only. + * The encoder used to reject such inputs outright. + * + * Round-tripping alone would not prove the split is right: a frame whose block + * chain is mis-sized can still decode when the decoder is lenient about where + * it stops. So every case here also walks the chain with + * [FrameInspector.blocks], which asserts the last block's body ends exactly at + * the end of the frame. + */ +class MultiBlockFrameTest { + + private val blockMax = 1 shl 17 // 128 KiB + + /** Compressible: a 26-byte cycle, so blocks stay small and the tests stay fast. */ + private fun cyclic(size: Int) = ByteArray(size) { ('a'.code + (it % 26)).toByte() } + + /** Incompressible, so blocks fall back to Raw and the chain is at its widest. */ + private fun noisy(size: Int, seed: Int) = Random(seed).nextBytes(size) + + @Test + fun inputAtTheBlockLimitStaysASingleBlock() { + val data = cyclic(blockMax) + val frame = Zstd.compress(data) + val blocks = FrameInspector.blocks(frame) + assertEquals(1, blocks.size, "128 KiB exactly still fits one block") + assertTrue(blocks[0].last, "the only block must be the last block") + assertContentEquals(data, Zstd.decompress(frame, maxSize = blockMax + 16)) + } + + @Test + fun oneByteOverTheBlockLimitSplitsIntoTwoBlocks() { + val data = cyclic(blockMax + 1) + val frame = Zstd.compress(data) + val blocks = FrameInspector.blocks(frame) + assertEquals(2, blocks.size, "128 KiB + 1 needs a second block") + assertEquals(listOf(false, true), blocks.map { it.last }, "Last_Block belongs to the final block only") + assertContentEquals(data, Zstd.decompress(frame, maxSize = blockMax + 16)) + } + + @Test + fun emptyInputIsStillASingleEmptyBlock() { + val frame = Zstd.compress(ByteArray(0)) + val blocks = FrameInspector.blocks(frame) + assertEquals(1, blocks.size) + assertTrue(blocks[0].last) + assertContentEquals(ByteArray(0), Zstd.decompress(frame, maxSize = 16)) + } + + @Test + fun lastBlockIsSetOnlyOnTheFinalBlockOfALongChain() { + val data = cyclic(blockMax * 3 + 7) + val blocks = FrameInspector.blocks(Zstd.compress(data)) + assertEquals(4, blocks.size, "three full blocks plus a 7-byte remainder") + assertEquals(listOf(false, false, false, true), blocks.map { it.last }) + } + + @Test + fun multiBlockFramesRoundTrip() { + // Each shape drives a different block type across the chain: compressible + // input -> Compressed blocks, noise -> Raw blocks, a constant run -> RLE + // blocks (whose Block_Size is the REGENERATED size, not the body length). + val samples = listOf( + cyclic(blockMax + 1), + cyclic(blockMax * 2), + noisy(blockMax + 5000, seed = 7), + ByteArray(blockMax * 2 + 3) { 'Q'.code.toByte() }, + ) + for (data in samples) { + val frame = Zstd.compress(data) + val blocks = FrameInspector.blocks(frame) + assertTrue(blocks.size > 1, "expected a multi-block frame for ${data.size} bytes") + val back = Zstd.decompress(frame, maxSize = data.size + 64) + assertContentEquals(data, back, "round-trip of ${data.size} bytes") + } + } + + @Test + fun multiBlockFramesRoundTripWithADictionary() { + val dict = ZstdDictionary(TestVectors.trainedDict) + // Noise first, so the second block's only useful history is the + // dictionary — which by then sits a whole block further back. + val data = noisy(blockMax, seed = 11) + TestVectors.structured[0] + val frame = Zstd.compress(data, dict) + assertTrue(FrameInspector.blocks(frame).size > 1, "expected a multi-block dictionary frame") + assertContentEquals(data, Zstd.decompress(frame, dict, maxSize = data.size + 64)) + } +} diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt new file mode 100644 index 0000000..26865fd --- /dev/null +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.meshtastic.kzstd + +import com.github.luben.zstd.ZstdDictDecompress +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import com.github.luben.zstd.Zstd as LibZstd + +/** + * Multi-block frames against the real-libzstd oracle: kzstd's own decoder + * agreeing with kzstd's own encoder proves nothing about the block chain, since + * both halves would share the same mistake. libzstd reading these frames is + * what proves the split is conformant. + * + * These inputs are multi-megabyte, so they live in `jvmTest` rather than + * `commonTest`, which runs on thirteen targets. + */ +class MultiBlockInteropTest { + + private val blockMax = 1 shl 17 // 128 KiB + + @Test + fun multiMegabyteFrameDecodesUnderLibzstd() { + val data = telemetry(3 shl 20) + val frame = Zstd.compress(data) + + val blocks = FrameInspector.blocks(frame) + assertEquals((data.size + blockMax - 1) / blockMax, blocks.size, "one block per 128 KiB chunk") + assertEquals(1, blocks.count { it.last }, "exactly one Last_Block") + assertTrue(blocks.last().last, "Last_Block must be the final block") + + assertContentEquals(data, LibZstd.decompress(frame, data.size), "kzstd -> libzstd, ${data.size} bytes") + assertContentEquals(data, Zstd.decompress(frame, maxSize = data.size), "kzstd -> kzstd, ${data.size} bytes") + } + + @Test + fun multiMegabyteIncompressibleFrameDecodesUnderLibzstd() { + // Raw blocks all the way down: the chain's headers are the only thing + // holding the frame together, so a mis-sized one shows up immediately. + val data = Random(0x5EED).nextBytes(2 shl 20) + val frame = Zstd.compress(data) + + val blocks = FrameInspector.blocks(frame) + assertEquals((data.size + blockMax - 1) / blockMax, blocks.size) + assertTrue(blocks.all { it.type == 0 }, "expected Raw blocks for random input") + assertContentEquals(data, LibZstd.decompress(frame, data.size)) + assertContentEquals(data, Zstd.decompress(frame, maxSize = data.size)) + } + + /** + * Sequences carry the decoder's three repeat-offset slots forward across + * block boundaries (they are per-FRAME state, not per-block). An encoder + * that restarted them at every block would emit repeat codes meaning some + * other distance, and libzstd would hand back different bytes — which a + * kzstd-only round-trip, restarting them the same wrong way on both sides, + * would not catch. + */ + @Test + fun repeatOffsetsCarryAcrossBlockBoundaries() { + // Long-range structure, so most sequences reuse a recent offset and the + // slots are in constant motion when each boundary is crossed. + val unit = telemetry(1 shl 14) + val data = ByteArray(unit.size * 40) { unit[it % unit.size] } + val frame = Zstd.compress(data) + assertTrue(FrameInspector.blocks(frame).size > 4, "expected several blocks") + assertContentEquals(data, LibZstd.decompress(frame, data.size)) + } + + /** + * A dictionary match's distance is measured from the CURRENT position, so in + * the second and later blocks the dictionary sits a whole block further back + * than it does in the first. Here the first block is pure noise, so the + * second block's only compressible history is the dictionary itself. + */ + @Test + fun multiBlockDictionaryFrameDecodesUnderLibzstd() { + val dictBytes = TestVectors.trainedDict + val kdict = ZstdDictionary(dictBytes) + val ddict = ZstdDictDecompress(dictBytes) + + for (sample in TestVectors.structured) { + val data = Random(0xD1C7).nextBytes(blockMax) + sample + val frame = Zstd.compress(data, kdict) + val blocks = FrameInspector.blocks(frame) + assertEquals(2, blocks.size, "expected two blocks for ${data.size} bytes") + assertEquals(2, blocks[1].type, "the tail block should be a Compressed_Block") + // Well under half: the tail is a sample the dictionary was trained on, + // so it only shrinks this far if the second block really is matching + // into the dictionary. Entropy-coding the literals alone would not. + assertTrue( + blocks[1].size < sample.size / 2, + "tail block ${blocks[1].size} B for a ${sample.size} B sample — dictionary matches lost?", + ) + assertContentEquals( + data, + LibZstd.decompress(frame, ddict, data.size), + "kzstd(dict) -> libzstd, tail sample ${sample.size} bytes", + ) + assertContentEquals(data, Zstd.decompress(frame, kdict, maxSize = data.size)) + } + } + + /** JSON-ish telemetry records, generated to at least [size] bytes. */ + private fun telemetry(size: Int): ByteArray { + var seed = 0x2468ACE + fun next(): Int { + seed = (seed * 1103515245 + 12345) and 0x7FFFFFFF + return seed ushr 8 + } + val states = listOf("ok", "warn", "offline", "degraded", "recovering", "unknown") + val words = listOf( + "region", "latency", "throughput", "stable", "nominal", + "monitored", "link", "quick", "fox", "dog", + ) + val sb = StringBuilder(size + 256) + while (sb.length < size) { + val msg = (0 until 3 + next() % 6).joinToString(" ") { words[next() % words.size] } + sb.append("{\"type\":\"telemetry\",\"seq\":").append(next() % 1000000) + .append(",\"node\":\"node-").append(next() % 64) + .append("\",\"state\":\"").append(states[next() % states.size]) + .append("\",\"lat\":").append(next() % 90).append('.').append(next() % 100000) + .append(",\"msg\":\"").append(msg).append("\"}") + } + return sb.toString().encodeToByteArray().copyOf(size) + } +} From ecc28f5a6e404f04bd8df37f4dd508e5091e5e55 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 14:55:15 -0500 Subject: [PATCH 07/11] feat(internal): carry entropy tables across block boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Repeat" is per-FRAME, not per-block: Symbol_Compression_Mode 3 names the FSE table the previous block described, and Treeless literals name the previous block's Huffman table. Only for a frame's first block do those mean the dictionary's tables. Multi-block frames therefore had both modes switched off after the first block — correct, but it left every later block paying for a table description it could have had for nothing. The encoder now keeps the same state the decoder does. FrameEntropy mirrors ZstdDecoder's DecodeState field for field, is seeded from the dictionary the same way, and is updated by the same rules: - a stream's chosen table becomes what Repeat names next, for EVERY mode including Predefined and RLE — the decoder stores its resolved table the same way however it got it; - a block that describes no Huffman tree (Raw or RLE literals) leaves the Huffman slot alone, so Treeless keeps naming the older table; - a block with no sequences touches nothing, matching the decoder's return before the Symbol_Compression_Modes byte such a block does not carry; - and a Raw or RLE BLOCK moves none of it, which is why the block is still encoded against a copy adopted only when the compressed form wins. FreshFseTable and FreshHuffmanTable now carry the decode-side table they were derived from, so what the encoder records as "current" is by construction the table the description it just wrote will rebuild — the same reason those types hand back a decoder-built encode table in the first place. The predefined tables gain their decode-side halves for the same reason. Choosing Repeat stays safe by construction: FseEncTable.streamBitCost returns null for a code the table cannot represent, so a carried-forward table that no longer fits the block — an RLE table from a constant stream being the sharp case — is simply not a candidate. Measured. 3 MB of synthetic JSON telemetry: 557,565 -> 556,657 bytes, as the offset and match-length streams settle onto one table for the whole frame. A 128 KiB noise block followed by a dictionary-trained sample: the tail block 54 -> 46, 49 -> 41, 51 -> 44, 56 -> 48, 59 -> 56, 58 -> 51 bytes, because the noise block describes nothing and so leaves the dictionary's own tables live for the block after it. Single-block frames see the seeded state and nothing else, so they are byte-identical. The tests assert the mode rather than the round-trip: a dictionary-less frame's first block cannot repeat anything (0, 0, 0) while its successors do (3, 3, 3), the multi-megabyte oracle case requires some later block to repeat before handing the frame to libzstd, and the dictionary case requires the block after the noise to still be Treeless. Signed-off-by: James Rich --- .../kzstd/internal/FseTableWriter.kt | 11 +- .../kzstd/internal/HuffmanBuilder.kt | 12 +- .../meshtastic/kzstd/internal/ZstdEncoder.kt | 238 ++++++++++++------ .../org/meshtastic/kzstd/FrameInspector.kt | 27 +- .../meshtastic/kzstd/MultiBlockFrameTest.kt | 23 ++ .../meshtastic/kzstd/MultiBlockInteropTest.kt | 18 ++ 6 files changed, 237 insertions(+), 92 deletions(-) diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt index f6277d8..3bc893b 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/FseTableWriter.kt @@ -19,8 +19,14 @@ import org.meshtastic.kzstd.ZstdException */ internal const val FSE_MIN_TABLELOG: Int = 5 -/** A freshly built sequence-stream FSE table plus its wire description. */ -internal class FreshFseTable(val encoder: FseEncTable, val description: ByteArray) +/** + * A freshly built sequence-stream FSE table plus its wire description. + * + * [decoder] is the decode-side table [encoder] was derived from — the same one a + * reader rebuilds from [description]. The encoder keeps it so that a later block + * choosing "Repeat" mode names exactly the table the decoder is holding. + */ +internal class FreshFseTable(val encoder: FseEncTable, val decoder: FseTable, val description: ByteArray) /** * Build an FSE table for [codes] (one sequence stream's symbols, in any order — @@ -47,6 +53,7 @@ internal fun buildFreshFseTable(codes: IntArray, maxSymbol: Int, maxLog: Int): F val decode = FseTable.build(normalized, maxSymbol, tableLog) return FreshFseTable( FseEncTable.fromDecodeTable(decode, maxSymbol), + decode, writeFseTableDescription(normalized, maxSymbol, tableLog), ) } diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt index 42c65c8..43a2e99 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/HuffmanBuilder.kt @@ -33,8 +33,14 @@ internal const val MAX_LITERAL_CODE_BITS: Int = 11 */ private const val MAX_DIRECT_WEIGHTS = 128 -/** A freshly built literals Huffman table plus its wire description. */ -internal class FreshHuffmanTable(val encoder: HuffmanEncTable, val description: ByteArray) +/** + * A freshly built literals Huffman table plus its wire description. + * + * [decoder] is the decode-side table [encoder] was derived from — the same one a + * reader rebuilds from [description]. The encoder keeps it so that a later + * block's Treeless literals name exactly the table the decoder is holding. + */ +internal class FreshHuffmanTable(val encoder: HuffmanEncTable, val decoder: HuffmanTable, val description: ByteArray) /** * Build a Huffman table for the alphabet [histogram] describes (counts per byte @@ -65,7 +71,7 @@ internal fun buildLiteralsHuffman(histogram: IntArray): FreshHuffmanTable? { // fromWeights recomputes. val explicit = IntArray(lastSymbol) { weights[it] } val decode = HuffmanTable.fromWeights(explicit, lastSymbol) - return FreshHuffmanTable(HuffmanEncTable.fromDecodeTable(decode), writeDirectWeights(explicit)) + return FreshHuffmanTable(HuffmanEncTable.fromDecodeTable(decode), decode, writeDirectWeights(explicit)) } /** diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt index dbac950..37baecd 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt @@ -29,12 +29,12 @@ import org.meshtastic.kzstd.ZstdException * once per `ZstdDictionary` instance, so the dict content is hashed once and * reused across calls. Lazy (1-step lookahead) matching for a better ratio. * - **Literals:** RLE (litType 1) when every literal is the same byte, the - * dictionary's trained Huffman table (Treeless, litType 3, single-stream + * Huffman table the decoder already holds (Treeless, litType 3, single-stream * only) when it covers every literal byte in this block, or RAW (litType 0) * — whichever is smallest. * - **Sequences:** FSE-coded, independently per LL/OF/ML stream, each stream - * picking the cheapest of: the dictionary's trained "Repeat" table (mode 3) - * when the dict has one that covers this block's codes; RLE (mode 1) when + * picking the cheapest of: the "Repeat" table (mode 3) the decoder already + * holds, when it covers this block's codes; RLE (mode 1) when * every sequence uses the same code; a fresh FSE table built from this * block's own code counts (mode 2); or the PREDEFINED table (mode 0) — * always total over its symbol range, and exactly what [PureZstdDecoder] / @@ -45,6 +45,10 @@ import org.meshtastic.kzstd.ZstdException * one of the three most-recently-used offsets (rotated exactly as * [PureZstdDecoder]'s `applyOffset` does), else an explicit literal offset * (`offset_code = distance + 3`). + * - **Per-frame state:** the three repeat offsets and the tables the "Repeat" + * modes above name are FRAME state, not block state — seeded from the + * dictionary and then carried block to block, exactly as [PureZstdDecoder] + * carries them (see `FrameEntropy`). * * Pure common Kotlin: no `java.*`, no `expect/actual`. */ @@ -95,12 +99,9 @@ internal object PureZstdEncoder { // block before it. out.add(windowDescriptor(dict.content.size + data.size)) - // The three repeat-offset slots are per-FRAME state that the decoder - // rotates as it executes sequences, so they must be carried from block to - // block here too -- restarting them per block would make every repeat code - // after the first block mean a different distance. Held in a local: this - // singleton keeps no mutable state. - val repeatOffsets = dict.repeatOffsets.copyOf() + // Per-FRAME entropy state, seeded from the dictionary and carried from + // block to block. Held in a local: this singleton keeps no mutable state. + val state = FrameEntropy(dict) // Cut the input into Block_Maximum_Size chunks, one block each. `do/while` // rather than `while`, so an empty input still emits the one (empty, @@ -108,7 +109,7 @@ internal object PureZstdEncoder { var start = 0 do { val end = minOf(start + MAX_BLOCK_SIZE, data.size) - encodeBlock(out, data, start, end, lastBlock = end == data.size, dict, index, depth, repeatOffsets) + encodeBlock(out, data, start, end, lastBlock = end == data.size, dict, index, depth, state) start = end } while (start < data.size) @@ -135,10 +136,12 @@ internal object PureZstdEncoder { * chunk is identical), the chunk length for Raw, or the compressed body. Ties * go to the simpler form. * - * [repeatOffsets] is the frame's running repeat-offset state. Sequences move - * it, so the block is encoded against a COPY that is adopted only if the - * Compressed form actually wins -- a Raw or RLE block carries no sequences and - * so leaves the decoder's slots exactly where they were. + * [state] is the frame's running entropy state. Encoding this block moves it + * -- sequences rotate the repeat offsets, and any table this block describes + * becomes what "Repeat" names next -- so the block is encoded against a COPY + * that is adopted only if the Compressed form actually wins. A Raw or RLE + * block describes nothing and executes no sequence, so it must leave the + * decoder's state exactly where it was. */ @Suppress("LongParameterList") private fun encodeBlock( @@ -150,12 +153,12 @@ internal object PureZstdEncoder { dict: ParsedDictionary, index: MatchIndex, depth: SearchDepth, - repeatOffsets: IntArray, + state: FrameEntropy, ) { val chunk = data.copyOfRange(start, end) val program = buildSequences(chunk, dict, index, depth, priorBytes = start) - val blockOffsets = repeatOffsets.copyOf() - val compressedBlock = encodeCompressedBlock(program, chunk, dict, blockOffsets, firstBlock = start == 0) + val blockState = state.copy() + val compressedBlock = encodeCompressedBlock(program, chunk, blockState) val constantByte = constantByteOrNull(chunk) when { @@ -171,7 +174,7 @@ internal object PureZstdEncoder { // Block_Header (3 bytes LE): last, type=2 (Compressed), size=blockSize writeBlockHeader(out, lastBlock, blockType = 2, blockSize = compressedBlock.size) compressedBlock.forEach { out.add(it) } - blockOffsets.copyInto(repeatOffsets) + state.adopt(blockState) } else -> { @@ -182,6 +185,48 @@ internal object PureZstdEncoder { } } + /** + * The state a zstd decoder carries from block to block within one frame: the + * three repeat offsets, and the tables that "Repeat" modes name -- the + * Huffman table Treeless literals reuse ([huffman]) and the previous block's + * FSE table per sequence stream. + * + * It mirrors [PureZstdDecoder]'s `DecodeState` field for field, and is seeded + * the same way, from the dictionary: that is exactly what those modes mean + * for a frame's first block. The two must agree at every block boundary or a + * Repeat mode names one table on the encode side and another on the decode + * side, which no round-trip through this codec alone would reveal. + * + * Instances live in [encode]'s locals, never on the singleton. + */ + private class FrameEntropy( + val repeatOffsets: IntArray, + var huffman: HuffmanTable?, + var litLenFse: FseTable?, + var offsetFse: FseTable?, + var matchLenFse: FseTable?, + ) { + constructor(dict: ParsedDictionary) : this( + repeatOffsets = dict.repeatOffsets.copyOf(), + huffman = dict.literalsHuffman, + litLenFse = dict.literalLengthFse, + offsetFse = dict.offsetFse, + matchLenFse = dict.matchLengthFse, + ) + + /** A detached copy to encode one candidate block against. */ + fun copy(): FrameEntropy = FrameEntropy(repeatOffsets.copyOf(), huffman, litLenFse, offsetFse, matchLenFse) + + /** Take on [other]'s state, once the block encoded against it is emitted. */ + fun adopt(other: FrameEntropy) { + other.repeatOffsets.copyInto(repeatOffsets) + huffman = other.huffman + litLenFse = other.litLenFse + offsetFse = other.offsetFse + matchLenFse = other.matchLenFse + } + } + /** The byte every element of [bytes] equals, or null (including when empty). */ private fun constantByteOrNull(bytes: ByteArray): Byte? { if (bytes.isEmpty()) return null @@ -446,19 +491,13 @@ internal object PureZstdEncoder { * FSE-coded sequences section. Returns null only if it could not be built * (it always can for our inputs). */ - private fun encodeCompressedBlock( - program: Program, - data: ByteArray, - dict: ParsedDictionary, - repeatOffsets: IntArray, - firstBlock: Boolean, - ): ByteArray? { + private fun encodeCompressedBlock(program: Program, data: ByteArray, state: FrameEntropy): ByteArray? { val out = ArrayList(data.size + 16) - writeLiteralsSection(out, program.literals, dict, firstBlock) + writeLiteralsSection(out, program.literals, state) // Sequences_Section. - writeSequences(out, program.sequences, dict, repeatOffsets, firstBlock) + writeSequences(out, program.sequences, state) return ByteArray(out.size) { out[it] } } @@ -474,16 +513,16 @@ internal object PureZstdEncoder { * Literals_Section_Header + body, in whichever encoding is smallest: * * - **RLE (litType 1)** -- one stored byte regenerates the whole run. - * - **Treeless (litType 3)** -- reuses the dictionary's trained Huffman - * table, costing no tree description at all, when the dict has one that - * covers every literal byte in this block (a dict's training corpus - * commonly never produced every byte value). Only offered for the - * frame's FIRST block: after that, "repeat" means the table the previous - * block described, which the dictionary's is not. + * - **Treeless (litType 3)** -- reuses the Huffman table the decoder is + * already holding, costing no tree description at all, when it covers + * every literal byte in this block. That table is the dictionary's for + * the frame's first block, and afterwards whichever table the last + * Huffman-coded block described. * - **Huffman_Compressed (litType 2)** -- a table built from THIS block's * own histogram, plus the tree description a decoder needs to rebuild * it. Pays for the description, so it wins only once the literals are - * both numerous and skewed enough. + * both numerous and skewed enough. Choosing it replaces the table + * Treeless will reuse in later blocks. * - **Raw (litType 0)** -- the fallback, and the winner for short or * high-entropy literal runs. * @@ -493,22 +532,22 @@ internal object PureZstdEncoder { * and measured. Ties keep the earlier, simpler candidate in the list order * above -- which is also what keeps the dictionary's Treeless path from * being displaced by an equally-sized fresh table. + * + * Updates [state] when the chosen form describes a table, and only then: + * Raw and RLE literals describe none, and the decoder likewise keeps + * whatever it was holding. */ - private fun writeLiteralsSection( - out: ArrayList, - literals: ByteArray, - dict: ParsedDictionary, - firstBlock: Boolean, - ) { + private fun writeLiteralsSection(out: ArrayList, literals: ByteArray, state: FrameEntropy) { val rawCost = rawLiteralsHeaderLen(literals.size) + literals.size val best = listOfNotNull( buildRleLiterals(literals), - if (firstBlock) buildTreelessLiterals(literals, dict) else null, + buildTreelessLiterals(literals, state.huffman), buildHuffmanLiterals(literals), - ).minByOrNull { it.size } + ).minByOrNull { it.section.size } - if (best != null && best.size < rawCost) { - best.forEach { out.add(it) } + if (best != null && best.section.size < rawCost) { + best.section.forEach { out.add(it) } + if (best.described != null) state.huffman = best.described return } // Literals_Section_Header (Raw, litType 0). Regenerated_Size = literals @@ -517,29 +556,39 @@ internal object PureZstdEncoder { literals.forEach { out.add(it) } } + /** + * One candidate Literals_Section: the bytes it would occupy, plus the + * Huffman table it DESCRIBES on the wire (null unless it carries a tree + * description), which becomes what a later block's Treeless literals reuse. + */ + private class LiteralsCandidate(val section: ByteArray, val described: HuffmanTable?) + /** RLE literals (litType 1): header + the single repeated byte, or null. */ - private fun buildRleLiterals(literals: ByteArray): ByteArray? { + private fun buildRleLiterals(literals: ByteArray): LiteralsCandidate? { val constant = constantByteOrNull(literals) ?: return null val section = ArrayList(4) writeRawOrRleLiteralsHeader(section, literals.size, litType = 1) section.add(constant) - return ByteArray(section.size) { section[it] } + return LiteralsCandidate(ByteArray(section.size) { section[it] }, described = null) } /** - * Treeless literals (litType 3): the dictionary's own Huffman table, so the + * Treeless literals (litType 3): the table the decoder already holds, so the * section carries no tree description -- only the header and the stream. - * Null when there is no dict table, it does not cover some literal byte, or + * Null when there is no such table, it does not cover some literal byte, or * the result overflows the single-stream size fields. */ - private fun buildTreelessLiterals(literals: ByteArray, dict: ParsedDictionary): ByteArray? { - val huffman = dict.literalsHuffman ?: return null + private fun buildTreelessLiterals(literals: ByteArray, huffman: HuffmanTable?): LiteralsCandidate? { + if (huffman == null) return null if (literals.isEmpty() || literals.size > MAX_SINGLE_STREAM_LITERALS) return null val encTable = HuffmanEncTable.fromDecodeTable(huffman) for (b in literals) if (!encTable.isCovered(b.toInt() and 0xFF)) return null val stream = huffmanLiteralsStream(literals, encTable) if (stream.size > MAX_SINGLE_STREAM_LITERALS) return null - return literalsSection(litType = 3, regenSize = literals.size, body = stream) + return LiteralsCandidate( + literalsSection(litType = 3, regenSize = literals.size, body = stream), + described = null, + ) } /** @@ -549,7 +598,7 @@ internal object PureZstdEncoder { * ([buildLiteralsHuffman]) or the result overflows the single-stream size * fields. */ - private fun buildHuffmanLiterals(literals: ByteArray): ByteArray? { + private fun buildHuffmanLiterals(literals: ByteArray): LiteralsCandidate? { if (literals.isEmpty() || literals.size > MAX_SINGLE_STREAM_LITERALS) return null val histogram = IntArray(256) for (b in literals) histogram[b.toInt() and 0xFF]++ @@ -558,7 +607,10 @@ internal object PureZstdEncoder { // Compressed_Size counts the tree description AND the stream. val body = table.description + stream if (body.size > MAX_SINGLE_STREAM_LITERALS) return null - return literalsSection(litType = 2, regenSize = literals.size, body = body) + return LiteralsCandidate( + literalsSection(litType = 2, regenSize = literals.size, body = body), + described = table.decoder, + ) } /** @@ -638,18 +690,13 @@ internal object PureZstdEncoder { } /** - * Sequences_Section for one block. [repeatOffsets] is the running per-frame - * repeat-offset state, advanced in place by the sequences written here -- - * except when the block carries none, where the decoder likewise never - * reaches the sequence machinery. + * Sequences_Section for one block, advancing [state]: the sequences written + * here rotate its repeat offsets, and each stream's chosen table becomes what + * "Repeat" names in the next block. A block with no sequences leaves both + * alone, exactly as the decoder does -- it returns before the + * Symbol_Compression_Modes byte, which such a block does not even carry. */ - private fun writeSequences( - out: ArrayList, - sequences: List, - dict: ParsedDictionary, - repeatOffsets: IntArray, - firstBlock: Boolean, - ) { + private fun writeSequences(out: ArrayList, sequences: List, state: FrameEntropy) { val nbSeq = sequences.size // Number_of_Sequences. when { @@ -680,7 +727,7 @@ internal object PureZstdEncoder { // ZstdDecoder.applyOffset rotates `rep` per sequence on decode. This // must happen BEFORE picking LL/OF/ML tables below: which table each // stream needs is a function of the codes actually produced. - val codes = Array(nbSeq) { computeCodes(sequences[it], repeatOffsets) } + val codes = Array(nbSeq) { computeCodes(sequences[it], state.repeatOffsets) } // Symbol_Compression_Modes: 2 bits each for LL, OF, ML (high bits // first), low 2 bits reserved (0). Each stream picks its own cheapest @@ -688,30 +735,36 @@ internal object PureZstdEncoder { val llCodes = IntArray(nbSeq) { codes[it].llCode } val ofCodes = IntArray(nbSeq) { codes[it].ofCode } val mlCodes = IntArray(nbSeq) { codes[it].mlCode } - // The dictionary's tables are only what "Repeat" means for the frame's - // FIRST block; after that the decoder's repeat slot holds the previous - // block's table instead. val ll = chooseSequenceTable( - dict.literalLengthFse.takeIf { firstBlock }, + state.litLenFse, predefinedLiteralLengthEnc, + predefinedLiteralLengthDec, LITERAL_LENGTH_MAX_SYMBOL, LITERAL_LENGTH_MAX_LOG, llCodes, ) val of = chooseSequenceTable( - dict.offsetFse.takeIf { firstBlock }, + state.offsetFse, predefinedOffsetEnc, + predefinedOffsetDec, OFFSET_MAX_SYMBOL, OFFSET_MAX_LOG, ofCodes, ) val ml = chooseSequenceTable( - dict.matchLengthFse.takeIf { firstBlock }, + state.matchLenFse, predefinedMatchLengthEnc, + predefinedMatchLengthDec, MATCH_LENGTH_MAX_SYMBOL, MATCH_LENGTH_MAX_LOG, mlCodes, ) + // Whatever each stream settled on is what the NEXT block's Repeat mode + // names -- for every mode, including Predefined and RLE (the decoder + // stores its resolved table the same way regardless of how it got it). + state.litLenFse = ll.next + state.offsetFse = of.next + state.matchLenFse = ml.next val llTable = ll.table val ofTable = of.table val mlTable = ml.table @@ -860,19 +913,24 @@ internal object PureZstdEncoder { /** * One stream's chosen FSE encode table, its 2-bit Symbol_Compression_Mode, - * and the table description bytes (if any) that must follow the mode byte. + * the table description bytes (if any) that must follow the mode byte, and + * [next] -- the decode-side table the reader ends up holding for this stream, + * which is what the NEXT block's Repeat mode would name. */ - private class SeqTableChoice(val table: FseEncTable, val mode: Int, val description: ByteArray) + private class SeqTableChoice(val table: FseEncTable, val mode: Int, val description: ByteArray, val next: FseTable) /** * Choose the cheapest valid table for one sequence symbol stream, costed in * BITS ([FseEncTable.streamBitCost] is exact, not an entropy estimate) plus * 8 bits per description byte: * - * - **Repeat (3)** -- the dictionary's trained table, when it exists and - * assigns nonzero probability to every code this block needs (a dict's - * training corpus commonly never produced some symbol). Costs no - * description bytes. + * - **Repeat (3)** -- [repeatTable], the table the decoder is already + * holding for this stream (the dictionary's trained one in the frame's + * first block, the previous block's after that), when it assigns nonzero + * probability to every code this block needs. Costs no description bytes, + * which makes it the cheapest option outright whenever it fits -- and it + * fails safe, since [FseEncTable.streamBitCost] returns null for a code + * the table cannot represent. * - **Predefined (0)** -- the spec's default distribution; also free of * description bytes, and total over its declared symbol range. * - **RLE (1)** -- when every sequence uses the SAME code: one description @@ -887,9 +945,11 @@ internal object PureZstdEncoder { * cheaper to win, so ties keep the mode that costs no description bytes and * leaves the dictionary paths in place. */ + @Suppress("LongParameterList") private fun chooseSequenceTable( - dictTable: FseTable?, + repeatTable: FseTable?, predefined: FseEncTable, + predefinedDecode: FseTable, maxSymbol: Int, maxLog: Int, codes: IntArray, @@ -906,23 +966,27 @@ internal object PureZstdEncoder { } } - if (dictTable != null) { - consider(SeqTableChoice(FseEncTable.fromDecodeTable(dictTable, maxSymbol), 3, ByteArray(0))) + if (repeatTable != null) { + consider( + SeqTableChoice(FseEncTable.fromDecodeTable(repeatTable, maxSymbol), 3, ByteArray(0), repeatTable), + ) } - consider(SeqTableChoice(predefined, 0, ByteArray(0))) + consider(SeqTableChoice(predefined, 0, ByteArray(0), predefinedDecode)) val constantCode = codes[0].takeIf { first -> codes.all { it == first } } if (constantCode != null) { + val rle = FseTable.rle(constantCode) consider( SeqTableChoice( - FseEncTable.fromDecodeTable(FseTable.rle(constantCode), maxSymbol), + FseEncTable.fromDecodeTable(rle, maxSymbol), 1, byteArrayOf(constantCode.toByte()), + rle, ), ) } val fresh = buildFreshFseTable(codes, maxSymbol, maxLog) - if (fresh != null) consider(SeqTableChoice(fresh.encoder, 2, fresh.description)) + if (fresh != null) consider(SeqTableChoice(fresh.encoder, 2, fresh.description, fresh.decoder)) // Predefined covers every code a block of any reachable size produces, so // this is unreachable in practice -- but silently falling back to a table @@ -970,4 +1034,12 @@ internal object PureZstdEncoder { private val predefinedOffsetEnc: FseEncTable by lazy { FseEncTable.build(OF_DEFAULT_DISTRIBUTION, OF_DEFAULT_DISTRIBUTION.size - 1, OF_DEFAULT_LOG) } + + // The decode-side halves of the same three tables. A block that picks + // Predefined leaves the decoder holding THIS table for the stream, so it is + // also what a following block's Repeat mode would name -- the encoder has to + // be able to hand it back. Same immutable + `by lazy` rule as above. + private val predefinedLiteralLengthDec: FseTable by lazy { predefinedLiteralLengthTable() } + private val predefinedMatchLengthDec: FseTable by lazy { predefinedMatchLengthTable() } + private val predefinedOffsetDec: FseTable by lazy { predefinedOffsetTable() } } diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt index 0a66cb5..51ece7b 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/FrameInspector.kt @@ -26,8 +26,8 @@ internal object FrameInspector { /** Block_Size field of the frame's first block header. */ fun blockSize(frame: ByteArray): Int = blockHeader(frame, frameHeaderEnd(frame)) ushr 3 - /** One block's header fields, as read off the wire. */ - class Block(val type: Int, val size: Int, val last: Boolean) + /** One block's header fields, as read off the wire, plus where its body starts. */ + class Block(val type: Int, val size: Int, val last: Boolean, val bodyStart: Int) /** * Every block of the frame, in order, by walking the block chain and @@ -44,9 +44,14 @@ internal object FrameInspector { val blocks = ArrayList() while (true) { val header = blockHeader(frame, p) - val block = Block(type = (header ushr 1) and 0x3, size = header ushr 3, last = (header and 1) == 1) + val block = Block( + type = (header ushr 1) and 0x3, + size = header ushr 3, + last = (header and 1) == 1, + bodyStart = p + 3, + ) blocks.add(block) - p += 3 + if (block.type == 1) 1 else block.size + p = block.bodyStart + if (block.type == 1) 1 else block.size if (block.last) break check(p < frame.size) { "block chain ran off the end of a ${frame.size}-byte frame" } } @@ -54,6 +59,20 @@ internal object FrameInspector { return blocks } + /** [literalsType] for any [block] of the frame, not just the first. */ + fun literalsTypeOf(frame: ByteArray, block: Block): Int = + if (block.type != 2) -1 else frame[block.bodyStart].toInt() and 0x3 + + /** [sequenceModes] for any [block] of the frame, not just the first. */ + fun sequenceModesOf(frame: ByteArray, block: Block): Triple? { + check(block.type == 2) { "not a Compressed_Block" } + var p = skipLiteralsSection(frame, block.bodyStart) + if (sequenceCountAt(frame, p) == 0) return null + p += sequenceCountFieldLen(frame, p) + val modes = frame[p].toInt() and 0xFF + return Triple((modes ushr 6) and 0x3, (modes ushr 4) and 0x3, (modes ushr 2) and 0x3) + } + /** * Literals_Block_Type (0 Raw, 1 RLE, 2 Huffman_Compressed, 3 Treeless) of * the first block, or -1 when that block is not a Compressed_Block. diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt index c2c6795..e2fbdc1 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/MultiBlockFrameTest.kt @@ -66,6 +66,29 @@ class MultiBlockFrameTest { assertEquals(listOf(false, false, false, true), blocks.map { it.last }) } + /** + * A block's entropy tables stay live for the blocks after it: "Repeat" + * (Symbol_Compression_Mode 3) means the previous block's table, and costs no + * description bytes at all. The first block of a dictionary-less frame has no + * previous table, so it cannot use the mode — which is what makes the + * contrast worth asserting rather than just the presence of a 3 somewhere. + */ + @Test + fun laterBlocksRepeatTheEarlierBlocksSequenceTables() { + val frame = Zstd.compress(cyclic(blockMax * 3 + 7)) + val blocks = FrameInspector.blocks(frame) + + val first = FrameInspector.sequenceModesOf(frame, blocks[0]) + assertEquals(Triple(0, 0, 0), first, "no dictionary, so the first block has nothing to repeat") + for (i in 1..2) { + assertEquals( + Triple(3, 3, 3), + FrameInspector.sequenceModesOf(frame, blocks[i]), + "block $i should repeat the tables already described", + ) + } + } + @Test fun multiBlockFramesRoundTrip() { // Each shape drives a different block type across the chain: compressible diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt index 26865fd..9210ab2 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt @@ -32,6 +32,13 @@ class MultiBlockInteropTest { assertEquals(1, blocks.count { it.last }, "exactly one Last_Block") assertTrue(blocks.last().last, "Last_Block must be the final block") + // Later blocks reuse tables described by earlier ones, which only libzstd + // reading the frame can confirm the encoder and decoder agree about. + val repeated = blocks.drop(1).count { block -> + FrameInspector.sequenceModesOf(frame, block)?.toList()?.contains(3) == true + } + assertTrue(repeated > 0, "no later block repeated an earlier block's table") + assertContentEquals(data, LibZstd.decompress(frame, data.size), "kzstd -> libzstd, ${data.size} bytes") assertContentEquals(data, Zstd.decompress(frame, maxSize = data.size), "kzstd -> kzstd, ${data.size} bytes") } @@ -74,6 +81,11 @@ class MultiBlockInteropTest { * the second and later blocks the dictionary sits a whole block further back * than it does in the first. Here the first block is pure noise, so the * second block's only compressible history is the dictionary itself. + * + * It also pins the rule that a Raw block leaves the frame's entropy state + * untouched: the noise block describes nothing, so Treeless literals in the + * second block still name the DICTIONARY's Huffman table — and libzstd has to + * resolve it the same way for the frame to come back intact. */ @Test fun multiBlockDictionaryFrameDecodesUnderLibzstd() { @@ -86,7 +98,13 @@ class MultiBlockInteropTest { val frame = Zstd.compress(data, kdict) val blocks = FrameInspector.blocks(frame) assertEquals(2, blocks.size, "expected two blocks for ${data.size} bytes") + assertEquals(0, blocks[0].type, "the noise block should be Raw, describing nothing") assertEquals(2, blocks[1].type, "the tail block should be a Compressed_Block") + assertEquals( + 3, + FrameInspector.literalsTypeOf(frame, blocks[1]), + "the tail block should still reach the dictionary's Huffman table (Treeless)", + ) // Well under half: the tail is a sample the dictionary was trained on, // so it only shrinks this far if the second block really is matching // into the dictionary. Entropy-coding the literals alone would not. From a5115a38b4ef41f5686c2bd0f8fea4230ed3394e Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 14:57:36 -0500 Subject: [PATCH 08/11] docs: record multi-block encoding in the API docs, README and CHANGELOG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State what Zstd.compress now does with an input of any size, and — more usefully for whoever reads this next — what it still does NOT do: blocks are compressed independently, so a match never reaches back into an earlier block's output. That, together with the 1023-byte single-stream literals cap already documented, is why a full 128 KiB block keeps raw literals and takes its ratio from the sequence tables alone, and why 3 MB of telemetry lands between libzstd's levels 3 and 19 rather than near 19. A windowed matcher is named as the follow-up. AGENTS.md's "one block per frame" invariant is replaced rather than deleted: what now needs protecting is that PureZstdEncoder's FrameEntropy mirrors PureZstdDecoder's DecodeState exactly, since a divergence makes a Repeat mode name a different table on each side and produces a frame that still round-trips through kzstd while libzstd reads garbage. The 0.1.0 release notes keep their original wording — they described that release accurately. Documentation only; no public symbol changes, so the API baseline is unchanged. Signed-off-by: James Rich --- AGENTS.md | 12 +++++--- CHANGELOG.md | 29 +++++++++++++++++-- README.md | 15 ++++++---- .../kotlin/org/meshtastic/kzstd/Zstd.kt | 10 ++++--- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6096bb1..dcb42c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,10 +36,14 @@ would be over-engineering here. - **One-shot only (no incremental streaming API yet).** Every frame is independently decodable; there is no cross-call state. -- **One block per frame (current limit).** The encoder emits a single block, so - `compress` rejects inputs > 128 KiB (zstd's `Block_Maximum_Size`) with a - `ZstdException`. Keep that guard until multi-block encoding lands — without it a - large input silently produces a frame neither libzstd nor kzstd can decode. +- **Blocks are independent, and per-frame state is threaded through them.** The + encoder cuts input into 128 KiB (`Block_Maximum_Size`) chunks and emits a + multi-block frame; a chunk is matched only against itself and the dictionary, + never against an earlier block's output. The three repeat offsets and the tables + the "Repeat" / "Treeless" modes name are FRAME state — `PureZstdEncoder`'s + `FrameEntropy` must keep mirroring `PureZstdDecoder`'s `DecodeState` exactly, or + a mode names one table on each side and the frame decodes to garbage that only + a real-libzstd oracle catches. - **No shared mutable state, no lock.** A `ZstdDictionary` digests its dictionary once in its constructor and is immutable thereafter; the engine objects keep all per-call state in locals. Do not reintroduce global caches. The encoder's diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dbdef3..d7f64ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] Encoder-side parity work closing several gaps against the libzstd/RFC 8878 -spec — real ratio improvements for both dictionary-compressed and -dictionary-free frames, plus dictionary-ID and content-checksum validation. -No wire format or public-API changes, except where noted below. +spec — the 128 KiB single-block input limit lifted, real ratio improvements +for both dictionary-compressed and dictionary-free frames, plus +dictionary-ID and content-checksum validation. No wire format or +public-API changes, except where noted below. ### Fixed @@ -29,6 +30,13 @@ No wire format or public-API changes, except where noted below. true, the encoder sets `Content_Checksum_Flag` and appends the XXH64 checksum of the input. Defaults to false, so every existing call site's frame bytes are unchanged. +- `Zstd.compress` now accepts input up to 128 MiB. It cuts the input into + 128 KiB (`Block_Maximum_Size`) chunks and emits them as one multi-block + frame, `Last_Block` set on the final block only; anything larger than + 128 KiB used to be rejected with a `ZstdException`. `Zstd.decompress` has + always read multi-block frames from any encoder, and real libzstd reads + these. 3 MB of synthetic JSON telemetry compresses to 556,657 bytes across + 24 blocks — between libzstd's level 3 (579,374) and level 19 (362,967). ### Changed @@ -59,6 +67,15 @@ No wire format or public-API changes, except where noted below. alternative and taking the smallest, so a form is used only when it actually wins. Ties keep the previous behaviour, and dictionary-compressed frames come out the same size as before. +- Entropy tables and the three repeat offsets are now carried from block to + block within a frame, which is what the format means by them: "Repeat" + sequence tables and "Treeless" literals name the PREVIOUS block's tables, and + the dictionary's only for a frame's first block. A later block therefore + reuses a table for nothing instead of describing its own, and a `Raw` or + `RLE` block describes nothing and so leaves the state untouched — the block + after a stretch of incompressible data still reaches the dictionary's own + tables. A 128 KiB noise block followed by a dictionary-trained sample + compresses that sample's block to 46 bytes rather than 54. - `level` (1–22) now governs match-finding search depth: higher levels search more candidate matches per position, which can shrink output at the cost of more work. Level 19 (`Zstd.DEFAULT_LEVEL`) maps to exactly the search depth @@ -80,6 +97,12 @@ No wire format or public-API changes, except where noted below. ### Notes +- Blocks are compressed independently: a match never reaches back into an + earlier block's output, only into this block and the dictionary. Large + inputs therefore compress less well than a windowed encoder would manage — + and combined with the 1023-byte literals cap below, a full 128 KiB block + keeps raw literals and takes its ratio from the sequence tables alone. A + windowed, cross-block matcher is the follow-up. - Huffman-coded literals stay single-stream, so they apply to at most 1023 bytes of literals per block, and their tree description uses the direct 4-bit weight form, so a block containing a literal byte above 128 falls back diff --git a/README.md b/README.md index 8e45093..6dd1b2d 100644 --- a/README.md +++ b/README.md @@ -66,17 +66,22 @@ val back = Zstd.decompress(small, dict, maxSize = 64 * 1024) other per-level parameters (window log, target length, etc.) — but a higher level does search more candidate matches per position, which can shrink output at the cost of more work. Frames remain fully libzstd-compatible at every level. -- **Single block per frame (≤ 128 KiB input).** `Zstd.compress` emits one zstd block, - so its input is bounded by zstd's 128 KiB `Block_Maximum_Size`; a larger input - throws `ZstdException`. (`Zstd.decompress` reads multi-block frames from any - encoder.) Multi-block encoding to lift the cap is planned. +- **Blocks are compressed independently (no cross-block matching).** `Zstd.compress` + takes an input of any size, cutting it into zstd's 128 KiB `Block_Maximum_Size` + chunks and emitting one multi-block frame. Each chunk is matched only against + itself and the dictionary, never against an earlier block's output, so a large + input compresses less well than a windowed encoder manages — 3 MB of synthetic + JSON telemetry lands between libzstd's levels 3 and 19. Entropy tables and the + repeat offsets ARE carried across blocks. A windowed matcher is a planned + improvement. - **Huffman-coded literals are single-stream and directly described.** The encoder builds a Huffman table from a block's own literals, but writes only the single-stream layout (so it applies to at most 1023 bytes of literals per block) and only the direct 4-bit weight tree description (so a block containing a literal byte above 128 falls back to raw literals). Both limits are encoder-side only — `Zstd.decompress` reads the 4-stream layout and FSE-compressed weight descriptions - that libzstd emits. + that libzstd emits. The 1023-byte cap is why a full 128 KiB block keeps raw + literals: on large inputs the ratio comes from the sequence tables alone. ## Interoperability diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/Zstd.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/Zstd.kt index 177ba85..cfbf160 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/Zstd.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/Zstd.kt @@ -13,10 +13,12 @@ import org.meshtastic.kzstd.internal.PureZstdEncoder * no cross-call state, so every frame is independently decodable (what packet / * mesh transports need). * - * [compress] emits a single zstd block per frame, so its input is bounded by zstd's - * 128 KiB `Block_Maximum_Size`; a larger input throws [ZstdException] (multi-block - * encoding is a planned addition). [decompress] reads any conformant frame, - * including multi-block frames produced by other encoders. + * [compress] takes an input of any size: it cuts the input into zstd's 128 KiB + * `Block_Maximum_Size` chunks and emits them as one multi-block frame. The chunks + * are compressed independently — a match never reaches back into an earlier + * block — so a large input compresses somewhat less well than a windowed encoder + * would manage. [decompress] reads any conformant frame, including multi-block + * frames produced by other encoders. * * Pass a [ZstdDictionary] for dictionary compression; the dictionary-less * overloads operate on plain frames. From 4ac77a169b5028acd676bc03ce5c948ed8aa8888 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 15:04:56 -0500 Subject: [PATCH 09/11] test: prove an RLE block does not disturb a frame's entropy state The dictionary case already covers a RAW block in the middle of a frame leaving the tables live for the block after it. The RLE block type takes the same path in the encoder but had no oracle coverage, so a mistake that reset the state only for RLE blocks would have passed everything. A frame of telemetry, a 128 KiB constant run and more telemetry produces exactly that shape: the middle block is an RLE_Block, and the third block still repeats a table the FIRST block described. libzstd has to resolve that repeat the same way for the frame to come back intact. Signed-off-by: James Rich --- .../meshtastic/kzstd/MultiBlockInteropTest.kt | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt index 9210ab2..feb60a2 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt @@ -76,6 +76,29 @@ class MultiBlockInteropTest { assertContentEquals(data, LibZstd.decompress(frame, data.size)) } + /** + * The companion to the dictionary case below: an RLE BLOCK in the middle of a + * frame must leave the entropy state alone just as a Raw one does, so the + * block after it can still repeat the table described two blocks back. Both + * sides have to agree on that, and only libzstd reading the frame proves it. + */ + @Test + fun anRleBlockBetweenCompressedBlocksLeavesTheTablesLive() { + val data = telemetry(blockMax) + ByteArray(blockMax) { 'Q'.code.toByte() } + telemetry(blockMax) + val frame = Zstd.compress(data) + + val blocks = FrameInspector.blocks(frame) + assertEquals(3, blocks.size) + assertEquals(1, blocks[1].type, "the constant chunk should be an RLE_Block") + assertTrue( + FrameInspector.sequenceModesOf(frame, blocks[2])?.toList()?.contains(3) == true, + "the block after the RLE block should still repeat a table described before it", + ) + + assertContentEquals(data, LibZstd.decompress(frame, data.size)) + assertContentEquals(data, Zstd.decompress(frame, maxSize = data.size)) + } + /** * A dictionary match's distance is measured from the CURRENT position, so in * the second and later blocks the dictionary sits a whole block further back From 4946f784dd828a8f29b1fec3b0f88087ca21deed Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 16:03:00 -0500 Subject: [PATCH 10/11] fix(internal): cap total history at libzstd's default window-log limit Lifting the single-block 128 KiB guard removed the encoder's only ceiling on total frame history. Without a replacement, a large enough input made the declared windowLog exceed libzstd's default decompression limit (ZSTD_WINDOWLOG_LIMIT_DEFAULT = 27, 128 MiB) -- a frame kzstd's own decoder reads fine but real-world libzstd consumers reject by default, breaking this codec's own "frames stay libzstd-interoperable in both directions" invariant. AGENTS.md's removed "one block per frame" note already framed the old guard as something to hold "until multi-block encoding lands" -- this is the replacement that should have landed with it. encode() now rejects (dict content + input) beyond 128 MiB up front, before any block work, mirroring the old guard's fail-fast shape. A jvmTest confirms the boundary (commonTest would multiply a 128 MiB allocation across all thirteen targets, same reasoning as the rest of MultiBlockInteropTest.kt). The offset-code integer-overflow path this also would have made reachable (distance approaching Int.MAX_VALUE) is closed by the same fix: 128 MiB is nowhere near where `distance + 3` could overflow a signed Int. Also, in the same file since it's the same per-block hot path: - perf: buildSequences allocated and zero-initialized a fresh 131,072-entry (512 KB) hash-chain head table on every call. Before multi-block encoding this ran once per encode(); now it runs once per 128 KiB block, so a multi-MB input pays that allocation dozens of times over (measured: a 3 MB/24-block input did ~3.1M wasted writes purely re-establishing a table encode() could hand down once). Matching is still reset per block by design (no cross-block matching), so the O(n) clear itself is unavoidable -- this just moves the allocation out of the loop, into encode(), reusing one instance across every block via head.fill(-1) instead of a fresh array literal each time. - docs: encodeBlock's doc comment claimed "ties go to the simpler form" (RLE), but the actual selection only picks RLE when strictly smaller than a Compressed candidate -- an equal-size Compressed block wins the tie. Corrected to describe what the code does. - docs: TestVectors.largeLogRecords' doc comment still referenced the "128 KiB single-block limit" language removed elsewhere in this PR now that a block is a chunk within a multi-block frame, not an encoder-wide cap. Signed-off-by: James Rich --- .../meshtastic/kzstd/internal/ZstdEncoder.kt | 47 ++++++++++++++++--- .../org/meshtastic/kzstd/TestVectors.kt | 5 +- .../meshtastic/kzstd/MultiBlockInteropTest.kt | 25 ++++++++++ 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt index 37baecd..8c6c342 100644 --- a/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt +++ b/src/commonMain/kotlin/org/meshtastic/kzstd/internal/ZstdEncoder.kt @@ -66,6 +66,17 @@ internal object PureZstdEncoder { // so 128 KiB is the binding half of that minimum. private const val MAX_BLOCK_SIZE = 1 shl 17 // 128 KiB + // libzstd's ZSTD_WINDOWLOG_LIMIT_DEFAULT: the largest windowLog a conformant + // decoder accepts without an explicit opt-in raise (ZSTD_d_windowLogMax). + // windowDescriptor() always declares a window covering the FULL history (dict + // + input), so total history beyond this bound would produce a frame most + // real-world libzstd consumers reject outright -- violating this codec's own + // "frames stay libzstd-interoperable in both directions" invariant. This + // replaces the old single-block 128 KiB input guard as the encoder's size + // ceiling now that multi-block encoding lifts that guard. + private const val MAX_WINDOW_LOG = 27 + private const val MAX_HISTORY_SIZE = 1L shl MAX_WINDOW_LOG // 128 MiB + /** * Encode [data] into a complete zstd frame using [dict] (parsed entropy * tables + content) and its [index] as match history. [level] governs @@ -90,6 +101,13 @@ internal object PureZstdEncoder { level: Int = 19, checksum: Boolean = false, ): ByteArray { + val historySize = dict.content.size.toLong() + data.size.toLong() + if (historySize > MAX_HISTORY_SIZE) { + throw ZstdException( + "input ${data.size} bytes (+ ${dict.content.size} dictionary) would require a window " + + "beyond libzstd's default decompression limit of 2^$MAX_WINDOW_LOG bytes", + ) + } val depth = searchDepthFor(level) val out = ArrayList(data.size + 20) FRAME_MAGIC.forEach { out.add(it) } @@ -97,19 +115,27 @@ internal object PureZstdEncoder { // The window must span the whole frame's back-reference history, not just // one block's: a later block's dictionary match reaches back past every // block before it. - out.add(windowDescriptor(dict.content.size + data.size)) + out.add(windowDescriptor(historySize.toInt())) // Per-FRAME entropy state, seeded from the dictionary and carried from // block to block. Held in a local: this singleton keeps no mutable state. val state = FrameEntropy(dict) + // Per-block match-index scratch space, allocated once and reused across + // every block in this call (rather than fresh per block): matching is + // deliberately reset per chunk (no cross-block matching, see buildSequences), + // so this still needs a full clear before each block, but reusing one + // instance avoids a fresh 128K-entry allocation (and the JVM's zero-init + // of it) on every one of a multi-MB input's blocks. + val matchHead = IntArray(HASH_SIZE) + // Cut the input into Block_Maximum_Size chunks, one block each. `do/while` // rather than `while`, so an empty input still emits the one (empty, // Last_Block) block a frame must have. var start = 0 do { val end = minOf(start + MAX_BLOCK_SIZE, data.size) - encodeBlock(out, data, start, end, lastBlock = end == data.size, dict, index, depth, state) + encodeBlock(out, data, start, end, lastBlock = end == data.size, dict, index, depth, state, matchHead) start = end } while (start < data.size) @@ -133,8 +159,10 @@ internal object PureZstdEncoder { * * All three carry the same 3-byte header, so the smallest block is simply the * one with the smallest payload: 1 byte for RLE (only when every byte of the - * chunk is identical), the chunk length for Raw, or the compressed body. Ties - * go to the simpler form. + * chunk is identical), the chunk length for Raw, or the compressed body. RLE + * wins only when strictly smaller than any Compressed candidate; an + * equal-size Compressed block is preferred, since it's already built and + * decodes to the same content either way. * * [state] is the frame's running entropy state. Encoding this block moves it * -- sequences rotate the repeat offsets, and any table this block describes @@ -154,9 +182,10 @@ internal object PureZstdEncoder { index: MatchIndex, depth: SearchDepth, state: FrameEntropy, + matchHead: IntArray, ) { val chunk = data.copyOfRange(start, end) - val program = buildSequences(chunk, dict, index, depth, priorBytes = start) + val program = buildSequences(chunk, dict, index, depth, priorBytes = start, head = matchHead) val blockState = state.copy() val compressedBlock = encodeCompressedBlock(program, chunk, blockState) @@ -328,6 +357,7 @@ internal object PureZstdEncoder { index: MatchIndex, depth: SearchDepth, priorBytes: Int, + head: IntArray, ): Program { val dictContent = dict.content val dictLen = dictContent.size @@ -340,8 +370,11 @@ internal object PureZstdEncoder { // Per-input hash chain (continues the dict's chain). head/prev index the // combined history. We only INSERT input positions here; dict positions - // live in the cached index. - val head = IntArray(HASH_SIZE) { -1 } + // live in the cached index. [head] is caller-owned scratch space reused + // across blocks (matching is reset per chunk regardless, see the class + // doc above, so it still needs a full clear -- this just avoids a fresh + // allocation per block). + head.fill(-1) val prev = IntArray(n) { -1 } fun hashAt(p: Int): Int { diff --git a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt index 32d62ed..34054e9 100644 --- a/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt +++ b/src/commonTest/kotlin/org/meshtastic/kzstd/TestVectors.kt @@ -228,8 +228,9 @@ internal object TestVectors { * thousand) to push the literal-length and match-length Accuracy_Log to the * format's ceiling, and far more than 1023 literals, so this is the * "raw literals + a fresh FSE table on every stream" combination, with the - * longest table descriptions the encoder can write. Sized to stay under the - * 128 KiB single-block limit. + * longest table descriptions the encoder can write. Sized to stay within one + * 128 KiB block (Block_Maximum_Size), so the case is exercised without also + * spanning a multi-block chain. */ val largeLogRecords: ByteArray = buildRecords(400) diff --git a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt index feb60a2..edbaff5 100644 --- a/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt +++ b/src/jvmTest/kotlin/org/meshtastic/kzstd/MultiBlockInteropTest.kt @@ -6,6 +6,7 @@ import kotlin.random.Random import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals +import kotlin.test.assertFailsWith import kotlin.test.assertTrue import com.github.luben.zstd.Zstd as LibZstd @@ -167,4 +168,28 @@ class MultiBlockInteropTest { } return sb.toString().encodeToByteArray().copyOf(size) } + + /** + * Lifting the single-block cap removed the only ceiling on total history + * size. Without a replacement, the encoder would declare a windowLog beyond + * libzstd's default decompression limit (`ZSTD_WINDOWLOG_LIMIT_DEFAULT` = + * 27, i.e. 128 MiB) for large-enough input — a frame real-world libzstd + * consumers reject by default (confirmed below: even a real libzstd + * decompressor with no explicit window-log override refuses it), breaking + * this codec's own libzstd-interoperability invariant. `encode()` now + * rejects that case up front instead of silently emitting a + * non-interoperable frame. 128 MiB+ is JVM-only (not commonTest) for the + * same reason as the rest of this file: too large to allocate on all + * thirteen targets. + */ + @Test + fun inputBeyondTheDefaultWindowLogLimitIsRejected() { + val overLimit = (1L shl 27) + 1 // one byte past 128 MiB + val data = ByteArray(overLimit.toInt()) + val error = assertFailsWith { Zstd.compress(data) } + assertTrue( + error.message.orEmpty().contains("window"), + "expected a window-size error, got: ${error.message}", + ) + } } From 2adc660a1544fc58161ff6b5dbe54d42aad5c269 Mon Sep 17 00:00:00 2001 From: James Rich Date: Mon, 17 Aug 2026 16:04:04 -0500 Subject: [PATCH 11/11] docs: document the 128 MiB total-history ceiling README, CHANGELOG and AGENTS.md still described compress() as accepting input of any size once the single-block cap lifted -- update all three for the 128 MiB replacement ceiling from the previous commit. Signed-off-by: James Rich --- AGENTS.md | 8 ++++++++ CHANGELOG.md | 19 ++++++++++++------- README.md | 13 +++++++++---- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dcb42c5..0885386 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,14 @@ would be over-engineering here. `FrameEntropy` must keep mirroring `PureZstdDecoder`'s `DecodeState` exactly, or a mode names one table on each side and the frame decodes to garbage that only a real-libzstd oracle catches. +- **Total history (dictionary content + input) is capped at 128 MiB.** The + frame header always declares a window covering the full history; beyond + 128 MiB that window exceeds libzstd's default decompression limit + (`ZSTD_WINDOWLOG_LIMIT_DEFAULT` = windowLog 27), so `encode()` rejects it + with a `ZstdException` up front rather than emit a frame most real-world + libzstd consumers refuse. Keep this guard — it's what "frames stay + libzstd-interoperable in both directions" actually requires now that + multi-block encoding has no other size limit. - **No shared mutable state, no lock.** A `ZstdDictionary` digests its dictionary once in its constructor and is immutable thereafter; the engine objects keep all per-call state in locals. Do not reintroduce global caches. The encoder's diff --git a/CHANGELOG.md b/CHANGELOG.md index d7f64ed..796dd3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,13 +30,18 @@ public-API changes, except where noted below. true, the encoder sets `Content_Checksum_Flag` and appends the XXH64 checksum of the input. Defaults to false, so every existing call site's frame bytes are unchanged. -- `Zstd.compress` now accepts input up to 128 MiB. It cuts the input into - 128 KiB (`Block_Maximum_Size`) chunks and emits them as one multi-block - frame, `Last_Block` set on the final block only; anything larger than - 128 KiB used to be rejected with a `ZstdException`. `Zstd.decompress` has - always read multi-block frames from any encoder, and real libzstd reads - these. 3 MB of synthetic JSON telemetry compresses to 556,657 bytes across - 24 blocks — between libzstd's level 3 (579,374) and level 19 (362,967). +- `Zstd.compress` now accepts input up to 128 MiB (dictionary content + data + combined), up from 128 KiB. It cuts the input into 128 KiB + (`Block_Maximum_Size`) chunks and emits them as one multi-block frame, + `Last_Block` set on the final block only; anything larger than 128 KiB used + to be rejected with a `ZstdException`. The new 128 MiB ceiling replaces that + guard: beyond it, the frame's required window would exceed libzstd's default + decompression limit (`ZSTD_WINDOWLOG_LIMIT_DEFAULT`), so it's still rejected + rather than emitting a frame most real-world libzstd consumers would refuse. + `Zstd.decompress` has always read multi-block frames from any encoder, and + real libzstd reads these. 3 MB of synthetic JSON telemetry compresses to + 556,657 bytes across 24 blocks — between libzstd's level 3 (579,374) and + level 19 (362,967). ### Changed diff --git a/README.md b/README.md index 6dd1b2d..4709126 100644 --- a/README.md +++ b/README.md @@ -67,13 +67,18 @@ val back = Zstd.decompress(small, dict, maxSize = 64 * 1024) level does search more candidate matches per position, which can shrink output at the cost of more work. Frames remain fully libzstd-compatible at every level. - **Blocks are compressed independently (no cross-block matching).** `Zstd.compress` - takes an input of any size, cutting it into zstd's 128 KiB `Block_Maximum_Size` - chunks and emitting one multi-block frame. Each chunk is matched only against - itself and the dictionary, never against an earlier block's output, so a large - input compresses less well than a windowed encoder manages — 3 MB of synthetic + cuts input into zstd's 128 KiB `Block_Maximum_Size` chunks and emits one + multi-block frame. Each chunk is matched only against itself and the + dictionary, never against an earlier block's output, so a large input + compresses less well than a windowed encoder manages — 3 MB of synthetic JSON telemetry lands between libzstd's levels 3 and 19. Entropy tables and the repeat offsets ARE carried across blocks. A windowed matcher is a planned improvement. +- **Total input (dictionary content + data) is capped at 128 MiB.** Beyond that, + the window a frame must declare exceeds libzstd's default decompression limit + (`ZSTD_WINDOWLOG_LIMIT_DEFAULT`), so `Zstd.compress` throws `ZstdException` + rather than emit a frame most real-world libzstd consumers would refuse to + decode. - **Huffman-coded literals are single-stream and directly described.** The encoder builds a Huffman table from a block's own literals, but writes only the single-stream layout (so it applies to at most 1023 bytes of literals per block)