From 57aa53206959f88f87d01818757916bdbdfaf50a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tizian=20=E3=80=8C=E3=83=86=E3=82=A3=E3=83=84=E3=82=A3?= =?UTF-8?q?=E3=82=A2=E3=83=BC=E3=83=B3=E3=80=8D?= Date: Fri, 2 Oct 2026 04:28:06 +0200 Subject: [PATCH] Decode frames that declare no content size Streaming encoders leave Frame_Content_Size out unless told the size up front, and their readers often can't know it either: every zstd frame in an Anki package (its collection, its media list, each media file) is written that way. Such a frame needed the exact size passed as expectedSize, and anything else was refused. Now it gets the most its blocks can hold, Block_Maximum_Size (128 KiB) each, found by walking the block headers alone, and the output is trimmed to what the blocks held. The output is still allocated once. expectedSize keeps its exact check when given, and maxOutputSize bounds the allocation as before. Checked against libzstd (python-zstandard) on Anki 26.09.3's exports: a small package's four frames and a 30 MB collection decode byte for byte the same. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++++ README.md | 7 +++++-- lib/src/decoder.dart | 43 ++++++++++++++++++++++++++++++++----------- test/corpus_test.dart | 26 ++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 668fd23..4abfc63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased + +- Decodes a frame that declares no content size without `expectedSize`, as + streaming encoders write them (Anki's packages, for one): the output is + allocated at the most the frame's blocks can hold, 128 KiB each, found from + their headers, and trimmed to what they held. `expectedSize` still checks + the size exactly when given. + ## 1.0.0 First release. diff --git a/README.md b/README.md index e9a220c..cfead66 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,11 @@ for (final frame in frames) { } ``` -A frame that declares no content size needs `expectedSize`, because the -output is allocated whole rather than grown. +The output is allocated whole rather than grown. A frame that declares no +content size, as streaming encoders write them, gets the most its blocks can +hold (128 KiB each, found from their headers alone) and is trimmed to what +they held; `expectedSize` turns that into an exact check when the caller +knows the size. ## What it implements diff --git a/lib/src/decoder.dart b/lib/src/decoder.dart index 5c30786..d6b387d 100644 --- a/lib/src/decoder.dart +++ b/lib/src/decoder.dart @@ -61,9 +61,12 @@ class ZstdDecoder { /// Decodes exactly one frame, which must be the whole of [src], into a new /// list. /// - /// [expectedSize] is required when the frame omits its content size, and - /// otherwise must match it exactly. Throws a [ZstdException] for any frame - /// that is malformed, too large, fails its checksum, or uses a dictionary. + /// [expectedSize], when given, must match the frame's content size + /// exactly. A frame that omits its content size, as streaming encoders + /// write them, needs none: its output is allocated at the most its blocks + /// can hold, Block_Maximum_Size each, and trimmed to what they held. + /// Throws a [ZstdException] for any frame that is malformed, too large, + /// fails its checksum, or uses a dictionary. Uint8List decode(Uint8List src, {int? expectedSize}) { if (src.length < 6) { throw const ZstdException('too short to be a zstd frame'); @@ -110,12 +113,10 @@ class ZstdDecoder { at += contentSizeBytes; } - final size = contentSize ?? expectedSize; - if (size == null) { - throw const ZstdException( - 'frame declares no content size and none was given', - ); - } + // Without a size from the frame or the caller, the most its blocks can + // hold, so the output is still allocated once; trimmed at the end. + final bounded = contentSize == null && expectedSize == null; + final size = contentSize ?? expectedSize ?? _blockBound(src, at); // Before the allocation, and before any of it is trusted. if (size < 0 || size > maxOutputSize) { throw ZstdException('frame declares $size bytes, over the maximum'); @@ -209,7 +210,7 @@ class ZstdDecoder { } at += 4; } - if (written != out.length) { + if (!bounded && written != out.length) { throw ZstdException('frame decoded to $written bytes, not ${out.length}'); } // Reading one frame is the whole API, so anything after it is input this @@ -221,7 +222,27 @@ class ZstdDecoder { throw ZstdException('${src.length - at} bytes after the frame'); } - return out; + return bounded ? Uint8List.sublistView(out, 0, written) : out; + } + + /// The most the blocks from [at] can decode to: [maxBlockSize] each, as + /// RFC 8878 bounds every block's output. Only their headers are read, each + /// skipping its block's content, so this costs one step per block. + static int _blockBound(Uint8List src, int at) { + var blocks = 0; + while (true) { + if (at + 3 > src.length) { + throw const ZstdException('truncated block header'); + } + final header = src[at] | (src[at + 1] << 8) | (src[at + 2] << 16); + final type = (header >> 1) & 3; + // An RLE block's content is its one byte, whatever its Block_Size. + at += 3 + (type == 1 ? 1 : header >> 3); + blocks++; + if (header & 1 == 1) { + return blocks * maxBlockSize; + } + } } /// Decodes one compressed block, returning the new output position. diff --git a/test/corpus_test.dart b/test/corpus_test.dart index d025dc9..6b25736 100644 --- a/test/corpus_test.dart +++ b/test/corpus_test.dart @@ -121,5 +121,31 @@ void main() { ); }); } + + // Streaming encoders write frames like this one, and their callers often + // can't know the size: an Anki package's collection, for one. + test('decodes without being told its size, to exactly what it holds', () { + final out = ZstdDecoder().decode(entry.frame); + expect(out.length, size); + expect(out, orderedEquals(entry.plain)); + }); + + test('is refused, not crashed on, when cut short', () { + for (final cut in [1, 3, entry.frame.length ~/ 2]) { + expect( + () => ZstdDecoder().decode( + Uint8List.sublistView(entry.frame, 0, entry.frame.length - cut), + ), + throwsA(isA()), + ); + } + }); + + test('is refused when its blocks could hold more than allowed', () { + expect( + () => ZstdDecoder(maxOutputSize: size).decode(entry.frame), + throwsA(isA()), + ); + }); }); }