diff --git a/CHANGELOG.md b/CHANGELOG.md index 668fd23..4abfc63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Unreleased + +- Decodes a frame that declares no content size without `expectedSize`, as + streaming encoders write them (Anki's packages, for one): the output is + allocated at the most the frame's blocks can hold, 128 KiB each, found from + their headers, and trimmed to what they held. `expectedSize` still checks + the size exactly when given. + ## 1.0.0 First release. diff --git a/README.md b/README.md index e9a220c..cfead66 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,11 @@ for (final frame in frames) { } ``` -A frame that declares no content size needs `expectedSize`, because the -output is allocated whole rather than grown. +The output is allocated whole rather than grown. A frame that declares no +content size, as streaming encoders write them, gets the most its blocks can +hold (128 KiB each, found from their headers alone) and is trimmed to what +they held; `expectedSize` turns that into an exact check when the caller +knows the size. ## What it implements diff --git a/lib/src/decoder.dart b/lib/src/decoder.dart index 5c30786..d6b387d 100644 --- a/lib/src/decoder.dart +++ b/lib/src/decoder.dart @@ -61,9 +61,12 @@ class ZstdDecoder { /// Decodes exactly one frame, which must be the whole of [src], into a new /// list. /// - /// [expectedSize] is required when the frame omits its content size, and - /// otherwise must match it exactly. Throws a [ZstdException] for any frame - /// that is malformed, too large, fails its checksum, or uses a dictionary. + /// [expectedSize], when given, must match the frame's content size + /// exactly. A frame that omits its content size, as streaming encoders + /// write them, needs none: its output is allocated at the most its blocks + /// can hold, Block_Maximum_Size each, and trimmed to what they held. + /// Throws a [ZstdException] for any frame that is malformed, too large, + /// fails its checksum, or uses a dictionary. Uint8List decode(Uint8List src, {int? expectedSize}) { if (src.length < 6) { throw const ZstdException('too short to be a zstd frame'); @@ -110,12 +113,10 @@ class ZstdDecoder { at += contentSizeBytes; } - final size = contentSize ?? expectedSize; - if (size == null) { - throw const ZstdException( - 'frame declares no content size and none was given', - ); - } + // Without a size from the frame or the caller, the most its blocks can + // hold, so the output is still allocated once; trimmed at the end. + final bounded = contentSize == null && expectedSize == null; + final size = contentSize ?? expectedSize ?? _blockBound(src, at); // Before the allocation, and before any of it is trusted. if (size < 0 || size > maxOutputSize) { throw ZstdException('frame declares $size bytes, over the maximum'); @@ -209,7 +210,7 @@ class ZstdDecoder { } at += 4; } - if (written != out.length) { + if (!bounded && written != out.length) { throw ZstdException('frame decoded to $written bytes, not ${out.length}'); } // Reading one frame is the whole API, so anything after it is input this @@ -221,7 +222,27 @@ class ZstdDecoder { throw ZstdException('${src.length - at} bytes after the frame'); } - return out; + return bounded ? Uint8List.sublistView(out, 0, written) : out; + } + + /// The most the blocks from [at] can decode to: [maxBlockSize] each, as + /// RFC 8878 bounds every block's output. Only their headers are read, each + /// skipping its block's content, so this costs one step per block. + static int _blockBound(Uint8List src, int at) { + var blocks = 0; + while (true) { + if (at + 3 > src.length) { + throw const ZstdException('truncated block header'); + } + final header = src[at] | (src[at + 1] << 8) | (src[at + 2] << 16); + final type = (header >> 1) & 3; + // An RLE block's content is its one byte, whatever its Block_Size. + at += 3 + (type == 1 ? 1 : header >> 3); + blocks++; + if (header & 1 == 1) { + return blocks * maxBlockSize; + } + } } /// Decodes one compressed block, returning the new output position. diff --git a/test/corpus_test.dart b/test/corpus_test.dart index d025dc9..6b25736 100644 --- a/test/corpus_test.dart +++ b/test/corpus_test.dart @@ -121,5 +121,31 @@ void main() { ); }); } + + // Streaming encoders write frames like this one, and their callers often + // can't know the size: an Anki package's collection, for one. + test('decodes without being told its size, to exactly what it holds', () { + final out = ZstdDecoder().decode(entry.frame); + expect(out.length, size); + expect(out, orderedEquals(entry.plain)); + }); + + test('is refused, not crashed on, when cut short', () { + for (final cut in [1, 3, entry.frame.length ~/ 2]) { + expect( + () => ZstdDecoder().decode( + Uint8List.sublistView(entry.frame, 0, entry.frame.length - cut), + ), + throwsA(isA()), + ); + } + }); + + test('is refused when its blocks could hold more than allowed', () { + expect( + () => ZstdDecoder(maxOutputSize: size).decode(entry.frame), + throwsA(isA()), + ); + }); }); }