Never publish this fork

Halkyon Learn uses this fork as a git dependency, from
https://code.halkyon.app/project-halkyon/pure-zstd, until upstream decodes frames
without a content size. The package name pure_zstd belongs to upstream on pub.dev,
so `dart pub publish` should refuse outright rather than depend on pub.dev refusing
the upload.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [tagadvance]

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
-## 0.1.0
+## Unreleased
+
+- Decodes a frame that declares no content size without `expectedSize`, as
+  streaming encoders write them (Anki's packages, for one): the output is
+  allocated at the most the frame's blocks can hold, 128 KiB each, found from
+  their headers, and trimmed to what they held. `expectedSize` still checks
+  the size exactly when given.
+
+## 1.0.0
 
 First release.
 
@@ -23,8 +31,8 @@ one frame per call.
 
 Correctness is measured against libzstd through pyzstd rather than against
 expectations written by hand: 2,615 blocks of real elevation data across 32
-containers, plus a committed corpus of 24 fixtures covering the format branches
+containers, plus a committed corpus of 23 cases covering the format branches
-that real data never reaches. 103 tests.
+that real data never reaches. 131 tests.
 
 Roughly four times slower than native zstd. On a Pixel 10 Pro XL, profile AOT,
 a 128 KiB block decodes in 0.414 ms, which is 302 MB/s.

README.md

@@ -1,5 +1,11 @@
 # pure_zstd
 
+[![CI](https://github.com/tagadvance/pure-zstd/actions/workflows/ci.yml/badge.svg)](https://github.com/tagadvance/pure-zstd/actions/workflows/ci.yml)
+[![pub package](https://img.shields.io/pub/v/pure_zstd.svg)](https://pub.dev/packages/pure_zstd)
+[![pub points](https://img.shields.io/pub/points/pure_zstd)](https://pub.dev/packages/pure_zstd/score)
+[![Dart](https://img.shields.io/badge/dart-%3E%3D3.8-blue)](https://dart.dev)
+[![License: MIT](https://img.shields.io/github/license/tagadvance/pure-zstd)](LICENSE)
+
 A zstd decompressor in plain Dart, with no native dependency.
 
 It exists because every zstd binding on pub.dev is a wrapper over libzstd, and
@@ -26,8 +32,11 @@ for (final frame in frames) {
 }
 ```
 
-A frame that declares no content size needs `expectedSize`, because the
+The output is allocated whole rather than grown. A frame that declares no
-output is allocated whole rather than grown.
+content size, as streaming encoders write them, gets the most its blocks can
+hold (128 KiB each, found from their headers alone) and is trimmed to what
+they held; `expectedSize` turns that into an exact check when the caller
+knows the size.
 
 ## What it implements
 
@@ -96,10 +105,6 @@ Run it yourself:
 
     dart compile exe benchmark/bench.dart -o /tmp/bench && /tmp/bench
 
-## Licence
-
-MIT. See [LICENSE](LICENSE).
-
 ## Acknowledgements
 
 This is an independent implementation. It contains no code from libzstd, and
@@ -161,3 +166,7 @@ Do not benchmark against `test/fixtures/N51E000.hgtz`. Its samples are a
 synthetic pattern that compresses to almost no sequences, and decode cost
 tracks sequence count, so it reads about six times too fast. That mistake was
 made once already.
+
+---
+
+If this saved you some time, please consider [sponsoring the work](https://github.com/sponsors/tagadvance).

benchmark/levels.dart

@@ -1,91 +0,0 @@
-// Times several extracted-block files against each other, interleaved, so that
-// CPU frequency drift lands on every file equally.
-//
-//     dart compile exe levels.dart -o /tmp/levels && /tmp/levels a.blocks b.blocks
-//
-// Each file is the format benchmark/same_blocks.dart reads: uint32 count, then
-// per block a uint32 frame length, a uint32 decompressed size, and the frame.
-// Every file must hold the same number of blocks, of the same output size, so
-// the medians are like for like.
-import 'dart:io';
-import 'dart:typed_data';
-
-import 'package:pure_zstd/pure_zstd.dart';
-
-const int warmSeconds = 4;
-const int passes = 60;
-
-List<Uint8List> load(String path, List<int> sizes) {
-  final bytes = File(path).readAsBytesSync();
-  final data = ByteData.sublistView(bytes);
-  final count = data.getUint32(0, Endian.big);
-  final frames = <Uint8List>[];
-  var at = 4;
-  for (var i = 0; i < count; i++) {
-    final length = data.getUint32(at, Endian.big);
-    final size = data.getUint32(at + 4, Endian.big);
-    if (sizes.length <= i) {
-      sizes.add(size);
-    } else if (sizes[i] != size) {
-      throw StateError(
-        '$path block $i is $size bytes out, expected ${sizes[i]}',
-      );
-    }
-    at += 8;
-    frames.add(Uint8List.sublistView(bytes, at, at + length));
-    at += length;
-  }
-  return frames;
-}
-
-void main(List<String> args) {
-  final sizes = <int>[];
-  final sets = args.map((path) => load(path, sizes)).toList();
-  for (final set in sets) {
-    if (set.length != sets.first.length) {
-      throw StateError('the files do not hold the same number of blocks');
-    }
-    for (var i = 0; i < set.length; i++) {
-      if (zstdDecode(set[i]).length != sizes[i]) {
-        throw StateError('block $i decoded to the wrong size');
-      }
-    }
-  }
-
-  // Spin long enough for the governor to raise the clock before anything is
-  // timed, otherwise the first file measured pays for the ramp.
-  final warm = Stopwatch()..start();
-  while (warm.elapsed.inSeconds < warmSeconds) {
-    for (final set in sets) {
-      for (final frame in set) {
-        zstdDecode(frame);
-      }
-    }
-  }
-
-  final micros = [for (final _ in sets) <int>[]];
-  for (var pass = 0; pass < passes; pass++) {
-    for (var b = 0; b < sets.first.length; b++) {
-      for (var s = 0; s < sets.length; s++) {
-        final watch = Stopwatch()..start();
-        zstdDecode(sets[s][b]);
-        watch.stop();
-        micros[s].add(watch.elapsedMicroseconds);
-      }
-    }
-  }
-
-  for (var s = 0; s < sets.length; s++) {
-    final xs = micros[s]..sort();
-    double q(double f) => xs[(xs.length * f).floor()] / 1000;
-    final median = q(0.5);
-    final bytes = sizes.reduce((a, b) => a + b) / sizes.length;
-    stdout.writeln(
-      '${args[s].split('/').last.padRight(24)} n=${xs.length} '
-      'median=${median.toStringAsFixed(3)}ms '
-      'p10=${q(0.1).toStringAsFixed(3)} p90=${q(0.9).toStringAsFixed(3)} '
-      'MBps=${(bytes / 1048576 / (median / 1000)).toStringAsFixed(0)} '
-      'sweep113=${(median * 113).toStringAsFixed(0)}ms',
-    );
-  }
-}

benchmark/same_blocks.dart

@@ -1,51 +0,0 @@
-// Times the same extracted blocks the on-device benchmark uses, so the two
-// numbers are like for like.
-import 'dart:io';
-import 'dart:typed_data';
-
-import 'package:pure_zstd/pure_zstd.dart';
-
-void main(List<String> args) {
-  final bytes = File(args.first).readAsBytesSync();
-  final data = ByteData.sublistView(bytes);
-  final count = data.getUint32(0, Endian.big);
-  final frames = <Uint8List>[];
-  final sizes = <int>[];
-  var at = 4;
-  for (var i = 0; i < count; i++) {
-    final length = data.getUint32(at, Endian.big);
-    sizes.add(data.getUint32(at + 4, Endian.big));
-    at += 8;
-    frames.add(Uint8List.sublistView(bytes, at, at + length));
-    at += length;
-  }
-  for (var i = 0; i < frames.length; i++) {
-    if (zstdDecode(frames[i]).length != sizes[i]) {
-      throw StateError('block $i decoded to the wrong size');
-    }
-  }
-  for (var pass = 0; pass < 20; pass++) {
-    for (final frame in frames) {
-      zstdDecode(frame);
-    }
-  }
-  final micros = <int>[];
-  for (var pass = 0; pass < 60; pass++) {
-    for (final frame in frames) {
-      final watch = Stopwatch()..start();
-      zstdDecode(frame);
-      watch.stop();
-      micros.add(watch.elapsedMicroseconds);
-    }
-  }
-  micros.sort();
-  double q(double f) => micros[(micros.length * f).floor()] / 1000;
-  final median = q(0.5);
-  stdout.writeln(
-    'samples=${micros.length} bytesOut=${sizes.first} '
-    'median=${median.toStringAsFixed(3)}ms p10=${q(0.1).toStringAsFixed(3)} '
-    'p90=${q(0.9).toStringAsFixed(3)} '
-    'MBps=${(sizes.first / 1048576 / (median / 1000)).toStringAsFixed(0)} '
-    'sweep113=${(median * 113).toStringAsFixed(0)}ms',
-  );
-}

lib/pure_zstd.dart

@@ -1,7 +1,7 @@
 /// A pure Dart zstd decompressor.
 ///
-/// Decompression only, and only the subset of RFC 8878 that the elevation
+/// Decompression only, one RFC 8878 frame per call, on native platforms. It
-/// containers this was written for use. See the README for what is left out.
+/// refuses dictionaries and skippable frames; see the README for the rest.
 library;
 
 export 'src/decoder.dart' show ZstdDecoder, zstdDecode;

lib/src/decoder.dart

@@ -13,33 +13,17 @@ import 'xxhash.dart';
 /// One instance is not safe to share between isolates, and a frame must be
 /// decoded to completion before the next one starts.
 class ZstdDecoder {
+  /// Creates a decoder that refuses any frame larger than [maxOutputSize].
   ZstdDecoder({this.maxOutputSize = defaultMaxOutputSize});
 
+  /// The four bytes every zstd frame starts with, read little-endian.
   static const int magic = 0xFD2FB528;
 
-  /// The largest output this decoder will allocate for one frame.
+  /// The largest output a decoder allocates for one frame unless told
+  /// otherwise, since a frame header can claim up to 2^64 bytes.
   ///
-  /// `Frame_Content_Size` is up to a 64-bit number taken straight from the
+  /// The top-level [zstdDecode] always uses it; raising it takes a
-  /// input and it used to size the output buffer unchecked, so a sixteen-byte
+  /// [ZstdDecoder] of one's own.
-  /// frame could ask for four exbibytes. It is reachable by accident as well
-  /// as by design: one bit flipped in a real tile's frame descriptor moves
-  /// the content-size flag from 2 to 3, eight bytes of compressed data are
-  /// read as the size, and the eighth lands on the sign bit. That is a
-  /// `RangeError` for a negative length, or an `OutOfMemoryError` for a large
-  /// positive one, and both are `Error` rather than `Exception`, so a caller
-  /// writing `on ZstdException` does not catch either and the isolate dies.
-  ///
-  /// RFC 8878 section 3.1.1.1.2 lets a decoder refuse a frame that asks for
-  /// more memory than it is willing to give, which is the job of the window
-  /// descriptor this package skips. Allocating the output whole is why it can
-  /// skip it, and this is the bound that replaces it.
-  ///
-  /// A caller that knows the exact size should pass [decode]'s `expectedSize`
-  /// as well, and gets an exact check *in addition to* this ceiling rather
-  /// than instead of it: a frame declaring 70 MiB is refused here before the
-  /// two are ever compared. Raising it takes a `ZstdDecoder` of one's own,
-  /// since the top-level [zstdDecode] does not offer it. The `.hgtz` reader
-  /// knows its size: a block is `height * width * 2`.
   static const int defaultMaxOutputSize = 64 * 1024 * 1024;
 
   /// The ceiling this instance enforces. See [defaultMaxOutputSize].
@@ -74,10 +58,15 @@ class ZstdDecoder {
   int _literalAt = 0;
   int _literalEnd = 0;
 
-  /// Decodes one frame.
+  /// Decodes exactly one frame, which must be the whole of [src], into a new
+  /// list.
   ///
-  /// [expectedSize] is only needed when the frame header omits its content
+  /// [expectedSize], when given, must match the frame's content size
-  /// size, which the containers this was written for never do.
+  /// exactly. A frame that omits its content size, as streaming encoders
+  /// write them, needs none: its output is allocated at the most its blocks
+  /// can hold, Block_Maximum_Size each, and trimmed to what they held.
+  /// Throws a [ZstdException] for any frame that is malformed, too large,
+  /// fails its checksum, or uses a dictionary.
   Uint8List decode(Uint8List src, {int? expectedSize}) {
     if (src.length < 6) {
       throw const ZstdException('too short to be a zstd frame');
@@ -124,12 +113,10 @@ class ZstdDecoder {
       at += contentSizeBytes;
     }
 
-    final size = contentSize ?? expectedSize;
+    // Without a size from the frame or the caller, the most its blocks can
-    if (size == null) {
+    // hold, so the output is still allocated once; trimmed at the end.
-      throw const ZstdException(
+    final bounded = contentSize == null && expectedSize == null;
-        'frame declares no content size and none was given',
+    final size = contentSize ?? expectedSize ?? _blockBound(src, at);
-      );
-    }
     // Before the allocation, and before any of it is trusted.
     if (size < 0 || size > maxOutputSize) {
       throw ZstdException('frame declares $size bytes, over the maximum');
@@ -151,9 +138,7 @@ class ZstdDecoder {
     _repeat2 = 4;
     _repeat3 = 8;
 
-    // Only when the frame carries one. Hashing every frame would cost the
+    // Only when the frame carries one, so a frame without pays nothing.
-    // .hgtz reader throughput it has no use for, since pyzstd writes no
-    // checksum and the container's own SHA-256 is verified on download.
     final hash = checksum == 1 ? (_hash..reset()) : null;
 
     var written = 0;
@@ -225,7 +210,7 @@ class ZstdDecoder {
       }
       at += 4;
     }
-    if (written != out.length) {
+    if (!bounded && written != out.length) {
       throw ZstdException('frame decoded to $written bytes, not ${out.length}');
     }
     // Reading one frame is the whole API, so anything after it is input this
@@ -237,7 +222,27 @@ class ZstdDecoder {
       throw ZstdException('${src.length - at} bytes after the frame');
     }
 
-    return out;
+    return bounded ? Uint8List.sublistView(out, 0, written) : out;
+  }
+
+  /// The most the blocks from [at] can decode to: [maxBlockSize] each, as
+  /// RFC 8878 bounds every block's output. Only their headers are read, each
+  /// skipping its block's content, so this costs one step per block.
+  static int _blockBound(Uint8List src, int at) {
+    var blocks = 0;
+    while (true) {
+      if (at + 3 > src.length) {
+        throw const ZstdException('truncated block header');
+      }
+      final header = src[at] | (src[at + 1] << 8) | (src[at + 2] << 16);
+      final type = (header >> 1) & 3;
+      // An RLE block's content is its one byte, whatever its Block_Size.
+      at += 3 + (type == 1 ? 1 : header >> 3);
+      blocks++;
+      if (header & 1 == 1) {
+        return blocks * maxBlockSize;
+      }
+    }
   }
 
   /// Decodes one compressed block, returning the new output position.
@@ -674,7 +679,7 @@ class ZstdDecoder {
 
 /// Decodes one zstd frame into a new list.
 ///
-/// Allocates a decoder per call. Decoding many frames, which is what reading
+/// Allocates a decoder per call, so decoding many frames should hold a
-/// a tile does, should hold a [ZstdDecoder] instead.
+/// [ZstdDecoder] instead. Throws as [ZstdDecoder.decode] does.
 Uint8List zstdDecode(Uint8List src, {int? expectedSize}) =>
     ZstdDecoder().decode(src, expectedSize: expectedSize);

lib/src/exception.dart

@@ -1,7 +1,12 @@
 /// Thrown when a frame is malformed, or uses a feature this decoder omits.
+///
+/// Every refusal from this package is one of these, so `on ZstdException` is
+/// enough to survive untrusted input.
 class ZstdException implements Exception {
+  /// Creates an exception carrying [message].
   const ZstdException(this.message);
 
+  /// What was wrong with the frame, for a log rather than for matching on.
   final String message;
 
   @override

pubspec.yaml

@@ -2,7 +2,9 @@ name: pure_zstd
 description: >-
   A zstd decompressor written in plain Dart. Decodes RFC 8878 frames with no
   native library and no platform channel, verified against libzstd.
-version: 0.1.0
+version: 1.0.0
+# Halkyon Learn's fork, used as a git dependency; pure_zstd on pub.dev is upstream's.
+publish_to: none
 
 repository: https://github.com/tagadvance/pure-zstd
 issue_tracker: https://github.com/tagadvance/pure-zstd/issues

test/corpus_test.dart

@@ -121,5 +121,31 @@ void main() {
         );
       });
     }
+
+    // Streaming encoders write frames like this one, and their callers often
+    // can't know the size: an Anki package's collection, for one.
+    test('decodes without being told its size, to exactly what it holds', () {
+      final out = ZstdDecoder().decode(entry.frame);
+      expect(out.length, size);
+      expect(out, orderedEquals(entry.plain));
+    });
+
+    test('is refused, not crashed on, when cut short', () {
+      for (final cut in [1, 3, entry.frame.length ~/ 2]) {
+        expect(
+          () => ZstdDecoder().decode(
+            Uint8List.sublistView(entry.frame, 0, entry.frame.length - cut),
+          ),
+          throwsA(isA<ZstdException>()),
+        );
+      }
+    });
+
+    test('is refused when its blocks could hold more than allowed', () {
+      expect(
+        () => ZstdDecoder(maxOutputSize: size).decode(entry.frame),
+        throwsA(isA<ZstdException>()),
+      );
+    });
   });
 }

test/libzstd_shapes_test.dart

@@ -65,9 +65,7 @@ void main() {
     ),
   };
 
-  group(
+  group('a frame libzstd wrote decodes back to what went in', () {
-    'a frame libzstd wrote decodes back to what went in',
-    () {
     for (final shape in shapes.entries) {
       test(shape.key, () {
         final frame = zstdCli(shape.value.arguments, shape.value.input);
@@ -76,7 +74,5 @@ void main() {
         expect(out, orderedEquals(shape.value.input));
       }, timeout: const Timeout(Duration(minutes: 2)));
     }
-    },
+  }, skip: hasZstdCli ? null : 'the zstd CLI is not on the path');
-    skip: hasZstdCli ? null : 'the zstd CLI is not on the path',
-  );
 }