diff --git a/.gitignore b/.gitignore
index 3150b40..f8dd747 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,7 @@
.dart_tool/
.packages
build/
-# If you're building an application, you may want to check-in your pubspec.lock
+# A library, not an application: the lockfile is the consumer's business.
pubspec.lock
# Directory created by dartdoc
diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 0000000..4b43bed
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,30 @@
+# Changelog
+
+## 0.1.0
+
+First release.
+
+A zstd decompressor in plain Dart. Decompression only, native platforms only,
+one frame per call.
+
+- Decodes RFC 8878 frames: raw, RLE and compressed blocks; raw, RLE, compressed
+ and treeless literals in one-stream and four-stream form; Huffman weights in
+ both the direct nibble and FSE-coded forms; predefined, RLE, FSE and repeat
+ sequence tables for all three symbols; repeated offsets including the
+ no-literals shift; multi-block frames reusing an earlier block's tables.
+- Verifies the content checksum when a frame carries one, with XXH64 in
+ `lib/src/xxhash.dart`.
+- Refuses dictionaries, several frames in one buffer, and any input left over
+ after a frame.
+- Bounds what a frame header can make it allocate. `ZstdDecoder` takes a
+ `maxOutputSize`, 64 MiB by default, and a caller passing `expectedSize` gets
+ an exact check against the frame's own declared size.
+- Every failure is a `ZstdException`.
+
+Correctness is measured against libzstd through pyzstd rather than against
+expectations written by hand: 2,615 blocks of real elevation data across 32
+containers, plus a committed corpus of 24 fixtures covering the format branches
+that real data never reaches. 103 tests.
+
+Roughly four times slower than native zstd. On a Pixel 10 Pro XL, profile AOT,
+a 128 KiB block decodes in 0.414 ms, which is 302 MB/s.
diff --git a/README.md b/README.md
index 03b5897..38a68fe 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,163 @@
-# pure-zstd
\ No newline at end of file
+# pure_zstd
+
+A zstd decompressor in plain Dart, with no native dependency.
+
+It exists because every zstd binding on pub.dev is a wrapper over libzstd, and
+an app that reads block-compressed elevation tiles on a phone would rather not
+carry a native library per platform to do it. Decompression only. There is no
+compressor and there is no plan for one.
+
+It decodes the real data it was written for, correctly and completely, and it
+is roughly four times slower than native zstd. See **Speed** below before
+depending on it.
+
+## Using it
+
+```dart
+import 'package:pure_zstd/pure_zstd.dart';
+
+final plain = zstdDecode(frame);
+
+// Decoding many frames: hold the decoder, which keeps its entropy tables and
+// scratch buffers between calls and then allocates only the output.
+final decoder = ZstdDecoder();
+for (final frame in frames) {
+ use(decoder.decode(frame));
+}
+```
+
+A frame that declares no content size needs `expectedSize`, because the
+output is allocated whole rather than grown.
+
+## What it implements
+
+RFC 8878 is the specification. Of it:
+
+| Part of the format | State |
+| --- | --- |
+| Frame header, single segment and windowed | Decoded. The window descriptor is skipped, since the output is allocated from the content size. |
+| Raw, RLE and compressed blocks | All three. |
+| Multi-block frames | Yes, including a later block reusing an earlier one's tables and recent offsets. |
+| Raw, RLE, compressed and treeless literals | All four, in one-stream and four-stream form. |
+| Huffman weights | Both the direct nibble form and the FSE-coded form. |
+| Sequence tables | Predefined, RLE, FSE-compressed and repeat, for all three of literal length, match length and offset. |
+| Repeated offsets | Yes, including the no-literals shift. |
+| Content checksum | Verified. XXH64 over the output, compared with the stored low 32 bits. Hashed only when the frame carries one. |
+| Dictionaries | Refused. |
+| Skippable frames, several frames in one buffer | Refused. Reading one frame is the whole API, and input left over after it is an error. |
+| Streaming, or decoding part of a frame | Not offered. |
+
+It targets native Dart and not the web: the literals header reads a five-byte
+field with a 32-bit shift, and the bit reader carries 56 bits in an `int`.
+Both are exact on the VM and on AOT, and neither is on JavaScript.
+
+## Correctness
+
+The reference is pyzstd, which wraps libzstd. `tool/oracle.py` extracts every
+block of a `.hgtz` container as its own frame, decompresses it with pyzstd,
+hands it to `tool/verify.dart`, and compares byte for byte.
+
+ ./tool/oracle.py
+ ./tool/oracle.py --every 1700
+
+As of 2026-09-21 that is 32 real containers, 2,615 blocks and 0.3 GB of
+output, all identical, spanning both the one and three arc-second sets, every
+thinned column count from 3601 down to 361, and latitudes from 88 S to 80 N.
+
+`dart test` needs no Python. It decodes the five blocks of the container in
+`test/fixtures/`, whose every sample is a known function of its position, so a
+wrong decode cannot pass.
+
+## Speed
+
+Measured 2026-09-21 on this eight-core Linux box, `dart compile exe`, Dart
+3.12.2. One 256 by 256 block is 131,072 bytes out. The frames are read into
+memory first, so only decoding is timed, and the figure includes allocating
+the output buffer because the API allocates it.
+
+| Block set | pure_zstd, AOT | pyzstd (libzstd) |
+| --- | --- | --- |
+| Colorado, `N39W105`, 196 blocks, median | **0.66 ms** | 0.18 ms |
+| Alaska, `N60W160`, 98 blocks, median | 0.94 ms | not measured |
+| One block, repeated 1000 times | 0.85 ms | 0.22 ms |
+| 113 blocks, a 360 degree sweep | 67 ms | 19 ms |
+
+That is 140 to 200 MB/s of output. The JIT is about 25% slower than AOT, so
+measure with `dart compile exe`.
+
+The cost is almost all in the sequence loop rather than in Huffman literals.
+A block of real terrain carries around 1,700 literals and 14,400 sequences,
+so the hot path is three FSE state updates, six bit reads and a nine-byte
+match copy, about 46 ns per sequence against native's dozen or so. The
+headroom left is bounds checks and the byte-at-a-time match copy; widening
+that copy to 64-bit words, the way libzstd does, is the next thing to try.
+
+Run it yourself:
+
+ dart compile exe benchmark/bench.dart -o /tmp/bench && /tmp/bench
+
+## Licence
+
+MIT. See [LICENSE](LICENSE).
+
+## Acknowledgements
+
+This is an independent implementation. It contains no code from libzstd, and
+nothing about its licence reaches this package. It would not exist without the
+following, and they are named here because that is where the debt actually is.
+
+**[RFC 8878](https://www.rfc-editor.org/rfc/rfc8878.html), "Zstandard
+Compression and the application/zstd Media Type"**, by Yann Collet and Murray
+Kucherawy, is the specification this was written from. Every table in
+`lib/src/tables.dart` was checked against section 3.1.1.3.2 entry by entry, and
+the FSE and Huffman decoders follow the reference algorithms it describes. A
+format specification written clearly enough to implement from is a gift, and
+this one is.
+
+**[Zstandard](https://github.com/facebook/zstd)**, by Yann Collet and
+contributors at Meta, is the format and the reference implementation. Its
+licence is BSD-3-Clause, and it obliges nothing here; it is named because
+without it there would be nothing to decode.
+
+**[pyzstd](https://pypi.org/project/pyzstd/)** wraps libzstd and is the oracle
+every test in this package is measured against. Correctness here means
+"agrees with libzstd, byte for byte", and pyzstd is how that is asked. The
+committed fixtures under `test/fixtures/corpus/` are output of the `zstd` CLI
+and of pyzstd, which is tool output rather than anything derived from their
+source.
+
+**The Copernicus Programme.** One committed fixture,
+`test/fixtures/corpus/direct_nibble.zst`, is a block of real terrain, kept
+because it is the only small input that exercises the direct-nibble Huffman
+weight form. Its licence requires this to travel with it:
+
+> Produced using Copernicus WorldDEM-30 © DLR e.V. 2010-2014 and © Airbus Defence and Space GmbH 2014-2018 provided under COPERNICUS by the European Union and ESA; all rights reserved.
+
+**The Dart team**, for making `int` an exact 64-bit integer on the VM and on
+AOT. The bit reader carries 56 bits in one and XXH64's multiplies wrap exactly,
+neither of which would be true on a platform with doubles underneath.
+
+## Measured on the target device, 2026-09-21
+
+Sixteen real 256x256 blocks lifted out of one elevation container,
+131,072 bytes of output each, the same blocks in every row so the numbers are
+comparable. Median of 960 timed calls after 320 warm-up calls.
+
+| where | mode | median | MB/s | 113-block sweep |
+| --- | --- | --- | --- | --- |
+| Pixel 10 Pro XL | **profile, AOT** | **0.414 ms** | 302 | **47 ms** |
+| Pixel 10 Pro XL | debug, JIT | 0.540 ms | 231 | 61 ms |
+| this laptop | AOT | 0.917 ms | 136 | 104 ms |
+| this laptop | JIT | 1.125 ms | 111 | 127 ms |
+
+The phone is about twice as fast as the laptop at this, which is why the
+desktop figures in the commit message read worse than reality.
+
+Native, for scale: pyzstd on the laptop is 0.175 ms on comparable blocks. That
+is a Python C extension rather than Dart FFI, so take it as an indication. No
+native-on-phone figure was taken.
+
+Do not benchmark against `test/fixtures/N51E000.hgtz`. Its samples are a
+synthetic pattern that compresses to almost no sequences, and decode cost
+tracks sequence count, so it reads about six times too fast. That mistake was
+made once already.
diff --git a/analysis_options.yaml b/analysis_options.yaml
new file mode 100644
index 0000000..572dd23
--- /dev/null
+++ b/analysis_options.yaml
@@ -0,0 +1 @@
+include: package:lints/recommended.yaml
diff --git a/benchmark/bench.dart b/benchmark/bench.dart
new file mode 100644
index 0000000..2536569
--- /dev/null
+++ b/benchmark/bench.dart
@@ -0,0 +1,125 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+/// Times decoding real `.hgtz` blocks.
+///
+/// dart run benchmark/bench.dart [path to a .hgtz]
+/// dart compile exe benchmark/bench.dart -o /tmp/bench && /tmp/bench
+///
+/// Reports the cost of one 256 by 256 block, which is 131,072 bytes out, and
+/// the cost of the 113 blocks a 360 degree skyline sweep touches. The frames
+/// are read into memory first, so nothing but decoding is timed. The output
+/// buffer is allocated per call, as the API allocates it, so the figure
+/// includes that allocation and the garbage it makes.
+const int sweepBlocks = 113;
+
+void main(List arguments) {
+ // A .hgtz container of real terrain. There is no useful default: the
+ // committed fixture is synthetic and compresses to almost no sequences, so
+ // it reads about six times too fast and is not worth timing.
+ final path = arguments.isEmpty
+ ? (throw ArgumentError('usage: bench '))
+ : arguments.single;
+ final bytes = File(path).readAsBytesSync();
+ final header = ByteData.sublistView(bytes);
+ final rows = header.getUint16(6);
+ final columns = header.getUint16(8);
+ final block = header.getUint16(10);
+ final blockColumns = (columns + block - 1) ~/ block;
+ final blockRows = (rows + block - 1) ~/ block;
+
+ // Only the square blocks, so every timing is over the same 131,072 bytes.
+ final full = [];
+ for (var index = 0; index < blockRows * blockColumns; index++) {
+ final entry = 12 + 8 * index;
+ final top = (index ~/ blockColumns) * block;
+ final left = (index % blockColumns) * block;
+ if (rows - top < block || columns - left < block) {
+ continue;
+ }
+ final offset = header.getUint32(entry);
+ full.add(
+ Uint8List.sublistView(
+ bytes,
+ offset,
+ offset + header.getUint32(entry + 4),
+ ),
+ );
+ }
+ if (full.isEmpty) {
+ stderr.writeln('$path has no full $block by $block block');
+ exit(2);
+ }
+
+ final decoder = ZstdDecoder();
+ final size = decoder.decode(full.first).length;
+ stdout.writeln(path);
+ stdout.writeln(
+ '${full.length} full blocks of $size bytes, '
+ '${(full.map((f) => f.length).reduce((a, b) => a + b) / full.length).round()}'
+ ' compressed bytes on average',
+ );
+
+ // One block, over and over. The same frame every time, so the caches are as
+ // warm as they will ever be. This is the optimistic figure.
+ final one = full[full.length ~/ 2];
+ for (var i = 0; i < 300; i++) {
+ decoder.decode(one);
+ }
+ final repeated = [];
+ for (var i = 0; i < 1000; i++) {
+ final watch = Stopwatch()..start();
+ decoder.decode(one);
+ watch.stop();
+ repeated.add(watch.elapsedMicroseconds);
+ }
+ _report('one block repeated, warm', repeated, size);
+
+ // Every full block once, which is the mix a real sweep meets: different
+ // entropy tables, different literal counts, different sequence counts.
+ final each = [];
+ for (var pass = 0; pass < 8; pass++) {
+ for (final frame in full) {
+ final watch = Stopwatch()..start();
+ decoder.decode(frame);
+ watch.stop();
+ if (pass > 0) {
+ each.add(watch.elapsedMicroseconds);
+ }
+ }
+ }
+ _report('every full block, warm', each, size);
+
+ // A sweep as one unit, since that is the latency the user feels.
+ final sweeps = [];
+ for (var pass = 0; pass < 30; pass++) {
+ final watch = Stopwatch()..start();
+ for (var i = 0; i < sweepBlocks; i++) {
+ decoder.decode(full[i % full.length]);
+ }
+ watch.stop();
+ sweeps.add(watch.elapsedMicroseconds);
+ }
+ sweeps.sort();
+ final sweep = sweeps[sweeps.length ~/ 2] / 1000;
+ stdout.writeln(
+ '$sweepBlocks block sweep, median ${sweep.toStringAsFixed(2)} ms '
+ '(${(sweep / sweepBlocks * 1000).toStringAsFixed(0)} us per block)',
+ );
+}
+
+void _report(String label, List microseconds, int size) {
+ microseconds.sort();
+ final median = microseconds[microseconds.length ~/ 2];
+ final low = microseconds[microseconds.length ~/ 10];
+ final high = microseconds[microseconds.length * 9 ~/ 10];
+ final mbPerSecond = size / median; // bytes per microsecond is MB/s.
+ stdout.writeln(
+ '${label.padRight(28)} ${(median / 1000).toStringAsFixed(3)} ms '
+ 'median (p10 ${(low / 1000).toStringAsFixed(3)}, '
+ 'p90 ${(high / 1000).toStringAsFixed(3)}, '
+ 'n=${microseconds.length}) ${mbPerSecond.toStringAsFixed(0)} MB/s',
+ );
+}
diff --git a/benchmark/levels.dart b/benchmark/levels.dart
new file mode 100644
index 0000000..3f08627
--- /dev/null
+++ b/benchmark/levels.dart
@@ -0,0 +1,91 @@
+// Times several extracted-block files against each other, interleaved, so that
+// CPU frequency drift lands on every file equally.
+//
+// dart compile exe levels.dart -o /tmp/levels && /tmp/levels a.blocks b.blocks
+//
+// Each file is the format benchmark/same_blocks.dart reads: uint32 count, then
+// per block a uint32 frame length, a uint32 decompressed size, and the frame.
+// Every file must hold the same number of blocks, of the same output size, so
+// the medians are like for like.
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+const int warmSeconds = 4;
+const int passes = 60;
+
+List load(String path, List sizes) {
+ final bytes = File(path).readAsBytesSync();
+ final data = ByteData.sublistView(bytes);
+ final count = data.getUint32(0, Endian.big);
+ final frames = [];
+ var at = 4;
+ for (var i = 0; i < count; i++) {
+ final length = data.getUint32(at, Endian.big);
+ final size = data.getUint32(at + 4, Endian.big);
+ if (sizes.length <= i) {
+ sizes.add(size);
+ } else if (sizes[i] != size) {
+ throw StateError(
+ '$path block $i is $size bytes out, expected ${sizes[i]}',
+ );
+ }
+ at += 8;
+ frames.add(Uint8List.sublistView(bytes, at, at + length));
+ at += length;
+ }
+ return frames;
+}
+
+void main(List args) {
+ final sizes = [];
+ final sets = args.map((path) => load(path, sizes)).toList();
+ for (final set in sets) {
+ if (set.length != sets.first.length) {
+ throw StateError('the files do not hold the same number of blocks');
+ }
+ for (var i = 0; i < set.length; i++) {
+ if (zstdDecode(set[i]).length != sizes[i]) {
+ throw StateError('block $i decoded to the wrong size');
+ }
+ }
+ }
+
+ // Spin long enough for the governor to raise the clock before anything is
+ // timed, otherwise the first file measured pays for the ramp.
+ final warm = Stopwatch()..start();
+ while (warm.elapsed.inSeconds < warmSeconds) {
+ for (final set in sets) {
+ for (final frame in set) {
+ zstdDecode(frame);
+ }
+ }
+ }
+
+ final micros = [for (final _ in sets) []];
+ for (var pass = 0; pass < passes; pass++) {
+ for (var b = 0; b < sets.first.length; b++) {
+ for (var s = 0; s < sets.length; s++) {
+ final watch = Stopwatch()..start();
+ zstdDecode(sets[s][b]);
+ watch.stop();
+ micros[s].add(watch.elapsedMicroseconds);
+ }
+ }
+ }
+
+ for (var s = 0; s < sets.length; s++) {
+ final xs = micros[s]..sort();
+ double q(double f) => xs[(xs.length * f).floor()] / 1000;
+ final median = q(0.5);
+ final bytes = sizes.reduce((a, b) => a + b) / sizes.length;
+ stdout.writeln(
+ '${args[s].split('/').last.padRight(24)} n=${xs.length} '
+ 'median=${median.toStringAsFixed(3)}ms '
+ 'p10=${q(0.1).toStringAsFixed(3)} p90=${q(0.9).toStringAsFixed(3)} '
+ 'MBps=${(bytes / 1048576 / (median / 1000)).toStringAsFixed(0)} '
+ 'sweep113=${(median * 113).toStringAsFixed(0)}ms',
+ );
+ }
+}
diff --git a/benchmark/same_blocks.dart b/benchmark/same_blocks.dart
new file mode 100644
index 0000000..74bd159
--- /dev/null
+++ b/benchmark/same_blocks.dart
@@ -0,0 +1,51 @@
+// Times the same extracted blocks the on-device benchmark uses, so the two
+// numbers are like for like.
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+void main(List args) {
+ final bytes = File(args.first).readAsBytesSync();
+ final data = ByteData.sublistView(bytes);
+ final count = data.getUint32(0, Endian.big);
+ final frames = [];
+ final sizes = [];
+ var at = 4;
+ for (var i = 0; i < count; i++) {
+ final length = data.getUint32(at, Endian.big);
+ sizes.add(data.getUint32(at + 4, Endian.big));
+ at += 8;
+ frames.add(Uint8List.sublistView(bytes, at, at + length));
+ at += length;
+ }
+ for (var i = 0; i < frames.length; i++) {
+ if (zstdDecode(frames[i]).length != sizes[i]) {
+ throw StateError('block $i decoded to the wrong size');
+ }
+ }
+ for (var pass = 0; pass < 20; pass++) {
+ for (final frame in frames) {
+ zstdDecode(frame);
+ }
+ }
+ final micros = [];
+ for (var pass = 0; pass < 60; pass++) {
+ for (final frame in frames) {
+ final watch = Stopwatch()..start();
+ zstdDecode(frame);
+ watch.stop();
+ micros.add(watch.elapsedMicroseconds);
+ }
+ }
+ micros.sort();
+ double q(double f) => micros[(micros.length * f).floor()] / 1000;
+ final median = q(0.5);
+ stdout.writeln(
+ 'samples=${micros.length} bytesOut=${sizes.first} '
+ 'median=${median.toStringAsFixed(3)}ms p10=${q(0.1).toStringAsFixed(3)} '
+ 'p90=${q(0.9).toStringAsFixed(3)} '
+ 'MBps=${(sizes.first / 1048576 / (median / 1000)).toStringAsFixed(0)} '
+ 'sweep113=${(median * 113).toStringAsFixed(0)}ms',
+ );
+}
diff --git a/lib/pure_zstd.dart b/lib/pure_zstd.dart
new file mode 100644
index 0000000..833338e
--- /dev/null
+++ b/lib/pure_zstd.dart
@@ -0,0 +1,8 @@
+/// A pure Dart zstd decompressor.
+///
+/// Decompression only, and only the subset of RFC 8878 that the elevation
+/// containers this was written for use. See the README for what is left out.
+library;
+
+export 'src/decoder.dart' show ZstdDecoder, zstdDecode;
+export 'src/exception.dart' show ZstdException;
diff --git a/lib/src/bit_reader.dart b/lib/src/bit_reader.dart
new file mode 100644
index 0000000..bae6766
--- /dev/null
+++ b/lib/src/bit_reader.dart
@@ -0,0 +1,140 @@
+import 'dart:typed_data';
+
+import 'exception.dart';
+
+/// Reads a zstd entropy bitstream, which runs backwards.
+///
+/// Every FSE and Huffman stream in a zstd frame is written from its end: the
+/// last byte holds the first bits, and within a byte the most significant bit
+/// comes first. Read as a little-endian integer the whole stream is one big
+/// number consumed from the top down, which is what this does.
+///
+/// The final byte carries a set bit marking where the data stops; the zeroes
+/// above it are padding and the marker itself is not data.
+class ReverseBitReader {
+ ReverseBitReader();
+
+ Uint8List _bytes = Uint8List(0);
+
+ /// A view over [_bytes], so a refill can take four bytes at a time. Kept
+ /// between resets, since every stream in one block shares a buffer.
+ ByteData _view = ByteData(0);
+
+ /// Index of the first byte of the stream. Reading stops when it is reached.
+ int _start = 0;
+
+ /// Index of the next byte to pull into [_acc], counting down to [_start].
+ int _next = 0;
+
+ /// The unread bits, in the low [_available] bits, next bit at the top.
+ int _acc = 0;
+ int _available = 0;
+
+ /// Zero bits shifted in past the front of the stream. They are only ever
+ /// at the bottom of [_acc], so any unread count below this means the
+ /// stream was over-read.
+ int _padding = 0;
+
+ void reset(Uint8List bytes, int start, int end) {
+ if (end <= start) {
+ throw const ZstdException('empty entropy stream');
+ }
+ final last = bytes[end - 1];
+ if (last == 0) {
+ throw const ZstdException('entropy stream has no end marker');
+ }
+ if (!identical(bytes, _bytes)) {
+ _bytes = bytes;
+ _view = ByteData.sublistView(bytes);
+ }
+ _start = start;
+ _next = end - 1;
+ _acc = last;
+ // bitLength - 1 is the marker's position; the data is everything below.
+ _available = last.bitLength - 1;
+ _padding = 0;
+ }
+
+ /// True once more bits have been taken than the stream held.
+ bool get exhausted => _available < _padding;
+
+ /// Tops [_acc] up to at least 49 bits, so a [peek] of up to 32 is safe and
+ /// several small reads can follow one call.
+ void fill() {
+ if (_available > 48) {
+ return;
+ }
+ var acc = _acc & ((1 << _available) - 1);
+ var available = _available;
+ var next = _next;
+ // Four bytes at a time while there is room for them. The stream is one
+ // little-endian number read from the top down, so a little-endian word
+ // lands the four bytes in the order the byte-at-a-time loop would.
+ while (available <= 24 && next - 4 >= _start) {
+ next -= 4;
+ acc = (acc << 32) | _view.getUint32(next, Endian.little);
+ available += 32;
+ }
+ while (available <= 48) {
+ if (next > _start) {
+ next--;
+ acc = (acc << 8) | _bytes[next];
+ } else {
+ acc <<= 8;
+ _padding += 8;
+ }
+ available += 8;
+ }
+ _acc = acc;
+ _available = available;
+ _next = next;
+ }
+
+ /// The next [count] bits without consuming them. Caller has called [fill].
+ int peek(int count) => (_acc >> (_available - count)) & ((1 << count) - 1);
+
+ void skip(int count) {
+ _available -= count;
+ }
+
+ /// Reads [count] bits, refilling as needed. `count == 0` yields 0.
+ int read(int count) {
+ if (_available < count) {
+ fill();
+ }
+ _available -= count;
+ return (_acc >> _available) & ((1 << count) - 1);
+ }
+}
+
+/// Reads the forward, least-significant-bit-first bitstream that an FSE table
+/// description uses. Table descriptions are tiny and read once per block, so
+/// this favours being obviously correct over being quick.
+class ForwardBitReader {
+ ForwardBitReader(this._bytes, this._end, int startByte)
+ : _position = startByte * 8;
+
+ final Uint8List _bytes;
+ final int _end;
+ int _position;
+
+ /// The next 32 bits, unconsumed. Reads past the end come back as zeroes.
+ int peek32() {
+ final index = _position >> 3;
+ var value = 0;
+ for (var k = 0; k < 5; k++) {
+ final at = index + k;
+ if (at < _end) {
+ value |= _bytes[at] << (8 * k);
+ }
+ }
+ return value >> (_position & 7);
+ }
+
+ void skip(int count) {
+ _position += count;
+ }
+
+ /// The number of bytes consumed, counting a partial byte as whole.
+ int bytesUsed(int startByte) => ((_position + 7) >> 3) - startByte;
+}
diff --git a/lib/src/decoder.dart b/lib/src/decoder.dart
new file mode 100644
index 0000000..f9c2baf
--- /dev/null
+++ b/lib/src/decoder.dart
@@ -0,0 +1,680 @@
+import 'dart:typed_data';
+
+import 'bit_reader.dart';
+import 'exception.dart';
+import 'fse.dart';
+import 'huffman.dart';
+import 'tables.dart';
+import 'xxhash.dart';
+
+/// Decodes zstd frames, holding its entropy tables and scratch between calls
+/// so that decoding a second frame allocates nothing but the output.
+///
+/// One instance is not safe to share between isolates, and a frame must be
+/// decoded to completion before the next one starts.
+class ZstdDecoder {
+ ZstdDecoder({this.maxOutputSize = defaultMaxOutputSize});
+
+ static const int magic = 0xFD2FB528;
+
+ /// The largest output this decoder will allocate for one frame.
+ ///
+ /// `Frame_Content_Size` is up to a 64-bit number taken straight from the
+ /// input and it used to size the output buffer unchecked, so a sixteen-byte
+ /// frame could ask for four exbibytes. It is reachable by accident as well
+ /// as by design: one bit flipped in a real tile's frame descriptor moves
+ /// the content-size flag from 2 to 3, eight bytes of compressed data are
+ /// read as the size, and the eighth lands on the sign bit. That is a
+ /// `RangeError` for a negative length, or an `OutOfMemoryError` for a large
+ /// positive one, and both are `Error` rather than `Exception`, so a caller
+ /// writing `on ZstdException` does not catch either and the isolate dies.
+ ///
+ /// RFC 8878 section 3.1.1.1.2 lets a decoder refuse a frame that asks for
+ /// more memory than it is willing to give, which is the job of the window
+ /// descriptor this package skips. Allocating the output whole is why it can
+ /// skip it, and this is the bound that replaces it.
+ ///
+ /// A caller that knows the exact size should pass [decode]'s `expectedSize`
+ /// as well, and gets an exact check *in addition to* this ceiling rather
+ /// than instead of it: a frame declaring 70 MiB is refused here before the
+ /// two are ever compared. Raising it takes a `ZstdDecoder` of one's own,
+ /// since the top-level [zstdDecode] does not offer it. The `.hgtz` reader
+ /// knows its size: a block is `height * width * 2`.
+ static const int defaultMaxOutputSize = 64 * 1024 * 1024;
+
+ /// The ceiling this instance enforces. See [defaultMaxOutputSize].
+ final int maxOutputSize;
+
+ /// The largest block a frame may contain, and so the most literals one
+ /// block can hold.
+ static const int maxBlockSize = 128 * 1024;
+
+ final Xxh64 _hash = Xxh64();
+ final ReverseBitReader _bits = ReverseBitReader();
+ final HuffmanTable _huffman = HuffmanTable();
+ final FseTable _literalLengths = FseTable(literalLengthMaxLog);
+ final FseTable _matchLengths = FseTable(matchLengthMaxLog);
+ final FseTable _offsets = FseTable(offsetCodeMaxLog);
+
+ bool _literalLengthsReady = false;
+ bool _matchLengthsReady = false;
+ bool _offsetsReady = false;
+
+ /// The three recent offsets. They live for the whole frame, so a later
+ /// block can refer back to one a previous block established.
+ int _repeat1 = 1;
+ int _repeat2 = 4;
+ int _repeat3 = 8;
+
+ Uint8List _literals = Uint8List(maxBlockSize);
+
+ /// Where the current block's literals live: either [_literals] or, for raw
+ /// literals, the frame itself.
+ Uint8List _literalSource = Uint8List(0);
+ int _literalAt = 0;
+ int _literalEnd = 0;
+
+ /// Decodes one frame.
+ ///
+ /// [expectedSize] is only needed when the frame header omits its content
+ /// size, which the containers this was written for never do.
+ Uint8List decode(Uint8List src, {int? expectedSize}) {
+ if (src.length < 6) {
+ throw const ZstdException('too short to be a zstd frame');
+ }
+ if (src[0] | (src[1] << 8) | (src[2] << 16) | (src[3] << 24) != magic) {
+ throw const ZstdException('not a zstd frame: bad magic');
+ }
+
+ final descriptor = src[4];
+ var at = 5;
+ final contentSizeFlag = descriptor >> 6;
+ final singleSegment = (descriptor >> 5) & 1;
+ if ((descriptor >> 3) & 1 != 0) {
+ throw const ZstdException('frame header reserved bit set');
+ }
+ final checksum = (descriptor >> 2) & 1;
+ final dictionaryFlag = descriptor & 3;
+
+ if (singleSegment == 0) {
+ at++; // Window_Descriptor. The output is allocated whole, so unused.
+ }
+ if (dictionaryFlag != 0) {
+ throw const ZstdException('dictionaries are not supported');
+ }
+
+ final contentSizeBytes = switch (contentSizeFlag) {
+ 0 => singleSegment == 1 ? 1 : 0,
+ 1 => 2,
+ 2 => 4,
+ _ => 8,
+ };
+ if (at + contentSizeBytes > src.length) {
+ throw const ZstdException('truncated frame header');
+ }
+ int? contentSize;
+ if (contentSizeBytes > 0) {
+ var value = 0;
+ for (var k = 0; k < contentSizeBytes; k++) {
+ value |= src[at + k] << (8 * k);
+ }
+ // The two-byte form is biased, since a frame that small would use the
+ // one-byte form.
+ contentSize = contentSizeBytes == 2 ? value + 256 : value;
+ at += contentSizeBytes;
+ }
+
+ final size = contentSize ?? expectedSize;
+ if (size == null) {
+ throw const ZstdException(
+ 'frame declares no content size and none was given',
+ );
+ }
+ // Before the allocation, and before any of it is trusted.
+ if (size < 0 || size > maxOutputSize) {
+ throw ZstdException('frame declares $size bytes, over the maximum');
+ }
+ // When the caller knows the size, the frame agreeing with it is the
+ // check. Left to the end, a frame declaring less decoded its own
+ // content happily and a frame declaring more allocated whatever it
+ // asked for first.
+ if (contentSize != null && expectedSize != null && size != expectedSize) {
+ throw ZstdException('frame declares $size bytes, not $expectedSize');
+ }
+ final out = Uint8List(size);
+
+ _huffman.log = 0;
+ _literalLengthsReady = false;
+ _matchLengthsReady = false;
+ _offsetsReady = false;
+ _repeat1 = 1;
+ _repeat2 = 4;
+ _repeat3 = 8;
+
+ // Only when the frame carries one. Hashing every frame would cost the
+ // .hgtz reader throughput it has no use for, since pyzstd writes no
+ // checksum and the container's own SHA-256 is verified on download.
+ final hash = checksum == 1 ? (_hash..reset()) : null;
+
+ var written = 0;
+ while (true) {
+ final blockStart = written;
+ if (at + 3 > src.length) {
+ throw const ZstdException('truncated block header');
+ }
+ final header = src[at] | (src[at + 1] << 8) | (src[at + 2] << 16);
+ at += 3;
+ final last = header & 1;
+ final type = (header >> 1) & 3;
+ final blockSize = header >> 3;
+ // Every type, RLE included. Its Block_Size is its regenerated size and
+ // the format bounds it like any other, and exempting it let a 97-byte
+ // input return 44 MB that a caller cannot tell from a good tile.
+ if (blockSize > maxBlockSize) {
+ throw ZstdException('block of $blockSize bytes is over the maximum');
+ }
+
+ switch (type) {
+ case 0:
+ if (at + blockSize > src.length || written + blockSize > out.length) {
+ throw const ZstdException('raw block does not fit');
+ }
+ out.setRange(written, written + blockSize, src, at);
+ written += blockSize;
+ at += blockSize;
+ case 1:
+ if (at >= src.length || written + blockSize > out.length) {
+ throw const ZstdException('RLE block does not fit');
+ }
+ out.fillRange(written, written + blockSize, src[at]);
+ written += blockSize;
+ at += 1;
+ case 2:
+ if (at + blockSize > src.length) {
+ throw const ZstdException('truncated compressed block');
+ }
+ written = _decodeBlock(src, at, at + blockSize, out, written);
+ at += blockSize;
+ default:
+ throw const ZstdException('reserved block type');
+ }
+
+ hash?.update(out, blockStart, written);
+
+ if (last == 1) {
+ break;
+ }
+ }
+
+ if (checksum == 1) {
+ if (at + 4 > src.length) {
+ throw const ZstdException('truncated content checksum');
+ }
+ // The low 32 bits of XXH64 over the whole output, little-endian.
+ // Skipping it left the decoder with no integrity check at all: over
+ // 600,000 mutations of real block data, a third decoded into wrong
+ // bytes and raised nothing, because a damaged match or literal length
+ // usually still produces *some* plausible output of the right size.
+ final stored =
+ src[at] |
+ (src[at + 1] << 8) |
+ (src[at + 2] << 16) |
+ (src[at + 3] << 24);
+ if (stored != hash!.digest32) {
+ throw const ZstdException('the frame checksum does not match');
+ }
+ at += 4;
+ }
+ if (written != out.length) {
+ throw ZstdException('frame decoded to $written bytes, not ${out.length}');
+ }
+ // Reading one frame is the whole API, so anything after it is input this
+ // decoder does not understand. Nothing used to compare the cursor with
+ // the length, so a second frame, a trailing skippable frame, and four
+ // bytes of junk were all accepted and silently ignored: two frames
+ // concatenated returned the first one's bytes and raised nothing.
+ if (at != src.length) {
+ throw ZstdException('${src.length - at} bytes after the frame');
+ }
+
+ return out;
+ }
+
+ /// Decodes one compressed block, returning the new output position.
+ int _decodeBlock(
+ Uint8List src,
+ int start,
+ int end,
+ Uint8List out,
+ int written,
+ ) {
+ var at = _decodeLiterals(src, start, end);
+ if (at == end) {
+ throw const ZstdException('block has no sequences section');
+ }
+
+ // Number_of_Sequences, one to three bytes.
+ final first = src[at++];
+ int sequences;
+ if (first == 0) {
+ sequences = 0;
+ } else if (first < 128) {
+ sequences = first;
+ } else if (first < 255) {
+ if (at >= end) {
+ throw const ZstdException('truncated sequence count');
+ }
+ sequences = ((first - 128) << 8) + src[at++];
+ } else {
+ if (at + 2 > end) {
+ throw const ZstdException('truncated sequence count');
+ }
+ sequences = src[at] + (src[at + 1] << 8) + 0x7F00;
+ at += 2;
+ }
+
+ if (sequences == 0) {
+ return _copyRemainingLiterals(out, written);
+ }
+
+ if (at >= end) {
+ throw const ZstdException('truncated compression modes');
+ }
+ final modes = src[at++];
+ if (modes & 3 != 0) {
+ throw const ZstdException('sequence modes reserved bits set');
+ }
+ at = _readSequenceTable(
+ _literalLengths,
+ modes >> 6,
+ literalLengthDefault,
+ 6,
+ literalLengthMaxSymbol,
+ src,
+ at,
+ end,
+ _literalLengthsReady,
+ );
+ _literalLengthsReady = true;
+ at = _readSequenceTable(
+ _offsets,
+ (modes >> 4) & 3,
+ offsetCodeDefault,
+ 5,
+ offsetCodeMaxSymbol,
+ src,
+ at,
+ end,
+ _offsetsReady,
+ );
+ _offsetsReady = true;
+ at = _readSequenceTable(
+ _matchLengths,
+ (modes >> 2) & 3,
+ matchLengthDefault,
+ 6,
+ matchLengthMaxSymbol,
+ src,
+ at,
+ end,
+ _matchLengthsReady,
+ );
+ _matchLengthsReady = true;
+
+ return _executeSequences(src, at, end, out, written, sequences);
+ }
+
+ int _readSequenceTable(
+ FseTable table,
+ int mode,
+ List predefined,
+ int predefinedLog,
+ int maxSymbol,
+ Uint8List src,
+ int at,
+ int end,
+ bool ready,
+ ) {
+ switch (mode) {
+ case 0:
+ buildPredefined(table, predefined, predefinedLog);
+ return at;
+ case 1:
+ if (at >= end) {
+ throw const ZstdException('truncated RLE sequence table');
+ }
+ if (src[at] > maxSymbol) {
+ throw ZstdException('RLE sequence symbol ${src[at]} out of range');
+ }
+ table.setRle(src[at]);
+ return at + 1;
+ case 2:
+ return at + readFseTable(table, src, at, end, maxSymbol);
+ default:
+ if (!ready) {
+ throw const ZstdException('repeat mode with no previous table');
+ }
+ return at;
+ }
+ }
+
+ /// Reads the literals section, leaving the literals addressable through
+ /// [_literalSource]. Returns the offset just past the section.
+ int _decodeLiterals(Uint8List src, int start, int end) {
+ if (start >= end) {
+ throw const ZstdException('empty compressed block');
+ }
+ var at = start;
+ final header = src[at];
+ final type = header & 3;
+ final sizeFormat = (header >> 2) & 3;
+
+ if (type < 2) {
+ int regenerated;
+ if (sizeFormat & 1 == 0) {
+ regenerated = header >> 3;
+ at += 1;
+ } else if (sizeFormat == 1) {
+ if (at + 2 > end) {
+ throw const ZstdException('truncated literals header');
+ }
+ regenerated = (header >> 4) | (src[at + 1] << 4);
+ at += 2;
+ } else {
+ if (at + 3 > end) {
+ throw const ZstdException('truncated literals header');
+ }
+ regenerated = (header >> 4) | (src[at + 1] << 4) | (src[at + 2] << 12);
+ at += 3;
+ }
+ if (regenerated > maxBlockSize) {
+ throw ZstdException('$regenerated literals is over the maximum');
+ }
+ if (type == 0) {
+ if (at + regenerated > end) {
+ throw const ZstdException('truncated raw literals');
+ }
+ // No copy: the literals are already contiguous in the frame.
+ _literalSource = src;
+ _literalAt = at;
+ _literalEnd = at + regenerated;
+ return at + regenerated;
+ }
+ if (at >= end) {
+ throw const ZstdException('truncated RLE literals');
+ }
+ _literals.fillRange(0, regenerated, src[at]);
+ _literalSource = _literals;
+ _literalAt = 0;
+ _literalEnd = regenerated;
+ return at + 1;
+ }
+
+ final int regenerated;
+ final int compressed;
+ final int streams;
+ switch (sizeFormat) {
+ case 0:
+ case 1:
+ if (at + 3 > end) {
+ throw const ZstdException('truncated literals header');
+ }
+ final value = src[at] | (src[at + 1] << 8) | (src[at + 2] << 16);
+ regenerated = (value >> 4) & 0x3FF;
+ compressed = (value >> 14) & 0x3FF;
+ streams = sizeFormat == 0 ? 1 : 4;
+ at += 3;
+ case 2:
+ if (at + 4 > end) {
+ throw const ZstdException('truncated literals header');
+ }
+ final value =
+ src[at] |
+ (src[at + 1] << 8) |
+ (src[at + 2] << 16) |
+ (src[at + 3] << 24);
+ regenerated = (value >> 4) & 0x3FFF;
+ compressed = (value >> 18) & 0x3FFF;
+ streams = 4;
+ at += 4;
+ default:
+ if (at + 5 > end) {
+ throw const ZstdException('truncated literals header');
+ }
+ final value =
+ src[at] |
+ (src[at + 1] << 8) |
+ (src[at + 2] << 16) |
+ (src[at + 3] << 24) |
+ (src[at + 4] << 32);
+ regenerated = (value >> 4) & 0x3FFFF;
+ compressed = (value >> 22) & 0x3FFFF;
+ streams = 4;
+ at += 5;
+ }
+ if (regenerated > maxBlockSize) {
+ throw ZstdException('$regenerated literals is over the maximum');
+ }
+ if (at + compressed > end) {
+ throw const ZstdException('truncated compressed literals');
+ }
+ if (_literals.length < regenerated) {
+ _literals = Uint8List(regenerated);
+ }
+
+ var streamStart = at;
+ final streamEnd = at + compressed;
+ if (type == 2) {
+ streamStart += _huffman.read(src, at, streamEnd, _bits);
+ }
+
+ if (streams == 1) {
+ _huffman.decodeStream(
+ _bits,
+ src,
+ streamStart,
+ streamEnd,
+ _literals,
+ 0,
+ regenerated,
+ );
+ } else {
+ if (streamStart + 6 > streamEnd) {
+ throw const ZstdException('truncated literals jump table');
+ }
+ final one = src[streamStart] | (src[streamStart + 1] << 8);
+ final two = src[streamStart + 2] | (src[streamStart + 3] << 8);
+ final three = src[streamStart + 4] | (src[streamStart + 5] << 8);
+ var from = streamStart + 6;
+ final four = streamEnd - from - one - two - three;
+ if (four < 0) {
+ throw const ZstdException('literals jump table does not add up');
+ }
+ // The first three streams carry a quarter each, rounded up.
+ final segment = (regenerated + 3) >> 2;
+ final sizes = [one, two, three, four];
+ var outAt = 0;
+ for (var i = 0; i < 4; i++) {
+ final count = i == 3 ? regenerated - 3 * segment : segment;
+ if (count < 0) {
+ throw const ZstdException('literals do not split into four');
+ }
+ if (count > 0) {
+ _huffman.decodeStream(
+ _bits,
+ src,
+ from,
+ from + sizes[i],
+ _literals,
+ outAt,
+ count,
+ );
+ }
+ from += sizes[i];
+ outAt += count;
+ }
+ }
+
+ _literalSource = _literals;
+ _literalAt = 0;
+ _literalEnd = regenerated;
+
+ return streamEnd;
+ }
+
+ int _copyRemainingLiterals(Uint8List out, int written) {
+ final count = _literalEnd - _literalAt;
+ if (written + count > out.length) {
+ throw const ZstdException('literals overflow the frame content size');
+ }
+ out.setRange(written, written + count, _literalSource, _literalAt);
+ _literalAt = _literalEnd;
+
+ return written + count;
+ }
+
+ int _executeSequences(
+ Uint8List src,
+ int start,
+ int end,
+ Uint8List out,
+ int written,
+ int sequences,
+ ) {
+ final bits = _bits;
+ bits.reset(src, start, end);
+
+ final llSymbol = _literalLengths.symbol;
+ final llBits = _literalLengths.nbBits;
+ final llNext = _literalLengths.newState;
+ final mlSymbol = _matchLengths.symbol;
+ final mlBits = _matchLengths.nbBits;
+ final mlNext = _matchLengths.newState;
+ final ofSymbol = _offsets.symbol;
+ final ofBits = _offsets.nbBits;
+ final ofNext = _offsets.newState;
+
+ var llState = bits.read(_literalLengths.log);
+ var ofState = bits.read(_offsets.log);
+ var mlState = bits.read(_matchLengths.log);
+
+ final literals = _literalSource;
+ var literalAt = _literalAt;
+ final literalEnd = _literalEnd;
+ var at = written;
+
+ var repeat1 = _repeat1;
+ var repeat2 = _repeat2;
+ var repeat3 = _repeat3;
+
+ for (var n = 0; n < sequences; n++) {
+ // Every table's symbols were bounded when it was built, so a code out
+ // of range is impossible here and is not re-checked per sequence.
+ final llCode = llSymbol[llState];
+ final mlCode = mlSymbol[mlState];
+ final ofCode = ofSymbol[ofState];
+
+ // Additional bits come off the stream offset first, then match
+ // length, then literals length.
+ bits.fill();
+ final offsetValue = (1 << ofCode) + bits.read(ofCode);
+ final matchLength =
+ matchLengthBaseline[mlCode] + bits.read(matchLengthExtra[mlCode]);
+ final literalLength =
+ literalLengthBaseline[llCode] + bits.read(literalLengthExtra[llCode]);
+
+ if (n + 1 < sequences) {
+ // The three updates together need at most 26 bits, which one fill
+ // covers, so they peek rather than each testing for a refill.
+ bits.fill();
+ final llTake = llBits[llState];
+ llState = llNext[llState] + bits.peek(llTake);
+ bits.skip(llTake);
+ final mlTake = mlBits[mlState];
+ mlState = mlNext[mlState] + bits.peek(mlTake);
+ bits.skip(mlTake);
+ final ofTake = ofBits[ofState];
+ ofState = ofNext[ofState] + bits.peek(ofTake);
+ bits.skip(ofTake);
+ }
+
+ int offset;
+ if (offsetValue > 3) {
+ offset = offsetValue - 3;
+ repeat3 = repeat2;
+ repeat2 = repeat1;
+ repeat1 = offset;
+ } else {
+ // One to three is a reference to a recent offset, and a sequence
+ // with no literals shifts which one it means.
+ final which = offsetValue - 1 + (literalLength == 0 ? 1 : 0);
+ if (which == 0) {
+ offset = repeat1;
+ } else {
+ offset = which == 3 ? repeat1 - 1 : (which == 1 ? repeat2 : repeat3);
+ if (offset == 0) {
+ throw const ZstdException('repeated offset of zero');
+ }
+ if (which != 1) {
+ repeat3 = repeat2;
+ }
+ repeat2 = repeat1;
+ repeat1 = offset;
+ }
+ }
+
+ if (literalAt + literalLength > literalEnd) {
+ throw const ZstdException('sequence wants more literals than exist');
+ }
+ if (at + literalLength + matchLength > out.length) {
+ throw const ZstdException('sequences overflow the content size');
+ }
+ final from = at + literalLength - offset;
+ if (from < 0) {
+ throw const ZstdException('match reaches before the output');
+ }
+
+ if (literalLength > 0) {
+ if (literalLength < 16) {
+ for (var i = 0; i < literalLength; i++) {
+ out[at++] = literals[literalAt++];
+ }
+ } else {
+ out.setRange(at, at + literalLength, literals, literalAt);
+ at += literalLength;
+ literalAt += literalLength;
+ }
+ }
+
+ if (matchLength < 16 || offset < matchLength) {
+ // A short or overlapping match is quicker byte by byte than through
+ // a range copy, and the median match here is nine bytes.
+ var source = from;
+ for (var i = 0; i < matchLength; i++) {
+ out[at++] = out[source++];
+ }
+ } else {
+ out.setRange(at, at + matchLength, out, from);
+ at += matchLength;
+ }
+ }
+
+ if (bits.exhausted) {
+ throw const ZstdException('sequence stream ran past its end');
+ }
+
+ _literalAt = literalAt;
+ _repeat1 = repeat1;
+ _repeat2 = repeat2;
+ _repeat3 = repeat3;
+
+ return _copyRemainingLiterals(out, at);
+ }
+}
+
+/// Decodes one zstd frame into a new list.
+///
+/// Allocates a decoder per call. Decoding many frames, which is what reading
+/// a tile does, should hold a [ZstdDecoder] instead.
+Uint8List zstdDecode(Uint8List src, {int? expectedSize}) =>
+ ZstdDecoder().decode(src, expectedSize: expectedSize);
diff --git a/lib/src/exception.dart b/lib/src/exception.dart
new file mode 100644
index 0000000..f4496f6
--- /dev/null
+++ b/lib/src/exception.dart
@@ -0,0 +1,9 @@
+/// Thrown when a frame is malformed, or uses a feature this decoder omits.
+class ZstdException implements Exception {
+ const ZstdException(this.message);
+
+ final String message;
+
+ @override
+ String toString() => 'ZstdException: $message';
+}
diff --git a/lib/src/fse.dart b/lib/src/fse.dart
new file mode 100644
index 0000000..4f25923
--- /dev/null
+++ b/lib/src/fse.dart
@@ -0,0 +1,234 @@
+import 'dart:typed_data';
+
+import 'bit_reader.dart';
+import 'exception.dart';
+
+/// An FSE decoding table: one entry per state, giving the symbol that state
+/// emits, how many bits to read next, and the base the read is added to.
+///
+/// The three arrays are parallel and sized for the largest accuracy log the
+/// format allows, so a table is built in place and never reallocated.
+class FseTable {
+ FseTable(this.maxLog)
+ : symbol = Uint8List(1 << maxLog),
+ nbBits = Uint8List(1 << maxLog),
+ newState = Uint16List(1 << maxLog),
+ _spread = Uint8List(1 << maxLog);
+
+ final int maxLog;
+
+ final Uint8List symbol;
+ final Uint8List nbBits;
+ final Uint16List newState;
+
+ /// Accuracy log. Zero for an RLE table, which has a single state.
+ int log = 0;
+
+ /// The normalised distribution this table was last built from. Written by
+ /// [readFseTable] and [buildPredefined], read by [build].
+ final Int16List distribution = Int16List(256);
+
+ final Uint8List _spread;
+ final Int32List _next = Int32List(256);
+
+ /// Makes this a table with one state emitting [value] and reading no bits.
+ void setRle(int value) {
+ log = 0;
+ symbol[0] = value;
+ nbBits[0] = 0;
+ newState[0] = 0;
+ }
+
+ /// Builds the table from [distribution], where a count of -1 is the "less
+ /// than one in the table" symbol the format reserves.
+ void build(int maxSymbol, int accuracyLog) {
+ if (accuracyLog > maxLog) {
+ throw ZstdException('FSE accuracy log $accuracyLog above $maxLog');
+ }
+ final size = 1 << accuracyLog;
+ log = accuracyLog;
+
+ // Low-probability symbols go at the top of the table, working down, and
+ // then count as one when the states are numbered.
+ var highThreshold = size - 1;
+ for (var s = 0; s <= maxSymbol; s++) {
+ final count = distribution[s];
+ if (count == -1) {
+ _spread[highThreshold--] = s;
+ _next[s] = 1;
+ } else {
+ _next[s] = count;
+ }
+ }
+
+ // The step is coprime with the table size, so walking it visits every
+ // slot exactly once. Skipping the low-probability slots at the top is
+ // what keeps those symbols where they were just put.
+ final mask = size - 1;
+ final step = (size >> 1) + (size >> 3) + 3;
+ var position = 0;
+ for (var s = 0; s <= maxSymbol; s++) {
+ final count = distribution[s];
+ for (var i = 0; i < count; i++) {
+ _spread[position] = s;
+ position = (position + step) & mask;
+ while (position > highThreshold) {
+ position = (position + step) & mask;
+ }
+ }
+ }
+ if (position != 0) {
+ throw const ZstdException('FSE distribution does not fill its table');
+ }
+
+ for (var state = 0; state < size; state++) {
+ final s = _spread[state];
+ final nth = _next[s]++;
+ final bits = accuracyLog - (nth.bitLength - 1);
+ symbol[state] = s;
+ nbBits[state] = bits;
+ newState[state] = (nth << bits) - size;
+ }
+ }
+}
+
+/// Reads an FSE table description into [table.distribution] and builds it.
+///
+/// Returns the number of bytes the description occupied. The description is a
+/// forward, least-significant-bit-first bitstream: an accuracy log, then one
+/// value per symbol coded in as few bits as the remaining probability mass
+/// allows, with a repeat flag after any zero.
+int readFseTable(
+ FseTable table,
+ Uint8List src,
+ int start,
+ int end,
+ int maxSymbol,
+) {
+ final bits = ForwardBitReader(src, end, start);
+ final counts = table.distribution;
+ for (var s = 0; s <= maxSymbol; s++) {
+ counts[s] = 0;
+ }
+
+ final accuracyLog = (bits.peek32() & 0xF) + 5;
+ bits.skip(4);
+ if (accuracyLog > table.maxLog) {
+ throw ZstdException('FSE accuracy log $accuracyLog above ${table.maxLog}');
+ }
+
+ var remaining = (1 << accuracyLog) + 1;
+ var threshold = 1 << accuracyLog;
+ var nbBits = accuracyLog + 1;
+ var symbol = 0;
+ var previousZero = false;
+
+ while (remaining > 1 && symbol <= maxSymbol) {
+ if (previousZero) {
+ var repeat = 0;
+ while (true) {
+ final flag = bits.peek32() & 3;
+ bits.skip(2);
+ repeat += flag;
+ if (flag != 3) {
+ break;
+ }
+ }
+ while (repeat-- > 0) {
+ if (symbol > maxSymbol) {
+ throw const ZstdException('FSE zero run past the last symbol');
+ }
+ counts[symbol++] = 0;
+ }
+ previousZero = false;
+ continue;
+ }
+
+ final value = bits.peek32();
+ final small = (2 * threshold - 1) - remaining;
+ int count;
+ if ((value & (threshold - 1)) < small) {
+ count = value & (threshold - 1);
+ bits.skip(nbBits - 1);
+ } else {
+ count = value & (2 * threshold - 1);
+ if (count >= threshold) {
+ count -= small;
+ }
+ bits.skip(nbBits);
+ }
+
+ count -= 1;
+ remaining -= count < 0 ? -count : count;
+ counts[symbol++] = count;
+ previousZero = count == 0;
+ while (remaining < threshold) {
+ nbBits--;
+ threshold >>= 1;
+ }
+ }
+ if (remaining != 1) {
+ throw const ZstdException('FSE distribution is short');
+ }
+
+ table.build(symbol - 1, accuracyLog);
+
+ return bits.bytesUsed(start);
+}
+
+/// Builds [table] from one of the format's fixed distributions.
+void buildPredefined(FseTable table, List values, int accuracyLog) {
+ final counts = table.distribution;
+ for (var s = 0; s < values.length; s++) {
+ counts[s] = values[s];
+ }
+ table.build(values.length - 1, accuracyLog);
+}
+
+/// Decodes a whole FSE bitstream with the two interleaved states the format
+/// uses, appending symbols to [out] from [outStart] and returning the count.
+///
+/// The symbol count is not recorded anywhere, so decoding runs until the
+/// stream is spent. This is how the Huffman weights are coded.
+int fseDecodeInterleaved(
+ FseTable table,
+ ReverseBitReader bits,
+ Uint8List out,
+ int outStart,
+ int outLimit,
+) {
+ final symbol = table.symbol;
+ final nbBits = table.nbBits;
+ final newState = table.newState;
+ final log = table.log;
+
+ var state1 = bits.read(log);
+ var state2 = bits.read(log);
+ var at = outStart;
+
+ while (true) {
+ // Three, not two: both exhausted branches below write a third symbol in
+ // the same pass. Guarding for two let `at` reach outLimit exactly, which
+ // is one past the end of the caller's buffer, and a crafted weight
+ // stream turned that into an IndexError rather than a refusal.
+ if (at + 3 > outLimit) {
+ throw const ZstdException('FSE stream longer than its output allows');
+ }
+ out[at++] = symbol[state1];
+ bits.fill();
+ state1 = newState[state1] + bits.read(nbBits[state1]);
+ if (bits.exhausted) {
+ out[at++] = symbol[state2];
+ break;
+ }
+ out[at++] = symbol[state2];
+ bits.fill();
+ state2 = newState[state2] + bits.read(nbBits[state2]);
+ if (bits.exhausted) {
+ out[at++] = symbol[state1];
+ break;
+ }
+ }
+
+ return at - outStart;
+}
diff --git a/lib/src/huffman.dart b/lib/src/huffman.dart
new file mode 100644
index 0000000..f26aa53
--- /dev/null
+++ b/lib/src/huffman.dart
@@ -0,0 +1,208 @@
+import 'dart:typed_data';
+
+import 'bit_reader.dart';
+import 'exception.dart';
+import 'fse.dart';
+
+/// The Huffman table for a block's literals, flattened so that decoding one
+/// literal is a peek of [log] bits and one array index.
+///
+/// Weights, not code lengths, are what the format stores: a symbol of weight
+/// `w` occupies `1 << (w - 1)` of the table's slots, and the last symbol's
+/// weight is left out and recovered from the slack.
+class HuffmanTable {
+ /// The largest table log the format allows for literals.
+ static const int maxLog = 12;
+
+ /// One larger than the 255 weights a table may carry, deliberately.
+ ///
+ /// `fseDecodeInterleaved` guards each pass for the three symbols its
+ /// exhausted branches can write, so a 255-weight stream whose final pass
+ /// writes two has to be able to start that pass at 253. Shrinking this to
+ /// 255 looks like a tightening and would refuse every 255-weight table,
+ /// which the real elevation containers are full of.
+ static const int _maxSymbols = 256;
+
+ /// One entry per table slot: the symbol in the low byte, the code length
+ /// in the next. One load per literal rather than two.
+ final Uint16List entries = Uint16List(1 << maxLog);
+
+ /// Bits to peek per literal. Zero until a table has been read.
+ int log = 0;
+
+ final Uint8List _weights = Uint8List(_maxSymbols);
+ final Int32List _rankCount = Int32List(maxLog + 2);
+ final Int32List _rankStart = Int32List(maxLog + 2);
+
+ /// The FSE table the weights are coded with, when they are coded at all.
+ final FseTable _weightTable = FseTable(6);
+
+ /// Reads a Huffman tree description starting at [start], returning the
+ /// number of bytes it occupied.
+ int read(Uint8List src, int start, int end, ReverseBitReader bitReader) {
+ if (start >= end) {
+ throw const ZstdException('literals section has no tree');
+ }
+ final header = src[start];
+ int nbWeights;
+ int used;
+ if (header >= 128) {
+ // Directly encoded: one nibble per weight, high nibble first.
+ nbWeights = header - 127;
+ final packed = (nbWeights + 1) >> 1;
+ if (start + 1 + packed > end) {
+ throw const ZstdException('truncated Huffman weights');
+ }
+ for (var i = 0; i < nbWeights; i += 2) {
+ final byte = src[start + 1 + (i >> 1)];
+ _weights[i] = byte >> 4;
+ if (i + 1 < nbWeights) {
+ _weights[i + 1] = byte & 0xF;
+ }
+ }
+ used = 1 + packed;
+ } else {
+ final size = header;
+ if (start + 1 + size > end) {
+ throw const ZstdException('truncated Huffman weight stream');
+ }
+ final read = readFseTable(
+ _weightTable,
+ src,
+ start + 1,
+ start + 1 + size,
+ 255,
+ );
+ bitReader.reset(src, start + 1 + read, start + 1 + size);
+ nbWeights = fseDecodeInterleaved(
+ _weightTable,
+ bitReader,
+ _weights,
+ 0,
+ _maxSymbols,
+ );
+ used = 1 + size;
+ }
+
+ _buildFromWeights(nbWeights);
+
+ return used;
+ }
+
+ void _buildFromWeights(int nbWeights) {
+ if (nbWeights < 1 || nbWeights >= _maxSymbols) {
+ throw ZstdException('Huffman table with $nbWeights weights');
+ }
+ var total = 0;
+ for (var i = 0; i < nbWeights; i++) {
+ final weight = _weights[i];
+ if (weight > maxLog) {
+ throw ZstdException('Huffman weight $weight out of range');
+ }
+ if (weight > 0) {
+ total += 1 << (weight - 1);
+ }
+ }
+ if (total == 0) {
+ throw const ZstdException('Huffman table with no used symbol');
+ }
+ final tableLog = total.bitLength;
+ if (tableLog > maxLog) {
+ throw ZstdException('Huffman table log $tableLog above $maxLog');
+ }
+ final size = 1 << tableLog;
+ final rest = size - total;
+ if (rest & (rest - 1) != 0) {
+ throw const ZstdException('Huffman weights do not complete a tree');
+ }
+ // The weight left out of the description is whatever fills the slack.
+ _weights[nbWeights] = rest.bitLength;
+ final nbSymbols = nbWeights + 1;
+ log = tableLog;
+
+ for (var n = 0; n < _rankCount.length; n++) {
+ _rankCount[n] = 0;
+ }
+ for (var s = 0; s < nbSymbols; s++) {
+ final weight = _weights[s];
+ if (weight > 0) {
+ _rankCount[tableLog + 1 - weight]++;
+ }
+ }
+ // Codes are handed out from the lowest weight up, so the longest codes
+ // take the lowest table indices and the shortest take the highest.
+ var start = 0;
+ for (var n = tableLog; n >= 1; n--) {
+ _rankStart[n] = start;
+ start += _rankCount[n] << (tableLog - n);
+ }
+ if (start != size) {
+ throw const ZstdException('Huffman ranks do not fill the table');
+ }
+
+ // Within one code length, symbols keep their natural order.
+ for (var s = 0; s < nbSymbols; s++) {
+ final weight = _weights[s];
+ if (weight == 0) {
+ continue;
+ }
+ final length = tableLog + 1 - weight;
+ final span = 1 << (weight - 1);
+ var at = _rankStart[length];
+ _rankStart[length] = at + span;
+ final entry = s | (length << 8);
+ for (var i = 0; i < span; i++) {
+ entries[at++] = entry;
+ }
+ }
+ }
+
+ /// Decodes [count] literals from the reverse bitstream in [src] between
+ /// [start] and [end] into [out] at [outStart].
+ void decodeStream(
+ ReverseBitReader reader,
+ Uint8List src,
+ int start,
+ int end,
+ Uint8List out,
+ int outStart,
+ int count,
+ ) {
+ if (log == 0) {
+ throw const ZstdException('literals coded with no Huffman table');
+ }
+ reader.reset(src, start, end);
+ final table = entries;
+ final peekBits = log;
+ var at = outStart;
+ final stop = outStart + count;
+ // One fill leaves at least 49 bits, which covers four literals at the
+ // format's twelve-bit maximum, so the inner steps are a peek and an
+ // index with nothing between them.
+ final groups = stop - 4;
+ while (at <= groups) {
+ reader.fill();
+ var entry = table[reader.peek(peekBits)];
+ out[at++] = entry & 0xFF;
+ reader.skip(entry >> 8);
+ entry = table[reader.peek(peekBits)];
+ out[at++] = entry & 0xFF;
+ reader.skip(entry >> 8);
+ entry = table[reader.peek(peekBits)];
+ out[at++] = entry & 0xFF;
+ reader.skip(entry >> 8);
+ entry = table[reader.peek(peekBits)];
+ out[at++] = entry & 0xFF;
+ reader.skip(entry >> 8);
+ }
+ while (at < stop) {
+ reader.fill();
+ final entry = table[reader.peek(peekBits)];
+ out[at++] = entry & 0xFF;
+ reader.skip(entry >> 8);
+ }
+ if (reader.exhausted) {
+ throw const ZstdException('Huffman stream ran past its end');
+ }
+ }
+}
diff --git a/lib/src/tables.dart b/lib/src/tables.dart
new file mode 100644
index 0000000..50ea8cf
--- /dev/null
+++ b/lib/src/tables.dart
@@ -0,0 +1,67 @@
+/// The fixed tables RFC 8878 defines for sequence decoding.
+library;
+
+import 'dart:typed_data';
+
+/// Literals length code to the smallest length it can mean.
+final Int32List literalLengthBaseline = Int32List.fromList(const [
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, //
+ 16, 18, 20, 22, 24, 28, 32, 40, //
+ 48, 64, 128, 256, 512, 1024, 2048, 4096, //
+ 8192, 16384, 32768, 65536,
+]);
+
+/// Extra bits read after a literals length code.
+final Uint8List literalLengthExtra = Uint8List.fromList(const [
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //
+ 1, 1, 1, 1, 2, 2, 3, 3, //
+ 4, 6, 7, 8, 9, 10, 11, 12, //
+ 13, 14, 15, 16,
+]);
+
+/// Match length code to the smallest length it can mean.
+final Int32List matchLengthBaseline = Int32List.fromList(const [
+ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, //
+ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, //
+ 35, 37, 39, 41, 43, 47, 51, 59, //
+ 67, 83, 99, 131, 259, 515, 1027, 2051, //
+ 4099, 8195, 16387, 32771, 65539,
+]);
+
+/// Extra bits read after a match length code.
+final Uint8List matchLengthExtra = Uint8List.fromList(const [
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //
+ 1, 1, 1, 1, 2, 2, 3, 3, //
+ 4, 4, 5, 7, 8, 9, 10, 11, //
+ 12, 13, 14, 15, 16,
+]);
+
+/// The default literals length distribution, accuracy log 6.
+const List literalLengthDefault = [
+ 4, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, //
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 1, 1, 1, 1, 1, //
+ -1, -1, -1, -1,
+];
+
+/// The default match length distribution, accuracy log 6.
+const List matchLengthDefault = [
+ 1, 4, 3, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, //
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, //
+ -1, -1, -1, -1, -1,
+];
+
+/// The default offset code distribution, accuracy log 5.
+const List offsetCodeDefault = [
+ 1, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, //
+ 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, -1,
+];
+
+const int literalLengthMaxLog = 9;
+const int matchLengthMaxLog = 9;
+const int offsetCodeMaxLog = 8;
+
+const int literalLengthMaxSymbol = 35;
+const int matchLengthMaxSymbol = 52;
+const int offsetCodeMaxSymbol = 31;
diff --git a/lib/src/xxhash.dart b/lib/src/xxhash.dart
new file mode 100644
index 0000000..93de96a
--- /dev/null
+++ b/lib/src/xxhash.dart
@@ -0,0 +1,169 @@
+/// XXH64, the hash zstd uses for `Content_Checksum`.
+///
+/// RFC 8878 section 3.1.1 stores the low 32 bits of the XXH64 of the whole
+/// decompressed frame content, seeded with zero, little-endian, after the last
+/// block. [Xxh64.digest32] is that value.
+///
+/// The algorithm is defined on unsigned 64-bit words that wrap. Dart's `int`
+/// on the VM and on AOT is a 64-bit two's complement integer whose `+`, `-`,
+/// `*` and `<<` wrap on overflow, and `>>>` shifts the raw bits in, so every
+/// operation here is exact with no masking. None of that holds under dart2js,
+/// where `int` is a double: the multiplies, the rotates and the wrapping adds
+/// would all be wrong. Like the rest of this package, it targets native only.
+library;
+
+import 'dart:typed_data';
+
+// The five primes, written as they appear in the reference implementation.
+// Dart reads a hex literal above 2^63 as its two's complement, so _prime1,
+// _prime2 and _prime4 are negative here. The bits are what matter.
+const int _prime1 = 0x9E3779B185EBCA87;
+const int _prime2 = 0xC2B2AE3D27D4EB4F;
+const int _prime3 = 0x165667B19E3779F9;
+const int _prime4 = 0x85EBCA77C2B2AE63;
+const int _prime5 = 0x27D4EB2F165667C5;
+
+/// Hashes [data] in one pass. Equivalent to `(Xxh64()..update(data)).digest`.
+int xxh64(Uint8List data, {int seed = 0}) =>
+ (Xxh64(seed: seed)..update(data)).digest;
+
+/// XXH64 over as many chunks as the caller has.
+///
+/// A multi-block frame can feed each block to [update] as it is written,
+/// rather than hashing the output again once it is whole. Reading [digest]
+/// does not end the hash; more may be fed in afterwards.
+class Xxh64 {
+ Xxh64({int seed = 0}) : _seed = seed {
+ reset();
+ }
+
+ final int _seed;
+
+ /// Whole 32-byte stripes are consumed straight from the caller's bytes.
+ /// This holds only the remainder between calls.
+ final Uint8List _tail = Uint8List(32);
+ late final ByteData _tailView = ByteData.sublistView(_tail);
+
+ int _tailLength = 0;
+ int _length = 0;
+
+ int _v1 = 0;
+ int _v2 = 0;
+ int _v3 = 0;
+ int _v4 = 0;
+
+ /// Discards everything fed in so far, back to the seed given to the
+ /// constructor.
+ void reset() {
+ _v1 = _seed + _prime1 + _prime2;
+ _v2 = _seed + _prime2;
+ _v3 = _seed;
+ _v4 = _seed - _prime1;
+ _tailLength = 0;
+ _length = 0;
+ }
+
+ /// Feeds `data[start..end)`, defaulting to all of [data].
+ ///
+ /// The range form is so a decoder can hand over the part of an output buffer
+ /// it just filled without taking a view of it.
+ void update(Uint8List data, [int start = 0, int? end]) {
+ final stop = RangeError.checkValidRange(start, end, data.length);
+ var at = start;
+ _length += stop - at;
+
+ if (_tailLength > 0) {
+ final take = stop - at < 32 - _tailLength ? stop - at : 32 - _tailLength;
+ _tail.setRange(_tailLength, _tailLength + take, data, at);
+ _tailLength += take;
+ at += take;
+ if (_tailLength < 32) {
+ return;
+ }
+ _stripe(_tailView, 0);
+ _tailLength = 0;
+ }
+
+ final view = ByteData.sublistView(data);
+ while (at + 32 <= stop) {
+ _stripe(view, at);
+ at += 32;
+ }
+
+ if (at < stop) {
+ _tail.setRange(0, stop - at, data, at);
+ _tailLength = stop - at;
+ }
+ }
+
+ /// The 64-bit hash of everything fed in so far.
+ int get digest {
+ // Under 32 bytes the accumulators were never touched, so the hash starts
+ // from the seed instead of from their merge. _v3 still holds the seed.
+ var h = _length >= 32
+ ? _merge(
+ _merge(
+ _merge(
+ _merge(
+ _rotl(_v1, 1) +
+ _rotl(_v2, 7) +
+ _rotl(_v3, 12) +
+ _rotl(_v4, 18),
+ _v1,
+ ),
+ _v2,
+ ),
+ _v3,
+ ),
+ _v4,
+ )
+ : _v3 + _prime5;
+ h += _length;
+
+ // The remainder is consumed eight bytes at a time, then four, then one.
+ var at = 0;
+ while (at + 8 <= _tailLength) {
+ h ^= _round(0, _tailView.getInt64(at, Endian.little));
+ h = _rotl(h, 27) * _prime1 + _prime4;
+ at += 8;
+ }
+ if (at + 4 <= _tailLength) {
+ h ^= _tailView.getUint32(at, Endian.little) * _prime1;
+ h = _rotl(h, 23) * _prime2 + _prime3;
+ at += 4;
+ }
+ while (at < _tailLength) {
+ h ^= _tail[at] * _prime5;
+ h = _rotl(h, 11) * _prime1;
+ at++;
+ }
+
+ h ^= h >>> 33;
+ h *= _prime2;
+ h ^= h >>> 29;
+ h *= _prime3;
+ h ^= h >>> 32;
+ return h;
+ }
+
+ /// The low 32 bits of [digest]: the value zstd stores as
+ /// `Content_Checksum`.
+ int get digest32 => digest & 0xFFFFFFFF;
+
+ void _stripe(ByteData view, int at) {
+ _v1 = _round(_v1, view.getInt64(at, Endian.little));
+ _v2 = _round(_v2, view.getInt64(at + 8, Endian.little));
+ _v3 = _round(_v3, view.getInt64(at + 16, Endian.little));
+ _v4 = _round(_v4, view.getInt64(at + 24, Endian.little));
+ }
+
+ static int _round(int accumulator, int input) {
+ final v = accumulator + input * _prime2;
+ return _rotl(v, 31) * _prime1;
+ }
+
+ static int _merge(int h, int accumulator) =>
+ (h ^ _round(0, accumulator)) * _prime1 + _prime4;
+
+ static int _rotl(int v, int bits) => (v << bits) | (v >>> (64 - bits));
+}
diff --git a/pubspec.yaml b/pubspec.yaml
new file mode 100644
index 0000000..de50c0d
--- /dev/null
+++ b/pubspec.yaml
@@ -0,0 +1,25 @@
+name: pure_zstd
+description: >-
+ A zstd decompressor written in plain Dart. Decodes RFC 8878 frames with no
+ native library and no platform channel, verified against libzstd.
+version: 0.1.0
+
+repository: https://github.com/tagadvance/pure-zstd
+issue_tracker: https://github.com/tagadvance/pure-zstd/issues
+
+# Remove this line to publish. A published version can be retracted but never
+# deleted, so it stays until the first release is deliberate.
+publish_to: none
+
+topics:
+ - zstd
+ - zstandard
+ - compression
+ - decompression
+
+environment:
+ sdk: ^3.8.0
+
+dev_dependencies:
+ lints: ^6.0.0
+ test: ^1.25.0
diff --git a/test/corpus.dart b/test/corpus.dart
new file mode 100644
index 0000000..4d125fa
--- /dev/null
+++ b/test/corpus.dart
@@ -0,0 +1,209 @@
+import 'dart:convert';
+import 'dart:io';
+import 'dart:typed_data';
+
+/// Loading and generating the test corpus.
+///
+/// The committed half lives in `test/fixtures/corpus`, built by
+/// `tool/make_corpus.py`. Every frame there was either written by libzstd or
+/// hand-built and then accepted by pyzstd, and the expected plain bytes of a
+/// legal frame are what pyzstd decompressed it to, gzipped. `manifest.json`
+/// records the exact command behind each one, so a fixture is never a blob
+/// nobody can reproduce.
+///
+/// The half that is too large to commit is generated here instead, from the
+/// seeded generator below and the `zstd` CLI.
+const String corpusDirectory = 'test/fixtures/corpus';
+
+/// One entry of `manifest.json`.
+class CorpusCase {
+ CorpusCase(Map json)
+ : name = json['name']! as String,
+ command = json['command']! as String,
+ frameFile = json['frame']! as String,
+ legal = json['verdict'] == 'legal',
+ why = json['why'] as String?,
+ derive = json['derive'] as String?,
+ size = json['size'] as int?,
+ expectedSize = json['expectedSize'] as int?,
+ trailing = json['trailing'] as String?,
+ _plain = (json['plain'] as List? ?? const [])
+ .cast()
+ .toList();
+
+ /// The case name, which is also the `test()` description.
+ final String name;
+
+ /// The exact command that produced the frame.
+ final String command;
+
+ final String frameFile;
+
+ /// True when libzstd decompresses the frame, false when it refuses it.
+ final bool legal;
+
+ /// For a refused case, what is wrong with it and what pyzstd said.
+ final String? why;
+
+ /// For a refused case built by damaging another case's frame, the damage.
+ final String? derive;
+
+ /// Length of the plain bytes, for a legal case.
+ final int? size;
+
+ /// What the caller must pass to `decode`, for a frame declaring no content
+ /// size of its own.
+ final int? expectedSize;
+
+ /// What follows the first frame, for a buffer libzstd reads as a stream of
+ /// frames and this decoder reads as one frame and then leftovers.
+ final String? trailing;
+
+ final List _plain;
+
+ /// The frame, with any recorded damage applied.
+ Uint8List get frame {
+ final bytes = Uint8List.fromList(
+ File('$corpusDirectory/$frameFile').readAsBytesSync(),
+ );
+ return switch (derive) {
+ null => bytes,
+ 'the last checksum byte flipped' => Uint8List.fromList(
+ bytes,
+ )..[bytes.length - 1] ^= 0x01,
+ 'all four checksum bytes zeroed' => Uint8List.fromList(
+ bytes,
+ )..fillRange(bytes.length - 4, bytes.length, 0),
+ 'the last two bytes cut' => Uint8List.sublistView(
+ bytes,
+ 0,
+ bytes.length - 2,
+ ),
+ _ => throw StateError('unknown damage: $derive'),
+ };
+ }
+
+ /// What pyzstd decompressed the frame to. Only for a legal case.
+ Uint8List get plain {
+ final out = BytesBuilder(copy: false);
+ for (final part in _plain) {
+ out.add(gzip.decode(File('$corpusDirectory/$part').readAsBytesSync()));
+ }
+ return out.takeBytes();
+ }
+
+ @override
+ String toString() => '$name ($command)';
+}
+
+/// Every case in `manifest.json`, in the order the generator wrote them.
+List loadCorpus() {
+ final text = File('$corpusDirectory/manifest.json').readAsStringSync();
+ return (jsonDecode(text) as List)
+ .cast>()
+ .map(CorpusCase.new)
+ .toList();
+}
+
+/// xorshift64, the same three shifts `tool/make_corpus.py` uses, so an input
+/// built here and one built there are the same bytes.
+class Xorshift64 {
+ Xorshift64(this._state);
+
+ int _state;
+
+ int next() {
+ var state = _state;
+ state ^= state << 13;
+ state ^= state >>> 7;
+ state ^= state << 17;
+ _state = state;
+ return state;
+ }
+
+ Uint8List bytes(int count) {
+ final out = Uint8List(count);
+ for (var i = 0; i < count; i++) {
+ out[i] = (next() >>> 33) & 0xFF;
+ }
+ return out;
+ }
+}
+
+/// `count` bytes of pseudo-random noise. Incompressible, so a compressor has
+/// to fall back on raw blocks and raw literals.
+Uint8List noise(int count, {int seed = 0x9E3779B97F4A7C15}) =>
+ Xorshift64(seed).bytes(count);
+
+/// A stream built by drawing `count` tokens of `tokenLength` bytes from a
+/// pool of `tokens` of them.
+///
+/// Short tokens from a small pool are what puts tens of thousands of
+/// sequences in one block: every token is a match, and at three bytes each
+/// there are more of them than the two-byte sequence count can hold.
+Uint8List tokenStream({
+ required int tokens,
+ required int tokenLength,
+ required int count,
+ int seed = 0x2545F4914F6CDD1D,
+}) {
+ final random = Xorshift64(seed);
+ final pool = random.bytes(tokens * tokenLength);
+ final out = Uint8List(count * tokenLength);
+ for (var i = 0; i < count; i++) {
+ final token = (random.next() >>> 16) % tokens;
+ out.setRange(
+ i * tokenLength,
+ (i + 1) * tokenLength,
+ pool,
+ token * tokenLength,
+ );
+ }
+ return out;
+}
+
+/// `count` bytes drawn from a lopsided alphabet: `a` eight times as likely as
+/// the rare letters, and no matches to speak of.
+///
+/// This is the shape that makes libzstd build one Huffman table and then
+/// reuse it for the blocks that follow, which is the treeless literals form.
+Uint8List skewedAlphabet(int count, {int seed = 0x853C49E6748FEA9B}) {
+ const alphabet = 'aaaaaaaabbbbccdefghijklmnopqrstuvwxyz';
+ final random = Xorshift64(seed);
+ final out = Uint8List(count);
+ for (var i = 0; i < count; i++) {
+ out[i] = alphabet.codeUnitAt((random.next() >>> 16) % alphabet.length);
+ }
+ return out;
+}
+
+/// True when the `zstd` CLI is on the path.
+bool get hasZstdCli {
+ try {
+ return Process.runSync('zstd', const ['--version']).exitCode == 0;
+ } on ProcessException {
+ return false;
+ }
+}
+
+/// Compresses [input] with the `zstd` CLI, as `zstd -q -c `.
+///
+/// A file argument rather than a pipe, so the frame carries a content size.
+Uint8List zstdCli(List args, Uint8List input) {
+ final directory = Directory.systemTemp.createTempSync('pure_zstd_corpus_');
+ try {
+ final file = File('${directory.path}/input.bin')..writeAsBytesSync(input);
+ final run = Process.runSync('zstd', [
+ '-q',
+ '-c',
+ ...args,
+ file.path,
+ ], stdoutEncoding: null);
+ if (run.exitCode != 0) {
+ throw StateError('zstd ${args.join(' ')} failed: ${run.stderr}');
+ }
+ return Uint8List.fromList(run.stdout as List);
+ } finally {
+ directory.deleteSync(recursive: true);
+ }
+}
diff --git a/test/corpus_test.dart b/test/corpus_test.dart
new file mode 100644
index 0000000..d025dc9
--- /dev/null
+++ b/test/corpus_test.dart
@@ -0,0 +1,125 @@
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+import 'package:test/test.dart';
+
+import 'corpus.dart';
+
+/// The committed corpus, which exists because `hgtz_block_test.dart` decodes
+/// one container from one producer at one level and so proves the sequence
+/// loop and Huffman literals on level-19 terrain and nothing else.
+///
+/// The frames here reach the parts of RFC 8878 that 16,490 real `.hgtz`
+/// blocks never touch: raw blocks, RLE blocks, RLE literals, a frame with no
+/// content size, a frame with a checksum, the eight-byte content size, and
+/// the direct-nibble Huffman weight form.
+///
+/// Nothing below asserts what this decoder happens to do. A legal frame is
+/// compared against what pyzstd decompressed it to; a refused frame is one
+/// pyzstd also refuses. `tool/make_corpus.py` records both, along with the
+/// exact command behind each frame, in `test/fixtures/corpus/manifest.json`.
+void main() {
+ final corpus = loadCorpus();
+
+ group('a legal frame decodes to the bytes pyzstd gives', () {
+ for (final entry in corpus.where(
+ (entry) => entry.legal && entry.trailing == null,
+ )) {
+ test('${entry.name}: ${entry.command}', () {
+ final out = ZstdDecoder().decode(
+ entry.frame,
+ expectedSize: entry.expectedSize,
+ );
+ expect(out.length, entry.size);
+ expect(out, orderedEquals(entry.plain));
+ });
+ }
+ });
+
+ group('a malformed frame is refused, never crashed on', () {
+ for (final entry in corpus.where((entry) => !entry.legal)) {
+ // A `ZstdException` and nothing else. `RangeError`, `OutOfMemoryError`
+ // and `IndexError` are all `Error` rather than `Exception`, so a caller
+ // catching `ZstdException` does not catch them and the isolate dies.
+ test('${entry.name}: ${entry.why}', () {
+ expect(
+ () => ZstdDecoder().decode(entry.frame),
+ throwsA(isA()),
+ );
+ });
+ }
+ });
+
+ group('two frames in one buffer', () {
+ final entry = corpus.singleWhere((entry) => entry.name == 'twoframes');
+ final first = 5000;
+
+ test('${entry.command}: decodes to both or refuses, never to the first '
+ 'alone', () {
+ Uint8List? out;
+ try {
+ out = ZstdDecoder().decode(entry.frame);
+ } on ZstdException {
+ return; // Refusing is the other acceptable answer.
+ }
+ expect(
+ out.length,
+ isNot(first),
+ reason: 'returned the first frame and stopped, raising nothing',
+ );
+ expect(out, orderedEquals(entry.plain));
+ });
+ });
+
+ group('a frame whose declared size is not the one asked for', () {
+ // A guard callers rest on, and it had no test:
+ // deleting the whole check left every other case green. A caller that
+ // knows the size, as the .hgtz reader does, gets an exact comparison
+ // against the frame's own figure rather than only the ceiling. Left to
+ // the end, a frame declaring less decoded its own content happily and a
+ // frame declaring more allocated whatever it asked for first.
+ for (final entry in corpus.where(
+ // Not a buffer holding more than one frame: this decoder refuses
+ // those whatever size it is told to expect.
+ (entry) =>
+ entry.legal &&
+ entry.expectedSize == null &&
+ entry.trailing == null &&
+ entry.size! > 1,
+ )) {
+ test('${entry.name} is refused when asked for one byte more', () {
+ expect(
+ () =>
+ ZstdDecoder().decode(entry.frame, expectedSize: entry.size! + 1),
+ throwsA(isA()),
+ );
+ });
+
+ test('${entry.name} still decodes when asked for the size it has', () {
+ expect(
+ ZstdDecoder().decode(entry.frame, expectedSize: entry.size),
+ orderedEquals(entry.plain),
+ );
+ });
+ }
+ });
+
+ group('a frame declaring no content size', () {
+ final entry = corpus.singleWhere((entry) => entry.name == 'nofcs_windowed');
+ final size = entry.expectedSize!;
+
+ test('decodes when given the size it really has', () {
+ final out = ZstdDecoder().decode(entry.frame, expectedSize: size);
+ expect(out, orderedEquals(entry.plain));
+ });
+
+ for (final wrong in [size - 1, size + 1, 0]) {
+ test('is refused when given $wrong instead of $size', () {
+ expect(
+ () => ZstdDecoder().decode(entry.frame, expectedSize: wrong),
+ throwsA(isA()),
+ );
+ });
+ }
+ });
+}
diff --git a/test/fixtures/N51E000.hgtz b/test/fixtures/N51E000.hgtz
new file mode 100644
index 0000000..8cd6cf8
Binary files /dev/null and b/test/fixtures/N51E000.hgtz differ
diff --git a/test/fixtures/corpus/A.raw.gz b/test/fixtures/corpus/A.raw.gz
new file mode 100644
index 0000000..b172352
Binary files /dev/null and b/test/fixtures/corpus/A.raw.gz differ
diff --git a/test/fixtures/corpus/B.raw.gz b/test/fixtures/corpus/B.raw.gz
new file mode 100644
index 0000000..6d146b9
Binary files /dev/null and b/test/fixtures/corpus/B.raw.gz differ
diff --git a/test/fixtures/corpus/checksum_good.zst b/test/fixtures/corpus/checksum_good.zst
new file mode 100644
index 0000000..a5b9117
Binary files /dev/null and b/test/fixtures/corpus/checksum_good.zst differ
diff --git a/test/fixtures/corpus/dictflag.zst b/test/fixtures/corpus/dictflag.zst
new file mode 100644
index 0000000..b4fca2c
Binary files /dev/null and b/test/fixtures/corpus/dictflag.zst differ
diff --git a/test/fixtures/corpus/direct_nibble.raw.gz b/test/fixtures/corpus/direct_nibble.raw.gz
new file mode 100644
index 0000000..c5f09ba
Binary files /dev/null and b/test/fixtures/corpus/direct_nibble.raw.gz differ
diff --git a/test/fixtures/corpus/direct_nibble.zst b/test/fixtures/corpus/direct_nibble.zst
new file mode 100644
index 0000000..cc5ebff
Binary files /dev/null and b/test/fixtures/corpus/direct_nibble.zst differ
diff --git a/test/fixtures/corpus/fcs8.zst b/test/fixtures/corpus/fcs8.zst
new file mode 100644
index 0000000..0b10670
Binary files /dev/null and b/test/fixtures/corpus/fcs8.zst differ
diff --git a/test/fixtures/corpus/fcs_2p40.zst b/test/fixtures/corpus/fcs_2p40.zst
new file mode 100644
index 0000000..ec03b05
Binary files /dev/null and b/test/fixtures/corpus/fcs_2p40.zst differ
diff --git a/test/fixtures/corpus/fcs_max_u64.zst b/test/fixtures/corpus/fcs_max_u64.zst
new file mode 100644
index 0000000..6f90fe3
Binary files /dev/null and b/test/fixtures/corpus/fcs_max_u64.zst differ
diff --git a/test/fixtures/corpus/huf_weight_overrun.zst b/test/fixtures/corpus/huf_weight_overrun.zst
new file mode 100644
index 0000000..ac3797d
Binary files /dev/null and b/test/fixtures/corpus/huf_weight_overrun.zst differ
diff --git a/test/fixtures/corpus/lit_one_stream.raw.gz b/test/fixtures/corpus/lit_one_stream.raw.gz
new file mode 100644
index 0000000..8cf50af
Binary files /dev/null and b/test/fixtures/corpus/lit_one_stream.raw.gz differ
diff --git a/test/fixtures/corpus/lit_one_stream.zst b/test/fixtures/corpus/lit_one_stream.zst
new file mode 100644
index 0000000..6b21a78
Binary files /dev/null and b/test/fixtures/corpus/lit_one_stream.zst differ
diff --git a/test/fixtures/corpus/manifest.json b/test/fixtures/corpus/manifest.json
new file mode 100644
index 0000000..05dfb0b
--- /dev/null
+++ b/test/fixtures/corpus/manifest.json
@@ -0,0 +1,209 @@
+[
+ {
+ "name": "twoframes",
+ "frame": "twoframes.zst",
+ "command": "python3: pyzstd.compress(A, 3) + pyzstd.compress(B, 3), A = 5000 xorshift64 bytes, B = b'the quick brown fox ' * 500",
+ "verdict": "legal",
+ "plain": [
+ "A.raw.gz",
+ "B.raw.gz"
+ ],
+ "size": 15000,
+ "trailing": "a second complete frame"
+ },
+ {
+ "name": "checksum_good",
+ "frame": "checksum_good.zst",
+ "command": "zstd -q -c -3 --check A.bin",
+ "verdict": "legal",
+ "plain": [
+ "A.raw.gz"
+ ],
+ "size": 5000
+ },
+ {
+ "name": "checksum_flipped",
+ "frame": "checksum_good.zst",
+ "command": "zstd -q -c -3 --check A.bin, then the last checksum byte flipped",
+ "verdict": "refused",
+ "why": "pyzstd refuses it: ZstdError",
+ "derive": "the last checksum byte flipped"
+ },
+ {
+ "name": "checksum_zeroed",
+ "frame": "checksum_good.zst",
+ "command": "zstd -q -c -3 --check A.bin, then all four checksum bytes zeroed",
+ "verdict": "refused",
+ "why": "pyzstd refuses it: ZstdError",
+ "derive": "all four checksum bytes zeroed"
+ },
+ {
+ "name": "checksum_truncated",
+ "frame": "checksum_good.zst",
+ "command": "zstd -q -c -3 --check A.bin, then the last two bytes cut",
+ "verdict": "refused",
+ "why": "pyzstd refuses it: ZstdError",
+ "derive": "the last two bytes cut"
+ },
+ {
+ "name": "fcs_max_u64",
+ "frame": "fcs_max_u64.zst",
+ "command": "python3: pyzstd.compress(b'hello world', 3) with the descriptor rewritten to Frame_Content_Size_flag=3 declaring 18446744073709551615",
+ "verdict": "refused",
+ "why": "a 27-byte frame asking for a size no allocator will give; pyzstd refuses it: ZstdError"
+ },
+ {
+ "name": "fcs_2p40",
+ "frame": "fcs_2p40.zst",
+ "command": "python3: pyzstd.compress(b'hello world', 3) with the descriptor rewritten to Frame_Content_Size_flag=3 declaring 1099511627776",
+ "verdict": "refused",
+ "why": "a 27-byte frame asking for a size no allocator will give; pyzstd refuses it: MemoryError"
+ },
+ {
+ "name": "fcs8",
+ "frame": "fcs8.zst",
+ "command": "python3: pyzstd.compress(B, 3) with the descriptor rewritten to Frame_Content_Size_flag=3 and an 8-byte size",
+ "verdict": "legal",
+ "plain": [
+ "B.raw.gz"
+ ],
+ "size": 10000
+ },
+ {
+ "name": "huf_weight_overrun",
+ "frame": "huf_weight_overrun.zst",
+ "command": "hand-built: a compressed block whose Huffman weight description is [0x10, 0x3F] and whose FSE weight stream is 33 x 0xAA then 0x02",
+ "verdict": "refused",
+ "why": "the weight stream decodes past the 256th weight; pyzstd refuses it: ZstdError"
+ },
+ {
+ "name": "rlelit_fmt0_R31",
+ "frame": "rlelit_fmt0_R31.zst",
+ "command": "hand-built: one compressed block, RLE literals Size_Format=0, Regenerated_Size=31, byte 0x5A, Number_of_Sequences=0",
+ "verdict": "legal",
+ "plain": [
+ "rlelit_fmt0_R31.raw.gz"
+ ],
+ "size": 31
+ },
+ {
+ "name": "rlelit_fmt1_R4000",
+ "frame": "rlelit_fmt1_R4000.zst",
+ "command": "hand-built: one compressed block, RLE literals Size_Format=1, Regenerated_Size=4000, byte 0x5A, Number_of_Sequences=0",
+ "verdict": "legal",
+ "plain": [
+ "rlelit_fmt1_R4000.raw.gz"
+ ],
+ "size": 4000
+ },
+ {
+ "name": "rlelit_fmt2_R70000",
+ "frame": "rlelit_fmt2_R70000.zst",
+ "command": "hand-built: one compressed block, RLE literals Size_Format=2, Regenerated_Size=70000, byte 0x5A, Number_of_Sequences=0",
+ "verdict": "legal",
+ "plain": [
+ "rlelit_fmt2_R70000.raw.gz"
+ ],
+ "size": 70000
+ },
+ {
+ "name": "rlelit_fmt2_R131072",
+ "frame": "rlelit_fmt2_R131072.zst",
+ "command": "hand-built: one compressed block, RLE literals Size_Format=2, Regenerated_Size=131072, byte 0x5A, Number_of_Sequences=0",
+ "verdict": "legal",
+ "plain": [
+ "rlelit_fmt2_R131072.raw.gz"
+ ],
+ "size": 131072
+ },
+ {
+ "name": "mixed_rle_raw",
+ "frame": "mixed_rle_raw.zst",
+ "command": "hand-built: [RLE 5000 x 0xAB][raw 33 bytes][RLE 40000 x 0x07] in one frame",
+ "verdict": "legal",
+ "plain": [
+ "mixed_rle_raw.raw.gz"
+ ],
+ "size": 45033
+ },
+ {
+ "name": "nofcs_windowed",
+ "frame": "nofcs_windowed.zst",
+ "command": "zstd -3 --no-check from stdin over b'the quick brown fox jumps over the lazy dog. ' * 24000",
+ "verdict": "legal",
+ "plain": [
+ "text.raw.gz"
+ ],
+ "size": 1080000,
+ "expectedSize": 1080000
+ },
+ {
+ "name": "direct_nibble",
+ "frame": "direct_nibble.zst",
+ "command": "block 183 of a Copernicus GLO-30 elevation tile transcoded into a .hgtz container, lifted out as its own frame; its literals carry the direct-nibble Huffman weight form",
+ "verdict": "legal",
+ "plain": [
+ "direct_nibble.raw.gz"
+ ],
+ "size": 131072,
+ "attribution": "Produced using Copernicus WorldDEM-30 \u00a9 DLR e.V. 2010-2014 and \u00a9 Airbus Defence and Space GmbH 2014-2018 provided under COPERNICUS by the European Union and ESA; all rights reserved."
+ },
+ {
+ "name": "dictflag",
+ "frame": "dictflag.zst",
+ "command": "python3: pyzstd.compress(A, 3) with Dictionary_ID_flag forced to 1 in the frame header descriptor",
+ "verdict": "refused",
+ "why": "a dictionary this decoder does not have; pyzstd refuses it: ZstdError"
+ },
+ {
+ "name": "rle_over_max",
+ "frame": "rle_over_max.zst",
+ "command": "hand-built: one last RLE block declaring Block_Size 2,000,000",
+ "verdict": "refused",
+ "why": "RFC 8878 bounds Block_Size by Block_Maximum_Size, never over 128 KB; libzstd's one-shot decoder accepts it and returns 2 MB"
+ },
+ {
+ "name": "rle_at_max",
+ "frame": "rle_at_max.zst",
+ "command": "hand-built: one last RLE block at exactly Block_Maximum_Size, 131072",
+ "verdict": "legal",
+ "plain": [
+ "rle_at_max.raw.gz"
+ ],
+ "size": 131072
+ },
+ {
+ "name": "lit_one_stream",
+ "frame": "lit_one_stream.zst",
+ "command": "zstd -19 -c --no-check over a file holding b'aaadaabaaaddbbaaacabbada'; from a file rather than stdin so the frame declares its own content size",
+ "verdict": "legal",
+ "plain": [
+ "lit_one_stream.raw.gz"
+ ],
+ "size": 24
+ },
+ {
+ "name": "multiblock_check",
+ "frame": "multiblock_check.zst",
+ "command": "python3: pyzstd.compress(body, {level: 9, checksumFlag: 1, targetCBlockSize: 8192}), body = 165,450 bytes of runs",
+ "verdict": "legal",
+ "plain": [
+ "multiblock_check.raw.gz"
+ ],
+ "size": 165450
+ },
+ {
+ "name": "trailing_junk",
+ "frame": "trailing_junk.zst",
+ "command": "hand-built: the lit_one_stream frame with four bytes of junk appended",
+ "verdict": "refused",
+ "why": "reading one frame is the whole API, so input left over after it is input this decoder does not understand; it used to be ignored"
+ },
+ {
+ "name": "trailing_skippable",
+ "frame": "trailing_skippable.zst",
+ "command": "hand-built: the lit_one_stream frame followed by a skippable frame",
+ "verdict": "refused",
+ "why": "a trailing skippable frame was accepted and silently ignored; only a leading one was refused, and only because the magic check fired"
+ }
+]
diff --git a/test/fixtures/corpus/mixed_rle_raw.raw.gz b/test/fixtures/corpus/mixed_rle_raw.raw.gz
new file mode 100644
index 0000000..a437cad
Binary files /dev/null and b/test/fixtures/corpus/mixed_rle_raw.raw.gz differ
diff --git a/test/fixtures/corpus/mixed_rle_raw.zst b/test/fixtures/corpus/mixed_rle_raw.zst
new file mode 100644
index 0000000..c6b7058
Binary files /dev/null and b/test/fixtures/corpus/mixed_rle_raw.zst differ
diff --git a/test/fixtures/corpus/multiblock_check.raw.gz b/test/fixtures/corpus/multiblock_check.raw.gz
new file mode 100644
index 0000000..32aff9f
Binary files /dev/null and b/test/fixtures/corpus/multiblock_check.raw.gz differ
diff --git a/test/fixtures/corpus/multiblock_check.zst b/test/fixtures/corpus/multiblock_check.zst
new file mode 100644
index 0000000..0f137e6
Binary files /dev/null and b/test/fixtures/corpus/multiblock_check.zst differ
diff --git a/test/fixtures/corpus/nofcs_windowed.zst b/test/fixtures/corpus/nofcs_windowed.zst
new file mode 100644
index 0000000..6a3e3c9
Binary files /dev/null and b/test/fixtures/corpus/nofcs_windowed.zst differ
diff --git a/test/fixtures/corpus/rle_at_max.raw.gz b/test/fixtures/corpus/rle_at_max.raw.gz
new file mode 100644
index 0000000..3558a98
Binary files /dev/null and b/test/fixtures/corpus/rle_at_max.raw.gz differ
diff --git a/test/fixtures/corpus/rle_at_max.zst b/test/fixtures/corpus/rle_at_max.zst
new file mode 100644
index 0000000..61fcf1b
Binary files /dev/null and b/test/fixtures/corpus/rle_at_max.zst differ
diff --git a/test/fixtures/corpus/rle_over_max.zst b/test/fixtures/corpus/rle_over_max.zst
new file mode 100644
index 0000000..ce9943f
Binary files /dev/null and b/test/fixtures/corpus/rle_over_max.zst differ
diff --git a/test/fixtures/corpus/rlelit_fmt0_R31.raw.gz b/test/fixtures/corpus/rlelit_fmt0_R31.raw.gz
new file mode 100644
index 0000000..147b0d8
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt0_R31.raw.gz differ
diff --git a/test/fixtures/corpus/rlelit_fmt0_R31.zst b/test/fixtures/corpus/rlelit_fmt0_R31.zst
new file mode 100644
index 0000000..681ec82
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt0_R31.zst differ
diff --git a/test/fixtures/corpus/rlelit_fmt1_R4000.raw.gz b/test/fixtures/corpus/rlelit_fmt1_R4000.raw.gz
new file mode 100644
index 0000000..692e114
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt1_R4000.raw.gz differ
diff --git a/test/fixtures/corpus/rlelit_fmt1_R4000.zst b/test/fixtures/corpus/rlelit_fmt1_R4000.zst
new file mode 100644
index 0000000..0fcb013
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt1_R4000.zst differ
diff --git a/test/fixtures/corpus/rlelit_fmt2_R131072.raw.gz b/test/fixtures/corpus/rlelit_fmt2_R131072.raw.gz
new file mode 100644
index 0000000..443bbfd
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt2_R131072.raw.gz differ
diff --git a/test/fixtures/corpus/rlelit_fmt2_R131072.zst b/test/fixtures/corpus/rlelit_fmt2_R131072.zst
new file mode 100644
index 0000000..8ccc14f
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt2_R131072.zst differ
diff --git a/test/fixtures/corpus/rlelit_fmt2_R70000.raw.gz b/test/fixtures/corpus/rlelit_fmt2_R70000.raw.gz
new file mode 100644
index 0000000..2fd8780
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt2_R70000.raw.gz differ
diff --git a/test/fixtures/corpus/rlelit_fmt2_R70000.zst b/test/fixtures/corpus/rlelit_fmt2_R70000.zst
new file mode 100644
index 0000000..d27068e
Binary files /dev/null and b/test/fixtures/corpus/rlelit_fmt2_R70000.zst differ
diff --git a/test/fixtures/corpus/text.raw.gz b/test/fixtures/corpus/text.raw.gz
new file mode 100644
index 0000000..9ca0054
Binary files /dev/null and b/test/fixtures/corpus/text.raw.gz differ
diff --git a/test/fixtures/corpus/trailing_junk.zst b/test/fixtures/corpus/trailing_junk.zst
new file mode 100644
index 0000000..e021f4b
Binary files /dev/null and b/test/fixtures/corpus/trailing_junk.zst differ
diff --git a/test/fixtures/corpus/trailing_skippable.zst b/test/fixtures/corpus/trailing_skippable.zst
new file mode 100644
index 0000000..7cbd6a8
Binary files /dev/null and b/test/fixtures/corpus/trailing_skippable.zst differ
diff --git a/test/fixtures/corpus/twoframes.zst b/test/fixtures/corpus/twoframes.zst
new file mode 100644
index 0000000..d36aab7
Binary files /dev/null and b/test/fixtures/corpus/twoframes.zst differ
diff --git a/test/fse_interleaved_test.dart b/test/fse_interleaved_test.dart
new file mode 100644
index 0000000..8379d9e
--- /dev/null
+++ b/test/fse_interleaved_test.dart
@@ -0,0 +1,63 @@
+import 'dart:typed_data';
+
+import 'package:pure_zstd/src/bit_reader.dart';
+import 'package:pure_zstd/src/exception.dart';
+import 'package:pure_zstd/src/fse.dart';
+import 'package:test/test.dart';
+
+/// `fseDecodeInterleaved` has no symbol count to stop on: it runs until the
+/// bitstream is spent, so the only thing keeping it inside the caller's
+/// buffer is its own room check. Both of its exhausted branches write a third
+/// symbol in the same pass, so a check for two let `at` reach `outLimit`,
+/// which is one past the end.
+///
+/// `HuffmanTable` calls it with a 256-byte buffer, so overrunning it indexes
+/// `_weights[256]`. This is the same reproducer as the 53-byte frame in the
+/// corpus, with the frame stripped away: the weight table description and the
+/// bitstream the block carried, handed straight to the function.
+void main() {
+ group('fseDecodeInterleaved', () {
+ // The two bytes are the whole FSE table description: accuracy log 5 and a
+ // distribution that gives the stream below two symbols per state pair.
+ final description = Uint8List.fromList([0x10, 0x3F]);
+
+ // 33 bytes of 0xAA and then the end marker, which is the stream the
+ // crafted block put after the description.
+ final stream = Uint8List.fromList([...List.filled(33, 0xAA), 0x02]);
+
+ test('refuses a stream longer than the caller has room for', () {
+ final table = FseTable(6);
+ expect(readFseTable(table, description, 0, description.length, 255), 2);
+
+ final bits = ReverseBitReader()..reset(stream, 0, stream.length);
+ final weights = Uint8List(256);
+
+ expect(
+ () => fseDecodeInterleaved(table, bits, weights, 0, weights.length),
+ throwsA(isA()),
+ );
+ });
+
+ test(
+ 'writes exactly what the stream encodes, and knows where that ends',
+ () {
+ // The old version of this asserted only that the count was over 256
+ // and under 1024, which passes with the guard at `at + 2`, at
+ // `at + 3`, or removed entirely. This stream encodes 257 symbols, and
+ // the boundary is the whole point: a buffer of exactly 257 takes it
+ // and one of 256 does not.
+ int run(int limit) {
+ final table = FseTable(6);
+ readFseTable(table, description, 0, description.length, 255);
+ final bits = ReverseBitReader()..reset(stream, 0, stream.length);
+
+ return fseDecodeInterleaved(table, bits, Uint8List(1024), 0, limit);
+ }
+
+ expect(run(1024), 257);
+ expect(run(257), 257);
+ expect(() => run(256), throwsA(isA()));
+ },
+ );
+ });
+}
diff --git a/test/hgtz_block_test.dart b/test/hgtz_block_test.dart
new file mode 100644
index 0000000..9bb99ea
--- /dev/null
+++ b/test/hgtz_block_test.dart
@@ -0,0 +1,65 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+import 'package:test/test.dart';
+
+/// The committed container was written by the Python transcoder these frames
+/// come from, so every block is a real level-19 pyzstd frame rather than
+/// anything this package produced. Its samples are synthetic, though: this
+/// file carries no terrain.
+///
+/// Every sample in it is `(northRow * 1000 + column) % 30000`, which makes the
+/// expected bytes of each block computable without a reference decompressor.
+void main() {
+ group('a real .hgtz container', () {
+ final bytes = File('test/fixtures/N51E000.hgtz').readAsBytesSync();
+ final header = ByteData.sublistView(bytes);
+ final rows = header.getUint16(6);
+ final columns = header.getUint16(8);
+ final block = header.getUint16(10);
+ final blockColumns = (columns + block - 1) ~/ block;
+ final blockRows = (rows + block - 1) ~/ block;
+
+ test('has the shape its README documents', () {
+ expect(rows, 1201);
+ expect(columns, 121);
+ expect(block, 256);
+ expect(blockRows * blockColumns, 5);
+ });
+
+ test('decodes every block to the samples the generator wrote', () {
+ final decoder = ZstdDecoder();
+ for (var index = 0; index < blockRows * blockColumns; index++) {
+ final entry = 12 + 8 * index;
+ final offset = header.getUint32(entry);
+ final length = header.getUint32(entry + 4);
+ final blockRow = index ~/ blockColumns;
+ final blockColumn = index % blockColumns;
+ final top = blockRow * block;
+ final left = blockColumn * block;
+ final height = block < rows - top ? block : rows - top;
+ final width = block < columns - left ? block : columns - left;
+
+ final frame = Uint8List.sublistView(bytes, offset, offset + length);
+ final out = decoder.decode(frame);
+
+ expect(out.length, height * width * 2, reason: 'block $index size');
+ final samples = ByteData.sublistView(out);
+ for (var row = 0; row < height; row++) {
+ for (var column = 0; column < width; column++) {
+ // Row 0 of the container is the north edge, which is what the
+ // generator counted from.
+ final northRow = top + row;
+ final expected = (northRow * 1000 + left + column) % 30000;
+ expect(
+ samples.getInt16((row * width + column) * 2),
+ expected,
+ reason: 'block $index at $row,$column',
+ );
+ }
+ }
+ }
+ });
+ });
+}
diff --git a/test/libzstd_shapes_test.dart b/test/libzstd_shapes_test.dart
new file mode 100644
index 0000000..ce61cdb
--- /dev/null
+++ b/test/libzstd_shapes_test.dart
@@ -0,0 +1,82 @@
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+import 'package:test/test.dart';
+
+import 'corpus.dart';
+
+/// The shapes whose frames are too large to commit.
+///
+/// Each input is built here from a seeded generator and handed to the `zstd`
+/// CLI, so the frame is libzstd's own output and the expectation is the exact
+/// bytes that went in. The round trip is the oracle; nothing is asserted
+/// against what this decoder happens to produce.
+///
+/// The shapes matter because the real `.hgtz` corpus is 16,490 blocks from
+/// one producer at one level, and none of this appears in it:
+///
+/// * short tokens from a small pool put more than 32,512 sequences in one
+/// block, which is the only thing that writes the three-byte sequence
+/// count. The pool size decides which sequence tables libzstd picks: 200
+/// tokens gets RLE and repeat tables for the literal lengths, 2,000 gets an
+/// RLE table for the match lengths;
+/// * incompressible noise at a low level makes libzstd give up and emit raw
+/// blocks and raw literals;
+/// * a lopsided alphabet with no matches makes it build one Huffman table and
+/// have the blocks that follow reuse it, which is treeless literals.
+void main() {
+ // 600 KB each, as 200,000 three-byte tokens drawn from a pool.
+ final Uint8List narrowTokens = tokenStream(
+ tokens: 200,
+ tokenLength: 3,
+ count: 200000,
+ );
+ final Uint8List wideTokens = tokenStream(
+ tokens: 2000,
+ tokenLength: 3,
+ count: 200000,
+ );
+
+ // 200 KB of noise, which no compressor can do anything with.
+ final Uint8List random = noise(200000);
+
+ final shapes = arguments, Uint8List input})>{
+ 'zstd --ultra -22 --zstd=minMatch=3 over 600 KB of 3-byte tokens, '
+ 'pool 200': (
+ arguments: ['--ultra', '-22', '--zstd=minMatch=3'],
+ input: narrowTokens,
+ ),
+ 'zstd -19 --zstd=minMatch=3 over 600 KB of 3-byte tokens, pool 200': (
+ arguments: ['-19', '--zstd=minMatch=3'],
+ input: narrowTokens,
+ ),
+ 'zstd -19 --zstd=minMatch=3 over 600 KB of 3-byte tokens, pool 2000': (
+ arguments: ['-19', '--zstd=minMatch=3'],
+ input: wideTokens,
+ ),
+ 'zstd -1 over 200 KB of noise': (arguments: ['-1'], input: random),
+ 'zstd --fast=5 over 200 KB of noise': (
+ arguments: ['--fast=5'],
+ input: random,
+ ),
+ 'zstd -3 over 300 KB of a skewed alphabet': (
+ arguments: ['-3'],
+ input: skewedAlphabet(300000),
+ ),
+ };
+
+ group(
+ 'a frame libzstd wrote decodes back to what went in',
+ () {
+ for (final shape in shapes.entries) {
+ test(shape.key, () {
+ final frame = zstdCli(shape.value.arguments, shape.value.input);
+ final out = ZstdDecoder().decode(frame);
+ expect(out.length, shape.value.input.length);
+ expect(out, orderedEquals(shape.value.input));
+ }, timeout: const Timeout(Duration(minutes: 2)));
+ }
+ },
+ skip: hasZstdCli ? null : 'the zstd CLI is not on the path',
+ );
+}
diff --git a/test/shared_decoder_test.dart b/test/shared_decoder_test.dart
new file mode 100644
index 0000000..8b43605
--- /dev/null
+++ b/test/shared_decoder_test.dart
@@ -0,0 +1,74 @@
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+import 'package:test/test.dart';
+
+import 'corpus.dart';
+
+/// One `ZstdDecoder` over a mixed list of frames, forward and then reversed.
+///
+/// A decoder holds its Huffman table, its three FSE tables, the three "table
+/// is ready to repeat" flags and the three recent offsets between calls, and
+/// every one of those is per frame. A frame that leaves a table behind is
+/// invisible while every frame in the list looks alike, which is exactly what
+/// the real `.hgtz` corpus is. Reversing the order moves each frame to a
+/// different predecessor, so a leak shows up as a difference between the two
+/// passes or against what pyzstd gave.
+void main() {
+ final mixed = loadCorpus()
+ .where((entry) => entry.legal && entry.trailing == null)
+ .toList();
+
+ test('decodes ${mixed.length} mixed frames the same in either order', () {
+ expect(mixed.length, greaterThan(5), reason: 'the list must be mixed');
+
+ final decoder = ZstdDecoder();
+ final forward = {
+ for (final entry in mixed)
+ entry.name: decoder.decode(
+ entry.frame,
+ expectedSize: entry.expectedSize,
+ ),
+ };
+
+ final backward = {};
+ for (final entry in mixed.reversed) {
+ backward[entry.name] = decoder.decode(
+ entry.frame,
+ expectedSize: entry.expectedSize,
+ );
+ }
+
+ for (final entry in mixed) {
+ expect(
+ forward[entry.name],
+ orderedEquals(entry.plain),
+ reason: '${entry.name} decoded wrongly in order',
+ );
+ expect(
+ backward[entry.name],
+ orderedEquals(entry.plain),
+ reason: '${entry.name} decoded wrongly reversed',
+ );
+ }
+ });
+
+ test('a refused frame does not poison the next one', () {
+ final decoder = ZstdDecoder();
+ final bad = loadCorpus().where((entry) => !entry.legal).toList();
+ for (final good in mixed) {
+ for (final entry in bad) {
+ try {
+ decoder.decode(entry.frame);
+ } on ZstdException {
+ // The point of the case. Some are open defects and decode fine.
+ }
+ }
+ expect(
+ decoder.decode(good.frame, expectedSize: good.expectedSize),
+ orderedEquals(good.plain),
+ reason: '${good.name} after a run of malformed frames',
+ );
+ }
+ });
+}
diff --git a/test/xxhash_test.dart b/test/xxhash_test.dart
new file mode 100644
index 0000000..0cda652
--- /dev/null
+++ b/test/xxhash_test.dart
@@ -0,0 +1,241 @@
+import 'dart:typed_data';
+
+import 'package:pure_zstd/src/xxhash.dart';
+import 'package:test/test.dart';
+
+// Every digest below came from the `xxhash` Python package, version 3.0.0,
+// which wraps the reference C implementation. The two byte generators are
+// written identically on both sides, so the inputs are reproducible here
+// without carrying the bytes themselves.
+//
+// The lengths are chosen around the boundaries of the tail, which is
+// consumed in 32-byte stripes, then 8, then 4, then 1. Each of those is a
+// separate branch and each gets a length either side of it.
+
+/// A repeating pattern. 251 is prime, so it does not line up with the stripe.
+Uint8List _pattern(int length) =>
+ Uint8List.fromList(List.generate(length, (i) => i % 251));
+
+/// Bytes with no structure, from a deterministic generator rather than
+/// `Random`, so the expected digests stay valid.
+Uint8List _pseudoRandom(int length) {
+ final out = Uint8List(length);
+ var x = 1;
+ for (var i = 0; i < length; i++) {
+ x = (x * 1103515245 + 12345) & 0x7FFFFFFF;
+ out[i] = (x >> 16) & 0xFF;
+ }
+ return out;
+}
+
+const List<(int, int)> _patternDigests = [
+ (0, 0xEF46DB3751D8E999),
+ (1, 0xE934A84ADB052768),
+ (2, 0x4865A00BA85B7CD6),
+ (3, 0xE5C7BB4533BC65DD),
+ (4, 0xFFCED8604453CC1E),
+ (5, 0xDD0274386E26030C),
+ (7, 0x14CC643F630C72D2),
+ (8, 0x884A173614B81B8D),
+ (9, 0x67D85784A7C78C5B),
+ (15, 0xA948F5F0F6ABAC2D),
+ (16, 0x44B6EF2FB84169F7),
+ (17, 0x5603E60C527599B6),
+ (31, 0xC346D2B59B4D8EE1),
+ (32, 0xCBF59C5116FF32B4),
+ (33, 0x0C535D1ACAFB8EAD),
+ (63, 0xE26AA9E2A95F8E4F),
+ (64, 0xF7C67301DB6713F0),
+ (65, 0xC31EB63B2AE4465B),
+ (127, 0x464D085810CE0199),
+ (128, 0x7A7FE14647B9AB92),
+ (129, 0x0BA25DFD6E891FCF),
+ (255, 0x566D96B832B967C1),
+ (256, 0xF33944343EE85824),
+ (1000, 0xF306F04AA88B54D3),
+ (4096, 0x122A8C8D994AD3EC),
+ (5000, 0xA6833D648FD6A332),
+];
+
+const List<(int, int)> _pseudoRandomDigests = [
+ (0, 0xEF46DB3751D8E999),
+ (1, 0xD2DA77930F69647C),
+ (2, 0x9F33D4BD50F99C34),
+ (3, 0xF7C331CF2042B940),
+ (4, 0x1B042F821FC4A793),
+ (5, 0x6E1F98107D9DB571),
+ (7, 0x41D39023940006C8),
+ (8, 0xEF7823FCE4AD9AFB),
+ (9, 0x81CF88E9D8340150),
+ (15, 0x6F0095B1C00E82FB),
+ (16, 0x29965039DF6047BB),
+ (17, 0x5236FCB96D8FB228),
+ (31, 0x68C3DBB752F3FA97),
+ (32, 0xAD7DC5A569BB047C),
+ (33, 0x475B642AC692E380),
+ (63, 0xC18C4743DE1B99A8),
+ (64, 0x06872C5D6370DE25),
+ (65, 0x8B5F4E34A32C5A09),
+ (127, 0x77DC651915D836B5),
+ (128, 0x2E9C4CB0B29BC8EC),
+ (129, 0xB5E3A45BFF735CFA),
+ (255, 0x9AD1F660B88ABC38),
+ (256, 0x9AE30993C3F4B05A),
+ (1000, 0x7AD5B044FB6FA81D),
+ (4096, 0x4AFD74BF8BD52DF5),
+ (5000, 0xCAEF4049E5C55187),
+];
+
+/// `Mt Herman 2764 m\n` 241 times, through `zstd -19 --check`. The frame is
+/// embedded whole so the trailing `Content_Checksum` is read from a real
+/// frame rather than asserted from a constant.
+const String _checkedFrame =
+ '28b52ffd64010fcd0000884d74204865726d616e2032373634206d0a01'
+ '00db4ffe5c02fe13e790';
+
+Uint8List _hex(String text) {
+ final out = Uint8List(text.length ~/ 2);
+ for (var i = 0; i < out.length; i++) {
+ out[i] = int.parse(text.substring(i * 2, i * 2 + 2), radix: 16);
+ }
+ return out;
+}
+
+void main() {
+ group('xxh64 in one shot', () {
+ for (final (length, expected) in _patternDigests) {
+ test('matches the reference over $length repeating bytes', () {
+ expect(xxh64(_pattern(length)), expected);
+ });
+ }
+
+ for (final (length, expected) in _pseudoRandomDigests) {
+ test('matches the reference over $length unstructured bytes', () {
+ expect(xxh64(_pseudoRandom(length)), expected);
+ });
+ }
+
+ test('honours a seed, including one that is negative as a Dart int', () {
+ expect(xxh64(_pattern(0), seed: 1), 0xD5AFBA1336A3BE4B);
+ expect(xxh64(_pattern(5), seed: 1), 0x27CE45508B78A340);
+ expect(xxh64(_pattern(32), seed: 1), 0xD74E6766CE9DBA94);
+ expect(xxh64(_pattern(200), seed: 1), 0x20BD094800CB1DFA);
+
+ const seed = 0x9E3779B185EBCA87;
+ expect(seed.isNegative, isTrue);
+ expect(xxh64(_pattern(0), seed: seed), 0x6EC6D05F61C7E7A7);
+ expect(xxh64(_pattern(5), seed: seed), 0xF60DC54C180A9098);
+ expect(xxh64(_pattern(32), seed: seed), 0xBFB3E4EF6096C49C);
+ expect(xxh64(_pattern(200), seed: seed), 0x6EA98426E0B64F4A);
+ });
+ });
+
+ group('the incremental form', () {
+ // Lengths either side of every boundary, plus enough stripes that a
+ // chunking can straddle several.
+ const lengths = [
+ 0,
+ 1,
+ 3,
+ 4,
+ 7,
+ 8,
+ 15,
+ 16,
+ 31,
+ 32,
+ 33,
+ 63,
+ 64,
+ 65,
+ 127,
+ 128,
+ 129,
+ 1000,
+ 4096,
+ 5001,
+ ];
+
+ for (final chunk in [1, 3, 7, 8, 17, 31, 32, 33, 64, 250]) {
+ test('in $chunk-byte chunks equals the one shot', () {
+ for (final length in lengths) {
+ final data = _pseudoRandom(length);
+ final hash = Xxh64();
+ for (var at = 0; at < length; at += chunk) {
+ final end = at + chunk < length ? at + chunk : length;
+ hash.update(data, at, end);
+ }
+ expect(hash.digest, xxh64(data), reason: '$length bytes');
+ }
+ });
+ }
+
+ test('ignores empty chunks, leading, trailing and interleaved', () {
+ final data = _pattern(300);
+ final empty = Uint8List(0);
+ final hash = Xxh64()
+ ..update(empty)
+ ..update(data, 0, 1)
+ ..update(empty)
+ ..update(data, 1, 40)
+ ..update(data, 40, 40)
+ ..update(data, 40, 300)
+ ..update(empty);
+ expect(hash.digest, xxh64(data));
+ });
+
+ test('accepts a whole list with no range', () {
+ final data = _pattern(100);
+ final hash = Xxh64()
+ ..update(Uint8List.sublistView(data, 0, 37))
+ ..update(Uint8List.sublistView(data, 37));
+ expect(hash.digest, xxh64(data));
+ });
+
+ test('reads the same digest twice and keeps hashing afterwards', () {
+ final data = _pattern(500);
+ final hash = Xxh64()..update(data, 0, 200);
+ expect(hash.digest, xxh64(_pattern(200)));
+ expect(hash.digest, xxh64(_pattern(200)));
+ hash.update(data, 200, 500);
+ expect(hash.digest, xxh64(data));
+ });
+
+ test('reset returns it to the state of a fresh instance', () {
+ final hash = Xxh64()..update(_pattern(999));
+ hash.reset();
+ expect(hash.digest, xxh64(_pattern(0)));
+ hash.update(_pattern(65));
+ expect(hash.digest, xxh64(_pattern(65)));
+ });
+
+ test('rejects a range outside the list', () {
+ expect(() => Xxh64().update(_pattern(10), 0, 11), throwsRangeError);
+ expect(() => Xxh64().update(_pattern(10), -1), throwsRangeError);
+ expect(() => Xxh64().update(_pattern(10), 6, 5), throwsRangeError);
+ });
+ });
+
+ group('a real zstd frame', () {
+ final frame = _hex(_checkedFrame);
+ final content = Uint8List.fromList(
+ List.filled(
+ 241,
+ 'Mt Herman 2764 m\n'.codeUnits,
+ ).expand((e) => e).toList(),
+ );
+
+ test('declares a content checksum', () {
+ expect((frame[4] >> 2) & 1, 1);
+ });
+
+ test('ends with the low 32 bits of the content digest', () {
+ final stored = ByteData.sublistView(
+ frame,
+ frame.length - 4,
+ ).getUint32(0, Endian.little);
+ expect((Xxh64()..update(content)).digest32, stored);
+ expect(xxh64(content) & 0xFFFFFFFF, stored);
+ });
+ });
+}
diff --git a/tool/make_corpus.py b/tool/make_corpus.py
new file mode 100644
index 0000000..3bdabcb
--- /dev/null
+++ b/tool/make_corpus.py
@@ -0,0 +1,399 @@
+#!/usr/bin/env python3
+"""Builds the committed test corpus in test/fixtures/corpus/.
+
+Every frame here is either produced by libzstd (through the `zstd` CLI or
+pyzstd) or hand-built and then handed to pyzstd for a verdict, so "legal" and
+"malformed" are libzstd's opinion rather than this file's. The expected plain
+bytes of a legal frame are whatever pyzstd decompressed it to, gzipped so the
+committed fixture stays small; nothing here writes an expectation by hand.
+
+manifest.json records, per case, the exact command that made the frame. The
+Dart tests read that manifest and never carry a literal of their own.
+
+ ./tool/make_corpus.py # regenerates test/fixtures/corpus
+
+Needs pyzstd, the zstd CLI, and for the one real-tile case a readable
+a directory of .hgtz containers, given in HGTZ_TILE.
+"""
+
+import gzip
+import os
+import json
+import pathlib
+import struct
+import subprocess
+import sys
+
+import pyzstd
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+OUT = ROOT / "test" / "fixtures" / "corpus"
+# One fixture is a block of real terrain lifted out of an elevation
+# container. It is committed, so rebuilding it is optional: point HGTZ_TILE at
+# a .hgtz file to regenerate it, and without one that case is left alone.
+_TILE = os.environ.get("HGTZ_TILE")
+TILE = pathlib.Path(_TILE) if _TILE else None
+TILE_BLOCK = 183
+
+MAGIC = struct.pack("> 7
+ state ^= (state << 17) & MASK64
+ out[i] = (state >> 33) & 0xFF
+ return bytes(out)
+
+
+A = xorshift(0x9E3779B97F4A7C15, 5000)
+B = b"the quick brown fox " * 500
+TEXT = b"the quick brown fox jumps over the lazy dog. " * 24000
+
+
+def build():
+ OUT.mkdir(parents=True, exist_ok=True)
+ for stale in OUT.glob("*"):
+ stale.unlink()
+
+ a = plain("A", A)
+ b = plain("B", B)
+
+ # 1. Two frames in one buffer. pyzstd returns A+B; a decoder that reads
+ # one frame must refuse rather than return A and stop.
+ legal(
+ "twoframes",
+ pyzstd.compress(A, 3) + pyzstd.compress(B, 3),
+ [a, b],
+ "python3: pyzstd.compress(A, 3) + pyzstd.compress(B, 3), "
+ "A = 5000 xorshift64 bytes, B = b'the quick brown fox ' * 500",
+ )
+ cases[-1]["trailing"] = "a second complete frame"
+
+ # 2. A frame with a content checksum. The three malformed variants are
+ # derived from this one in the test; pyzstd's verdict on each is
+ # recorded here so nothing is asserted on faith.
+ checked = cli(["-3", "--check"], A, "A.bin")
+ legal("checksum_good", checked, [a], "zstd -q -c -3 --check A.bin")
+ for name, label, variant in (
+ (
+ "checksum_flipped",
+ "the last checksum byte flipped",
+ checked[:-1] + bytes([checked[-1] ^ 0x01]),
+ ),
+ (
+ "checksum_zeroed",
+ "all four checksum bytes zeroed",
+ checked[:-4] + b"\0\0\0\0",
+ ),
+ (
+ "checksum_truncated",
+ "the last two bytes cut",
+ checked[:-2],
+ ),
+ ):
+ try:
+ pyzstd.decompress(variant)
+ except Exception as error:
+ verdict = f"pyzstd refuses it: {type(error).__name__}"
+ else:
+ verdict = "NOTE pyzstd accepts it"
+ cases.append(
+ {
+ "name": name,
+ "frame": "checksum_good.zst",
+ "command": f"zstd -q -c -3 --check A.bin, then {label}",
+ "verdict": "refused",
+ "why": verdict,
+ "derive": label,
+ }
+ )
+
+ # 3. Frame_Content_Size_flag = 3 declaring a size no machine will give.
+ # Built by rewriting a real frame's descriptor, so everything after the
+ # header is a frame libzstd wrote.
+ base = pyzstd.compress(b"hello world", 3)
+ assert base[4] >> 6 == 0 and (base[4] >> 5) & 1 == 1, base[4]
+ for label, declared in (
+ ("fcs_max_u64", (1 << 64) - 1),
+ ("fcs_2p40", 1 << 40),
+ ):
+ frame = base[:4] + bytes([(3 << 6) | (base[4] & 0x3F)])
+ frame += declared.to_bytes(8, "little") + base[6:]
+ illegal(
+ label,
+ frame,
+ f"python3: pyzstd.compress(b'hello world', 3) with the descriptor "
+ f"rewritten to Frame_Content_Size_flag=3 declaring {declared}",
+ "a 27-byte frame asking for a size no allocator will give",
+ )
+
+ # 3b. The same eight-byte content size on a frame that is legal, so the
+ # branch is reached by something that decodes and not only by two
+ # frames refused on the way past it.
+ legal(
+ "fcs8",
+ rewrite_fcs8(pyzstd.compress(B, 3), len(B)),
+ [b],
+ "python3: pyzstd.compress(B, 3) with the descriptor rewritten to "
+ "Frame_Content_Size_flag=3 and an 8-byte size",
+ )
+
+ # 4. The 53-byte frame that walked fseDecodeInterleaved past _weights[255].
+ # Hand-built: a compressed block whose literals carry an FSE-coded
+ # Huffman weight stream long enough to emit 257 weights.
+ weights = bytes([0x10, 0x3F]) + bytes([0xAA] * 33) + bytes([0x02])
+ body = bytes([0xA2, 0x00, 0x0A]) # compressed literals, 10 regenerated
+ body += bytes([len(weights)]) + weights
+ body += bytes([0x01, 0x01, 0x01, 0x00]) # sequence section, never reached
+ frame = MAGIC + bytes([0x20, 0x0A]) + blockhdr(1, 2, len(body)) + body
+ illegal(
+ "huf_weight_overrun",
+ frame,
+ "hand-built: a compressed block whose Huffman weight description is "
+ "[0x10, 0x3F] and whose FSE weight stream is 33 x 0xAA then 0x02",
+ "the weight stream decodes past the 256th weight",
+ )
+
+ # 5. RLE literals with Number_of_Sequences = 0, at every size format.
+ # libzstd never emits these: it writes an RLE block instead.
+ for fmt, count in ((0, 31), (1, 4000), (2, 70000), (2, 131072)):
+ body = rle_literals(count, 0x5A, fmt)
+ name = f"rlelit_fmt{fmt}_R{count}"
+ expect = plain(name, bytes([0x5A]) * count)
+ legal(
+ name,
+ single_frame([blockhdr(1, 2, len(body)) + body], count),
+ [expect],
+ f"hand-built: one compressed block, RLE literals Size_Format={fmt}, "
+ f"Regenerated_Size={count}, byte 0x5A, Number_of_Sequences=0",
+ )
+
+ # 6. RLE and raw blocks, which the real .hgtz corpus contains none of.
+ content = b"\xab" * 5000 + b"hello world hello world hello wor" + b"\x07" * 40000
+ assert len(content) == 5000 + 33 + 40000
+ blocks = [
+ blockhdr(0, 1, 5000) + b"\xab",
+ blockhdr(0, 0, 33) + content[5000:5033],
+ blockhdr(1, 1, 40000) + b"\x07",
+ ]
+ expect = plain("mixed_rle_raw", content)
+ legal(
+ "mixed_rle_raw",
+ single_frame(blocks, len(content)),
+ [expect],
+ "hand-built: [RLE 5000 x 0xAB][raw 33 bytes][RLE 40000 x 0x07] "
+ "in one frame",
+ )
+
+ # 7. No Frame_Content_Size at all: the shape the CLI writes from a pipe.
+ frame = cli(["-3", "--no-check"], TEXT)
+ assert frame[4] >> 6 == 0 and (frame[4] >> 5) & 1 == 0, frame[4]
+ expect = plain("text", TEXT)
+ legal(
+ "nofcs_windowed",
+ frame,
+ [expect],
+ "zstd -3 --no-check from stdin over "
+ "b'the quick brown fox jumps over the lazy dog. ' * 24000",
+ expected_size=len(TEXT),
+ )
+
+ # 11. A real block that codes its Huffman weights as direct nibbles. The
+ # synthetic shapes above never produce one, and the committed
+ # container has none. Terrain is not ours to redistribute freely, so
+ # the one committed here carries its licence's attribution in the
+ # manifest; see the README's acknowledgements.
+ if TILE is None:
+ print("HGTZ_TILE not set: leaving direct_nibble as committed")
+ else:
+ tile = TILE.read_bytes()
+ offset, length = struct.unpack_from(">II", tile, 12 + 8 * TILE_BLOCK)
+ frame = bytes(tile[offset : offset + length])
+ assert direct_nibble_weights(frame), "that block is not direct nibbles"
+ expect = plain("direct_nibble", pyzstd.decompress(frame))
+ legal(
+ "direct_nibble",
+ frame,
+ [expect],
+ f"block {TILE_BLOCK} of a Copernicus GLO-30 elevation tile "
+ "transcoded into a .hgtz container, lifted out as its own frame; "
+ "its literals carry the direct-nibble Huffman weight form",
+ )
+
+ # 13. Dictionary_ID_flag set on a frame that carries no dictionary id.
+ base = pyzstd.compress(A, 3)
+ illegal(
+ "dictflag",
+ base[:4] + bytes([base[4] | 1]) + base[5:],
+ "python3: pyzstd.compress(A, 3) with Dictionary_ID_flag forced to 1 "
+ "in the frame header descriptor",
+ "a dictionary this decoder does not have",
+ )
+
+ (OUT / "manifest.json").write_text(json.dumps(cases, indent=2) + "\n")
+ total = sum(f.stat().st_size for f in OUT.iterdir())
+ print(f"{len(cases)} cases, {len(list(OUT.iterdir()))} files, {total} bytes")
+
+
+def rewrite_fcs8(frame, size):
+ """The same frame with an 8-byte Frame_Content_Size in place of its own."""
+ descriptor = frame[4]
+ single = (descriptor >> 5) & 1
+ at = 5 + (0 if single else 1)
+ width = {0: 1 if single else 0, 1: 2, 2: 4, 3: 8}[descriptor >> 6]
+ return (
+ frame[:4]
+ + bytes([(3 << 6) | (descriptor & 0x3F)])
+ + frame[5:at]
+ + size.to_bytes(8, "little")
+ + frame[at + width :]
+ )
+
+
+def blockhdr(last, btype, size):
+ return struct.pack("> 4, byte])
+ else:
+ assert count < (1 << 20)
+ body = bytes(
+ [
+ ((count & 0xF) << 4) | (3 << 2) | 1,
+ (count >> 4) & 0xFF,
+ count >> 12,
+ byte,
+ ]
+ )
+ return body + b"\x00"
+
+
+def single_frame(blocks, size):
+ """A single-segment frame declaring `size` and carrying `blocks`."""
+ if size < 256:
+ descriptor, field = (0 << 6) | (1 << 5), bytes([size])
+ elif size < 65536 + 256:
+ descriptor, field = (1 << 6) | (1 << 5), struct.pack("> 5) & 1 else 1)
+ at += {0: 1 if (descriptor >> 5) & 1 else 0, 1: 2, 2: 4, 3: 8}[descriptor >> 6]
+ header = frame[at] | (frame[at + 1] << 8) | (frame[at + 2] << 16)
+ at += 3
+ if (header >> 1) & 3 != 2:
+ return False
+ literals = frame[at]
+ if literals & 3 != 2:
+ return False
+ at += {0: 3, 1: 3, 2: 4, 3: 5}[(literals >> 2) & 3]
+ return frame[at] >= 128
+
+
+if __name__ == "__main__":
+ sys.exit(build())
diff --git a/tool/oracle.py b/tool/oracle.py
new file mode 100755
index 0000000..ad05ca8
--- /dev/null
+++ b/tool/oracle.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+"""Check the Dart decoder against pyzstd over real .hgtz containers.
+
+ ./tool/oracle.py
+ ./tool/oracle.py --every 1500 --limit 8
+
+pyzstd is the reference. Every block of every container named is extracted as
+its own frame, decompressed by pyzstd, and handed to tool/verify.dart, which
+decodes it again and compares byte for byte. One tile at a time, through a
+temporary directory that is deleted as it goes, because a whole tile's blocks
+decompress to around 29 MB.
+"""
+
+from __future__ import annotations
+
+import argparse
+import struct
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+import pyzstd
+
+HEADER = ">4sBbHHH"
+ENTRY = ">II"
+
+
+def frames(path: Path):
+ """Yields (index, frame bytes, expected decompressed size) per block."""
+ data = path.read_bytes()
+ magic, version, _level, rows, cols, block = struct.unpack_from(HEADER, data)
+ if magic != b"HGTZ" or version != 1:
+ raise SystemExit(f"{path}: not a version 1 .hgtz")
+ block_cols = (cols + block - 1) // block
+ block_rows = (rows + block - 1) // block
+ for index in range(block_rows * block_cols):
+ offset, length = struct.unpack_from(ENTRY, data, 12 + 8 * index)
+ top = (index // block_cols) * block
+ left = (index % block_cols) * block
+ height = min(block, rows - top)
+ width = min(block, cols - left)
+ yield index, data[offset : offset + length], height * width * 2
+
+
+def check(path: Path, root: Path) -> bool:
+ with tempfile.TemporaryDirectory() as name:
+ directory = Path(name)
+ count = 0
+ for index, frame, expected in frames(path):
+ plain = pyzstd.decompress(frame)
+ if len(plain) != expected:
+ raise SystemExit(
+ f"{path} block {index}: pyzstd gave {len(plain)},"
+ f" the index implies {expected}"
+ )
+ (directory / f"{index:05d}.zst").write_bytes(frame)
+ (directory / f"{index:05d}.raw").write_bytes(plain)
+ count += 1
+ result = subprocess.run(
+ ["dart", "run", "tool/verify.dart", str(directory)],
+ cwd=root,
+ capture_output=True,
+ text=True,
+ )
+ line = result.stdout.strip().splitlines()[-1] if result.stdout else ""
+ print(f"{path.name}: {count} blocks, {line or result.stderr.strip()}")
+ return result.returncode == 0
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("paths", nargs="+", type=Path)
+ parser.add_argument(
+ "--every",
+ type=int,
+ default=1,
+ help="when a directory is named, take every Nth container",
+ )
+ parser.add_argument("--limit", type=int, default=0)
+ arguments = parser.parse_args()
+
+ containers: list[Path] = []
+ for path in arguments.paths:
+ if path.is_dir():
+ found = sorted(path.glob("*.hgtz"))
+ containers += found[:: arguments.every]
+ else:
+ containers.append(path)
+ if arguments.limit:
+ containers = containers[: arguments.limit]
+
+ root = Path(__file__).resolve().parent.parent
+ failures = [c for c in containers if not check(c, root)]
+ if failures:
+ print(f"FAILED on {len(failures)} of {len(containers)} containers")
+ return 1
+ print(f"all {len(containers)} containers match pyzstd byte for byte")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tool/verify.dart b/tool/verify.dart
new file mode 100644
index 0000000..5790d42
--- /dev/null
+++ b/tool/verify.dart
@@ -0,0 +1,52 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+/// Decodes every `*.zst` in a directory and compares it to the `*.raw`
+/// beside it. Driven by tool/oracle.py, which writes the pair.
+void main(List arguments) {
+ if (arguments.length != 1) {
+ stderr.writeln('usage: verify.dart ');
+ exit(2);
+ }
+ final directory = Directory(arguments.single);
+ final frames =
+ directory
+ .listSync()
+ .whereType()
+ .where((file) => file.path.endsWith('.zst'))
+ .toList()
+ ..sort((a, b) => a.path.compareTo(b.path));
+
+ final decoder = ZstdDecoder();
+ var checked = 0;
+ var bytes = 0;
+ for (final frame in frames) {
+ final expected = File(
+ frame.path.replaceRange(frame.path.length - 4, null, '.raw'),
+ ).readAsBytesSync();
+ final Uint8List actual;
+ try {
+ actual = decoder.decode(frame.readAsBytesSync());
+ } on ZstdException catch (error) {
+ stderr.writeln('${frame.path}: $error');
+ exit(1);
+ }
+ if (actual.length != expected.length) {
+ stderr.writeln(
+ '${frame.path}: ${actual.length} bytes, expected ${expected.length}',
+ );
+ exit(1);
+ }
+ for (var i = 0; i < actual.length; i++) {
+ if (actual[i] != expected[i]) {
+ stderr.writeln('${frame.path}: byte $i differs');
+ exit(1);
+ }
+ }
+ checked++;
+ bytes += actual.length;
+ }
+ stdout.writeln('$checked frames, $bytes bytes, identical');
+}