Decode the corpus against pyzstd, never against ourselves

A test that asserts what the decoder currently does proves nothing. Every
legal frame here is compared byte for byte with what pyzstd decompressed it
to, and every malformed frame is one pyzstd also refuses.

The refusal has to be a ZstdException. RangeError, OutOfMemoryError and
IndexError are all Error rather than Exception, so a caller writing
"on ZstdException" does not catch them and the isolate dies.

test/corpus.dart

@@ -0,0 +1,209 @@
+import 'dart:convert';
+import 'dart:io';
+import 'dart:typed_data';
+
+/// Loading and generating the test corpus.
+///
+/// The committed half lives in `test/fixtures/corpus`, built by
+/// `tool/make_corpus.py`. Every frame there was either written by libzstd or
+/// hand-built and then accepted by pyzstd, and the expected plain bytes of a
+/// legal frame are what pyzstd decompressed it to, gzipped. `manifest.json`
+/// records the exact command behind each one, so a fixture is never a blob
+/// nobody can reproduce.
+///
+/// The half that is too large to commit is generated here instead, from the
+/// seeded generator below and the `zstd` CLI.
+const String corpusDirectory = 'test/fixtures/corpus';
+
+/// One entry of `manifest.json`.
+class CorpusCase {
+  CorpusCase(Map<String, Object?> json)
+    : name = json['name']! as String,
+      command = json['command']! as String,
+      frameFile = json['frame']! as String,
+      legal = json['verdict'] == 'legal',
+      why = json['why'] as String?,
+      derive = json['derive'] as String?,
+      size = json['size'] as int?,
+      expectedSize = json['expectedSize'] as int?,
+      trailing = json['trailing'] as String?,
+      _plain = (json['plain'] as List<Object?>? ?? const [])
+          .cast<String>()
+          .toList();
+
+  /// The case name, which is also the `test()` description.
+  final String name;
+
+  /// The exact command that produced the frame.
+  final String command;
+
+  final String frameFile;
+
+  /// True when libzstd decompresses the frame, false when it refuses it.
+  final bool legal;
+
+  /// For a refused case, what is wrong with it and what pyzstd said.
+  final String? why;
+
+  /// For a refused case built by damaging another case's frame, the damage.
+  final String? derive;
+
+  /// Length of the plain bytes, for a legal case.
+  final int? size;
+
+  /// What the caller must pass to `decode`, for a frame declaring no content
+  /// size of its own.
+  final int? expectedSize;
+
+  /// What follows the first frame, for a buffer libzstd reads as a stream of
+  /// frames and this decoder reads as one frame and then leftovers.
+  final String? trailing;
+
+  final List<String> _plain;
+
+  /// The frame, with any recorded damage applied.
+  Uint8List get frame {
+    final bytes = Uint8List.fromList(
+      File('$corpusDirectory/$frameFile').readAsBytesSync(),
+    );
+    return switch (derive) {
+      null => bytes,
+      'the last checksum byte flipped' => Uint8List.fromList(
+        bytes,
+      )..[bytes.length - 1] ^= 0x01,
+      'all four checksum bytes zeroed' => Uint8List.fromList(
+        bytes,
+      )..fillRange(bytes.length - 4, bytes.length, 0),
+      'the last two bytes cut' => Uint8List.sublistView(
+        bytes,
+        0,
+        bytes.length - 2,
+      ),
+      _ => throw StateError('unknown damage: $derive'),
+    };
+  }
+
+  /// What pyzstd decompressed the frame to. Only for a legal case.
+  Uint8List get plain {
+    final out = BytesBuilder(copy: false);
+    for (final part in _plain) {
+      out.add(gzip.decode(File('$corpusDirectory/$part').readAsBytesSync()));
+    }
+    return out.takeBytes();
+  }
+
+  @override
+  String toString() => '$name ($command)';
+}
+
+/// Every case in `manifest.json`, in the order the generator wrote them.
+List<CorpusCase> loadCorpus() {
+  final text = File('$corpusDirectory/manifest.json').readAsStringSync();
+  return (jsonDecode(text) as List<Object?>)
+      .cast<Map<String, Object?>>()
+      .map(CorpusCase.new)
+      .toList();
+}
+
+/// xorshift64, the same three shifts `tool/make_corpus.py` uses, so an input
+/// built here and one built there are the same bytes.
+class Xorshift64 {
+  Xorshift64(this._state);
+
+  int _state;
+
+  int next() {
+    var state = _state;
+    state ^= state << 13;
+    state ^= state >>> 7;
+    state ^= state << 17;
+    _state = state;
+    return state;
+  }
+
+  Uint8List bytes(int count) {
+    final out = Uint8List(count);
+    for (var i = 0; i < count; i++) {
+      out[i] = (next() >>> 33) & 0xFF;
+    }
+    return out;
+  }
+}
+
+/// `count` bytes of pseudo-random noise. Incompressible, so a compressor has
+/// to fall back on raw blocks and raw literals.
+Uint8List noise(int count, {int seed = 0x9E3779B97F4A7C15}) =>
+    Xorshift64(seed).bytes(count);
+
+/// A stream built by drawing `count` tokens of `tokenLength` bytes from a
+/// pool of `tokens` of them.
+///
+/// Short tokens from a small pool are what puts tens of thousands of
+/// sequences in one block: every token is a match, and at three bytes each
+/// there are more of them than the two-byte sequence count can hold.
+Uint8List tokenStream({
+  required int tokens,
+  required int tokenLength,
+  required int count,
+  int seed = 0x2545F4914F6CDD1D,
+}) {
+  final random = Xorshift64(seed);
+  final pool = random.bytes(tokens * tokenLength);
+  final out = Uint8List(count * tokenLength);
+  for (var i = 0; i < count; i++) {
+    final token = (random.next() >>> 16) % tokens;
+    out.setRange(
+      i * tokenLength,
+      (i + 1) * tokenLength,
+      pool,
+      token * tokenLength,
+    );
+  }
+  return out;
+}
+
+/// `count` bytes drawn from a lopsided alphabet: `a` eight times as likely as
+/// the rare letters, and no matches to speak of.
+///
+/// This is the shape that makes libzstd build one Huffman table and then
+/// reuse it for the blocks that follow, which is the treeless literals form.
+Uint8List skewedAlphabet(int count, {int seed = 0x853C49E6748FEA9B}) {
+  const alphabet = 'aaaaaaaabbbbccdefghijklmnopqrstuvwxyz';
+  final random = Xorshift64(seed);
+  final out = Uint8List(count);
+  for (var i = 0; i < count; i++) {
+    out[i] = alphabet.codeUnitAt((random.next() >>> 16) % alphabet.length);
+  }
+  return out;
+}
+
+/// True when the `zstd` CLI is on the path.
+bool get hasZstdCli {
+  try {
+    return Process.runSync('zstd', const ['--version']).exitCode == 0;
+  } on ProcessException {
+    return false;
+  }
+}
+
+/// Compresses [input] with the `zstd` CLI, as `zstd -q -c <args> <file>`.
+///
+/// A file argument rather than a pipe, so the frame carries a content size.
+Uint8List zstdCli(List<String> args, Uint8List input) {
+  final directory = Directory.systemTemp.createTempSync('pure_zstd_corpus_');
+  try {
+    final file = File('${directory.path}/input.bin')..writeAsBytesSync(input);
+    final run = Process.runSync('zstd', [
+      '-q',
+      '-c',
+      ...args,
+      file.path,
+    ], stdoutEncoding: null);
+    if (run.exitCode != 0) {
+      throw StateError('zstd ${args.join(' ')} failed: ${run.stderr}');
+    }
+    return Uint8List.fromList(run.stdout as List<int>);
+  } finally {
+    directory.deleteSync(recursive: true);
+  }
+}