Bring the decoder over from the project it grew in

Eighteen commits of history, grafted onto the scaffold rather than over
it. It leaves behind everything that only meant something inside that
project: paths under one machine's mount, references to a transcoder
that is not here, and a benchmark defaulting to a tile nobody else has.

One fixture is real terrain and stays, because it is the only small
input that exercises the direct-nibble Huffman weight form. Its licence
requires attribution to travel with it, so the README's
acknowledgements and the corpus manifest both carry the Copernicus
notice now. Everything else committed here is synthetic or was written
by the zstd CLI, including the .hgtz container, whose name looks like a
tile and whose samples are a function of their position.

benchmark/bench.dart

@@ -0,0 +1,125 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+/// Times decoding real `.hgtz` blocks.
+///
+///     dart run benchmark/bench.dart [path to a .hgtz]
+///     dart compile exe benchmark/bench.dart -o /tmp/bench && /tmp/bench
+///
+/// Reports the cost of one 256 by 256 block, which is 131,072 bytes out, and
+/// the cost of the 113 blocks a 360 degree skyline sweep touches. The frames
+/// are read into memory first, so nothing but decoding is timed. The output
+/// buffer is allocated per call, as the API allocates it, so the figure
+/// includes that allocation and the garbage it makes.
+const int sweepBlocks = 113;
+
+void main(List<String> arguments) {
+  // A .hgtz container of real terrain. There is no useful default: the
+  // committed fixture is synthetic and compresses to almost no sequences, so
+  // it reads about six times too fast and is not worth timing.
+  final path = arguments.isEmpty
+      ? (throw ArgumentError('usage: bench <a .hgtz container>'))
+      : arguments.single;
+  final bytes = File(path).readAsBytesSync();
+  final header = ByteData.sublistView(bytes);
+  final rows = header.getUint16(6);
+  final columns = header.getUint16(8);
+  final block = header.getUint16(10);
+  final blockColumns = (columns + block - 1) ~/ block;
+  final blockRows = (rows + block - 1) ~/ block;
+
+  // Only the square blocks, so every timing is over the same 131,072 bytes.
+  final full = <Uint8List>[];
+  for (var index = 0; index < blockRows * blockColumns; index++) {
+    final entry = 12 + 8 * index;
+    final top = (index ~/ blockColumns) * block;
+    final left = (index % blockColumns) * block;
+    if (rows - top < block || columns - left < block) {
+      continue;
+    }
+    final offset = header.getUint32(entry);
+    full.add(
+      Uint8List.sublistView(
+        bytes,
+        offset,
+        offset + header.getUint32(entry + 4),
+      ),
+    );
+  }
+  if (full.isEmpty) {
+    stderr.writeln('$path has no full $block by $block block');
+    exit(2);
+  }
+
+  final decoder = ZstdDecoder();
+  final size = decoder.decode(full.first).length;
+  stdout.writeln(path);
+  stdout.writeln(
+    '${full.length} full blocks of $size bytes, '
+    '${(full.map((f) => f.length).reduce((a, b) => a + b) / full.length).round()}'
+    ' compressed bytes on average',
+  );
+
+  // One block, over and over. The same frame every time, so the caches are as
+  // warm as they will ever be. This is the optimistic figure.
+  final one = full[full.length ~/ 2];
+  for (var i = 0; i < 300; i++) {
+    decoder.decode(one);
+  }
+  final repeated = <int>[];
+  for (var i = 0; i < 1000; i++) {
+    final watch = Stopwatch()..start();
+    decoder.decode(one);
+    watch.stop();
+    repeated.add(watch.elapsedMicroseconds);
+  }
+  _report('one block repeated, warm', repeated, size);
+
+  // Every full block once, which is the mix a real sweep meets: different
+  // entropy tables, different literal counts, different sequence counts.
+  final each = <int>[];
+  for (var pass = 0; pass < 8; pass++) {
+    for (final frame in full) {
+      final watch = Stopwatch()..start();
+      decoder.decode(frame);
+      watch.stop();
+      if (pass > 0) {
+        each.add(watch.elapsedMicroseconds);
+      }
+    }
+  }
+  _report('every full block, warm', each, size);
+
+  // A sweep as one unit, since that is the latency the user feels.
+  final sweeps = <int>[];
+  for (var pass = 0; pass < 30; pass++) {
+    final watch = Stopwatch()..start();
+    for (var i = 0; i < sweepBlocks; i++) {
+      decoder.decode(full[i % full.length]);
+    }
+    watch.stop();
+    sweeps.add(watch.elapsedMicroseconds);
+  }
+  sweeps.sort();
+  final sweep = sweeps[sweeps.length ~/ 2] / 1000;
+  stdout.writeln(
+    '$sweepBlocks block sweep, median      ${sweep.toStringAsFixed(2)} ms '
+    '(${(sweep / sweepBlocks * 1000).toStringAsFixed(0)} us per block)',
+  );
+}
+
+void _report(String label, List<int> microseconds, int size) {
+  microseconds.sort();
+  final median = microseconds[microseconds.length ~/ 2];
+  final low = microseconds[microseconds.length ~/ 10];
+  final high = microseconds[microseconds.length * 9 ~/ 10];
+  final mbPerSecond = size / median; // bytes per microsecond is MB/s.
+  stdout.writeln(
+    '${label.padRight(28)} ${(median / 1000).toStringAsFixed(3)} ms '
+    'median  (p10 ${(low / 1000).toStringAsFixed(3)}, '
+    'p90 ${(high / 1000).toStringAsFixed(3)}, '
+    'n=${microseconds.length})  ${mbPerSecond.toStringAsFixed(0)} MB/s',
+  );
+}

benchmark/levels.dart

@@ -0,0 +1,91 @@
+// Times several extracted-block files against each other, interleaved, so that
+// CPU frequency drift lands on every file equally.
+//
+//     dart compile exe levels.dart -o /tmp/levels && /tmp/levels a.blocks b.blocks
+//
+// Each file is the format benchmark/same_blocks.dart reads: uint32 count, then
+// per block a uint32 frame length, a uint32 decompressed size, and the frame.
+// Every file must hold the same number of blocks, of the same output size, so
+// the medians are like for like.
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+const int warmSeconds = 4;
+const int passes = 60;
+
+List<Uint8List> load(String path, List<int> sizes) {
+  final bytes = File(path).readAsBytesSync();
+  final data = ByteData.sublistView(bytes);
+  final count = data.getUint32(0, Endian.big);
+  final frames = <Uint8List>[];
+  var at = 4;
+  for (var i = 0; i < count; i++) {
+    final length = data.getUint32(at, Endian.big);
+    final size = data.getUint32(at + 4, Endian.big);
+    if (sizes.length <= i) {
+      sizes.add(size);
+    } else if (sizes[i] != size) {
+      throw StateError(
+        '$path block $i is $size bytes out, expected ${sizes[i]}',
+      );
+    }
+    at += 8;
+    frames.add(Uint8List.sublistView(bytes, at, at + length));
+    at += length;
+  }
+  return frames;
+}
+
+void main(List<String> args) {
+  final sizes = <int>[];
+  final sets = args.map((path) => load(path, sizes)).toList();
+  for (final set in sets) {
+    if (set.length != sets.first.length) {
+      throw StateError('the files do not hold the same number of blocks');
+    }
+    for (var i = 0; i < set.length; i++) {
+      if (zstdDecode(set[i]).length != sizes[i]) {
+        throw StateError('block $i decoded to the wrong size');
+      }
+    }
+  }
+
+  // Spin long enough for the governor to raise the clock before anything is
+  // timed, otherwise the first file measured pays for the ramp.
+  final warm = Stopwatch()..start();
+  while (warm.elapsed.inSeconds < warmSeconds) {
+    for (final set in sets) {
+      for (final frame in set) {
+        zstdDecode(frame);
+      }
+    }
+  }
+
+  final micros = [for (final _ in sets) <int>[]];
+  for (var pass = 0; pass < passes; pass++) {
+    for (var b = 0; b < sets.first.length; b++) {
+      for (var s = 0; s < sets.length; s++) {
+        final watch = Stopwatch()..start();
+        zstdDecode(sets[s][b]);
+        watch.stop();
+        micros[s].add(watch.elapsedMicroseconds);
+      }
+    }
+  }
+
+  for (var s = 0; s < sets.length; s++) {
+    final xs = micros[s]..sort();
+    double q(double f) => xs[(xs.length * f).floor()] / 1000;
+    final median = q(0.5);
+    final bytes = sizes.reduce((a, b) => a + b) / sizes.length;
+    stdout.writeln(
+      '${args[s].split('/').last.padRight(24)} n=${xs.length} '
+      'median=${median.toStringAsFixed(3)}ms '
+      'p10=${q(0.1).toStringAsFixed(3)} p90=${q(0.9).toStringAsFixed(3)} '
+      'MBps=${(bytes / 1048576 / (median / 1000)).toStringAsFixed(0)} '
+      'sweep113=${(median * 113).toStringAsFixed(0)}ms',
+    );
+  }
+}

benchmark/same_blocks.dart

@@ -0,0 +1,51 @@
+// Times the same extracted blocks the on-device benchmark uses, so the two
+// numbers are like for like.
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:pure_zstd/pure_zstd.dart';
+
+void main(List<String> args) {
+  final bytes = File(args.first).readAsBytesSync();
+  final data = ByteData.sublistView(bytes);
+  final count = data.getUint32(0, Endian.big);
+  final frames = <Uint8List>[];
+  final sizes = <int>[];
+  var at = 4;
+  for (var i = 0; i < count; i++) {
+    final length = data.getUint32(at, Endian.big);
+    sizes.add(data.getUint32(at + 4, Endian.big));
+    at += 8;
+    frames.add(Uint8List.sublistView(bytes, at, at + length));
+    at += length;
+  }
+  for (var i = 0; i < frames.length; i++) {
+    if (zstdDecode(frames[i]).length != sizes[i]) {
+      throw StateError('block $i decoded to the wrong size');
+    }
+  }
+  for (var pass = 0; pass < 20; pass++) {
+    for (final frame in frames) {
+      zstdDecode(frame);
+    }
+  }
+  final micros = <int>[];
+  for (var pass = 0; pass < 60; pass++) {
+    for (final frame in frames) {
+      final watch = Stopwatch()..start();
+      zstdDecode(frame);
+      watch.stop();
+      micros.add(watch.elapsedMicroseconds);
+    }
+  }
+  micros.sort();
+  double q(double f) => micros[(micros.length * f).floor()] / 1000;
+  final median = q(0.5);
+  stdout.writeln(
+    'samples=${micros.length} bytesOut=${sizes.first} '
+    'median=${median.toStringAsFixed(3)}ms p10=${q(0.1).toStringAsFixed(3)} '
+    'p90=${q(0.9).toStringAsFixed(3)} '
+    'MBps=${(sizes.first / 1048576 / (median / 1000)).toStringAsFixed(0)} '
+    'sweep113=${(median * 113).toStringAsFixed(0)}ms',
+  );
+}