Add XXH64 so the content checksum can be verified

The decoder parses Content_Checksum and steps over it without reading it.
An audit mutated real block data 600,000 times behind a valid frame header
and 34 percent decoded into wrong bytes with no error raised. A frame can
also set the checksum flag and supply no bytes at all. This is the only
mechanism that would catch a wrong decode from anything else, and the
package decodes elevation tiles on a phone.

RFC 8878 section 3.1.1 stores the low 32 bits of the XXH64 of the whole
content, seed zero, little-endian. Xxh64.digest32 is that value. The
incremental form is for the decoder, which writes the output block by
block and should not walk it a second time.

Every operation relies on int being a wrapping 64-bit two's complement
integer, which holds on the VM and on AOT and not under dart2js. The
package already targets native only.

Verified against the xxhash Python package, version 3.0.0, over both byte
patterns at every length class around the 32, 8, 4 and 1 byte boundaries,
over 400 random inputs one shot and in random chunkings, and end to end
against the trailing four bytes of a real zstd -19 --check frame.

lib/src/xxhash.dart

@@ -0,0 +1,169 @@
+/// XXH64, the hash zstd uses for `Content_Checksum`.
+///
+/// RFC 8878 section 3.1.1 stores the low 32 bits of the XXH64 of the whole
+/// decompressed frame content, seeded with zero, little-endian, after the last
+/// block. [Xxh64.digest32] is that value.
+///
+/// The algorithm is defined on unsigned 64-bit words that wrap. Dart's `int`
+/// on the VM and on AOT is a 64-bit two's complement integer whose `+`, `-`,
+/// `*` and `<<` wrap on overflow, and `>>>` shifts the raw bits in, so every
+/// operation here is exact with no masking. None of that holds under dart2js,
+/// where `int` is a double: the multiplies, the rotates and the wrapping adds
+/// would all be wrong. Like the rest of this package, it targets native only.
+library;
+
+import 'dart:typed_data';
+
+// The five primes, written as they appear in the reference implementation.
+// Dart reads a hex literal above 2^63 as its two's complement, so _prime1,
+// _prime2 and _prime4 are negative here. The bits are what matter.
+const int _prime1 = 0x9E3779B185EBCA87;
+const int _prime2 = 0xC2B2AE3D27D4EB4F;
+const int _prime3 = 0x165667B19E3779F9;
+const int _prime4 = 0x85EBCA77C2B2AE63;
+const int _prime5 = 0x27D4EB2F165667C5;
+
+/// Hashes [data] in one pass. Equivalent to `(Xxh64()..update(data)).digest`.
+int xxh64(Uint8List data, {int seed = 0}) =>
+    (Xxh64(seed: seed)..update(data)).digest;
+
+/// XXH64 over as many chunks as the caller has.
+///
+/// A multi-block frame can feed each block to [update] as it is written,
+/// rather than hashing the output again once it is whole. Reading [digest]
+/// does not end the hash; more may be fed in afterwards.
+class Xxh64 {
+  Xxh64({int seed = 0}) : _seed = seed {
+    reset();
+  }
+
+  final int _seed;
+
+  /// Whole 32-byte stripes are consumed straight from the caller's bytes.
+  /// This holds only the remainder between calls.
+  final Uint8List _tail = Uint8List(32);
+  late final ByteData _tailView = ByteData.sublistView(_tail);
+
+  int _tailLength = 0;
+  int _length = 0;
+
+  int _v1 = 0;
+  int _v2 = 0;
+  int _v3 = 0;
+  int _v4 = 0;
+
+  /// Discards everything fed in so far, back to the seed given to the
+  /// constructor.
+  void reset() {
+    _v1 = _seed + _prime1 + _prime2;
+    _v2 = _seed + _prime2;
+    _v3 = _seed;
+    _v4 = _seed - _prime1;
+    _tailLength = 0;
+    _length = 0;
+  }
+
+  /// Feeds `data[start..end)`, defaulting to all of [data].
+  ///
+  /// The range form is so a decoder can hand over the part of an output buffer
+  /// it just filled without taking a view of it.
+  void update(Uint8List data, [int start = 0, int? end]) {
+    final stop = RangeError.checkValidRange(start, end, data.length);
+    var at = start;
+    _length += stop - at;
+
+    if (_tailLength > 0) {
+      final take = stop - at < 32 - _tailLength ? stop - at : 32 - _tailLength;
+      _tail.setRange(_tailLength, _tailLength + take, data, at);
+      _tailLength += take;
+      at += take;
+      if (_tailLength < 32) {
+        return;
+      }
+      _stripe(_tailView, 0);
+      _tailLength = 0;
+    }
+
+    final view = ByteData.sublistView(data);
+    while (at + 32 <= stop) {
+      _stripe(view, at);
+      at += 32;
+    }
+
+    if (at < stop) {
+      _tail.setRange(0, stop - at, data, at);
+      _tailLength = stop - at;
+    }
+  }
+
+  /// The 64-bit hash of everything fed in so far.
+  int get digest {
+    // Under 32 bytes the accumulators were never touched, so the hash starts
+    // from the seed instead of from their merge. _v3 still holds the seed.
+    var h = _length >= 32
+        ? _merge(
+            _merge(
+              _merge(
+                _merge(
+                  _rotl(_v1, 1) +
+                      _rotl(_v2, 7) +
+                      _rotl(_v3, 12) +
+                      _rotl(_v4, 18),
+                  _v1,
+                ),
+                _v2,
+              ),
+              _v3,
+            ),
+            _v4,
+          )
+        : _v3 + _prime5;
+    h += _length;
+
+    // The remainder is consumed eight bytes at a time, then four, then one.
+    var at = 0;
+    while (at + 8 <= _tailLength) {
+      h ^= _round(0, _tailView.getInt64(at, Endian.little));
+      h = _rotl(h, 27) * _prime1 + _prime4;
+      at += 8;
+    }
+    if (at + 4 <= _tailLength) {
+      h ^= _tailView.getUint32(at, Endian.little) * _prime1;
+      h = _rotl(h, 23) * _prime2 + _prime3;
+      at += 4;
+    }
+    while (at < _tailLength) {
+      h ^= _tail[at] * _prime5;
+      h = _rotl(h, 11) * _prime1;
+      at++;
+    }
+
+    h ^= h >>> 33;
+    h *= _prime2;
+    h ^= h >>> 29;
+    h *= _prime3;
+    h ^= h >>> 32;
+    return h;
+  }
+
+  /// The low 32 bits of [digest]: the value zstd stores as
+  /// `Content_Checksum`.
+  int get digest32 => digest & 0xFFFFFFFF;
+
+  void _stripe(ByteData view, int at) {
+    _v1 = _round(_v1, view.getInt64(at, Endian.little));
+    _v2 = _round(_v2, view.getInt64(at + 8, Endian.little));
+    _v3 = _round(_v3, view.getInt64(at + 16, Endian.little));
+    _v4 = _round(_v4, view.getInt64(at + 24, Endian.little));
+  }
+
+  static int _round(int accumulator, int input) {
+    final v = accumulator + input * _prime2;
+    return _rotl(v, 31) * _prime1;
+  }
+
+  static int _merge(int h, int accumulator) =>
+      (h ^ _round(0, accumulator)) * _prime1 + _prime4;
+
+  static int _rotl(int v, int bits) => (v << bits) | (v >>> (64 - bits));
+}

test/xxhash_test.dart

@@ -0,0 +1,241 @@
+import 'dart:typed_data';
+
+import 'package:pure_zstd/src/xxhash.dart';
+import 'package:test/test.dart';
+
+// Every digest below came from the `xxhash` Python package, version 3.0.0,
+// which wraps the reference C implementation. The two byte generators are
+// written identically on both sides, so the inputs are reproducible here
+// without carrying the bytes themselves.
+//
+// The lengths are chosen around the boundaries of the tail, which is
+// consumed in 32-byte stripes, then 8, then 4, then 1. Each of those is a
+// separate branch and each gets a length either side of it.
+
+/// A repeating pattern. 251 is prime, so it does not line up with the stripe.
+Uint8List _pattern(int length) =>
+    Uint8List.fromList(List.generate(length, (i) => i % 251));
+
+/// Bytes with no structure, from a deterministic generator rather than
+/// `Random`, so the expected digests stay valid.
+Uint8List _pseudoRandom(int length) {
+  final out = Uint8List(length);
+  var x = 1;
+  for (var i = 0; i < length; i++) {
+    x = (x * 1103515245 + 12345) & 0x7FFFFFFF;
+    out[i] = (x >> 16) & 0xFF;
+  }
+  return out;
+}
+
+const List<(int, int)> _patternDigests = [
+  (0, 0xEF46DB3751D8E999),
+  (1, 0xE934A84ADB052768),
+  (2, 0x4865A00BA85B7CD6),
+  (3, 0xE5C7BB4533BC65DD),
+  (4, 0xFFCED8604453CC1E),
+  (5, 0xDD0274386E26030C),
+  (7, 0x14CC643F630C72D2),
+  (8, 0x884A173614B81B8D),
+  (9, 0x67D85784A7C78C5B),
+  (15, 0xA948F5F0F6ABAC2D),
+  (16, 0x44B6EF2FB84169F7),
+  (17, 0x5603E60C527599B6),
+  (31, 0xC346D2B59B4D8EE1),
+  (32, 0xCBF59C5116FF32B4),
+  (33, 0x0C535D1ACAFB8EAD),
+  (63, 0xE26AA9E2A95F8E4F),
+  (64, 0xF7C67301DB6713F0),
+  (65, 0xC31EB63B2AE4465B),
+  (127, 0x464D085810CE0199),
+  (128, 0x7A7FE14647B9AB92),
+  (129, 0x0BA25DFD6E891FCF),
+  (255, 0x566D96B832B967C1),
+  (256, 0xF33944343EE85824),
+  (1000, 0xF306F04AA88B54D3),
+  (4096, 0x122A8C8D994AD3EC),
+  (5000, 0xA6833D648FD6A332),
+];
+
+const List<(int, int)> _pseudoRandomDigests = [
+  (0, 0xEF46DB3751D8E999),
+  (1, 0xD2DA77930F69647C),
+  (2, 0x9F33D4BD50F99C34),
+  (3, 0xF7C331CF2042B940),
+  (4, 0x1B042F821FC4A793),
+  (5, 0x6E1F98107D9DB571),
+  (7, 0x41D39023940006C8),
+  (8, 0xEF7823FCE4AD9AFB),
+  (9, 0x81CF88E9D8340150),
+  (15, 0x6F0095B1C00E82FB),
+  (16, 0x29965039DF6047BB),
+  (17, 0x5236FCB96D8FB228),
+  (31, 0x68C3DBB752F3FA97),
+  (32, 0xAD7DC5A569BB047C),
+  (33, 0x475B642AC692E380),
+  (63, 0xC18C4743DE1B99A8),
+  (64, 0x06872C5D6370DE25),
+  (65, 0x8B5F4E34A32C5A09),
+  (127, 0x77DC651915D836B5),
+  (128, 0x2E9C4CB0B29BC8EC),
+  (129, 0xB5E3A45BFF735CFA),
+  (255, 0x9AD1F660B88ABC38),
+  (256, 0x9AE30993C3F4B05A),
+  (1000, 0x7AD5B044FB6FA81D),
+  (4096, 0x4AFD74BF8BD52DF5),
+  (5000, 0xCAEF4049E5C55187),
+];
+
+/// `Mt Herman 2764 m\n` 241 times, through `zstd -19 --check`. The frame is
+/// embedded whole so the trailing `Content_Checksum` is read from a real
+/// frame rather than asserted from a constant.
+const String _checkedFrame =
+    '28b52ffd64010fcd0000884d74204865726d616e2032373634206d0a01'
+    '00db4ffe5c02fe13e790';
+
+Uint8List _hex(String text) {
+  final out = Uint8List(text.length ~/ 2);
+  for (var i = 0; i < out.length; i++) {
+    out[i] = int.parse(text.substring(i * 2, i * 2 + 2), radix: 16);
+  }
+  return out;
+}
+
+void main() {
+  group('xxh64 in one shot', () {
+    for (final (length, expected) in _patternDigests) {
+      test('matches the reference over $length repeating bytes', () {
+        expect(xxh64(_pattern(length)), expected);
+      });
+    }
+
+    for (final (length, expected) in _pseudoRandomDigests) {
+      test('matches the reference over $length unstructured bytes', () {
+        expect(xxh64(_pseudoRandom(length)), expected);
+      });
+    }
+
+    test('honours a seed, including one that is negative as a Dart int', () {
+      expect(xxh64(_pattern(0), seed: 1), 0xD5AFBA1336A3BE4B);
+      expect(xxh64(_pattern(5), seed: 1), 0x27CE45508B78A340);
+      expect(xxh64(_pattern(32), seed: 1), 0xD74E6766CE9DBA94);
+      expect(xxh64(_pattern(200), seed: 1), 0x20BD094800CB1DFA);
+
+      const seed = 0x9E3779B185EBCA87;
+      expect(seed.isNegative, isTrue);
+      expect(xxh64(_pattern(0), seed: seed), 0x6EC6D05F61C7E7A7);
+      expect(xxh64(_pattern(5), seed: seed), 0xF60DC54C180A9098);
+      expect(xxh64(_pattern(32), seed: seed), 0xBFB3E4EF6096C49C);
+      expect(xxh64(_pattern(200), seed: seed), 0x6EA98426E0B64F4A);
+    });
+  });
+
+  group('the incremental form', () {
+    // Lengths either side of every boundary, plus enough stripes that a
+    // chunking can straddle several.
+    const lengths = [
+      0,
+      1,
+      3,
+      4,
+      7,
+      8,
+      15,
+      16,
+      31,
+      32,
+      33,
+      63,
+      64,
+      65,
+      127,
+      128,
+      129,
+      1000,
+      4096,
+      5001,
+    ];
+
+    for (final chunk in [1, 3, 7, 8, 17, 31, 32, 33, 64, 250]) {
+      test('in $chunk-byte chunks equals the one shot', () {
+        for (final length in lengths) {
+          final data = _pseudoRandom(length);
+          final hash = Xxh64();
+          for (var at = 0; at < length; at += chunk) {
+            final end = at + chunk < length ? at + chunk : length;
+            hash.update(data, at, end);
+          }
+          expect(hash.digest, xxh64(data), reason: '$length bytes');
+        }
+      });
+    }
+
+    test('ignores empty chunks, leading, trailing and interleaved', () {
+      final data = _pattern(300);
+      final empty = Uint8List(0);
+      final hash = Xxh64()
+        ..update(empty)
+        ..update(data, 0, 1)
+        ..update(empty)
+        ..update(data, 1, 40)
+        ..update(data, 40, 40)
+        ..update(data, 40, 300)
+        ..update(empty);
+      expect(hash.digest, xxh64(data));
+    });
+
+    test('accepts a whole list with no range', () {
+      final data = _pattern(100);
+      final hash = Xxh64()
+        ..update(Uint8List.sublistView(data, 0, 37))
+        ..update(Uint8List.sublistView(data, 37));
+      expect(hash.digest, xxh64(data));
+    });
+
+    test('reads the same digest twice and keeps hashing afterwards', () {
+      final data = _pattern(500);
+      final hash = Xxh64()..update(data, 0, 200);
+      expect(hash.digest, xxh64(_pattern(200)));
+      expect(hash.digest, xxh64(_pattern(200)));
+      hash.update(data, 200, 500);
+      expect(hash.digest, xxh64(data));
+    });
+
+    test('reset returns it to the state of a fresh instance', () {
+      final hash = Xxh64()..update(_pattern(999));
+      hash.reset();
+      expect(hash.digest, xxh64(_pattern(0)));
+      hash.update(_pattern(65));
+      expect(hash.digest, xxh64(_pattern(65)));
+    });
+
+    test('rejects a range outside the list', () {
+      expect(() => Xxh64().update(_pattern(10), 0, 11), throwsRangeError);
+      expect(() => Xxh64().update(_pattern(10), -1), throwsRangeError);
+      expect(() => Xxh64().update(_pattern(10), 6, 5), throwsRangeError);
+    });
+  });
+
+  group('a real zstd frame', () {
+    final frame = _hex(_checkedFrame);
+    final content = Uint8List.fromList(
+      List.filled(
+        241,
+        'Mt Herman 2764 m\n'.codeUnits,
+      ).expand((e) => e).toList(),
+    );
+
+    test('declares a content checksum', () {
+      expect((frame[4] >> 2) & 1, 1);
+    });
+
+    test('ends with the low 32 bits of the content digest', () {
+      final stored = ByteData.sublistView(
+        frame,
+        frame.length - 4,
+      ).getUint32(0, Endian.little);
+      expect((Xxh64()..update(content)).digest32, stored);
+      expect(xxh64(content) & 0xFFFFFFFF, stored);
+    });
+  });
+}