d970a0cca8
A feasibility spike, standalone so it can move to its own repository. Not wired into the app and not a dependency of it. Decompression only, and only what our containers need: no dictionaries, no multi-frame, no streaming, and the content checksum is parsed but never verified. Native only, since the bit reader carries 56 bits in an int. It is correct on real data. The committed fixture decodes to the samples the generator wrote, and tool/oracle.py drives a byte-for-byte comparison against pyzstd over 32 containers and 2,615 blocks from both tiers, every thinned column count from 3601 to 361, and 88 S to 80 N. All identical. The numbers, AOT on this box, 131,072 bytes a block: 0.66 ms median against pyzstd's 0.175, so about 200 MB/s against 750. Across three tiles the range is 0.66 to 0.94 ms, and a 113 block sweep is 67 to 105 ms. The cost is the sequence loop and nothing else: a terrain block at level 19 carries around 1,700 literals and 14,400 sequences, so optimising the literal path bought nothing.
52 lines
1.5 KiB
Dart
52 lines
1.5 KiB
Dart
import 'dart:io';
|
|
import 'dart:typed_data';
|
|
|
|
import 'package:pure_zstd/pure_zstd.dart';
|
|
|
|
/// Decodes every `*.zst` in a directory and compares it to the `*.raw`
|
|
/// beside it. Driven by tool/oracle.py, which writes the pair.
|
|
void main(List<String> arguments) {
|
|
if (arguments.length != 1) {
|
|
stderr.writeln('usage: verify.dart <directory>');
|
|
exit(2);
|
|
}
|
|
final directory = Directory(arguments.single);
|
|
final frames =
|
|
directory
|
|
.listSync()
|
|
.whereType<File>()
|
|
.where((file) => file.path.endsWith('.zst'))
|
|
.toList()
|
|
..sort((a, b) => a.path.compareTo(b.path));
|
|
|
|
final decoder = ZstdDecoder();
|
|
var checked = 0;
|
|
var bytes = 0;
|
|
for (final frame in frames) {
|
|
final expected = File(
|
|
frame.path.replaceRange(frame.path.length - 4, null, '.raw'),
|
|
).readAsBytesSync();
|
|
final Uint8List actual;
|
|
try {
|
|
actual = decoder.decode(frame.readAsBytesSync());
|
|
} on ZstdException catch (error) {
|
|
stderr.writeln('${frame.path}: $error');
|
|
exit(1);
|
|
}
|
|
if (actual.length != expected.length) {
|
|
stderr.writeln(
|
|
'${frame.path}: ${actual.length} bytes, expected ${expected.length}',
|
|
);
|
|
exit(1);
|
|
}
|
|
for (var i = 0; i < actual.length; i++) {
|
|
if (actual[i] != expected[i]) {
|
|
stderr.writeln('${frame.path}: byte $i differs');
|
|
exit(1);
|
|
}
|
|
}
|
|
checked++;
|
|
bytes += actual.length;
|
|
}
|
|
stdout.writeln('$checked frames, $bytes bytes, identical');
|
|
}
|