d970a0cca8
A feasibility spike, standalone so it can move to its own repository. Not wired into the app and not a dependency of it. Decompression only, and only what our containers need: no dictionaries, no multi-frame, no streaming, and the content checksum is parsed but never verified. Native only, since the bit reader carries 56 bits in an int. It is correct on real data. The committed fixture decodes to the samples the generator wrote, and tool/oracle.py drives a byte-for-byte comparison against pyzstd over 32 containers and 2,615 blocks from both tiers, every thinned column count from 3601 to 361, and 88 S to 80 N. All identical. The numbers, AOT on this box, 131,072 bytes a block: 0.66 ms median against pyzstd's 0.175, so about 200 MB/s against 750. Across three tiles the range is 0.66 to 0.94 ms, and a 113 block sweep is 67 to 105 ms. The cost is the sequence loop and nothing else: a terrain block at level 19 carries around 1,700 literals and 14,400 sequences, so optimising the literal path bought nothing.
103 lines
3.4 KiB
Python
Executable file
103 lines
3.4 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Check the Dart decoder against pyzstd over real .hgtz containers.
|
|
|
|
./tool/oracle.py path/to/srtm1z/N39W105.hgtz
|
|
./tool/oracle.py --every 1500 --limit 8 path/to/srtm1z
|
|
|
|
pyzstd is the reference. Every block of every container named is extracted as
|
|
its own frame, decompressed by pyzstd, and handed to tool/verify.dart, which
|
|
decodes it again and compares byte for byte. One tile at a time, through a
|
|
temporary directory that is deleted as it goes, because a whole tile's blocks
|
|
decompress to around 29 MB.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pyzstd
|
|
|
|
HEADER = ">4sBbHHH"
|
|
ENTRY = ">II"
|
|
|
|
|
|
def frames(path: Path):
|
|
"""Yields (index, frame bytes, expected decompressed size) per block."""
|
|
data = path.read_bytes()
|
|
magic, version, _level, rows, cols, block = struct.unpack_from(HEADER, data)
|
|
if magic != b"HGTZ" or version != 1:
|
|
raise SystemExit(f"{path}: not a version 1 .hgtz")
|
|
block_cols = (cols + block - 1) // block
|
|
block_rows = (rows + block - 1) // block
|
|
for index in range(block_rows * block_cols):
|
|
offset, length = struct.unpack_from(ENTRY, data, 12 + 8 * index)
|
|
top = (index // block_cols) * block
|
|
left = (index % block_cols) * block
|
|
height = min(block, rows - top)
|
|
width = min(block, cols - left)
|
|
yield index, data[offset : offset + length], height * width * 2
|
|
|
|
|
|
def check(path: Path, root: Path) -> bool:
|
|
with tempfile.TemporaryDirectory() as name:
|
|
directory = Path(name)
|
|
count = 0
|
|
for index, frame, expected in frames(path):
|
|
plain = pyzstd.decompress(frame)
|
|
if len(plain) != expected:
|
|
raise SystemExit(
|
|
f"{path} block {index}: pyzstd gave {len(plain)},"
|
|
f" the index implies {expected}"
|
|
)
|
|
(directory / f"{index:05d}.zst").write_bytes(frame)
|
|
(directory / f"{index:05d}.raw").write_bytes(plain)
|
|
count += 1
|
|
result = subprocess.run(
|
|
["dart", "run", "tool/verify.dart", str(directory)],
|
|
cwd=root,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
line = result.stdout.strip().splitlines()[-1] if result.stdout else ""
|
|
print(f"{path.name}: {count} blocks, {line or result.stderr.strip()}")
|
|
return result.returncode == 0
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("paths", nargs="+", type=Path)
|
|
parser.add_argument(
|
|
"--every",
|
|
type=int,
|
|
default=1,
|
|
help="when a directory is named, take every Nth container",
|
|
)
|
|
parser.add_argument("--limit", type=int, default=0)
|
|
arguments = parser.parse_args()
|
|
|
|
containers: list[Path] = []
|
|
for path in arguments.paths:
|
|
if path.is_dir():
|
|
found = sorted(path.glob("*.hgtz"))
|
|
containers += found[:: arguments.every]
|
|
else:
|
|
containers.append(path)
|
|
if arguments.limit:
|
|
containers = containers[: arguments.limit]
|
|
|
|
root = Path(__file__).resolve().parent.parent
|
|
failures = [c for c in containers if not check(c, root)]
|
|
if failures:
|
|
print(f"FAILED on {len(failures)} of {len(containers)} containers")
|
|
return 1
|
|
print(f"all {len(containers)} containers match pyzstd byte for byte")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|