103 lines
3.4 KiB
Python
103 lines
3.4 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Check the Dart decoder against pyzstd over real .hgtz containers.
|
||
|
|
|
||
|
|
./tool/oracle.py path/to/srtm1z/N39W105.hgtz
|
||
|
|
./tool/oracle.py --every 1500 --limit 8 path/to/srtm1z
|
||
|
|
|
||
|
|
pyzstd is the reference. Every block of every container named is extracted as
|
||
|
|
its own frame, decompressed by pyzstd, and handed to tool/verify.dart, which
|
||
|
|
decodes it again and compares byte for byte. One tile at a time, through a
|
||
|
|
temporary directory that is deleted as it goes, because a whole tile's blocks
|
||
|
|
decompress to around 29 MB.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import struct
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pyzstd
|
||
|
|
|
||
|
|
HEADER = ">4sBbHHH"
|
||
|
|
ENTRY = ">II"
|
||
|
|
|
||
|
|
|
||
|
|
def frames(path: Path):
|
||
|
|
"""Yields (index, frame bytes, expected decompressed size) per block."""
|
||
|
|
data = path.read_bytes()
|
||
|
|
magic, version, _level, rows, cols, block = struct.unpack_from(HEADER, data)
|
||
|
|
if magic != b"HGTZ" or version != 1:
|
||
|
|
raise SystemExit(f"{path}: not a version 1 .hgtz")
|
||
|
|
block_cols = (cols + block - 1) // block
|
||
|
|
block_rows = (rows + block - 1) // block
|
||
|
|
for index in range(block_rows * block_cols):
|
||
|
|
offset, length = struct.unpack_from(ENTRY, data, 12 + 8 * index)
|
||
|
|
top = (index // block_cols) * block
|
||
|
|
left = (index % block_cols) * block
|
||
|
|
height = min(block, rows - top)
|
||
|
|
width = min(block, cols - left)
|
||
|
|
yield index, data[offset : offset + length], height * width * 2
|
||
|
|
|
||
|
|
|
||
|
|
def check(path: Path, root: Path) -> bool:
|
||
|
|
with tempfile.TemporaryDirectory() as name:
|
||
|
|
directory = Path(name)
|
||
|
|
count = 0
|
||
|
|
for index, frame, expected in frames(path):
|
||
|
|
plain = pyzstd.decompress(frame)
|
||
|
|
if len(plain) != expected:
|
||
|
|
raise SystemExit(
|
||
|
|
f"{path} block {index}: pyzstd gave {len(plain)},"
|
||
|
|
f" the index implies {expected}"
|
||
|
|
)
|
||
|
|
(directory / f"{index:05d}.zst").write_bytes(frame)
|
||
|
|
(directory / f"{index:05d}.raw").write_bytes(plain)
|
||
|
|
count += 1
|
||
|
|
result = subprocess.run(
|
||
|
|
["dart", "run", "tool/verify.dart", str(directory)],
|
||
|
|
cwd=root,
|
||
|
|
capture_output=True,
|
||
|
|
text=True,
|
||
|
|
)
|
||
|
|
line = result.stdout.strip().splitlines()[-1] if result.stdout else ""
|
||
|
|
print(f"{path.name}: {count} blocks, {line or result.stderr.strip()}")
|
||
|
|
return result.returncode == 0
|
||
|
|
|
||
|
|
|
||
|
|
def main() -> int:
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
parser.add_argument("paths", nargs="+", type=Path)
|
||
|
|
parser.add_argument(
|
||
|
|
"--every",
|
||
|
|
type=int,
|
||
|
|
default=1,
|
||
|
|
help="when a directory is named, take every Nth container",
|
||
|
|
)
|
||
|
|
parser.add_argument("--limit", type=int, default=0)
|
||
|
|
arguments = parser.parse_args()
|
||
|
|
|
||
|
|
containers: list[Path] = []
|
||
|
|
for path in arguments.paths:
|
||
|
|
if path.is_dir():
|
||
|
|
found = sorted(path.glob("*.hgtz"))
|
||
|
|
containers += found[:: arguments.every]
|
||
|
|
else:
|
||
|
|
containers.append(path)
|
||
|
|
if arguments.limit:
|
||
|
|
containers = containers[: arguments.limit]
|
||
|
|
|
||
|
|
root = Path(__file__).resolve().parent.parent
|
||
|
|
failures = [c for c in containers if not check(c, root)]
|
||
|
|
if failures:
|
||
|
|
print(f"FAILED on {len(failures)} of {len(containers)} containers")
|
||
|
|
return 1
|
||
|
|
print(f"all {len(containers)} containers match pyzstd byte for byte")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|