Test the guards that had no test
A sweep found no defect in the decoder and several in its tests. Three of the five fixes could be reverted with all 103 tests still green. The content size against expectedSize check is the worst of those, because the .hgtz reader was simplified to rest on it: deleting it left everything passing. Eleven cases cover it now. The RLE block ceiling and the requirement that a frame be the whole input had none either, so there are fixtures for a 13-byte frame that returns 2 MB, for the same shape at exactly the maximum, for four bytes of junk after a frame, and for a trailing skippable frame. Two of the new fixtures close coverage gaps rather than guards. No legal frame in the suite decoded a one-stream Huffman literals section, since the only one present had to be refused. And the only test of hashing each block as it lands rode on the zstd CLI being installed, so a multi-block frame carrying a checksum is committed now. Also replaces an assertion that constrained nothing. "Fills the buffer exactly" checked that the count was over 256 and under 1024, which passes with the guard at two, at three, or absent. The stream encodes 257 symbols, a buffer of 257 takes it and one of 256 does not. And two comments that were wrong. The weight buffer is one larger than the maximum on purpose, and shrinking it to 255 breaks 124 tests. A caller passing expectedSize gets the exact check in addition to the ceiling, not instead of it.
This commit is contained in:
Tag
parent
b221ae0e98
commit
45cab0620f
12 changed files with 272 additions and 170 deletions
|
|
@ -71,6 +71,39 @@ void main() {
|
|||
});
|
||||
});
|
||||
|
||||
group('a frame whose declared size is not the one asked for', () {
|
||||
// A guard callers rest on, and it had no test:
|
||||
// deleting the whole check left every other case green. A caller that
|
||||
// knows the size, as the .hgtz reader does, gets an exact comparison
|
||||
// against the frame's own figure rather than only the ceiling. Left to
|
||||
// the end, a frame declaring less decoded its own content happily and a
|
||||
// frame declaring more allocated whatever it asked for first.
|
||||
for (final entry in corpus.where(
|
||||
// Not a buffer holding more than one frame: this decoder refuses
|
||||
// those whatever size it is told to expect.
|
||||
(entry) =>
|
||||
entry.legal &&
|
||||
entry.expectedSize == null &&
|
||||
entry.trailing == null &&
|
||||
entry.size! > 1,
|
||||
)) {
|
||||
test('${entry.name} is refused when asked for one byte more', () {
|
||||
expect(
|
||||
() =>
|
||||
ZstdDecoder().decode(entry.frame, expectedSize: entry.size! + 1),
|
||||
throwsA(isA<ZstdException>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('${entry.name} still decodes when asked for the size it has', () {
|
||||
expect(
|
||||
ZstdDecoder().decode(entry.frame, expectedSize: entry.size),
|
||||
orderedEquals(entry.plain),
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
group('a frame declaring no content size', () {
|
||||
final entry = corpus.singleWhere((entry) => entry.name == 'nofcs_windowed');
|
||||
final size = entry.expectedSize!;
|
||||
|
|
|
|||
BIN
test/fixtures/corpus/lit_one_stream.raw.gz
vendored
Normal file
BIN
test/fixtures/corpus/lit_one_stream.raw.gz
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/lit_one_stream.zst
vendored
Normal file
BIN
test/fixtures/corpus/lit_one_stream.zst
vendored
Normal file
Binary file not shown.
361
test/fixtures/corpus/manifest.json
vendored
361
test/fixtures/corpus/manifest.json
vendored
|
|
@ -1,157 +1,208 @@
|
|||
[
|
||||
{
|
||||
"name": "twoframes",
|
||||
"frame": "twoframes.zst",
|
||||
"command": "python3: pyzstd.compress(A, 3) + pyzstd.compress(B, 3), A = 5000 xorshift64 bytes, B = b'the quick brown fox ' * 500",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"A.raw.gz",
|
||||
"B.raw.gz"
|
||||
],
|
||||
"size": 15000,
|
||||
"trailing": "a second complete frame"
|
||||
},
|
||||
{
|
||||
"name": "checksum_good",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"A.raw.gz"
|
||||
],
|
||||
"size": 5000
|
||||
},
|
||||
{
|
||||
"name": "checksum_flipped",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin, then the last checksum byte flipped",
|
||||
"verdict": "refused",
|
||||
"why": "pyzstd refuses it: ZstdError",
|
||||
"derive": "the last checksum byte flipped"
|
||||
},
|
||||
{
|
||||
"name": "checksum_zeroed",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin, then all four checksum bytes zeroed",
|
||||
"verdict": "refused",
|
||||
"why": "pyzstd refuses it: ZstdError",
|
||||
"derive": "all four checksum bytes zeroed"
|
||||
},
|
||||
{
|
||||
"name": "checksum_truncated",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin, then the last two bytes cut",
|
||||
"verdict": "refused",
|
||||
"why": "pyzstd refuses it: ZstdError",
|
||||
"derive": "the last two bytes cut"
|
||||
},
|
||||
{
|
||||
"name": "fcs_max_u64",
|
||||
"frame": "fcs_max_u64.zst",
|
||||
"command": "python3: pyzstd.compress(b'hello world', 3) with the descriptor rewritten to Frame_Content_Size_flag=3 declaring 18446744073709551615",
|
||||
"verdict": "refused",
|
||||
"why": "a 27-byte frame asking for a size no allocator will give; pyzstd refuses it: ZstdError"
|
||||
},
|
||||
{
|
||||
"name": "fcs_2p40",
|
||||
"frame": "fcs_2p40.zst",
|
||||
"command": "python3: pyzstd.compress(b'hello world', 3) with the descriptor rewritten to Frame_Content_Size_flag=3 declaring 1099511627776",
|
||||
"verdict": "refused",
|
||||
"why": "a 27-byte frame asking for a size no allocator will give; pyzstd refuses it: MemoryError"
|
||||
},
|
||||
{
|
||||
"name": "fcs8",
|
||||
"frame": "fcs8.zst",
|
||||
"command": "python3: pyzstd.compress(B, 3) with the descriptor rewritten to Frame_Content_Size_flag=3 and an 8-byte size",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"B.raw.gz"
|
||||
],
|
||||
"size": 10000
|
||||
},
|
||||
{
|
||||
"name": "huf_weight_overrun",
|
||||
"frame": "huf_weight_overrun.zst",
|
||||
"command": "hand-built: a compressed block whose Huffman weight description is [0x10, 0x3F] and whose FSE weight stream is 33 x 0xAA then 0x02",
|
||||
"verdict": "refused",
|
||||
"why": "the weight stream decodes past the 256th weight; pyzstd refuses it: ZstdError"
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt0_R31",
|
||||
"frame": "rlelit_fmt0_R31.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=0, Regenerated_Size=31, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt0_R31.raw.gz"
|
||||
],
|
||||
"size": 31
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt1_R4000",
|
||||
"frame": "rlelit_fmt1_R4000.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=1, Regenerated_Size=4000, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt1_R4000.raw.gz"
|
||||
],
|
||||
"size": 4000
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt2_R70000",
|
||||
"frame": "rlelit_fmt2_R70000.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=2, Regenerated_Size=70000, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt2_R70000.raw.gz"
|
||||
],
|
||||
"size": 70000
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt2_R131072",
|
||||
"frame": "rlelit_fmt2_R131072.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=2, Regenerated_Size=131072, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt2_R131072.raw.gz"
|
||||
],
|
||||
"size": 131072
|
||||
},
|
||||
{
|
||||
"name": "mixed_rle_raw",
|
||||
"frame": "mixed_rle_raw.zst",
|
||||
"command": "hand-built: [RLE 5000 x 0xAB][raw 33 bytes][RLE 40000 x 0x07] in one frame",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"mixed_rle_raw.raw.gz"
|
||||
],
|
||||
"size": 45033
|
||||
},
|
||||
{
|
||||
"name": "nofcs_windowed",
|
||||
"frame": "nofcs_windowed.zst",
|
||||
"command": "zstd -3 --no-check from stdin over b'the quick brown fox jumps over the lazy dog. ' * 24000",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"text.raw.gz"
|
||||
],
|
||||
"size": 1080000,
|
||||
"expectedSize": 1080000
|
||||
},
|
||||
{
|
||||
"name": "direct_nibble",
|
||||
"frame": "direct_nibble.zst",
|
||||
"command": "block 183 of path/to/srtm1z/N00E042.hgtz, lifted out as its own frame; its literals carry the direct-nibble Huffman weight form",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"direct_nibble.raw.gz"
|
||||
],
|
||||
"size": 131072
|
||||
},
|
||||
{
|
||||
"name": "dictflag",
|
||||
"frame": "dictflag.zst",
|
||||
"command": "python3: pyzstd.compress(A, 3) with Dictionary_ID_flag forced to 1 in the frame header descriptor",
|
||||
"verdict": "refused",
|
||||
"why": "a dictionary this decoder does not have; pyzstd refuses it: ZstdError"
|
||||
}
|
||||
{
|
||||
"name": "twoframes",
|
||||
"frame": "twoframes.zst",
|
||||
"command": "python3: pyzstd.compress(A, 3) + pyzstd.compress(B, 3), A = 5000 xorshift64 bytes, B = b'the quick brown fox ' * 500",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"A.raw.gz",
|
||||
"B.raw.gz"
|
||||
],
|
||||
"size": 15000,
|
||||
"trailing": "a second complete frame"
|
||||
},
|
||||
{
|
||||
"name": "checksum_good",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"A.raw.gz"
|
||||
],
|
||||
"size": 5000
|
||||
},
|
||||
{
|
||||
"name": "checksum_flipped",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin, then the last checksum byte flipped",
|
||||
"verdict": "refused",
|
||||
"why": "pyzstd refuses it: ZstdError",
|
||||
"derive": "the last checksum byte flipped"
|
||||
},
|
||||
{
|
||||
"name": "checksum_zeroed",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin, then all four checksum bytes zeroed",
|
||||
"verdict": "refused",
|
||||
"why": "pyzstd refuses it: ZstdError",
|
||||
"derive": "all four checksum bytes zeroed"
|
||||
},
|
||||
{
|
||||
"name": "checksum_truncated",
|
||||
"frame": "checksum_good.zst",
|
||||
"command": "zstd -q -c -3 --check A.bin, then the last two bytes cut",
|
||||
"verdict": "refused",
|
||||
"why": "pyzstd refuses it: ZstdError",
|
||||
"derive": "the last two bytes cut"
|
||||
},
|
||||
{
|
||||
"name": "fcs_max_u64",
|
||||
"frame": "fcs_max_u64.zst",
|
||||
"command": "python3: pyzstd.compress(b'hello world', 3) with the descriptor rewritten to Frame_Content_Size_flag=3 declaring 18446744073709551615",
|
||||
"verdict": "refused",
|
||||
"why": "a 27-byte frame asking for a size no allocator will give; pyzstd refuses it: ZstdError"
|
||||
},
|
||||
{
|
||||
"name": "fcs_2p40",
|
||||
"frame": "fcs_2p40.zst",
|
||||
"command": "python3: pyzstd.compress(b'hello world', 3) with the descriptor rewritten to Frame_Content_Size_flag=3 declaring 1099511627776",
|
||||
"verdict": "refused",
|
||||
"why": "a 27-byte frame asking for a size no allocator will give; pyzstd refuses it: MemoryError"
|
||||
},
|
||||
{
|
||||
"name": "fcs8",
|
||||
"frame": "fcs8.zst",
|
||||
"command": "python3: pyzstd.compress(B, 3) with the descriptor rewritten to Frame_Content_Size_flag=3 and an 8-byte size",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"B.raw.gz"
|
||||
],
|
||||
"size": 10000
|
||||
},
|
||||
{
|
||||
"name": "huf_weight_overrun",
|
||||
"frame": "huf_weight_overrun.zst",
|
||||
"command": "hand-built: a compressed block whose Huffman weight description is [0x10, 0x3F] and whose FSE weight stream is 33 x 0xAA then 0x02",
|
||||
"verdict": "refused",
|
||||
"why": "the weight stream decodes past the 256th weight; pyzstd refuses it: ZstdError"
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt0_R31",
|
||||
"frame": "rlelit_fmt0_R31.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=0, Regenerated_Size=31, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt0_R31.raw.gz"
|
||||
],
|
||||
"size": 31
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt1_R4000",
|
||||
"frame": "rlelit_fmt1_R4000.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=1, Regenerated_Size=4000, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt1_R4000.raw.gz"
|
||||
],
|
||||
"size": 4000
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt2_R70000",
|
||||
"frame": "rlelit_fmt2_R70000.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=2, Regenerated_Size=70000, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt2_R70000.raw.gz"
|
||||
],
|
||||
"size": 70000
|
||||
},
|
||||
{
|
||||
"name": "rlelit_fmt2_R131072",
|
||||
"frame": "rlelit_fmt2_R131072.zst",
|
||||
"command": "hand-built: one compressed block, RLE literals Size_Format=2, Regenerated_Size=131072, byte 0x5A, Number_of_Sequences=0",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rlelit_fmt2_R131072.raw.gz"
|
||||
],
|
||||
"size": 131072
|
||||
},
|
||||
{
|
||||
"name": "mixed_rle_raw",
|
||||
"frame": "mixed_rle_raw.zst",
|
||||
"command": "hand-built: [RLE 5000 x 0xAB][raw 33 bytes][RLE 40000 x 0x07] in one frame",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"mixed_rle_raw.raw.gz"
|
||||
],
|
||||
"size": 45033
|
||||
},
|
||||
{
|
||||
"name": "nofcs_windowed",
|
||||
"frame": "nofcs_windowed.zst",
|
||||
"command": "zstd -3 --no-check from stdin over b'the quick brown fox jumps over the lazy dog. ' * 24000",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"text.raw.gz"
|
||||
],
|
||||
"size": 1080000,
|
||||
"expectedSize": 1080000
|
||||
},
|
||||
{
|
||||
"name": "direct_nibble",
|
||||
"frame": "direct_nibble.zst",
|
||||
"command": "block 183 of path/to/srtm1z/N00E042.hgtz, lifted out as its own frame; its literals carry the direct-nibble Huffman weight form",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"direct_nibble.raw.gz"
|
||||
],
|
||||
"size": 131072
|
||||
},
|
||||
{
|
||||
"name": "dictflag",
|
||||
"frame": "dictflag.zst",
|
||||
"command": "python3: pyzstd.compress(A, 3) with Dictionary_ID_flag forced to 1 in the frame header descriptor",
|
||||
"verdict": "refused",
|
||||
"why": "a dictionary this decoder does not have; pyzstd refuses it: ZstdError"
|
||||
},
|
||||
{
|
||||
"name": "rle_over_max",
|
||||
"frame": "rle_over_max.zst",
|
||||
"command": "hand-built: one last RLE block declaring Block_Size 2,000,000",
|
||||
"verdict": "refused",
|
||||
"why": "RFC 8878 bounds Block_Size by Block_Maximum_Size, never over 128 KB; libzstd's one-shot decoder accepts it and returns 2 MB"
|
||||
},
|
||||
{
|
||||
"name": "rle_at_max",
|
||||
"frame": "rle_at_max.zst",
|
||||
"command": "hand-built: one last RLE block at exactly Block_Maximum_Size, 131072",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"rle_at_max.raw.gz"
|
||||
],
|
||||
"size": 131072
|
||||
},
|
||||
{
|
||||
"name": "lit_one_stream",
|
||||
"frame": "lit_one_stream.zst",
|
||||
"command": "zstd -19 -c --no-check over a file holding b'aaadaabaaaddbbaaacabbada'; from a file rather than stdin so the frame declares its own content size",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"lit_one_stream.raw.gz"
|
||||
],
|
||||
"size": 24
|
||||
},
|
||||
{
|
||||
"name": "multiblock_check",
|
||||
"frame": "multiblock_check.zst",
|
||||
"command": "python3: pyzstd.compress(body, {level: 9, checksumFlag: 1, targetCBlockSize: 8192}), body = 165,450 bytes of runs",
|
||||
"verdict": "legal",
|
||||
"plain": [
|
||||
"multiblock_check.raw.gz"
|
||||
],
|
||||
"size": 165450
|
||||
},
|
||||
{
|
||||
"name": "trailing_junk",
|
||||
"frame": "trailing_junk.zst",
|
||||
"command": "hand-built: the lit_one_stream frame with four bytes of junk appended",
|
||||
"verdict": "refused",
|
||||
"why": "reading one frame is the whole API, so input left over after it is input this decoder does not understand; it used to be ignored"
|
||||
},
|
||||
{
|
||||
"name": "trailing_skippable",
|
||||
"frame": "trailing_skippable.zst",
|
||||
"command": "hand-built: the lit_one_stream frame followed by a skippable frame",
|
||||
"verdict": "refused",
|
||||
"why": "a trailing skippable frame was accepted and silently ignored; only a leading one was refused, and only because the magic check fired"
|
||||
}
|
||||
]
|
||||
|
|
|
|||
BIN
test/fixtures/corpus/multiblock_check.raw.gz
vendored
Normal file
BIN
test/fixtures/corpus/multiblock_check.raw.gz
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/multiblock_check.zst
vendored
Normal file
BIN
test/fixtures/corpus/multiblock_check.zst
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/rle_at_max.raw.gz
vendored
Normal file
BIN
test/fixtures/corpus/rle_at_max.raw.gz
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/rle_at_max.zst
vendored
Normal file
BIN
test/fixtures/corpus/rle_at_max.zst
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/rle_over_max.zst
vendored
Normal file
BIN
test/fixtures/corpus/rle_over_max.zst
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/trailing_junk.zst
vendored
Normal file
BIN
test/fixtures/corpus/trailing_junk.zst
vendored
Normal file
Binary file not shown.
BIN
test/fixtures/corpus/trailing_skippable.zst
vendored
Normal file
BIN
test/fixtures/corpus/trailing_skippable.zst
vendored
Normal file
Binary file not shown.
|
|
@ -38,18 +38,26 @@ void main() {
|
|||
);
|
||||
});
|
||||
|
||||
test('fills the buffer exactly when there is room for the last pass', () {
|
||||
final table = FseTable(6);
|
||||
readFseTable(table, description, 0, description.length, 255);
|
||||
test(
|
||||
'writes exactly what the stream encodes, and knows where that ends',
|
||||
() {
|
||||
// The old version of this asserted only that the count was over 256
|
||||
// and under 1024, which passes with the guard at `at + 2`, at
|
||||
// `at + 3`, or removed entirely. This stream encodes 257 symbols, and
|
||||
// the boundary is the whole point: a buffer of exactly 257 takes it
|
||||
// and one of 256 does not.
|
||||
int run(int limit) {
|
||||
final table = FseTable(6);
|
||||
readFseTable(table, description, 0, description.length, 255);
|
||||
final bits = ReverseBitReader()..reset(stream, 0, stream.length);
|
||||
|
||||
// Same stream, a buffer with slack. The count it returns must be the
|
||||
// symbols it actually wrote, so the last pass cannot have been longer
|
||||
// than the room the check left.
|
||||
final bits = ReverseBitReader()..reset(stream, 0, stream.length);
|
||||
final room = Uint8List(1024);
|
||||
final written = fseDecodeInterleaved(table, bits, room, 0, room.length);
|
||||
expect(written, greaterThan(256));
|
||||
expect(written, lessThanOrEqualTo(room.length));
|
||||
});
|
||||
return fseDecodeInterleaved(table, bits, Uint8List(1024), 0, limit);
|
||||
}
|
||||
|
||||
expect(run(1024), 257);
|
||||
expect(run(257), 257);
|
||||
expect(() => run(256), throwsA(isA<ZstdException>()));
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue