From dfcb4adc7a11f3b35399587503435870c3cc3cef Mon Sep 17 00:00:00 2001 From: Peter Tos <38345014+bmxcode@users.noreply.github.com> Date: Fri, 21 Aug 2026 02:22:29 +1000 Subject: [PATCH] AKAI: a volume's emptiness is explained by the allocation map Ten volumes across three discs listed empty with no explanation -- the ADR-0012 signature -- for two different reasons (#16, #17). Each now carries a note naming the block and what the partition's own block allocation map at 0x70A says is in it. No volume or file count moves: 872 volumes and 56 662 files across the 79-image collection, before and after. The map explains an emptiness and never gates a listing, because the gating fixes cost real audio. Settles #17 against the container: the four allocated volumes on Kickin' Lunatic Beats 2 CD1 are not a mis-decode and not stale start blocks. That image is short of the disc by four 32 KB MDX blocks, which two independent structures agree on. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 10 + ...lume-is-explained-by-the-allocation-map.md | 71 ++++++ docs/adr/README.md | 1 + docs/formats/akai-fs.md | 84 +++++++- src/samplerdisc/fs/akai.py | 115 +++++++++- tests/fixtures.py | 85 +++++++- tests/test_akai_fs.py | 181 ++++++++++++++++ tests/test_discs.py | 202 ++++++++++++++++-- 8 files changed, 719 insertions(+), 30 deletions(-) create mode 100644 docs/adr/0022-a-volume-is-explained-by-the-allocation-map.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f58e3c..ff6fe8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,16 @@ Notable changes to `samplerdisc`. Format-level findings live in [docs/formats/]( - **`protozoa`'s two Formula 4000 banks extract.** `Orbit Presets 4k` and `Phatt Presets 4K` open with `EMU SI-32 v3` where every other bank on that disc opens with `EMULATOR 3X`. Nothing recognised the signature, so neither was located — and an unlocated bank is not merely unread, it is also not a boundary, so the bank in front of each was handed its region too. They now give **535 and 239 samples** under their own names. ([docs/formats/emu3.md](docs/formats/emu3.md), [ADR-0021](docs/adr/0021-a-bank-owns-the-run-its-header-declares.md)) +- **The AKAI partition's block allocation map is read.** At `0x70A`, one u16 per block, as many as the partition declares at `0x00`. It is the disc's own record of what every block holds, and it is verified rather than merely plausible: a file's chain length and the size its directory entry declares come from two different structures, and across all 44 AKAI discs they agree for **14 607 of 14 607 files**, exactly. `tests/test_discs.py` asserts that per disc, so an AKAI image that starts decoding wrongly now has something to fail against instead of presenting as a disc with less on it. ([docs/formats/akai-fs.md](docs/formats/akai-fs.md), [ADR-0022](docs/adr/0022-a-volume-is-explained-by-the-allocation-map.md)) + ### Fixed +- **An AKAI volume that lists nothing now says why, in the disc's own words.** Ten volumes across three discs listed empty with no explanation, which is exactly the [ADR-0012](docs/adr/0012-a-probe-must-confirm-a-file.md) signature — it reads as an empty volume rather than as a wrong answer. Each now carries a note naming the block and what the partition's allocation map says is in it: file data on `Advance Orchestra` ×4 and the OMI disc, a free block on `Kickin' Lunatic Beats 2 CD1`, and — for four more on that disc — a block the disc says *is* a volume directory and the image has none at. ([#16](https://github.com/bmxcode/samplerdisc/issues/16), [#17](https://github.com/bmxcode/samplerdisc/issues/17), [ADR-0022](docs/adr/0022-a-volume-is-explained-by-the-allocation-map.md)) + + **No volume or file count moves anywhere** — 872 volumes and 56 662 files across the collection, before and after, unchanged to the number. That is the point rather than a happy accident: the map explains an emptiness and never gates a listing, because the one-line fixes that *do* gate cost real audio. Rejecting volumes whose type byte is 0 discards four volumes carrying 63 files, and trusting the map as an allocation flag discards those and every volume on every S3000 and CD3000 disc besides. + +- **The AKAI volume entry's type is read as a byte.** It was unpacked with the start block as one ` bool: return not any(entry) +def allocation_map(image: SectorImage, offset: int) -> list[int]: + """The partition's block allocation map, or ``[]`` when it cannot be read. + + Bounded by the partition's own declared block count rather than by the + image, which is the whole point: the map is the disc speaking about itself, + so a disc that declares nothing usable gets no map and no notes derived + from one, instead of a map invented out of whatever follows the header. + """ + declared = image.read(offset + PARTITION_BLOCKS_OFFSET, 2) + if len(declared) < 2: + return [] + (blocks,) = struct.unpack(" str: + """Why a volume holding no files holds none, in the disc's own words. + + Every branch reports what the allocation map *declares* about the block and + stops there. It deliberately does not diagnose: a free block under a volume + that lists nothing is a slot formatted and never used on one disc and a + damaged image on another, and the map cannot tell them apart. Saying which + would be inventing the half that is not written down (ADR-0022). + + An empty string means the map does not account for the emptiness, which is + the ADR-0012 signature and has to stay visible as such. + """ + if not 0 <= start_block < len(allocation): + return "" + code = allocation[start_block] + if code == FAT_FREE: + return f"block {start_block} is free in the partition's allocation map" + if code == FAT_VOLUME_DIR: + return ( + f"the partition's allocation map marks block {start_block} a volume " + f"directory, but no file entry could be read there" + ) + if 0 < code < FAT_VOLUME_DIR or code == FAT_CHAIN_END: + return ( + f"the partition's allocation map assigns block {start_block} to file " + f"data, not to a volume directory" + ) + return ( + f"the partition's allocation map does not mark block {start_block} a " + f"volume directory (code 0x{code:04x})" + ) + + class AkaiBackend: name = "akai" @@ -129,11 +219,15 @@ def probe(self, image: SectorImage, offset: int) -> bool: raw_name = entry[:NAME_LEN] if not is_plausible_name(raw_name): return False - _type, start = struct.unpack(" Iterator[Volume]: header = image.read(offset, VOLUME_DIR_OFFSET + _MAX_VOLUMES * VOLUME_ENTRY_LEN) max_block = (image.size - offset) // BLOCK_SIZE + # Read once for the whole partition: 7680 entries is 15 KB at the + # largest seen, against a walk that opens a block per volume anyway. + allocation = allocation_map(image, offset) for index in range(_MAX_VOLUMES): base = VOLUME_DIR_OFFSET + index * VOLUME_ENTRY_LEN entry = header[base : base + VOLUME_ENTRY_LEN] @@ -200,11 +297,19 @@ def volumes(self, image: SectorImage, offset: int) -> Iterator[Volume]: if not is_plausible_name(raw_name): continue name = decode_name(raw_name) - _type, start = struct.unpack(" max_block: continue volume = Volume(name=name, start_block=start) volume.files = list(self._files(image, offset, start, max_block)) + if not volume.files: + # A volume that lists nothing has to say why, and the answer + # comes from the disc rather than from a judgement about how + # plausible its directory looks (ADR-0012, ADR-0022). A slot + # is still listed either way: the four type-0 volumes on this + # collection that read as free carry 63 files between them, + # so "the map calls it free" is not grounds for dropping one. + volume.note = why_empty(allocation, start) yield volume def _files( diff --git a/tests/fixtures.py b/tests/fixtures.py index c43dc15..64a5c9a 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -166,38 +166,89 @@ def akai_name(text: str) -> bytes: return bytes(CHARSET.index(c) if c in CHARSET else CHARSET.index(" ") for c in text) -def akai_partition(volumes, blocks_total: int = 512) -> bytes: +def akai_partition( + volumes, + blocks_total: int = 512, + *, + stale_slots=(), + phantom_directories=(), + allocation_map: bool = True, + volume_type: int = 1, +) -> bytes: """Build an AKAI partition image. ``volumes`` is a list of ``(name, [(file_name, type_byte, size, payload)])``. Returns a byte image whose block 0 is the partition header. + + The header carries the partition's block count and a **block allocation + map**, both of which a real one has and neither of which this fixture had + while the volume walk ignored them. The map is built to match what is + written: a live volume's directory block reads ``FAT_VOLUME_DIR``, a file's + blocks chain to ``FAT_CHAIN_END``, everything else stays free. + + The three keyword arguments model the three ways a real disc departs from + that, one per situation measured on the shelf: + + ``stale_slots`` + ``(name, start_block)`` pairs written into the volume directory with a + type byte of 0 -- a slot AKAI pre-formatted, whose start block was left + pointing wherever formatting left it. Point one at a file's block to + reproduce `Advance Orchestra`, or at an unused one to reproduce + `Kickin' Lunatic Beats 2 CD1`'s `VOLUME 018`. + ``phantom_directories`` + blocks marked ``FAT_VOLUME_DIR`` in the map with no directory written + at them -- what an image short of the disc it came from looks like from + inside the filesystem. + ``allocation_map`` + False leaves the map area zeroed and the block count unwritten, for the + case where the disc declares nothing usable and no note may be drawn. """ from samplerdisc.fs.akai import ( BLOCK_SIZE, + FAT_CHAIN_END, + FAT_OFFSET, + FAT_VOLUME_DIR, FILE_ENTRY_LEN, NAME_LEN, + PARTITION_BLOCKS_OFFSET, VOLUME_DIR_OFFSET, VOLUME_ENTRY_LEN, + VOLUME_START_OFFSET, + VOLUME_TYPE_INACTIVE, + VOLUME_TYPE_OFFSET, ) image = bytearray(blocks_total * BLOCK_SIZE) next_block = 1 header = bytearray(BLOCK_SIZE) + allocation = [0] * blocks_total - for index, (volume_name, files) in enumerate(volumes): - volume_block = next_block - next_block += 1 + def slot(index: int, name: str, type_byte: int, start: int) -> None: entry = bytearray(VOLUME_ENTRY_LEN) - entry[:NAME_LEN] = akai_name(volume_name) - struct.pack_into(" bytes: record[18] = (size >> 8) & 0xFF record[19] = (size >> 16) & 0xFF struct.pack_into("= FAT_VOLUME_DIR: + break + block = allocation[block] + assert length == want + assert allocation[block] == FAT_CHAIN_END + + +def test_a_stale_slot_pointing_into_file_data_says_so(tmp_path): + """Issue #16: a slot AKAI formatted, whose start block was never cleared. + + The four on `Advance Orchestra` and the one on the OMI disc present as a + default name, a type byte of 0 and a start block that lands inside a + file's extent -- so the block holds PCM, and the walk stops on the first + entry. What says it is not a volume is the map, which has that block + chained to a file. + """ + payload = fixtures.akai_sample("KICK", words=8192) + data = fixtures.akai_partition( + [("VOL 1", [("KICK", 0x73, len(payload), payload)])], + # Block 2 is the first file's; block 3 is the second block of its + # extent, which is where a stale pointer lands mid-sample. + stale_slots=[("VOLUME 016", 3)], + ) + volumes = {v.name: v for v in BACKEND.volumes(image_of(tmp_path, data, "s.iso"), 0)} + assert [f.name for f in volumes["VOL 1"].files] == ["KICK"] + stale = volumes["VOLUME 016"] + assert stale.files == [] + assert "file data" in stale.note and "block 3" in stale.note + + +def test_a_stale_slot_pointing_at_a_free_block_says_so(tmp_path): + """The other shape of an unused slot: a start block belonging to nothing.""" + payload = fixtures.akai_sample("KICK") + data = fixtures.akai_partition( + [("VOL 1", [("KICK", 0x73, len(payload), payload)])], + stale_slots=[("VOLUME 018", 200)], + ) + volumes = {v.name: v for v in BACKEND.volumes(image_of(tmp_path, data, "fr.iso"), 0)} + empty = volumes["VOLUME 018"] + assert empty.files == [] + assert empty.note == "block 200 is free in the partition's allocation map" + + +def test_a_declared_directory_that_is_not_there_says_so(tmp_path): + """Issue #17: the map says volume directory, the image has none. + + This is what an image short of the disc it was made from looks like from + inside the filesystem, and it is the case that must not be confused with + an unused slot: here the disc is asserting that a directory belongs at + that block, so the volume is real and the *image* is what is wrong. + """ + import struct + + from samplerdisc.fs.akai import VOLUME_DIR_OFFSET, VOLUME_ENTRY_LEN, VOLUME_START_OFFSET + + payload = fixtures.akai_sample("KICK") + data = bytearray( + fixtures.akai_partition( + [ + ("VOL 1", [("KICK", 0x73, len(payload), payload)]), + ("14-TRK06 MF1", [("X", 0x73, 32, b"\x00" * 32)]), + ], + phantom_directories=[300], + ) + ) + # Point the second volume at the block the map calls a directory and the + # image leaves empty, exactly as the four on `Kickin' Lunatic Beats 2 CD1` + # point past the four 32 KB blocks their image is missing. + struct.pack_into(" set[int]: *(size for size, _ in _ROLAND_S7XX.values()), *(size for size, _, _ in _ISO9660.values()), *(size for size, _, _ in _EMU3.values()), + *(size for size, _, _, _ in _AKAI.values()), } @@ -165,15 +166,24 @@ def _ids(paths: list[Path]) -> list[str]: @pytest.mark.parametrize("path", _discs(), ids=_ids(_discs())) -def test_a_claimed_disc_yields_at_least_one_file(path: Path) -> None: - """No backend may claim a disc and then produce nothing. +def test_every_claimed_volume_yields_a_file_or_says_why(path: Path) -> None: + """No backend may claim a **volume** and then produce nothing from it. Resolving to None is a legitimate outcome -- the container was understood and the filesystem inside it was not, which is what ``export-iso`` is for - (ADR-0009). Claiming a disc and walking out with zero files in every volume + (ADR-0009). Claiming a disc and walking out with zero files and no reason is not: it is a probe that matched arbitrary data, and it reports as an empty disc rather than as an error (ADR-0005, ADR-0012). + **Per volume, not per disc**, and that is the whole strength of it. The + per-disc form lets one volume that extracts cover for every volume that + does not, which is exactly how the EMU3 index banks of issue #15 stayed + invisible for as long as they did: those discs always had *some* bank with + records in it. Of the 79 images measured, three failed the per-volume form + and none failed the per-disc one -- the six AKAI volumes of issues #16 and + #17, plus the four on `Kickin' Lunatic Beats 2 CD1` whose blocks the disc + itself says are volume directories. + This is deliberately an invariant rather than a table of expected offsets, so it holds against whatever collection a contributor has. """ @@ -182,16 +192,19 @@ def test_a_claimed_disc_yields_at_least_one_file(path: Path) -> None: if origin is None: return volumes = list(origin.backend.volumes(image, origin.offset)) - files = sum(len(volume.files) for volume in volumes) - if files: - return - # No files is allowed only when a backend says why -- a variant it - # recognises and deliberately does not extract. Unexplained emptiness - # is the ADR-0012 signature. - explained = [v for v in volumes if v.note] - assert volumes and explained, ( + assert volumes, ( f"{path.name}: {origin.backend.name} claimed offset {origin.offset} " - f"but returned {len(volumes)} volumes, no files and no explanation" + f"but returned no volumes at all" + ) + # No files is allowed only where the backend says why -- a variant it + # recognises and deliberately does not extract, or a block the disc's + # own bookkeeping accounts for. Unexplained emptiness is the ADR-0012 + # signature, and one silent volume is enough to hide a wrong answer. + unexplained = [v.name for v in volumes if not v.files and not v.note] + assert not unexplained, ( + f"{path.name}: {origin.backend.name} claimed offset {origin.offset} " + f"but {len(unexplained)} of {len(volumes)} volumes hold no files and " + f"give no reason: {unexplained[:5]}" ) @@ -422,6 +435,171 @@ def test_protozoa_gives_each_bank_its_own_records() -> None: ) +#: AKAI discs pinned by size, with the volumes, files and *noted* volumes each +#: must yield. The noted count is pinned as tightly as the file count on +#: purpose: a note is what separates an emptiness the disc accounts for from +#: the ADR-0012 signature, so a note appearing where none was measured is a +#: backend explaining away something nobody looked at, and one disappearing is +#: the invariant above losing its teeth. The eight are the discs of issues #16 +#: and #17 together with the four counter-examples -- volumes whose type byte +#: is 0 and whose blocks the allocation map calls free, which carry 63 files +#: between them and must keep every one. +#: ``label: (size in bytes, volumes, files, volumes carrying a note)``. +_AKAI = { + "AKAI Advance Orchestra Upgrade 97 Vol.1": (545_720_320, 18, 464, 4), + "AMG - Kickin' Lunatic Beats 2 AKAI CD1": (378_443_564, 18, 669, 5), + "AMG - Kickin' Lunatic Beats 2 AKAI CD2": (371_768_845, 20, 1346, 0), + "OMI Universe Of Sounds Vol.1 (Roland S-770,S-750)": (295_837_696, 28, 900, 1), + "Back in Time Records - Big Bang": (269_979_648, 8, 380, 0), + "Best Service ProSamples vol.01 - Hip Hop and R&B Drumloops": (314_882_048, 7, 82, 0), + "Best Service ProSamples vol.19 - Pop Brass": (484_558_848, 3, 139, 0), + "Best Service ProSamples vol.24 - Breakbeat": (505_772_032, 15, 113, 0), +} + + +@pytest.mark.parametrize("label", sorted(_AKAI)) +def test_akai_discs_list_their_volumes_and_files(label: str) -> None: + """Pinned where present, skipped where the shelf is bare -- see _pinned_disc().""" + size, volumes_expected, files_expected, noted_expected = _AKAI[label] + with open_image(_pinned_disc(label, size)) as image: + origin = find_origin(image) + assert origin is not None, f"{label}: no filesystem found" + assert origin.backend.name == "akai" + assert origin.offset == 0 + volumes = list(origin.backend.volumes(image, origin.offset)) + assert len(volumes) == volumes_expected + assert sum(len(v.files) for v in volumes) == files_expected + assert sum(1 for v in volumes if v.note) == noted_expected + # The same rule as the collection-wide check above, tightened from + # "the collection has no silent volume" to "this disc has none", which + # only a disc whose expected shape is known can ask (ADR-0012). + assert all(v.files or v.note for v in volumes), [v.name for v in volumes if not v.files] + + +def test_akai_unused_slots_are_explained_by_the_allocation_map() -> None: + """The six volumes of issue #16 and the four of #17, and what tells them apart. + + All ten look identical from the volume entry alone: a name, a start block + inside the image, and nothing at that block a file walk will take. What + separates them is the partition's own allocation map, and each of the three + answers below is a different fact about the disc rather than a different + guess about it (ADR-0022). + + `Advance Orchestra` and the OMI disc keep a start block left over from + formatting that now points into a file's extent -- the map says those + blocks hold file data, so no directory was ever there. `Kickin' Lunatic + Beats 2 CD1` is the other way round: the map says all four blocks *are* + volume directories, and the image has none at them, because the image is + short of the disc by four 32 KB blocks. Its `VOLUME 018` is a third case + again, and the map declines to choose between them. + """ + expect = { + "AKAI Advance Orchestra Upgrade 97 Vol.1": { + "VOLUME 015": "file data", + "VOLUME 016": "file data", + "VOLUME 017": "file data", + "VOLUME 018": "file data", + }, + "OMI Universe Of Sounds Vol.1 (Roland S-770,S-750)": {"VOLUME 028": "file data"}, + "AMG - Kickin' Lunatic Beats 2 AKAI CD1": { + "14-TRK06 MF1": "a volume directory", + "15-TRK06 MF2": "a volume directory", + "16-TRACK 07": "a volume directory", + "17-TRK07 MF": "a volume directory", + "VOLUME 018": "is free", + }, + } + for label, wanted in expect.items(): + with open_image(_pinned_disc(label, _AKAI[label][0])) as image: + origin = find_origin(image) + assert origin is not None + volumes = {v.name: v for v in origin.backend.volumes(image, origin.offset)} + for name, fragment in wanted.items(): + volume = volumes[name] + assert not volume.files, f"{label}: {name} unexpectedly lists files" + assert fragment in volume.note, f"{label}: {name}: {volume.note!r}" + + +def test_akai_keeps_the_files_of_a_volume_the_allocation_map_calls_free() -> None: + """A free block under a volume that *does* list files is not grounds to drop it. + + Four volumes across three discs have a type byte of 0 and sit on blocks the + allocation map marks free, and every one holds a real directory: these are + volumes that were deleted, on read-only media that never reused the blocks. + They carry 63 files between them, so the one-line fix of rejecting type 0 -- + or of trusting the map as an allocation flag rather than reading it as an + explanation -- costs real audio. This is the test that says so. + """ + expect = { + "Back in Time Records - Big Bang": ("VOLUME 008", 14), + "Best Service ProSamples vol.01 - Hip Hop and R&B Drumloops": ("VOLUME 007", 10), + "Best Service ProSamples vol.19 - Pop Brass": ("VOLUME 003", 37), + "Best Service ProSamples vol.24 - Breakbeat": ("VOLUME 015", 2), + } + from samplerdisc.fs.akai import FAT_FREE, allocation_map + + total = 0 + for label, (name, count) in expect.items(): + with open_image(_pinned_disc(label, _AKAI[label][0])) as image: + origin = find_origin(image) + assert origin is not None + allocation = allocation_map(image, origin.offset) + volume = {v.name: v for v in origin.backend.volumes(image, origin.offset)}[name] + assert allocation[volume.start_block] == FAT_FREE, label + assert len(volume.files) == count, label + assert not volume.note + total += len(volume.files) + assert total == 63 + + +@pytest.mark.parametrize("path", _discs(), ids=_ids(_discs())) +def test_an_akai_file_chain_is_as_long_as_its_declared_size(path: Path) -> None: + """The allocation map has to agree with the directory about every file. + + This is what makes the map evidence rather than a hopeful reading of the + bytes that follow the volume directory: walk each file's chain of blocks + and it must be exactly as long as the size the directory declares, which + the map and the directory state independently of one another. Across the + 44 AKAI discs measured it holds for **14 607 of 14 607** files, exactly, + with no disc disagreeing anywhere. + + Volumes the map calls free are excluded, and that exclusion is a finding + rather than a fudge: those five are deleted volumes, so their blocks were + returned to the free list and their chains genuinely no longer describe + the audio still sitting in them. Their 63 files are the reason the + exclusion has to be by what the map says and not by a tolerance -- on + `ProSamples vol.19` they are 37 of 139 files, so any rate loose enough to + pass there would be loose enough to hide a real fault on a larger disc. + """ + from samplerdisc.fs.akai import BLOCK_SIZE, FAT_FREE, FAT_VOLUME_DIR, allocation_map + + with open_image(path) as image: + origin = find_origin(image) + if origin is None or origin.backend.name != "akai": + return + allocation = allocation_map(image, origin.offset) + assert allocation, f"{path.name}: no allocation map" + disagreed = [] + for volume in origin.backend.volumes(image, origin.offset): + if volume.start_block < len(allocation) and allocation[volume.start_block] == FAT_FREE: + continue + for entry in volume.files: + want = -(-entry.size // BLOCK_SIZE) + block, length, seen = entry.start_block, 0, set() + while 0 <= block < len(allocation) and block not in seen and length <= want: + seen.add(block) + length += 1 + if allocation[block] >= FAT_VOLUME_DIR: + break + block = allocation[block] + if length != want: + disagreed.append(f"{volume.name}/{entry.name}: {length} blocks, wanted {want}") + assert not disagreed, ( + f"{path.name}: {len(disagreed)} files whose chain is not as long as the " + f"size their directory declares, first: {disagreed[:3]}" + ) + + @pytest.mark.parametrize("path", _discs(), ids=_ids(_discs())) def test_an_iso9660_volume_lists_no_path_twice(path: Path) -> None: """On a hierarchical filesystem two entries under one path means one of