From 3cd79018ab5a65113e5c869a1e7f0328db8a0252 Mon Sep 17 00:00:00 2001 From: Peter Tos <38345014+bmxcode@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:28:15 +1000 Subject: [PATCH] ISO 9660: convert the AIFF, and write each sound once The Best Service ProSamples discs added a third collection: 29 images, 16 AKAI and 13 ISO 9660. The AKAI half needed nothing -- 472 volumes, 11 730 samples, zero skips. The ISO 9660 half ships every sound twice, as a full AIFF tree beside a full WAV tree, and extraction wrote both. AIFF is now carried to WAV rather than copied out as .aiff. The samples are big-endian and a WAV's are little-endian, so the bytes within each value are reversed and the values are untouched -- exactly reversible, no resampling, no change of depth. 8-bit is refused, because AIFF stores it signed and WAV unsigned and carrying that means adding 128 to every sample; AIFF-C is refused because it may be compressed and no disc here has one to check a reader against. The loop convention was not guessed. An AIFF marks its loop with two MARK positions and the spec does not say whether the frame at the second is played. These discs carry a WAV of every sound beside the AIFF, and a WAV states its loop in a smpl chunk where the end is inclusive, so the two files answer the question about each other: on 195 of 195 pairs carrying a loop on both sides the end marker is exclusive, and on all 198 carrying an INST the root key matches exactly. That is the convention SampleLoop already used for AKAI, so nothing had to change -- the disc confirmed a choice already made rather than settling a new one. 6 033 of the 7 498 AIFF hold audio that is already coming out as a WAV. Those are reported as duplicates naming the file that holds the audio instead of being written again. Matched on the audio and never on the filename, and vol.43 is why: its 1 386 AIFF all share a name with a WAV and not one shares its audio, being mastered a few frames longer -- 17 638 bytes against 17 616 on 43e-01chh01. A name-based rule would have discarded a whole disc and said nothing. A twin is kept where it carries something the written file lacks. On 314 pairs the AIFF has a root key and a loop and the WAV has no smpl chunk at all. Same audio is not the same file, and deduplicating on the audio alone would have dropped the metadata with it. These discs are the only ground truth in this project for what a conversion should produce. Every other format is checked against the bytes it came from, which proves the payload was copied and says nothing about whether it was understood; here the publisher shipped an independent answer 6 033 times. tests/test_discs.py asserts it, along with the disc that has no twins at all. .exs, .fxp and .fxb classify as programs, so --keep-originals writes them out. They hold the key ranges and envelopes a WAV cannot carry, which is the argument ADR-0011 already made for AKAI programs, and they are the shape ConvertWithMoss reads. Roughly 2 200 across the collection, previously dropped. Three things this found by running: Every audio file copied off an ISO 9660 disc reported 0 Hz and 0 frames -- 15 424 files whose manifest entry said nothing about the audio. read_header walks to the end of the file rather than stopping at data, because smpl is written after the audio as often as before it. A kept original was named twice over: BONGOS M.exs.exs. original_suffix appends what the backend supplies, which is right for a filesystem whose names carry no extension and wrong for one whose names do. Without the backend hook as well, every kept .exs landed in original/ renamed .bin. A clean disc reported itself as a damaged one. vol.42 printed "skipped 423 damaged or unreadable entries" when all 423 were sounds already written. Skipped grows a duplicate flag and the summary and manifest count the two apart. Across the 29 discs: 29 converted, 21 435 samples, 4 930 stereo pairs, 26 365 WAVs written and every one of them reads back as valid WAV with audio in it, 5 719 duplicates suppressed, and one entry not written that was not a duplicate -- a stereo pair on vol.20 whose halves declare 44 033 and 44 100, so the joiner refuses to fuse them and both mono halves are written instead. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 23 ++ README.md | 24 +- docs/README.md | 5 +- ...aiff-twin-is-converted-and-deduplicated.md | 59 +++++ docs/adr/README.md | 1 + docs/formats/README.md | 4 + docs/formats/aiff.md | 120 +++++++++ src/samplerdisc/batch.py | 10 +- src/samplerdisc/cli.py | 14 +- src/samplerdisc/extract.py | 175 ++++++++++-- src/samplerdisc/fs/iso9660.py | 25 +- src/samplerdisc/sample/__init__.py | 6 +- src/samplerdisc/sample/aiff.py | 248 ++++++++++++++++++ src/samplerdisc/wav.py | 60 ++++- tests/fixtures.py | 74 ++++++ tests/test_aiff.py | 154 +++++++++++ tests/test_batch.py | 1 + tests/test_discs.py | 154 +++++++++++ tests/test_extract.py | 91 ++++++- tests/test_iso9660.py | 32 +++ 20 files changed, 1250 insertions(+), 30 deletions(-) create mode 100644 docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md create mode 100644 docs/formats/aiff.md create mode 100644 src/samplerdisc/sample/aiff.py create mode 100644 tests/test_aiff.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fe32e32..19c4b38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ Notable changes to `samplerdisc`. Format-level findings live in [docs/formats/]( ### Added +- **AIFF payloads are converted, not copied out as `.aiff`.** An AIFF's samples are big-endian and a WAV's are little-endian, so the bytes within each value are reversed and the values are left alone — a re-ordering, exactly reversible, with no resampling or change of depth. Root key, tuning and the sustain loop come across from the AIFF's `INST` and `MARK` chunks into the WAV's `smpl`. ([docs/formats/aiff.md](docs/formats/aiff.md), [ADR-0024](docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)) + + AIFF-C is refused rather than guessed at — its payload may be compressed, and compressed data written out as PCM opens, plays as noise and reports nothing wrong. 8-bit is refused too: AIFF stores it signed and WAV unsigned, so carrying it would mean adding 128 to every sample, which is a change to the values and not to their order. Nothing in the collection is either. + + **The loop convention was not guessed.** An AIFF marks its loop with two `MARK` positions and the spec does not say whether the frame at the second is played. The Best Service ProSamples discs carry a WAV of every sound beside the AIFF, and a WAV states its loop in a `smpl` chunk where the end *is* inclusive — so the two files answer the question about each other. On **195 of 195** pairs carrying a loop on both sides, the end marker is exclusive; on all **198** carrying an `INST`, the root key matches exactly. + +- **Duplicate audio on a disc is written once.** The thirteen ISO 9660 ProSamples discs ship each sound twice, as a full AIFF tree beside a full WAV tree — 7 926 WAV and 7 498 AIFF, of which **6 033 hold audio already coming out as a WAV**. Those are now reported as duplicates naming the file that holds the audio, instead of being written a second time. ([ADR-0024](docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)) + + Matched on the audio, never on the filename, and `vol.43` is why: its 1 386 AIFF all share a name with a WAV and **not one shares its audio**, being mastered a few frames longer — 17 638 bytes against 17 616 on `43e-01chh01`. A name-based rule would have discarded that whole disc and said nothing. + + A twin is also kept where it carries something the written file lacks. On **314** pairs the AIFF has a root key and a loop and the WAV has no `smpl` chunk at all; the audio is the same and the files are not. + +- **EXS24 and HALion instruments are kept by `--keep-originals`.** `.exs`, `.fxp` and `.fxb` classify as programs, joining the vocabulary AKAI programs already used — they hold the key ranges and envelopes a WAV cannot carry, which is the argument [ADR-0011](docs/adr/0011-the-deliverable-is-daw-ready-wav.md) already made, and they are the shape [ConvertWithMoss](https://github.com/git-moss/ConvertWithMoss) reads. Roughly 2 200 across the collection, previously dropped. + - **Every partition of an AKAI disc is read, not just the first.** An AKAI disc is a disk image — several partitions laid end to end — and the walk stopped at the one the origin resolved to. Across the 44 AKAI discs on the shelf that is the difference between **448 volumes and 14 670 files** and **2 154 volumes and 68 997 files**; the collection goes from 872 volumes and 56 662 files to **2 578 and 110 989**. `Loop Soup` alone goes from 7 volumes to 60. ([docs/formats/akai-fs.md](docs/formats/akai-fs.md), [ADR-0023](docs/adr/0023-partitions-come-from-the-table-the-disc-declares.md), [#22](https://github.com/bmxcode/samplerdisc/issues/22)) The partitions are not guessed at. The disk declares them, in a table at `0x4500` of the first partition: a count, that many sizes in blocks, then the disk's total. All 44 discs carry one and on all 44 the sizes sum to the total. A partition is read only where a header sits at the position the table gives and restates the size the table gave it. @@ -47,8 +61,17 @@ Notable changes to `samplerdisc`. Format-level findings live in [docs/formats/]( - **The bank-count baselines are asserted rather than written down.** The whole-disc figures in the format doc are now a table in `tests/test_discs.py`, pinned by disc size. They were the stated regression guard for the shared record parser and nothing checked them, which is how two of them shipped wrong. - **A bank with no header on an EIII/ESI disc says so in its own terms.** `E3 Main Code` and `E3X Main Code` — the sampler's operating system, occupying a bank slot — were told they had "no sample directory", naming an E-IV structure those discs do not use. +- **Every audio file copied off an ISO 9660 disc was reported as 0 Hz and 0 frames.** Nothing read the payload's header, so 15 424 files across thirteen discs came back with a manifest entry that said nothing about the audio. `wav.read_header` now reads the real rate and length. It walks to the end of the file rather than stopping at `data`, because a `smpl` chunk is written after the audio as often as before it. +- **An original kept from an ISO 9660 disc was named twice over.** `original_suffix` appends the suffix a backend supplies, which is right for a sampler filesystem — names there carry no extension — and gave `BONGOS M.exs.exs` for a filesystem whose names already do. The ISO 9660 backend now supplies the file's own extension, and the suffix is not appended when the name already ends in it. Without both halves every kept `.exs` landed in `original/` renamed `.bin`: the bytes survived and nothing would open them. +- **A clean disc reported itself as a damaged one.** `Skipped` covered both an entry lost to damage and an entry deliberately not written, and the summary called every one of them damage — `vol.42` printed *"skipped 423 damaged or unreadable entries"* when all 423 were sounds already written under another name. Duplicates are now counted and reported apart, in the summary and in the batch manifest. +- **`--keep-originals` no longer calls its output "AKAI files"** in the summary. An ISO 9660 disc keeps EXS24 and HALion instruments through the same path. + ### Known limits +- **The ISO 9660 directory hierarchy is not preserved.** A disc's audio is written flat into one directory per volume, so `PS-34 AIFF …/056_Ballad de Boo/34a-bas-56Dmin.aif` becomes `34a-bas-56Dmin.wav` and the folder that grouped it by tempo is gone. Nothing is lost on these discs — Best Service named every file uniquely, and all thirteen list zero collisions after flattening — but a disc that reused a basename would rely on `unique_path` suffixing it. + +- **AIFF-C is refused, and no disc exercises a reader for it.** Nothing in the collection is AIFF-C, so there is nothing to check one against; a compressed payload written out as PCM would open, play as noise, and report nothing wrong. Same for 8-bit AIFF, where the sign convention differs from WAV's and carrying it would change the sample values. + - **`AMG - Kickin' Lunatic Beats 2 AKAI CD1.mdx` is an incomplete image, and nine of the 669 files it yields are wrong.** It is short of the disc it was made from by four 32 KB blocks, so everything past the first gap has slid: the last nine files of `13-TRACK 06` extract audio belonging to other samples, with payload headers that no longer match the names their directory gives them. The container decodes every block the file does contain correctly — the file is not a complete copy of the disc, which two independent structures agree on. Nothing detects this yet; that is [#23](https://github.com/bmxcode/samplerdisc/issues/23). If you have extracted that disc, treat its last volume with suspicion, and prefer a fresh rip. - **Only the first partition of an AKAI disc is read.** Partitions are laid end to end at multiples of the size declared at `0x00`, and they hold real content — `Advance Orchestra` declares 7680 blocks of an image of 66 616, with a second partition's volume directory sitting right where the first ends. Most AKAI discs therefore list a fraction of what they hold. The format doc has said to walk the partition table since it was written and nothing does; the measurement is now in it, and the work is [#22](https://github.com/bmxcode/samplerdisc/issues/22). - **100 of `Studio Essentials`'s 230 banks list without extracting.** They have no confirmed sample directory, and carry a note saying so rather than being guessed at. That disc holds 901 `E4P1` presets and preset-only banks are the likely explanation — likely is not established, so it is not claimed. diff --git a/README.md b/README.md index 386ce6f..7acba29 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ The audio is a byte-for-byte copy: AKAI stores signed 16-bit little-endian PCM a **Filesystems** — AKAI S1000/S3000 family, E-mu `EMU3` (EIIIX, ESI-32/4000, Emulator IV), Roland `S770 MR25A` (S-770, S-750, S-760), and plain ISO 9660 for discs whose payload is already WAV or AIFF. +**Audio payloads** — a WAV inside an ISO 9660 disc is copied out untouched. An AIFF is carried to WAV: its samples are big-endian and a WAV's are little-endian, so the bytes within each value are reversed and the values are left alone. Root key, tuning and loop points come across from the AIFF's `INST` and `MARK` chunks into the WAV's `smpl`. + Compressed `.mdx` is the piece no other open-source tool reads today. The format is documented byte by byte in [docs/formats/mdx.md](docs/formats/mdx.md), along with [`.nrg`](docs/formats/nrg.md), [raw CD sectors](docs/formats/rawcd.md), the [AKAI](docs/formats/akai-fs.md), [E-mu](docs/formats/emu3.md) and [Roland S-7xx](docs/formats/roland-s7xx.md) filesystems, and [audio CDs](docs/formats/audio-cd.md). ## Tested against @@ -90,6 +92,25 @@ Every WAV was checked against the disc it came from — **67 of 67 discs match e The five Roland S-7xx discs contribute **6 392 samples and 1 341 stereo pairs, with nothing skipped**. +### Best Service ProSamples + +A third collection, [29 discs](https://archive.org/details/best-service-pro-samples-vol.-12-dance-vocals-akai-1-cd), added after the two above and measured separately. Sixteen are AKAI and thirteen are ISO 9660 discs holding plain audio. + +| | | +|---|---| +| Discs converted | 29 of 29 | +| Samples | 21 435 | +| Stereo pairs rejoined | 4 930 | +| WAV files written | 26 365 | +| Duplicate AIFF suppressed | 5 719 | +| Entries skipped (damage) | 0 | + +Every one of the 26 365 files written back reads as a valid WAV with audio in it. The single entry not written that was not a duplicate is a stereo pair on `vol.20` whose two halves declare different rates — 44 033 against 44 100 — so the joiner refuses to fuse them and both mono halves are written instead. + +The thirteen ISO 9660 discs ship each sound twice, once as AIFF and once as WAV. 6 033 of the 7 498 AIFF hold audio that is already coming out as a WAV, and those are written once; the other 1 779 are converted, along with 314 whose AIFF carries a root key or a loop that its WAV twin does not. `vol.43` is the reason the check is on the audio and not the filename: its 1 386 AIFF all share a name with a WAV and **none of them share its audio**, being mastered a few frames longer ([ADR-0024](docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)). + +These discs are also the only place in this project where the correct output is known independently. Where a sound exists as both, the publisher's own WAV says what the AIFF conversion should produce — including the loop convention the AIFF spec leaves open, settled on 195 pairs out of 195. + ## What doesn't work yet - **Roland S-550, Ensoniq and Kurzweil filesystems.** `Roland LCD1.iso` opens `* ROLAND S-550 *`, which shares nothing with the S-7xx format ([ADR-0014](docs/adr/0014-one-backend-per-on-disc-format.md)); neither archive holds a second specimen to check a backend against. The container layer opens all of these, so `export-iso` gets you the sectors meanwhile. Each backend is a self-contained module ([ADR-0003](docs/adr/0003-brand-neutral-pluggable-backends.md)), so adding one touches nothing else. @@ -99,7 +120,8 @@ The five Roland S-7xx discs contribute **6 392 samples and 1 341 stereo pairs, w - **A `.mds`/`.mdf` pair is read, but its descriptor is not parsed.** One pair has now been through it end to end, and it reads the `.mdf` and sniffs its geometry rather than parsing the `.mds` — correct for a single-track data disc, which is what these are. A multi-track or offset image would be read from byte 0 and come out wrong; the track table is unread. Please open an issue if you have such a disc. - **`CUES` chunks in NRG** are not parsed; only `CUEX`. No disc using the older form was to hand to check the layout against. - **An audio CD with no cue sheet cannot be split into tracks.** `samplerdisc info` tells you when a disc's content looks like Red Book audio, and `extract --assume-audio-cd` writes the whole stream as one WAV, but the track boundaries live in a cue, not in the bytes. -- **AIFF payloads on ISO 9660 discs are copied, not converted** — they come out as `.aiff`. +- **AIFF-C, and 8-bit AIFF, are refused rather than read.** AIFF-C may be compressed, and compressed data written out as PCM plays as noise while reporting nothing wrong. 8-bit AIFF is signed where 8-bit WAV is unsigned, so carrying it would change every sample value rather than reorder its bytes — which is the one thing this tool does not do ([ADR-0024](docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)). +- **EXS24 and HALion instruments are kept, not read.** `--keep-originals` writes the `.exs` and `.fxp` files out byte for byte; turning them into a playable instrument is ConvertWithMoss's job. - **AKAI S900, floppy images and the DD partition.** Use [akaiutil](https://sourceforge.net/projects/akaiutil/). ## If a disc doesn't work diff --git a/docs/README.md b/docs/README.md index bd707c6..8318f4f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -55,6 +55,7 @@ All of that vanishes into a working parser. Six months on, the code says *what* | D13 | Roland `S770 MR25A` backend ([ADR-0016](adr/0016-the-s7xx-hierarchy-is-located-not-walked.md), [ADR-0017](adr/0017-the-stereo-side-marker-is-a-character-class.md), [ADR-0018](adr/0018-the-s7xx-sample-rate-is-measured.md)) | done | | D14 | E-mu Emulator IV bank extraction ([ADR-0020](adr/0020-read-e-iv-through-its-sample-directory.md)) | done | | D15 | Every partition of an AKAI disc, from the table it declares ([ADR-0023](adr/0023-partitions-come-from-the-table-the-disc-declares.md)) | done | +| D16 | AIFF payloads converted, deduplicated against their WAV twin, and EXS24/HALion instruments kept ([ADR-0024](adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)) | done | Across the local collection, by listing: 71 of 79 images claimed, 2 578 volumes, 110 989 files, 77 620 of them samples. The AKAI discs are 44 of those images and 68 997 of those files, read across 276 partitions — before D15 they were 14 670 files, because only the partition at the origin was read. @@ -66,4 +67,6 @@ Across the local collection, by listing: 71 of 79 images claimed, 2 578 volumes, - **`E4P1` presets are not read.** The three E-IV discs carry 916, 901 and 284 of them. On `eiv-studio` 100 of 230 banks have no sample directory and are listed with a note; preset-only banks are the likely explanation, and it is not established. - **The `.mds` track table is unread.** One real pair now reads end to end, but geometry is sniffed from the `.mdf` rather than taken from the descriptor, so a multi-track or offset image would be read from byte 0. What the one specimen's descriptor holds is written down in [formats/mdx.md](formats/mdx.md) without being relied on. - **CUES chunks in NRG.** Only `CUEX` is parsed; `CUES` encodes position as MSF and no disc using it was available to check the layout against. -- **AIFF payloads on ISO 9660 discs are copied, not converted.** They come out as `.aiff`. +- **EXS24 and HALion instruments are kept, not read.** `--keep-originals` writes the `.exs` and `.fxp` files out byte for byte, because they hold the key ranges and envelopes a WAV cannot. Nothing parses them, and nothing should: turning them into a playable instrument is [ConvertWithMoss](https://github.com/git-moss/ConvertWithMoss)'s job ([ADR-0011](adr/0011-the-deliverable-is-daw-ready-wav.md)). +- **AIFF-C is refused rather than read.** Its payload may be compressed, and compressed data written out as PCM opens, plays as noise and reports nothing wrong. No disc in the collection has one, so there is nothing to check a reader against ([ADR-0024](adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)). +- **8-bit AIFF is refused.** AIFF stores 8-bit signed and WAV unsigned, so carrying one to the other changes every sample value rather than reordering its bytes. That is the line ADR-0024 draws; nothing in the collection is other than 16-bit. diff --git a/docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md b/docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md new file mode 100644 index 0000000..c0ee284 --- /dev/null +++ b/docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md @@ -0,0 +1,59 @@ +# ADR-0024: Convert AIFF, and drop the twin only when it says nothing new + +**Status:** accepted + +## Context + +The Best Service ProSamples discs come in two kinds. Sixteen are AKAI. Thirteen are ISO 9660 discs holding plain audio, and every one of those carries **the same sounds twice**: a `PS-nn AIFF …` tree beside a `PS-nn WAV …` tree, matching file counts, matching stems. + +Extraction copied both out untouched. That is 7 926 WAV and 7 498 AIFF from thirteen discs — and 6 033 of the AIFF hold audio that is already coming out of the same disc as a WAV. A user got each of those sounds twice, at double the disk, with the second copy in a format the README calls a gap: *"AIFF payloads are copied, not converted."* + +So there were two questions, and they turned out to be one. Converting the AIFF closes the stated gap. Whether to then write it is the real decision, because on these discs the conversion mostly produces a file the user already has. + +Three things the discs settled before the decision could be made: + +**The audio really is identical.** Matched by hashing the PCM rather than by name: 6 033 of 7 498. Where both sides carry a loop, the loop agrees exactly, on 195 of 195 pairs; where both carry a root key, it agrees on 198 of 198. + +**On one disc it is identical nowhere.** `vol.43` ships 1 386 AIFF and 1 386 WAV under matching names, and **not one pair shares its audio** — the AIFF are mastered a few frames longer, 17 638 bytes against 17 616 on `43e-01chh01`. Two masterings of one take. + +**On 314 pairs the AIFF knows something the WAV does not.** Those AIFF carry an `INST` chunk — root key, tuning, a sustain loop — and their WAV twins carry no `smpl` chunk at all. The audio is the same; the files are not. + +## Decision + +**Convert AIFF to WAV, and skip it only when its audio has already been written *and* it carries nothing the written file lacks.** + +Three parts, each answering one of the findings: + +1. **The WAV is written verbatim, the AIFF is converted.** The disc's own WAV is preferred where there is a choice: it is the publisher's file and carries the `smpl`, `acid` and `LIST` chunks a conversion would have to rebuild. +2. **The twin is recognised by its audio, never by its name.** A `sha256` of the PCM, not of the file — the two trees differ in their metadata chunks and agree on every audio byte. +3. **A twin that carries a root key or a loop the written WAV lacks is written too.** Same audio is not the same file. + +A suppressed twin is reported as a `Skipped` naming the file that already holds the audio, and counted apart from damage in both the summary and the manifest. + +On the collection: 5 719 duplicates suppressed, 1 779 AIFF written — 1 465 with no twin at all, 314 carrying metadata their twin lacks. + +## Rejected + +**Write both trees.** The simplest rule, and it doubles 8 GB of output for no extra sound. The duplication is not the user's to sort out afterwards: the two files have different names in different folders and nothing in either says they are the same take. + +**Write the disc's WAV and never convert.** Cheapest, and wrong three ways. It drops the 1 465 AIFF with no WAV twin, it drops all 1 386 of vol.43 — where the AIFF are a different mastering, not a copy — and it leaves the README's stated gap open for the next disc that ships AIFF alone. + +**Deduplicate by name.** What the file counts suggest and what vol.43 refutes: 1 386 files that share a name with a WAV and share nothing else. A name-based rule discards a whole disc's worth of audio and reports nothing. + +**Prefer the AIFF and drop the WAV wherever both exist.** Symmetrical, and loses more: 1 704 pairs have a `smpl` on the WAV side and no `INST` on the AIFF side, against 314 the other way. Neither side is reliably richer, which is why the rule compares the two files rather than picking a format. + +**Merge the AIFF's metadata into the copied WAV.** Strictly the best output — one file, all the information — and it means rewriting the publisher's file. A verbatim copy is checkable against the disc, and every other format here is checked that way. Writing a second file costs 314 files out of 26 365 and keeps that property. + +**Treat the byte swap as forbidden by [ADR-0011](0011-the-deliverable-is-daw-ready-wav.md).** The rule there is that audio is copied, not converted — no resampling, no bit-depth change, no dithering. Reversing the bytes within a sample value changes no value, loses no precision, and is exactly reversible; running it twice returns the input. It is a re-ordering, and the same argument does not extend to 8-bit, where AIFF is signed and WAV is unsigned: carrying that means adding 128 to every sample, so `sample/aiff.py` refuses 8-bit rather than blurring the line. + +**Wait for a second publisher's discs before deciding.** These are thirteen discs from one publisher, and the shape may not generalise. It does not need to: the rule is stated over what the *files* contain, not over what Best Service did. A disc with no duplicates triggers nothing. + +## Consequences + +`sample/aiff.py` joins `sample/`, and `sample/__init__.py` no longer claims that no module there converts anything — it says what is re-ordered and why that is not the same. + +`wav.read_header` exists, so a copied WAV reports its real rate and length. Every ISO 9660 payload used to come back as 0 Hz and 0 frames — 15 424 files across thirteen discs whose manifest entry said nothing about the audio. + +`Skipped` grows a `duplicate` flag and `DiscReport` a `duplicates` count. Without it a clean disc reads as a broken one: `vol.42` reported *"skipped 423 damaged or unreadable entries"* when all 423 were sounds already written. + +The oracle in `tests/test_discs.py` is the part worth keeping. Every other format here is tested by comparing output against the bytes it came from, which proves the payload was copied and says nothing about whether it was understood. Here the publisher shipped an independent answer 6 033 times, and it is what settled the loop convention rather than leaving it a guess — see [formats/aiff.md](../formats/aiff.md). diff --git a/docs/adr/README.md b/docs/adr/README.md index 4018f6c..3f9d77b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -29,3 +29,4 @@ If you find yourself writing an ADR with no rejected alternative, you are writin | [0021](0021-a-bank-owns-the-run-its-header-declares.md) | An EIII/ESI bank owns the record run its own header declares | Bounding by the directory's `length`; deduplicating headers by address; dropping repeated names; listing the `4k` banks with a note; a looser header match | | [0022](0022-a-volume-is-explained-by-the-allocation-map.md) | An AKAI volume's emptiness is explained by the partition's allocation map | Rejecting type 0; requiring the directory to parse; using the map as an allocation flag; recovering the displaced directories; calling the damage in the note | | [0023](0023-partitions-come-from-the-table-the-disc-declares.md) | An AKAI disc's partitions come from the table it declares | Tiling at multiples of the first size; chaining each header's own size; locating headers by signature; walking partitions in the probe; rewriting block numbers as disc-relative; nesting extraction only where a disc has several | +| [0024](0024-the-aiff-twin-is-converted-and-deduplicated.md) | Convert AIFF, and drop the twin only when it says nothing new | Writing both trees; never converting; deduplicating by name; preferring the AIFF; merging its metadata into the copied WAV; calling the byte swap a conversion | diff --git a/docs/formats/README.md b/docs/formats/README.md index a58e686..8820a9f 100644 --- a/docs/formats/README.md +++ b/docs/formats/README.md @@ -13,6 +13,7 @@ Without these docs, the next session re-derives the MDX block chain by hexdumpin | [emu3.md](emu3.md) | E-mu `EMU3` filesystem — EIIIX, ESI/Formula 4000, E-IV | | [roland-s7xx.md](roland-s7xx.md) | Roland `S770 MR25A` filesystem — S-770, S-750, S-760 | | [iso9660.md](iso9660.md) | ISO 9660 and Joliet — discs whose payload is already audio | +| [aiff.md](aiff.md) | AIFF — the payload inside those discs, and the one format carried rather than copied | | [audio-cd.md](audio-cd.md) | Red Book audio CDs — no filesystem at all | ## How to read the constants @@ -34,6 +35,9 @@ The reference discs are not in this repository ([ADR-0008](../adr/0008-no-media- | `l-cdx-01` | `Roland - L-CDX-01 - Rhythm Section Instruments (Roland Sxx CD-ROM).iso` | 629 149 696 | | `vintage-pro` | `Digital Sound Factory - E-MU Vintage Pro.bin` + `.cue` | 45 558 240 | | `bsbssd2` | `Best Service - Brass Super Section (CD2).bin` + `.cue` | 539 584 080 | +| `prosamples-42` | `Best Service ProSamples vol.42 - Session Instruments [AIFF, EXS24, HALion, WAV] 1CD.iso` | 263 153 664 | +| `prosamples-43` | `Best Service ProSamples vol.43 - Real Drum Kits [AIFF, EXS24, HALion, WAV] 1CD.iso` | 414 228 480 | +| `prosamples-45` | `Best Service ProSamples vol.45 - Techno ID [AIFF, EXS24, HALion, WAV] 1CD.iso` | 433 889 280 | Point `SAMPLERDISC_TEST_DISCS` at a directory containing them to run the disc-backed tests. The scan recurses, so pointing it at the collection root rather than one folder is fine. diff --git a/docs/formats/aiff.md b/docs/formats/aiff.md new file mode 100644 index 0000000..90017c0 --- /dev/null +++ b/docs/formats/aiff.md @@ -0,0 +1,120 @@ +# AIFF + +Apple's Audio Interchange File Format, as it appears on the ProSamples discs — the payload inside an ISO 9660 disc rather than a sampler format. The standard is well documented elsewhere and this file does not restate it. What it records is what these discs actually contain, and the two questions the spec leaves open that the discs answered. + +## Why this format needed a document at all + +An AIFF is a WAV with the bytes the other way round, and that sounds like it needs no notes. It needed three: which variants are on the discs (one, uniformly), whether the loop end marker is inclusive (it is not, and the spec does not say), and where a conversion may stop being a conversion (at the sign convention, not before). + +## The census + +*Verified across 780 AIFF sampled from all 13 ISO 9660 ProSamples discs:* + +| | Value | Count | +|---|---|---| +| Form type | `AIFF` | 780 | +| | `AIFC` | **0** | +| Bit depth | 16 | 780 | +| Sample rate | 44 100 | 780 | +| Channels | 2 | 410 | +| | 1 | 370 | +| Chunks present | `COMM`, `SSND` | 780 | +| | `minf`, `elmo` (Sound Designer II leftovers) | 60 | +| | `MARK` + `INST` | 53 | +| | `APPL` | 49 | + +Every one is uncompressed. Nothing in the collection is AIFF-C, so nothing here is written against a compressed payload, and `sample/aiff.py` refuses `AIFC` rather than guessing that its `sowt`/`ima4`/`ulaw` compression type is one it could pass through. + +## Layout + +A `FORM` container: `FORM`, a big-endian length, the form type `AIFF`, then chunks. Every chunk is a four-character id, a big-endian length, and a body padded to an even length — **the pad byte is not counted in the declared length**, which is the usual off-by-one in a chunk walk. + +### `COMM` — what the audio is + +| Offset | Size | Meaning | +|---|---|---| +| 0 | 2 | Channel count | +| 2 | 4 | Frames | +| 6 | 2 | Bits per sample | +| 8 | 10 | Sample rate, 80-bit IEEE 754 extended | + +The rate field is a sign bit, a 15-bit exponent biased by 16 383, and a **64-bit mantissa with an explicit leading bit** — unlike a `double`, where the leading bit is implied. The value is `mantissa × 2^(exponent − 16383 − 63)`. + +Compute it in integer arithmetic, as a shift. A `double` holds 53 bits of mantissa and this field has 64, so `mantissa * 2.0 ** shift` is lossy for exactly the values this field carries. It rounds 44 100 correctly and is not guaranteed to for every rate; `44 033` — which `ProSamples vol.20` really contains — is the kind of number that makes the difference visible. + +### `SSND` — the audio + +| Offset | Size | Meaning | +|---|---|---| +| 0 | 4 | Offset: a gap of this many bytes before the audio begins | +| 4 | 4 | Block size, for alignment | +| 8 + offset | — | The samples, big-endian, interleaved | + +**The offset field is a gap, not a position.** Reading from `body[8:]` and ignoring it shifts the audio by up to a block, which produces a file that opens, plays as noise and reports nothing wrong. All the discs here use 0; the field is honoured anyway because a wrong answer here is invisible. + +### `MARK` — named positions + +A `uint16` count, then that many markers: a `int16` id, a `uint32` frame position, and a Pascal string — a length byte followed by that many characters, **the two together padded to an even length**. So a one-character name occupies two bytes and needs no pad; a three-character name occupies four and needs one. Getting this wrong shifts every marker after the first. + +### `INST` — how to play it + +| Offset | Size | Meaning | +|---|---|---| +| 0 | 1 | `baseNote` — the root key, MIDI | +| 1 | 1 | `detune`, in cents, signed | +| 2–7 | 6 | Key and velocity range, gain | +| 8 | 2 | Sustain loop `playMode` — 0 none, 1 forward, 2 alternating | +| 10 | 2 | Sustain loop begin: a **marker id**, not a position | +| 12 | 2 | Sustain loop end: a marker id | +| 14 | 6 | Release loop, same shape | + +The loop points are one indirection away: `INST` names two markers and `MARK` holds their frame positions. + +## The end marker is exclusive + +**This is the finding.** The AIFF spec says a loop runs between two markers and does not say whether the frame at the end marker is played. Guess wrong and every loop in the collection is one frame long in the wrong direction — inaudible on a long sustain, and wrong. + +It did not have to be guessed. Best Service mastered these discs with a WAV of every sound beside the AIFF, and a WAV states its loop in a `smpl` chunk, where the end **is** inclusive. So the two files answer the question about each other. + +*Verified across 195 pairs that carry a loop on both sides — 175 on `prosamples-42`, 20 on `prosamples-45`:* + +| Reading | Agrees with the twin's `smpl` | +|---|---| +| end marker is exclusive (`end − 1`) | **195 of 195** | +| end marker is inclusive | 0 of 195 | + +The same pairs settle the root key: on all 198 AIFF that carry an `INST`, `baseNote` equals the twin's `smpl` MIDI unity note exactly. + +That convention — an exclusive `end`, made inclusive by the WAV writer — is the one `SampleLoop` already used for AKAI. Nothing had to change to accommodate AIFF; the disc confirmed the choice already made. + +## The trees are not always the same audio + +Each of these discs carries a `PS-nn AIFF …` tree and a `PS-nn WAV …` tree with the same file count and matching stems. On most discs they are the same sounds. On one they are not. + +*Verified, matching by audio rather than by name:* + +| Disc | AIFF | WAV | AIFF whose audio is also a WAV | +|---|---:|---:|---:| +| `prosamples-37` | 992 | 992 | 992 | +| `prosamples-55` | 1 302 | 1 302 | 1 302 | +| `prosamples-42` | 423 | 851 | 423 | +| `prosamples-45` | 850 | 850 | 850 | +| **`prosamples-43`** | **1 386** | **1 386** | **0** | + +Every one of vol.43's AIFF has a same-named WAV and not one of them holds the same audio: the AIFF are mastered a few frames longer. `43e-01chh01.aif` carries 17 638 bytes of audio against the WAV's 17 616 — eleven frames, at the same rate, in the same channel count. They are two masterings of one take, not two containers for one file. + +Across all 13 discs: 7 498 AIFF, of which 6 033 share their audio with a WAV on the same disc and 1 465 do not. + +## What may be changed, and what may not + +Carrying an AIFF to a WAV reverses the bytes within each sample value. That is a re-ordering: the values are untouched, no rate or depth changes, and running it twice returns the original bytes exactly. + +**8-bit is where that stops being true.** AIFF stores 8-bit samples signed and WAV stores them unsigned, so carrying one to the other means adding 128 to every sample — a change to the values, not to their order. `sample/aiff.py` refuses 8-bit for that reason rather than doing it quietly. 16-bit and 24-bit are both pure reversals and are carried. Nothing in the collection is other than 16-bit. + +## Reference discs + +| Short name | File | Size | +|---|---|---| +| `prosamples-42` | `Best Service ProSamples vol.42 - Session Instruments [AIFF, EXS24, HALion, WAV] 1CD.iso` | 263 153 664 | +| `prosamples-43` | `Best Service ProSamples vol.43 - Real Drum Kits [AIFF, EXS24, HALion, WAV] 1CD.iso` | 414 228 480 | +| `prosamples-45` | `Best Service ProSamples vol.45 - Techno ID [AIFF, EXS24, HALion, WAV] 1CD.iso` | 433 889 280 | diff --git a/src/samplerdisc/batch.py b/src/samplerdisc/batch.py index d2bdd16..46e7d26 100644 --- a/src/samplerdisc/batch.py +++ b/src/samplerdisc/batch.py @@ -38,8 +38,12 @@ class DiscReport: samples: int = 0 stereo_pairs: int = 0 originals: int = 0 + #: Entries read, understood, and deliberately not written because their + #: audio was already written from another file on the same disc. Counted + #: apart from ``skipped`` so a clean disc does not read as a damaged one. + duplicates: int = 0 audio_tracks: int = 0 - skipped: list[dict[str, str]] = field(default_factory=list) + skipped: list[dict[str, object]] = field(default_factory=list) error: str | None = None @property @@ -108,12 +112,15 @@ def convert_disc( elif isinstance(result, Kept): report.originals += 1 elif isinstance(result, Skipped): + if result.duplicate: + report.duplicates += 1 report.skipped.append( { "volume": result.volume, "partition": result.partition, "name": result.name, "reason": result.reason, + "duplicate": result.duplicate, } ) report.volumes = list(volumes.values()) @@ -142,6 +149,7 @@ def write_manifest(path: str, reports: list[DiscReport]) -> None: "audio_tracks": sum(r.audio_tracks for r in reports), "originals": sum(r.originals for r in reports), "skipped": sum(len(r.skipped) for r in reports), + "duplicates": sum(r.duplicates for r in reports), }, } directory = os.path.dirname(os.path.abspath(path)) diff --git a/src/samplerdisc/cli.py b/src/samplerdisc/cli.py index fb2d1fb..008dede 100644 --- a/src/samplerdisc/cli.py +++ b/src/samplerdisc/cli.py @@ -105,6 +105,7 @@ def cmd_extract(args: argparse.Namespace) -> int: joined = 0 kept = 0 skipped = 0 + duplicates = 0 results = extract_disc( image, origin.backend, @@ -128,13 +129,22 @@ def cmd_extract(args: argparse.Namespace) -> int: if args.verbose: print(f" {result.volume}/original/{result.name} kept ({result.kind})") else: - skipped += 1 + if result.duplicate: + duplicates += 1 + else: + skipped += 1 print(f" skipped {result.volume}/{result.name}: {result.reason}", file=sys.stderr) print(f"wrote {written} WAV files to {args.out}") if joined: print(f"joined {joined} stereo pairs (mono originals kept)") if kept: - print(f"kept {kept} original AKAI files") + # Not "AKAI files": an ISO 9660 disc keeps EXS24 and HALion + # instruments through the same path. + print(f"kept {kept} original files") + if duplicates: + # Not damage, and saying so matters: a disc that lists 423 skips reads + # as a bad rip when every one of them is a sound already written. + print(f"skipped {duplicates} duplicates of audio already written") if skipped: # A disc that yields most of its samples is a good outcome; say so # plainly rather than burying it. diff --git a/src/samplerdisc/extract.py b/src/samplerdisc/extract.py index 189c4d4..a907646 100644 --- a/src/samplerdisc/extract.py +++ b/src/samplerdisc/extract.py @@ -1,22 +1,28 @@ """Disc in, WAV files out. -The deliverable is uncompressed WAV that works anywhere (ADR-0011): audio is -copied, never converted, and what the disc knows about a sample -- root key, +The deliverable is uncompressed WAV that works anywhere (ADR-0011): sample +values are never altered, and what the disc knows about a sample -- root key, tuning -- rides along in the WAV's own smpl chunk. + +Sampler payloads are copied into the data chunk unchanged. AIFF is the single +exception, and only in byte order: its samples are big-endian and a WAV's are +little-endian, so the bytes within each sample are reversed and the values are +left alone (ADR-0024). """ from __future__ import annotations +import hashlib import os import re from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol from samplerdisc.fs.base import original_suffix -from samplerdisc.sample import NotASample +from samplerdisc.sample import NotASample, aiff from samplerdisc.sample.akai import parse from samplerdisc.stereo import find_pairs, interleave -from samplerdisc.wav import Loop, write_wav +from samplerdisc.wav import LOOP_FORWARD, Loop, read_header, write_wav if TYPE_CHECKING: from collections.abc import Iterator @@ -42,9 +48,6 @@ class _Pairable(Protocol): _UNSAFE = re.compile(r"[^A-Za-z0-9 ._+#-]") -#: Kinds that are already audio files and need copying, not decoding. -_AUDIO_FILE_KINDS = frozenset({"wav", "aiff"}) - #: Kinds --keep-originals writes out verbatim. Programs are here because they #: carry the key ranges and envelopes, which the WAVs cannot: dropping them #: loses the only copy. Drum settings and effects are not, being unusable @@ -97,6 +100,11 @@ class Skipped: name: str reason: str partition: int = 0 + #: True where the entry was read and understood and deliberately not + #: written -- its audio is already out under another name. Damage and a + #: duplicate are both "skipped" and they are not the same news, so the + #: summary must be able to tell them apart (ADR-0024). + duplicate: bool = False @dataclass @@ -139,6 +147,11 @@ def extract_volume( made = False originals_made = False parsed: dict[str, _Pairable] = {} + #: sha256 of each WAV payload written, against the name it was written + #: from and whether it carried a smpl chunk, so a duplicate can say which + #: file already holds the audio and whether it holds the metadata too. + written_audio: dict[bytes, tuple[str, bool]] = {} + deferred: list[File] = [] for entry in volume.files: if keep_originals and entry.kind in _KEEP_KINDS: payload = backend.read_file(image, origin, entry) @@ -148,8 +161,7 @@ def extract_volume( originals_made = True kept_path = unique_path( os.path.join(out_dir, ORIGINALS_DIR), - safe_name(entry.name), - original_suffix(backend, entry), + *_original_name(backend, entry), ) with open(kept_path, "wb") as out: out.write(payload) @@ -160,13 +172,23 @@ def extract_volume( kind=entry.kind, partition=volume.partition, ) - if entry.kind in _AUDIO_FILE_KINDS: - # Already an audio file -- an ISO 9660 disc whose payload is plain - # WAV or AIFF. Copy it out untouched; there is nothing to decode. + if entry.kind == "aiff": + # Held back until the WAVs of this volume have been written, so a + # twin can be recognised. Not stylistic: the AIFF tree sorts ahead + # of the WAV tree on every ProSamples disc, so a forward pass meets + # the AIFF with nothing yet to compare it against (ADR-0024). + deferred.append(entry) + continue + if entry.kind == "wav": + # Already a WAV -- an ISO 9660 disc whose payload is plain audio. + # Copied untouched: it is the publisher's own file and carries the + # smpl, acid and LIST chunks we would otherwise have to rebuild. if not made: os.makedirs(out_dir, exist_ok=True) made = True - result = _copy_audio(image, backend, origin, volume, entry, out_dir) + result, digest, has_smpl = _copy_wav(image, backend, origin, volume, entry, out_dir) + if digest is not None: + written_audio.setdefault(digest, (entry.name, has_smpl)) yield result continue if entry.kind != "sample": @@ -215,6 +237,12 @@ def extract_volume( partition=volume.partition, ) + for entry in deferred: + if not made: + os.makedirs(out_dir, exist_ok=True) + made = True + yield _convert_aiff(image, backend, origin, volume, entry, out_dir, written_audio) + if join_stereo: yield from _join_pairs(volume, parsed, out_dir) @@ -283,31 +311,138 @@ def _wav_loops(sample) -> list[Loop]: A sample format with no loop information yields none, rather than an invented one. """ - return [Loop(start=loop.start, end=loop.end - 1) for loop in getattr(sample, "loops", ())] + return [ + Loop( + start=loop.start, + end=loop.end - 1, + loop_type=getattr(loop, "loop_type", LOOP_FORWARD), + ) + for loop in getattr(sample, "loops", ()) + ] + + +def _original_name(backend: Backend, entry: File) -> tuple[str, str]: + """Stem and suffix for one file kept verbatim. + + A sampler filesystem has names and a type byte, so the suffix is supplied + and appended. ISO 9660 has real filenames that already carry it, and + appending it again gives ``BONGOS M.exs.exs``. + """ + suffix = original_suffix(backend, entry) + name = entry.name + if suffix and name.lower().endswith(suffix.lower()): + name = name[: -len(suffix)] + return safe_name(name), suffix -def _copy_audio( +def _copy_wav( image: SectorImage, backend: Backend, origin: int, volume: Volume, entry: File, out_dir: str, -) -> Extracted | Skipped: +) -> tuple[Extracted | Skipped, bytes | None, bool]: + """Write one already-WAV payload out untouched. + + Returns the record and a digest of the audio it holds, so the AIFF pass can + recognise the same sound arriving a second time. The digest covers the data + chunk alone, not the file: the two trees of a ProSamples disc differ in + their metadata chunks and agree on every audio byte. + """ payload = backend.read_file(image, origin, entry) if not payload: - return Skipped(volume.name, entry.name, "no data on disc", volume.partition) + return Skipped(volume.name, entry.name, "no data on disc", volume.partition), None, False stem, suffix = os.path.splitext(os.path.basename(entry.name)) path = unique_path(out_dir, safe_name(stem), suffix.lower() or ".wav") with open(path, "wb") as out: out.write(payload) + header = read_header(payload) + digest = None + if header is not None: + digest = hashlib.sha256(payload[header.offset : header.offset + header.length]).digest() + return ( + Extracted( + volume=volume.name, + name=entry.name, + path=path, + # A payload whose header will not parse is still written -- it is + # the disc's own bytes -- but it cannot be described, and zero is + # the honest answer rather than a guess. + rate=header.rate if header else 0, + frames=header.frames if header else 0, + pitch=0, + partition=volume.partition, + ), + digest, + bool(header and header.has_smpl), + ) + + +def _convert_aiff( + image: SectorImage, + backend: Backend, + origin: int, + volume: Volume, + entry: File, + out_dir: str, + written_audio: dict[bytes, str], +) -> Extracted | Skipped: + """Write one AIFF payload as a WAV, unless its audio is already out. + + Best Service mastered these discs with a full AIFF tree beside a full WAV + tree of the same sounds, so converting both writes every sample twice for + no extra audio. The twin is recognised by its PCM and not by its name: + vol.43 carries pairs that share a name and differ by eleven frames, and + those are two different sounds (ADR-0024). + """ + payload = backend.read_file(image, origin, entry) + if not payload: + return Skipped(volume.name, entry.name, "no data on disc", volume.partition) + try: + sample = aiff.parse(payload, fallback_name=os.path.basename(entry.name)) + except NotASample as exc: + return Skipped(volume.name, entry.name, str(exc), volume.partition) + if sample.frames == 0: + return Skipped(volume.name, entry.name, "zero-length sample", volume.partition) + + twin = written_audio.get(hashlib.sha256(sample.pcm).digest()) + if twin is not None: + twin_name, twin_has_smpl = twin + # Same audio is not the same file. On 314 of these pairs the AIFF + # carries a root key and a loop the WAV has nowhere to put, so + # dropping it for having identical audio would drop the metadata with + # it -- and where both carry it they agree exactly, checked on 173 + # pairs against the publisher's own smpl chunk (ADR-0024). + if twin_has_smpl or (sample.pitch is None and not sample.loops): + return Skipped( + volume.name, + entry.name, + f"same audio as {twin_name}, already written", + volume.partition, + duplicate=True, + ) + + stem, _ = os.path.splitext(os.path.basename(entry.name)) + path = unique_path(out_dir, safe_name(stem)) + write_wav( + path, + sample.pcm, + rate=sample.rate, + channels=sample.channels, + sample_width=sample.width, + midi_note=sample.pitch, + cents=sample.cents, + loops=_wav_loops(sample), + name=sample.name, + ) return Extracted( volume=volume.name, name=entry.name, path=path, - rate=0, - frames=0, - pitch=0, + rate=sample.rate, + frames=sample.frames, + pitch=sample.pitch if sample.pitch is not None else 0, partition=volume.partition, ) diff --git a/src/samplerdisc/fs/iso9660.py b/src/samplerdisc/fs/iso9660.py index 4608000..de9cc6d 100644 --- a/src/samplerdisc/fs/iso9660.py +++ b/src/samplerdisc/fs/iso9660.py @@ -13,11 +13,12 @@ from __future__ import annotations +import os import struct from typing import TYPE_CHECKING, NamedTuple from samplerdisc.container.base import SECTOR_SIZE -from samplerdisc.fs.base import File, Volume, register +from samplerdisc.fs.base import DEFAULT_ORIGINAL_SUFFIX, File, Volume, register if TYPE_CHECKING: from collections.abc import Iterator @@ -205,6 +206,26 @@ def _walk( def read_file(self, image: SectorImage, origin: int, entry: File) -> bytes: return image.read(origin + entry.start_block * SECTOR_SIZE, entry.size) + def original_suffix(self, entry: File) -> str: + """The file's own extension. + + Unlike a sampler filesystem, this one has real filenames, so the + suffix is not inferred from a type byte -- it is already there. Without + this the default applies and every kept .exs lands in original/ named + .bin: the bytes survive and nothing will open them. + """ + _, suffix = os.path.splitext(entry.name) + return suffix.lower() or DEFAULT_ORIGINAL_SUFFIX + + +#: Instrument definitions: Logic's EXS24 (.exs) and Steinberg's HALion (.fxp, +#: with .fxb for a whole bank). They are to one of these discs what a program +#: is to an AKAI one -- the key ranges and envelopes, which a WAV cannot carry +#: -- so they are kept for the same reason (ADR-0011), and they are the shape +#: ConvertWithMoss reads. Calling them "program" puts them in a vocabulary +#: --keep-originals already understands. +_PROGRAM_SUFFIXES = (".exs", ".fxp", ".fxb") + def _decode(raw: bytes, joliet: bool) -> str: """A directory record's name, from whichever name space it came. @@ -237,6 +258,8 @@ def _classify(name: str) -> str: return "wav" if lower.endswith((".aif", ".aiff")): return "aiff" + if lower.endswith(_PROGRAM_SUFFIXES): + return "program" return "file" diff --git a/src/samplerdisc/sample/__init__.py b/src/samplerdisc/sample/__init__.py index cb2729c..f584fea 100644 --- a/src/samplerdisc/sample/__init__.py +++ b/src/samplerdisc/sample/__init__.py @@ -7,8 +7,10 @@ keeps its parameters in a different region of the disc entirely, so its filesystem layer hands them over on the ``File``. -None of them convert anything. The payload becomes a WAV data chunk unchanged -(ADR-0011). +None of them alter a sample value. A sampler payload becomes a WAV data chunk +unchanged; AIFF alone is re-ordered, because its samples are big-endian and a +WAV's are little-endian, and reversing the bytes within a value is not the same +as changing it (ADR-0011, ADR-0024). """ from __future__ import annotations diff --git a/src/samplerdisc/sample/aiff.py b/src/samplerdisc/sample/aiff.py new file mode 100644 index 0000000..9aa0a87 --- /dev/null +++ b/src/samplerdisc/sample/aiff.py @@ -0,0 +1,248 @@ +"""AIFF payloads on ISO 9660 discs. See docs/formats/aiff.md. + +The one place in this project where the audio bytes are not handed on exactly +as the disc stored them: AIFF is big-endian and WAV is little-endian, so each +sample's bytes are reversed. That is a re-ordering of the bytes *within* a +sample value, not a re-sampling -- no rate change, no bit-depth change, no +dithering, and it is exactly reversible (ADR-0024). + +The line is drawn at byte order. 8-bit AIFF is signed and 8-bit WAV is +unsigned, so carrying one to the other means adding 128 to every sample, which +changes the values; that is a conversion and this module refuses it rather than +doing it quietly. +""" + +from __future__ import annotations + +import struct +from dataclasses import dataclass + +from samplerdisc.sample import NotASample as _NotASample +from samplerdisc.wav import LOOP_ALTERNATING, LOOP_FORWARD + +MAGIC = b"FORM" +FORM_AIFF = b"AIFF" +#: AIFF-C. Its payload may be compressed -- ``sowt``, ``ima4``, ``ulaw`` -- and +#: a compressed payload emitted as PCM opens, plays as noise, and reports +#: nothing wrong. Rejected rather than guessed; no disc in the collection has +#: one. See docs/formats/aiff.md. +FORM_AIFC = b"AIFC" + +#: Byte orders we can carry to WAV by reversing bytes within a sample. +SUPPORTED_WIDTHS = (2, 3) + +#: An AIFF chunk header is a four-character id and a big-endian length. +CHUNK_HEADER_LEN = 8 + +#: INST playMode values (AIFF 1.3). +PLAY_NONE = 0 +PLAY_FORWARD = 1 +PLAY_FORWARD_BACKWARD = 2 + +_PLAY_MODES = {PLAY_FORWARD: LOOP_FORWARD, PLAY_FORWARD_BACKWARD: LOOP_ALTERNATING} + +#: The 80-bit IEEE 754 extended sample rate in COMM: sign and 15-bit exponent, +#: then a 64-bit mantissa with an explicit leading bit. +EXTENDED_BIAS = 16383 +MANTISSA_BITS = 63 + + +class NotASample(_NotASample): + """The payload is not an uncompressed AIFF we can carry to WAV.""" + + +@dataclass(frozen=True) +class SampleLoop: + """One loop, in frames. ``end`` is exclusive, as elsewhere in this project; + the WAV writer makes it inclusive as the RIFF spec requires.""" + + start: int + end: int + loop_type: int = LOOP_FORWARD + + +@dataclass(frozen=True) +class AiffSample: + name: str + rate: int + frames: int + channels: int + width: int + pcm: bytes + #: None where the file carries no INST chunk, so the WAV is written without + #: an invented root key rather than with one. + pitch: int | None = None + cents: float = 0.0 + loops: tuple[SampleLoop, ...] = () + + @property + def duration(self) -> float: + return self.frames / self.rate if self.rate else 0.0 + + +def _extended(raw: bytes) -> int: + """The 80-bit extended float in COMM, as an integer sample rate. + + Done in integer arithmetic rather than through a float: the mantissa is 64 + bits and a double has 53, so the obvious ``mantissa * 2.0 ** shift`` is + lossy for exactly the large mantissas this field uses. + """ + if len(raw) < 10: + raise NotASample("COMM sample rate is truncated") + exponent = struct.unpack_from(">H", raw, 0)[0] + mantissa = struct.unpack_from(">Q", raw, 2)[0] + negative = bool(exponent & 0x8000) + exponent &= 0x7FFF + if exponent == 0x7FFF: + raise NotASample("COMM sample rate is infinity or NaN") + if mantissa == 0: + return 0 + shift = exponent - EXTENDED_BIAS - MANTISSA_BITS + value = mantissa << shift if shift >= 0 else mantissa >> -shift + return -value if negative else value + + +def _chunks(payload: bytes): + """Walk the chunks of a FORM, yielding ``(id, body)``. + + Stops at the first header that does not fit rather than raising: these are + rips and a truncated tail is normal, so a file that loses its last chunk + still yields the audio ahead of it. + """ + pos = 12 + while pos + CHUNK_HEADER_LEN <= len(payload): + chunk_id = payload[pos : pos + 4] + size = struct.unpack_from(">I", payload, pos + 4)[0] + body = payload[pos + CHUNK_HEADER_LEN : pos + CHUNK_HEADER_LEN + size] + yield chunk_id, body + # Chunks are padded to an even length; the pad byte is not counted in + # the declared size. + pos += CHUNK_HEADER_LEN + size + (size & 1) + + +def _markers(body: bytes) -> dict[int, int]: + """MARK, as ``{marker id: frame position}``. + + Marker names are Pascal strings padded to an even total, and a malformed + one would shift every marker after it -- so the walk stops at the first + marker that does not fit rather than reading past it. + """ + if len(body) < 2: + return {} + count = struct.unpack_from(">H", body, 0)[0] + found: dict[int, int] = {} + pos = 2 + for _ in range(count): + if pos + 7 > len(body): + break + marker_id, position, name_len = struct.unpack_from(">hIB", body, pos) + found[marker_id] = position + pos += 7 + name_len + (1 - (name_len & 1)) + return found + + +def _swap(pcm: bytes, width: int) -> bytes: + """Reverse the bytes within each sample. Big-endian in, little-endian out. + + Whole samples only: a payload whose length is not a multiple of the sample + width has a partial sample at the end, and reversing that would move real + audio bytes into the wrong sample rather than dropping a fragment. + """ + usable = len(pcm) - len(pcm) % width + out = bytearray(usable) + for offset in range(width): + out[offset::width] = pcm[width - 1 - offset : usable : width] + return bytes(out) + + +def parse(payload: bytes, fallback_name: str = "") -> AiffSample: + """One AIFF payload, with its audio carried to little-endian PCM.""" + if len(payload) < 12 or payload[:4] != MAGIC: + raise NotASample("not a FORM") + form = payload[8:12] + if form == FORM_AIFC: + raise NotASample("AIFF-C, which may be compressed and is not read") + if form != FORM_AIFF: + raise NotASample(f"FORM type {form!r} is not AIFF") + + channels = width = frames = rate = 0 + audio: bytes | None = None + name = "" + inst: bytes | None = None + marks: dict[int, int] = {} + + for chunk_id, body in _chunks(payload): + if chunk_id == b"COMM": + if len(body) < 18: + raise NotASample("COMM chunk is truncated") + channels, frames, bits = struct.unpack_from(">HIH", body, 0) + rate = _extended(body[8:18]) + width = (bits + 7) // 8 + elif chunk_id == b"SSND": + if len(body) < 8: + raise NotASample("SSND chunk is truncated") + # The offset field is a gap before the audio, for block alignment. + audio = body[8 + struct.unpack_from(">I", body, 0)[0] :] + elif chunk_id == b"NAME": + name = body.split(b"\x00", 1)[0].decode("ascii", "replace").strip() + elif chunk_id == b"INST": + inst = body + elif chunk_id == b"MARK": + marks = _markers(body) + + if channels == 0 or width == 0: + raise NotASample("no COMM chunk") + if audio is None: + raise NotASample("no SSND chunk") + if width not in SUPPORTED_WIDTHS: + raise NotASample( + f"{width * 8}-bit AIFF is not carried to WAV: only byte order is " + f"changed, and this depth would need the sample values changed too" + ) + if channels < 1: + raise NotASample(f"COMM declares {channels} channels") + + pcm = _swap(audio, width) + block = channels * width + # The disc is the authority on how much audio there is, not the header: a + # truncated rip declares the frames it was mastered with and carries fewer. + frames = min(frames, len(pcm) // block) if block else 0 + pcm = pcm[: frames * block] + + pitch, cents, loops = _instrument(inst, marks) + return AiffSample( + name=name or fallback_name, + rate=rate, + frames=frames, + channels=channels, + width=width, + pcm=pcm, + pitch=pitch, + cents=cents, + loops=loops, + ) + + +def _instrument( + inst: bytes | None, marks: dict[int, int] +) -> tuple[int | None, float, tuple[SampleLoop, ...]]: + """Root key, tuning and the sustain loop, from INST and the markers it names. + + A file with no INST gets ``None`` for the root key rather than a plausible + 60, so ``write_wav`` leaves the smpl chunk out entirely (ADR-0011). + """ + if inst is None or len(inst) < 14: + return None, 0.0, () + base_note, detune = struct.unpack_from(">bb", inst, 0) + play_mode, begin_id, end_id = struct.unpack_from(">hhh", inst, 8) + loop_type = _PLAY_MODES.get(play_mode) + loops: tuple[SampleLoop, ...] = () + if loop_type is not None and begin_id in marks and end_id in marks: + start, end = marks[begin_id], marks[end_id] + # A loop that does not run forwards is not a loop, whatever the + # markers say. Dropping it beats writing one a DAW would refuse. + if end > start: + loops = (SampleLoop(start=start, end=end, loop_type=loop_type),) + if not 0 <= base_note <= 127: + return None, 0.0, loops + return base_note, float(detune), loops diff --git a/src/samplerdisc/wav.py b/src/samplerdisc/wav.py index 6551232..d618c96 100644 --- a/src/samplerdisc/wav.py +++ b/src/samplerdisc/wav.py @@ -18,8 +18,9 @@ WAVE_FORMAT_PCM = 1 -#: smpl loop types (RIFF spec). 0 is a plain forward loop. +#: smpl loop types (RIFF spec). 0 is a plain forward loop, 1 alternates. LOOP_FORWARD = 0 +LOOP_ALTERNATING = 1 @dataclass(frozen=True) @@ -29,6 +30,63 @@ class Loop: loop_type: int = LOOP_FORWARD +@dataclass(frozen=True) +class Header: + """What a WAV declares about its audio, and where the audio sits.""" + + channels: int + rate: int + width: int # bytes per sample + offset: int # of the data chunk body, within the payload + length: int # of the data chunk body, in bytes + #: Whether the file carries a smpl chunk -- root key and loop points. Used + #: to tell a WAV that already knows what the disc knows from one that does + #: not, when an AIFF of the same audio turns up (ADR-0024). + has_smpl: bool = False + + @property + def frames(self) -> int: + block = self.channels * self.width + return self.length // block if block else 0 + + +def read_header(payload: bytes) -> Header | None: + """Read a WAV's fmt, data and smpl chunks, or None if it is not a WAV. + + Needed because a WAV copied off an ISO 9660 disc is passed through + untouched, and a run that cannot say what rate it wrote is a run that + reported nothing. The chunks are walked rather than assumed at fixed + offsets: ten of the ProSamples WAVs put LIST or PAD ahead of data. + + The walk runs to the end rather than stopping at ``data``, because smpl is + written after the audio as often as before it and stopping early would + report half the collection as carrying no root key. + """ + if len(payload) < 12 or payload[:4] != b"RIFF" or payload[8:12] != b"WAVE": + return None + channels = rate = width = 0 + data: tuple[int, int] | None = None + has_smpl = False + pos = 12 + while pos + 8 <= len(payload): + tag = payload[pos : pos + 4] + size = struct.unpack_from("= 16: + _, channels, rate, _, _, bits = struct.unpack_from(" bytes: """A RIFF chunk, padded to an even length as the spec requires.""" padding = b"\x00" if len(body) % 2 else b"" diff --git a/tests/fixtures.py b/tests/fixtures.py index bd55581..8c9adb3 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -518,6 +518,80 @@ def tiny_wav(tmp_path, frames: int = 32, rate: int = 44100) -> bytes: return path.read_bytes() +def aiff_pcm(frames: int = 32, channels: int = 1) -> bytes: + """Big-endian PCM, the way an AIFF stores it.""" + return b"".join(struct.pack(">h", (i * 211) % 8000 - 4000) for i in range(frames * channels)) + + +def make_aiff( + frames: int = 32, + rate: int = 44100, + channels: int = 1, + bits: int = 16, + pcm: bytes | None = None, + form: bytes = b"AIFF", + loop: tuple[int, int] | None = None, + base_note: int = 60, + detune: int = 0, + play_mode: int = 1, + name: str = "", + ssnd_offset: int = 0, + declared_frames: int | None = None, +) -> bytes: + """One AIFF payload. See docs/formats/aiff.md. + + ``loop`` is a ``(start, end)`` frame pair; supplying it adds the MARK and + INST chunks that carry a loop and a root key. ``form`` is exposed so a test + can build the AIFF-C the parser must refuse. + """ + if pcm is None: + pcm = aiff_pcm(frames, channels) + + def chunk(tag: bytes, body: bytes) -> bytes: + return tag + struct.pack(">I", len(body)) + body + (b"\x00" if len(body) % 2 else b"") + + # The 80-bit IEEE extended sample rate: exponent, then a mantissa with an + # explicit leading bit. + exponent = 16383 + 63 + mantissa = rate + while mantissa and not mantissa & (1 << 63): + mantissa <<= 1 + exponent -= 1 + extended = struct.pack(">HQ", exponent, mantissa) if rate else b"\x00" * 10 + + body = chunk( + b"COMM", + struct.pack( + ">HIH", channels, declared_frames if declared_frames is not None else frames, bits + ) + + extended, + ) + if name: + body += chunk(b"NAME", name.encode("ascii")) + if loop is not None: + start, end = loop + markers = struct.pack(">H", 2) + # Marker names are Pascal strings padded so the count byte and the + # characters together come to an even length. "L" needs no pad; "END" + # does, and using both exercises the walk either way. + for marker_id, position, label in ((1, start, b"L"), (2, end, b"END")): + pstring = bytes([len(label)]) + label + markers += ( + struct.pack(">hI", marker_id, position) + + pstring + + (b"\x00" if len(pstring) % 2 else b"") + ) + body += chunk(b"MARK", markers) + body += chunk( + b"INST", + struct.pack(">bbbbbbh", base_note, detune, 0, 127, 0, 127, 0) + + struct.pack(">hhh", play_mode, 1, 2) + + struct.pack(">hhh", 0, 0, 0), + ) + body += chunk(b"SSND", struct.pack(">II", ssnd_offset, 0) + b"\x00" * ssnd_offset + pcm) + return b"FORM" + struct.pack(">I", 4 + len(body)) + form + body + + def subchannel_block(seed: int = 0, sectors: int = 15) -> tuple[bytes, bytes]: """One MDX block of 2144-byte sectors: 2048 of data plus 96 of subchannel. diff --git a/tests/test_aiff.py b/tests/test_aiff.py new file mode 100644 index 0000000..df4ddcb --- /dev/null +++ b/tests/test_aiff.py @@ -0,0 +1,154 @@ +"""AIFF payloads, and the one place this project re-orders audio bytes. + +The parser's real oracle is a disc: Best Service mastered the ProSamples discs +with an AIFF tree beside a WAV tree of the same sounds, so the publisher's own +WAV says what our conversion should produce. That check lives in test_discs.py +because it needs the discs. What is here is the shape of the format and the +edges no disc in hand exercises. +""" + +from __future__ import annotations + +import struct + +import pytest + +from samplerdisc.sample import NotASample +from samplerdisc.sample.aiff import parse +from samplerdisc.wav import LOOP_ALTERNATING, LOOP_FORWARD +from tests import fixtures + + +def test_the_audio_is_the_same_values_with_their_bytes_reversed(): + """The conversion is a byte order change and nothing else. + + Asserted as an identity on the sample values rather than on the bytes, so + it stays true if the fixture's audio changes: reading the payload + big-endian and the result little-endian must give the same numbers. + """ + sample = parse(fixtures.make_aiff(frames=64)) + source = fixtures.aiff_pcm(64) + assert struct.unpack(">64h", source) == struct.unpack("<64h", sample.pcm) + assert sample.pcm != source # and it did have to be reversed + + +def test_the_swap_is_exactly_reversible(): + payload = fixtures.make_aiff(frames=48) + once = parse(payload).pcm + twice = bytes(b for pair in zip(once[1::2], once[0::2], strict=True) for b in pair) + assert twice == fixtures.aiff_pcm(48) + + +def test_the_header_is_read(): + sample = parse(fixtures.make_aiff(frames=100, rate=22050, channels=2), fallback_name="x") + assert (sample.rate, sample.channels, sample.width, sample.frames) == (22050, 2, 2, 100) + assert len(sample.pcm) == 100 * 2 * 2 + + +@pytest.mark.parametrize("rate", [8000, 22050, 44100, 48000, 96000, 33075, 44033]) +def test_the_80_bit_extended_sample_rate_round_trips(rate): + """Including the odd rates these libraries really use. + + Done in integer arithmetic in the parser because the mantissa is 64 bits + and a double holds 53; 44 033 is a rate one of these discs actually + carries, so this is not a hypothetical. + """ + assert parse(fixtures.make_aiff(rate=rate)).rate == rate + + +def test_a_name_chunk_is_preferred_over_the_filename(): + named = parse(fixtures.make_aiff(name="Conga Mute"), fallback_name="42x.aif") + assert named.name == "Conga Mute" + assert parse(fixtures.make_aiff(), fallback_name="42x.aif").name == "42x.aif" + + +def test_the_ssnd_offset_is_a_gap_before_the_audio(): + """Skipping it is what keeps the audio aligned; reading from the wrong + place produces a file that plays as noise and reports nothing wrong.""" + sample = parse(fixtures.make_aiff(frames=32, ssnd_offset=8)) + assert struct.unpack("<32h", sample.pcm) == struct.unpack(">32h", fixtures.aiff_pcm(32)) + + +def test_a_loop_and_a_root_key_come_from_inst_and_the_markers_it_names(): + sample = parse(fixtures.make_aiff(frames=200, loop=(64, 192), base_note=48, detune=-7)) + assert sample.pitch == 48 + assert sample.cents == -7.0 + assert [(loop.start, loop.end, loop.loop_type) for loop in sample.loops] == [ + (64, 192, LOOP_FORWARD) + ] + + +def test_an_alternating_loop_keeps_its_type(): + sample = parse(fixtures.make_aiff(frames=200, loop=(10, 100), play_mode=2)) + assert sample.loops[0].loop_type == LOOP_ALTERNATING + + +def test_play_mode_none_means_no_loop_however_the_markers_read(): + sample = parse(fixtures.make_aiff(frames=200, loop=(10, 100), play_mode=0)) + assert sample.loops == () + assert sample.pitch == 60 # the root key is still good + + +def test_a_file_with_no_inst_offers_no_root_key(): + """None rather than a plausible 60, so write_wav leaves the smpl chunk out + entirely rather than mapping every sample to middle C (ADR-0011).""" + sample = parse(fixtures.make_aiff()) + assert sample.pitch is None + assert sample.loops == () + + +def test_aifc_is_refused_rather_than_read_as_pcm(): + """Its payload may be compressed, and compressed data emitted as PCM opens, + plays as noise, and reports nothing wrong.""" + with pytest.raises(NotASample, match="AIFF-C"): + parse(fixtures.make_aiff(form=b"AIFC")) + + +def test_eight_bit_is_refused_because_carrying_it_would_change_the_values(): + """AIFF 8-bit is signed and WAV 8-bit is unsigned. Converting means adding + 128 to every sample, which is a conversion, not a byte order change.""" + with pytest.raises(NotASample, match="8-bit"): + parse(fixtures.make_aiff(bits=8, pcm=b"\x01\x02\x03\x04")) + + +def test_twenty_four_bit_is_carried_because_only_the_order_changes(): + payload = fixtures.make_aiff(bits=24, frames=4, pcm=bytes(range(12))) + sample = parse(payload) + assert sample.width == 3 + assert sample.pcm == bytes([2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9]) + + +def test_a_truncated_payload_yields_the_audio_it_has(): + """These are rips and a lost tail is normal: the file declares 400 frames + and carries 40, and the 40 are worth having.""" + sample = parse(fixtures.make_aiff(frames=40, declared_frames=400)) + assert sample.frames == 40 + assert len(sample.pcm) == 80 + + +def test_a_partial_sample_at_the_end_is_dropped_not_reversed(): + """Reversing a fragment would move real audio bytes into the wrong sample. + Losing an odd byte is right; shifting the tail is not.""" + sample = parse(fixtures.make_aiff(frames=4, pcm=fixtures.aiff_pcm(4) + b"\x7f")) + assert len(sample.pcm) % 2 == 0 + assert struct.unpack("<4h", sample.pcm) == struct.unpack(">4h", fixtures.aiff_pcm(4)) + + +@pytest.mark.parametrize( + ("payload", "match"), + [ + (b"", "not a FORM"), + (b"RIFF" + b"\x00" * 20, "not a FORM"), + (b"FORM" + struct.pack(">I", 4) + b"8SVX", "not AIFF"), + ], +) +def test_something_that_is_not_an_aiff_is_refused(payload, match): + with pytest.raises(NotASample, match=match): + parse(payload) + + +def test_a_form_with_no_ssnd_is_refused(): + payload = fixtures.make_aiff() + payload = payload[: payload.index(b"SSND")] + with pytest.raises(NotASample, match="no SSND"): + parse(payload) diff --git a/tests/test_batch.py b/tests/test_batch.py index 10cb71c..19977c9 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -93,6 +93,7 @@ def test_manifest_records_totals_and_failures(tmp_path): "originals": 0, "audio_tracks": 0, "skipped": 0, + "duplicates": 0, } failed = [d for d in payload["discs"] if d["error"]] assert len(failed) == 1 diff --git a/tests/test_discs.py b/tests/test_discs.py index 1ea610e..45b1b54 100644 --- a/tests/test_discs.py +++ b/tests/test_discs.py @@ -22,6 +22,8 @@ from samplerdisc.container.detect import open_image, sniff from samplerdisc.container.mdsmdf import find_mdf from samplerdisc.fs.probe import find_origin +from samplerdisc.sample import aiff +from samplerdisc.wav import read_header #: ``.mds`` and not ``.mdf``: the pair is reached through the descriptor, the #: member that is actually opened, so listing both would open the disc twice. @@ -342,8 +344,26 @@ def test_roland_s7xx_payloads_are_byte_identical_to_the_disc(label: str) -> None _ISO9660 = { "Digital Sound Factory - E-MU Vintage Pro": (45_558_240, "VintagePro", 1062), "Best Service - Brass Super Section (CD2)": (539_584_080, "BSBSS", 2059), + "Best Service ProSamples vol.42 - Session Instruments": (263_153_664, "PS_42", 1347), + "Best Service ProSamples vol.43 - Real Drum Kits": (414_228_480, "PS_43", 2801), } +#: ProSamples discs carrying a full AIFF tree beside a full WAV tree of the +#: same sounds. They are the only ground truth in this project for what a +#: conversion should produce: the publisher shipped the answer (ADR-0024). +#: ``label: (size in bytes, twins, pairs whose AIFF names a root key, pairs +#: that carry a loop on both sides)``. +_AIFF_TWINS = { + "Best Service ProSamples vol.42 - Session Instruments": (263_153_664, 423, 178, 175), + "Best Service ProSamples vol.45 - Techno ID": (433_889_280, 850, 20, 20), +} + +#: The disc that says the twin trees are not always the same audio. vol.43 +#: ships 1 386 AIFF and 1 386 WAV under matching names and **not one pair +#: shares its audio** -- the AIFF are mastered a few frames longer. Pinned +#: because it is what makes deduplicating by name wrong (ADR-0024). +_NO_TWINS = ("Best Service ProSamples vol.43 - Real Drum Kits", 414_228_480, 1386) + @pytest.mark.parametrize("label", sorted(_ISO9660)) def test_iso9660_discs_list_every_file_under_a_distinct_path(label: str) -> None: @@ -365,6 +385,140 @@ def test_iso9660_discs_list_every_file_under_a_distinct_path(label: str) -> None assert len(set(names)) == count +def _audio_index(backend, image, offset, files) -> tuple[dict, dict]: + """Every WAV on the disc indexed by its audio, split by whether it carries + a smpl chunk. Matching on audio and not on name is the point: the two trees + do not agree on their directory names.""" + plain: dict[bytes, str] = {} + rich: dict[bytes, bytes] = {} + for entry in (f for f in files if f.kind == "wav"): + payload = backend.read_file(image, offset, entry) + header = read_header(payload) + if header is None: + continue + pcm = payload[header.offset : header.offset + header.length] + if header.has_smpl: + rich[pcm] = payload + else: + plain[pcm] = entry.name + return plain, rich + + +@pytest.mark.parametrize("label", sorted(_AIFF_TWINS)) +def test_a_converted_aiff_matches_the_publishers_own_wav_of_the_same_sound(label: str) -> None: + """The disc is its own oracle. + + Every other sample format here is checked against the bytes it came from, + which proves the payload was copied and says nothing about whether it was + understood. These discs carry each sound twice -- once as AIFF, once as WAV + -- so the publisher's WAV is an independent statement of what the AIFF + conversion should produce, down to the byte. Nothing else in this project + has that. + """ + size, twins, _, _ = _AIFF_TWINS[label] + with open_image(_pinned_disc(label, size)) as image: + origin = find_origin(image) + assert origin is not None, f"{label}: no filesystem found" + backend, offset = origin.backend, origin.offset + files = [f for v in backend.volumes(image, offset) for f in v.files] + plain, rich = _audio_index(backend, image, offset, files) + + matched = 0 + for entry in (f for f in files if f.kind == "aiff"): + sample = aiff.parse(backend.read_file(image, offset, entry)) + if sample.pcm in plain or sample.pcm in rich: + matched += 1 + assert matched == twins + + +@pytest.mark.parametrize("label", sorted(_AIFF_TWINS)) +def test_an_aiff_agrees_with_the_smpl_chunk_of_its_wav_twin(label: str) -> None: + """What settled the loop-end convention, kept so it cannot drift. + + An AIFF marks its loop with two MARK positions and the spec does not say + whether the frame at the second one is played. The WAV twin's smpl chunk + does say, and ours is right only if the end marker is exclusive -- which is + the convention ``SampleLoop`` already used everywhere else. + + Asserted only where each side has something to say. Most of these AIFF + carry no INST at all, and a file with no root key must not be read as + claiming one (ADR-0011) -- so the count of pairs that *do* agree is pinned + too, or an INST that stopped parsing would pass this vacuously. + """ + size, _, notes, loop_pairs = _AIFF_TWINS[label] + with open_image(_pinned_disc(label, size)) as image: + origin = find_origin(image) + assert origin is not None + backend, offset = origin.backend, origin.offset + files = [f for v in backend.volumes(image, offset) for f in v.files] + _, rich = _audio_index(backend, image, offset, files) + + notes_seen = loops_seen = 0 + for entry in (f for f in files if f.kind == "aiff"): + sample = aiff.parse(backend.read_file(image, offset, entry)) + twin = rich.get(sample.pcm) + if twin is None: + continue + note, loops = _smpl(twin) + if sample.pitch is not None: + assert sample.pitch == note, entry.name + notes_seen += 1 + if sample.loops and loops: + # The disc's own answer to the question the AIFF spec leaves + # open: our exclusive end, minus one, is the WAV's end. + assert (sample.loops[0].start, sample.loops[0].end - 1) == loops[0], entry.name + loops_seen += 1 + assert (notes_seen, loops_seen) == (notes, loop_pairs) + + +def test_a_disc_whose_twins_are_not_the_same_audio_keeps_both() -> None: + """vol.43's two trees agree on every name and on no single sound. + + The AIFF are mastered a few frames longer than the WAVs -- 17 638 bytes + against 17 616 on ``43e-01chh01``. Deduplicating on the name would drop + 1 386 files that are not duplicates of anything, which is why the dedupe + hashes the audio (ADR-0024). + """ + label, size, aiff_count = _NO_TWINS + with open_image(_pinned_disc(label, size)) as image: + origin = find_origin(image) + assert origin is not None + backend, offset = origin.backend, origin.offset + files = [f for v in backend.volumes(image, offset) for f in v.files] + plain, rich = _audio_index(backend, image, offset, files) + + aiffs = [f for f in files if f.kind == "aiff"] + assert len(aiffs) == aiff_count + by_name = {f.name.rsplit("/", 1)[-1].rsplit(".", 1)[0] for f in aiffs} + wav_names = {f.name.rsplit("/", 1)[-1].rsplit(".", 1)[0] for f in files if f.kind == "wav"} + # Every AIFF has a same-named WAV ... + assert by_name <= wav_names + # ... and not one of them holds the same audio. + for entry in aiffs: + sample = aiff.parse(backend.read_file(image, offset, entry)) + assert sample.pcm not in plain and sample.pcm not in rich, entry.name + + +def _smpl(payload: bytes) -> tuple[int, list[tuple[int, int]]]: + """Root key and loop points from a WAV's own smpl chunk.""" + pos = 12 + while pos + 8 <= len(payload): + tag = payload[pos : pos + 4] + size = struct.unpack_from(" bytes: + """The same 16-bit samples, little-endian.""" + return bytes(b for pair in zip(pcm[1::2], pcm[0::2], strict=True) for b in pair) + + +def _wav_bytes(tmp_path, pcm: bytes, rate: int) -> bytes: + path = tmp_path / "w.wav" + write_wav(path, pcm, rate=rate) + return path.read_bytes() diff --git a/tests/test_iso9660.py b/tests/test_iso9660.py index 4432239..208f1bc 100644 --- a/tests/test_iso9660.py +++ b/tests/test_iso9660.py @@ -280,3 +280,35 @@ def test_the_fallback_does_not_fire_on_a_healthy_disc(tmp_path): ) volume = next(iter(BACKEND.volumes(image, 0))) assert [f.name for f in volume.files] == ["Kick 01.wav"] + + +def test_instrument_definitions_are_classified_as_programs(tmp_path, payload): + """EXS24 and HALion files are to one of these discs what a program is to an + AKAI one: the key ranges and envelopes, which the WAVs cannot carry.""" + image = iso_image( + tmp_path, + {"P.EXS": b"exs", "H.FXP": b"fxp", "B.FXB": b"fxb", "A.WAV": payload, "R.TXT": b"read me"}, + ) + kinds = {f.name: f.kind for f in next(iter(BACKEND.volumes(image, 0))).files} + assert kinds == { + "P.EXS": "program", + "H.FXP": "program", + "B.FXB": "program", + "A.WAV": "wav", + "R.TXT": "file", + } + + +def test_an_original_keeps_the_extension_the_disc_gave_it(tmp_path, payload): + """Unlike a sampler filesystem, this one has real filenames. Falling back + to the default suffix writes every kept .exs out as .bin -- the bytes + survive and nothing will open them.""" + image = iso_image(tmp_path, {"P.EXS": b"exs", "A.WAV": payload}) + files = {f.name: f for f in next(iter(BACKEND.volumes(image, 0))).files} + assert BACKEND.original_suffix(files["P.EXS"]) == ".exs" + + +def test_an_original_with_no_extension_falls_back_to_the_default(tmp_path, payload): + image = iso_image(tmp_path, {"README": b"notes", "A.WAV": payload}) + entry = next(f for f in next(iter(BACKEND.volumes(image, 0))).files if f.name == "README") + assert BACKEND.original_suffix(entry) == ".bin"