From d4ffb605c79684f0fe2417365adf792b0417cb12 Mon Sep 17 00:00:00 2001 From: Peter Tos <38345014+bmxcode@users.noreply.github.com> Date: Fri, 21 Aug 2026 21:25:52 +1000 Subject: [PATCH] EMU3: decode the loop from the record's pointer block The 92-byte sample record's undecoded middle is an eight-pointer block -- a start, an end, a loop start and a loop end per channel, as byte offsets from the record's own start. 8 039 of the 14 738 E-mu samples now carry their loop points in the WAV smpl chunk. Established from the audio: the waveform at the loop start correlates with the waveform at the loop end at +0.64 to +0.86 on six of seven discs, against a control of about zero. A loop end past the audio is refused rather than clamped, which is where this format parts company with AKAI and Roland -- clamping turns +0.86 into -0.10 on protozoa's own records. No root key ships: the record does not carry one, measured against the notes in the sample names over 2 658 records. Additive by construction, and asserted -- the disc-backed suite now pins the SHA-256 of every sample payload per disc, and all seven are unchanged. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 14 ++ docs/README.md | 5 +- ...-read-e-iv-through-its-sample-directory.md | 2 + ...the-loop-is-decoded-the-root-key-is-not.md | 101 +++++++++++++ docs/adr/README.md | 3 +- docs/formats/emu3.md | 138 ++++++++++++++++-- src/samplerdisc/fs/emu3.py | 96 +++++++++++- src/samplerdisc/sample/emu3.py | 110 +++++++++++++- src/samplerdisc/wav.py | 23 ++- tests/fixtures.py | 40 ++++- tests/test_discs.py | 73 +++++++-- tests/test_emu3.py | 128 +++++++++++++++- tests/test_loops.py | 43 ++++++ tests/test_wav.py | 25 +++- 14 files changed, 755 insertions(+), 46 deletions(-) create mode 100644 docs/adr/0025-the-loop-is-decoded-the-root-key-is-not.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 227a642..e8e213f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ Notable changes to `samplerdisc`. Format-level findings live in [docs/formats/]( ### Added +- **E-mu samples carry their loop points.** The 92-byte sample record's undecoded middle is an eight-pointer block — a start, an end, a loop start and a loop end, per channel, as byte offsets from the record's own start — and **8 039 of the 14 738** E-mu samples now come out with those loop points in the WAV's `smpl` chunk: 107, 1 157, 1 260, 1 689, 449, 2 551 and 826 across the seven reference discs. Every other backend has carried this since D7; the E-mu path carried none. ([docs/formats/emu3.md](docs/formats/emu3.md), [ADR-0025](docs/adr/0025-the-loop-is-decoded-the-root-key-is-not.md)) + + **The loop was established from the audio, not from the structure fitting.** Correlating the waveform at the loop start against the waveform at the loop end gives **+0.64 to +0.86 on six of the seven discs**, against a control — the same loop end with the start put somewhere else — of about zero on all of them. Three publishers and both E-mu generations agree. `eiv-analogia` is the exception and is reported as one: only 34 of its records carry audio loud enough at both ends to score, and those 34 show nothing, so its loops rest on the rule the other six establish. + + **A loop end past the audio is refused, not clamped.** AKAI and Roland both clamp a declared end back to the audio present, because these rips run short of their directories. Doing it here destroys the loop, and `protozoa` shows it on one disc: of its one-channel records, the 689 whose end already fits correlate at **+0.86** and the 525 whose end overshoots score **−0.10** once clamped. That is why `esi32-gm` — the format doc's own reference disc — yields loops on only 5% of its samples. + + **No root key ships, because the record does not carry one.** No byte of the 92 tracks the note written in the sample's own name above chance, over 1 741 named records of one disc and 917 of another. The field that looks most like a root key turns out to be the sample rate. Deriving one from the name was considered and rejected: it would be indistinguishable, in the output, from a value the disc actually stated. The WAV carries its loop with the RIFF neutral root key of 60, which the docs say plainly is a placeholder. + + **Nothing else moved, and that is asserted rather than claimed.** The disc-backed suite now pins the SHA-256 of every sample payload per disc alongside the counts — a count table cannot see a payload that shifted by a byte while staying the same length. All seven digests are unchanged. + - **AIFF payloads are converted, not copied out as `.aiff`.** An AIFF's samples are big-endian and a WAV's are little-endian, so the bytes within each value are reversed and the values are left alone — a re-ordering, exactly reversible, with no resampling or change of depth. Root key, tuning and the sustain loop come across from the AIFF's `INST` and `MARK` chunks into the WAV's `smpl`. ([docs/formats/aiff.md](docs/formats/aiff.md), [ADR-0024](docs/adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)) AIFF-C is refused rather than guessed at — its payload may be compressed, and compressed data written out as PCM opens, plays as noise and reports nothing wrong. 8-bit is refused too: AIFF stores it signed and WAV unsigned, so carrying it would mean adding 128 to every sample, which is a change to the values and not to their order. Nothing in the collection is either. @@ -44,6 +54,8 @@ Notable changes to `samplerdisc`. Format-level findings live in [docs/formats/]( ### Fixed +- **The E-mu sample record's field offsets were listed wrong, and one field was misnamed.** The eight "undecoded fields" at `+18`, `+24`, `+28` … were a four-byte stride begun in the wrong place, straddling two real fields at every step. The real block sits at `+22`, `+26`, `+30` … `+50`. `OFF_SAMPLE_HEADER_LEN` at `+22` was never a header length: it is the sample's start pointer, which reads 92 because the header is 92 bytes — and that is why it reads **0** on 547 of `eiv-studio`'s records, something [docs/formats/emu3.md](docs/formats/emu3.md) recorded as an oddity and can now explain. It also explains the `+34` "bias of two": the pointer addresses the last word rather than one past it. + - **An AKAI volume that lists nothing now says why, in the disc's own words.** Ten volumes across three discs listed empty with no explanation, which is exactly the [ADR-0012](docs/adr/0012-a-probe-must-confirm-a-file.md) signature — it reads as an empty volume rather than as a wrong answer. Each now carries a note naming the block and what the partition's allocation map says is in it: file data on `Advance Orchestra` ×4 and the OMI disc, a free block on `Kickin' Lunatic Beats 2 CD1`, and — for four more on that disc — a block the disc says *is* a volume directory and the image has none at. ([#16](https://github.com/bmxcode/samplerdisc/issues/16), [#17](https://github.com/bmxcode/samplerdisc/issues/17), [ADR-0022](docs/adr/0022-a-volume-is-explained-by-the-allocation-map.md)) **No volume or file count moves anywhere** — 872 volumes and 56 662 files across the collection, before and after, unchanged to the number. That is the point rather than a happy accident: the map explains an emptiness and never gates a listing, because the one-line fixes that *do* gate cost real audio. Rejecting volumes whose type byte is 0 discards four volumes carrying 63 files, and trusting the map as an allocation flag discards those and every volume on every S3000 and CD3000 disc besides. @@ -72,6 +84,8 @@ Notable changes to `samplerdisc`. Format-level findings live in [docs/formats/]( ### Known limits +- **E-mu stereo samples are written as double-length mono.** Roughly **2 700 of the 14 738** E-mu samples declare two channels in their record, stored as a block of left followed by a block of right. Reading the payload as one mono stream concatenates them. The earlier finding that "everything is mono" tested *interleaved* stereo — a sound refutation of the wrong hypothesis, since de-interleaving cannot detect a block split. Measured directly, the two halves of a two-channel record are the same performance, correlating by RMS envelope at 0.99, 0.96 and 0.95 on three discs against 0.13–0.26 for one-channel records. Not fixed here: correcting it moves audio, and this release was scoped to add metadata without touching a byte of payload. ([docs/formats/emu3.md](docs/formats/emu3.md), [ADR-0025](docs/adr/0025-the-loop-is-decoded-the-root-key-is-not.md)) + - **The ISO 9660 directory hierarchy is not preserved.** A disc's audio is written flat into one directory per volume, so `PS-34 AIFF …/056_Ballad de Boo/34a-bas-56Dmin.aif` becomes `34a-bas-56Dmin.wav` and the folder that grouped it by tempo is gone. Nothing is lost on these discs — Best Service named every file uniquely, and all thirteen list zero collisions after flattening — but a disc that reused a basename would rely on `unique_path` suffixing it. - **AIFF-C is refused, and no disc exercises a reader for it.** Nothing in the collection is AIFF-C, so there is nothing to check one against; a compressed payload written out as PCM would open, play as noise, and report nothing wrong. Same for 8-bit AIFF, where the sign convention differs from WAV's and carrying it would change the sample values. diff --git a/docs/README.md b/docs/README.md index abaa941..9ede166 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,7 @@ All of that vanishes into a working parser. Six months on, the code says *what* | D14 | E-mu Emulator IV bank extraction ([ADR-0020](adr/0020-read-e-iv-through-its-sample-directory.md)) | done | | D15 | Every partition of an AKAI disc, from the table it declares ([ADR-0023](adr/0023-partitions-come-from-the-table-the-disc-declares.md)) | done | | D16 | AIFF payloads converted, deduplicated against their WAV twin, and EXS24/HALion instruments kept ([ADR-0024](adr/0024-the-aiff-twin-is-converted-and-deduplicated.md)) | done | +| D17 | E-mu loop points in the WAV `smpl` chunk ([ADR-0025](adr/0025-the-loop-is-decoded-the-root-key-is-not.md)) | done | Across the local collection, by listing: 71 of 79 images claimed, 2 578 volumes, 110 989 files, 77 620 of them samples. The AKAI discs are 44 of those images and 68 997 of those files, read across 275 partitions — before D15 they were 14 670 files, because only the partition at the origin was read. @@ -63,7 +64,9 @@ Across the local collection, by listing: 71 of 79 images claimed, 2 578 volumes, - **Roland S-550.** `Roland LCD1.iso`/`.nrg` opens `* ROLAND S-550 *` and is a different format from the S-7xx entirely ([ADR-0014](adr/0014-one-backend-per-on-disc-format.md)). Neither archive holds a second specimen, so it stays deferred rather than being reverse-engineered from one disc. - **Ensoniq and Kurzweil backends.** The archives are full of these discs and the containers already open them; each needs a module in `fs/` and nothing else ([ADR-0003](adr/0003-brand-neutral-pluggable-backends.md)). -- **Loop points and root key for E-mu.** The 92-byte sample header has eight undecoded fields (`+18`, `+24`, `+28`, `+32`, `+36`, `+40`, `+44`, `+48`); some are very likely loop points and root key. [ADR-0011](adr/0011-the-deliverable-is-daw-ready-wav.md) wants them in the WAV `smpl` chunk and the E-mu path writes none. Deferred rather than done because it changes the *shared* record parser and would alter every E-mu sample already extracted. +- **E-mu stereo samples come out as double-length mono.** The sample record's pointer block declares a channel count, and roughly **2 700 of the 14 738** E-mu samples declare two — stored as a block of left followed by a block of right, not interleaved. Reading the payload as one mono stream concatenates the channels. D17 established this and deliberately did not fix it: correcting it moves audio, and that deliverable was scoped to add metadata without touching a byte of payload. It is the largest known defect in the E-mu path ([formats/emu3.md](formats/emu3.md), [ADR-0025](adr/0025-the-loop-is-decoded-the-root-key-is-not.md)). +- **`esi32-gm` and `protozoa` declare a longer extent than their record length gives.** On about 2 200 records of one and 3 800 of the other, `end_L` runs some 45 frames past the payload `+34` produces. Either the reader is 90 bytes short on those samples or the extent field means something else; the splice test does not settle it — a scan for the true loop end peaks within ±2 frames of the declared one on 10% and 20% of records, against about 3% for chance. Those records are refused a loop rather than given a clamped one, and nothing is changed on the strength of an unsettled measurement. +- **No root key for E-mu.** It is not in the sample record: no byte of the 92 tracks the note in the sample's own name above chance, over 1 741 named records of `esi32-gm` and 917 of `eiiix-1`. The E3 keeps it in the preset, and presets are not read — see the `E4P1` entry below. E-mu WAVs carry their loop with the RIFF neutral root key of 60 ([ADR-0025](adr/0025-the-loop-is-decoded-the-root-key-is-not.md)). - **`E4P1` presets are not read.** The three E-IV discs carry 916, 901 and 284 of them. On `eiv-studio` 100 of 230 banks have no sample directory and are listed with a note; preset-only banks are the likely explanation, and it is not established. - **The `.mds` track table is unread.** One real pair now reads end to end, but geometry is sniffed from the `.mdf` rather than taken from the descriptor, so a multi-track or offset image would be read from byte 0. What the one specimen's descriptor holds is written down in [formats/mdx.md](formats/mdx.md) without being relied on. - **CUES chunks in NRG.** Only `CUEX` is parsed; `CUES` encodes position as MSF and no disc using it was available to check the layout against. diff --git a/docs/adr/0020-read-e-iv-through-its-sample-directory.md b/docs/adr/0020-read-e-iv-through-its-sample-directory.md index c7c69f8..ed573c8 100644 --- a/docs/adr/0020-read-e-iv-through-its-sample-directory.md +++ b/docs/adr/0020-read-e-iv-through-its-sample-directory.md @@ -42,6 +42,8 @@ Three things follow from that, and each is the same rule applied at a different **Treat the paired length fields as a channel count.** `+34 == 2 × (+30) − 90` on both EIII and E-IV looks exactly like a mono/stereo flag, and building a stereo path on it was planned. Rejected on measurement: de-interleaving a payload as stereo roughly doubles its sample-to-sample delta, which is what decimating a smooth mono signal does, and the known-good `esi32-gm` `Piano E0` scores the same as every E-IV record. Everything is mono. E-IV pairs samples into stereo the way the rest of the collection does, by name ([ADR-0017](0017-the-stereo-side-marker-is-a-character-class.md)). +> **This rejection is overturned by [ADR-0025](0025-the-loop-is-decoded-the-root-key-is-not.md); the rest of this record stands.** The measurement above is sound about *interleaved* stereo and tested the wrong hypothesis: the format splits into blocks — all of the left channel, then all of the right — which de-interleaving cannot detect. The two halves of a record whose pointers declare two channels are the same performance, correlating at 0.99, 0.96 and 0.95 by RMS envelope on three discs against 0.13–0.26 for one-channel records. The paired fields *are* a channel count. Nothing else here depends on it: bank binding, the chain and the directory's length are untouched. + ## Consequences **Good.** Three discs that could only be listed now extract: 449, 2 822 and 828 samples. The four EIII/ESI counts are unchanged, which is the check that the shared record parser was not disturbed. diff --git a/docs/adr/0025-the-loop-is-decoded-the-root-key-is-not.md b/docs/adr/0025-the-loop-is-decoded-the-root-key-is-not.md new file mode 100644 index 0000000..b5d5135 --- /dev/null +++ b/docs/adr/0025-the-loop-is-decoded-the-root-key-is-not.md @@ -0,0 +1,101 @@ +# ADR-0025 · The E-mu loop is decoded from the record; the root key is not there to decode + +**Status:** accepted · 2026-08-21 + +## Context + +Every other backend carries what its disc knows about a sample into the WAV's own `smpl` chunk — AKAI since D7, Roland since D13, AIFF since D16. The E-mu path carried none, so 14 738 samples across seven discs shipped as plain WAV. [ADR-0011](0011-the-deliverable-is-daw-ready-wav.md) asks for exactly this and calls dropping it *data loss, not neutrality*. + +It was deferred for a good reason: the 92-byte sample record is read by **one parser shared across all seven discs and two generations**, so decoding it risks moving output that four earlier deliverables pinned. + +The eight fields `docs/README.md` listed as undecoded — `+18`, `+24`, `+28`, `+32`, `+36`, `+40`, `+44`, `+48` — are not fields. They are a four-byte stride started at the wrong place, and a `u32` read at `+28` straddles two real fields, which is why they dump as nine-digit noise. The record carries an **eight-pointer block at `+22`, `+26`, `+30`, `+34`, `+38`, `+42`, `+46`, `+50`**: a start, an end, a loop start and a loop end, per channel, each a byte offset from the record's own start naming the first byte of a 16-bit word. The full layout and every disc it was checked against are in [formats/emu3.md](../formats/emu3.md). + +Three things the project already recorded fall out of that reading at no cost, which is the strongest argument that it is right: + +- **`OFF_SAMPLE_HEADER_LEN` was never a header length.** It is `start_L`, and it reads 92 because the header is 92 bytes and the audio begins immediately after it. The format doc recorded that this field "reads 92 on most E-IV records and **0 on 547 of `eiv-studio`'s** — those carry 92 at `+26` instead" and left it as an oddity. Those records declare no left channel. It is not a broken field, it is a different value of a working one. +- **`RECORD_LEN_BIAS = 2` stops being a bias.** `+34` is `end_R`, which addresses the *last word* rather than one past it, so the record ends two bytes further on. +- **`+34 == 2 × (+30) − 90`**, which the "Everything is mono" section reported without explaining, is `end_R = end_L + P/2` written out. + +## Decision + +**Decode the loop from the pointer block. Do not decode a root key, because the record does not carry one. Write the `smpl` chunk for the loop alone.** + +### The loop is established by content, on six discs and three publishers + +The oracle is the one [formats/roland-s7xx.md](../formats/roland-s7xx.md) used for the S-7xx sustain loop, guards included. Two tests on different evidence: the **join**, `|x[E−1] − x[L]|` over the mean step measured in a window at each end; and the **shape**, the correlation of the waveform at `L` against the waveform at `E`, which is the stronger instrument here. + +| Disc | Publisher | scored | shape *r* | control | join | control | seamless | control | +|---|---|---|---|---|---|---|---|---| +| `eiiix-1` | E-mu | 512 | **+0.70** | +0.02 | 0.66 | — | 91% | — | +| `eiiix-2` | E-mu | 526 | **+0.73** | −0.01 | 1.12 | — | 81% | — | +| `protozoa` | E-mu | 723 | **+0.83** | −0.01 | 1.22 | — | 83% | — | +| `esi32-gm` | E-mu | 16 | **+0.64** | +0.17 | 0.32 | — | 81% | — | +| `eiv-studio` | Producer Series | 1 051 | **+0.86** | −0.02 | 1.49 | 5.29 | 79% | 34% | +| `eiv-vitous` | Miroslav Vitous | 144 | **+0.68** | −0.06 | 2.30 | 7.21 | 64% | 25% | +| `eiv-analogia` | Producer Series | 34 | −0.00 | +0.02 | 2.44 | 5.36 | 53% | 25% | + +The control is the same loop end with the start put somewhere else, which is the only thing that isolates the *start* — the end cannot be isolated the same way, because `+46` and `+30` sit six frames apart on most records and no test can separate them. + +`eiv-analogia` is stated as it came out. 441 of its 449 records pass the gates but only 34 carry audio loud enough at both ends to score, and those 34 show nothing. That is a lack of power, not a refutation, and analogia's loops rest on the rule the other six discs establish rather than on its own evidence. [ADR-0020](0020-read-e-iv-through-its-sample-directory.md)'s independence requirement is still met without it: two E-mu-published generations, Producer Series through `eiv-studio`, and Vitous. + +### A loop end past the audio is refused, not clamped + +This is the one place the format parts company with the rest of the project, and it is the finding that made the measurement work at all. AKAI and Roland both **clamp** a declared end back to the audio present, because a rip is routinely a little shorter than its directory claims. + +Doing that here destroys the loop, and `protozoa` proves it on a single disc within a single record shape: + +| `protozoa`, mono-shaped records | count | shape *r* | control | +|---|---|---|---| +| loop end already inside the payload | 689 | **+0.86** | −0.04 | +| loop end past the payload, clamped back | 525 | **−0.10** | +0.01 | + +Same disc, same shape, separated only by whether the end fits. A clamped end is a loop point the disc did not state. So the end is a **gate**: a record whose loop end lies past its audio yields no loop. + +That is most of why `esi32-gm` — the format doc's own reference disc — yields only **107 loops of 2 265 samples**. On almost every record it declares an extent about 45 frames longer than the payload the record's own length field produces. Which of the two is right is not established here and is not guessed at. + +### The root key is not in the record + +No byte of the 92 tracks the note written in the sample's own name. Across **1 741 named records on `esi32-gm` the best byte matches at any constant offset on 8%**, and 917 on `eiiix-1` on 6% — chance, and the "winners" are constant-zero bytes. `+58` looked promising and turned out to track the sample **rate**: 64 096 ↔ 12 000, 64 184 ↔ 13 000, 64 266 ↔ 14 000, 64 388 ↔ 15 625. + +The E3 keeps root key in its preset. `E4P1` presets are not read and `docs/README.md` already says so. + +So `Emu3Sample.pitch` is `None`, always — the same value [sample/aiff.py](../../src/samplerdisc/sample/aiff.py) gives an AIFF with no `INST` — and `write_wav` writes the `smpl` chunk for the loop with `MIDIUnityNote` 60. The chunk has no way to say "no root key"; the field is mandatory. **60 is the neutral value, not a finding**, and it is recorded as such here and in the format doc. + +## Alternatives rejected + +**Derive the root key from the sample name.** `Piano E0` → 28, `CP70 D#2` → 51; it would cover 1 741 of `esi32-gm`'s records and reads as a free win. Rejected, and it is the most important rejection here. A name is a label someone typed; this project's whole discipline is content over declared text ([ADR-0004](0004-detect-by-signature.md)), and the same reasoning already made `_pinned_disc` identify a disc by size rather than by filename. Worse, the output would be **indistinguishable from a decoded field** — nothing downstream could tell a root key the disc stated from one this tool inferred from a string, which is the failure mode [ADR-0012](0012-a-probe-must-confirm-a-file.md) exists to prevent. + +**Write no `smpl` chunk without a root key.** Strictly conservative, and it changes nothing for any other backend. Rejected: it would measure 8 039 loops, document them, and then ship none of them, on the strength of a field the format does not have. ADR-0011 wants what the disc knows carried; the disc knows the loop. + +**Clamp the loop end, as AKAI and Roland do.** Consistent with the rest of the project and it would raise `esi32-gm` from 107 loops to about 2 262. Rejected on the `protozoa` measurement above: the clamped loops score −0.10 against a control of −0.01, which is to say they are not loops. This is a case where consistency across backends would have been consistency in the wrong thing. + +**Emit the right-hand pointer set as a second loop.** On a two-channel record both sets name the same loop, in each half. Rejected as duplication: one loop in the audio, listed once. + +**Trust the declared extent over the record length on `esi32-gm`.** Its `end_L` runs about 45 frames past the payload on ~2 200 records, so reading 90 more bytes would let those loops through. Rejected: the shape test does **not** peak at the declared end when the extra audio is read — 10% within ±2 frames on `esi32-gm`, against 20% on `protozoa` — so nothing confirms that the reader is short. Changing the extent would also move every payload on two discs, which is exactly the shared-parser risk this deliverable was deferred over. It is recorded as open in [formats/emu3.md](../formats/emu3.md) instead. + +## Consequences + +**Good.** 8 039 of 14 738 E-mu samples now carry their loop points: 107, 1 157, 1 260, 1 689, 449, 2 551 and 826 across the seven discs. They were on the disc, they are lost when the disc is, and they cost a chunk a DAW may ignore. + +**Good, and asserted.** The change is additive by construction — nothing in it touches `read_file` or the offset arithmetic — and the disc-backed suite now pins the **SHA-256 of every sample payload per disc** rather than only the counts. All seven are unchanged from the release before D17. A count table cannot see a payload that shifted by a byte while staying the same length; a digest can. + +**Bad, and the headline.** `esi32-gm` is the format doc's reference disc and gets loops on 5% of its samples. The reference bank `8M GeneralMidi X` is largely among them. Refusing is the right call on the measurement, but a reader coming to the doc will find its worked example is the disc that yields least. + +**Bad.** The pointer block was decoded from these seven discs and the two record shapes they show. A generation that writes a third shape gets no loops and nothing says so — it is one more case of a gate that is silent when it declines. + +**Watch for.** The gates loosening. Every one of them — the end inside the audio, the minimum loop length, the whole-extent rejection — exists because a plausible alternative was measured and found to be wrong on real data. The minimum-length guard in particular is [ADR-0011](0011-the-deliverable-is-daw-ready-wav.md)'s bargain held up: a metric that rewards silence will find plenty of it on a sampler disc, and the Roland record says so from its own experience. + +## What this exposes about ADR-0020 + +[ADR-0020](0020-read-e-iv-through-its-sample-directory.md) rejected "treat the paired length fields as a channel count" on a measurement, and **that measurement tested the wrong hypothesis.** It de-interleaved payloads as `LRLR` and found the roughness roughly doubled, which is what decimating a mono signal does — a sound conclusion about *interleaved* stereo. The pointer block says the layout would be **block** split, all left then all right, which de-interleaving cannot detect. + +Measured directly, on the records whose pointers declare two channels, the two halves are the same performance. Median RMS-envelope correlation between them, per disc, against the same measurement on records declaring one channel: + +| | `esi32-gm` | `eiiix-1` | `eiiix-2` | `protozoa` | `analogia` | `studio` | `vitous` | +|---|---|---|---|---|---|---|---| +| two channels declared | **0.99** | **0.71** | **0.95** | **0.55** | **0.91** | **0.96** | **0.82** | +| one channel declared | 0.13 | 0.16 | 0.59 | 0.26 | 0.21 | 0.22 | — | + +Those samples are stereo, and this project currently writes each as a double-length mono WAV. `eiiix-2`'s one-channel figure of 0.59 is the weakest separation and is left standing rather than explained away; `protozoa` has only 18 two-channel records with enough audio to score. + +Nothing in D17 changes that — fixing it moves audio, which is the one thing this deliverable must not do. It is written up in [formats/emu3.md](../formats/emu3.md) and left open for a deliverable of its own. The loop frames decoded here survive it unchanged: `(pointer − start) / 2` is a per-channel frame index either way. diff --git a/docs/adr/README.md b/docs/adr/README.md index 3f9d77b..1c112ce 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -25,8 +25,9 @@ If you find yourself writing an ADR with no rejected alternative, you are writin | [0017](0017-the-stereo-side-marker-is-a-character-class.md) | The stereo side marker is a character class | A per-backend hook; renaming Roland to AKAI's spelling; an optional separator | | [0018](0018-the-s7xx-sample-rate-is-measured.md) | The S-7xx rate is 44 100 by measurement, not by a field | Refusing to extract; a `--rate` override; per-sample pitch inference | | [0019](0019-prefer-joliet-names.md) | Prefer Joliet names over the ISO 9660 short names | De-duplicating the short names; merging the two trees; Rock Ridge; naming from the payload | -| [0020](0020-read-e-iv-through-its-sample-directory.md) | Read E-IV through its `E3S1` sample directory | Waiting for a fourth disc; arithmetic on `start`; a signature walk; an `EMU4` backend; a stereo channel count | +| [0020](0020-read-e-iv-through-its-sample-directory.md) | Read E-IV through its `E3S1` sample directory | Waiting for a fourth disc; arithmetic on `start`; a signature walk; an `EMU4` backend; ~~a stereo channel count~~ *overturned by 0025* | | [0021](0021-a-bank-owns-the-run-its-header-declares.md) | An EIII/ESI bank owns the record run its own header declares | Bounding by the directory's `length`; deduplicating headers by address; dropping repeated names; listing the `4k` banks with a note; a looser header match | | [0022](0022-a-volume-is-explained-by-the-allocation-map.md) | An AKAI volume's emptiness is explained by the partition's allocation map | Rejecting type 0; requiring the directory to parse; using the map as an allocation flag; recovering the displaced directories; calling the damage in the note | | [0023](0023-partitions-come-from-the-table-the-disc-declares.md) | An AKAI disc's partitions come from the table it declares | Tiling at multiples of the first size; chaining each header's own size; locating headers by signature; walking partitions in the probe; rewriting block numbers as disc-relative; nesting extraction only where a disc has several | | [0024](0024-the-aiff-twin-is-converted-and-deduplicated.md) | Convert AIFF, and drop the twin only when it says nothing new | Writing both trees; never converting; deduplicating by name; preferring the AIFF; merging its metadata into the copied WAV; calling the byte swap a conversion | +| [0025](0025-the-loop-is-decoded-the-root-key-is-not.md) | The E-mu loop is decoded from the record; the root key is not there to decode | Deriving the root key from the sample name; no `smpl` chunk without a root key; clamping the loop end as AKAI and Roland do; emitting the second channel's loop as well; trusting the declared extent over the record length | diff --git a/docs/formats/emu3.md b/docs/formats/emu3.md index 892fa06..cf371e1 100644 --- a/docs/formats/emu3.md +++ b/docs/formats/emu3.md @@ -182,13 +182,98 @@ A record starts **two bytes before its name**; those two bytes are `00 00` on ev | Offset in record | Size | Meaning | |---|---|---| | 2 | 16 | name, ASCII | -| 18 | 4 | u32 LE checksum | -| 22 | 4 | u32 LE **header length — 92 on every record measured** | -| 34 | 4 | u32 LE record length, **two short** of the distance to the next (EIII only) | +| 18 | 4 | u32 LE, not identified — see below | +| 22 | 4 | u32 LE `start_L` — **92 on every record measured**, and the signature the walk scans for | +| 26 | 4 | u32 LE `start_R` | +| 30 | 4 | u32 LE `end_L` | +| 34 | 4 | u32 LE `end_R` — the same field as the record length, **two short** of the distance to the next (EIII only) | +| 38 | 4 | u32 LE `loop_start_L` | +| 42 | 4 | u32 LE `loop_start_R` | +| 46 | 4 | u32 LE `loop_end_L` | +| 50 | 4 | u32 LE `loop_end_R` | | 54 | 4 | u32 LE **sample rate** | | 92 | … | sample data | -The signature — header length exactly 92, a plausible rate, sixteen printable name bytes — is specific enough to scan megabytes of audio without false hits. On `esi32-gm`'s `8M GeneralMidi X` bank it yields **452 records with 452 distinct names** totalling 7.00 MiB inside a bank declaring 8 MiB. +The signature — 92 at `+22`, a plausible rate, sixteen printable name bytes — is specific enough to scan megabytes of audio without false hits. On `esi32-gm`'s `8M GeneralMidi X` bank it yields **452 records with 452 distinct names** totalling 7.00 MiB inside a bank declaring 8 MiB. + +### The eight pointers + +`+22` through `+50` are one block: a start, an end, a loop start and a loop end, **per channel**. Every one is a **byte offset from the record's own start**, naming the first byte of a 16-bit word. + +An earlier revision of this doc listed eight *undecoded* fields at `+18`, `+24`, `+28`, `+32`, `+36`, `+40`, `+44`, `+48`. Those are a four-byte stride started at the wrong place: a `u32` read at `+28` straddles `end_L` and `end_R`, which is why they dump as nine-digit noise and why nothing was ever made of them. + +Reading them as pointers explains three things this doc already recorded and could not account for: + +- **`+22` is not a header length.** It is `start_L`, and it reads 92 because the header is 92 bytes and the audio begins immediately after it. That is why it also serves as the signature. It reads **0** on 542 of `eiv-studio`'s records and 146 of `eiv-analogia`'s — this doc previously noted 547 "carry 92 at `+26` instead" and left it as an oddity. Those records declare no left channel. Not a broken field: a different value of a working one. +- **`+34`'s "bias of two" is not a bias.** The pointer addresses the *last word* rather than one past it, so the record ends two bytes further on. +- **`+34 == 2 × (+30) − 90`**, reported below without an explanation, is `end_R = end_L + P/2`. + +Two record shapes occur, and a reader must handle both. `start_R == start_L + P/2` declares **two channels**, the payload being all of the left then all of the right. `start_R` equal to `start_L`, or zero, declares **one**. Counts per disc: + +| Disc | two channels | one channel | neither | +|---|---|---|---| +| `esi32-gm` | 28 | 2 230 | 7 | +| `eiiix-1` | 641 | 380 | 168 | +| `eiiix-2` | 598 | 680 | 55 | +| `protozoa` | 27 | 5 791 | 34 | +| `eiv-analogia` | 279 | 146 | 24 | +| `eiv-studio` | 320 | 1 882 | 620 | +| `eiv-vitous` | 828 | 0 | 0 | + +**Do not derive one channel's pointers from the other's.** `loop_start_R == loop_start_L + P/2` holds exactly on `eiv-vitous`'s 828 records and on 1.2% of `esi32-gm`'s. Read the set whose start is 92 and use it. + +`+18` is **not identified**. An earlier revision of this doc called it a u32 checksum and `docs/README.md` listed it undecoded; neither cited a measurement and neither is followed here. It is not needed for anything. + +## Loop points + +The loop is `(+38, +46)` read against `+22` — or `(+42, +50)` against `+26` on a record that declares no left channel. In frames: `(pointer − start) / 2`. + +**Established by content, not by the structure fitting.** Two measurements on different evidence, the method [roland-s7xx.md](roland-s7xx.md) used for the S-7xx sustain loop, guards included — a minimum loop length of 64 frames and a window RMS at both ends of at least 15% of the sample's peak, because a metric that rewards silence will find plenty of it. + +The **join** is `|x[E−1] − x[L]|` over the mean sample-to-sample step measured in a 64-frame window at each end. The **shape** test correlates the waveform at `L` against the waveform at `E`: the same phase of the same note correlates, a wrong start lands at a random phase. The control for both is *the same loop end with the start put somewhere else*, which is what isolates the start field. + +| Disc | scored | shape *r* | control | join | seamless (<3×) | control seamless | +|---|---|---|---|---|---|---| +| `eiiix-1` | 512 | **+0.70** | +0.02 | 0.66 | 91% | — | +| `eiiix-2` | 526 | **+0.73** | −0.01 | 1.12 | 81% | — | +| `protozoa` | 723 | **+0.83** | −0.01 | 1.22 | 83% | — | +| `esi32-gm` | 16 | **+0.64** | +0.17 | 0.32 | 81% | — | +| `eiv-studio` | 1 051 | **+0.86** | −0.02 | 1.49 | 79% | 34% | +| `eiv-vitous` | 144 | **+0.68** | −0.06 | 2.30 | 64% | 25% | +| `eiv-analogia` | 34 | −0.00 | +0.02 | 2.44 | 53% | 25% | + +The **end** cannot be isolated the same way: `+46` and `+30` sit six frames apart on most records, so no test separates them. What is established is the start, and that the pair splices. + +`eiv-analogia` is stated as it came out. 441 of its 449 records pass every structural gate but only 34 carry audio loud enough at both ends to score, and those 34 show nothing. That is a lack of power rather than a refutation, and its loops rest on the rule the other six discs establish. + +### A loop end past the audio is refused, not clamped + +[akai-fs.md](akai-fs.md) and [roland-s7xx.md](roland-s7xx.md) both clamp a declared loop end back to the audio actually present, because a rip is routinely a little short of its directory. **Here the same move destroys the loop**, and `protozoa` shows it on one disc within one record shape: + +| `protozoa`, one-channel records | count | shape *r* | control | +|---|---|---|---| +| loop end already inside the payload | 689 | **+0.86** | −0.04 | +| loop end past the payload, clamped back | 525 | **−0.10** | +0.01 | + +Same disc, same shape, separated only by whether the end fits. A clamped end is a loop point the disc did not state, so a record whose loop end lies past its audio yields **no** loop ([ADR-0025](../adr/0025-the-loop-is-decoded-the-root-key-is-not.md)). + +That is most of why the reference disc yields fewest. On about 2 200 of `esi32-gm`'s records `end_L` runs roughly 45 frames past the payload its own length field produces, and 95% of the disc is refused on it. + +**Which of the two is right is open.** If the record's extent is correct, the reader is 90 bytes short on those samples. Scanning candidate ends across ±96 frames and taking the shape-test peak does *not* confirm it: the peak lands within ±2 frames of the declared end on **10%** of `esi32-gm`'s records and 20% of `protozoa`'s, where a uniform peak would give about 3%. Better than chance, nowhere near an answer. Nothing is changed on the strength of it. + +### There is no root key + +**No byte of the 92 tracks the note written in the sample's own name.** Measured against the names themselves — `Piano E0`, `CP70 D#2`, `Arco Violin F#2` — over every byte position and every constant offset: + +| Disc | named records | best byte matches | +|---|---|---| +| `esi32-gm` | 1 741 | 8% | +| `eiiix-1` | 917 | 6% | +| | | *chance, on constant-zero bytes* | + +`+58` is the field that looks most like it and is the **sample rate** in another form: 64 096 ↔ 12 000, 64 184 ↔ 13 000, 64 266 ↔ 14 000, 64 388 ↔ 15 625. + +The E3 keeps root key in its preset, and presets are not read. So an E-mu WAV carries its loop with `MIDIUnityNote` 60 — the RIFF neutral value, written because the field is mandatory and a loop cannot be carried without it. **60 is a placeholder, not a finding** ([ADR-0025](../adr/0025-the-loop-is-decoded-the-root-key-is-not.md)). ## E-IV: the `E3S1` sample directory @@ -258,13 +343,28 @@ The EIII rule — `+34` plus a bias of two equals the distance to the next recor `OFF_SAMPLE_HEADER_LEN` is not usable as a validity test either. It reads 92 on most E-IV records and **0 on 547 of `eiv-studio`'s** — those carry 92 at `+26` instead. Requiring it drops a fifth of the disc. It is not needed: the directory already says where the record is and what it is called. -## Everything is mono +## Stereo: the payload is split, not interleaved — and this was got wrong -The 92-byte header carries two paired length fields — `+26`/`+30` against `+34`/`+50` — with `+34 == 2 × (+30) − 90` holding identically on `esi32-gm` and on E-IV. The obvious reading is a channel count, and it is wrong. +**A previous revision of this section concluded "everything is mono". That conclusion is wrong, and the way it was wrong is worth more than the answer.** -Measured by comparing the mean absolute sample-to-sample delta of the payload read as mono against the same payload de-interleaved as stereo: de-interleaving roughly **doubles** the roughness (ratio ≈ 0.5), which is what taking every other sample of a smooth mono signal does. Interleaved stereo would come out smoother, not rougher. The known-good `esi32-gm` `Piano E0` — mono, verified byte-identical in the section below — scores 0.58, the same as the E-IV records. So ≈ 0.5 is the signature of mono, confirmed against the reference. +What it measured was real: de-interleaving a payload as `LRLR` roughly **doubles** its mean sample-to-sample delta (ratio ≈ 0.5), which is what taking every other sample of a smooth mono signal does, and `esi32-gm`'s `Piano E0` scored 0.58 exactly like the E-IV records. That is a sound refutation of **interleaved** stereo. It says nothing about the layout the format actually uses. -There is no stereo record. E-IV discs do pair samples into stereo, and they do it the way the rest of the collection does: two mono records that [ADR-0017](../adr/0017-the-stereo-side-marker-is-a-character-class.md) joins by name. +The pointer block says what that layout is. `start_R == start_L + P/2` is a **block** split — all of the left channel, then all of the right — which de-interleaving cannot detect, because reading a block-split payload as mono gives one continuous waveform with a single join in the middle. + +Measured directly, on the records whose own pointers declare two channels, the halves are the same performance. Median RMS-envelope correlation between them, against the same measurement on records declaring one channel: + +| | `esi32-gm` | `eiiix-1` | `eiiix-2` | `protozoa` | `analogia` | `studio` | `vitous` | +|---|---|---|---|---|---|---|---| +| two channels declared | **0.99** | **0.71** | **0.95** | **0.55** | **0.91** | **0.96** | **0.82** | +| one channel declared | 0.13 | 0.16 | 0.59 | 0.26 | 0.21 | 0.22 | — | + +Envelope rather than sample-level correlation, because the two channels of a real recording differ in phase and agree in shape; sample-level correlation on the same records is modest and inconclusive, which is why it was not the instrument. `eiiix-2`'s one-channel figure of 0.59 is the weakest separation and is left standing rather than explained away, and `protozoa` has only 18 two-channel records with enough audio to score. + +**So roughly 2 700 of the 14 738 samples are stereo, and this project currently writes each as a double-length mono WAV.** That is not fixed here: correcting it moves audio, and [D17](../README.md) was scoped to add metadata without touching a byte of payload. It is open, and it is the largest known defect in the E-mu path. + +The loop points decoded above survive the fix unchanged. `(pointer − start) / 2` is a per-channel frame index either way: in the double-length mono file it lands in the left block, and in a correctly interleaved stereo file it is the frame number. + +E-IV discs *also* pair separate mono records into stereo by name, the way the rest of the collection does ([ADR-0017](../adr/0017-the-stereo-side-marker-is-a-character-class.md)). Both things are true, which is part of why one hid the other. ## The payload is little-endian @@ -302,7 +402,21 @@ Whole-disc listings: | `eiv-studio` | 230 | 2 822 | | `eiv-vitous` | 44 | 828 | -These are the regression baseline: any change to the shared record parser is a bug if they move. **They are now asserted by `tests/test_discs.py`, pinned by disc size** — a table in a document is a note, not a test, and two of these numbers were wrong for a release with a green suite. +Samples carrying a loop, of those totals: + +| Disc | Samples | With a loop | | +|---|---|---|---| +| `esi32-gm` | 2 265 | 107 | 5% | +| `eiiix-1` | 1 189 | 1 157 | 97% | +| `eiiix-2` | 1 333 | 1 260 | 95% | +| `protozoa` | 5 852 | 1 689 | 29% | +| `eiv-analogia` | 449 | 449 | 100% | +| `eiv-studio` | 2 822 | 2 551 | 90% | +| `eiv-vitous` | 828 | 826 | 100% | + +**8 039 of 14 738.** The two low ones are the discs that declare a loop end past the audio they carry, which is refused rather than clamped. + +These are the regression baseline: any change to the shared record parser is a bug if they move. **They are now asserted by `tests/test_discs.py`, pinned by disc size** — a table in a document is a note, not a test, and two of these numbers were wrong for a release with a green suite. The suite also pins the **SHA-256 of every sample payload per disc**, because a count table cannot see a payload that shifted by a byte while staying the same length. `esi32-gm`'s 2 424 and `protozoa`'s 6 788 are what the previous revision of this table gave, and both counted another bank's records; [ADR-0021](../adr/0021-a-bank-owns-the-run-its-header-declares.md) has the accounting. `esi32-gm` is the instructive one: it was believed clean, and its last bank ran to the end of the image and was credited with 193 records belonging to the two banks in front of it. @@ -324,7 +438,11 @@ Each of the four EIII/ESI discs lists one index bank with a note and no samples, - Each folder's bank directory must be bounded by the next folder's start block; they sit two blocks apart on `eiv-studio`. - On E-IV the record's own length field is unusable and the directory's is authoritative. The EIII rule matches 0 of 5 349 consecutive pairs. - An E-IV sample directory can appear twice. Deduplicate by address or every one of its records is listed twice. -- The paired length fields are not a channel count. Everything is mono, measured. +- The paired length fields **are** a channel count, and the measurement that said otherwise tested interleaved stereo when the format splits into blocks. Roughly 2 700 samples are stereo and currently come out as double-length mono. +- The sample record's fields are at `+22`, `+26`, `+30` … `+50`, not at `+24`, `+28`, `+32` … A four-byte stride begun at `+18` straddles two real fields at every step and reads as nine-digit noise. +- `+22` is a start pointer, not a header length. It reads 0 where a record declares no left channel, and requiring 92 drops a fifth of `eiv-studio`. +- A loop end past the payload must be **refused**, not clamped back the way AKAI and Roland clamp theirs. Clamping turns a splice correlation of +0.86 into −0.10 on `protozoa`'s own records. +- There is no root key in the sample record. No byte tracks the note in the sample's name above chance, and `+58` — the field that looks most like one — is the sample rate. ## What `protozoa` taught, in one place diff --git a/src/samplerdisc/fs/emu3.py b/src/samplerdisc/fs/emu3.py index 382631a..6b925a4 100644 --- a/src/samplerdisc/fs/emu3.py +++ b/src/samplerdisc/fs/emu3.py @@ -107,12 +107,56 @@ #: Sample record, relative to its own start. A record begins two bytes before #: its name; those two bytes are zero on every record after the first. SAMPLE_NAME_OFFSET = 2 -OFF_SAMPLE_HEADER_LEN = 22 OFF_SAMPLE_RECORD_LEN = 34 OFF_SAMPLE_RATE = 54 +#: The eight-pointer block, and the reason the fields either side of it read as +#: nonsense when sampled at four-byte strides from ``+18``. Every one is a +#: **byte offset from the record's own start**, naming the first byte of a +#: 16-bit word, and they come in (left, right) pairs -- the EIII is a +#: stereo-capable sampler and writes a pointer per channel (ADR-0025). +#: +#: ``OFF_SAMPLE_START_L`` is the field this walk scans for. It reads 92 because +#: the header is 92 bytes and the audio begins immediately after it, so the +#: same value serves as the record signature and as the start pointer -- which +#: is what it always was. It reads **0** on 542 of `studio`'s records and 146 +#: of `analogia`'s: those declare no left channel and put 92 at +#: ``OFF_SAMPLE_START_R`` instead, which is a different value of a working +#: field rather than a broken one. +#: +#: ``OFF_SAMPLE_END_R`` and ``OFF_SAMPLE_RECORD_LEN`` are one field under two +#: names, and that also explains ``RECORD_LEN_BIAS``: the pointer names the +#: first byte of the *last* word, so the record ends two bytes further on. +OFF_SAMPLE_START_L = 22 +OFF_SAMPLE_START_R = 26 +OFF_SAMPLE_END_L = 30 +OFF_SAMPLE_END_R = 34 +OFF_SAMPLE_LOOP_START_L = 38 +OFF_SAMPLE_LOOP_START_R = 42 +OFF_SAMPLE_LOOP_END_L = 46 +OFF_SAMPLE_LOOP_END_R = 50 + +#: The two channels' pointer sets, each as ``(start, end, loop start, loop +#: end)``. The set whose start reads 92 is the one that describes the audio; +#: the other is its mirror, or zeroed where the record declares one channel. +POINTER_SETS = ( + (OFF_SAMPLE_START_L, OFF_SAMPLE_END_L, OFF_SAMPLE_LOOP_START_L, OFF_SAMPLE_LOOP_END_L), + (OFF_SAMPLE_START_R, OFF_SAMPLE_END_R, OFF_SAMPLE_LOOP_START_R, OFF_SAMPLE_LOOP_END_R), +) + +#: What the pointers are carried on the ``File`` as. The filesystem layer reads +#: them and does not judge them; which set describes the audio, and whether the +#: loop it names is usable, is decided in ``sample/emu3.py`` -- the same split +#: Roland S-7xx uses, where the parameters also arrive beside the audio rather +#: than in front of it. +POINTER_KEYS = ( + ("start_l", "end_l", "loop_start_l", "loop_end_l"), + ("start_r", "end_r", "loop_start_r", "loop_end_r"), +) + #: Records sit back to back and each declares its own length two short of the -#: distance to the next, verified across a 12-record chain. +#: distance to the next, verified across a 12-record chain. See the pointer +#: block above for why two: ``+34`` addresses the last word, not past it. RECORD_LEN_BIAS = 2 #: Every EIII/ESI sample header measured is this long. Treated as a validity @@ -181,6 +225,28 @@ def is_plausible_name(raw: bytes) -> bool: return set(raw[len(text) :]) <= {0} +def sample_pointers(head: bytes) -> tuple[tuple[str, int], ...]: + """The eight-pointer block off a record header, as ``File.meta`` pairs. + + Read and carried, not judged: a pointer that is zero, odd or past the end + of the audio travels exactly as the disc wrote it, and ``sample/emu3.py`` + decides what is usable. Keeping the reading here and the judgement there is + what stops a loop rule from having to be re-derived if a fourth E-mu + generation turns up with a ninth pointer. + + Returns ``()`` for a header too short to hold the block, which is tail + damage: a record that was not fully read must not present as one that + declared nothing. + """ + if len(head) < OFF_SAMPLE_LOOP_END_R + 4: + return () + return tuple( + (key, struct.unpack_from(" str | None: """The name of the sample record eight bytes into ``window``, if any. A record is confirmed by its own tag and its name, *not* by the header - length at ``OFF_SAMPLE_HEADER_LEN``. That field reads 92 on most E-IV + length at ``OFF_SAMPLE_START_L``. That field reads 92 on most E-IV records and 0 on 547 of `studio`'s, so requiring it drops a fifth of the disc -- and it is not needed, because the directory already says where the record is and what it is called. @@ -632,6 +698,9 @@ def _eiv_samples(self, tags: dict[int, bytes], base: int, entries) -> Iterator[F size=size, start_block=record + SAMPLE_HEADER_LEN, raw_type=rate, + # The tag window already holds the record's header, so the + # pointers cost no extra read on either path. + meta=sample_pointers(window[EIV_RECORD_OFFSET:]), ) def volumes(self, image: SectorImage, offset: int) -> Iterator[Volume]: @@ -736,7 +805,7 @@ def _samples( window = image.read(offset + bank_at, max(limit - bank_at, 0)) needle = struct.pack(" tuple[str, int, int, int] | None: @@ -760,7 +830,7 @@ def _parse_record(self, head: bytes) -> tuple[str, int, int, int] | None: name = decode_name(raw) if not name: return None - (header_len,) = struct.unpack_from(" bytes: return image.read(offset + entry.start_block, entry.size) def parse_sample(self, entry: File, payload: bytes): - """The record's rate travelled on the File; the payload is already PCM.""" + """The record's rate and pointers travelled on the File; the payload is + already PCM. + + The 92 header bytes are in hand during the walk and gone by the time + the audio is read, so the loop pointers come across on the ``File`` + rather than being parsed out of the payload -- the same route the + Roland parameters take, for the same reason. + """ from samplerdisc.sample import emu3 as sample_emu3 - return sample_emu3.parse(payload, rate=entry.raw_type, fallback_name=entry.name) + return sample_emu3.parse( + payload, + rate=entry.raw_type, + fallback_name=entry.name, + pointers={key: entry.get(key) for key in sum(POINTER_KEYS, ())} if entry.meta else {}, + ) def original_suffix(self, entry: File) -> str: return ".e3s" if entry.kind == "sample" else ".bin" diff --git a/src/samplerdisc/sample/emu3.py b/src/samplerdisc/sample/emu3.py index 1edae90..a28ffda 100644 --- a/src/samplerdisc/sample/emu3.py +++ b/src/samplerdisc/sample/emu3.py @@ -1,9 +1,14 @@ -"""E-mu EIII/ESI sample payloads. See docs/formats/emu3.md. +"""E-mu EIII/ESI/E-IV sample payloads. See docs/formats/emu3.md. -There is nothing to decode. The filesystem layer already located the record and -read its rate, and the payload that follows the 92-byte header is signed 16-bit -little-endian PCM -- the same thing a WAV data chunk holds. So this module -carries the rate and the bytes and converts nothing (ADR-0011). +The audio needs nothing done to it. The filesystem layer already located the +record and read its rate, and the payload that follows the 92-byte header is +signed 16-bit little-endian PCM -- the same thing a WAV data chunk holds. So +this module converts nothing (ADR-0011). + +What it does decode is the record's **eight-pointer block**: a start, an end, a +loop start and a loop end, per channel, as byte offsets from the record's own +start. Those become the WAV's smpl chunk. There is no root key anywhere in the +92 bytes -- see ``pitch`` below and ADR-0025. The endianness is worth a note, because it was got wrong first. Sampling this data at 2048-byte sector boundaries makes it read as big-endian, convincingly @@ -23,19 +28,66 @@ class NotASample(_NotASample): """The payload is not usable as audio.""" +#: A record's audio begins immediately after its 92-byte header, so the pointer +#: set that describes it opens with exactly this. The other set is its mirror, +#: or zeroed on a record declaring a single channel. +DATA_START = 92 + +#: Shorter than this and it is not a loop worth writing. The guard is the one +#: docs/formats/roland-s7xx.md records the need for: without a floor on length +#: a loop metric finds pairs a few frames apart in a fade-out and calls them +#: seamless, because the signal there is silent rather than matching. +MIN_LOOP_FRAMES = 64 + +#: A loop that spans the whole declared extent is the format's "no loop": the +#: sampler fills the pointers with the sample's own bounds when nothing set +#: them. Emitting it would tell a DAW to loop the entire file, which is not +#: what the disc means and is worse than saying nothing. +FULL_EXTENT_SLACK = 16 + +#: The channel pointer sets, in the order they are tried. +_SETS = ( + ("start_l", "end_l", "loop_start_l", "loop_end_l"), + ("start_r", "end_r", "loop_start_r", "loop_end_r"), +) + + +@dataclass(frozen=True) +class SampleLoop: + """One loop, in frames. ``end`` is exclusive here, as it is for AKAI and + Roland; the WAV writer makes it inclusive as the RIFF spec requires.""" + + start: int + end: int + + @dataclass(frozen=True) class Emu3Sample: name: str rate: int frames: int pcm: bytes + #: Always None. The 92-byte record states no root key: no byte in it tracks + #: the note written in the sample's own name on 1 741 named records of + #: `esi32-gm` or 917 of `eiiix-1` -- the best constant-offset match is 8% + #: and 6%, which is chance. The E3 keeps root key in its preset, and + #: presets are not read. Kept as a field, and None, so that ``extract`` + #: treats this format exactly as it treats an AIFF with no INST rather + #: than through a special case (ADR-0025). + pitch: int | None = None + loops: tuple[SampleLoop, ...] = () @property def duration(self) -> float: return self.frames / self.rate if self.rate else 0.0 -def parse(payload: bytes, rate: int, fallback_name: str = "") -> Emu3Sample: +def parse( + payload: bytes, + rate: int, + fallback_name: str = "", + pointers: dict[str, int] | None = None, +) -> Emu3Sample: """Wrap an already-located payload. Raises NotASample if it is unusable.""" if not payload: raise NotASample("no data on disc") @@ -47,4 +99,50 @@ def parse(payload: bytes, rate: int, fallback_name: str = "") -> Emu3Sample: rate=rate, frames=frames, pcm=payload[: frames * 2], + loops=_loops(pointers or {}, frames), ) + + +def _loops(pointers: dict[str, int], frames: int) -> tuple[SampleLoop, ...]: + """The sustain loop, where the record declares one this audio can carry. + + Which channel's pointers to read is decided by the record, not guessed: the + set that describes the audio opens at ``DATA_START``. Both sets do on a + two-channel record and they name the same loop in each half; exactly one + does where a record declares a single channel, and on 542 of `studio`'s + records that is the right-hand set, with the left zeroed. + + **The loop end is gated, not clamped, and that is the one place this format + parts company with AKAI and Roland.** Both of those clamp a declared end + back to the audio actually present, because a rip is often marginally short + of its directory. Here the same move destroys the loop, and `protozoa` + proves it on a single disc: of its mono-shaped records, the 689 whose end + already lies inside the payload correlate at their splice at **+0.86**, and + the 525 whose end lies past it -- clamped back to the last frame -- score + **-0.10**, against a control of -0.01 either way. Same disc, same shape, + separated only by whether the end fits. A clamped end is a loop point the + disc did not state, so it is refused (ADR-0025). + """ + for start_key, end_key, loop_start_key, loop_end_key in _SETS: + start = pointers.get(start_key, 0) + if start != DATA_START: + continue + end = pointers.get(end_key, 0) + loop_start = pointers.get(loop_start_key, 0) + loop_end = pointers.get(loop_end_key, 0) + if not start <= loop_start < loop_end <= end: + continue + if (loop_start - start) % 2 or (loop_end - start) % 2: + continue + # The end must be audio this file actually holds. See the docstring: + # clamping is what the other two formats do and what this one cannot. + if loop_end > start + frames * 2: + continue + a, b = (loop_start - start) // 2, (loop_end - start) // 2 + extent = min((end - start) // 2, frames) + if b - a < MIN_LOOP_FRAMES: + continue + if a == 0 and b >= extent - FULL_EXTENT_SLACK: + continue + return (SampleLoop(start=a, end=b),) + return () diff --git a/src/samplerdisc/wav.py b/src/samplerdisc/wav.py index d618c96..cb38fad 100644 --- a/src/samplerdisc/wav.py +++ b/src/samplerdisc/wav.py @@ -22,6 +22,17 @@ LOOP_FORWARD = 0 LOOP_ALTERNATING = 1 +#: The root key written when the disc carries loop points and no root key. +#: The smpl chunk has no way to say "this sample has no root key" -- the field +#: is mandatory -- so carrying a loop at all means writing something here, and +#: 60 is the neutral value: middle C, no transposition, what a sampler assumes +#: when nothing tells it otherwise. +#: +#: It is a placeholder and not a finding. The E-mu sample record states no root +#: key anywhere in its 92 bytes, and a WAV written this way is saying "here are +#: the loop points" rather than "this sample is middle C" (ADR-0025). +DEFAULT_ROOT_KEY = 60 + @dataclass(frozen=True) class Loop: @@ -196,8 +207,16 @@ def write_wav( body = b"WAVE" + _fmt_chunk(channels, rate, sample_width) if name: body += _info_chunk(name) - if midi_note is not None: - body += _smpl_chunk(rate, midi_note, cents, loops or []) + # A loop is worth carrying even where the disc states no root key, so the + # chunk is written for either. E-mu is the format that needs this: it + # declares loop points in every sample record and a root key in none. + if midi_note is not None or loops: + body += _smpl_chunk( + rate, + midi_note if midi_note is not None else DEFAULT_ROOT_KEY, + cents, + loops or [], + ) body += _chunk(b"data", pcm) with open(path, "wb") as out: diff --git a/tests/fixtures.py b/tests/fixtures.py index 8c9adb3..b9249ed 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -878,6 +878,28 @@ def address(frames: int, fraction: int = 0) -> int: return bytes(image) +def _emu3_pointers(head: bytearray, payload_bytes: int, loop) -> None: + """Write a record's left-hand extent and loop pointers. + + Byte offsets from the record's own start, naming the first byte of a word, + which is what the reference discs hold -- so the end addresses the *last* + word rather than one past it. + """ + from samplerdisc.fs.emu3 import ( + OFF_SAMPLE_END_L, + OFF_SAMPLE_LOOP_END_L, + OFF_SAMPLE_LOOP_START_L, + SAMPLE_HEADER_LEN, + ) + + struct.pack_into(" bytes: """Build a synthetic EMU3 image. @@ -933,6 +956,11 @@ def emu3_disc( ``folder_flags`` overrides the folder entries' flags word. ``studio`` writes 0x0013 and 0x0018 there rather than 0xFFFF, and requiring 0xFFFF loses every folder on that disc. + + ``loops`` maps a sample name to ``(start frame, end frame)`` and writes the + record's left-hand loop pointers accordingly. A record not named here gets + the extent pointers and zeroed loop pointers, which is a record declaring + no loop -- so every fixture written before this existed still describes one. """ from samplerdisc.fs.emu3 import ( BANK_MAGICS, @@ -948,9 +976,9 @@ def emu3_disc( OFF_EIV_LENGTH, OFF_EIV_NAME, OFF_EIV_POSITION, - OFF_SAMPLE_HEADER_LEN, OFF_SAMPLE_RATE, OFF_SAMPLE_RECORD_LEN, + OFF_SAMPLE_START_L, RECORD_LEN_BIAS, SAMPLE_AREA_PREAMBLE, SAMPLE_HEADER_LEN, @@ -1021,8 +1049,9 @@ def name16(text: str) -> bytes: image[record - EIV_RECORD_OFFSET : record - EIV_RECORD_OFFSET + 4] = EIV_MAGIC head = bytearray(SAMPLE_HEADER_LEN) head[2:18] = name16(sample_name) - struct.pack_into(" bytes: record_len = SAMPLE_HEADER_LEN + len(pcm) head = bytearray(SAMPLE_HEADER_LEN) head[2:18] = name16(sample_name) - struct.pack_into(" bytes: record_len = SAMPLE_HEADER_LEN + len(pcm) head = bytearray(SAMPLE_HEADER_LEN) head[2:18] = name16(stale_name) - struct.pack_into(" bytes: record_len = SAMPLE_HEADER_LEN + len(pcm) head = bytearray(SAMPLE_HEADER_LEN) head[2:18] = name16("Older Revision") - struct.pack_into(" set[int]: *_EXPECT_NO_FILESYSTEM.values(), *(size for size, _ in _ROLAND_S7XX.values()), *(size for size, _, _ in _ISO9660.values()), - *(size for size, _, _ in _EMU3.values()), + *(size for size, _, _, _, _ in _EMU3.values()), *(size for size, _, _, _, _, _ in _AKAI.values()), } @@ -527,21 +528,37 @@ def _smpl(payload: bytes) -> tuple[int, list[tuple[int, int]]]: #: Labelled by the short names docs/formats/emu3.md uses, which is what the #: measurements there are recorded against. #: ``label: (size in bytes, volumes, samples)``. +#: ``label: (size in bytes, volumes, samples, samples carrying a loop, the +#: SHA-256 of every sample payload on the disc, concatenated in walk order)``. +#: +#: The digest is the pin that matters most. D17 decoded the sample record's +#: pointer block and taught the E-mu path to write a ``smpl`` chunk, which is +#: **additive by construction**: nothing in it touches ``read_file`` or the +#: offset arithmetic, and the digest is what says so rather than the diff. All +#: seven were computed against the release before D17 and none of them moved. +#: +#: The loop counts are pinned as tightly as the sample counts, for the reason +#: the noted-volume counts are on the AKAI table: a loop appearing where none +#: was measured is a gate that has come loose, and one disappearing is the +#: decode silently failing on a disc nobody looked at. `esi32-gm` yields the +#: fewest by far -- 107 of 2 265 -- because that disc declares a loop end past +#: the audio it carries on almost every record, and those are refused rather +#: than clamped (ADR-0025). _EMU3 = { - "esi32-gm": (93_077_504, 10, 2265), - "protozoa": (131_690_496, 16, 5852), - "eiiix-1": (304_128_000, 46, 1189), - "eiiix-2": (304_435_200, 46, 1333), - "eiv-analogia": (293_912_576, 12, 449), - "eiv-studio": (399_077_376, 230, 2822), - "eiv-vitous": (532_443_136, 44, 828), + "esi32-gm": (93_077_504, 10, 2265, 107, "b7964228d84cfc50"), + "protozoa": (131_690_496, 16, 5852, 1689, "ac4b74a601955ca1"), + "eiiix-1": (304_128_000, 46, 1189, 1157, "c26ee8fb959b3f91"), + "eiiix-2": (304_435_200, 46, 1333, 1260, "2d5d002be060cc52"), + "eiv-analogia": (293_912_576, 12, 449, 449, "5d8faa38572914cb"), + "eiv-studio": (399_077_376, 230, 2822, 2551, "8802808655deea30"), + "eiv-vitous": (532_443_136, 44, 828, 826, "66c179be5b78cbd2"), } @pytest.mark.parametrize("label", sorted(_EMU3)) def test_emu3_discs_list_their_banks_and_samples(label: str) -> None: """Pinned where present, skipped where the shelf is bare -- see _pinned_disc().""" - size, volumes_expected, samples_expected = _EMU3[label] + size, volumes_expected, samples_expected, _, _ = _EMU3[label] with open_image(_pinned_disc(label, size)) as image: origin = find_origin(image) assert origin is not None, f"{label}: no filesystem found" @@ -556,6 +573,44 @@ def test_emu3_discs_list_their_banks_and_samples(label: str) -> None: assert all(v.files or v.note for v in volumes), [v.name for v in volumes if not v.files] +@pytest.mark.parametrize("label", sorted(_EMU3)) +def test_emu3_loops_are_decoded_without_disturbing_the_audio(label: str) -> None: + """The D17 invariant, both halves of it (ADR-0025). + + The payload digest is the whole point: decoding the record's pointer block + must add a ``smpl`` chunk and change nothing else, and a table of sample + counts cannot see a payload that shifted by a byte while staying the same + length. Every loop must also lie inside the audio of its own sample -- + that is the gate that separates a decoded loop from a clamped one. + """ + size, _, samples_expected, loops_expected, digest_expected = _EMU3[label] + with open_image(_pinned_disc(label, size)) as image: + origin = find_origin(image) + assert origin is not None and origin.backend.name == "emu3" + digest = hashlib.sha256() + samples = looped = 0 + for volume in origin.backend.volumes(image, origin.offset): + for entry in volume.samples(): + samples += 1 + payload = origin.backend.read_file(image, origin.offset, entry) + digest.update(payload) + sample = origin.backend.parse_sample(entry, payload) + # No root key is stated anywhere in the 92-byte record, on any + # of the seven discs. Inventing one is what ADR-0025 refuses. + assert sample.pitch is None + for loop in sample.loops: + assert 0 <= loop.start < loop.end <= sample.frames, ( + f"{entry.name}: loop ({loop.start}, {loop.end}) is not " + f"inside {sample.frames} frames" + ) + looped += bool(sample.loops) + assert samples == samples_expected + assert looped == loops_expected + assert digest.hexdigest()[:16] == digest_expected, ( + f"{label}: sample payloads moved -- D17 must be additive" + ) + + def test_protozoa_gives_each_bank_its_own_records() -> None: """The three banks of issue #15, and the two that were invisible. diff --git a/tests/test_emu3.py b/tests/test_emu3.py index 8599468..8cce1fc 100644 --- a/tests/test_emu3.py +++ b/tests/test_emu3.py @@ -8,11 +8,13 @@ from samplerdisc.container.flat import FlatImage from samplerdisc.fs.emu3 import ( BANK_MAGICS, - OFF_SAMPLE_HEADER_LEN, + OFF_SAMPLE_START_L, Emu3Backend, is_plausible_name, ) from samplerdisc.fs.probe import find_origin +from samplerdisc.sample.emu3 import DATA_START, MIN_LOOP_FRAMES +from samplerdisc.sample.emu3 import parse as parse_sample from tests import fixtures BACKEND = Emu3Backend() @@ -194,7 +196,7 @@ def test_an_unexplained_empty_bank_gets_no_note(tmp_path): """ data = bytearray(fixtures.emu3_disc(INDEX_BANK)) record = data.index(b"Arco C1") - 2 # a record begins two bytes before its name - struct.pack_into(" bytes: @@ -82,3 +82,26 @@ def test_odd_sampler_rates_survive(tmp_path, rate): write_wav(path, pcm(50), rate, midi_note=60) with wave.open(str(path)) as w: assert w.getframerate() == rate + + +def test_a_loop_is_carried_even_where_the_disc_states_no_root_key(tmp_path): + """The smpl chunk's root key is mandatory, so carrying a loop means + writing one. 60 is the neutral value, not a claim (ADR-0025).""" + path = tmp_path / "loop.wav" + write_wav( + path, b"\x01\x00" * 1000, rate=22050, midi_note=None, loops=[Loop(start=100, end=899)] + ) + raw = path.read_bytes() + at = raw.find(b"smpl") + assert at != -1 + body = at + 8 + assert struct.unpack_from("