From a8776e5c82da4105743ba5b427614f49ec648078 Mon Sep 17 00:00:00 2001 From: Peter Tos <38345014+bmxcode@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:22:41 +1000 Subject: [PATCH] D24: recover five EIII/ESI banks whose header name is a mistyped copy of the directory's Five named banks across three discs (Elements of Sound 1MB, Ditto Drums, Heavy Guitars) claimed a volume, found no bank header for their directory name and read nothing. Measured: each has a real EMULATOR header at exactly the address its placement predicts, holding audio named for the bank, but the header's own +16 name is the directory name mistyped by a shifted space, a case change or one doubled/dropped character. _bank_offsets gains a recovery pass: a directory entry no header names exactly binds the header at its placement-predicted address when that header's name is within one normalised edit and no other entry claims it (ADR-0031). Nothing is placed; downstream is unchanged. Binding by address alone was rejected because ditto-drums' E3 Main Code / E3X Main Code slots predict addresses that fall on other banks' headers -- the name and unclaimed gates reject them. ditto-drums 948 -> 979 samples; elements1mb and heavy pinned for the first time. No reference disc moves. The separate duplicate-name double-listing found in passing is deferred to #47. Closes #43. Co-Authored-By: Claude Opus 4.8 --- docs/README.md | 2 + ...ear-named-header-its-placement-predicts.md | 59 ++++++++++ docs/adr/README.md | 1 + docs/formats/emu3.md | 25 ++++- src/samplerdisc/fs/emu3.py | 74 +++++++++++++ tests/fixtures.py | 12 +- tests/test_discs.py | 91 ++++++++++++++- tests/test_emu3.py | 104 ++++++++++++++++++ 8 files changed, 361 insertions(+), 7 deletions(-) create mode 100644 docs/adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md diff --git a/docs/README.md b/docs/README.md index a753f45..db291dc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -62,6 +62,7 @@ All of that vanishes into a working parser. Six months on, the code says *what* | D20 | The partitions eight short AKAI images displaced, found by an anchored search ([ADR-0028](adr/0028-a-displaced-partition-is-anchored-quantised-and-floored.md)) | done | | D21 | An E-mu record is closed by the channel it declares, and found by either one ([ADR-0029](adr/0029-a-record-is-closed-by-the-channel-it-declares.md)) | done | | D23 | The E-mu whole-extent "no loop" is refused at both ends, not only at frame 0 ([ADR-0030](adr/0030-the-whole-extent-no-loop-is-refused-at-both-ends.md)) | done | +| D24 | An EIII/ESI bank binds the near-named header its placement predicts, recovering five banks a mistyped header name hid ([ADR-0031](adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md)) | done | Across the local collection, by listing: 74 of 82 images claimed, 3 096 volumes, 132 802 files, 98 061 of them samples. The AKAI discs are 44 of those images and 86 177 of those files, read across 314 partitions — 275 where the disk's table puts them and 39 displaced by blocks the rip lost. Before D15 they were 14 670 files, because only the partition at the origin was read. @@ -77,6 +78,7 @@ Across the local collection, by listing: 74 of 82 images claimed, 3 096 volumes, - **Ensoniq and Kurzweil backends.** The archives are full of these discs and the containers already open them; each needs a module in `fs/` and nothing else ([ADR-0003](adr/0003-brand-neutral-pluggable-backends.md)). - **No root key for E-mu.** It is not in the sample record: no byte of the 92 tracks the note in the sample's own name above chance, over 1 741 named records of `esi32-gm` and 917 of `eiiix-1`. The E3 keeps it in the preset, and presets are not read — see the `E4P1` entry below. E-mu WAVs carry their loop with the RIFF neutral root key of 60 ([ADR-0025](adr/0025-the-loop-is-decoded-the-root-key-is-not.md)). - **`E4P1` presets are not read.** The three E-IV discs carry 916, 901 and 284 of them. On `eiv-studio` 100 of 230 banks have no sample directory and are listed with a note; preset-only banks are the likely explanation, and it is not established. +- **A duplicate directory name double-lists on EIII/ESI.** `located` is keyed by bank name, so where a directory lists one name twice both entries resolve to a single header and report the same records — `Harpsichord X` on `Vol. 10 – Elements of Sound 1MB`, `HvyGtr Maj.Open` on `Vol. 17 – Heavy Guitars`. One arm of the `heavy` pair points at a real but **blank-named** 6.3 MB header, which is a second mystery. It is a different mechanism from D24's mistyped-header recovery — it reads the same audio twice rather than nothing — and needs per-entry header resolution; the D24 pins for those two discs include the double-count until it is fixed ([#47](https://github.com/bmxcode/samplerdisc/issues/47)). - **The `.mds` track table is unread.** One real pair now reads end to end, but geometry is sniffed from the `.mdf` rather than taken from the descriptor, so a multi-track or offset image would be read from byte 0. What the one specimen's descriptor holds is written down in [formats/mdx.md](formats/mdx.md) without being relied on. - **CUES chunks in NRG.** Only `CUEX` is parsed; `CUES` encodes position as MSF and no disc using it was available to check the layout against. - **EXS24 and HALion instruments are kept, not read.** `--keep-originals` writes the `.exs` and `.fxp` files out byte for byte, because they hold the key ranges and envelopes a WAV cannot. Nothing parses them, and nothing should: turning them into a playable instrument is [ConvertWithMoss](https://github.com/git-moss/ConvertWithMoss)'s job ([ADR-0011](adr/0011-the-deliverable-is-daw-ready-wav.md)). diff --git a/docs/adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md b/docs/adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md new file mode 100644 index 0000000..57c2a52 --- /dev/null +++ b/docs/adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md @@ -0,0 +1,59 @@ +# ADR-0031 · A bank binds the near-named header its placement predicts + +**Status:** accepted · 2026-08-24 + +## Context + +[ADR-0015](0015-locate-banks-by-signature.md) settled that an EIII/ESI bank is *found* by its own `EMULATOR` header, which repeats the directory's bank name, and [ADR-0021](0021-a-bank-owns-the-run-its-header-declares.md) settled how a name written twice is disambiguated: fit `header address == unit × start + bias` from the headers already located by signature, and use it *only* to say which of two same-named headers a directory entry meant. `_bank_offsets` implements both by keying `located` on the exact bank name. + +[Issue #43](https://github.com/bmxcode/samplerdisc/issues/43) is the case that keying on the exact name cannot reach. Five named banks across three discs claimed a volume, found no header for their directory name and read nothing — `Electric Grand X` on `Vol. 10 – Elements of Sound 1MB`, `PERCUSSION#1 X` on `Vol. 16 – Ditto Drums`, and `HvyGtr FX5 X`, `Misc Gtr FX 2MbX`, `HvGtrFdBkTxtr2Mb` on `Vol. 17 – Heavy Guitars`. Each carried the ADR-0012 note `no bank header found for this bank; listed only`, so the invariant held — but unlike the 28 `E3 Main Code`/`E3X Main Code` operating-system slots that share it, these are named like ordinary sample banks. + +Measured against the discs, every one has a real header sitting at *exactly* the address ADR-0021's placement fit predicts, carrying real audio named after the bank — but its own 16-byte name field at `+16` is a corrupted copy of the directory name: + +| Disc | Directory name | Header `+16` name | Records | First record | +|---|---|---|---:|---| +| `elements1mb` | `Electric Grand X` | `Eelectric GrandX` | 9 | `ELEC GRAND _000` | +| `ditto-drums` | `PERCUSSION#1 X` | `PERCUSSION #1 X` | 31 | `TAMB BRASS` | +| `heavy` | `HvyGtr FX5 X` | `HvyGtr FX5 XX` | 2 | `Gtr FX 11` | +| `heavy` | `Misc Gtr FX 2MbX` | `Misc Gtr FX 2mbX` | 6 | `Gtr Feedback Shr` | +| `heavy` | `HvGtrFdBkTxtr2Mb` | `HvGtrFdBkTxtr2M ` | 1 | `Gtr FeedbackLoop` | + +The corruptions are a shifted space, a case change, or a single doubled or dropped character — an edit of at most one once the name is lowercased and its spaces stripped. `_bank_headers` finds these headers (their `+16` names are plausible), but `_bank_offsets` never binds them to a directory entry, because the directory says `Electric Grand X` and the header says `Eelectric GrandX`. + +They are not the operating-system slots and not index banks. An OS slot has no header at all where it points; an index bank declares a zero-length run. These declare a run, hold audio, and name it for the bank. + +## Decision + +**A directory entry that no header names exactly binds the header sitting at the address its placement predicts, when that header carries a near-copy of the entry's name and no other entry already claims it.** + +Three gates, each measured: + +**The header must sit exactly at the predicted address.** `want = unit × start + bias`, the same fit ADR-0021 already computes, requiring the same three agreeing single-header banks before it says anything. A disc that has shown no placement rule recovers nothing and keeps the note. Nothing is *placed*: a bank whose predicted address holds no header binds nothing. + +**The header's name must be a near-copy.** Normalise both names — lowercase, strip spaces — and require a Levenshtein distance of at most one. Every one of the five is a distance of zero or one; the OS-slot collisions below are a dozen. The name is what confirms the hit, exactly as in ADR-0015 — the placement only says which header, and the name says it is the right one. + +**The header must be unclaimed.** A bank may never bind a header that a name-matched entry already owns. This is the gate that decides `ditto-drums`: `E3 Main Code`'s predicted address lands on the `Ditto Drums X` index bank's header and `E3X Main Code`'s lands on `DAVE W KIT1 X`'s — both real banks the directory names elsewhere. The name gate rejects them (`e3maincode` is nothing like `dittodrumsx`), and the unclaimed gate is the belt to that suspenders. + +This is [ADR-0021](0021-a-bank-owns-the-run-its-header-declares.md)'s instrument used one step wider: from *which of two headers wearing this exact name* to *the near-named header this entry points at*. Downstream nothing changes — a recovered entry flows through the same `_declared_run` and record walk, bounded by the same next-header cap, and a recovered bank with records gets no note while one that recovers nothing keeps it, so [ADR-0012](0012-a-probe-must-confirm-a-file.md) still holds. + +## Alternatives rejected + +**Bind by the predicted address alone, without a name gate.** The smallest change, and it recovers all five. Rejected on `ditto-drums`, measured: `E3 Main Code` and `E3X Main Code` are real slots on every EIII disc, and their predicted addresses fall on the `Ditto Drums X` and `DAVE W KIT1 X` headers. Address alone would hand each OS slot another bank's audio under the wrong name — the exact failure ADR-0015 exists to prevent, one layer in. The unclaimed gate catches these two, but a future disc whose OS slot points at an *orphan* stale header would slip through it; the name gate is what makes the rule safe rather than lucky. + +**Widen the exact-name match to a fuzzy match everywhere in `_bank_offsets`.** Tempting, because it would need no placement. Rejected: it reopens the whole of ADR-0015. A fuzzy match between a directory name and *any* header on the disc would, on a disc full of `Gtr FX 11`-style near-identical names, bind a bank to a neighbour that is one edit away and holds different audio. The placement is what makes the near-name safe — it is only ever consulted at the one address the disc itself points the entry at. + +**Leave them listed with the note.** The conservative reading, and the state issue #43 filed. Rejected on the evidence: the header is there, at the predicted address, carrying `ELEC GRAND` and `TAMB BRASS` and `Gtr Feedback`. Listing them empty withholds 49 samples the disc plainly offers, on the strength of a one-character typo the mastering left in a name field. + +**Correct the header name to the directory's and match.** Would also work. Rejected as the wrong shape: it invents an authority the disc does not grant. The directory name and the header name disagree, and this project does not get to decide the header's is wrong — only that the two are the same bank. Binding by address with a near-name gate says exactly that and no more. + +## Consequences + +**Good.** Five banks that read nothing now read their records: `elements1mb` gains 9, `ditto-drums` 31, `heavy` 9 — 49 samples across three discs, each named for its bank. + +**Good.** No reference disc moves. Simulated across all ten EMU3 reference discs and the three size-twins in the collection, the recovery binds nothing: every bank on those discs either matches a header exactly or has no header at its predicted address. The change fires only where a name was mistyped. + +**Good.** The OS-code slots stay noted. `E3 Main Code` and `E3X Main Code` are refused by the name gate and the unclaimed gate together, so the note still names exactly the slots that hold an operating system and no audio. + +**Watch for.** A disc that mistypes a header name by *more* than one normalised edit. It will list empty with the note, which is the honest floor — visible, and recoverable if a later disc shows the corruption is systematic enough to widen the threshold against. + +**Deliberately not claimed.** Two discs here list a bank name **twice** in the directory — `Harpsichord X` on `elements1mb`, `HvyGtr Maj.Open` on `heavy` — and because `located` is keyed by name both entries resolve to one header and double-list the same records. That is a different mechanism from the five (it reads the same audio twice, not nothing), one arm of it points at a real but blank-named header, and it is left to [#47](https://github.com/bmxcode/samplerdisc/issues/47). This record recovers a header the directory names once and mistypes; it does not touch a name the directory writes twice. diff --git a/docs/adr/README.md b/docs/adr/README.md index 14b944f..b060198 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -36,3 +36,4 @@ If you find yourself writing an ADR with no rejected alternative, you are writin | [0028](0028-a-displaced-partition-is-anchored-quantised-and-floored.md) | A displaced AKAI partition is searched for from its declared position, in the container's unit, and never inside one already read | Leaving them unread; scanning for the signature; filtering a scan by file yield; searching in AKAI blocks; truncating an overlapping recovery; moving the present partition instead; refusing a whole disc for one clash; relaxing the floor for `Alpha Dance I`; searching forward; per-file provenance; fixing #35 here | | [0029](0029-a-record-is-closed-by-the-channel-it-declares.md) | An E-mu record is closed by the channel it declares, and found by either one | Falling back to `end_L` only where `end_R` fails; `max(end_L, end_R)` unconditionally; widening the signature alone; bounding the extent by the next record found; fixing the whole-extent loop guard here | | [0030](0030-the-whole-extent-no-loop-is-refused-at-both-ends.md) | The whole-extent "no loop" is refused within the slack of both ends, not only frame 0 | Leaving the guard at `a == 0`; a content guard on end silence; matching each disc's fixed inset; widening the end slack instead | +| [0031](0031-a-bank-binds-the-near-named-header-its-placement-predicts.md) | A bank whose name no header matches binds the near-named header its placement predicts, when it is unclaimed | Binding by address alone; fuzzy-matching names everywhere; leaving them noted; rewriting the header name; fixing the twice-written-name double-listing here | diff --git a/docs/formats/emu3.md b/docs/formats/emu3.md index 38bf60f..d741eed 100644 --- a/docs/formats/emu3.md +++ b/docs/formats/emu3.md @@ -149,6 +149,22 @@ What decides is where the directory put the bank. `header address == unit × sta This does **not** reopen [ADR-0015](../adr/0015-locate-banks-by-signature.md). Nothing is placed by the arithmetic; it only says which of two headers already carrying the right name a directory entry meant, and only names with a single header are allowed to vote for the fit. It is the same instrument, used the same way, as the E-IV allocation-unit fit below. +### A header's name can be a corrupted copy of the directory's + +A bank's header repeats the directory's name at `+16`, and matching on it is what confirms a hit. On a handful of banks the mastering **mistyped that copy**, so the header no longer matches the directory verbatim and keying the lookup on exact-name equality lists the bank empty — with the `no bank header found for this bank; listed only` note — while its audio sits right there behind a header the signature scan did find. Five such banks across three discs: + +| Disc | Directory name | Header `+16` name | Records | First record | +|---|---|---|---:|---| +| `elements1mb` | `Electric Grand X` | `Eelectric GrandX` | 9 | `ELEC GRAND _000` | +| `ditto-drums` | `PERCUSSION#1 X` | `PERCUSSION #1 X` | 31 | `TAMB BRASS` | +| `heavy` | `HvyGtr FX5 X` | `HvyGtr FX5 XX` | 2 | `Gtr FX 11` | +| `heavy` | `Misc Gtr FX 2MbX` | `Misc Gtr FX 2mbX` | 6 | `Gtr Feedback Shr` | +| `heavy` | `HvGtrFdBkTxtr2Mb` | `HvGtrFdBkTxtr2M ` | 1 | `Gtr FeedbackLoop` | + +Every corruption is a shifted space, a case change, or one doubled or dropped character — an edit of at most one once the name is lowercased and its spaces stripped. The header sits at exactly the address `unit × start + bias` predicts, holds a declared run, and names its records for the bank, so all three of "where the directory points", "what the header is called" and "what the audio is" agree that it is the same bank. + +So a directory entry that no header names exactly binds the header at its predicted address, when that header's name is within one normalised edit of the entry's and no other entry already owns it ([ADR-0031](../adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md)). The near-name gate is load-bearing, not cosmetic: `ditto-drums`'s `E3 Main Code` and `E3X Main Code` slots — real operating-system banks with no audio — predict addresses that fall on the `Ditto Drums X` and `DAVE W KIT1 X` headers, a dozen edits from their own names, and binding by address alone would hand each OS slot another bank's records. `elements1mb` and `heavy` are pinned by this recovery in `tests/test_discs.py`; it binds nothing on any of the ten reference discs. + An earlier revision of this doc gave `0x38` as the bank size and claimed `0x30 + 0x34 == 0x38`. **That sum holds on 0 of the 114 located banks** across the four EIII/ESI discs, and nothing checked it. `0x38` reads a constant 8 388 608 on every `EMULATOR 3X` and `EMU SI-32` bank — including a 256 KiB one — which is the sampler's memory size and not a property of the bank; on `EMULATOR THREE` banks it reads 0 on 60 of 90 and a small number unrelated to the bank's extent on the rest. Do not use it for anything. ### A bank's region holds more than the bank @@ -589,12 +605,12 @@ Whole-disc listings, with the samples the record declares stereo: | `eiiix-2` | 46 | 1 337 | 592 | | `emu-classics` | 22 | 1 516 | 185 | | `vintage` | 16 | 993 | 2 | -| `ditto-drums` | 48 | 948 | 0 | +| `ditto-drums` | 48 | 979 | 0 | | `eiv-analogia` | 12 | 449 | 279 | | `eiv-studio` | 230 | 2 822 | 320 | | `eiv-vitous` | 44 | 828 | 828 | -**2 843 of 19 371**, and a stereo sample is still one sample: the counts above do not move when the channel count is read, only the file's shape does. +**2 843 of 19 402**, and a stereo sample is still one sample: the counts above do not move when the channel count is read, only the file's shape does. `ditto-drums` gained 31 with D24 — its `PERCUSSION#1 X` recovered from a mistyped header name, from 948 — and two discs not in this reference table were pinned for the first time by the same recovery: `elements1mb` (102 volumes, 1 465 samples, `Electric Grand X`'s 9 recovered) and `heavy` (68 volumes, 870 samples, three banks recovered) ([ADR-0031](../adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md)). Samples carrying a loop, of those totals: @@ -606,12 +622,12 @@ Samples carrying a loop, of those totals: | `eiiix-2` | 1 337 | 892 | 67% | | `emu-classics` | 1 516 | 1 133 | 75% | | `vintage` | 993 | 846 | 85% | -| `ditto-drums` | 948 | 14 | 1% | +| `ditto-drums` | 979 | 14 | 1% | | `eiv-analogia` | 449 | 6 | 1% | | `eiv-studio` | 2 822 | 2 214 | 78% | | `eiv-vitous` | 828 | 198 | 24% | -**10 878 of 19 371.** These are D23's numbers: the whole-extent "no loop" is now refused at both ends rather than only where it starts at frame 0, so the loop-over-the-whole-file that every disc writes with a small fixed inset stops being emitted — `ditto-drums` from 948 to 14, `eiv-analogia` from 449 to 6, `eiv-vitous` from 826 to 198 ([ADR-0030](../adr/0030-the-whole-extent-no-loop-is-refused-at-both-ends.md), and "The whole-extent 'no loop'" above). The D21 revision's figures were `esi32-gm` 1 778, `protozoa` 5 244, `eiiix-1` 1 215, `eiiix-2` 1 264, `emu-classics` 1 435, `vintage` 953, `ditto-drums` 948, `eiv-analogia` 449, `eiv-studio` 2 551, `eiv-vitous` 826 — every one of them counting the no-loops. The **sample** counts, the stereo counts and every payload digest are unchanged across D23: only loop emission moved, which is what says the read path was not touched. +**10 878 of 19 402** — `ditto-drums`'s 31 newly recovered records carry no loop (percussion one-shots), so the loop total is unchanged and only the sample total moved. These are D23's numbers: the whole-extent "no loop" is now refused at both ends rather than only where it starts at frame 0, so the loop-over-the-whole-file that every disc writes with a small fixed inset stops being emitted — `ditto-drums` from 948 to 14, `eiv-analogia` from 449 to 6, `eiv-vitous` from 826 to 198 ([ADR-0030](../adr/0030-the-whole-extent-no-loop-is-refused-at-both-ends.md), and "The whole-extent 'no loop'" above). The D21 revision's figures were `esi32-gm` 1 778, `protozoa` 5 244, `eiiix-1` 1 215, `eiiix-2` 1 264, `emu-classics` 1 435, `vintage` 953, `ditto-drums` 948, `eiv-analogia` 449, `eiv-studio` 2 551, `eiv-vitous` 826 — every one of them counting the no-loops. The **sample** counts, the stereo counts and every payload digest are unchanged across D23: only loop emission moved, which is what says the read path was not touched. These are the regression baseline: any change to the shared record parser is a bug if they move. **They are now asserted by `tests/test_discs.py`, pinned by disc size** — a table in a document is a note, not a test, and two of these numbers were wrong for a release with a green suite. The suite also pins the **SHA-256 of every sample payload per disc**, because a count table cannot see a payload that shifted by a byte while staying the same length, and the **stereo counts** beside them, because the third condition of the gate is exactly the kind of thing a later simplification removes. On a stereo sample the suite additionally de-interleaves what was written and requires it to reproduce the disc's two blocks byte for byte: the audio moved, and it must be the same audio. @@ -630,6 +646,7 @@ Each of the seven EIII/ESI discs lists one index bank with a note and no samples - Sample records are found, not chained; the chain has gaps. - The bank signature is not always `EMULATOR`. `protozoa` writes `EMU SI-32 v3` on two banks, and a bank nobody locates hands its region to the bank in front of it. - One bank name can have two headers. `esi32-gm`'s duplicates sit *below* the directory's copy and `protozoa`'s sits *above* it, so neither "first" nor "last" is a rule. +- A header's `+16` name can be a mistyped copy of the directory's — `Electric Grand X` written `Eelectric GrandX`. Bind it by the address the placement predicts, gated on the name being within one normalised edit and unclaimed; never by address alone, or `ditto-drums`'s `E3 Main Code` binds the header its arithmetic lands on ([ADR-0031](../adr/0031-a-bank-binds-the-near-named-header-its-placement-predicts.md)). - A bank's region holds more than the bank. Everything past `0x30 + 74 + 0x34` is the previous occupant's, and it is inside the region, so no bound between banks excludes it. - The payload is little-endian. A sector-aligned measurement says otherwise and is wrong. - The E-IV **sample directory** is big-endian, alone in the format. The trap runs both ways. diff --git a/src/samplerdisc/fs/emu3.py b/src/samplerdisc/fs/emu3.py index c86819b..8fdaad2 100644 --- a/src/samplerdisc/fs/emu3.py +++ b/src/samplerdisc/fs/emu3.py @@ -212,6 +212,49 @@ def decode_name(raw: bytes) -> str: return raw.decode("ascii", "replace").rstrip("\x00 ").strip() +def _levenshtein(a: str, b: str) -> int: + """Edit distance, iterative single-row -- small strings, called rarely.""" + if a == b: + return 0 + previous = list(range(len(b) + 1)) + for i, ca in enumerate(a, 1): + current = [i] + for j, cb in enumerate(b, 1): + current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (ca != cb))) + previous = current + return previous[-1] + + +def _near_name(directory: str, header: str) -> bool: + """Whether a bank header's own name is a corrupted copy of the directory's. + + A handful of banks carry a header whose 16-byte name field at + ``OFF_BANK_NAME`` is the directory's name mis-typed by the mastering: a + doubled or dropped character and a shifted space, never more. Normalising + away case and spaces and allowing a single further edit matches every one + measured and clears the things that are genuinely a different bank by a wide + margin (see docs/formats/emu3.md, "A header's name can be a corrupted copy + of the directory's", and ADR-0031): + + * ``Electric Grand X`` / ``Eelectric GrandX`` -- one insert + * ``PERCUSSION#1 X`` / ``PERCUSSION #1 X`` -- equal once spaces go + * ``HvyGtr FX5 X`` / ``HvyGtr FX5 XX`` -- one insert + * ``Misc Gtr FX 2MbX`` / ``Misc Gtr FX 2mbX`` -- equal once case folds + * ``HvGtrFdBkTxtr2Mb`` / ``HvGtrFdBkTxtr2M`` -- one delete + + against the operating-system slots, whose header addresses happen to fall on + another bank's header entirely: ``E3 Main Code`` / ``Ditto Drums X`` and + ``E3X Main Code`` / ``DAVE W KIT1 X`` are both a dozen edits apart. The + gate is what keeps the placement arithmetic from binding an OS slot to the + audio it points at by accident. + """ + + def norm(name: str) -> str: + return name.lower().replace(" ", "") + + return _levenshtein(norm(directory), norm(header)) <= 1 + + def is_plausible_name(raw: bytes) -> bool: """Printable text, padded to width with spaces *or* NULs. @@ -678,6 +721,18 @@ def _bank_offsets(self, banks: list[_Bank], headers: list[tuple[int, str]]) -> d directory does not allocate, both *before* the banks the directory points at; `protozoa` carries a second ``Phatt Presets X`` after them. Whichever end you take, one of those discs is read wrong. + + A bank whose name matches **no** header exactly is then given one last + chance: the header sitting at the address its placement predicts, when + that header carries a near-copy of the bank's name (ADR-0031). Five + banks across three discs -- ``Electric Grand X``, ``PERCUSSION#1 X``, + ``HvyGtr FX5 X``, ``Misc Gtr FX 2MbX``, ``HvGtrFdBkTxtr2Mb`` -- have + a real ``EMULATOR`` header the mastering mis-typed the name on, and were + listed empty with a note where their audio was plainly there. This never + *places* a bank: the header is one the signature scan already found, and + the name at ``+16`` is what confirms it -- the same instrument the + placement fit already uses to arbitrate a name written twice, one step + wider. """ at_name: dict[str, list[int]] = {} for at, name in headers: @@ -691,6 +746,25 @@ def _bank_offsets(self, banks: list[_Bank], headers: list[tuple[int, str]]) -> d want = unit * bank.start + bias if want in at_name.get(bank.name, ()): found[bank.name] = want + # Recovery. ``taken`` is the addresses that name-matched entries own, so + # a bank can never be bound to a header another bank already claims -- + # which is what keeps ``ditto-drums``'s ``E3 Main Code`` off the + # ``Ditto Drums X`` header its arithmetic lands on. ``by_address`` + # names the header actually sitting at ``want``; a bank whose predicted + # address holds no header, or one whose name is not a near-copy of it, + # keeps its note. + by_address = {at: name for at, name in headers} + taken = {found[bank.name] for bank in banks if bank.name in at_name} + for bank in banks: + if bank.name in at_name: + continue + want = unit * bank.start + bias + header_name = by_address.get(want) + if header_name is None or want in taken: + continue + if _near_name(bank.name, header_name): + found[bank.name] = want + taken.add(want) return found def _declared_run( diff --git a/tests/fixtures.py b/tests/fixtures.py index fb69c1a..2dee646 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -953,6 +953,7 @@ def emu3_disc( formula_4000: tuple[str, ...] = (), stale_tail: tuple[tuple[str, int, int], ...] = (), second_header: str | None = None, + header_names: dict[str, str] | None = None, eiv: bool = False, duplicate_sample_dir: bool = False, folder_flags: int | None = None, @@ -985,6 +986,10 @@ def emu3_disc( directory points at, so taking the first header of a name reads the wrong one. + ``header_names`` maps a bank name to the name its header writes at +16, + when that differs from the directory entry: the D24 case, where the + mastering mistyped the header's own copy so only its address places it. + A bank given no samples gets a header declaring a zero-length sample area, which is what the index banks on `esi32-gm`, `eiiix-1` and `eiiix-2` do. @@ -1111,7 +1116,12 @@ def name16(text: str) -> bytes: if bank_header: magic = BANK_MAGICS[1] if bank_name in formula_4000 else BANK_MAGICS[0] image[at : at + len(magic)] = magic - image[at + 16 : at + 32] = name16(bank_name) + # ``header_names`` writes a *different* name at +16 from the + # directory entry, which is the D24 case: the mastering mistyped + # the header's own copy of the name, so it no longer matches the + # directory verbatim and only its address gives it away. + written = (header_names or {}).get(bank_name, bank_name) + image[at + 16 : at + 32] = name16(written) # The declared sample area starts at 0x30 and its first record # sits SAMPLE_AREA_PREAMBLE bytes into it, which is what every # populated bank on the reference discs does. diff --git a/tests/test_discs.py b/tests/test_discs.py index 24353e8..9617c7b 100644 --- a/tests/test_discs.py +++ b/tests/test_discs.py @@ -132,6 +132,10 @@ def _pinned_sizes() -> set[int]: _HEAD_DIGEST = { "emu-classics": "3882c2319cc27871", "eiv-studio": "f1f1c805136d4881", + # Vol. 10 - Elements of Sound 1MB shares its size with Vol. 11 (the 2MB + # cut of the same library), so it needs the first-megabyte digest to be + # told apart -- their bank directories differ. + "elements1mb": "250858faa4d17ceb", } @@ -614,10 +618,22 @@ def _smpl(payload: bytes) -> tuple[int, list[tuple[int, int]]]: "eiiix-2": (304_435_200, 46, 1337, 892, 592, "8bfcddf6d8dbd806"), "emu-classics": (526_723_072, 22, 1516, 1133, 185, "e1398c11a9e7cb02"), "vintage": (527_030_272, 16, 993, 846, 2, "fbcc378dd173f96c"), - "ditto-drums": (308_121_600, 48, 948, 14, 0, "3c756586711810de"), + "ditto-drums": (308_121_600, 48, 979, 14, 0, "47b17bcd6028ec29"), "eiv-analogia": (293_912_576, 12, 449, 6, 279, "5d8faa38572914cb"), "eiv-studio": (399_077_376, 230, 2822, 2214, 320, "8802808655deea30"), "eiv-vitous": (532_443_136, 44, 828, 198, 828, "66c179be5b78cbd2"), + # D24 recovers a bank whose header carries a mis-typed copy of its + # directory name (ADR-0031). ``ditto-drums`` gains ``PERCUSSION#1 X``'s + # 31 records above; these two discs are pinned for the first time by it. + # + # Their sample totals still include a *separate*, pre-existing + # duplicate-directory-name double-listing that D24 does not touch: + # ``elements1mb`` lists ``Harpsichord X`` twice and ``heavy`` lists + # ``HvyGtr Maj.Open`` twice, each pair reading the same records because + # ``located`` is keyed by name. When that is fixed these counts move; it is + # filed separately (see the PR). + "elements1mb": (296_042_496, 102, 1465, 1181, 0, "299901d661807f2d"), + "heavy": (524_599_296, 68, 870, 680, 745, "2c22387cd8be8344"), } @@ -816,6 +832,72 @@ def test_emu3_banks_that_declared_a_sample_area_and_yielded_nothing(label: str) assert end_r == SAMPLE_HEADER_LEN + 2 * frame - 2 +#: The banks of [issue #43](https://github.com/bmxcode/samplerdisc/issues/43), +#: with the records each yields and the mis-typed header name that hid it. +#: Five named EIII/ESI banks across three discs claimed a volume, found no +#: header for their directory name and read nothing, and unlike the OS-code +#: slots that share that note they are ordinary sample banks (ADR-0031). +#: +#: They are pinned by the **mechanism** and not only by the count. Each carries +#: a real ``EMULATOR`` header at the address its placement predicts, whose own +#: name at ``+16`` is the directory name corrupted by a shifted space, a case +#: change or a single doubled/dropped character -- which is why keying +#: ``located`` on exact-name equality missed it. The test asserts the bound +#: header's name is that near-copy and is *not* the directory name, so the day +#: this stops holding it says whether the bank went empty, was located wrong, +#: or matched by exact name after all. +_EMU3_RECOVERED_BANKS = { + "elements1mb": (("Electric Grand X", 9, "Eelectric GrandX"),), + "ditto-drums": (("PERCUSSION#1 X", 31, "PERCUSSION #1 X"),), + "heavy": ( + ("HvyGtr FX5 X", 2, "HvyGtr FX5 XX"), + ("Misc Gtr FX 2MbX", 6, "Misc Gtr FX 2mbX"), + ("HvGtrFdBkTxtr2Mb", 1, "HvGtrFdBkTxtr2M"), + ), +} + + +@pytest.mark.parametrize("label", sorted(_EMU3_RECOVERED_BANKS)) +def test_emu3_banks_recovered_from_a_mistyped_header_name(label: str) -> None: + """Issue #43, pinned by the mis-typed header rather than by the totals. + + A regression here is a bank falling back to its note, or -- worse and + quieter -- binding a *different* header. Asserting the bound header's own + name is the corrupted copy, distinct from the directory name, is what tells + a genuine recovery apart from an exact-name match that would mean the + corruption was never really there. + """ + from samplerdisc.fs.emu3 import ( + BANK_NAME_LEN, + OFF_BANK_NAME, + _near_name, + decode_name, + ) + + size = _EMU3[label][0] + with open_image(_pinned_disc(label, size)) as image: + origin = find_origin(image) + assert origin is not None and origin.backend.name == "emu3" + backend = origin.backend + offset = origin.offset + banks = backend._banks(image, offset) + headers = backend._bank_headers(image, offset) + placement = backend._placement(banks, headers) + assert placement is not None, f"{label}: no placement fit to recover through" + unit, bias = placement + volumes = {v.name: v for v in backend.volumes(image, offset)} + by_start = {bank.name: bank.start for bank in banks} + for name, expected, header_name in _EMU3_RECOVERED_BANKS[label]: + volume = volumes[name] + assert len(volume.files) == expected, f"{label}/{name}: {len(volume.files)} samples" + assert not volume.note, f"{label}/{name}: recovered but still noted" + want = unit * by_start[name] + bias + raw = image.read(offset + want + OFF_BANK_NAME, BANK_NAME_LEN) + assert decode_name(raw) == header_name, f"{label}/{name}: header name {raw!r}" + assert header_name != name, "a recovered header's name is a corrupted copy, not exact" + assert _near_name(name, header_name) + + #: Banks whose last record must end exactly at ``0x30 + 74 + 0x34``, out of the #: banks that yield records at all. This is the independent half of D21's #: evidence: the bank header's declared run is a different field, written by a @@ -830,7 +912,12 @@ def test_emu3_banks_that_declared_a_sample_area_and_yielded_nothing(label: str) "eiiix-2": (44, 39), "emu-classics": (19, 16), "vintage": (13, 13), - "ditto-drums": (44, 44), + "ditto-drums": (45, 45), + # Newly pinned by D24. ``ditto-drums`` gains its recovered ``PERCUSSION#1``; + # ``heavy``'s two non-exact banks are the ordinary payload-overshoot D21 + # already documents, not the recovery. + "elements1mb": (100, 100), + "heavy": (65, 63), } diff --git a/tests/test_emu3.py b/tests/test_emu3.py index 13ed71b..9315e97 100644 --- a/tests/test_emu3.py +++ b/tests/test_emu3.py @@ -424,6 +424,110 @@ def test_a_bank_with_neither_header_nor_directory_is_listed_with_a_note(tmp_path assert all(v.note for v in volumes) +# --- a mistyped header name (ADR-0031) ---------------------------------- + + +def test_near_name_matches_the_mistyped_headers_and_nothing_else(): + """The five real corruptions match; the two OS-slot collisions do not. + + Every recovery on the real discs is a shifted space, a case change or one + doubled/dropped character -- normalised away, an edit of at most one. The + ``E3 Main Code``/``E3X Main Code`` slots whose arithmetic lands on another + bank's header are a dozen edits off, which is the margin the gate lives in. + """ + from samplerdisc.fs.emu3 import _near_name + + for directory, header in ( + ("Electric Grand X", "Eelectric GrandX"), + ("PERCUSSION#1 X", "PERCUSSION #1 X"), + ("HvyGtr FX5 X", "HvyGtr FX5 XX"), + ("Misc Gtr FX 2MbX", "Misc Gtr FX 2mbX"), + ("HvGtrFdBkTxtr2Mb", "HvGtrFdBkTxtr2M"), + ): + assert _near_name(directory, header), (directory, header) + assert not _near_name("E3 Main Code", "Ditto Drums X") + assert not _near_name("E3X Main Code", "DAVE W KIT1 X") + + +def test_bank_offsets_recovers_a_mistyped_header_only_when_unclaimed(): + """The recovery binds a near-named header the placement predicts, and only + then (ADR-0031). + + Three exactly-named banks pin the fit at ``unit == 100``. ``Delta``'s + header is mistyped ``Deltaa`` at the address it predicts and is recovered; + ``Echo``'s predicted address holds an unrelated name and is not; and + ``Alphaa`` -- near ``Alpha`` but predicting an address ``Alpha`` already + owns -- is refused, because a bank may never take a header another entry + already claims. + """ + from samplerdisc.fs.emu3 import _Bank + + backend = Emu3Backend() + headers = [ + (100, "Alpha"), + (200, "Beta"), + (300, "Gamma"), + (400, "Deltaa"), + (500, "Zulu Foxtrot"), + ] + banks = [ + _Bank("Alpha", "", 1, 1), + _Bank("Beta", "", 2, 1), + _Bank("Gamma", "", 3, 1), + _Bank("Delta", "", 4, 1), + _Bank("Echo", "", 5, 1), + _Bank("Alphaa", "", 1, 1), + ] + found = backend._bank_offsets(banks, headers) + assert found["Delta"] == 400 # mistyped header, recovered by its address + assert "Echo" not in found # predicted address holds a far name + assert "Alphaa" not in found # would steal Alpha's header, refused + assert found["Alpha"] == 100 # the claim it would have stolen is intact + + +def test_a_mistyped_bank_header_is_recovered_end_to_end(tmp_path): + """A bank whose header name the mastering mistyped reads its records. + + ``Gutar Leads X`` carries a real ``EMULATOR`` header at the address its + directory placement predicts, but the header's own name is one edit off, so + keying ``located`` on exact-name equality listed it empty with a note. The + address plus the near-name gate binds it. ``OS Reserved X`` sits at an + address whose header carries an unrelated name -- the OS-slot shape -- and + stays noted, which is what proves the gate is not merely accepting whatever + header the arithmetic reaches. + """ + banks = [ + ( + "Default Folder", + [ + ("Piano Grand X", [("Grand C1", 20000, 512)]), + ("Strings Warm X", [("Str A2", 22000, 512)]), + ("Brass Bright X", [("Brs D3", 24000, 512)]), + ("Gutar Leads X", [("Lead E1", 26000, 512)]), + ("OS Reserved X", [("Unread", 28000, 256)]), + ], + ) + ] + image = image_of( + tmp_path, + fixtures.emu3_disc( + banks, + header_names={ + "Gutar Leads X": "Gutarr Leads X", # one edit: recovered + "OS Reserved X": "Zebra Marimba X", # unrelated: not recovered + }, + ), + "mistyped.iso", + ) + volumes = {v.name: v for v in BACKEND.volumes(image, 0)} + assert [f.name for f in volumes["Gutar Leads X"].files] == ["Lead E1"] + assert not volumes["Gutar Leads X"].note + assert volumes["OS Reserved X"].files == [] + assert volumes["OS Reserved X"].note == "no bank header found for this bank; listed only" + # The banks that pinned the fit are untouched. + assert [f.name for f in volumes["Piano Grand X"].files] == ["Grand C1"] + + # --- E-IV ---------------------------------------------------------------