From 80193cccb07adf384570d49bed3cdc4016392cea Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 4 Aug 2026 16:37:00 -0400 Subject: [PATCH 1/6] Resolve companion ranges across filename case differences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A LIFT folder written on Windows can spell its pair inconsistently — Dict.LIFT beside Dict.lift-ranges, or the reverse — and load fine there, because the filesystem folds case. On Linux the sibling candidate is built from the .lift's own suffix, so it missed, the companion was skipped without a word, and every range it defined went absent. Candidates that match no file exactly now fall back to one whose name differs only in case. The fallback is reached only after an exact miss, so a case-folding filesystem never enters it and behaves as before; a case-sensitive one gets one directory read per folder, cached across the candidate list. Where several names fold together the lexicographically first wins. The choice is arbitrary but fixed, which matters more than which file it picks: directory order varies between filesystems and runs, and a companion that loads differently on consecutive reads would be worse than one that never loads. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 10 ++++++++ src/sil_lift/_model.py | 48 ++++++++++++++++++++++++++++++++---- tests/test_ranges_folder.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08dcbe8..9e1b10e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,16 @@ releases may contain breaking changes. ## [Unreleased] +### Fixed + +- Companion `.lift-ranges` files now resolve when the folder's filenames + disagree in case (`Dict.LIFT` beside `Dict.lift-ranges`, or the reverse). + Such a folder loads on Windows and macOS, whose filesystems fold case, but + on Linux the companion was silently skipped and its ranges went missing. + A candidate that matches no file exactly now falls back to one whose name + differs only in case; where several fold together the lexicographically + first wins, so resolution is stable across runs. + ## [0.1.0] - 2026-07-TBD ### Added diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index e130f9f..3895046 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -452,6 +452,38 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: + """``candidate`` if it is a file, else one whose name differs only in case. + + LIFT folders are written on Windows, where the filesystem folds case, and + read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in + case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before + this fallback, silently did not on a case-sensitive filesystem. + + The fallback fires only where the exact name missed, so a case-folding + filesystem never reaches it and nothing changes there. Where several names + fold together, the lexicographically first wins — arbitrary, but stable + across runs, which "whatever the directory yields first" would not be. + ``listings`` caches one directory read per folder. + """ + try: + if candidate.is_file(): + return candidate + except OSError: + return None + folder = candidate.parent + if folder not in listings: + entries: dict[str, Path] = {} + try: + for path in sorted(folder.iterdir()): + if path.is_file(): + entries.setdefault(path.name.lower(), path) + except OSError: + pass # unreadable folder: no candidate resolves out of it + listings[folder] = entries + return listings[folder].get(candidate.name.lower()) + + def _same_dir(left: Path, right: Path | None) -> bool: """Whether two paths denote the same directory, spelling aside. @@ -510,7 +542,10 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the - exporting machine, so the basename is what resolves locally). + exporting machine, so the basename is what resolves locally). A + candidate no file matches exactly still resolves to one whose name + differs only in case, so a folder authored on Windows loads the same + way on a case-sensitive filesystem. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -544,14 +579,17 @@ def _resolve_ranges(self) -> None: basename = range_.href.replace("\\", "/").rpartition("/")[2] if basename: candidates.append(base / basename) + listings: dict[Path, dict[str, Path]] = {} for candidate in candidates: + found = _existing_file(candidate, listings) + if found is None: + continue try: - resolved = candidate.resolve() - exists = candidate.is_file() + resolved = found.resolve() except OSError: continue - if exists and resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(candidate) + if resolved not in self.ranges_files: + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9544db9..336a6bc 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -225,6 +225,55 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: assert [r.href for r in missing] == ["pictures\\sdd.png"] +def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: + """A loadable .lift plus companion under arbitrary filename casing. + + Named off the fixture stem so the header's ``range/@href`` basename + candidate finds nothing — only the sibling candidate can resolve these. + """ + folder.mkdir(parents=True, exist_ok=True) + (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes()) + return folder / lift_name + + +def _case_sensitive_fs(folder: Path) -> bool: + (folder / "CaseProbe").mkdir() + return not (folder / "caseprobe").exists() + + +def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: + if not _case_sensitive_fs(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the + # tie-break picks one: lexicographically first, the same one every run. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") + (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"] + + +def test_absent_companion_stays_absent(tmp_path: Path) -> None: + # The fallback must not reach past a folder for a name that isn't in it. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + assert sil_lift.load(folder / "Dict.lift").ranges_files == {} + + @pytest.mark.parametrize( ("href", "expected"), [ From 7d14ad7aa00a5dfea72d1b3a05eac2c8790ae7f1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 4 Aug 2026 16:43:47 -0400 Subject: [PATCH 2/6] Describe case-tolerant companion discovery under 0.1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.1.0 has not shipped, so there is no released behavior for an Unreleased entry to be fixing — the tolerance is simply part of what companion discovery does in the first release. Fold it into that bullet. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1b10e..d4ed83a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,16 +18,6 @@ releases may contain breaking changes. ## [Unreleased] -### Fixed - -- Companion `.lift-ranges` files now resolve when the folder's filenames - disagree in case (`Dict.LIFT` beside `Dict.lift-ranges`, or the reverse). - Such a folder loads on Windows and macOS, whose filesystems fold case, but - on Linux the companion was silently skipped and its ranges went missing. - A candidate that matches no file exactly now falls back to one whose name - differs only in case; where several fold together the lexicographically - first wins, so resolution is stable across runs. - ## [0.1.0] - 2026-07-TBD ### Added @@ -58,13 +48,14 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`), `save()` writes companions together, - `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, - build-from-scratch helpers `Lexicon.add_ranges_file()` / - `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and - header-references a new companion beside the `.lift`); vendored - `schemas/lift-ranges-0.13.rng` — the first schema for standalone - ranges documents. + (`Lexicon.ranges_files`, resolving a companion whose filename differs from + the `.lift` only in case, as Windows-authored folders often do), `save()` + writes companions together, `all_ranges()` merged view, `media_refs()` / + `missing_media()` helpers, build-from-scratch helpers + `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / + `Range.add_element()` (`save()` writes and header-references a new companion + beside the `.lift`); vendored `schemas/lift-ranges-0.13.rng` — the first + schema for standalone ranges documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other From a565d67a49514d11f46ba980bac87a9ea73842e1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 12 Aug 2026 15:16:08 -0400 Subject: [PATCH 3/6] Keep "entry" for LIFT entries in the companion-case fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The directory listing the fallback builds called its files "entries", the word this module uses for a LIFT everywhere else — the same collision that keeps byte regions from being called spans. Name them files. Spell the surrounding prose the way the rest of the package does: a fallback that runs rather than fires, a name that matched no file rather than missed, a helper named for the filesystem it probes rather than abbreviating it, and fixture names deliberately not taken from the corpus file. Unpack the two densest clauses so each reads in one pass. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 22 +++++++++++----------- tests/test_ranges_folder.py | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 3895046..2bde198 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -460,11 +460,11 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before this fallback, silently did not on a case-sensitive filesystem. - The fallback fires only where the exact name missed, so a case-folding - filesystem never reaches it and nothing changes there. Where several names - fold together, the lexicographically first wins — arbitrary, but stable - across runs, which "whatever the directory yields first" would not be. - ``listings`` caches one directory read per folder. + The fallback runs only where the exact name matched no file, so a + case-folding filesystem never reaches it and nothing changes there. Where + several names fold together, the lexicographically first wins — arbitrary, + but stable across runs, which "whatever the directory yields first" would + not be. ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -473,14 +473,14 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa return None folder = candidate.parent if folder not in listings: - entries: dict[str, Path] = {} + files: dict[str, Path] = {} try: for path in sorted(folder.iterdir()): if path.is_file(): - entries.setdefault(path.name.lower(), path) + files.setdefault(path.name.lower(), path) except OSError: pass # unreadable folder: no candidate resolves out of it - listings[folder] = entries + listings[folder] = files return listings[folder].get(candidate.name.lower()) @@ -543,9 +543,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). A - candidate no file matches exactly still resolves to one whose name - differs only in case, so a folder authored on Windows loads the same - way on a case-sensitive filesystem. + candidate that no file matches exactly still resolves to a file whose + name differs only in case, so a folder authored on Windows loads the + same way on a case-sensitive filesystem. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 336a6bc..4869bc2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -228,8 +228,8 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: """A loadable .lift plus companion under arbitrary filename casing. - Named off the fixture stem so the header's ``range/@href`` basename - candidate finds nothing — only the sibling candidate can resolve these. + Deliberately not named after the fixture, so the header's ``range/@href`` + basename candidate finds nothing — only the sibling candidate resolves these. """ folder.mkdir(parents=True, exist_ok=True) (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) @@ -237,7 +237,7 @@ def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> return folder / lift_name -def _case_sensitive_fs(folder: Path) -> bool: +def _case_sensitive_filesystem(folder: Path) -> bool: (folder / "CaseProbe").mkdir() return not (folder / "caseprobe").exists() @@ -255,7 +255,7 @@ def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) - def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: - if not _case_sensitive_fs(tmp_path): + if not _case_sensitive_filesystem(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the # tie-break picks one: lexicographically first, the same one every run. @@ -267,7 +267,7 @@ def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> Non def test_absent_companion_stays_absent(tmp_path: Path) -> None: - # The fallback must not reach past a folder for a name that isn't in it. + # The fallback must not look outside the folder for a name not in it. folder = tmp_path / "pkg" folder.mkdir() (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) From 8bf134f81227c0c360624150db4f98e016ca9b58 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:23:33 -0400 Subject: [PATCH 4/6] Resolve companions by folded name, and never load the .lift as one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion lookup folds names with casefold() over NFC rather than lower(), so a Turkish-cased or NFD-spelled name resolves the way it does on the filesystem that wrote it — FLEx mixes normalization forms within one export. Ties break in code point order on every platform; sorting Path objects left the choice to directory order on Windows, where PurePath ordering is itself case-folded. An unstattable exact spelling now falls through to the folded lookup instead of giving up. A candidate that folds onto the .lift itself is skipped: RangesFile.load rejects a root, so a header href naming the lexicon in another case took the whole load down. One that folds onto a companion already tracked is skipped too — Path.resolve() leaves case alone on macOS, so a single file reached under two spellings was loaded and tracked twice, and written twice by save(). The sibling candidate is built with with_name, which agrees with with_suffix on every name that has an extension and does not raise on a name without one. Nothing upstream requires the .lift extension: parse_document never inspects it. dangling-ranges-href decides existence with that same lookup, so a companion spelled in another case is no longer reported missing on a case-sensitive filesystem. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 85 ++++++++++++++++++++++++++++++------- src/sil_lift/_validate.py | 9 ++-- tests/test_ranges_folder.py | 66 ++++++++++++++++++++++++++-- 3 files changed, 138 insertions(+), 22 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 2bde198..b1e6155 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -11,6 +11,7 @@ from __future__ import annotations +import unicodedata from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath, PureWindowsPath @@ -452,36 +453,73 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _fold(text: str) -> str: + """A filename reduced to what a forgiving filesystem treats as one name. + + ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted + and dotless I), and NFC because normalization forms get mixed within a + single export — FLEx writes the ``.lift`` in NFC and its companion in NFD — + and macOS folds them together where Linux does not. Neither NTFS's nor + APFS's own folding table is reproduced exactly; this is an approximation + over LIFT filenames, not a general equivalence. + """ + return unicodedata.normalize("NFC", text).casefold() + + def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name differs only in case. + """``candidate`` if it is a file, else one whose name differs only in spelling. LIFT folders are written on Windows, where the filesystem folds case, and read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before - this fallback, silently did not on a case-sensitive filesystem. + this fallback, silently did not on a case-sensitive filesystem. See + :func:`_fold` for what counts as the same name. The fallback runs only where the exact name matched no file, so a - case-folding filesystem never reaches it and nothing changes there. Where - several names fold together, the lexicographically first wins — arbitrary, - but stable across runs, which "whatever the directory yields first" would - not be. ``listings`` caches one directory read per folder. + case-folding filesystem never reaches it and nothing changes there. Only + the final path component is folded: a candidate under a *directory* spelled + in another case still does not resolve, which the hrefs this serves — bare + basenames, or relatives within the folder — do not need. Where several + names fold together, the first in code point order wins (so ``Dict.LIFT`` + ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms, + which "whatever the directory yields first" would not be. ``listings`` + caches one directory read per folder. """ try: if candidate.is_file(): return candidate except OSError: - return None + pass # unstattable exact spelling: a case variant of it may still stat folder = candidate.parent if folder not in listings: files: dict[str, Path] = {} try: - for path in sorted(folder.iterdir()): + # By name, not by Path: PurePath ordering is case-folded on Windows, + # which would leave the tie-break to directory order there. + for path in sorted(folder.iterdir(), key=lambda entry: entry.name): if path.is_file(): - files.setdefault(path.name.lower(), path) + files.setdefault(_fold(path.name), path) except OSError: pass # unreadable folder: no candidate resolves out of it listings[folder] = files - return listings[folder].get(candidate.name.lower()) + return listings[folder].get(_fold(candidate.name)) + + +def _same_file(left: Path, right: Path) -> bool: + """Whether two paths differing only in spelling denote one file. + + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on + the latter one file reached under two spellings yields two distinct keys — + tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that + :func:`_fold` together are compared, since the inode check alone would + conflate distinct files on the filesystems that report ``st_ino`` as 0. + """ + if _fold(str(left)) != _fold(str(right)): + return False + try: + return left.samefile(right) + except OSError: + return False def _same_dir(left: Path, right: Path | None) -> bool: @@ -544,8 +582,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). A candidate that no file matches exactly still resolves to a file whose - name differs only in case, so a folder authored on Windows loads the - same way on a case-sensitive filesystem. + name differs only in case or Unicode normalization, so a folder + authored on Windows loads the same way on a case-sensitive filesystem. + The ``.lift`` itself is never taken as its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -567,9 +606,16 @@ def _resolve_ranges(self) -> None: if self.path is None: return base = self.path.parent + try: + own = self.path.resolve() + except OSError: + own = self.path candidates: list[Path] = [] - sibling = self.path.with_suffix(self.path.suffix + "-ranges") - candidates.append(sibling) + # with_name, not with_suffix: they agree on every name that has an + # extension, but with_suffix rejects "-ranges" outright on a name + # without one, and nothing upstream requires the document to be named + # ``.lift`` — parse_document never looks at the extension. + candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: continue @@ -588,8 +634,15 @@ def _resolve_ranges(self) -> None: resolved = found.resolve() except OSError: continue - if resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(found) + # Skip a spelling of something already tracked (macOS keeps two + # keys for one file) and the .lift itself, which a header href + # naming it in another case now folds onto — RangesFile.load would + # reject its root and take the whole load down with it. + if resolved in self.ranges_files or any( + _same_file(resolved, other) for other in (own, *self.ranges_files) + ): + continue + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 39e0b25..9680eb9 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError -from ._model import GrammaticalInfo, Lexicon, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href from ._text import Multitext, Trait if TYPE_CHECKING: @@ -435,9 +435,12 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve # (they are resolved by basename when the companion is in the same folder) # and are not checked here; this catches an exporter that writes a relative - # href but not the file. + # href but not the file. Existence is the same notion load resolves + # companions by (_existing_file), so a companion spelled in another case is + # not reported missing on a case-sensitive filesystem. if lexicon.path is not None: base = lexicon.path.parent + listings: dict[Path, dict[str, Path]] = {} for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue @@ -447,7 +450,7 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: resolved = all_ranges.get(range_.id) if resolved is not None and resolved.elements: continue # supplied by a sibling companion instead - if not (base / relative).is_file(): + if _existing_file(base / relative, listings) is None: yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 4869bc2..fc0d2b2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -1,4 +1,5 @@ import shutil +import unicodedata from pathlib import Path import pytest @@ -226,7 +227,7 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: - """A loadable .lift plus companion under arbitrary filename casing. + """A loadable .lift plus companion under arbitrary filename spellings. Deliberately not named after the fixture, so the header's ``range/@href`` basename candidate finds nothing — only the sibling candidate resolves these. @@ -237,9 +238,22 @@ def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> return folder / lift_name +def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: + """The fixture .lift under another name, its companion href rewritten.""" + folder.mkdir(parents=True, exist_ok=True) + source = (PAIR_DIR / "test20080407.lift").read_bytes() + patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode()) + assert patched != source, "fixture href changed; the replacement no longer matches" + (folder / lift_name).write_bytes(patched) + return folder / lift_name + + def _case_sensitive_filesystem(folder: Path) -> bool: - (folder / "CaseProbe").mkdir() - return not (folder / "caseprobe").exists() + probe = folder / "CaseProbe" + probe.mkdir(exist_ok=True) + sensitive = not (folder / "caseprobe").exists() + probe.rmdir() + return sensitive def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: @@ -274,6 +288,52 @@ def test_absent_companion_stays_absent(tmp_path: Path) -> None: assert sil_lift.load(folder / "Dict.lift").ranges_files == {} +def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None: + # FLEx mixes NFC and NFD within one export, and the mismatch reaches the + # filenames; only macOS folds the two forms together on its own. + composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift" + decomposed = unicodedata.normalize("NFD", f"{composed}-ranges") + lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed) + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_lift_without_an_extension_loads(tmp_path: Path) -> None: + # Loading never inspects the extension, so the sibling candidate is built + # from a name that may have none; this companion is the href's basename. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder) + lexicon = sil_lift.load(folder / "Dict") + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None: + # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it + # as one would raise on the root and take the whole load down. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + assert sil_lift.load(lift).ranges_files == {} + + +# Defines the range the header points at, but no elements — so the merged view +# cannot vouch for the href and the check falls through to the filesystem. +ELEMENTLESS_RANGES = b""" + + + +""" + + +def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None: + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges") + (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files # the companion resolved + assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == [] + + @pytest.mark.parametrize( ("href", "expected"), [ From 15109b5e38fc536af7089594aaef25db41d5c88f Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:23:53 -0400 Subject: [PATCH 5/6] Describe case- and normalization-tolerant companion discovery The 0.1.0 entry said companion names fold on case alone; they fold on Unicode normalization form as well. The folder guide listed the candidates tried but never mentioned the folding at all. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 15 ++++++++------- docs/en/guides/folder-media.md | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ed83a..52d35bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,13 +49,14 @@ releases may contain breaking changes. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load (`Lexicon.ranges_files`, resolving a companion whose filename differs from - the `.lift` only in case, as Windows-authored folders often do), `save()` - writes companions together, `all_ranges()` merged view, `media_refs()` / - `missing_media()` helpers, build-from-scratch helpers - `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / - `Range.add_element()` (`save()` writes and header-references a new companion - beside the `.lift`); vendored `schemas/lift-ranges-0.13.rng` — the first - schema for standalone ranges documents. + the `.lift` only in case or Unicode normalization form, as Windows- and + FLEx-authored folders do), `save()` writes companions together, + `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, + build-from-scratch helpers `Lexicon.add_ranges_file()` / + `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and + header-references a new companion beside the `.lift`); vendored + `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges + documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other diff --git a/docs/en/guides/folder-media.md b/docs/en/guides/folder-media.md index 16497fb..ddbeb56 100644 --- a/docs/en/guides/folder-media.md +++ b/docs/en/guides/folder-media.md @@ -12,7 +12,7 @@ lex.all_ranges() # merged {id: Range} view lex.all_ranges()["grammatical-info"].elements ``` -Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. +Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem. `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use: From 7cee80f9abece1abfb3263d6bdaaf96f2f7da181 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:55:45 -0400 Subject: [PATCH 6/6] Trim the companion-folding prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three helpers each stated a rule, defended it, then disclaimed it. What is left is the reasoning the code cannot show: casefold over lower, NFC, why only the final component folds, why code point order, and what the fold pre-check protects the inode comparison from. The folder guide drops the folding sentence outright — it describes behavior no reader acts on, in a paragraph otherwise about which candidate wins. The 0.1.0 entry keeps the fact and loses the justification, which now lives only in the docstrings. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 5 ++-- docs/en/guides/folder-media.md | 2 +- src/sil_lift/_model.py | 49 ++++++++++++++-------------------- 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52d35bb..4138872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,9 +48,8 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`, resolving a companion whose filename differs from - the `.lift` only in case or Unicode normalization form, as Windows- and - FLEx-authored folders do), `save()` writes companions together, + (`Lexicon.ranges_files`, matching companion filenames across case and + Unicode normalization differences), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and diff --git a/docs/en/guides/folder-media.md b/docs/en/guides/folder-media.md index ddbeb56..16497fb 100644 --- a/docs/en/guides/folder-media.md +++ b/docs/en/guides/folder-media.md @@ -12,7 +12,7 @@ lex.all_ranges() # merged {id: Range} view lex.all_ranges()["grammatical-info"].elements ``` -Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem. +Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use: diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index b1e6155..0a7d42c 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -454,36 +454,28 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: - """A filename reduced to what a forgiving filesystem treats as one name. - - ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted - and dotless I), and NFC because normalization forms get mixed within a - single export — FLEx writes the ``.lift`` in NFC and its companion in NFD — - and macOS folds them together where Linux does not. Neither NTFS's nor - APFS's own folding table is reproduced exactly; this is an approximation - over LIFT filenames, not a general equivalence. + """A filename reduced to what a case-folding filesystem treats as one name. + + ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC + because FLEx mixes normalization forms within one export. An approximation + of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name differs only in spelling. + """``candidate`` if it is a file, else one whose name folds onto it. LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in - case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before - this fallback, silently did not on a case-sensitive filesystem. See - :func:`_fold` for what counts as the same name. - - The fallback runs only where the exact name matched no file, so a - case-folding filesystem never reaches it and nothing changes there. Only - the final path component is folded: a candidate under a *directory* spelled - in another case still does not resolve, which the hrefs this serves — bare - basenames, or relatives within the folder — do not need. Where several - names fold together, the first in code point order wins (so ``Dict.LIFT`` - ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms, - which "whatever the directory yields first" would not be. ``listings`` - caches one directory read per folder. + read everywhere: ``Dict.LIFT`` beside ``Dict.lift-ranges`` resolves there + and, before this fallback, silently did not on a case-sensitive filesystem. + + Only the final component folds — the hrefs this serves are basenames or + same-folder relatives — and only after the exact name misses, so a + case-folding filesystem never reaches this. Among names that fold together + the first in code point order wins: arbitrary, but stable across runs and + platforms, which directory order is not. ``listings`` caches one directory + read per folder. """ try: if candidate.is_file(): @@ -506,13 +498,12 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa def _same_file(left: Path, right: Path) -> bool: - """Whether two paths differing only in spelling denote one file. + """Whether two paths that fold together denote one file. - ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on - the latter one file reached under two spellings yields two distinct keys — - tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that - :func:`_fold` together are compared, since the inode check alone would - conflate distinct files on the filesystems that report ``st_ino`` as 0. + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where + one file reached under two spellings yields two keys — tracked twice, and + written twice by :meth:`Lexicon.save`. The fold pre-check keeps the inode + comparison from conflating distinct files where ``st_ino`` is 0. """ if _fold(str(left)) != _fold(str(right)): return False