Resolve companion ranges across filename case differences

CHANGELOG.md

@@ -48,13 +48,14 @@ releases may contain breaking changes.
   against the most recent `save()`.
 - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents,
   same fidelity guarantees), automatic companion discovery/tracking on load
-  (`Lexicon.ranges_files`), `save()` writes companions together,
+  (`Lexicon.ranges_files`, matching companion filenames across case and
+  Unicode normalization differences), `save()` writes companions together,
   `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers,
   build-from-scratch helpers `Lexicon.add_ranges_file()` /
   `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and
   header-references a new companion beside the `.lift`); vendored
-  `schemas/lift-ranges-0.13.rng` — the first schema for standalone
-  ranges documents.
+  `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges
+  documents.
 - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and
   folder-wrapped layouts, junk entries like `__MACOSX` ignored),
   `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other

src/sil_lift/_model.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import unicodedata
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path, PurePosixPath, PureWindowsPath
@@ -452,6 +453,75 @@ def _normalize_href(href: str) -> Path | None:
     return Path(normalized)
 
 
+def _fold(text: str) -> str:
+    """A filename reduced to what a case-folding filesystem treats as one name.
+
+    ``casefold`` for what ``lower`` gets wrong (e.g. the Turkish dotless i);
+    NFC for names that arrive decomposed (e.g. ones zipped on macOS). An
+    approximation of NTFS's and APFS's tables, not a general equivalence.
+    """
+    return unicodedata.normalize("NFC", text).casefold()
+
+
+def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None:
+    """The file ``candidate`` names, matched exactly or by folded name.
+
+    Where the authoring filesystem folds case, as on Windows and macOS, an
+    inconsistently spelled pair goes unnoticed: ``Dict.LIFT`` beside
+    ``Dict.lift-ranges`` is a pair there but not everywhere.
+
+    An exact hit is always returned unchanged: folding runs only after the
+    exact name misses, and then only on the final component — the hrefs this
+    serves are basenames or same-folder relatives.
+
+    Among names that fold together the first in code point order wins:
+    arbitrary, but deterministic, which directory order is not.
+
+    ``listings`` caches one directory read per folder.
+    """
+    try:
+        if candidate.is_file():
+            return candidate
+        if candidate.is_dir():
+            # An href of "" or "sub/" lands here; folding a folder's own name
+            # would search its parent and match anything spelled like it.
+            return None
+    except OSError:
+        pass  # unstattable exact spelling: a case variant of it may still stat
+    folder = candidate.parent
+    if folder not in listings:
+        files: dict[str, Path] = {}
+        try:
+            # By name, not by Path: PurePath ordering is case-folded on Windows,
+            # which would leave the tie-break to directory order there.
+            for path in sorted(folder.iterdir(), key=lambda entry: entry.name):
+                if path.is_file():
+                    files.setdefault(_fold(path.name), path)
+        except OSError:
+            pass  # unreadable folder: no candidate resolves out of it
+        listings[folder] = files
+    return listings[folder].get(_fold(candidate.name))
+
+
+def _same_file(left: Path, right: Path) -> bool:
+    """Whether two paths that fold together denote one file.
+
+    ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where
+    one file reached under two spellings yields two keys — tracked twice, and
+    written twice by :meth:`Lexicon.save`.
+
+    Both sides resolve first, so ``..`` segments and symlinks compare alike;
+    the fold pre-check then keeps the inode comparison from conflating
+    distinct files where ``st_ino`` is 0.
+    """
+    try:
+        if _fold(str(left.resolve())) != _fold(str(right.resolve())):
+            return False
+        return left.samefile(right)
+    except OSError:
+        return False
+
+
 def _same_dir(left: Path, right: Path | None) -> bool:
     """Whether two paths denote the same directory, spelling aside.
 
@@ -505,13 +575,17 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L
 
         With ``resolve_ranges`` (the default), companion ``.lift-ranges``
         files are loaded and tracked in :attr:`ranges_files`. Several
-        candidates are tried and every one that exists is loaded: the
-        conventional ``<name>.lift-ranges`` sibling, and for each header
+        candidates are tried and every distinct file among them is loaded:
+        the conventional ``<name>.lift-ranges`` sibling, and for each header
         ``range/@href`` both the href resolved as a path relative to the
         ``.lift`` file and its bare basename in the same directory (FLEx
         hrefs are usually dangling absolute ``file://C:/...`` paths from the
         exporting machine, so the basename is what resolves locally).
 
+        A candidate matching no file exactly resolves across differences in
+        case or Unicode normalization, so a folder authored on Windows loads
+        the same way everywhere; the ``.lift`` is never its own companion.
+
         A ``.zip`` path is treated as a packaged LIFT folder: it is extracted
         to a temporary directory (kept alive for the returned lexicon's
         lifetime) and the single contained ``.lift`` is loaded.
@@ -533,8 +607,10 @@ def _resolve_ranges(self) -> None:
             return
         base = self.path.parent
         candidates: list[Path] = []
-        sibling = self.path.with_suffix(self.path.suffix + "-ranges")
-        candidates.append(sibling)
+        # with_name and with_suffix agree on every name that has an extension,
+        # but with_suffix would raise on a name that has none — which
+        # parse_document accepts, since it never inspects the extension.
+        candidates.append(self.path.with_name(self.path.name + "-ranges"))
         for range_ in self.header.ranges:
             if range_.href is None:
                 continue
@@ -544,14 +620,24 @@ def _resolve_ranges(self) -> None:
             basename = range_.href.replace("\\", "/").rpartition("/")[2]
             if basename:
                 candidates.append(base / basename)
+        listings: dict[Path, dict[str, Path]] = {}
         for candidate in candidates:
+            found = _existing_file(candidate, listings)
+            if found is None:
+                continue
             try:
-                resolved = candidate.resolve()
-                exists = candidate.is_file()
+                resolved = found.resolve()
             except OSError:
                 continue
-            if exists and resolved not in self.ranges_files:
-                self.ranges_files[resolved] = RangesFile.load(candidate)
+            # A header href naming the .lift in another case folds onto it, and
+            # RangesFile.load rejects that root, failing the whole load; two
+            # spellings of one companion, which resolve() leaves distinct on
+            # macOS, would load and write it twice.
+            if resolved in self.ranges_files or any(
+                _same_file(resolved, other) for other in (self.path, *self.ranges_files)
+            ):
+                continue
+            self.ranges_files[resolved] = RangesFile.load(found)
 
     def save(self, path: str | os.PathLike[str] | None = None) -> None:
         """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion.

src/sil_lift/_validate.py

@@ -40,7 +40,7 @@
 from lxml import etree
 
 from ._errors import LiftValidationError
-from ._model import GrammaticalInfo, Lexicon, _normalize_href
+from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href, _same_file
 from ._text import Multitext, Trait
 
 if TYPE_CHECKING:
@@ -431,13 +431,13 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]:
                     file=file,
                 )
 
-    # Header <range href> references (relative) that resolve to no companion.
-    # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve
-    # (they are resolved by basename when the companion is in the same folder)
-    # and are not checked here; this catches an exporter that writes a relative
-    # href but not the file.
+    # Header <range href> references that resolve to no companion — an exporter
+    # that wrote the href but not the file. Absolute and file:// hrefs are
+    # skipped: FLEx writes those knowing they will not resolve, and load reaches
+    # their companions by basename in the same folder instead.
     if lexicon.path is not None:
         base = lexicon.path.parent
+        listings: dict[Path, dict[str, Path]] = {}
         for range_ in lexicon.header.ranges:
             if not range_.href or range_.elements:
                 continue
@@ -447,7 +447,10 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]:
             resolved = all_ranges.get(range_.id)
             if resolved is not None and resolved.elements:
                 continue  # supplied by a sibling companion instead
-            if not (base / relative).is_file():
+            found = _existing_file(base / relative, listings)
+            # _resolve_ranges refuses the lexicon as its own companion, so an
+            # href folding onto it supplies nothing and dangles too.
+            if found is None or _same_file(found, lexicon.path):
                 yield Problem(
                     "warning",
                     "dangling-ranges-href",

tests/test_ranges_folder.py

@@ -1,4 +1,5 @@
 import shutil
+import unicodedata
 from pathlib import Path
 
 import pytest
@@ -225,6 +226,134 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None:
     assert [r.href for r in missing] == ["pictures\\sdd.png"]
 
 
+def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path:
+    """A loadable .lift plus companion under arbitrary filename spellings.
+
+    Deliberately not named after the fixture, so the header's ``range/@href``
+    basename candidate finds nothing — only the sibling candidate resolves these.
+    """
+    folder.mkdir(parents=True, exist_ok=True)
+    (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes())
+    (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes())
+    return folder / lift_name
+
+
+def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path:
+    """The fixture .lift under another name, its companion href rewritten."""
+    folder.mkdir(parents=True, exist_ok=True)
+    source = (PAIR_DIR / "test20080407.lift").read_bytes()
+    patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode())
+    assert patched != source, "fixture href changed; the replacement no longer matches"
+    (folder / lift_name).write_bytes(patched)
+    return folder / lift_name
+
+
+def _case_sensitive(folder: Path) -> bool:
+    probe = folder / "CaseProbe"
+    probe.mkdir(exist_ok=True)
+    sensitive = not (folder / "caseprobe").exists()
+    probe.rmdir()
+    return sensitive
+
+
+def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None:
+    lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges")
+    lexicon = sil_lift.load(lift)
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None:
+    lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES")
+    lexicon = sil_lift.load(lift)
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None:
+    if not _case_sensitive(tmp_path):
+        pytest.skip("needs a case-sensitive filesystem to hold both spellings at once")
+    # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the
+    # tie-break picks one: lexicographically first, the same one every run.
+    folder = tmp_path / "pkg"
+    lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges")
+    (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes())
+    lexicon = sil_lift.load(lift)
+    assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"]
+
+
+def test_absent_companion_stays_absent(tmp_path: Path) -> None:
+    # The fallback must not look outside the folder for a name not in it.
+    folder = tmp_path / "pkg"
+    folder.mkdir()
+    (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes())
+    assert sil_lift.load(folder / "Dict.lift").ranges_files == {}
+
+
+def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None:
+    # FLEx mixes NFC and NFD within one export, and the mismatch reaches the
+    # filenames; only macOS folds the two forms together on its own.
+    composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift"
+    decomposed = unicodedata.normalize("NFD", f"{composed}-ranges")
+    lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed)
+    lexicon = sil_lift.load(lift)
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_lift_without_an_extension_loads(tmp_path: Path) -> None:
+    # Loading never inspects the extension, so the sibling candidate is built
+    # from a name that may have none; this companion is the href's basename.
+    folder = tmp_path / "pkg"
+    folder.mkdir()
+    (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes())
+    shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder)
+    lexicon = sil_lift.load(folder / "Dict")
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None:
+    # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it
+    # as one would raise on the root and take the whole load down.
+    lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift")
+    lexicon = sil_lift.load(lift)
+    assert lexicon.ranges_files == {}
+    assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()]
+
+
+def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> None:
+    # The ".." keeps the href from matching the lexicon's path as spelled, so
+    # both sides have to resolve before deciding what the reference supplies.
+    lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "../pkg/Dict.lift")
+    lexicon = sil_lift.load(lift)
+    assert lexicon.ranges_files == {}
+    assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()]
+
+
+def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None:
+    if not _case_sensitive(tmp_path):
+        pytest.skip("needs a case-sensitive filesystem to hold both spellings at once")
+    # An empty href names the folder itself; folding it would search the parent.
+    lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "")
+    (tmp_path / "PKG").write_bytes(b"<lift/>")
+    assert sil_lift.load(lift).ranges_files == {}
+
+
+# Defines the range the header points at, but no elements — so the merged view
+# cannot vouch for the href and the check falls through to the filesystem.
+ELEMENTLESS_RANGES = b"""<?xml version="1.0" encoding="UTF-8"?>
+<lift-ranges>
+  <range id="grammatical-info"/>
+</lift-ranges>
+"""
+
+
+def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None:
+    folder = tmp_path / "pkg"
+    lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges")
+    (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES)
+    lexicon = sil_lift.load(lift)
+    assert lexicon.ranges_files  # the companion resolved
+    assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == []
+
+
 @pytest.mark.parametrize(
     ("href", "expected"),
     [