From 3a7943dd5bfe84396c9cb15d3f934bac53e395e5 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 12 Aug 2026 16:21:40 -0400 Subject: [PATCH 1/9] Compare a range-element's parent against sibling ids under NFC FLEx normalizes strings to NFC on export, but a few writes emit the NFD it holds in memory -- among them the grammatical-info and lexical-relation range-element ids, while the parent attribute on those same elements is normalized. Comparing the two as exact strings reported a sound parent link as dangling: both range-parent errors in the sango fixture are that encoding difference, not defects. Co-authored-by: Claude Opus 5 (1M context) --- CHANGELOG.md | 8 +++++--- src/sil_lift/_validate.py | 20 ++++++++++++-------- tests/corpus/PROVENANCE.md | 19 +++++++++++++++++++ tests/test_validate.py | 29 ++++++++++++++++++++++++++--- 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08dcbe8..c65db7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,9 +73,11 @@ releases may contain breaking changes. ranges/range-elements within their own document — matching the C# `Validator`'s document-wide guid scan), dangling-ref, range-parent, undefined-range-value (every grammatical-info and range-keyed trait - reachable from an entry, however nested \+ NFC-normalized), - duplicate-form-lang, missing-media, dangling-ranges-href, and (opt-in via - `require_ids`) missing-id. + reachable from an entry, however nested), duplicate-form-lang, + missing-media, dangling-ranges-href, and (opt-in via `require_ids`) + missing-id. Both checks that resolve a name against a range-element id + compare under NFC normalization: FLEx can write an id in NFD while the + value and the `parent` attribute referring to it are NFC. - Canonical sort: `Lexicon.sort()` / `RangesFile.sort()` (entries by case-folded guid/id, ranges/range-elements by id, field definitions by tag; informed by the C# LiftSorter, locale-independent) and diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 39e0b25..65fe125 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -417,12 +417,21 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: line=at(index), ) + # Comparisons of a range value or a parent link against a range-element id + # are NFC-normalized. FLEx normalizes strings to NFC on export, but a few + # writes bypass its normalizing helper and emit the NFD it holds in memory + # -- among them the grammatical-info and lexical-relation range-element + # ids, whose labels and own parent attribute are normalized. So an id can + # be NFD while the .lift value and the parent pointing at it are NFC. + def nfc(value: str) -> str: + return unicodedata.normalize("NFC", value) + # Range integrity over the merged view (inline + companions). all_ranges = lexicon.all_ranges() for range_ in all_ranges.values(): - element_ids = {element.id for element in range_.elements} + element_ids = {nfc(element.id) for element in range_.elements} for element in range_.elements: - if element.parent and element.parent not in element_ids: + if element.parent and nfc(element.parent) not in element_ids: yield Problem( "error", "range-parent", @@ -463,12 +472,7 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: # is-primary/complex-form-type inside and morph-type inside # (_iter_traits/_iter_grammatical_infos walk the whole entry). # Only ranges that actually enumerate elements can confirm a value; empty - # values skipped (FLEx writes them). Comparison is NFC-normalized: FLEx - # writes the .lift in NFC but the companion .lift-ranges in NFD within the - # same export. - def nfc(value: str) -> str: - return unicodedata.normalize("NFC", value) - + # values skipped (FLEx writes them). def defined(range_id: str) -> set[str] | None: range_: Range | None = all_ranges.get(range_id) if range_ is None or not range_.elements: diff --git a/tests/corpus/PROVENANCE.md b/tests/corpus/PROVENANCE.md index 45dcf60..0804848 100644 --- a/tests/corpus/PROVENANCE.md +++ b/tests/corpus/PROVENANCE.md @@ -155,6 +155,25 @@ with one `HashSet` and would flag this too — so sil-lift's `duplicate-guid` check reports it as an error, matching that behavior. `tests/test_validate.py` locks in the exact counts found (5 in AllFLExFields, 37 in Sango). +## Known Unicode normalization asymmetry in real fixtures + +FLEx holds strings as NFD and normalizes them to NFC on export, but a few writes +bypass its normalizing helper and emit the NFD unchanged. In +`large/sango/sango.lift-ranges` the `grammatical-info`, `from-part-of-speech`, +and `lexical-relation` range-element `id`s are NFD while everything around them +— their `label`/`abbrev`/`description`, the `parent` attribute of those same +elements, the `semantic-domain-ddp4`/`translation-type`/`usage-type` ids, and +every value in `large/sango/sango.lift` — is NFC. + +So one name appears in two spellings within a single element: the part of +speech whose id decomposes as `Comple` + U+0301 + `ments` is named by a child +element's `parent` attribute with the precomposed U+00E9 instead. Both +`range-parent` and `undefined-range-value` therefore compare under NFC +normalization; without it, this fixture reports two dangling parents and an +extra undefined part of speech that are artifacts of the encoding, not defects. +The one genuine `undefined-range-value` left in Sango is the part of speech +`prenom`, which no range defines in any normalization. + ## generated/ — synthetic large files (not committed) Produced by `tests/tools/generate_large.py` for streaming/perf tests; diff --git a/tests/test_validate.py b/tests/test_validate.py index 6249154..8468ca5 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -41,6 +41,27 @@ def test_range_parent_integrity() -> None: assert "Nooun" in problem.message +def test_range_parent_tolerates_flex_normalization_asymmetry() -> None: + # Regression: FLEx writes a grammatical-info range-element id straight from + # its NFD in-memory string but normalizes the parent attribute to NFC (see + # PROVENANCE.md), so the two spellings of one name differ within a single + # element. The parent link is sound; only the encoding differs. + lexicon = sil_lift.Lexicon() + ranges = sil_lift.RangesFile() + # "Compl\u00e9ments" spelled two ways: the id decomposed (e + U+0301), + # as FLEx writes range-element ids, and the parent composed. + nfd_id = "Comple\u0301ments" + nfc_parent = "Compl\u00e9ments" + range_ = ranges.add_range("grammatical-info") + range_.add_element(nfd_id) + range_.add_element("Comple\u0301ment du lieu", parent=nfc_parent) + lexicon.add_ranges_file(ranges, href="x.lift-ranges") + entry = sil_lift.Entry(id="e1", guid="bbbbbbbb-1111-4444-8888-bbbbbbbbbbbb") + entry.lexical_unit["en"] = "e1" + lexicon.entries.append(entry) + assert [p for p in lexicon.iter_problems() if p.code == "range-parent"] == [] + + def test_undefined_range_values_are_warnings() -> None: problems = problems_for(NEGATIVE_DIR / "undefined-range-value.lift") assert codes(problems) == {("warning", "undefined-range-value")} @@ -199,9 +220,11 @@ def test_sango_real_defects_are_found() -> None: by_code: dict[str, int] = {} for problem in problems: by_code[problem.code] = by_code.get(problem.code, 0) + 1 - # Two genuinely dangling range-element parents + one undefined POS value - # ('prenom') in the real export; NFC-normalization keeps the count at 1. - assert by_code.get("range-parent") == 2 + # One undefined POS value ('prenom') in the real export; NFC-normalization + # keeps the count at 1. No range-parent finding: the two parent links that + # differ from their target's spelling are FLEx's NFD ids under an NFC + # parent (see PROVENANCE.md), not dangling references. + assert by_code.get("range-parent") is None assert by_code.get("undefined-range-value") == 1 assert by_code.get("schema", 0) > 0 # companion's trait/field extensions # 37 real duplicate guids: FLEx aliases its POS list under both From a1572860b0f66d282ee5372b2159fa706ee24faa Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 12 Aug 2026 16:21:49 -0400 Subject: [PATCH 2/9] Say which FLEx writes skip normalization, not which file The validation notes described the .lift as NFC and the .lift-ranges as NFD. It is narrower than that: the normalizer is applied per write, so a range-element id can be NFD beside a label, a parent, and a .lift value that are all NFC. Co-authored-by: Claude Opus 5 (1M context) --- docs/en/csharp-differences.md | 2 +- docs/en/guides/lift-export-interop.md | 2 +- docs/en/guides/validate.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/csharp-differences.md b/docs/en/csharp-differences.md index cf0cf1e..5d05320 100644 --- a/docs/en/csharp-differences.md +++ b/docs/en/csharp-differences.md @@ -30,7 +30,7 @@ The C# `Validator` runs one RELAX NG pass and reports the first errors as string - **Invalid URIs are warnings, not errors.** The C# RELAX NG engine never enforced the `anyURI` datatype, so FieldWorks (FLEx) has been writing `file://C:/...` hrefs into real lexicons for years. Rejecting those files would flag virtually every FLEx export. - **Schematron rules are enforced** (as semantic checks): duplicate form languages and similar co-constraints in the LIFT grammar were silently ignored by both C# and raw lxml validation. -- **Cross-file comparisons are Unicode-normalized**, because FLEx writes the `.lift` in NFC and the companion `.lift-ranges` in NFD. +- **Range id comparisons are Unicode-normalized** (NFC), because FLEx's own export is not internally consistent: it normalizes to NFC on the way out, but a few writes bypass that step and emit the NFD it holds in memory. A `grammatical-info` or `lexical-relation` range-element `id` can therefore be NFD while its labels, the `parent` attribute on that same element, and the `.lift` value referring to it are all NFC. sil-lift also validates the `.lift-ranges` companions of a loaded lexicon against a schema for standalone ranges documents (vendored from `lift-standard` alongside the base LIFT grammar) — every tracked external ranges file is checked whenever the `.lift` is validated — with no such schema (or check) in the C# world. (There is no entry point for validating a `.lift-ranges` file on its own, detached from a `.lift`.) diff --git a/docs/en/guides/lift-export-interop.md b/docs/en/guides/lift-export-interop.md index 1777e35..2394d9a 100644 --- a/docs/en/guides/lift-export-interop.md +++ b/docs/en/guides/lift-export-interop.md @@ -92,7 +92,7 @@ The companion carries each range's full definition. Values are `` ``` -An entry then refers to a value by id: a sense's part of speech is ``, and a semantic domain is ``. `sil-lift validate` warns (`undefined-range-value`) when a value isn't defined in its range and errors (`range-parent`) when a `parent` isn't a sibling id — so emit the ranges your data actually uses. See also [Ranges and media](folder-media.md). +An entry then refers to a value by id: a sense's part of speech is ``, and a semantic domain is ``. `sil-lift validate` warns (`undefined-range-value`) when a value isn't defined in its range and errors (`range-parent`) when a `parent` isn't a sibling id — so emit the ranges your data actually uses. Both comparisons are NFC-normalized, so an id and the value or `parent` referring to it may differ in Unicode normalization; still, write one consistent normalization if you can. See also [Ranges and media](folder-media.md). If you build the export in Python, `Lexicon.add_ranges_file()`, `RangesFile.add_range()`, and `Range.add_element()` construct the companion and add the header references for you; `open_writer(..., ranges=...)` does the same on the streaming path. diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index c8545e7..1ead92e 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -32,5 +32,5 @@ FieldWorks systematically writes some content that strict tooling rejects. Here - `file://C:/...` hrefs (invalid URIs) are reported as **warnings** (`uri-not-rfc`), not schema errors — the C# validator never rejected them. - Legally interleaved children (e.g. `field, note, field, note` in a sense) are **not** flagged, working around a false positive in libxml2. -- Range values are compared under Unicode NFC normalization — FLEx writes the `.lift` in NFC but the `.lift-ranges` in NFD within the same export. +- Range values and `parent` links are compared under Unicode NFC normalization — FLEx normalizes to NFC on export but a few writes bypass that step, so a range-element `id` can be NFD while its labels, its own `parent` attribute, and the `.lift` value referring to it are NFC. Without normalization those spellings compare unequal and a sound export looks broken. - FLEx's `trait`/`field` extensions inside `range-element` **are** reported (schema errors against the ranges schema): they are genuine spec deviations. From eca6e8d653607c56c2684629e0961b92e794da91 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 12 Aug 2026 16:57:49 -0400 Subject: [PATCH 3/9] Warn when a name resolves only after NFC normalization Normalizing both sides of the range comparisons hid a real property of FLEx output: an id and the names referring to it can differ in encoding, and any consumer comparing raw strings -- a Send/Receive merge, say -- fails to resolve them. Report that as a warning, once per range-element id however many references differ (6 rather than 82 in the sango fixture), naming both spellings by code point because they render identically, and addressing it to the file the id lives in. The new negative pair carries the shape end to end: NFD ids, NFC parents and grammatical-info value, one parent that dangles in every normalization, and bytes that must survive the round trip unchanged -- normalization belongs to the comparison, not to what gets written. Co-authored-by: Claude Opus 5 (1M context) --- CHANGELOG.md | 13 ++-- docs/en/csharp-differences.md | 2 +- docs/en/guides/lift-export-interop.md | 2 +- docs/en/guides/validate.md | 4 +- src/sil_lift/_validate.py | 62 +++++++++++++++---- tests/corpus/PROVENANCE.md | 12 +++- tests/corpus/negative/nfd-range-ids.lift | 20 ++++++ .../corpus/negative/nfd-range-ids.lift-ranges | 14 +++++ tests/test_validate.py | 23 ++++++- 9 files changed, 129 insertions(+), 23 deletions(-) create mode 100644 tests/corpus/negative/nfd-range-ids.lift create mode 100644 tests/corpus/negative/nfd-range-ids.lift-ranges diff --git a/CHANGELOG.md b/CHANGELOG.md index c65db7e..2784957 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,11 +73,14 @@ releases may contain breaking changes. ranges/range-elements within their own document — matching the C# `Validator`'s document-wide guid scan), dangling-ref, range-parent, undefined-range-value (every grammatical-info and range-keyed trait - reachable from an entry, however nested), duplicate-form-lang, - missing-media, dangling-ranges-href, and (opt-in via `require_ids`) - missing-id. Both checks that resolve a name against a range-element id - compare under NFC normalization: FLEx can write an id in NFD while the - value and the `parent` attribute referring to it are NFC. + reachable from an entry, however nested), normalization-mismatch, + duplicate-form-lang, missing-media, dangling-ranges-href, and (opt-in via + `require_ids`) missing-id. Both checks that resolve a name against a + range-element id compare under NFC normalization: FLEx can write an id in + NFD while the value and the `parent` attribute referring to it are NFC. + What matched only after normalizing is reported as a normalization-mismatch + warning, once per range-element id, so the encoding split stays visible to + consumers that compare raw strings. - Canonical sort: `Lexicon.sort()` / `RangesFile.sort()` (entries by case-folded guid/id, ranges/range-elements by id, field definitions by tag; informed by the C# LiftSorter, locale-independent) and diff --git a/docs/en/csharp-differences.md b/docs/en/csharp-differences.md index 5d05320..3d0307d 100644 --- a/docs/en/csharp-differences.md +++ b/docs/en/csharp-differences.md @@ -30,7 +30,7 @@ The C# `Validator` runs one RELAX NG pass and reports the first errors as string - **Invalid URIs are warnings, not errors.** The C# RELAX NG engine never enforced the `anyURI` datatype, so FieldWorks (FLEx) has been writing `file://C:/...` hrefs into real lexicons for years. Rejecting those files would flag virtually every FLEx export. - **Schematron rules are enforced** (as semantic checks): duplicate form languages and similar co-constraints in the LIFT grammar were silently ignored by both C# and raw lxml validation. -- **Range id comparisons are Unicode-normalized** (NFC), because FLEx's own export is not internally consistent: it normalizes to NFC on the way out, but a few writes bypass that step and emit the NFD it holds in memory. A `grammatical-info` or `lexical-relation` range-element `id` can therefore be NFD while its labels, the `parent` attribute on that same element, and the `.lift` value referring to it are all NFC. +- **Range id comparisons are Unicode-normalized** (NFC), because FLEx's own export is not internally consistent: it normalizes to NFC on the way out, but a few writes bypass that step and emit the NFD it holds in memory. A `grammatical-info` or `lexical-relation` range-element `id` can therefore be NFD while its labels, the `parent` attribute on that same element, and the `.lift` value referring to it are all NFC. References that resolve only after normalizing are reported as `normalization-mismatch` warnings — a check with no C# counterpart — so the encoding split stays visible to anyone whose own comparisons are exact. sil-lift also validates the `.lift-ranges` companions of a loaded lexicon against a schema for standalone ranges documents (vendored from `lift-standard` alongside the base LIFT grammar) — every tracked external ranges file is checked whenever the `.lift` is validated — with no such schema (or check) in the C# world. (There is no entry point for validating a `.lift-ranges` file on its own, detached from a `.lift`.) diff --git a/docs/en/guides/lift-export-interop.md b/docs/en/guides/lift-export-interop.md index 2394d9a..a85d625 100644 --- a/docs/en/guides/lift-export-interop.md +++ b/docs/en/guides/lift-export-interop.md @@ -92,7 +92,7 @@ The companion carries each range's full definition. Values are `` ``` -An entry then refers to a value by id: a sense's part of speech is ``, and a semantic domain is ``. `sil-lift validate` warns (`undefined-range-value`) when a value isn't defined in its range and errors (`range-parent`) when a `parent` isn't a sibling id — so emit the ranges your data actually uses. Both comparisons are NFC-normalized, so an id and the value or `parent` referring to it may differ in Unicode normalization; still, write one consistent normalization if you can. See also [Ranges and media](folder-media.md). +An entry then refers to a value by id: a sense's part of speech is ``, and a semantic domain is ``. `sil-lift validate` warns (`undefined-range-value`) when a value isn't defined in its range and errors (`range-parent`) when a `parent` isn't a sibling id — so emit the ranges your data actually uses. Both comparisons are NFC-normalized, so an id and the value or `parent` referring to it may differ in Unicode normalization — that difference is a `normalization-mismatch` warning rather than an error, but write one consistent normalization if you can: consumers that compare raw strings will not resolve those references. See also [Ranges and media](folder-media.md). If you build the export in Python, `Lexicon.add_ranges_file()`, `RangesFile.add_range()`, and `Range.add_element()` construct the companion and add the header references for you; `open_writer(..., ranges=...)` does the same on the streaming path. diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index 1ead92e..6163ba8 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -24,7 +24,7 @@ Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `messag 1. **RELAX NG** against the LIFT 0.13 grammar (vendored from lift-standard — a byte-identical copy committed into this package). 2. **Ranges schema** — this project's `lift-ranges-0.13.rng` — over every tracked `.lift-ranges` companion. -3. **Semantic checks** the grammar cannot express: `duplicate-guid`, `dangling-ref`, `range-parent`, `undefined-range-value`, `duplicate-form-lang`, `missing-media`. +3. **Semantic checks** the grammar cannot express: `duplicate-guid`, `dangling-ref`, `range-parent`, `undefined-range-value`, `normalization-mismatch`, `duplicate-form-lang`, `missing-media`. ## Real-world FieldWorks (FLEx) output @@ -32,5 +32,5 @@ FieldWorks systematically writes some content that strict tooling rejects. Here - `file://C:/...` hrefs (invalid URIs) are reported as **warnings** (`uri-not-rfc`), not schema errors — the C# validator never rejected them. - Legally interleaved children (e.g. `field, note, field, note` in a sense) are **not** flagged, working around a false positive in libxml2. -- Range values and `parent` links are compared under Unicode NFC normalization — FLEx normalizes to NFC on export but a few writes bypass that step, so a range-element `id` can be NFD while its labels, its own `parent` attribute, and the `.lift` value referring to it are NFC. Without normalization those spellings compare unequal and a sound export looks broken. +- Range values and `parent` links are compared under Unicode NFC normalization — FLEx normalizes to NFC on export but a few writes bypass that step, so a range-element `id` can be NFD while its labels, its own `parent` attribute, and the `.lift` value referring to it are NFC. Without normalization those spellings compare unequal and a sound export looks broken. What did only match after normalizing is reported as a `normalization-mismatch` **warning**, once per range-element id however many references differ, addressed to the file that defines the id: the data is sound here, but a consumer comparing raw strings — including a Send/Receive merge — will not resolve those references. `sil-lift` never rewrites the ids; the file keeps the spellings it came with. - FLEx's `trait`/`field` extensions inside `range-element` **are** reported (schema errors against the ranges schema): they are genuine spec deviations. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 65fe125..91265f0 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -62,8 +62,8 @@ class Problem: level: Literal["error", "warning"] code: str # "schema", "duplicate-guid", "dangling-ref", "range-parent", - # "undefined-range-value", "duplicate-form-lang", "missing-media", - # "uri-not-rfc", "dangling-ranges-href", "missing-id" + # "undefined-range-value", "normalization-mismatch", "duplicate-form-lang", + # "missing-media", "uri-not-rfc", "dangling-ranges-href", "missing-id" message: str file: Path | None = None entry_id: str | None = None @@ -426,12 +426,23 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: def nfc(value: str) -> str: return unicodedata.normalize("NFC", value) + # Every name that resolved only because of that normalization, keyed by the + # range-element it resolved to: (range id, id as written) -> a differing + # reference. Reported once per element however many references differ -- + # in a real 3507-entry export that is 6 findings rather than 82. + mismatched: dict[tuple[str, str], str] = {} + # Range integrity over the merged view (inline + companions). all_ranges = lexicon.all_ranges() for range_ in all_ranges.values(): - element_ids = {nfc(element.id) for element in range_.elements} + element_ids: dict[str, str] = {} # NFC id -> id as written + for element in range_.elements: + element_ids.setdefault(nfc(element.id), element.id) for element in range_.elements: - if element.parent and nfc(element.parent) not in element_ids: + if not element.parent: + continue + target = element_ids.get(nfc(element.parent)) + if target is None: yield Problem( "error", "range-parent", @@ -439,6 +450,8 @@ def nfc(value: str) -> str: f"parent {element.parent!r} which is not a sibling id", file=file, ) + elif target != element.parent: + mismatched.setdefault((range_.id, target), element.parent) # Header references (relative) that resolve to no companion. # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve @@ -473,24 +486,31 @@ def nfc(value: str) -> str: # (_iter_traits/_iter_grammatical_infos walk the whole entry). # Only ranges that actually enumerate elements can confirm a value; empty # values skipped (FLEx writes them). - def defined(range_id: str) -> set[str] | None: + def defined(range_id: str) -> dict[str, str] | None: range_: Range | None = all_ranges.get(range_id) if range_ is None or not range_.elements: return None - return {nfc(element.id) for element in range_.elements} + ids: dict[str, str] = {} # NFC id -> id as written + for element in range_.elements: + ids.setdefault(nfc(element.id), element.id) + return ids grammatical_values = defined("grammatical-info") for index, entry in enumerate(lexicon.entries): - checks: list[tuple[str, str, set[str]]] = [] # (label, value, allowed) + # (label, range id, value, allowed ids) + checks: list[tuple[str, str, str, dict[str, str]]] = [] for info in _iter_grammatical_infos(entry): if info.value and grammatical_values is not None: - checks.append(("grammatical-info", info.value, grammatical_values)) + checks.append( + ("grammatical-info", "grammatical-info", info.value, grammatical_values) + ) for trait in _iter_traits(entry): allowed = defined(trait.name) if allowed is not None and trait.value: - checks.append((f"trait {trait.name!r}", trait.value, allowed)) - for label, value, allowed in checks: - if nfc(value) not in allowed: + checks.append((f"trait {trait.name!r}", trait.name, trait.value, allowed)) + for label, range_id, value, allowed in checks: + target = allowed.get(nfc(value)) + if target is None: yield Problem( "warning", "undefined-range-value", @@ -500,6 +520,26 @@ def defined(range_id: str) -> set[str] | None: guid=entry.guid, line=at(index), ) + elif target != value: + mismatched.setdefault((range_id, target), value) + + # The two spellings render identically, so name them by code point rather + # than with the !r other messages use; the file each id lives in is the + # companion that defines it, not necessarily the document being validated. + ranges_paths = { + range_.id: ranges_file.path + for ranges_file in lexicon.ranges_files.values() + for range_ in ranges_file.ranges + if all_ranges.get(range_.id) is range_ # an inline range wins the merge + } + for (range_id, element_id), reference in sorted(mismatched.items()): + yield Problem( + "warning", + "normalization-mismatch", + f"range {range_id!r}: range-element id {element_id!a} is referenced " + f"as {reference!a}; they match only under Unicode NFC normalization", + file=ranges_paths.get(range_id) or file, + ) # Missing media files. for media_ref in lexicon.missing_media(): diff --git a/tests/corpus/PROVENANCE.md b/tests/corpus/PROVENANCE.md index 0804848..6274d9d 100644 --- a/tests/corpus/PROVENANCE.md +++ b/tests/corpus/PROVENANCE.md @@ -174,6 +174,11 @@ extra undefined part of speech that are artifacts of the encoding, not defects. The one genuine `undefined-range-value` left in Sango is the part of speech `prenom`, which no range defines in any normalization. +What did match only after normalizing is still reported, as a +`normalization-mismatch` warning per range-element id: 6 in Sango, covering 2 +parent links and 80 grammatical-info values, and none in the other fixtures. +`negative/nfd-range-ids.lift` is the hand-authored version of the same shape. + ## generated/ — synthetic large files (not committed) Produced by `tests/tools/generate_large.py` for streaming/perf tests; @@ -185,8 +190,11 @@ Each file carries an XML comment documenting its defect and the expected Problem code: `duplicate-guid`, `dangling-ref`, `range-parent`, `undefined-range-value` (2 warnings \+ a clean control entry), `duplicate-form-lang` (the Schematron-only rule), `schema-invalid` -(structural), `missing-media/` (a folder fixture), and `flex-quirks` -(URI quirks that must yield warnings, never schema errors). +(structural), `missing-media/` (a folder fixture), `flex-quirks` +(URI quirks that must yield warnings, never schema errors), and +`nfd-range-ids` (a `.lift` \+ `.lift-ranges` pair carrying FLEx's +normalization asymmetry: NFD ids, NFC references, and one parent that +dangles in every normalization). `schema-invalid.lift` and `flex-quirks.lift` are raw-RNG-invalid (the latter only under libxml2's anyURI check) and appear in the corpus test's expected-invalid list. diff --git a/tests/corpus/negative/nfd-range-ids.lift b/tests/corpus/negative/nfd-range-ids.lift new file mode 100644 index 0000000..a9b3e47 --- /dev/null +++ b/tests/corpus/negative/nfd-range-ids.lift @@ -0,0 +1,20 @@ + + + +
+ + + +
+ +
de
 + + + diff --git a/tests/corpus/negative/nfd-range-ids.lift-ranges b/tests/corpus/negative/nfd-range-ids.lift-ranges new file mode 100644 index 0000000..ac0a6f1 --- /dev/null +++ b/tests/corpus/negative/nfd-range-ids.lift-ranges @@ -0,0 +1,14 @@ + + + + + + + + + + + + diff --git a/tests/test_validate.py b/tests/test_validate.py index 8468ca5..12e991c 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -62,6 +62,25 @@ def test_range_parent_tolerates_flex_normalization_asymmetry() -> None: assert [p for p in lexicon.iter_problems() if p.code == "range-parent"] == [] +def test_nfd_ids_warn_once_and_still_flag_the_real_dangling_parent(tmp_path: Path) -> None: + path = NEGATIVE_DIR / "nfd-range-ids.lift" + problems = problems_for(path) + assert codes(problems) == {("error", "range-parent"), ("warning", "normalization-mismatch")} + (dangling,) = [p for p in problems if p.code == "range-parent"] + assert "Preposicao" in dangling.message + # Two parent attributes and one grammatical-info value resolve to the same + # id: one finding, against the companion the id lives in, naming both + # spellings by code point (they are indistinguishable rendered). + (mismatch,) = [p for p in problems if p.code == "normalization-mismatch"] + assert mismatch.file is not None and mismatch.file.suffix == ".lift-ranges" + assert "Preposic\\u0327a\\u0303o" in mismatch.message + assert "Preposi\\xe7\\xe3o" in mismatch.message + # Normalization belongs to the comparison only: the mixed forms survive. + sil_lift.Lexicon.load(path).save(tmp_path / path.name) + for name in (path.name, "nfd-range-ids.lift-ranges"): + assert (tmp_path / name).read_bytes() == (NEGATIVE_DIR / name).read_bytes(), name + + def test_undefined_range_values_are_warnings() -> None: problems = problems_for(NEGATIVE_DIR / "undefined-range-value.lift") assert codes(problems) == {("warning", "undefined-range-value")} @@ -223,9 +242,11 @@ def test_sango_real_defects_are_found() -> None: # One undefined POS value ('prenom') in the real export; NFC-normalization # keeps the count at 1. No range-parent finding: the two parent links that # differ from their target's spelling are FLEx's NFD ids under an NFC - # parent (see PROVENANCE.md), not dangling references. + # parent (see PROVENANCE.md), not dangling references. Those two and 80 + # NFC grammatical-info values resolve to 6 NFD ids, one warning each. assert by_code.get("range-parent") is None assert by_code.get("undefined-range-value") == 1 + assert by_code.get("normalization-mismatch") == 6 assert by_code.get("schema", 0) > 0 # companion's trait/field extensions # 37 real duplicate guids: FLEx aliases its POS list under both # "grammatical-info" and "from-part-of-speech" (same range-element guids From 056e1c5660b92394c14676081dd05547254427e4 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 13 Aug 2026 08:56:12 -0400 Subject: [PATCH 4/9] Resolve a range name by its exact spelling before its NFC form Keying each range's lookup by both the ids as written and their NFC forms, with an exact id mapping to itself, makes a reference that matches a sibling exactly stay an exact match. Before, the first spelling of an NFC-equal pair won, so a range carrying one name in both normalizations -- distinct ids as strings, and FLEx normalizes some writes and not others -- would resolve the second element's references to the first and report them as mismatched. The lookup is now built once per range and read by the value checks, which were rebuilding it per trait: thousands of traits over a few ranges of hundreds of elements each. The sango fixture's semantic pass drops from 1.54s to 0.84s. The module docstring's semantic checks fed the API reference as one sentence running nine checks and their parentheticals together. They are a bullet each now, named by the Problem code the check reports, matching how layer 1 lays out its deviations; the comments around the normalization helpers drop the mechanism they were re-telling from that list. Also escapes the list markers in the new PROVENANCE sentence, per that file's convention for hard-wrapped prose, and drops a Markdown escape that had no business in an XML comment. Co-authored-by: Claude Opus 5 (1M context) --- docs/en/guides/validate.md | 2 +- src/sil_lift/_validate.py | 92 ++++++++++++++---------- tests/corpus/PROVENANCE.md | 4 +- tests/corpus/negative/nfd-range-ids.lift | 2 +- tests/test_validate.py | 37 ++++++++-- 5 files changed, 88 insertions(+), 49 deletions(-) diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index 6163ba8..ab79a26 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -18,7 +18,7 @@ lex = sil_lift.load("dictionary.lift") problems = list(lex.iter_problems()) ``` -Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `message`, and an address: `file`, `entry_id`, `guid`, `line`. +Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `message`, and an address: `file`, plus `entry_id`, `guid`, and `line` when the finding belongs to one entry. Findings about a document rather than an entry — `range-parent`, `normalization-mismatch`, `dangling-ranges-href` — carry the file alone, and the unset fields are `None` (`null` in `--format json`, where every key is always present). ## The layers diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 91265f0..556a994 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -20,14 +20,24 @@ left untouched. 2. The vendored ``lift-ranges-0.13.rng`` over each tracked ``.lift-ranges`` companion. -3. Semantic checks the grammar cannot express: duplicate GUIDs (entries, and - ranges/range-elements within their own document), dangling ``relation/@ref`` - and ``variant/@ref``, ``range-element/@parent`` integrity, undefined range - values (every grammatical-info and range-keyed trait anywhere in the entry, - however deeply nested), duplicate form languages (the RNG's Schematron - rule, which lxml ignores), missing media files, header ``range/@href`` - references that resolve to no companion, and — opt-in — entries/senses - missing a stable id. +3. Semantic checks that the grammar cannot express, one ``Problem`` code each: + + - ``duplicate-guid`` — a guid reused, among entries or among the + ranges/range-elements of one document. + - ``dangling-ref`` — a ``relation/@ref`` or ``variant/@ref`` matching no + entry or sense. + - ``range-parent`` — a ``range-element/@parent`` no sibling id defines. + - ``undefined-range-value`` — a grammatical-info or range-keyed trait value + the range does not list, checked however deeply nested in the entry. + - ``normalization-mismatch`` — a name that reaches its range-element id + only under Unicode NFC normalization, because FLEx writes some ids in + NFD and the references to them in NFC. + - ``duplicate-form-lang`` — two forms in one multitext sharing a language + (the RNG's Schematron rule, which lxml ignores). + - ``missing-media`` — a referenced audio or picture file not on disk. + - ``dangling-ranges-href`` — a header ``range/@href`` resolving to no + companion file. + - ``missing-id`` — opt-in: an entry without a guid, a sense without an id. """ from __future__ import annotations @@ -417,31 +427,46 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: line=at(index), ) - # Comparisons of a range value or a parent link against a range-element id - # are NFC-normalized. FLEx normalizes strings to NFC on export, but a few - # writes bypass its normalizing helper and emit the NFD it holds in memory - # -- among them the grammatical-info and lexical-relation range-element - # ids, whose labels and own parent attribute are normalized. So an id can - # be NFD while the .lift value and the parent pointing at it are NFC. + # FLEx writes some range-element ids in NFD but every reference to them in + # NFC, so resolving a name to an id has to compare both forms. def nfc(value: str) -> str: return unicodedata.normalize("NFC", value) - # Every name that resolved only because of that normalization, keyed by the - # range-element it resolved to: (range id, id as written) -> a differing - # reference. Reported once per element however many references differ -- - # in a real 3507-entry export that is 6 findings rather than 82. + # Names that resolved only after normalizing, keyed by the range-element + # they resolved to: (range id, id as written) -> one differing reference. + # Warning per id rather than per reference keeps a real export to 6 + # findings instead of 82. mismatched: dict[tuple[str, str], str] = {} - # Range integrity over the merged view (inline + companions). + def id_lookup(range_: Range) -> dict[str, str]: + """What each spelling of this range's ids resolves to: ids as written, + keyed by both themselves and their NFC form. An id present as written + maps to itself, so a range holding two normalizations of one name + resolves an exact reference to the sibling that matches it exactly. + """ + ids: dict[str, str] = {} + for element in range_.elements: + ids.setdefault(nfc(element.id), element.id) + for element in range_.elements: + ids[element.id] = element.id + return ids + + def resolve(name: str, ids: dict[str, str]) -> str | None: + """The id ``name`` names, exactly or after normalizing; None if no id.""" + exact = ids.get(name) + return exact if exact is not None else ids.get(nfc(name)) + + # Range integrity over the merged view (inline + companions). Each range's + # lookup is kept for the value checks below, which would otherwise rebuild + # it per trait. all_ranges = lexicon.all_ranges() + lookups: dict[str, dict[str, str]] = {} for range_ in all_ranges.values(): - element_ids: dict[str, str] = {} # NFC id -> id as written - for element in range_.elements: - element_ids.setdefault(nfc(element.id), element.id) + lookups[range_.id] = element_ids = id_lookup(range_) for element in range_.elements: if not element.parent: continue - target = element_ids.get(nfc(element.parent)) + target = resolve(element.parent, element_ids) if target is None: yield Problem( "error", @@ -478,22 +503,11 @@ def nfc(value: str) -> str: file=lexicon.path, ) - # Undefined range values: every grammatical-info (sense, reversal, and - # reversal main chains) against the grammatical-info range; every trait - # anywhere in the entry whose name matches a known range — not just entry- - # and sense-direct ones, since real FLEx exports nest traits like - # is-primary/complex-form-type inside and morph-type inside - # (_iter_traits/_iter_grammatical_infos walk the whole entry). - # Only ranges that actually enumerate elements can confirm a value; empty - # values skipped (FLEx writes them). + # Undefined range values: every grammatical-info and every trait whose + # name matches a known range, anywhere in the entry. Empty values are + # skipped -- FLEx writes them. def defined(range_id: str) -> dict[str, str] | None: - range_: Range | None = all_ranges.get(range_id) - if range_ is None or not range_.elements: - return None - ids: dict[str, str] = {} # NFC id -> id as written - for element in range_.elements: - ids.setdefault(nfc(element.id), element.id) - return ids + return lookups.get(range_id) or None # absent, or enumerating nothing grammatical_values = defined("grammatical-info") for index, entry in enumerate(lexicon.entries): @@ -509,7 +523,7 @@ def defined(range_id: str) -> dict[str, str] | None: if allowed is not None and trait.value: checks.append((f"trait {trait.name!r}", trait.name, trait.value, allowed)) for label, range_id, value, allowed in checks: - target = allowed.get(nfc(value)) + target = resolve(value, allowed) if target is None: yield Problem( "warning", diff --git a/tests/corpus/PROVENANCE.md b/tests/corpus/PROVENANCE.md index 6274d9d..18ce267 100644 --- a/tests/corpus/PROVENANCE.md +++ b/tests/corpus/PROVENANCE.md @@ -166,8 +166,8 @@ elements, the `semantic-domain-ddp4`/`translation-type`/`usage-type` ids, and every value in `large/sango/sango.lift` — is NFC. So one name appears in two spellings within a single element: the part of -speech whose id decomposes as `Comple` + U+0301 + `ments` is named by a child -element's `parent` attribute with the precomposed U+00E9 instead. Both +speech whose id decomposes as `Comple` \+ U+0301 \+ `ments` is named by a +child element's `parent` attribute with the precomposed U+00E9 instead. Both `range-parent` and `undefined-range-value` therefore compare under NFC normalization; without it, this fixture reports two dangling parents and an extra undefined part of speech that are artifacts of the encoding, not defects. diff --git a/tests/corpus/negative/nfd-range-ids.lift b/tests/corpus/negative/nfd-range-ids.lift index a9b3e47..a933045 100644 --- a/tests/corpus/negative/nfd-range-ids.lift +++ b/tests/corpus/negative/nfd-range-ids.lift @@ -5,7 +5,7 @@ grammatical-info value - is NFC, so three references resolve to one id only after NFC normalization. Expected: warning normalization-mismatch (once, for the id they resolve - to) \+ error range-parent for the one parent that no sibling defines in + to) + error range-parent for the one parent that no sibling defines in any normalization. Schema-valid otherwise. -->
diff --git a/tests/test_validate.py b/tests/test_validate.py index 12e991c..b2a5c2c 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,3 +1,4 @@ +import unicodedata from pathlib import Path import pytest @@ -17,6 +18,11 @@ def codes(problems: list[Problem]) -> set[tuple[str, str]]: return {(p.level, p.code) for p in problems} +def nfd(value: str) -> str: + """The decomposed spelling, as FLEx leaves the ids that skip its normalizer.""" + return unicodedata.normalize("NFD", value) + + def test_duplicate_guid_is_error_with_addressing() -> None: problems = problems_for(NEGATIVE_DIR / "duplicate-guid.lift") (problem,) = problems @@ -46,15 +52,12 @@ def test_range_parent_tolerates_flex_normalization_asymmetry() -> None: # its NFD in-memory string but normalizes the parent attribute to NFC (see # PROVENANCE.md), so the two spellings of one name differ within a single # element. The parent link is sound; only the encoding differs. + parent_name = "Compl\u00e9ments" # NFC, the form FLEx writes a parent in lexicon = sil_lift.Lexicon() ranges = sil_lift.RangesFile() - # "Compl\u00e9ments" spelled two ways: the id decomposed (e + U+0301), - # as FLEx writes range-element ids, and the parent composed. - nfd_id = "Comple\u0301ments" - nfc_parent = "Compl\u00e9ments" range_ = ranges.add_range("grammatical-info") - range_.add_element(nfd_id) - range_.add_element("Comple\u0301ment du lieu", parent=nfc_parent) + range_.add_element(nfd(parent_name)) # ids skip FLEx's normalizer + range_.add_element(nfd("Compl\u00e9ment du lieu"), parent=parent_name) lexicon.add_ranges_file(ranges, href="x.lift-ranges") entry = sil_lift.Entry(id="e1", guid="bbbbbbbb-1111-4444-8888-bbbbbbbbbbbb") entry.lexical_unit["en"] = "e1" @@ -62,6 +65,28 @@ def test_range_parent_tolerates_flex_normalization_asymmetry() -> None: assert [p for p in lexicon.iter_problems() if p.code == "range-parent"] == [] +def test_normalization_mismatch_prefers_an_exactly_matching_sibling() -> None: + # A range may hold both spellings of one name -- ids are unique as strings, + # and FLEx normalizes some writes and not others. Every reference here + # matches a sibling exactly, so none of them needed normalizing. + name = "Preposi\u00e7\u00e3o" + lexicon = sil_lift.Lexicon() + ranges = sil_lift.RangesFile() + range_ = ranges.add_range("grammatical-info") + range_.add_element(nfd(name)) + range_.add_element(name) + range_.add_element("Associativo", parent=nfd(name)) + range_.add_element("Prepositional phrase", parent=name) + lexicon.add_ranges_file(ranges, href="x.lift-ranges") + entry = sil_lift.Entry(id="e1", guid="cccccccc-1111-4444-8888-cccccccccccc") + entry.lexical_unit["en"] = "e1" + entry.senses.append( + sil_lift.Sense(id="s1", grammatical_info=sil_lift.GrammaticalInfo(nfd(name))) + ) + lexicon.entries.append(entry) + assert list(lexicon.iter_problems()) == [] + + def test_nfd_ids_warn_once_and_still_flag_the_real_dangling_parent(tmp_path: Path) -> None: path = NEGATIVE_DIR / "nfd-range-ids.lift" problems = problems_for(path) From 6acaf712162e13fd94ed6e1d5d7e9ab0304f1492 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 13 Aug 2026 11:40:30 -0400 Subject: [PATCH 5/9] Address a range finding to the companion that defines the range The file the mismatch warning reports against is worked out for that warning alone, three lines under a range-parent error about the very same range-element that still reports against the .lift. Both belong to the document the element lives in, so the lookup moves above the check that walks the ranges and answers for both. Co-authored-by: Claude Opus 5 (1M context) --- docs/en/guides/validate.md | 2 +- src/sil_lift/_validate.py | 25 +++++++++++++------------ tests/test_validate.py | 2 ++ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index ab79a26..e1d913a 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -18,7 +18,7 @@ lex = sil_lift.load("dictionary.lift") problems = list(lex.iter_problems()) ``` -Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `message`, and an address: `file`, plus `entry_id`, `guid`, and `line` when the finding belongs to one entry. Findings about a document rather than an entry — `range-parent`, `normalization-mismatch`, `dangling-ranges-href` — carry the file alone, and the unset fields are `None` (`null` in `--format json`, where every key is always present). +Each `Problem` carries `level` (`"error"`/`"warning"`), a stable `code`, `message`, and an address: `file`, plus `entry_id`, `guid`, and `line` when the finding belongs to one entry. Findings about a document rather than an entry — `range-parent`, `normalization-mismatch`, `dangling-ranges-href` — carry the file alone (for a finding about a range, the `.lift-ranges` companion that defines it), and the unset fields are `None` (`null` in `--format json`, where every key is always present). ## The layers diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 556a994..d11f512 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -456,10 +456,18 @@ def resolve(name: str, ids: dict[str, str]) -> str | None: exact = ids.get(name) return exact if exact is not None else ids.get(nfc(name)) - # Range integrity over the merged view (inline + companions). Each range's - # lookup is kept for the value checks below, which would otherwise rebuild - # it per trait. + # Range integrity over the merged view (inline + companions). A finding + # about a range is addressed to the companion that defines it, which is not + # necessarily the document being validated. all_ranges = lexicon.all_ranges() + ranges_paths = { + range_.id: ranges_file.path + for ranges_file in lexicon.ranges_files.values() + for range_ in ranges_file.ranges + if all_ranges.get(range_.id) is range_ # an inline range wins the merge + } + # Each range's lookup is kept for the value checks below, which would + # otherwise rebuild it per trait. lookups: dict[str, dict[str, str]] = {} for range_ in all_ranges.values(): lookups[range_.id] = element_ids = id_lookup(range_) @@ -473,7 +481,7 @@ def resolve(name: str, ids: dict[str, str]) -> str | None: "range-parent", f"range {range_.id!r}: range-element {element.id!r} has " f"parent {element.parent!r} which is not a sibling id", - file=file, + file=ranges_paths.get(range_.id) or file, ) elif target != element.parent: mismatched.setdefault((range_.id, target), element.parent) @@ -538,14 +546,7 @@ def defined(range_id: str) -> dict[str, str] | None: mismatched.setdefault((range_id, target), value) # The two spellings render identically, so name them by code point rather - # than with the !r other messages use; the file each id lives in is the - # companion that defines it, not necessarily the document being validated. - ranges_paths = { - range_.id: ranges_file.path - for ranges_file in lexicon.ranges_files.values() - for range_ in ranges_file.ranges - if all_ranges.get(range_.id) is range_ # an inline range wins the merge - } + # than with the !r other messages use. for (range_id, element_id), reference in sorted(mismatched.items()): yield Problem( "warning", diff --git a/tests/test_validate.py b/tests/test_validate.py index b2a5c2c..47c4396 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -93,6 +93,8 @@ def test_nfd_ids_warn_once_and_still_flag_the_real_dangling_parent(tmp_path: Pat assert codes(problems) == {("error", "range-parent"), ("warning", "normalization-mismatch")} (dangling,) = [p for p in problems if p.code == "range-parent"] assert "Preposicao" in dangling.message + # The element is in the companion, so that is where the error is addressed. + assert dangling.file is not None and dangling.file.suffix == ".lift-ranges" # Two parent attributes and one grammatical-info value resolve to the same # id: one finding, against the companion the id lives in, naming both # spellings by code point (they are indistinguishable rendered). From b7fa4964b1a3e602dd0bb1ea92d477777544030c Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 13 Aug 2026 11:42:03 -0400 Subject: [PATCH 6/9] Resolve the name keying a range under NFC too The comparison of a name against a range-element id was normalized while the comparison that reaches the range in the first place -- a trait name, a header range id -- stayed exact. A range whose id FLEx wrote in NFD is therefore never found by the NFC trait names keyed on it, and its values go unchecked: an unresolved name is indistinguishable from a trait no range defines, which is silently accepted, so the whole range drops out of validation with nothing to show for it. Both lookups are now built by one helper over a collection of ids, and a range reached only by normalizing is a normalization-mismatch warning like an element is. The value checks name their range through that resolution, which also drops the allowed-ids set from the per-check tuples: they carry the resolved range id, and the lookup kept for it answers by that key. The validate guide's normalization entry, five sentences beside four one-liners once this landed, keeps the mechanism on the bullet and gives the rest a sub-bullet each. It moves last as well, so the list runs schema deviations first and the semantic check after them, as the layers above are ordered. The Sango notes counted its 6 warnings as 6 ids. They are 5: FLEx aliases the part-of-speech list under two range ids, and the id under both is warned about under each, since each range defines its own element. Co-authored-by: Claude Opus 5 (1M context) --- CHANGELOG.md | 12 ++-- docs/en/guides/lift-export-interop.md | 2 +- docs/en/guides/validate.md | 5 +- src/sil_lift/_validate.py | 100 ++++++++++++++------------ tests/corpus/PROVENANCE.md | 5 +- tests/test_validate.py | 30 +++++++- 6 files changed, 98 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2784957..6afaf22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,12 +75,12 @@ releases may contain breaking changes. undefined-range-value (every grammatical-info and range-keyed trait reachable from an entry, however nested), normalization-mismatch, duplicate-form-lang, missing-media, dangling-ranges-href, and (opt-in via - `require_ids`) missing-id. Both checks that resolve a name against a - range-element id compare under NFC normalization: FLEx can write an id in - NFD while the value and the `parent` attribute referring to it are NFC. - What matched only after normalizing is reported as a normalization-mismatch - warning, once per range-element id, so the encoding split stays visible to - consumers that compare raw strings. + `require_ids`) missing-id. Names are resolved against range and + range-element ids under NFC normalization: FLEx can write an id in NFD while + the trait name, value, or `parent` referring to it is NFC. What matched only + after normalizing is reported as a normalization-mismatch warning, once per + id, so the encoding split stays visible to consumers that compare raw + strings. - Canonical sort: `Lexicon.sort()` / `RangesFile.sort()` (entries by case-folded guid/id, ranges/range-elements by id, field definitions by tag; informed by the C# LiftSorter, locale-independent) and diff --git a/docs/en/guides/lift-export-interop.md b/docs/en/guides/lift-export-interop.md index a85d625..046a8e8 100644 --- a/docs/en/guides/lift-export-interop.md +++ b/docs/en/guides/lift-export-interop.md @@ -92,7 +92,7 @@ The companion carries each range's full definition. Values are `` ``` -An entry then refers to a value by id: a sense's part of speech is ``, and a semantic domain is ``. `sil-lift validate` warns (`undefined-range-value`) when a value isn't defined in its range and errors (`range-parent`) when a `parent` isn't a sibling id — so emit the ranges your data actually uses. Both comparisons are NFC-normalized, so an id and the value or `parent` referring to it may differ in Unicode normalization — that difference is a `normalization-mismatch` warning rather than an error, but write one consistent normalization if you can: consumers that compare raw strings will not resolve those references. See also [Ranges and media](folder-media.md). +An entry then refers to a value by id: a sense's part of speech is ``, and a semantic domain is ``. `sil-lift validate` warns (`undefined-range-value`) when a value isn't defined in its range and errors (`range-parent`) when a `parent` isn't a sibling id — so emit the ranges your data actually uses. Those comparisons are NFC-normalized, so an id and the value or `parent` referring to it may differ in Unicode normalization — that difference is a `normalization-mismatch` warning rather than an error, but write one consistent normalization if you can: consumers that compare raw strings will not resolve those references. See also [Ranges and media](folder-media.md). If you build the export in Python, `Lexicon.add_ranges_file()`, `RangesFile.add_range()`, and `Range.add_element()` construct the companion and add the header references for you; `open_writer(..., ranges=...)` does the same on the streaming path. diff --git a/docs/en/guides/validate.md b/docs/en/guides/validate.md index e1d913a..1fb66f1 100644 --- a/docs/en/guides/validate.md +++ b/docs/en/guides/validate.md @@ -32,5 +32,8 @@ FieldWorks systematically writes some content that strict tooling rejects. Here - `file://C:/...` hrefs (invalid URIs) are reported as **warnings** (`uri-not-rfc`), not schema errors — the C# validator never rejected them. - Legally interleaved children (e.g. `field, note, field, note` in a sense) are **not** flagged, working around a false positive in libxml2. -- Range values and `parent` links are compared under Unicode NFC normalization — FLEx normalizes to NFC on export but a few writes bypass that step, so a range-element `id` can be NFD while its labels, its own `parent` attribute, and the `.lift` value referring to it are NFC. Without normalization those spellings compare unequal and a sound export looks broken. What did only match after normalizing is reported as a `normalization-mismatch` **warning**, once per range-element id however many references differ, addressed to the file that defines the id: the data is sound here, but a consumer comparing raw strings — including a Send/Receive merge — will not resolve those references. `sil-lift` never rewrites the ids; the file keeps the spellings it came with. - FLEx's `trait`/`field` extensions inside `range-element` **are** reported (schema errors against the ranges schema): they are genuine spec deviations. +- Names are resolved against range and range-element `id`s under Unicode **NFC normalization** — `parent` links, range values, and the `trait` name or header `range` id that keys a range. FLEx normalizes to NFC on export, but a few writes bypass that step, so a range-element `id` can be NFD while its labels, its own `parent`, and the `.lift` values naming it are NFC. + - Compared exactly, a sound export looks broken — and a range whose `id` is spelled the other way goes unchecked entirely, since a trait name that reaches no range is silently accepted. + - A name that matched only after normalizing is reported as a `normalization-mismatch` **warning**, once per id however many references differ, addressed to the file that defines it. The data is sound, but a consumer comparing raw strings — a Send/Receive merge, say — will not resolve those references. + - The ids are never rewritten: the file keeps the spellings it came with. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index d11f512..4400a99 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -29,9 +29,9 @@ - ``range-parent`` — a ``range-element/@parent`` no sibling id defines. - ``undefined-range-value`` — a grammatical-info or range-keyed trait value the range does not list, checked however deeply nested in the entry. - - ``normalization-mismatch`` — a name that reaches its range-element id - only under Unicode NFC normalization, because FLEx writes some ids in - NFD and the references to them in NFC. + - ``normalization-mismatch`` — a name that reaches the range or + range-element id it refers to only under Unicode NFC normalization, + because FLEx writes some ids in NFD and the references to them in NFC. - ``duplicate-form-lang`` — two forms in one multitext sharing a language (the RNG's Schematron rule, which lxml ignores). - ``missing-media`` — a referenced audio or picture file not on disk. @@ -55,7 +55,7 @@ if TYPE_CHECKING: import os - from collections.abc import Iterator + from collections.abc import Collection, Iterator from ._header import Range from ._model import Entry, Sense @@ -427,29 +427,29 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: line=at(index), ) - # FLEx writes some range-element ids in NFD but every reference to them in - # NFC, so resolving a name to an id has to compare both forms. + # FLEx writes some ids in NFD but every reference to them in NFC, so + # resolving a name to an id has to compare both forms. def nfc(value: str) -> str: return unicodedata.normalize("NFC", value) - # Names that resolved only after normalizing, keyed by the range-element - # they resolved to: (range id, id as written) -> one differing reference. + # Names that resolved only after normalizing, keyed by what they resolved + # to: (range id, what the id is, id as written) -> one differing reference. # Warning per id rather than per reference keeps a real export to 6 # findings instead of 82. - mismatched: dict[tuple[str, str], str] = {} + mismatched: dict[tuple[str, str, str], str] = {} - def id_lookup(range_: Range) -> dict[str, str]: - """What each spelling of this range's ids resolves to: ids as written, + def id_lookup(ids: Collection[str]) -> dict[str, str]: + """What each spelling of these ids resolves to: the ids as written, keyed by both themselves and their NFC form. An id present as written - maps to itself, so a range holding two normalizations of one name - resolves an exact reference to the sibling that matches it exactly. + maps to itself, so a collection holding two normalizations of one name + resolves an exact reference to the one that matches it exactly. """ - ids: dict[str, str] = {} - for element in range_.elements: - ids.setdefault(nfc(element.id), element.id) - for element in range_.elements: - ids[element.id] = element.id - return ids + lookup: dict[str, str] = {} + for id_ in ids: + lookup.setdefault(nfc(id_), id_) + for id_ in ids: + lookup[id_] = id_ + return lookup def resolve(name: str, ids: dict[str, str]) -> str | None: """The id ``name`` names, exactly or after normalizing; None if no id.""" @@ -466,11 +466,14 @@ def resolve(name: str, ids: dict[str, str]) -> str | None: for range_ in ranges_file.ranges if all_ranges.get(range_.id) is range_ # an inline range wins the merge } - # Each range's lookup is kept for the value checks below, which would - # otherwise rebuild it per trait. + # Each range's element lookup is kept for the value checks below, which + # would otherwise rebuild it per trait. Only ranges that enumerate elements + # can confirm a parent or a value. lookups: dict[str, dict[str, str]] = {} for range_ in all_ranges.values(): - lookups[range_.id] = element_ids = id_lookup(range_) + if not range_.elements: + continue + lookups[range_.id] = element_ids = id_lookup([e.id for e in range_.elements]) for element in range_.elements: if not element.parent: continue @@ -484,7 +487,20 @@ def resolve(name: str, ids: dict[str, str]) -> str | None: file=ranges_paths.get(range_.id) or file, ) elif target != element.parent: - mismatched.setdefault((range_.id, target), element.parent) + mismatched.setdefault((range_.id, "range-element id", target), element.parent) + + # The name keying a range -- a header id, a trait name -- is written + # separately from the range's own id, so it resolves the same way. + range_ids = id_lookup(lookups.keys()) + + def range_named(name: str) -> str | None: + """The id of the range ``name`` keys, or None if no such range + enumerates elements. Records a name that reached one by normalizing. + """ + range_id = resolve(name, range_ids) + if range_id is not None and range_id != name: + mismatched.setdefault((range_id, "range id", range_id), name) + return range_id # Header references (relative) that resolve to no companion. # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve @@ -499,8 +515,7 @@ def resolve(name: str, ids: dict[str, str]) -> str | None: relative = _normalize_href(range_.href) if relative is None: continue - resolved = all_ranges.get(range_.id) - if resolved is not None and resolved.elements: + if range_named(range_.id) is not None: continue # supplied by a sibling companion instead if not (base / relative).is_file(): yield Problem( @@ -514,24 +529,19 @@ def resolve(name: str, ids: dict[str, str]) -> str | None: # Undefined range values: every grammatical-info and every trait whose # name matches a known range, anywhere in the entry. Empty values are # skipped -- FLEx writes them. - def defined(range_id: str) -> dict[str, str] | None: - return lookups.get(range_id) or None # absent, or enumerating nothing - - grammatical_values = defined("grammatical-info") + grammatical_range = range_named("grammatical-info") for index, entry in enumerate(lexicon.entries): - # (label, range id, value, allowed ids) - checks: list[tuple[str, str, str, dict[str, str]]] = [] - for info in _iter_grammatical_infos(entry): - if info.value and grammatical_values is not None: - checks.append( - ("grammatical-info", "grammatical-info", info.value, grammatical_values) - ) + checks: list[tuple[str, str, str]] = [] # (label, range id, value) + if grammatical_range is not None: + for info in _iter_grammatical_infos(entry): + if info.value: + checks.append(("grammatical-info", grammatical_range, info.value)) for trait in _iter_traits(entry): - allowed = defined(trait.name) - if allowed is not None and trait.value: - checks.append((f"trait {trait.name!r}", trait.name, trait.value, allowed)) - for label, range_id, value, allowed in checks: - target = resolve(value, allowed) + range_id = range_named(trait.name) + if range_id is not None and trait.value: + checks.append((f"trait {trait.name!r}", range_id, trait.value)) + for label, range_id, value in checks: + target = resolve(value, lookups[range_id]) if target is None: yield Problem( "warning", @@ -543,16 +553,16 @@ def defined(range_id: str) -> dict[str, str] | None: line=at(index), ) elif target != value: - mismatched.setdefault((range_id, target), value) + mismatched.setdefault((range_id, "range-element id", target), value) # The two spellings render identically, so name them by code point rather # than with the !r other messages use. - for (range_id, element_id), reference in sorted(mismatched.items()): + for (range_id, kind, id_), reference in sorted(mismatched.items()): yield Problem( "warning", "normalization-mismatch", - f"range {range_id!r}: range-element id {element_id!a} is referenced " - f"as {reference!a}; they match only under Unicode NFC normalization", + f"range {range_id!r}: {kind} {id_!a} is referenced as {reference!a}; " + "they match only under Unicode NFC normalization", file=ranges_paths.get(range_id) or file, ) diff --git a/tests/corpus/PROVENANCE.md b/tests/corpus/PROVENANCE.md index 18ce267..9991473 100644 --- a/tests/corpus/PROVENANCE.md +++ b/tests/corpus/PROVENANCE.md @@ -175,8 +175,9 @@ The one genuine `undefined-range-value` left in Sango is the part of speech `prenom`, which no range defines in any normalization. What did match only after normalizing is still reported, as a -`normalization-mismatch` warning per range-element id: 6 in Sango, covering 2 -parent links and 80 grammatical-info values, and none in the other fixtures. +`normalization-mismatch` warning per range-element id: 2 parent links and 80 +grammatical-info values reach 5 ids in Sango, one of them under both aliases +of the part-of-speech list, so 6 warnings — and none in the other fixtures. `negative/nfd-range-ids.lift` is the hand-authored version of the same shape. ## generated/ — synthetic large files (not committed) diff --git a/tests/test_validate.py b/tests/test_validate.py index 47c4396..047f131 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -87,6 +87,33 @@ def test_normalization_mismatch_prefers_an_exactly_matching_sibling() -> None: assert list(lexicon.iter_problems()) == [] +def test_trait_name_reaches_a_range_id_in_another_normalization() -> None: + # A custom FLEx list becomes a range whose id is the list name and traits + # whose name is that same string -- separate writes, so they can differ in + # normalization exactly as an id and the values naming it do. The range has + # to be resolved for its values to be checked at all: an unresolved name + # looks like a trait no range keys, which is silently accepted. + name = "Catégorie" + lexicon = sil_lift.Lexicon() + ranges = sil_lift.RangesFile() + range_ = ranges.add_range(nfd(name)) + range_.add_element("Nom") + lexicon.add_ranges_file(ranges, href="x.lift-ranges") + entry = sil_lift.Entry(id="e1", guid="dddddddd-1111-4444-8888-dddddddddddd") + entry.lexical_unit["en"] = "e1" + entry.traits.append(sil_lift.Trait(name=name, value="Verbe")) + lexicon.entries.append(entry) + problems = list(lexicon.iter_problems()) + assert codes(problems) == { + ("warning", "undefined-range-value"), + ("warning", "normalization-mismatch"), + } + (mismatch,) = [p for p in problems if p.code == "normalization-mismatch"] + assert "range id" in mismatch.message + assert "Cate\\u0301gorie" in mismatch.message + assert "Cat\\xe9gorie" in mismatch.message + + def test_nfd_ids_warn_once_and_still_flag_the_real_dangling_parent(tmp_path: Path) -> None: path = NEGATIVE_DIR / "nfd-range-ids.lift" problems = problems_for(path) @@ -270,7 +297,8 @@ def test_sango_real_defects_are_found() -> None: # keeps the count at 1. No range-parent finding: the two parent links that # differ from their target's spelling are FLEx's NFD ids under an NFC # parent (see PROVENANCE.md), not dangling references. Those two and 80 - # NFC grammatical-info values resolve to 6 NFD ids, one warning each. + # NFC grammatical-info values reach 5 NFD ids; the aliased POS list below + # holds one of them under each of its two range ids, so 6 warnings. assert by_code.get("range-parent") is None assert by_code.get("undefined-range-value") == 1 assert by_code.get("normalization-mismatch") == 6 From 7a3a319b232a3ccac73aed9edbce31aba15f70f3 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 13 Aug 2026 15:19:06 -0400 Subject: [PATCH 7/9] Settle a header range id's spelling outside the file check Resolving the header's range id against the companion's own id was done inside the check for a dangling href, which runs only for a lexicon read from disk. Whether those two spellings agree is a property of the document, so a lexicon held in memory reported nothing where the same document on disk warned -- and validation is supposed to answer for what save() would write, wherever the document came from. Every header range id resolves before that check now, and the check reads the answer. Ranges that carry their own elements resolve to themselves and record nothing, so covering them all costs a lookup each. Co-authored-by: Claude Opus 5 (1M context) --- src/sil_lift/_validate.py | 7 ++++++- tests/test_validate.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 4400a99..eb94e45 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -502,6 +502,11 @@ def range_named(name: str) -> str | None: mismatched.setdefault((range_id, "range id", range_id), name) return range_id + # Whether a header range id and the companion's own id agree is a fact about + # the document, not about there being a file to look for -- so it is settled + # here, not inside the file check below, which a lexicon with no path skips. + header_ranges = {range_.id: range_named(range_.id) for range_ in lexicon.header.ranges} + # Header references (relative) that resolve to no companion. # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve # (they are resolved by basename when the companion is in the same folder) @@ -515,7 +520,7 @@ def range_named(name: str) -> str | None: relative = _normalize_href(range_.href) if relative is None: continue - if range_named(range_.id) is not None: + if header_ranges[range_.id] is not None: continue # supplied by a sibling companion instead if not (base / relative).is_file(): yield Problem( diff --git a/tests/test_validate.py b/tests/test_validate.py index 047f131..f95c694 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -114,6 +114,23 @@ def test_trait_name_reaches_a_range_id_in_another_normalization() -> None: assert "Cat\\xe9gorie" in mismatch.message +def test_header_range_id_reaches_a_companion_id_in_another_normalization() -> None: + # Regression: the resolution used to sit inside the dangling-href check, + # which runs only for a lexicon read from disk, so this went unreported. + name = "Catégorie" + lexicon = sil_lift.Lexicon() + ranges = sil_lift.RangesFile() + ranges.add_range(nfd(name)).add_element("Nom") + lexicon.add_ranges_file(ranges, href="x.lift-ranges") # header id as written + lexicon.header.ranges.append(sil_lift.Range(id=name, href="x.lift-ranges")) + entry = sil_lift.Entry(id="e1", guid="eeeeeeee-1111-4444-8888-eeeeeeeeeeee") + entry.lexical_unit["en"] = "e1" + lexicon.entries.append(entry) + problems = list(lexicon.iter_problems()) + assert codes(problems) == {("warning", "normalization-mismatch")} + assert "range id" in problems[0].message + + def test_nfd_ids_warn_once_and_still_flag_the_real_dangling_parent(tmp_path: Path) -> None: path = NEGATIVE_DIR / "nfd-range-ids.lift" problems = problems_for(path) From 139c4cf2cf8be3fb96fc57f1284e015e75c48f57 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 13 Aug 2026 16:57:40 -0400 Subject: [PATCH 8/9] Add order-justifying comment --- src/sil_lift/_validate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index eb94e45..1c9e4ce 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -542,6 +542,8 @@ def range_named(name: str) -> str | None: if info.value: checks.append(("grammatical-info", grammatical_range, info.value)) for trait in _iter_traits(entry): + # Resolved before the value guard: a trait name that reaches its range only + # by normalizing is worth reporting even without a value. range_id = range_named(trait.name) if range_id is not None and trait.value: checks.append((f"trait {trait.name!r}", range_id, trait.value)) From 5c560f092a749f10931506b56cba6629d35e964c Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Fri, 14 Aug 2026 15:55:20 -0400 Subject: [PATCH 9/9] Pin the mismatch dedup to the id, not the reference spelling The existing coverage for reporting a normalization mismatch once per id uses references that are all spelled the same way, so it pins only the collapse of identical references. Add a case where two references differ from the id and from each other -- a parent link in canonical mark order and a grammatical-info value with the same marks reversed -- and assert one warning. Which spelling the message names is deliberately left unasserted, so the order references are reached in stays free. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_validate.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/test_validate.py b/tests/test_validate.py index f95c694..6c5616e 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -152,6 +152,33 @@ def test_nfd_ids_warn_once_and_still_flag_the_real_dangling_parent(tmp_path: Pat assert (tmp_path / name).read_bytes() == (NEGATIVE_DIR / name).read_bytes(), name +def test_one_id_referenced_in_two_spellings_still_warns_once() -> None: + # The dedup is per id, not per spelling: references that differ from the id + # and from each other still collapse into one warning. Which of them the + # message names is left unasserted -- the contract is the count, not the + # order the references happen to be reached in. + name = "ṩ" # s with dot below and dot above, precomposed + # A third spelling: the same two marks in the other order, matching neither + # the id nor the parent link below (nfd() orders them canonically). + other_order = "ṩ" + lexicon = sil_lift.Lexicon() + ranges = sil_lift.RangesFile() + range_ = ranges.add_range("grammatical-info") + range_.add_element(name) + range_.add_element("Enclitic", parent=nfd(name)) + lexicon.add_ranges_file(ranges, href="x.lift-ranges") + entry = sil_lift.Entry(id="e1", guid="ffffffff-1111-4444-8888-ffffffffffff") + entry.lexical_unit["en"] = "e1" + entry.senses.append( + sil_lift.Sense(id="s1", grammatical_info=sil_lift.GrammaticalInfo(other_order)) + ) + lexicon.entries.append(entry) + problems = list(lexicon.iter_problems()) + assert codes(problems) == {("warning", "normalization-mismatch")} + assert len(problems) == 1 + assert "range-element id '\\u1e69'" in problems[0].message + + def test_undefined_range_values_are_warnings() -> None: problems = problems_for(NEGATIVE_DIR / "undefined-range-value.lift") assert codes(problems) == {("warning", "undefined-range-value")}