From e963674c7e8b6cdf360059a8d516e66ddcf0b7e3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 13:10:14 -0700 Subject: [PATCH 01/10] Pin the maiden rule to its vocabulary, and drop 'born' (#350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fix(#274) rule's alternation is a hand copy of MAIDEN_MARKERS that nothing checked. It offered 'born', which is in no config constant and never was, in v1 or v2 -- so that alternative could not correspond to a real #274 change. What it could do is claim any name containing "born", with fields maiden/middle/family, and absorb a genuine regression: the rule matches 'Max Born'. The pin is not the honorific pin's set equality, because these members are regex fragments rather than entries -- n[ée]e covers two markers and geb\.? covers one. It asserts the direction that matters instead: every alternative must match at least one entry the vocabulary ships. The covered set is recorded rather than equated, since widening to markers no corpus name exercises buys nothing and 旧姓 already has its own rule in the 2.0 ledger. Removing 'born' changes no classification -- no name in any of the three corpora contains it, which is also why it was never caught. English is absent from the vocabulary deliberately (it borrows née); promoting 'born' would be a behavior change with a real cost, since 'Bertha Born Smith' loses its family name to the marker. --- tests/v2/test_regex_sync.py | 93 +++++++++++++++++++- tools/differential/expected_since_1.4.0.toml | 19 +++- 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 9685d301..afbf53ad 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -30,6 +30,7 @@ from nameparser import _policy from nameparser._policy import Script from nameparser import _render +from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.suffixes import GLUED_HONORIFICS, SUFFIX_NOT_ACRONYMS @@ -658,12 +659,15 @@ def test_cjk_corpus_matches_the_case_table() -> None: _ALTERNATION = re.compile(r"\((?:\?:|(?!\?))((?:[^()|]+\|)+[^()|]+)\)") +def _alternations(name_regex: str) -> list[set[str]]: + """Every alternation group's members.""" + return [set(body.split("|")) for body in _ALTERNATION.findall(name_regex)] + + def _cjk_alternations(name_regex: str) -> list[set[str]]: """Every alternation in a rule with a script-classified member.""" has_classified = _policy._script_matcher(*_policy._SCRIPT_RANGES) - return [members - for body in _ALTERNATION.findall(name_regex) - for members in [set(body.split("|"))] + return [members for members in _alternations(name_regex) if any(has_classified(m) for m in members)] @@ -726,3 +730,86 @@ def test_differential_honorific_rules_match_their_vocabulary() -> None: f"group, a paren inside a character class, or a lone member), " f"which is how a still-present hand copy silently leaves this " f"pin.") + + +#: Ledger rules whose alternation is a hand copy of a LATIN vocabulary, +#: mapped to the constant it mirrors and the entries it is known to +#: cover. Kept apart from _HONORIFIC_SOURCES because the relationship +#: is not set equality: these members are regex FRAGMENTS, not entries. +#: "n[ée]e" covers two markers at once and "geb\.?" covers one, so +#: there is no set to compare against. +#: +#: What is pinned instead runs in two directions, and they are not +#: symmetric: +#: +#: Every member must mean something the vocabulary ships. This is the +#: over-claiming direction and the one that absorbs regressions -- a +#: member matching no entry cannot correspond to a real change, so it +#: can only ever claim OTHER names' diffs. It is how "born" sat in +#: this rule from the harness's first commit (#350) while never +#: appearing in any config constant: the rule's fields are maiden/ +#: middle/family, so a genuine regression on any name containing +#: "born" read as intended. +#: +#: The covered set is recorded, not equal to the vocabulary. Forcing +#: equality would widen the rule to markers no corpus name exercises +#: (README: 3 of 17 appear anywhere), and 旧姓 already has its own +#: rule in the 2.0 ledger, so folding it in here would make two rules +#: claim one diff. Recording it still catches removal: drop an entry +#: a member covers and the set shrinks. +_LATIN_ALTERNATION_SOURCES: dict[str, tuple[set[str], frozenset[str]]] = { + "fix(#274)": (MAIDEN_MARKERS, frozenset({"geb", "nee", "née", "roz"})), +} + + +def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: + """The Latin twin of the honorific pin, shaped by what a regex + alternation over a vocabulary can honestly promise. + + A member is matched against entries as a full regex, not compared + as a string, because that is what the rule does at classification + time -- "geb\\.?" is one member standing for the entry "geb", which + the config stores normalized (lowercase, no trailing period). + """ + used: set[str] = set() + for ledger in _LEDGERS: + for rule in _rules(ledger): + regex = rule.get("name_regex") + if not isinstance(regex, str): + continue + keys = [k for k in _LATIN_ALTERNATION_SOURCES + if k in rule["issue"]] + if not keys: + continue + assert len(keys) == 1, ( + f"{ledger.name}: {rule['issue']!r} matches {len(keys)} " + f"roster keys ({keys}); it must name exactly one") + vocabulary, recorded = _LATIN_ALTERNATION_SOURCES[keys[0]] + used.add(keys[0]) + alternations = _alternations(regex) + assert alternations, ( + f"{ledger.name}: {rule['issue']!r} is rostered as a " + f"vocabulary copy but carries no alternation _ALTERNATION " + f"can read; see its notes for the shapes that do not parse") + covered = set() + for members in alternations: + for member in members: + matched = {entry for entry in vocabulary + if re.fullmatch(member, entry, re.IGNORECASE)} + assert matched, ( + f"{ledger.name}: {rule['issue']!r} offers the " + f"alternative {member!r}, which matches no entry in " + f"the vocabulary it copies. It cannot correspond to " + f"a real change, so it can only claim other names' " + f"diffs as intended -- drop it, or ship it as " + f"vocabulary first") + covered |= matched + assert covered == recorded, ( + f"{ledger.name}: {rule['issue']!r} covers {sorted(covered)}; " + f"recorded {sorted(recorded)}. Lost: " + f"{sorted(recorded - covered)} (an entry the rule relied on " + f"left the vocabulary). Gained: {sorted(covered - recorded)} " + f"(record it)") + assert used == set(_LATIN_ALTERNATION_SOURCES), ( + f"_LATIN_ALTERNATION_SOURCES keys matching no rule: " + f"{sorted(set(_LATIN_ALTERNATION_SOURCES) - used)}") diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 4a4a58c1..61dbe3c4 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -63,7 +63,24 @@ fields = ["given", "middle", "family"] [[change]] issue = "fix(#274) maiden markers consumed" -name_regex = "(?i)\\b(n[ée]e|born|geb\\.?|roz\\.?)\\b" +# The alternation is a hand copy of MAIDEN_MARKERS, pinned by +# tests/v2/test_regex_sync.py: every alternative must match at least one +# entry the config actually ships. It covers the four the corpora +# exercise, not all seventeen -- widening it to markers no corpus name +# contains would buy nothing, and 旧姓 has its own rule in the 2.0 +# ledger, so claiming it here too would make two rules answer for one +# diff. +# +# 'born' was an alternative here from the harness's first commit and +# was removed in #350. It is not in MAIDEN_MARKERS and never was, in +# v1 or v2, so it could not correspond to a real #274 change. What it +# could do is claim any name containing "born" -- with fields +# maiden/middle/family -- and absorb a genuine regression. No corpus +# name contains it, so removing it changes no classification. English +# is absent from the vocabulary on purpose: it borrows née. Supporting +# 'born' would mean shipping it as vocabulary first, and that is not +# free: 'Bertha Born Smith' loses its family name to the marker. +name_regex = "(?i)\\b(n[ée]e|geb\\.?|roz\\.?)\\b" fields = ["maiden", "middle", "family"] [[change]] From a722048e0176cc1b58608074ab7a8e1dd70bf360 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 13:11:29 -0700 Subject: [PATCH 02/10] Let the open cycle's ledger be legitimately empty A ledger is created the day its baseline is released, before that cycle has produced a single behavior change, so it is empty and correct until the first one lands. Two things refused that state: the shipped-ledgers test asserted every file carries rules, and _rules read ["change"] where compare.py has always used .get("change", []). The carve-out is exactly the ledger DEFAULT_BASELINE names. Older ones are history, and history is not empty, so emptying one is still caught -- and the test now also asserts the open cycle's file exists at all, since _allowlist_for hard-errors on a bare run without it. --- tests/v2/test_differential.py | 18 ++++++++++++++++-- tests/v2/test_regex_sync.py | 6 +++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index 71244b52..739b8566 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -299,14 +299,28 @@ def test_validate_rules_rejects_a_rule_that_would_silently_widen( def test_validate_rules_accepts_the_shipped_ledgers() -> None: - """The guards above must not be so strict they reject real rules.""" + """The guards above must not be so strict they reject real rules. + + Every ledger but one must carry rules. The exception is the OPEN + cycle's -- the one DEFAULT_BASELINE names -- which is created the day + its baseline is released and is legitimately empty until that + cycle's first behavior change lands. An older ledger is history, and + history is not empty, so emptying one is still a mistake this + catches. + """ import tomllib + open_cycle = f"expected_since_{compare.DEFAULT_BASELINE}.toml" ledgers = sorted(_TOOLS.glob("expected_since_*.toml")) assert ledgers, "no ledgers found; this test would pass vacuously" + assert any(led.name == open_cycle for led in ledgers), ( + f"DEFAULT_BASELINE is {compare.DEFAULT_BASELINE!r} but {open_cycle} " + f"does not exist; a bare compare.py run would hard-error") for ledger in ledgers: rules = tomllib.loads( ledger.read_text(encoding="utf-8")).get("change", []) - assert rules, f"{ledger.name} has no [[change]] rules" + assert rules or ledger.name == open_cycle, ( + f"{ledger.name} has no [[change]] rules, and it is not the " + f"open cycle's ledger ({open_cycle})") compare.validate_rules(rules, ledger.name) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index afbf53ad..a473ffb1 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -197,7 +197,11 @@ def test_ledger_glob_is_not_empty() -> None: def _rules(ledger: Path) -> list[dict]: """The [[change]] table of one ledger.""" - return tomllib.loads(ledger.read_text(encoding="utf-8"))["change"] + # .get, matching compare.py: the open cycle's ledger is created + # empty at release and has no [[change]] table until that cycle's + # first behavior change lands. + return tomllib.loads( + ledger.read_text(encoding="utf-8")).get("change", []) def test_span_bearing_roster_names_exactly_the_ledgers_on_disk() -> None: From 180f616950acd720748a0b2b2e9f2818494d7765 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 13:13:56 -0700 Subject: [PATCH 03/10] Open the 2.1 cycle's ledger and advance DEFAULT_BASELINE Release step 8 was outstanding from 2.1.0: the baseline still named 2.0.0 and no expected_since_2.1.0.toml existed, so a bare compare.py run measured against a minor further back than it reported. The new ledger is empty and correct that way. nameparser/ is byte-identical to the v2.1.0 tag, so this cycle has produced no behavior change to classify yet. It has to exist anyway, because _allowlist_for treats a missing ledger as a hard error and the baseline cannot advance without it. Verified end to end rather than by the summary line alone. Against 2.1.0 the harness reports 751 names, 0 intentional, 0 unexplained -- which is also what a harness comparing the installed wheel against itself would print, so that run cannot stand on its own. Against 2.0.0 it reports 89 intentional and 0 unexplained, which is the discriminator: the comparison is real, and the empty result is a measurement rather than a broken run. _SPAN_BEARING_RULES records the file with an empty set, which is the enrollment AGENTS.md step 8 prescribes and this is its first exercise. --- tests/v2/test_regex_sync.py | 1 + tools/differential/compare.py | 2 +- tools/differential/expected_since_2.1.0.toml | 35 ++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tools/differential/expected_since_2.1.0.toml diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index a473ffb1..93357b58 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -426,6 +426,7 @@ def test_script_ranges_membership_is_decided() -> None: "fix(#271/#272/#298)", # the canonical class "fix(#298)", # the 间隔号 lookahead }), + "expected_since_2.1.0.toml": frozenset(), # open cycle, no rules yet } #: The leading `fix(...)`/`feat(...)` tag of a rule's `issue`, which is diff --git a/tools/differential/compare.py b/tools/differential/compare.py index cd3e711d..fc3a6f5d 100644 --- a/tools/differential/compare.py +++ b/tools/differential/compare.py @@ -23,7 +23,7 @@ FIELDS = ("title", "first", "middle", "last", "suffix", "nickname", "maiden") -DEFAULT_BASELINE = "2.0.0" +DEFAULT_BASELINE = "2.1.0" REPO_ROOT = HERE.parents[1] #: The v2 API's names for the same seven roles FIELDS names in v1 #: vocabulary. Both are compared from baseline 2.0 on. diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml new file mode 100644 index 00000000..5e9358a6 --- /dev/null +++ b/tools/differential/expected_since_2.1.0.toml @@ -0,0 +1,35 @@ +# Ledger for baseline 2.1.0 -- what changes for a user upgrading from +# the previous minor. Same rule grammar as the other ledgers: every rule +# needs `issue`; optional `name_regex` and `fields` narrow it, and +# compare.py sorts name_regex rules ahead of fields-only ones. +# +# EMPTY ON PURPOSE, and correct while it stays that way. A ledger is +# opened the day its baseline ships (AGENTS.md release step 8), before +# the cycle has produced a single behavior change -- so there is nothing +# to classify yet. `nameparser/` is byte-identical to the v2.1.0 tag as +# this file is added, so a run against this baseline reports zero diffs +# and zero UNEXPLAINED. +# +# It exists rather than waiting because DEFAULT_BASELINE and the ledger +# are coupled: _allowlist_for treats a missing file as a hard error, on +# the reasoning that an absent ledger classifies nothing and would make +# every diff report as unexplained. So the baseline cannot advance to +# 2.1.0 without this file. +# +# Add the first rule when the first 2.2 behavior change lands. Do not +# copy rules across from expected_since_2.0.0.toml: that ledger +# classifies what 2.1 changed on top of 2.0, and a rule copied without +# checking either over-matches -- hiding a real 2.2 regression behind a +# 2.1-era label -- or never fires at all. +# +# tests/v2/test_differential.py permits exactly one empty ledger, the +# one DEFAULT_BASELINE names. Once this stops being the open cycle it +# must carry rules like any other, so a 2.2 that changed no behavior at +# all would need that decision made deliberately rather than inherited. +# +# tests/v2/test_regex_sync.py's _SPAN_BEARING_RULES already records this +# file with an empty set: no rule here hand-copies _SCRIPT_RANGES yet, +# and the first one that does has to be recorded there or the sweep +# fails. + +change = [] From 351af1140b90087d4dc722f4f81b1d3a9457ebb2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 13:44:32 -0700 Subject: [PATCH 04/10] Give the Latin pin discovery, and close two ways it can be widened Review defeated the pin three ways, each measured through the harness's own classify() against the real ledger. It had no discovery direction. It iterated roster keys and skipped every rule that matched none, so a hand copy under an unrostered tag was pinned by nothing -- and one already was: the MA/DO acronym rule copies SUFFIX_ACRONYMS_AMBIGUOUS and no test could see it. It now iterates alternations, so an undeclared one fails; alternations that copy no vocabulary are declared as such rather than skipped. Appending "|[A-Za-z]" at depth 0 passed. The pinned alternation then governs one branch of an unchecked whole, and a family-only regression would be labelled fix(#274) on 675 of 783 corpus names instead of 4. _top_level_alternation already existed for exactly this -- it was applied one function away, inside the span-bearing sweep only. A member of "[a-z]{3}" or ".{3}" passed, because fullmatch against the vocabulary bounds what a member matches within 17 entries and says nothing about what it matches in a name -- and every entry this rule needs happens to be three characters. 320 of 783 names. Members are now probed against ordinary name fragments too. Also: a NamedTuple, since the roster's two set-shaped fields are not the same kind of thing and the positional tuple said nothing about which was the source of truth; re.error re-raised as an assertion naming the ledger, rule and member, since a mis-split alternation can manufacture an invalid fragment; and the vacuity guard its twin has. --- tests/v2/test_regex_sync.py | 183 +++++++++++++++++++++++++----------- 1 file changed, 129 insertions(+), 54 deletions(-) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 93357b58..0048184c 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -22,6 +22,7 @@ import re import tomllib from pathlib import Path +from typing import NamedTuple import pytest @@ -31,7 +32,8 @@ from nameparser._policy import Script from nameparser import _render from nameparser.config.maiden_markers import MAIDEN_MARKERS -from nameparser.config.suffixes import GLUED_HONORIFICS, SUFFIX_NOT_ACRONYMS +from nameparser.config.suffixes import ( + GLUED_HONORIFICS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS) def test_emoji_ranges_match_config() -> None: @@ -737,70 +739,140 @@ def test_differential_honorific_rules_match_their_vocabulary() -> None: f"pin.") -#: Ledger rules whose alternation is a hand copy of a LATIN vocabulary, -#: mapped to the constant it mirrors and the entries it is known to -#: cover. Kept apart from _HONORIFIC_SOURCES because the relationship -#: is not set equality: these members are regex FRAGMENTS, not entries. -#: "n[ée]e" covers two markers at once and "geb\.?" covers one, so -#: there is no set to compare against. -#: -#: What is pinned instead runs in two directions, and they are not -#: symmetric: +class _LatinCopy(NamedTuple): + """A ledger alternation that hand-copies a Latin-script vocabulary. + + Two set-shaped fields that are emphatically not the same kind of + thing, which is why they are named rather than positional: + `vocabulary` is the source of truth, `covers` an audited snapshot of + which of its entries the rule's members reach. + """ + vocabulary: set[str] + covers: frozenset[str] + + +#: Ledger rules whose alternation hand-copies a LATIN vocabulary, keyed +#: by a substring of the rule's `issue`. Kept apart from +#: _HONORIFIC_SOURCES because the relationship is not set equality: +#: these members are regex FRAGMENTS, not entries -- "n[ée]e" covers two +#: markers at once, "geb\.?" and "roz\.?" one each -- so there is no set +#: to compare against. #: -#: Every member must mean something the vocabulary ships. This is the -#: over-claiming direction and the one that absorbs regressions -- a -#: member matching no entry cannot correspond to a real change, so it -#: can only ever claim OTHER names' diffs. It is how "born" sat in -#: this rule from the harness's first commit (#350) while never -#: appearing in any config constant: the rule's fields are maiden/ -#: middle/family, so a genuine regression on any name containing -#: "born" read as intended. +#: `covers` is recorded rather than equated to the whole vocabulary. +#: Equality would force the rule to grow alternatives for markers it has +#: no reason to claim, and 旧姓 already has a dedicated rule further down +#: this same ledger which sorts AFTER this one -- so widening here would +#: shadow that rule rather than complement it. Recording still catches +#: removal: drop an entry a member covers and the snapshot shrinks. #: -#: The covered set is recorded, not equal to the vocabulary. Forcing -#: equality would widen the rule to markers no corpus name exercises -#: (README: 3 of 17 appear anywhere), and 旧姓 already has its own -#: rule in the 2.0 ledger, so folding it in here would make two rules -#: claim one diff. Recording it still catches removal: drop an entry -#: a member covers and the set shrinks. -_LATIN_ALTERNATION_SOURCES: dict[str, tuple[set[str], frozenset[str]]] = { - "fix(#274)": (MAIDEN_MARKERS, frozenset({"geb", "nee", "née", "roz"})), +#: Three nearby counts differ and are easy to conflate: MAIDEN_MARKERS +#: ships 17 entries; this rule's members reach 4 of them; the corpora +#: contain 3 markers in total (geb, née, 旧姓), only 2 of which this +#: rule covers. +_LATIN_ALTERNATION_SOURCES: dict[str, _LatinCopy] = { + "fix(#274)": _LatinCopy( + vocabulary=MAIDEN_MARKERS, + covers=frozenset({"geb", "nee", "née", "roz"})), + "ambiguous-surname-acronym": _LatinCopy( + vocabulary=SUFFIX_ACRONYMS_AMBIGUOUS, + covers=frozenset({"do", "ma"})), } +#: Alternations that copy no vocabulary, so discovery must not demand a +#: source for them. Declared rather than inferred, on the principle +#: _SOURCES' None entries already set: an undeclared alternation is a +#: question someone answers in writing, not something to skip past. +_NOT_A_VOCABULARY_COPY = frozenset({ + frozenset({"^", " "}), # the honorific rule's leading anchor +}) + +#: Ordinary name fragments no vocabulary member may match. fullmatch +#: against the vocabulary bounds what a member matches WITHIN those +#: entries and says nothing about what it matches in a NAME, so a +#: fixed-width or dot-bearing member slips past it: "[a-z]{3}" and +#: ".{3}" each cover exactly {geb, nee, née, roz}, because every entry +#: this rule needs happens to be three characters -- while the rule they +#: produce would absorb a family-only regression on a third of the +#: corpus. These probes are what make the over-claiming direction mean +#: something. +_NOT_VOCABULARY = ("Bob", "Nye", "Van", "der", "Jones", "Gen", "Get", "abc") + def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: """The Latin twin of the honorific pin, shaped by what a regex alternation over a vocabulary can honestly promise. - A member is matched against entries as a full regex, not compared - as a string, because that is what the rule does at classification - time -- "geb\\.?" is one member standing for the entry "geb", which - the config stores normalized (lowercase, no trailing period). + Discovery-first, like its twin: every alternation in every ledger + must be a declared vocabulary copy or a declared non-copy. Keying + off the roster and skipping everything else would mean a future rule + that hand-copies a vocabulary under a new tag is pinned only if its + author remembers to enroll it -- the failure this module exists to + prevent, not a shape it should adopt. + + Members are matched against entries as regexes rather than compared + as strings, because the members ARE regex syntax: "geb\\.?" stands + for the entry "geb", which the config stores normalized (lowercase, + no trailing period). + + Three ways a member can be wrong, so three assertions. It can match + nothing in the vocabulary -- then it cannot describe a real change + and can only claim other names' diffs, which is how "born" survived + from the harness's first commit to #350. It can match ordinary name + text as well as the vocabulary. And the rule around it can widen at + depth 0, leaving the pinned alternation governing one branch of an + unchecked whole -- the same hatch the span-bearing sweep closes. """ + has_classified = _policy._script_matcher(*_policy._SCRIPT_RANGES) used: set[str] = set() + found = 0 for ledger in _LEDGERS: for rule in _rules(ledger): regex = rule.get("name_regex") if not isinstance(regex, str): continue - keys = [k for k in _LATIN_ALTERNATION_SOURCES - if k in rule["issue"]] - if not keys: - continue - assert len(keys) == 1, ( - f"{ledger.name}: {rule['issue']!r} matches {len(keys)} " - f"roster keys ({keys}); it must name exactly one") - vocabulary, recorded = _LATIN_ALTERNATION_SOURCES[keys[0]] - used.add(keys[0]) - alternations = _alternations(regex) - assert alternations, ( - f"{ledger.name}: {rule['issue']!r} is rostered as a " - f"vocabulary copy but carries no alternation _ALTERNATION " - f"can read; see its notes for the shapes that do not parse") - covered = set() - for members in alternations: + for members in _alternations(regex): + if any(has_classified(m) for m in members): + continue # the honorific pin owns these + if frozenset(members) in _NOT_A_VOCABULARY_COPY: + continue + found += 1 + keys = [k for k in _LATIN_ALTERNATION_SOURCES + if k in rule["issue"]] + assert len(keys) == 1, ( + f"{ledger.name}: {rule['issue']!r} carries the Latin " + f"alternation {sorted(members)}, which matches " + f"{len(keys)} roster keys ({keys}). Declare the " + f"vocabulary it copies in _LATIN_ALTERNATION_SOURCES, " + f"or add it to _NOT_A_VOCABULARY_COPY if it copies " + f"nothing") + used.add(keys[0]) + source = _LATIN_ALTERNATION_SOURCES[keys[0]] + assert not _top_level_alternation(regex), ( + f"{ledger.name}: {rule['issue']!r} has a '|' at depth " + f"0, so the pinned alternation governs only one branch " + f"and the rest of the rule is unchecked. Wrap it in " + f"'(?:...)'") + covered = set() for member in members: - matched = {entry for entry in vocabulary - if re.fullmatch(member, entry, re.IGNORECASE)} + try: + matched = {entry for entry in source.vocabulary + if re.fullmatch(member, entry, + re.IGNORECASE)} + loose = [probe for probe in _NOT_VOCABULARY + if re.fullmatch(member, probe, + re.IGNORECASE)] + except re.error as exc: + raise AssertionError( + f"{ledger.name}: {rule['issue']!r} member " + f"{member!r} is not a valid regex ({exc}). A " + f"mis-split alternation can produce this -- see " + f"_ALTERNATION's notes") from None + assert not loose, ( + f"{ledger.name}: {rule['issue']!r} member " + f"{member!r} also matches ordinary name text " + f"{loose}. Spell it literally -- a member broad " + f"enough to hit a real name lets the rule claim " + f"that name's diff as intended") assert matched, ( f"{ledger.name}: {rule['issue']!r} offers the " f"alternative {member!r}, which matches no entry in " @@ -809,12 +881,15 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: f"diffs as intended -- drop it, or ship it as " f"vocabulary first") covered |= matched - assert covered == recorded, ( - f"{ledger.name}: {rule['issue']!r} covers {sorted(covered)}; " - f"recorded {sorted(recorded)}. Lost: " - f"{sorted(recorded - covered)} (an entry the rule relied on " - f"left the vocabulary). Gained: {sorted(covered - recorded)} " - f"(record it)") + assert covered == source.covers, ( + f"{ledger.name}: {rule['issue']!r} covers " + f"{sorted(covered)}; recorded {sorted(source.covers)}. " + f"Lost: {sorted(source.covers - covered)} (an entry the " + f"rule relied on left the vocabulary). Gained: " + f"{sorted(covered - source.covers)} (record it)") + assert found, ( + "no Latin vocabulary alternation found in any ledger; this pin is " + "passing vacuously") assert used == set(_LATIN_ALTERNATION_SOURCES), ( f"_LATIN_ALTERNATION_SOURCES keys matching no rule: " f"{sorted(set(_LATIN_ALTERNATION_SOURCES) - used)}") From 372b4a4ab17947017045250ae6bc3c3470e07c92 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 13:46:16 -0700 Subject: [PATCH 05/10] Stop the open cycle's ledger from hiding a broken one Two defects in how the empty ledger was shipped, both found by trying to use it. `change = []` blocks the next step the file's own comment asks for: TOML forbids appending a [[change]] table to a statically defined array, so adding the first 2.2 rule raises TOMLDecodeError pointing at the new rule rather than at the line that caused it. The line was never needed -- both readers use .get("change", []), so an absent key already means an empty ledger. Removed, and appending now works. The carve-out that lets this one ledger be empty also lets it be broken. A mistyped table header -- [[changes]], [[rules]] -- reads as a legitimately empty open cycle everywhere: every sweep gets zero rules and passes while the author believes they shipped a rule. The other ledgers are protected by having to be non-empty, so this is the one place it needs saying: nothing but `change` may be defined at the top level of the open cycle's file. The existence assertion moves out of test_validate_rules_accepts_the_ shipped_ledgers into a test named for what it checks. It is a fact about the DEFAULT_BASELINE-to-ledger coupling in compare.py, not about whether validate_rules accepts the shipped rules, and someone half way through release step 8 will look for it by name. --- tests/v2/test_differential.py | 33 ++++++++++++++++++-- tools/differential/expected_since_2.1.0.toml | 10 ++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/tests/v2/test_differential.py b/tests/v2/test_differential.py index 739b8566..443809ad 100644 --- a/tests/v2/test_differential.py +++ b/tests/v2/test_differential.py @@ -298,6 +298,36 @@ def test_validate_rules_rejects_a_rule_that_would_silently_widen( compare.validate_rules([rule], "expected_since_2.0.0.toml") +def test_default_baseline_has_a_ledger_and_nothing_else_in_it() -> None: + """Two facts about the OPEN cycle's ledger, which the carve-out + below leans on and neither of us should have to derive. + + It must exist: _allowlist_for treats a missing ledger as a hard + error, so a DEFAULT_BASELINE with no file makes a bare compare.py + run abort -- and it would also make the carve-out inert, since it + would name a file nothing iterates over. + + And it must define nothing at the top level except `change`. This is + the one ledger allowed to be empty, so a mistyped table header -- + `[[changes]]`, `[[rules]]` -- reads as a legitimately empty open + cycle everywhere instead of as a broken file: every sweep gets zero + rules and passes, while the author believes they shipped a rule. + The other ledgers are protected by having to be non-empty; this one + needs saying out loud. + """ + import tomllib + open_cycle = _TOOLS / f"expected_since_{compare.DEFAULT_BASELINE}.toml" + assert open_cycle.exists(), ( + f"DEFAULT_BASELINE is {compare.DEFAULT_BASELINE!r} but " + f"{open_cycle.name} does not exist; a bare compare.py run would " + f"hard-error, and the empty-ledger carve-out would be inert") + keys = set(tomllib.loads(open_cycle.read_text(encoding="utf-8"))) + assert keys <= {"change"}, ( + f"{open_cycle.name} defines {sorted(keys - {'change'})} at the top " + f"level. Only `change` is read, so anything else is a typo that " + f"would read as an empty ledger rather than as a broken one") + + def test_validate_rules_accepts_the_shipped_ledgers() -> None: """The guards above must not be so strict they reject real rules. @@ -312,9 +342,6 @@ def test_validate_rules_accepts_the_shipped_ledgers() -> None: open_cycle = f"expected_since_{compare.DEFAULT_BASELINE}.toml" ledgers = sorted(_TOOLS.glob("expected_since_*.toml")) assert ledgers, "no ledgers found; this test would pass vacuously" - assert any(led.name == open_cycle for led in ledgers), ( - f"DEFAULT_BASELINE is {compare.DEFAULT_BASELINE!r} but {open_cycle} " - f"does not exist; a bare compare.py run would hard-error") for ledger in ledgers: rules = tomllib.loads( ledger.read_text(encoding="utf-8")).get("change", []) diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 5e9358a6..3b9cf311 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -31,5 +31,11 @@ # file with an empty set: no rule here hand-copies _SCRIPT_RANGES yet, # and the first one that does has to be recorded there or the sweep # fails. - -change = [] +# +# There is deliberately no `change = []` line. TOML forbids appending a +# [[change]] table to a statically defined array, so that line would +# block the exact next step this comment asks for -- and both readers +# use .get("change", []), so the key's absence is already the empty +# ledger. tests/v2/test_differential.py checks that nothing else is +# defined at the top level here, since a mistyped table name would +# otherwise read as an empty ledger rather than as a broken one. From 91af1bd63908c4e5cffc0aab656fceb81a32b4d2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 13:47:59 -0700 Subject: [PATCH 06/10] Correct four claims in the maiden rule's notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review measured every factual claim in the last round. These did not survive. "It covers the four the corpora exercise" was wrong on both halves. The rule's members reach 4 of the 17 MAIDEN_MARKERS entries; the corpora contain 3 markers; only 2 are in both sets. The next clause then contradicted the first -- "widening it to markers no corpus name contains would buy nothing" -- when nee and roz are exactly such markers and were already in. "旧姓 has its own rule in the 2.0 ledger, so claiming it here too would make two rules answer for one diff" named the wrong file and a mechanism that cannot happen: compare.py loads exactly one ledger per run, so rules in different ledgers never compete. The rule that could actually be shadowed is fix(cjk-maiden-marker), 20 lines below in THIS file, which sorts after fix(#274) within the name_regex tier. "English is absent from the vocabulary on purpose: it borrows née" was invented. maiden_markers.py has a "Deliberately absent" note and it names Polish z domu and Scandinavian f., not English -- and 'nee' is in the vocabulary, which tests/v2/cases.py calls the spelling English writes most often. The note now points at the real one. _rules' comment described a key-absent shape the shipped ledger contradicted; the ledger no longer defines `change` at all, so the comment is now true, and it says where that leniency is policed. Also sweeps a pre-existing leftover from the Role-vocabulary migration in the same file: fields "maiden/middle/last" and "flips `first`" for a rule that ships given/middle/family. AGENTS.md release step 8 gains the third roster, and drops the claim that the sweep needs no discovery. tools/differential/README.md's quick-start hardcoded --baseline 2.0.0, which is now two minors back; the bare invocation is the release-time run. --- AGENTS.md | 11 ++++-- tests/v2/test_regex_sync.py | 10 ++++-- tools/differential/README.md | 2 +- tools/differential/expected_since_1.4.0.toml | 36 ++++++++++++-------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 204cbc09..eb7bf9e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,9 +88,10 @@ uv run sphinx-build -b html docs dist/docs # bare compare.py measures against two minors back while reporting the # previous one, and _allowlist_for hard-errors on the missing file. # tests/v2/test_regex_sync.py sweeps every expected_since_*.toml (#333), so -# the new ledger's hand copies of _SCRIPT_RANGES and the honorific -# vocabulary are checked from the day the file lands -- but it enrolls -# itself in neither roster, and both failures are loud, not silent: +# the new ledger's hand copies of _SCRIPT_RANGES and of the honorific and +# Latin vocabularies are checked from the day the file lands. Discovery +# finds the copies; the rosters record what they mirror, and a copy the +# rosters cannot account for fails loudly rather than going unpinned: # - _SPAN_BEARING_RULES: add the filename, mapped to the set of issue # tags whose rules carry a script-span class (empty set if none). # - _HONORIFIC_SOURCES: if the ledger has a CJK honorific rule, add a @@ -99,6 +100,10 @@ uv run sphinx-build -b html docs dist/docs # that issue. A retroactive ledger can repeat an older one's rule # verbatim (fix(#271/#272/#298) is in both today), and every rule # must match exactly one key. +# - _LATIN_ALTERNATION_SOURCES: same, for a rule copying a Latin +# vocabulary (maiden markers, ambiguous acronyms). An alternation +# matching no key fails as undeclared -- add it, or record it in +# _NOT_A_VOCABULARY_COPY if it copies nothing. ``` Enable debug logging to see the parser's internal decisions: diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 0048184c..e9326ce1 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -199,9 +199,13 @@ def test_ledger_glob_is_not_empty() -> None: def _rules(ledger: Path) -> list[dict]: """The [[change]] table of one ledger.""" - # .get, matching compare.py: the open cycle's ledger is created - # empty at release and has no [[change]] table until that cycle's - # first behavior change lands. + # .get, matching compare.py. The open cycle's ledger is created at + # release with no `change` key at all -- an empty [[change]] array + # cannot be appended to in TOML -- so an absent key IS the empty + # ledger here, not a malformed file. What stops that leniency from + # hiding a typo'd table header lives in tests/v2/test_differential.py: + # every other ledger must be non-empty, and the open one may define + # nothing but `change`. return tomllib.loads( ledger.read_text(encoding="utf-8")).get("change", []) diff --git a/tools/differential/README.md b/tools/differential/README.md index 9d133148..808cf89c 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -23,7 +23,7 @@ Two processes, two environments: ``` uv run python tools/differential/build_corpus.py --ref > tools/differential/corpus.jsonl # only when regenerating uv run python tools/differential/compare.py --baseline 1.4.0 -uv run python tools/differential/compare.py --baseline 2.0.0 +uv run python tools/differential/compare.py # bare = DEFAULT_BASELINE, the last release ``` `compare.py` spawns the worker as a subprocess, feeds it every corpus diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 61dbe3c4..9f69f3c9 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -64,32 +64,40 @@ fields = ["given", "middle", "family"] [[change]] issue = "fix(#274) maiden markers consumed" # The alternation is a hand copy of MAIDEN_MARKERS, pinned by -# tests/v2/test_regex_sync.py: every alternative must match at least one -# entry the config actually ships. It covers the four the corpora -# exercise, not all seventeen -- widening it to markers no corpus name -# contains would buy nothing, and 旧姓 has its own rule in the 2.0 -# ledger, so claiming it here too would make two rules answer for one -# diff. +# tests/v2/test_regex_sync.py: every alternative must match at least +# one entry the config ships, and none may match ordinary name text. +# +# Its members reach 4 of the 17 entries -- geb, nee, née, roz -- which +# is not the same set as the markers the corpora contain (geb, née and +# 旧姓; only the first two are covered here). Growing it toward the +# other 13 is not obviously right: the 旧姓 rule below sorts AFTER this +# one, so covering 旧姓 here would shadow it rather than complement it, +# and the rest have no corpus name to classify. Widen deliberately, and +# record the new coverage in _LATIN_ALTERNATION_SOURCES. # # 'born' was an alternative here from the harness's first commit and # was removed in #350. It is not in MAIDEN_MARKERS and never was, in # v1 or v2, so it could not correspond to a real #274 change. What it # could do is claim any name containing "born" -- with fields # maiden/middle/family -- and absorb a genuine regression. No corpus -# name contains it, so removing it changes no classification. English -# is absent from the vocabulary on purpose: it borrows née. Supporting -# 'born' would mean shipping it as vocabulary first, and that is not -# free: 'Bertha Born Smith' loses its family name to the marker. +# name contains it, so removing it changed no classification. +# +# Supporting 'born' would mean shipping it as vocabulary first, and it +# is not free: 'Bertha Born Smith' loses its family name to the marker. +# Note the vocabulary is not missing English -- 'nee' is there, and +# tests/v2/cases.py calls it the spelling English writes most often. +# See maiden_markers.py's own "Deliberately absent" note for what has +# actually been considered and declined. name_regex = "(?i)\\b(n[ée]e|geb\\.?|roz\\.?)\\b" fields = ["maiden", "middle", "family"] [[change]] issue = "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip" # Its own rule rather than a widening of fix(#274) above: that rule's -# fields stop at maiden/middle/last because a Latin marker moves only -# those, while a Han one also flips `first` -- the name left after the -# marker is consumed is wholly Han, so it reads family-first (#271). -# Adding `first` to #274's list would make that rule broader than its +# fields stop at maiden/middle/family because a Latin marker moves +# only those, while a Han one also flips `given` -- the name left after +# the marker is consumed is wholly Han, so it reads family-first +# (#271). Adding `given` to #274's list would make it broader than its # prose and let it absorb diffs that have nothing to do with markers # (#328). The regex is the marker itself, so this rule can claim # nothing else. From c87f1aa778b7557f5df8f355304f6aac90fe3281 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 14:11:46 -0700 Subject: [PATCH 07/10] Ask the corpus what a rule claims, instead of inspecting its syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four review rounds found the same defect in four costumes: a hardcoded slug, then a hardcoded count, then a hardcoded probe list, then a hardcoded nesting depth. Each guard closed the spelling that had just been demonstrated and left the next one open, because every one of them checked regex SYNTAX as a proxy for what a rule claims. The last round made the pattern undeniable. "[acdf-uw-z]{3,}" walks through all eight probe strings and claims 563 of 783 corpus names -- wider than the 320-name attack the probes were added to stop, and derivable rather than lucky, since every entry the rule needs is three characters so a member must accept one and is free everywhere else. And a copy written as a character class rather than an alternation was not undeclared but unseen: no roster demanded a key, nothing fired, 320 names claimed. So stop asking how a rule is spelled and ask what it reaches. The ledgers exist to explain diffs on the corpora and nothing else, so the corpus is the whole population a rule will ever be asked about -- and it answers without parsing anything, just a regex search. Three guards, all corpus-derived: - A span-bearing rule must claim no corpus name carrying an unclassified script. The depth-0 test could not see "(?:CJK|[A-Za-z])" hiding its pipe one level down; this does, because it never looks at the pipe. Both are kept -- the syntactic one gives a clearer message for the naive spelling and covers a widening toward a script the corpora do not happen to contain. - Every corpus fragment a vocabulary member matches must BE an entry, normalized (the corpus writes "geb.", the config stores "geb") and exempting only that member's own vocabulary. Replaces the probe tuple, which could never be more than a spot check. - Every rule claiming a role in _FIELD_VOCABULARIES must claim only names that carry that vocabulary. Keyed on `fields`, not on the regex, which is what closes the unseen-copy hole: a rule that does not claim `maiden` cannot be a maiden change, so no notation escapes it. It also reaches fix(cjk-maiden-marker), whose regex is the bare literal 旧姓 and which no roster in this module could see. Verified by replaying every attack from all four rounds: depth-0 widening, wrapped widening, [a-z]{3}, [acdf-uw-z]{3,}, geb[^o]*, the class-with-no-alternation rule, and 'born' itself. All fail; the shipped rules stay green; removing an entry from MAIDEN_MARKERS still fails. --- tests/v2/test_regex_sync.py | 185 ++++++++++++++++++++++++++++++++---- 1 file changed, 165 insertions(+), 20 deletions(-) diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index e9326ce1..ba94ad16 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -188,6 +188,44 @@ def test_comma_char_matches_the_pipeline_comma_set() -> None: #: release -- see AGENTS.md's release step 8. _LEDGERS = sorted(_TOOLS.glob("expected_since_*.toml")) +#: Every name the harness classifies, deduplicated. The ledgers exist +#: to explain diffs on THESE strings and no others, so "what does this +#: rule claim?" is answerable here without parsing anything -- a plain +#: regex search, no baseline wheel, no network. +#: +#: This is what the guards below check against, and it is why they hold +#: where four rounds of syntactic ones did not. Depth-0 pipes, nesting +#: levels and probe strings are all proxies for the question that +#: actually matters; a rule cannot widen its corpus reach and still +#: answer this one the same way, however it is spelled. +_CORPUS_NAMES = sorted({ + json.loads(line) + for path in sorted(_TOOLS.glob("corpus*.jsonl")) + for line in path.read_text(encoding="utf-8").splitlines() if line.strip()}) + + +def _claimed(name_regex: str) -> list[str]: + """Corpus names a rule's regex matches.""" + return [name for name in _CORPUS_NAMES if re.search(name_regex, name)] + + +def _unclassified_names() -> list[str]: + """Corpus names carrying no codepoint _SCRIPT_RANGES classifies.""" + has_classified = _policy._script_matcher(*_policy._SCRIPT_RANGES) + return [name for name in _CORPUS_NAMES if not has_classified(name)] + + +def _normalize(text: str) -> str: + """How the config stores vocabulary: lowercase, no edge punctuation.""" + return text.strip(".,()[]\"'\u2019 ").lower() + + +def test_corpus_is_loaded() -> None: + """The guards below all reduce to zero findings over an empty + corpus, which is what a silent load failure produces.""" + assert len(_CORPUS_NAMES) > 500, len(_CORPUS_NAMES) + assert _unclassified_names(), "no unclassified names; guard A is inert" + def test_ledger_glob_is_not_empty() -> None: """A parametrize over an empty list generates a single silent SKIP @@ -516,6 +554,23 @@ def test_every_span_bearing_rule_matches_the_script_ranges( f"{ledger.name}: {rule['issue']!r} has a '|' at depth 0, so " f"the whole rule is an alternation and the pinned class " f"governs only one branch. Wrap it in '(?:...)'") + # The property the syntactic check above is only a proxy for. + # A rule scoped to classified scripts must not reach a name + # written in none of them -- and unlike a depth test, this does + # not care how the widening is spelled. "(?:CJK|[A-Za-z])" + # hides the pipe at depth 1 where the check above stops + # looking, and claims 665 unclassified corpus names; this sees + # it. Both are kept: the depth test gives the clearer message + # for the naive spelling, and catches a widening toward a + # script the corpora happen not to contain. + reached = _claimed(regex) + latin = [name for name in reached if name in set(_unclassified_names())] + assert not latin, ( + f"{ledger.name}: {rule['issue']!r} declares the script table's " + f"spans but claims {len(latin)} corpus names carrying no " + f"classified codepoint at all, e.g. {latin[:3]}. A rule scoped " + f"to these scripts cannot explain a diff on those names, so it " + f"would absorb one instead") # Every rule, not just the discovered ones: this is what stops a # class being respelled out of discovery in the first place, and it # has to reach the rules that are NOT currently span-bearing to do @@ -790,16 +845,35 @@ class _LatinCopy(NamedTuple): frozenset({"^", " "}), # the honorific rule's leading anchor }) -#: Ordinary name fragments no vocabulary member may match. fullmatch -#: against the vocabulary bounds what a member matches WITHIN those -#: entries and says nothing about what it matches in a NAME, so a -#: fixed-width or dot-bearing member slips past it: "[a-z]{3}" and -#: ".{3}" each cover exactly {geb, nee, née, roz}, because every entry -#: this rule needs happens to be three characters -- while the rule they -#: produce would absorb a family-only regression on a third of the -#: corpus. These probes are what make the over-claiming direction mean -#: something. -_NOT_VOCABULARY = ("Bob", "Nye", "Van", "der", "Jones", "Gen", "Get", "abc") +def _reaches_non_vocabulary(member: str, vocabulary: set[str]) -> list[str]: + """Corpus text this member matches that is NOT a vocabulary entry. + + fullmatch against the vocabulary bounds what a member matches + WITHIN those entries and says nothing about what it matches in a + NAME. That gap was first filled with a tuple of eight hand-picked + probe strings, and the tuple was defeated by a wider rule than the + one it was added to stop: every entry this rule needs is three + characters, so a member must accept some 3-character string and is + unconstrained everywhere else -- "[acdf-uw-z]{3,}" covers `roz`, + dodges all eight probes, and claims 563 of the corpus. + + Eight strings could never be more than a spot check. The corpus is + the whole population the rule will ever be asked about, so ask it + instead: every fragment a member matches must BE an entry. + + Searched unanchored, which is stricter than the rule's own \\b + anchoring and so cannot produce a false negative from it. + Normalized before comparison because the corpus writes "geb." where + the config stores "geb" -- without that, the shipped member fails. + Exempting only THIS vocabulary, not the union of all of them, keeps + a maiden member from being excused by an acronym entry. + """ + matches = set() + for name in _CORPUS_NAMES: + for found in re.finditer(member, name, re.IGNORECASE): + if found.group() and _normalize(found.group()) not in vocabulary: + matches.add(found.group()) + return sorted(matches) def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: @@ -859,24 +933,22 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: covered = set() for member in members: try: - matched = {entry for entry in source.vocabulary - if re.fullmatch(member, entry, - re.IGNORECASE)} - loose = [probe for probe in _NOT_VOCABULARY - if re.fullmatch(member, probe, - re.IGNORECASE)] + pattern = re.compile(member, re.IGNORECASE) except re.error as exc: raise AssertionError( f"{ledger.name}: {rule['issue']!r} member " f"{member!r} is not a valid regex ({exc}). A " f"mis-split alternation can produce this -- see " f"_ALTERNATION's notes") from None + matched = {entry for entry in source.vocabulary + if pattern.fullmatch(entry)} + loose = _reaches_non_vocabulary(member, source.vocabulary) assert not loose, ( f"{ledger.name}: {rule['issue']!r} member " - f"{member!r} also matches ordinary name text " - f"{loose}. Spell it literally -- a member broad " - f"enough to hit a real name lets the rule claim " - f"that name's diff as intended") + f"{member!r} matches corpus text that is not " + f"vocabulary: {loose[:6]}. Spell it literally -- a " + f"member broad enough to reach a real name lets the " + f"rule claim that name's diff as intended") assert matched, ( f"{ledger.name}: {rule['issue']!r} offers the " f"alternative {member!r}, which matches no entry in " @@ -897,3 +969,76 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: assert used == set(_LATIN_ALTERNATION_SOURCES), ( f"_LATIN_ALTERNATION_SOURCES keys matching no rule: " f"{sorted(set(_LATIN_ALTERNATION_SOURCES) - used)}") + + +#: A role a rule can claim, mapped to the vocabulary a name must carry +#: for that claim to be possible. Keyed on `fields` rather than on the +#: regex, which is the point: the two pins above discover hand copies +#: by their SYNTAX, so a copy that is not written as an alternation or +#: a character class is not undeclared, it is unseen. A brand-new rule +#: whose name_regex is "(?i)\b[a-z]{3}\b" with maiden fields passed +#: every check in this module while claiming 320 corpus names. +#: +#: `fields` cannot be dodged the same way. A rule that does not claim +#: `maiden` cannot be labelled a maiden change at all, so keying here +#: reaches every spelling -- including fix(cjk-maiden-marker), whose +#: regex is the bare literal "旧姓" and which no roster in this module +#: could see. +#: +#: Only `maiden` today, because only its vocabulary is small and +#: mandatory enough for the implication to hold: a maiden diff needs a +#: marker in the name. `suffix` and `title` are not like that -- most +#: of their diffs come from routing, not from a vocabulary word being +#: present -- so adding them would be false rather than strict. +_FIELD_VOCABULARIES: dict[str, set[str]] = { + "maiden": MAIDEN_MARKERS, +} + + +def _carries(name: str, vocabulary: set[str]) -> bool: + """Whether a name contains a vocabulary entry. + + Whole-token for entries that have token boundaries, substring for + the ones that do not: 旧姓 is written against the name it marks. + """ + tokens = {_normalize(token) for token in name.split()} + return bool(tokens & vocabulary) or any( + entry in name for entry in vocabulary if not entry.isascii()) + + +def test_rules_claiming_a_vocabulary_role_need_the_vocabulary_present() -> None: + """The guard that does not care how a rule is spelled. + + Every pin above starts from regex syntax -- an alternation, a + character class, a span. Each closed the hole it was built for and + left the next spelling open, four rounds running. This one starts + from what the rule CLAIMS: if a rule says a diff is a maiden-marker + change, then every corpus name it claims must actually carry a + maiden marker. A rule cannot escape that by changing notation, + because it is not reading the notation. + + Deliberately narrow. It does not say the rule is correct, only that + it cannot be explaining a marker on a name that has none -- which + is exactly the shape every widening in this PR's review took. + """ + checked = 0 + for ledger in _LEDGERS: + for rule in _rules(ledger): + regex = rule.get("name_regex") + if not isinstance(regex, str): + continue + for field, vocabulary in _FIELD_VOCABULARIES.items(): + if field not in (rule.get("fields") or []): + continue + checked += 1 + bare = [name for name in _claimed(regex) + if not _carries(name, vocabulary)] + assert not bare, ( + f"{ledger.name}: {rule['issue']!r} claims {field!r} " + f"diffs on {len(bare)} corpus names carrying no {field} " + f"vocabulary, e.g. {bare[:3]}. It cannot be explaining a " + f"marker that is not there, so on those names it would " + f"absorb a regression instead") + assert checked, ( + "no rule claims a role in _FIELD_VOCABULARIES; this pin is passing " + "vacuously") From 6ad0e5fb4c79ff3879a230be82ada4ddc9aaa07e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 14:12:41 -0700 Subject: [PATCH 08/10] =?UTF-8?q?Correct=20the=20=E6=97=A7=E5=A7=93=20rati?= =?UTF-8?q?onale,=20for=20the=20third=20and=20last=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three rounds, three wrong versions of the same explanation, while the correct one sat 30 lines below in the sibling rule's own note. Round 2 said 旧姓 "has its own rule in the 2.0 ledger, so claiming it here too would make two rules answer for one diff" -- wrong file, and a mechanism that cannot happen, since compare.py loads one ledger per run. Round 3 said the 旧姓 rule "sorts AFTER this one, so widening here would shadow it" -- the sort order is right and the consequence is wrong: classify() needs the diff's fields to be a subset of the rule's, and both 旧姓 corpus names diff on `given`, which fix(#274) does not claim. Widening it to 旧姓 is inert, not shadowing. Which is exactly what fix(cjk-maiden-marker)'s own note has said all along: #274's fields stop short of `given`, and that is why the separate rule exists. Both copies now point there instead of inventing a third mechanism. Also: "the other 13" was 12 once 旧姓 is accounted for separately; the roster comment said "this rule" three times while the roster now holds two; the member-loop docstring counted three assertions where two are per-member and one is per-rule; and "both readers" was four call sites across three files. AGENTS.md promised that "a copy the rosters cannot account for fails loudly rather than going unpinned". That is true of the field-keyed corpus check and false of the two syntax-keyed rosters -- a hand copy that is not an alternation or a span class is invisible to them, as fix(cjk-maiden-marker)'s bare 旧姓 literal was until this branch. The step now says which mechanism covers what. --- AGENTS.md | 8 +++--- tests/v2/test_regex_sync.py | 26 ++++++++++---------- tools/differential/expected_since_1.4.0.toml | 10 +++++--- tools/differential/expected_since_2.1.0.toml | 3 ++- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index eb7bf9e4..399e920b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,9 +89,11 @@ uv run sphinx-build -b html docs dist/docs # previous one, and _allowlist_for hard-errors on the missing file. # tests/v2/test_regex_sync.py sweeps every expected_since_*.toml (#333), so # the new ledger's hand copies of _SCRIPT_RANGES and of the honorific and -# Latin vocabularies are checked from the day the file lands. Discovery -# finds the copies; the rosters record what they mirror, and a copy the -# rosters cannot account for fails loudly rather than going unpinned: +# Latin vocabularies are checked from the day the file lands. Two of the +# three rosters find copies by their SYNTAX -- an alternation, a span +# class -- so a copy spelled some other way is not undeclared but unseen; +# what catches those is the corpus check keyed on `fields`, which no +# notation escapes. Enrol the new ledger in whichever apply: # - _SPAN_BEARING_RULES: add the filename, mapped to the set of issue # tags whose rules carry a script-span class (empty set if none). # - _HONORIFIC_SOURCES: if the ledger has a CJK honorific rule, add a diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index ba94ad16..636f1b9b 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -818,16 +818,15 @@ class _LatinCopy(NamedTuple): #: to compare against. #: #: `covers` is recorded rather than equated to the whole vocabulary. -#: Equality would force the rule to grow alternatives for markers it has -#: no reason to claim, and 旧姓 already has a dedicated rule further down -#: this same ledger which sorts AFTER this one -- so widening here would -#: shadow that rule rather than complement it. Recording still catches +#: Equality would force a rule to grow alternatives for markers it has +#: no reason to claim -- see the note on fix(#274) in the 1.4 ledger for +#: why 旧姓 in particular is not one of them. Recording still catches #: removal: drop an entry a member covers and the snapshot shrinks. #: -#: Three nearby counts differ and are easy to conflate: MAIDEN_MARKERS -#: ships 17 entries; this rule's members reach 4 of them; the corpora -#: contain 3 markers in total (geb, née, 旧姓), only 2 of which this -#: rule covers. +#: Three nearby counts differ and are easy to conflate, all for +#: fix(#274) specifically: MAIDEN_MARKERS ships 17 entries; that rule's +#: members reach 4 of them; the corpora contain 3 markers in total +#: (geb, née, 旧姓), only 2 of which it covers. _LATIN_ALTERNATION_SOURCES: dict[str, _LatinCopy] = { "fix(#274)": _LatinCopy( vocabulary=MAIDEN_MARKERS, @@ -892,13 +891,14 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: for the entry "geb", which the config stores normalized (lowercase, no trailing period). - Three ways a member can be wrong, so three assertions. It can match + Two ways a MEMBER can be wrong, checked per member: it can match nothing in the vocabulary -- then it cannot describe a real change and can only claim other names' diffs, which is how "born" survived - from the harness's first commit to #350. It can match ordinary name - text as well as the vocabulary. And the rule around it can widen at - depth 0, leaving the pinned alternation governing one branch of an - unchecked whole -- the same hatch the span-bearing sweep closes. + from the harness's first commit to #350 -- or it can reach corpus + text that is not vocabulary. A third assertion is about the RULE + around them, which can widen at depth 0 and leave the pinned + alternation governing one branch of an unchecked whole. An invalid + member raises rather than asserts, since nothing else can proceed. """ has_classified = _policy._script_matcher(*_policy._SCRIPT_RANGES) used: set[str] = set() diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 9f69f3c9..b85165cb 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -70,10 +70,12 @@ issue = "fix(#274) maiden markers consumed" # Its members reach 4 of the 17 entries -- geb, nee, née, roz -- which # is not the same set as the markers the corpora contain (geb, née and # 旧姓; only the first two are covered here). Growing it toward the -# other 13 is not obviously right: the 旧姓 rule below sorts AFTER this -# one, so covering 旧姓 here would shadow it rather than complement it, -# and the rest have no corpus name to classify. Widen deliberately, and -# record the new coverage in _LATIN_ALTERNATION_SOURCES. +# other 12 buys nothing: none appears as a token in any corpus, so +# there is no diff for them to classify. Widening it to 旧姓 buys +# nothing either, and the reason is on the rule below rather than here +# -- this rule's `fields` stop short of `given`, so it can never claim +# a name the Han marker moves. Widen deliberately, and record the new +# coverage in _LATIN_ALTERNATION_SOURCES. # # 'born' was an alternative here from the harness's first commit and # was removed in #350. It is not in MAIDEN_MARKERS and never was, in diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index 3b9cf311..ffb1bd4c 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -35,7 +35,8 @@ # There is deliberately no `change = []` line. TOML forbids appending a # [[change]] table to a statically defined array, so that line would # block the exact next step this comment asks for -- and both readers -# use .get("change", []), so the key's absence is already the empty +# every reader uses .get("change", []) -- four call sites across +# three files -- so the key's absence is already the empty # ledger. tests/v2/test_differential.py checks that nothing else is # defined at the top level here, since a mistyped table name would # otherwise read as an empty ledger rather than as a broken one. From 85b70ebc6a79bd2ced328b6f1705f38fca844908 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 15:22:25 -0700 Subject: [PATCH 09/10] Bound every rule by the corpus it claims, not by its category Round five defeated the round-four guards twice more, and the pattern in the FIXES is now as clear as the one in the defects. Each guard was scoped to a category -- a span class, an alternation's members, a `maiden` field, a member's reach -- and each attack simply moved to a rule outside whichever category the last fix covered. feat(#273) is a nickname-delimiter character class: no span, so the span guard skips it; no alternation, so the vocabulary roster never discovers it; no `maiden` field, so the field guard skips it. Append A-Za-z and it goes from claiming 6 corpus names to 668, with the whole 2232-test suite green. And deleting the acronym rule's delimiter classes -- leaving its members untouched -- takes it from 0 to 193, because the members really do reach those names on their own; the narrowing lived in the context around them, which nothing measured. The categories are the test's, not the ledger's. What every rule shares is how much of the corpus it claims, and no widening can change what a rule matches without changing that. So _CORPUS_CLAIMS records the number per rule, per ledger, and refuses to let it move quietly. It is deliberately dumb. It knows nothing about vocabularies, scripts or roles and cannot say whether a number is right -- only that it moved. The specific guards stay, because they explain WHY a rule may claim what it claims and their messages are the ones worth reading; this is the backstop none of them could be. Keyed on the full `issue`, since the 1.4 ledger has two rules tagged feat(#269) and a tag-keyed roster cannot tell them apart. Verified by replaying every attack from all five rounds: both of round five's, the nested-and-widened acronym, [acdf-uw-z]{3,}, depth-0 |[A-Za-z], the class-with-no-alternation maiden rule, and 'born'. All fail. The counts move when the CORPORA move too, which is the intended cost -- a corpus name landing under an existing rule changes what that rule explains, and should be read once rather than absorbed. Also replaces the module's own "> 500" corpus floor with compare.py's per-file _CORPUS_FLOORS. A total cannot see a file vanish: emptying corpus_issues.jsonl left 583 names and every test green, silently dropping a quarter of the population every guard here measures. --- AGENTS.md | 11 +- tests/v2/test_regex_sync.py | 247 +++++++++++++++++-- tools/differential/expected_since_1.4.0.toml | 18 +- tools/differential/expected_since_2.1.0.toml | 2 +- 4 files changed, 247 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 399e920b..90d1c64a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,11 +89,12 @@ uv run sphinx-build -b html docs dist/docs # previous one, and _allowlist_for hard-errors on the missing file. # tests/v2/test_regex_sync.py sweeps every expected_since_*.toml (#333), so # the new ledger's hand copies of _SCRIPT_RANGES and of the honorific and -# Latin vocabularies are checked from the day the file lands. Two of the -# three rosters find copies by their SYNTAX -- an alternation, a span -# class -- so a copy spelled some other way is not undeclared but unseen; -# what catches those is the corpus check keyed on `fields`, which no -# notation escapes. Enrol the new ledger in whichever apply: +# Latin vocabularies are checked from the day the file lands. All three +# rosters find copies by their SYNTAX -- a span class or an alternation -- +# so a copy spelled some other way is not undeclared but unseen. What +# covers those is the pair of corpus checks: what a rule claims beyond +# its own vocabulary members, and what it claims without the vocabulary +# its `fields` assert. Enrol the new ledger in whichever apply: # - _SPAN_BEARING_RULES: add the filename, mapped to the set of issue # tags whose rules carry a script-span class (empty set if none). # - _HONORIFIC_SOURCES: if the ledger has a CJK honorific rule, add a diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index 636f1b9b..a1f63da7 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -12,10 +12,13 @@ Layering is the usual reason for a copy but not the only one, so this module's scope is the PROMISE rather than that one pair of packages: the comma-set pin below reads _pipeline._state instead of config, and -several tests reach outside the package altogether -- four to the +several tests reach outside the package altogether. Seven read the differential ledgers, which could not import a Python constant if they -wanted to, one to a generated corpus whose generator can, and must -stay run. +wanted to; one pins a generated corpus against its generator, which +can and must stay run; and four more read the corpora as a DATA +POPULATION rather than as an artifact -- asking what a ledger rule +actually claims, which is the question four rounds of syntactic guards +could not answer. """ import importlib.util import json @@ -31,6 +34,13 @@ from nameparser import _policy from nameparser._policy import Script from nameparser import _render +# The parser's own fold, imported rather than reimplemented: a +# hand-written one here stripped commas, parens, brackets and +# quotes, five classes neither the lexicon's fold nor config's +# assert_normalized touches -- looser in the dangerous direction, +# and a hand copy of a constant with a source of truth, inside the +# module written to forbid exactly that. +from nameparser._lexicon import _normalize from nameparser.config.maiden_markers import MAIDEN_MARKERS from nameparser.config.suffixes import ( GLUED_HONORIFICS, SUFFIX_ACRONYMS_AMBIGUOUS, SUFFIX_NOT_ACRONYMS) @@ -215,15 +225,47 @@ def _unclassified_names() -> list[str]: return [name for name in _CORPUS_NAMES if not has_classified(name)] -def _normalize(text: str) -> str: - """How the config stores vocabulary: lowercase, no edge punctuation.""" - return text.strip(".,()[]\"'\u2019 ").lower() +#: Built once: the guard that reads it runs per rule, and the +#: matcher rebuild is 700x the cost of the lookup on the failure +#: path, which is the path that matters. +_UNCLASSIFIED_NAMES = frozenset(_unclassified_names()) -def test_corpus_is_loaded() -> None: - """The guards below all reduce to zero findings over an empty - corpus, which is what a silent load failure produces.""" - assert len(_CORPUS_NAMES) > 500, len(_CORPUS_NAMES) +def test_every_corpus_meets_the_floor_compare_py_records() -> None: + """Per file, against compare.py's own floors rather than a total. + + A total cannot see a file disappear. Emptying or renaming + corpus_issues.jsonl -- 200 names -- leaves corpus.jsonl and + corpus_cjk.jsonl summing to 583, so a global "> 500" stays green + while every guard in this module silently stops asking about a + quarter of the population. compare.py already solved this for + itself with per-file floors, for the reason its own comment gives: + "without a floor a corpus can shrink to a handful of names and the + run still exits 0." + + Reusing that constant rather than restating it also inherits its + forcing function -- a corpus file with no floor is a hard error, so + a new corpus cannot join unnoticed -- and keeps one set of numbers. + """ + spec = importlib.util.spec_from_file_location( + "differential_compare", _TOOLS / "compare.py") + assert spec is not None and spec.loader is not None + compare = importlib.util.module_from_spec(spec) + spec.loader.exec_module(compare) + + present = {path.name: sum(1 for line in + path.read_text(encoding="utf-8").splitlines() + if line.strip()) + for path in sorted(_TOOLS.glob("corpus*.jsonl"))} + assert set(present) == set(compare._CORPUS_FLOORS), ( + f"corpora on disk {sorted(present)} do not match the files " + f"compare.py records floors for {sorted(compare._CORPUS_FLOORS)}; a " + f"renamed or added corpus silently changes what every guard in this " + f"module measures") + for name, floor in compare._CORPUS_FLOORS.items(): + assert present[name] >= floor, ( + f"{name} holds {present[name]} names, below compare.py's " + f"recorded floor of {floor}") assert _unclassified_names(), "no unclassified names; guard A is inert" @@ -559,16 +601,16 @@ def test_every_span_bearing_rule_matches_the_script_ranges( # written in none of them -- and unlike a depth test, this does # not care how the widening is spelled. "(?:CJK|[A-Za-z])" # hides the pipe at depth 1 where the check above stops - # looking, and claims 665 unclassified corpus names; this sees + # looking, and claims 644 of the 654 unclassified corpus names; + # this sees # it. Both are kept: the depth test gives the clearer message # for the naive spelling, and catches a widening toward a # script the corpora happen not to contain. - reached = _claimed(regex) - latin = [name for name in reached if name in set(_unclassified_names())] - assert not latin, ( + unclassified = _UNCLASSIFIED_NAMES.intersection(_claimed(regex)) + assert not unclassified, ( f"{ledger.name}: {rule['issue']!r} declares the script table's " - f"spans but claims {len(latin)} corpus names carrying no " - f"classified codepoint at all, e.g. {latin[:3]}. A rule scoped " + f"spans but claims {len(unclassified)} corpus names carrying " + f"no classified codepoint at all, e.g. {sorted(unclassified)[:3]}. A rule scoped " f"to these scripts cannot explain a diff on those names, so it " f"would absorb one instead") # Every rule, not just the discovered ones: this is what stops a @@ -844,6 +886,32 @@ class _LatinCopy(NamedTuple): frozenset({"^", " "}), # the honorific rule's leading anchor }) +def _unjustified_reach(name_regex: str, members: set[str]) -> list[str]: + """Corpus names the whole rule claims that none of its own members + reach. + + The member checks bound what each ALTERNATIVE matches. This bounds + the rule built around them, which is a different question and the + one four rounds of syntactic guards kept failing to ask. Nesting a + rule's own alternation one level down and adding a branch -- + "(?:[(\"'](m\\.?a\\.?|d\\.?o\\.?)[)\"']|[A-Za-z]{4,})" -- leaves every + member innocent, hides the pipe below the depth test, and hides the + outer group from _ALTERNATION, which refuses nested parens. It + claimed 622 of the corpus while every check passed. + + Keyed on the roster rather than on `fields`, which is what makes it + reach that rule: the acronym copy claims `suffix`/`nickname`, so + the field-keyed guard below never looks at it. + + A rule claiming nothing scores zero, which is the strongest answer + rather than a vacuous one -- so the non-vacuity assertion is over + the union of all rostered rules, not per rule. + """ + reachable = [re.compile(member, re.IGNORECASE) for member in members] + return [name for name in _claimed(name_regex) + if not any(pattern.search(name) for pattern in reachable)] + + def _reaches_non_vocabulary(member: str, vocabulary: set[str]) -> list[str]: """Corpus text this member matches that is NOT a vocabulary entry. @@ -854,7 +922,9 @@ def _reaches_non_vocabulary(member: str, vocabulary: set[str]) -> list[str]: one it was added to stop: every entry this rule needs is three characters, so a member must accept some 3-character string and is unconstrained everywhere else -- "[acdf-uw-z]{3,}" covers `roz`, - dodges all eight probes, and claims 563 of the corpus. + dodges all eight probes, and reaches 592 of the 751 corpus names + (the rule carrying it claims 542). Counts here and below are + against _CORPUS_NAMES, which deduplicates the 783 corpus lines. Eight strings could never be more than a spot check. The corpus is the whole population the rule will ever be asked about, so ask it @@ -903,6 +973,7 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: has_classified = _policy._script_matcher(*_policy._SCRIPT_RANGES) used: set[str] = set() found = 0 + reach_checked = 0 for ledger in _LEDGERS: for rule in _rules(ledger): regex = rule.get("name_regex") @@ -925,6 +996,14 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: f"nothing") used.add(keys[0]) source = _LATIN_ALTERNATION_SOURCES[keys[0]] + unjustified = _unjustified_reach(regex, members) + assert not unjustified, ( + f"{ledger.name}: {rule['issue']!r} claims " + f"{len(unjustified)} corpus names that none of its own " + f"vocabulary members reach, e.g. {unjustified[:3]}. The " + f"members are innocent and the rule around them is not " + f"-- whatever it matches beyond them, it claims") + reach_checked += len(_claimed(regex)) assert not _top_level_alternation(regex), ( f"{ledger.name}: {rule['issue']!r} has a '|' at depth " f"0, so the pinned alternation governs only one branch " @@ -966,6 +1045,9 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: assert found, ( "no Latin vocabulary alternation found in any ledger; this pin is " "passing vacuously") + assert reach_checked, ( + "no rostered rule claims any corpus name, so the reach check above " + "measured nothing -- verify the corpora loaded") assert used == set(_LATIN_ALTERNATION_SOURCES), ( f"_LATIN_ALTERNATION_SOURCES keys matching no rule: " f"{sorted(set(_LATIN_ALTERNATION_SOURCES) - used)}") @@ -977,7 +1059,7 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: #: by their SYNTAX, so a copy that is not written as an alternation or #: a character class is not undeclared, it is unseen. A brand-new rule #: whose name_regex is "(?i)\b[a-z]{3}\b" with maiden fields passed -#: every check in this module while claiming 320 corpus names. +#: every check in this module while claiming 308 corpus names. #: #: `fields` cannot be dodged the same way. A rule that does not claim #: `maiden` cannot be labelled a maiden change at all, so keying here @@ -998,8 +1080,18 @@ def test_latin_alternations_mean_something_the_vocabulary_ships() -> None: def _carries(name: str, vocabulary: set[str]) -> bool: """Whether a name contains a vocabulary entry. - Whole-token for entries that have token boundaries, substring for - the ones that do not: 旧姓 is written against the name it marks. + Whole-token for ASCII entries, substring for the rest, because a + marker like 旧姓 is written against the name it marks rather than + spaced off it. + + Note what the isascii() split actually covers: 12 of the 17 + entries, not only the CJK one. `né` is two characters, so the + substring branch reads `René` as carrying a marker. Every + over-match here SHRINKS the set of unexplained names and so + weakens the guard -- the direction this module exists to close -- + but exactly one corpus name reaches that branch today, and it is + the 旧姓 one. Tighten this before admitting a vocabulary whose + short non-ASCII entries occur inside ordinary names. """ tokens = {_normalize(token) for token in name.split()} return bool(tokens & vocabulary) or any( @@ -1042,3 +1134,118 @@ def test_rules_claiming_a_vocabulary_role_need_the_vocabulary_present() -> None: assert checked, ( "no rule claims a role in _FIELD_VOCABULARIES; this pin is passing " "vacuously") + + +#: How many corpus names each rule's name_regex matches, per ledger. +#: +#: The backstop the other guards each failed to be. Every one of them +#: is scoped to a CATEGORY -- a span class, an alternation's members, a +#: `maiden` field, a member's reach -- and five review rounds each found +#: a rule outside whichever category the last fix had covered. The +#: categories are the test's, not the ledger's. What every rule shares +#: is how much of the corpus it claims, and no widening can change what +#: a rule matches without changing that. +#: +#: So this is deliberately dumb: it knows nothing about vocabularies, +#: scripts or roles, and it cannot say whether a number is RIGHT. It +#: says only that it moved, which is the question a widening cannot +#: dodge. The specific guards above stay because they explain WHY a +#: rule may claim what it claims, and their messages are the ones worth +#: reading; this one just refuses to let the number change quietly. +#: +#: Keyed by the full `issue` rather than by tag: the 1.4 ledger has two +#: rules tagged feat(#269), and a tag-keyed roster cannot tell them +#: apart -- the same identity-free weakness recorded at +#: _SPAN_BEARING_RULES. +#: +#: These numbers move when the CORPORA move, not only when a rule does. +#: That is the intended cost: a corpus name added under an existing +#: rule is a real change in what that rule explains, and it should be +#: read once rather than absorbed silently. +_CORPUS_CLAIMS: dict[str, dict[str, int]] = { + "expected_since_1.4.0.toml": { + "fix(#271/#272/#298) native-script CJK: family-first order, hangul segmentation, the kana license and the dots": + 97, + "fix(#274) maiden markers consumed": + 4, + "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip": + 3, + "fix(comma-family) lone post-comma piece routes to suffix/title, not first": + 236, + "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": + 0, + "ambiguous-surname-acronym data change: parenthesized (MA)/(DO) now stays nickname": + 0, + "feat(#269) Arabic بن prefix chains onto family (non-Latin new-recognition)": + 2, + "feat(#273) typographic nickname delimiters recognized by default": + 6, + "fix(cjk-delimited-nickname) delimiter recognition compounds with the CJK order flip": + 6, + "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition compounds with the CJK order flip": + 1, + "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": + 20, + "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": + 14, + "feat(#269) non-Latin titles/conjunctions recognized": + 2, + "fix(leading-credential) a split 'Ph. D.' before the name stays one unit": + 1, + }, + "expected_since_2.0.0.toml": { + "fix(#271/#272/#298) native-script CJK: family-first order, hangul segmentation, the kana license and the dots": + 97, + "fix(#308/#312/#319/#320) glued CJK honorific peeled off the name into suffix": + 34, + "fix(#307/#308/#320) spaced CJK postnominal honorific routed to suffix": + 16, + "fix(#309) 旧姓 maiden marker consumed, compounding with the CJK order flip": + 3, + "fix(#272) nakaguro inside delimited content renders as a space, compounding with the CJK order flip": + 1, + "fix(#298) 间隔号 division changes the comma reading, sending the credential from title to suffix": + 1, + }, + "expected_since_2.1.0.toml": {}, +} + + +def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: + """The guard that knows nothing, and therefore cannot be dodged. + + Five rounds of review defeated five guards, each by moving to a + rule the last fix did not cover: a span rule, then an alternation's + members, then a non-maiden role, then a rule whose narrowing lived + outside its alternation entirely. Every one of those attacks + changed how much corpus the rule claimed -- 4 to 675, 0 to 193, 6 + to 668 -- because that is what widening a rule MEANS. + + A count is identity-free, which this module rejects elsewhere for + good reason. It is acceptable here because it is a backstop and not + the explanation: the guards above name what is wrong, and this one + only insists that nothing moved unnoticed. A rule can still be + wrong at a stable count -- it just cannot become wrong in the one + way five rounds of review actually found. + """ + for ledger in _LEDGERS: + assert ledger.name in _CORPUS_CLAIMS, ( + f"{ledger.name} is a new ledger with no recorded corpus claims; " + f"add it to _CORPUS_CLAIMS (an empty mapping if it has no rules)") + recorded = _CORPUS_CLAIMS[ledger.name] + actual = {rule["issue"]: len(_claimed(rule["name_regex"])) + for rule in _rules(ledger) + if isinstance(rule.get("name_regex"), str)} + moved = {issue: (recorded.get(issue), count) + for issue, count in actual.items() + if recorded.get(issue) != count} + assert actual == recorded, ( + f"{ledger.name}: the corpus each rule claims is not what is " + f"recorded. Moved (recorded, now): {moved}. Gone: " + f"{sorted(set(recorded) - set(actual))}. New: " + f"{sorted(set(actual) - set(recorded))}. A number that GREW " + f"means the rule claims more of the corpus than it did -- " + f"check it is not absorbing a regression, then record it") + assert set(_CORPUS_CLAIMS) == {led.name for led in _LEDGERS}, ( + f"_CORPUS_CLAIMS names ledgers that do not exist: " + f"{sorted(set(_CORPUS_CLAIMS) - {L.name for L in _LEDGERS})}") diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index b85165cb..36636572 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -71,11 +71,19 @@ issue = "fix(#274) maiden markers consumed" # is not the same set as the markers the corpora contain (geb, née and # 旧姓; only the first two are covered here). Growing it toward the # other 12 buys nothing: none appears as a token in any corpus, so -# there is no diff for them to classify. Widening it to 旧姓 buys -# nothing either, and the reason is on the rule below rather than here -# -- this rule's `fields` stop short of `given`, so it can never claim -# a name the Han marker moves. Widen deliberately, and record the new -# coverage in _LATIN_ALTERNATION_SOURCES. +# there is no diff for them to classify. +# +# 旧姓 is a different case, and it has been got wrong three times, so +# state it precisely. Widening this rule to cover it changes no corpus +# classification -- every 旧姓 name in the corpora leaves a wholly Han +# residual, which reads family-first (#271), so its diff includes +# `given`, and classify() requires the diff's fields to be a subset of +# the rule's. That is a fact about the CORPUS, not about the rules: a +# 旧姓 name with a Latin residual would diff without `given`, and since +# this rule sorts ahead of fix(cjk-maiden-marker) in the same tier it +# would take that name from it. Inert today, shadowing in principle. +# Widen deliberately, and record the new coverage in +# _LATIN_ALTERNATION_SOURCES. # # 'born' was an alternative here from the harness's first commit and # was removed in #350. It is not in MAIDEN_MARKERS and never was, in diff --git a/tools/differential/expected_since_2.1.0.toml b/tools/differential/expected_since_2.1.0.toml index ffb1bd4c..cb5f5bf4 100644 --- a/tools/differential/expected_since_2.1.0.toml +++ b/tools/differential/expected_since_2.1.0.toml @@ -34,7 +34,7 @@ # # There is deliberately no `change = []` line. TOML forbids appending a # [[change]] table to a statically defined array, so that line would -# block the exact next step this comment asks for -- and both readers +# block the exact next step this comment asks for -- and # every reader uses .get("change", []) -- four call sites across # three files -- so the key's absence is already the empty # ledger. tests/v2/test_differential.py checks that nothing else is From ae7b3d9dbe2f0f5169af7f7af87336f388a75159 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 8 Aug 2026 15:56:43 -0700 Subject: [PATCH 10/10] Record what a rule claims in all three dimensions classify() uses Round six defeated the count three ways, and the root of all three is that a count of regex matches is not what a rule claims. classify() narrows on `name_regex` AND `fields`. _CORPUS_CLAIMS recorded only the first, so widening a rule's roles moved nothing it could see: the comma rule kept its `,` regex and its 236 names while going from explaining 6 of the corpus to 242. The fields-only rule -- no name_regex at all, the most unbounded shape validate_rules permits -- was not recorded at all, and widening it to six roles absorbed 751/751 on every diff shape not touching `title`. A count is also identity-free, the weakness this module rejects at _SPAN_BEARING_RULES. Swapping feat(#273)'s delimiter class for a single accented letter held the count at 6 while claiming six entirely different names. And it was keyed on `issue` in a dict comprehension, so a duplicate silently kept the last rule written. Nothing asserts issue uniqueness -- validate_rules wants only a non-empty string -- and the collision is worse than it sounds: _sorted_rules is stable, so FIRST in file is the position classify reaches, and first in file is exactly what the comprehension discards. Coverage by file order is no coverage. So record a _Claim: the regex's corpus reach, the sorted roles, and a digest of which names. Plus the uniqueness assert, which the two substring-keyed rosters lean on as well. Also corrects the framing, which had the layers backwards. This is a change DETECTOR, not an enforcer -- inert for a new rule, and its own message invites the re-record that defeats it. The member and vocabulary guards are the walls, because they judge a rule wrong at any time including at recording time. This catches what none of them is scoped to see and holds it still long enough to be read. Deletes the duplicate corpus-floor test, which restated two tests test_differential.py already owns and re-copied their compare.py loader to do it. What is local to this module is the deduplicated population its guards measure, so that is what stays. --- AGENTS.md | 11 +- tests/v2/test_regex_sync.py | 216 ++++++++++++++++++++++-------------- 2 files changed, 143 insertions(+), 84 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 90d1c64a..da54fbdc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,9 +92,10 @@ uv run sphinx-build -b html docs dist/docs # Latin vocabularies are checked from the day the file lands. All three # rosters find copies by their SYNTAX -- a span class or an alternation -- # so a copy spelled some other way is not undeclared but unseen. What -# covers those is the pair of corpus checks: what a rule claims beyond -# its own vocabulary members, and what it claims without the vocabulary -# its `fields` assert. Enrol the new ledger in whichever apply: +# covers those is _CORPUS_CLAIMS, which records what every rule claims +# -- its regex's corpus reach, its roles, and which names -- and so +# needs no notion of how a copy is spelled. Enrol the new ledger in +# _CORPUS_CLAIMS always, and in the others where they apply: # - _SPAN_BEARING_RULES: add the filename, mapped to the set of issue # tags whose rules carry a script-span class (empty set if none). # - _HONORIFIC_SOURCES: if the ledger has a CJK honorific rule, add a @@ -103,6 +104,10 @@ uv run sphinx-build -b html docs dist/docs # that issue. A retroactive ledger can repeat an older one's rule # verbatim (fix(#271/#272/#298) is in both today), and every rule # must match exactly one key. +# - _CORPUS_CLAIMS: REQUIRED, not conditional -- a ledger with no +# entry hard-fails. Add the filename mapped to {} while the ledger +# is empty, then one _Claim per rule as rules land. The test prints +# the values to record. # - _LATIN_ALTERNATION_SOURCES: same, for a rule copying a Latin # vocabulary (maiden markers, ambiguous acronyms). An alternation # matching no key fails as undeclared -- add it, or record it in diff --git a/tests/v2/test_regex_sync.py b/tests/v2/test_regex_sync.py index a1f63da7..8c09fef1 100644 --- a/tests/v2/test_regex_sync.py +++ b/tests/v2/test_regex_sync.py @@ -12,14 +12,15 @@ Layering is the usual reason for a copy but not the only one, so this module's scope is the PROMISE rather than that one pair of packages: the comma-set pin below reads _pipeline._state instead of config, and -several tests reach outside the package altogether. Seven read the +several tests reach outside the package altogether: eight read the differential ledgers, which could not import a Python constant if they -wanted to; one pins a generated corpus against its generator, which -can and must stay run; and four more read the corpora as a DATA +wanted to, and four of those eight also read the corpora as a DATA POPULATION rather than as an artifact -- asking what a ledger rule -actually claims, which is the question four rounds of syntactic guards -could not answer. +actually claims, which is the question no amount of inspecting its +syntax could answer. One more pins a generated corpus against its +generator, which can and must stay run. """ +import hashlib import importlib.util import json import re @@ -225,48 +226,35 @@ def _unclassified_names() -> list[str]: return [name for name in _CORPUS_NAMES if not has_classified(name)] -#: Built once: the guard that reads it runs per rule, and the -#: matcher rebuild is 700x the cost of the lookup on the failure -#: path, which is the path that matters. +#: Built once. The expression this replaced sat inside a +#: comprehension's condition, so it rebuilt the script matcher AND +#: rescanned all 751 names per candidate name rather than per rule -- +#: measured around 400x a frozenset lookup, machine-dependent. The +#: rescan was the cost; the rebuild alone is minor. _UNCLASSIFIED_NAMES = frozenset(_unclassified_names()) -def test_every_corpus_meets_the_floor_compare_py_records() -> None: - """Per file, against compare.py's own floors rather than a total. +def test_the_corpus_population_is_not_degenerate() -> None: + """The floors themselves live in compare.py and are asserted by + tests/v2/test_differential.py, which already checks every shipped + corpus clears one and that every floor names a file that exists. + Restating that here was a second, independently-drifting copy of a + guarantee the harness owns. - A total cannot see a file disappear. Emptying or renaming - corpus_issues.jsonl -- 200 names -- leaves corpus.jsonl and - corpus_cjk.jsonl summing to 583, so a global "> 500" stays green - while every guard in this module silently stops asking about a - quarter of the population. compare.py already solved this for - itself with per-file floors, for the reason its own comment gives: - "without a floor a corpus can shrink to a handful of names and the - run still exits 0." - - Reusing that constant rather than restating it also inherits its - forcing function -- a corpus file with no floor is a hard error, so - a new corpus cannot join unnoticed -- and keeps one set of numbers. + What is local to THIS module is the population the guards actually + measure, which is not the same thing: _CORPUS_NAMES is + deduplicated, so a corpus rewritten as 486 copies of one line + clears its floor while the set collapses. And guard A is inert if + nothing in that set is unclassified. """ - spec = importlib.util.spec_from_file_location( - "differential_compare", _TOOLS / "compare.py") - assert spec is not None and spec.loader is not None - compare = importlib.util.module_from_spec(spec) - spec.loader.exec_module(compare) - - present = {path.name: sum(1 for line in - path.read_text(encoding="utf-8").splitlines() - if line.strip()) - for path in sorted(_TOOLS.glob("corpus*.jsonl"))} - assert set(present) == set(compare._CORPUS_FLOORS), ( - f"corpora on disk {sorted(present)} do not match the files " - f"compare.py records floors for {sorted(compare._CORPUS_FLOORS)}; a " - f"renamed or added corpus silently changes what every guard in this " - f"module measures") - for name, floor in compare._CORPUS_FLOORS.items(): - assert present[name] >= floor, ( - f"{name} holds {present[name]} names, below compare.py's " - f"recorded floor of {floor}") - assert _unclassified_names(), "no unclassified names; guard A is inert" + assert len(_CORPUS_NAMES) > 700, ( + f"_CORPUS_NAMES holds {len(_CORPUS_NAMES)} distinct names; the " + f"corpora clear their floors in compare.py but deduplicate to far " + f"fewer than usual, so every guard here is measuring a smaller " + f"population than it appears to") + assert _unclassified_names(), ( + "no corpus name lacks a classified codepoint, so the span rules' " + "unclassified-reach check has nothing to test against") def test_ledger_glob_is_not_empty() -> None: @@ -922,8 +910,10 @@ def _reaches_non_vocabulary(member: str, vocabulary: set[str]) -> list[str]: one it was added to stop: every entry this rule needs is three characters, so a member must accept some 3-character string and is unconstrained everywhere else -- "[acdf-uw-z]{3,}" covers `roz`, - dodges all eight probes, and reaches 592 of the 751 corpus names - (the rule carrying it claims 542). Counts here and below are + dodges all eight probes, and reaches 634 of the 751 corpus names + as a fourth alternative (the rule carrying it claims 542). + Measured with the IGNORECASE this function applies; the + case-sensitive figure, 592, is not what runs. Counts here and below are against _CORPUS_NAMES, which deduplicates the 783 corpus lines. Eight strings could never be more than a spot check. The corpus is @@ -1110,8 +1100,10 @@ def test_rules_claiming_a_vocabulary_role_need_the_vocabulary_present() -> None: because it is not reading the notation. Deliberately narrow. It does not say the rule is correct, only that - it cannot be explaining a marker on a name that has none -- which - is exactly the shape every widening in this PR's review took. + it cannot be explaining a marker on a name that has none. That + covers the maiden widenings review found; the ones claiming + `nickname` and `suffix` fall outside it, and _CORPUS_CLAIMS is + what catches those. """ checked = 0 for ledger in _LEDGERS: @@ -1136,6 +1128,41 @@ def test_rules_claiming_a_vocabulary_role_need_the_vocabulary_present() -> None: "vacuously") +class _Claim(NamedTuple): + """What a rule claims, in the three dimensions classify() uses. + + A count alone is identity-free -- the weakness this module rejects + at _SPAN_BEARING_RULES -- and review proved it here twice. Swapping + feat(#273)'s delimiter class for a single accented letter holds the + count at 6 while claiming six entirely different names. And + classify() narrows on `fields` as well as `name_regex`, so widening + a rule's roles moves nothing a regex-only count can see: the + comma rule kept its `,` regex and its 236 names while going from + explaining 6 of the corpus to 242. + """ + #: corpus names the name_regex reaches; the whole corpus when a + #: rule has none, which is the most unbounded shape validate_rules + #: permits and the one most worth writing down + names: int + #: the roles it narrows by, sorted; () when it narrows by regex alone + roles: tuple[str, ...] + #: sha256[:12] of the claimed names, so a swap that holds the count + #: still fails. Unreadable by design -- the count above is what a + #: reviewer reads, and the failure message prints what moved. + digest: str + + +def _claim(rule: dict) -> _Claim: + regex = rule.get("name_regex") + names = (_claimed(regex) if isinstance(regex, str) else list(_CORPUS_NAMES)) + fields = rule.get("fields") + return _Claim( + names=len(names), + roles=tuple(sorted(fields)) if isinstance(fields, list) else (), + digest=hashlib.sha256( + "\n".join(names).encode("utf-8")).hexdigest()[:12]) + + #: How many corpus names each rule's name_regex matches, per ledger. #: #: The backstop the other guards each failed to be. Every one of them @@ -1143,8 +1170,12 @@ def test_rules_claiming_a_vocabulary_role_need_the_vocabulary_present() -> None: #: `maiden` field, a member's reach -- and five review rounds each found #: a rule outside whichever category the last fix had covered. The #: categories are the test's, not the ledger's. What every rule shares -#: is how much of the corpus it claims, and no widening can change what -#: a rule matches without changing that. +#: is what it claims: how much corpus its regex reaches, which roles +#: it narrows by, and WHICH names those are. Scoped to the corpus -- +#: and only there -- a widening cannot change what a rule explains +#: without moving one of the three. Restoring 'born' moves none of +#: them, because no corpus name contains it; the member guards catch +#: that, which is why this does not replace them. #: #: So this is deliberately dumb: it knows nothing about vocabularies, #: scripts or roles, and it cannot say whether a number is RIGHT. It @@ -1162,50 +1193,52 @@ def test_rules_claiming_a_vocabulary_role_need_the_vocabulary_present() -> None: #: That is the intended cost: a corpus name added under an existing #: rule is a real change in what that rule explains, and it should be #: read once rather than absorbed silently. -_CORPUS_CLAIMS: dict[str, dict[str, int]] = { +_CORPUS_CLAIMS: dict[str, dict[str, _Claim]] = { "expected_since_1.4.0.toml": { "fix(#271/#272/#298) native-script CJK: family-first order, hangul segmentation, the kana license and the dots": - 97, + _Claim(97, ('family', 'given', 'middle'), "66e71d60a075"), "fix(#274) maiden markers consumed": - 4, + _Claim(4, ('family', 'maiden', 'middle'), "b31dc2e2bbc4"), "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip": - 3, + _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"), "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - 236, + _Claim(236, ('given', 'suffix', 'title'), "3416f69d0ce4"), + "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": + _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": - 0, + _Claim(0, ('suffix',), "e3b0c44298fc"), "ambiguous-surname-acronym data change: parenthesized (MA)/(DO) now stays nickname": - 0, + _Claim(0, ('nickname', 'suffix'), "e3b0c44298fc"), "feat(#269) Arabic بن prefix chains onto family (non-Latin new-recognition)": - 2, + _Claim(2, ('family', 'middle'), "3e2b5c6d1f4d"), "feat(#273) typographic nickname delimiters recognized by default": - 6, + _Claim(6, ('middle', 'nickname'), "a03c9763c8c4"), "fix(cjk-delimited-nickname) delimiter recognition compounds with the CJK order flip": - 6, + _Claim(6, ('family', 'given', 'nickname'), "ae1dffa01608"), "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition compounds with the CJK order flip": - 1, + _Claim(1, ('family', 'given', 'middle', 'nickname'), "cf370e856ae7"), "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": - 20, + _Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"), "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": - 14, + _Claim(14, ('family', 'given', 'middle', 'suffix'), "d49ce901bdce"), "feat(#269) non-Latin titles/conjunctions recognized": - 2, + _Claim(2, ('given', 'middle', 'title'), "c14187bb08f8"), "fix(leading-credential) a split 'Ph. D.' before the name stays one unit": - 1, + _Claim(1, ('given', 'middle', 'suffix', 'title'), "390e7f814d13"), }, "expected_since_2.0.0.toml": { "fix(#271/#272/#298) native-script CJK: family-first order, hangul segmentation, the kana license and the dots": - 97, + _Claim(97, ('_ambiguities', 'family', 'given', 'middle'), "66e71d60a075"), "fix(#308/#312/#319/#320) glued CJK honorific peeled off the name into suffix": - 34, + _Claim(34, ('family', 'given', 'suffix'), "877ab3246d33"), "fix(#307/#308/#320) spaced CJK postnominal honorific routed to suffix": - 16, + _Claim(16, ('family', 'given', 'middle', 'suffix'), "6d390e518bd2"), "fix(#309) 旧姓 maiden marker consumed, compounding with the CJK order flip": - 3, + _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"), "fix(#272) nakaguro inside delimited content renders as a space, compounding with the CJK order flip": - 1, + _Claim(1, ('family', 'given', 'nickname'), "d4069d459f23"), "fix(#298) 间隔号 division changes the comma reading, sending the credential from title to suffix": - 1, + _Claim(1, ('family', 'given', 'suffix', 'title'), "1d45596e6fdb"), }, "expected_since_2.1.0.toml": {}, } @@ -1218,24 +1251,45 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: rule the last fix did not cover: a span rule, then an alternation's members, then a non-maiden role, then a rule whose narrowing lived outside its alternation entirely. Every one of those attacks - changed how much corpus the rule claimed -- 4 to 675, 0 to 193, 6 - to 668 -- because that is what widening a rule MEANS. - - A count is identity-free, which this module rejects elsewhere for - good reason. It is acceptable here because it is a backstop and not - the explanation: the guards above name what is wrong, and this one - only insists that nothing moved unnoticed. A rule can still be - wrong at a stable count -- it just cannot become wrong in the one - way five rounds of review actually found. + changed what the rule claimed -- 6 to 668 names, 0 to 193, and the + comma rule from 3 roles to 6 while its regex and its 236 names + stood still -- because that is what widening a rule means WITHIN + the corpus. A widening reaching only names the corpora lack moves + nothing here; the guards above are what see those. + + Note which layer is which, because it is the opposite of what it + looks like. This is a change DETECTOR, not an enforcer: it is + inert for a brand-new rule, whose author simply records whatever + number it produces, and its own failure message invites the + remedy that defeats it -- re-record and the attack lands. The + member and vocabulary guards above are the walls, because they + judge a rule wrong at any time INCLUDING at recording time. This + catches the widenings none of them is scoped to see, and holds + them still long enough for someone to look. """ for ledger in _LEDGERS: assert ledger.name in _CORPUS_CLAIMS, ( f"{ledger.name} is a new ledger with no recorded corpus claims; " f"add it to _CORPUS_CLAIMS (an empty mapping if it has no rules)") recorded = _CORPUS_CLAIMS[ledger.name] - actual = {rule["issue"]: len(_claimed(rule["name_regex"])) - for rule in _rules(ledger) - if isinstance(rule.get("name_regex"), str)} + # Keyed on `issue`, so a duplicate silently collapses to the + # last rule written and the other goes unmeasured -- coverage + # by file order, which is no coverage. Nothing else asserts + # this: validate_rules only requires a non-empty string, and + # the tag-uniqueness check above covers span-bearing rules + # alone. _LATIN_ALTERNATION_SOURCES and _HONORIFIC_SOURCES key + # on issue SUBSTRINGS, so they lean on it too. + issues = [rule["issue"] for rule in _rules(ledger)] + assert len(set(issues)) == len(issues), ( + f"{ledger.name} has rules sharing an `issue`: " + f"{sorted({i for i in issues if issues.count(i) > 1})}. Every " + f"roster here keys on it, so one of them would go unmeasured") + # A rule with no name_regex narrows by `fields` alone and so + # reaches EVERY name -- the most unbounded shape validate_rules + # permits, and the one most worth recording. Counting it as the + # whole corpus is not a placeholder; it is what it claims. + actual = {rule["issue"]: _claim(rule) + for rule in _rules(ledger)} moved = {issue: (recorded.get(issue), count) for issue, count in actual.items() if recorded.get(issue) != count}