From 9f258cde6e5d009a210db003562ecfc7c030c106 Mon Sep 17 00:00:00 2001 From: David Straub Date: Sat, 15 Aug 2026 22:13:01 +0200 Subject: [PATCH 1/3] Add schema generation --- gedcom7/__init__.py | 3 +- gedcom7/format.py | 63 ++++++----- gedcom7/serializer.py | 92 +++++++++++++++- test/test_serializer.py | 229 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 351 insertions(+), 36 deletions(-) diff --git a/gedcom7/__init__.py b/gedcom7/__init__.py index 5b4a7e9..e70606f 100644 --- a/gedcom7/__init__.py +++ b/gedcom7/__init__.py @@ -5,7 +5,7 @@ from .exceptions import GedcomError, GedcomParseError, GedcomSerializeError from .format import format_value, set_value from .parser import load, loads -from .serializer import dump, dumps +from .serializer import dump, dumps, generate_schema __all__ = [ "GedcomError", @@ -14,6 +14,7 @@ "dump", "dumps", "format_value", + "generate_schema", "load", "loads", "set_value", diff --git a/gedcom7/format.py b/gedcom7/format.py index fc1a48b..f3ff8f8 100644 --- a/gedcom7/format.py +++ b/gedcom7/format.py @@ -29,20 +29,18 @@ def format_value(value: types.DataType | None, type_id: str) -> str | None: """Format a value as the payload string for its structure type. - ``None`` and the empty string mean different things, and a caller deciding - what to write has to tell them apart. ``None`` means there is no structure to - write: it comes back for a value of ``None``, and for a false ``Y|``, - which the specification expresses by leaving the structure out rather than by - writing it empty. The empty string means the structure is written with no - payload, as for an empty :class:`~gedcom7.types.DatePeriod`, which is a legal - date period and not the absence of one. - - Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value cannot - be written as a payload conforming to its structure type, including when the - structure type points at a record: a pointer belongs in the structure's - pointer, and writing it as text would escape its leading "@" and turn the - link into a line of text. A structure with no standard type has no data type - to format, so its text is written as it stands rather than passed here. + An empty payload and no structure at all are different answers:: + + format_value(Date(year=2000), DATE) -> "2000" + format_value(DatePeriod(), NO_DATE) -> "" write an empty payload + format_value(False, ADOP) -> None write no structure + + ``None`` also comes back for a value of ``None``. A false ``Y|`` gives + it because the specification expresses that by omitting the structure. + + Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value has no + conforming payload, or if the structure type points at a record, whose + pointer belongs in the structure's ``pointer`` rather than its ``text``. """ if value is None: return None @@ -62,30 +60,29 @@ def format_value(value: types.DataType | None, type_id: str) -> str | None: return format_function(value) +# The counterpart of GedcomStructure.value, but a function rather than a method: +# it formats, and a method would put an import of this module back into types. def set_value( structure: types.GedcomStructure, value: types.DataType, type_id: str | None = None, ) -> None: - """Set a structure's payload from a value of its structure type's data type. - - The counterpart of :attr:`~gedcom7.types.GedcomStructure.value`, which reads - a payload back. Reading is a property of a structure the parser has already - built; writing is something a writer does to one, which is why this is a - function here rather than a method there. - - Which data type applies follows from the structure type, and a structure only - has one once it sits under its superstructure, since that is what gives its - tag a meaning. Building a tree from the leaves up therefore means attaching a - structure before setting its value, or naming the structure type here. Where - no standard type applies the payload is carried uninterpreted, exactly as the - reader returns it, so a string is set as it stands and anything else is - refused. - - Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value cannot - be written as a payload, which includes a false ``Y|``: the - specification expresses that by leaving the structure out, and removing a - structure from its superstructure is the caller's to do. + """Set a structure's payload from a typed value, in place. + + Which data type applies comes from the structure type, which a structure only + has once it sits under its superstructure, so attach it first:: + + birth.append_child(date) + set_value(date, types.Date(day=1, month="JAN", year=2000)) + date.text # "1 JAN 2000" + + Pass ``type_id`` to name the structure type instead, for one not yet + attached. Where no standard type applies the payload is uninterpreted, so a + string is set as it stands and anything else is refused. + + Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value has no + payload, including a false ``Y|``: the specification expresses that by + omitting the structure, and removing it is the caller's to do. """ resolved = type_id if type_id is not None else structure.type_id if resolved is None: diff --git a/gedcom7/serializer.py b/gedcom7/serializer.py index f9218e0..fa8c849 100644 --- a/gedcom7/serializer.py +++ b/gedcom7/serializer.py @@ -7,19 +7,19 @@ from . import const, grammar from .exceptions import GedcomSerializeError +from .types import GedcomStructure if TYPE_CHECKING: from collections.abc import Iterable from typing import BinaryIO - from .types import GedcomStructure - _EOL = re.compile(r"\r\n|\r|\n") _TAG = re.compile(grammar.tag) _XREF = re.compile(grammar.xref) _POINTER = re.compile(grammar.pointer) _BANNED = re.compile(grammar.banned) _TAGDEF = re.compile(grammar.tagdef) +_EXTTAG = re.compile(grammar.exttag) _BOM = "\ufeff" @@ -52,6 +52,94 @@ def _schema(records: Iterable[GedcomStructure]) -> dict[str, str]: return uris +def _extension_tag(uri: str, taken: set[str]) -> str: + """Invent an extension tag for a URI, avoiding every tag already spoken for. + + The last path segment or fragment of the URI usually reads as a name, so it + is the basis for the tag; a URI that yields nothing usable falls back to a + plain counter. A tag already in use is stepped past rather than reused, since + a tag standing for two things resolves to neither on the way back in. + """ + segment = re.split(r"[/#]", uri)[-1] + base = "_" + re.sub(r"[^A-Z0-9_]", "", segment.upper()) + if _EXTTAG.fullmatch(base) is None: + base = "_EXT" + candidate, suffix = base, 1 + while candidate in taken: + suffix += 1 + candidate = f"{base}{suffix}" + return candidate + + +def generate_schema(records: list[GedcomStructure]) -> None: + """Add to HEAD the schema declarations dumps needs to write extension tags. + + Takes a whole dataset, HEAD included, and modifies it in place:: + + records = gedcom7.loads(text) + gedcom7.generate_schema(records) # records[0] gains HEAD.SCHMA.TAG + gedcom7.dumps(records) + + A tag held as a URI cannot be written until the header abbreviates it. Only + HEAD changes; the URIs stay on their own structures for :func:`dumps` to + substitute as it writes. Declarations already there are kept, so a second + call does nothing. + + Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if there is no HEAD, + or if a tag is neither writable nor a URI that can be abbreviated. + """ + head = next((record for record in records if record.tag == const.HEAD), None) + if head is None: + raise GedcomSerializeError( + "a schema is declared in the HEAD pseudo-structure, and these " + "records do not have one" + ) + + declared = _schema(records) + # Every literal tag in the document is spoken for as well: were a generated + # tag to collide with one, that structure would resolve to the extension's + # URI when the stream was read back. + taken = set(declared.values()) + undeclared: dict[str, None] = {} + + def visit(structure: GedcomStructure) -> None: + if _TAG.fullmatch(structure.tag) is None: + if structure.tag not in declared: + undeclared.setdefault(structure.tag, None) + else: + taken.add(structure.tag) + for child in structure.children: + visit(child) + + for record in records: + visit(record) + + if not undeclared: + return + + schema = next((c for c in head.children if c.tag == const.SCHMA), None) + if schema is None: + schema = GedcomStructure(tag=const.SCHMA) + schema.parent = head + gedc = next( + (i for i, c in enumerate(head.children) if c.tag == const.GEDC), None + ) + head.children.insert(0 if gedc is None else gedc + 1, schema) + + # One tag per URI and one URI per tag: dumps picks arbitrarily between two + # tags for a URI, and the parser refuses to resolve a tag that maps to two. + for uri in undeclared: + tag = _extension_tag(uri, taken) + taken.add(tag) + definition = f"{tag} {uri}" + if _TAGDEF.fullmatch(definition) is None: + raise GedcomSerializeError( + f"{uri!r} cannot be declared in a schema: it is neither a tag " + "this serializer can write nor a URI it can abbreviate" + ) + schema.append_child(GedcomStructure(tag=const.TAG, text=definition)) + + def _lines( structure: GedcomStructure, level: int, uris: dict[str, str] ) -> Iterable[str]: diff --git a/test/test_serializer.py b/test/test_serializer.py index 91d6e2c..14b23f8 100644 --- a/test/test_serializer.py +++ b/test/test_serializer.py @@ -272,3 +272,232 @@ def test_banned_character_in_payload_rejected() -> None: records[1].children[0].text = "bad\x7fchar" with pytest.raises(GedcomSerializeError, match="banned character"): gedcom7.dumps(records) + + +# -------------------------------------------------------------------------- +# Schema generation +# -------------------------------------------------------------------------- + +FOAF = "http://xmlns.com/foaf/0.1/skypeID" + + +def extension_record(*uris: str) -> types.GedcomStructure: + """An individual carrying one substructure per extension URI.""" + individual = types.GedcomStructure(tag="INDI", xref="@I1@") + for uri in uris: + individual.append_child(types.GedcomStructure(tag=uri, text="payload")) + return individual + + +def header(*children: types.GedcomStructure) -> types.GedcomStructure: + head = types.GedcomStructure(tag="HEAD") + gedc = types.GedcomStructure(tag="GEDC") + gedc.append_child(types.GedcomStructure(tag="VERS", text="7.0")) + head.append_child(gedc) + for child in children: + head.append_child(child) + return head + + +def test_generate_schema_declares_an_undeclared_uri() -> None: + """dumps refuses a URI tag it has no abbreviation for; this supplies one.""" + records = [header(), extension_record(FOAF), types.GedcomStructure(tag="TRLR")] + with pytest.raises(GedcomSerializeError, match="not a valid tag"): + gedcom7.dumps(records) + + gedcom7.generate_schema(records) + assert gedcom7.dumps(records, byte_order_mark=False) == ( + "0 HEAD\n1 GEDC\n2 VERS 7.0\n1 SCHMA\n2 TAG _SKYPEID " + FOAF + "\n" + "0 @I1@ INDI\n1 _SKYPEID payload\n0 TRLR\n" + ) + + +def test_generate_schema_round_trips_the_uri() -> None: + """The abbreviation has to resolve back to the URI it stood for.""" + records = [header(), extension_record(FOAF), types.GedcomStructure(tag="TRLR")] + gedcom7.generate_schema(records) + reparsed = gedcom7.loads(gedcom7.dumps(records)) + assert reparsed[1].children[0].tag == FOAF + + +def test_generate_schema_keeps_declarations_already_made() -> None: + """A tag chosen by hand is reused, not replaced.""" + schema = types.GedcomStructure(tag="SCHMA") + schema.append_child(types.GedcomStructure(tag="TAG", text=f"_MINE {FOAF}")) + records = [ + header(schema), + extension_record(FOAF), + types.GedcomStructure(tag="TRLR"), + ] + gedcom7.generate_schema(records) + definitions = [c.text for c in records[0].children[1].children] + assert definitions == [f"_MINE {FOAF}"] + assert "1 _MINE payload" in gedcom7.dumps(records) + + +def test_generate_schema_is_idempotent() -> None: + """Running it twice must not declare the same URI a second time.""" + records = [header(), extension_record(FOAF), types.GedcomStructure(tag="TRLR")] + gedcom7.generate_schema(records) + first = gedcom7.dumps(records) + gedcom7.generate_schema(records) + assert gedcom7.dumps(records) == first + + +def test_generate_schema_gives_each_uri_one_tag() -> None: + """One URI on many structures is declared once.""" + records = [ + header(), + extension_record(FOAF, FOAF, FOAF), + types.GedcomStructure(tag="TRLR"), + ] + gedcom7.generate_schema(records) + assert len(records[0].children[1].children) == 1 + + +def test_generate_schema_avoids_colliding_with_another_uris_tag() -> None: + """Two URIs whose last segments match must not be given the same tag.""" + other = "http://example.com/other/skypeID" + records = [ + header(), + extension_record(FOAF, other), + types.GedcomStructure(tag="TRLR"), + ] + gedcom7.generate_schema(records) + tags = [c.text.split()[0] for c in records[0].children[1].children] + assert tags == ["_SKYPEID", "_SKYPEID2"] + reparsed = gedcom7.loads(gedcom7.dumps(records)) + assert [c.tag for c in reparsed[1].children] == [FOAF, other] + + +def test_generate_schema_avoids_colliding_with_a_literal_tag() -> None: + """A tag used literally must not be made to stand for a URI as well. + + Were _SKYPEID already in the document as an undocumented extension tag, + declaring it here would make that structure resolve to the URI on the way + back in, silently changing what it means. + """ + individual = extension_record(FOAF) + individual.append_child(types.GedcomStructure(tag="_SKYPEID", text="unrelated")) + records = [header(), individual, types.GedcomStructure(tag="TRLR")] + gedcom7.generate_schema(records) + + assert records[0].children[1].children[0].text == f"_SKYPEID2 {FOAF}" + reparsed = gedcom7.loads(gedcom7.dumps(records)) + assert [c.tag for c in reparsed[1].children] == [FOAF, "_SKYPEID"] + + +def test_generate_schema_without_anything_to_declare() -> None: + """A document using no extensions gets no empty schema.""" + records = [ + header(), + types.GedcomStructure(tag="INDI", xref="@I1@"), + types.GedcomStructure(tag="TRLR"), + ] + gedcom7.generate_schema(records) + assert [c.tag for c in records[0].children] == ["GEDC"] + + +def test_generate_schema_falls_back_when_the_uri_yields_no_name() -> None: + """A URI whose last segment gives nothing usable still gets a tag.""" + records = [ + header(), + extension_record("http://example.com/"), + types.GedcomStructure(tag="TRLR"), + ] + gedcom7.generate_schema(records) + assert records[0].children[1].children[0].text == "_EXT http://example.com/" + + +def test_generate_schema_needs_a_header() -> None: + records = [types.GedcomStructure(tag="TRLR")] + with pytest.raises(GedcomSerializeError, match="HEAD"): + gedcom7.generate_schema(records) + + +def test_generate_schema_rejects_a_tag_that_is_no_uri_either() -> None: + """A tag that is neither writable nor abbreviatable cannot be rescued.""" + records = [ + header(), + extension_record("not a tag and not a uri"), + types.GedcomStructure(tag="TRLR"), + ] + with pytest.raises(GedcomSerializeError, match="neither a tag"): + gedcom7.generate_schema(records) + + +def test_generate_schema_places_the_schema_after_gedc() -> None: + """The slot is deterministic, so regenerating a file does not reshuffle it.""" + records = [header(), extension_record(FOAF), types.GedcomStructure(tag="TRLR")] + gedcom7.generate_schema(records) + assert [c.tag for c in records[0].children] == ["GEDC", "SCHMA"] + + +def test_generate_schema_reproduces_the_corpus_declarations() -> None: + """Stripping maximal70.ged's schema and regenerating it declares the same tags. + + The hand-written declarations in the official file are the check on the tag + naming: a generated tag has to be the one a person would have picked. + """ + filename = pathlib.Path(__file__).parent / "data" / "maximal70.ged" + original = gedcom7.loads(filename.read_text(encoding="utf-8")) + declarations = [ + definition.text + for schema in original[0].children + if schema.tag == "SCHMA" + for definition in schema.children + ] + + stripped = gedcom7.loads(filename.read_text(encoding="utf-8")) + stripped[0].children = [c for c in stripped[0].children if c.tag != "SCHMA"] + gedcom7.generate_schema(stripped) + + regenerated = [ + definition.text + for schema in stripped[0].children + if schema.tag == "SCHMA" + for definition in schema.children + ] + assert regenerated == declarations + # the schema moves to its deterministic slot, so the bytes differ, but every + # extension tag has to resolve to the URI it stood for before + assert gedcom7.loads(gedcom7.dumps(stripped)) == stripped + + +def test_generate_schema_touches_only_the_header() -> None: + """The one structure that changes is HEAD, which gains the declarations. + + The structures carrying the URIs keep them as their tags; dumps does the + abbreviating as it writes. Nothing else in the tree is rewritten, and the + records list is not itself changed. + """ + records = [header(), extension_record(FOAF), types.GedcomStructure(tag="TRLR")] + + def snapshot() -> dict[int, tuple[object, ...]]: + seen: dict[int, tuple[object, ...]] = {} + + def visit(s: types.GedcomStructure) -> None: + seen[id(s)] = ( + s.tag, + s.pointer, + s.text, + s.xref, + tuple(id(c) for c in s.children), + ) + for child in s.children: + visit(child) + + for record in records: + visit(record) + return seen + + before = snapshot() + contents = [id(r) for r in records] + gedcom7.generate_schema(records) + after = snapshot() + + changed = [before[k][0] for k in before if after[k] != before[k]] + assert changed == ["HEAD"] + assert [id(r) for r in records] == contents + # the extension structure still holds its URI, unabbreviated + assert records[1].children[0].tag == FOAF From 82d01d2665ff6b7b993c976da4cecdbaccdf98e0 Mon Sep 17 00:00:00 2001 From: David Straub Date: Sun, 16 Aug 2026 15:54:22 +0200 Subject: [PATCH 2/3] Address comment --- gedcom7/serializer.py | 7 +++++-- test/test_serializer.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/gedcom7/serializer.py b/gedcom7/serializer.py index fa8c849..21e0cd5 100644 --- a/gedcom7/serializer.py +++ b/gedcom7/serializer.py @@ -71,10 +71,10 @@ def _extension_tag(uri: str, taken: set[str]) -> str: return candidate -def generate_schema(records: list[GedcomStructure]) -> None: +def generate_schema(records: Iterable[GedcomStructure]) -> None: """Add to HEAD the schema declarations dumps needs to write extension tags. - Takes a whole dataset, HEAD included, and modifies it in place:: + Takes a whole dataset, HEAD included, and modifies its structures in place:: records = gedcom7.loads(text) gedcom7.generate_schema(records) # records[0] gains HEAD.SCHMA.TAG @@ -88,6 +88,9 @@ def generate_schema(records: list[GedcomStructure]) -> None: Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if there is no HEAD, or if a tag is neither writable nor a URI that can be abbreviated. """ + # Read once: the records are walked several times below, and a caller passing + # a one-shot iterator would otherwise have them silently disappear part way. + records = list(records) head = next((record for record in records if record.tag == const.HEAD), None) if head is None: raise GedcomSerializeError( diff --git a/test/test_serializer.py b/test/test_serializer.py index 14b23f8..467f702 100644 --- a/test/test_serializer.py +++ b/test/test_serializer.py @@ -501,3 +501,15 @@ def visit(s: types.GedcomStructure) -> None: assert [id(r) for r in records] == contents # the extension structure still holds its URI, unabbreviated assert records[1].children[0].tag == FOAF + + +def test_generate_schema_accepts_any_iterable() -> None: + """The records are walked several times, so a one-shot iterator must survive. + + Passing an iterator used to consume it during the search for HEAD, leaving + the traversal nothing to find and making the call a silent no-op. + """ + records = [header(), extension_record(FOAF), types.GedcomStructure(tag="TRLR")] + gedcom7.generate_schema(iter(records)) + assert [c.tag for c in records[0].children] == ["GEDC", "SCHMA"] + assert records[0].children[1].children[0].text == f"_SKYPEID {FOAF}" From 479c0d5a96f10862f59139a36b0da870ba814c3c Mon Sep 17 00:00:00 2001 From: David Straub Date: Sun, 16 Aug 2026 16:03:09 +0200 Subject: [PATCH 3/3] Address comment --- gedcom7/format.py | 10 ++++++---- test/test_format.py | 7 +++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/gedcom7/format.py b/gedcom7/format.py index f3ff8f8..27254ea 100644 --- a/gedcom7/format.py +++ b/gedcom7/format.py @@ -29,11 +29,13 @@ def format_value(value: types.DataType | None, type_id: str) -> str | None: """Format a value as the payload string for its structure type. - An empty payload and no structure at all are different answers:: + A structure type is named by its URI, and an empty payload and no structure + at all are different answers:: - format_value(Date(year=2000), DATE) -> "2000" - format_value(DatePeriod(), NO_DATE) -> "" write an empty payload - format_value(False, ADOP) -> None write no structure + V7 = "https://gedcom.io/terms/v7/" + format_value(types.Date(year=2000), V7 + "DATE") -> "2000" + format_value(types.DatePeriod(), V7 + "NO-DATE") -> "" empty payload + format_value(False, V7 + "ADOP") -> None no structure ``None`` also comes back for a value of ``None``. A false ``Y|`` gives it because the specification expresses that by omitting the structure. diff --git a/test/test_format.py b/test/test_format.py index 14517e9..d89b443 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -738,3 +738,10 @@ def test_set_value_round_trips_through_the_serializer() -> None: "1 BIRT\n2 DATE 1 JAN 2000\n0 TRLR\n" ) assert gedcom7.loads(text) == [head, individual, trlr] + + +def test_format_value_docstring_examples() -> None: + """The examples in format_value's docstring have to be true.""" + assert gedcom7.format_value(types.Date(year=2000), V7 + "DATE") == "2000" + assert gedcom7.format_value(types.DatePeriod(), V7 + "NO-DATE") == "" + assert gedcom7.format_value(False, V7 + "ADOP") is None