Add schema generation

gedcom7/__init__.py

@@ -5,7 +5,7 @@
 from .exceptions import GedcomError, GedcomParseError, GedcomSerializeError
 from .format import format_value, set_value
 from .parser import load, loads
-from .serializer import dump, dumps
+from .serializer import dump, dumps, generate_schema
 
 __all__ = [
     "GedcomError",
@@ -14,6 +14,7 @@
     "dump",
     "dumps",
     "format_value",
+    "generate_schema",
     "load",
     "loads",
     "set_value",

gedcom7/format.py

@@ -29,20 +29,20 @@
 def format_value(value: types.DataType | None, type_id: str) -> str | None:
     """Format a value as the payload string for its structure type.
 
-    ``None`` and the empty string mean different things, and a caller deciding
-    what to write has to tell them apart. ``None`` means there is no structure to
-    write: it comes back for a value of ``None``, and for a false ``Y|<NULL>``,
-    which the specification expresses by leaving the structure out rather than by
-    writing it empty. The empty string means the structure is written with no
-    payload, as for an empty :class:`~gedcom7.types.DatePeriod`, which is a legal
-    date period and not the absence of one.
-
-    Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value cannot
-    be written as a payload conforming to its structure type, including when the
-    structure type points at a record: a pointer belongs in the structure's
-    pointer, and writing it as text would escape its leading "@" and turn the
-    link into a line of text. A structure with no standard type has no data type
-    to format, so its text is written as it stands rather than passed here.
+    A structure type is named by its URI, and an empty payload and no structure
+    at all are different answers::
+
+        V7 = "https://gedcom.io/terms/v7/"
+        format_value(types.Date(year=2000), V7 + "DATE")   -> "2000"
+        format_value(types.DatePeriod(), V7 + "NO-DATE")   -> ""    empty payload
+        format_value(False, V7 + "ADOP")                   -> None  no structure
+
+    ``None`` also comes back for a value of ``None``. A false ``Y|<NULL>`` gives
+    it because the specification expresses that by omitting the structure.
+
+    Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value has no
+    conforming payload, or if the structure type points at a record, whose
+    pointer belongs in the structure's ``pointer`` rather than its ``text``.
     """
     if value is None:
         return None
@@ -62,30 +62,29 @@ def format_value(value: types.DataType | None, type_id: str) -> str | None:
     return format_function(value)
 
 
+# The counterpart of GedcomStructure.value, but a function rather than a method:
+# it formats, and a method would put an import of this module back into types.
 def set_value(
     structure: types.GedcomStructure,
     value: types.DataType,
     type_id: str | None = None,
 ) -> None:
-    """Set a structure's payload from a value of its structure type's data type.
-
-    The counterpart of :attr:`~gedcom7.types.GedcomStructure.value`, which reads
-    a payload back. Reading is a property of a structure the parser has already
-    built; writing is something a writer does to one, which is why this is a
-    function here rather than a method there.
-
-    Which data type applies follows from the structure type, and a structure only
-    has one once it sits under its superstructure, since that is what gives its
-    tag a meaning. Building a tree from the leaves up therefore means attaching a
-    structure before setting its value, or naming the structure type here. Where
-    no standard type applies the payload is carried uninterpreted, exactly as the
-    reader returns it, so a string is set as it stands and anything else is
-    refused.
-
-    Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value cannot
-    be written as a payload, which includes a false ``Y|<NULL>``: the
-    specification expresses that by leaving the structure out, and removing a
-    structure from its superstructure is the caller's to do.
+    """Set a structure's payload from a typed value, in place.
+
+    Which data type applies comes from the structure type, which a structure only
+    has once it sits under its superstructure, so attach it first::
+
+        birth.append_child(date)
+        set_value(date, types.Date(day=1, month="JAN", year=2000))
+        date.text   # "1 JAN 2000"
+
+    Pass ``type_id`` to name the structure type instead, for one not yet
+    attached. Where no standard type applies the payload is uninterpreted, so a
+    string is set as it stands and anything else is refused.
+
+    Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the value has no
+    payload, including a false ``Y|<NULL>``: the specification expresses that by
+    omitting the structure, and removing it is the caller's to do.
     """
     resolved = type_id if type_id is not None else structure.type_id
     if resolved is None:

gedcom7/serializer.py

@@ -7,19 +7,19 @@
 
 from . import const, grammar
 from .exceptions import GedcomSerializeError
+from .types import GedcomStructure
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
     from typing import BinaryIO
 
-    from .types import GedcomStructure
-
 _EOL = re.compile(r"\r\n|\r|\n")
 _TAG = re.compile(grammar.tag)
 _XREF = re.compile(grammar.xref)
 _POINTER = re.compile(grammar.pointer)
 _BANNED = re.compile(grammar.banned)
 _TAGDEF = re.compile(grammar.tagdef)
+_EXTTAG = re.compile(grammar.exttag)
 
 _BOM = "\ufeff"
 
@@ -52,6 +52,97 @@ def _schema(records: Iterable[GedcomStructure]) -> dict[str, str]:
     return uris
 
 
+def _extension_tag(uri: str, taken: set[str]) -> str:
+    """Invent an extension tag for a URI, avoiding every tag already spoken for.
+
+    The last path segment or fragment of the URI usually reads as a name, so it
+    is the basis for the tag; a URI that yields nothing usable falls back to a
+    plain counter. A tag already in use is stepped past rather than reused, since
+    a tag standing for two things resolves to neither on the way back in.
+    """
+    segment = re.split(r"[/#]", uri)[-1]
+    base = "_" + re.sub(r"[^A-Z0-9_]", "", segment.upper())
+    if _EXTTAG.fullmatch(base) is None:
+        base = "_EXT"
+    candidate, suffix = base, 1
+    while candidate in taken:
+        suffix += 1
+        candidate = f"{base}{suffix}"
+    return candidate
+
+
+def generate_schema(records: Iterable[GedcomStructure]) -> None:
+    """Add to HEAD the schema declarations dumps needs to write extension tags.
+
+    Takes a whole dataset, HEAD included, and modifies its structures in place::
+
+        records = gedcom7.loads(text)
+        gedcom7.generate_schema(records)   # records[0] gains HEAD.SCHMA.TAG
+        gedcom7.dumps(records)
+
+    A tag held as a URI cannot be written until the header abbreviates it. Only
+    HEAD changes; the URIs stay on their own structures for :func:`dumps` to
+    substitute as it writes. Declarations already there are kept, so a second
+    call does nothing.
+
+    Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if there is no HEAD,
+    or if a tag is neither writable nor a URI that can be abbreviated.
+    """
+    # Read once: the records are walked several times below, and a caller passing
+    # a one-shot iterator would otherwise have them silently disappear part way.
+    records = list(records)
+    head = next((record for record in records if record.tag == const.HEAD), None)
+    if head is None:
+        raise GedcomSerializeError(
+            "a schema is declared in the HEAD pseudo-structure, and these "
+            "records do not have one"
+        )
+
+    declared = _schema(records)
+    # Every literal tag in the document is spoken for as well: were a generated
+    # tag to collide with one, that structure would resolve to the extension's
+    # URI when the stream was read back.
+    taken = set(declared.values())
+    undeclared: dict[str, None] = {}
+
+    def visit(structure: GedcomStructure) -> None:
+        if _TAG.fullmatch(structure.tag) is None:
+            if structure.tag not in declared:
+                undeclared.setdefault(structure.tag, None)
+        else:
+            taken.add(structure.tag)
+        for child in structure.children:
+            visit(child)
+
+    for record in records:
+        visit(record)
+
+    if not undeclared:
+        return
+
+    schema = next((c for c in head.children if c.tag == const.SCHMA), None)
+    if schema is None:
+        schema = GedcomStructure(tag=const.SCHMA)
+        schema.parent = head
+        gedc = next(
+            (i for i, c in enumerate(head.children) if c.tag == const.GEDC), None
+        )
+        head.children.insert(0 if gedc is None else gedc + 1, schema)
+
+    # One tag per URI and one URI per tag: dumps picks arbitrarily between two
+    # tags for a URI, and the parser refuses to resolve a tag that maps to two.
+    for uri in undeclared:
+        tag = _extension_tag(uri, taken)
+        taken.add(tag)
+        definition = f"{tag} {uri}"
+        if _TAGDEF.fullmatch(definition) is None:
+            raise GedcomSerializeError(
+                f"{uri!r} cannot be declared in a schema: it is neither a tag "
+                "this serializer can write nor a URI it can abbreviate"
+            )
+        schema.append_child(GedcomStructure(tag=const.TAG, text=definition))
+
+
 def _lines(
     structure: GedcomStructure, level: int, uris: dict[str, str]
 ) -> Iterable[str]:

test/test_format.py

@@ -738,3 +738,10 @@ def test_set_value_round_trips_through_the_serializer() -> None:
         "1 BIRT\n2 DATE 1 JAN 2000\n0 TRLR\n"
     )
     assert gedcom7.loads(text) == [head, individual, trlr]
+
+
+def test_format_value_docstring_examples() -> None:
+    """The examples in format_value's docstring have to be true."""
+    assert gedcom7.format_value(types.Date(year=2000), V7 + "DATE") == "2000"
+    assert gedcom7.format_value(types.DatePeriod(), V7 + "NO-DATE") == ""
+    assert gedcom7.format_value(False, V7 + "ADOP") is None