diff --git a/gedcom7/__init__.py b/gedcom7/__init__.py index e70606f..da6c531 100644 --- a/gedcom7/__init__.py +++ b/gedcom7/__init__.py @@ -2,15 +2,23 @@ from importlib.metadata import PackageNotFoundError, version -from .exceptions import GedcomError, GedcomParseError, GedcomSerializeError -from .format import format_value, set_value +from .exceptions import ( + GedcomError, + GedcomParseError, + GedcomSerializeError, + GedcomValidationError, +) +from .formatter import format_value, set_value from .parser import load, loads from .serializer import dump, dumps, generate_schema +from .validator import Error, validate __all__ = [ "GedcomError", "GedcomParseError", "GedcomSerializeError", + "GedcomValidationError", + "Error", "dump", "dumps", "format_value", @@ -18,6 +26,7 @@ "load", "loads", "set_value", + "validate", ] try: diff --git a/gedcom7/exceptions.py b/gedcom7/exceptions.py index 7602849..54119d1 100644 --- a/gedcom7/exceptions.py +++ b/gedcom7/exceptions.py @@ -2,6 +2,11 @@ from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .validator import Error + class GedcomError(Exception): """Base class for all errors raised by this package.""" @@ -11,6 +16,17 @@ class GedcomSerializeError(GedcomError, ValueError): """Raised when structures cannot be encoded as a conforming data stream.""" +class GedcomValidationError(GedcomError, ValueError): + """Raised when a dataset fails validation, carrying every problem found.""" + + def __init__(self, errors: list[Error]) -> None: + """Record the errors and summarize them in the message.""" + self.errors = errors + first = "; ".join(f"{e.path}: {e.message}" for e in errors[:3]) + more = f", and {len(errors) - 3} more" if len(errors) > 3 else "" + super().__init__(f"{len(errors)} validation errors: {first}{more}") + + class GedcomParseError(GedcomError, ValueError): """Raised when a data stream does not conform to the GEDCOM 7 grammar. diff --git a/gedcom7/format.py b/gedcom7/formatter.py similarity index 100% rename from gedcom7/format.py rename to gedcom7/formatter.py diff --git a/gedcom7/serializer.py b/gedcom7/serializer.py index 21e0cd5..95bc5af 100644 --- a/gedcom7/serializer.py +++ b/gedcom7/serializer.py @@ -6,8 +6,9 @@ from typing import TYPE_CHECKING from . import const, grammar -from .exceptions import GedcomSerializeError +from .exceptions import GedcomSerializeError, GedcomValidationError from .types import GedcomStructure +from .validator import validate as _validate if TYPE_CHECKING: from collections.abc import Iterable @@ -203,6 +204,7 @@ def dumps( *, line_terminator: str = "\n", byte_order_mark: bool = True, + validate: bool = False, ) -> str: """Serialize structures to a GEDCOM 7 data stream. @@ -217,7 +219,11 @@ def dumps( terminators are not altered on the way out. Raises :class:`~gedcom7.exceptions.GedcomSerializeError` if the structures - cannot be encoded as conforming lines. + cannot be encoded as conforming lines. Pass ``validate=True`` to check the + dataset first and raise + :class:`~gedcom7.exceptions.GedcomValidationError`, whose ``errors`` holds + every problem :func:`~gedcom7.validator.validate` found, rather than only the + first one that stops a line being written. """ if line_terminator not in ("\n", "\r\n", "\r"): raise GedcomSerializeError( @@ -226,6 +232,10 @@ def dumps( ) records = list(records) + if validate: + errors = _validate(records) + if errors: + raise GedcomValidationError(errors) uris = _schema(records) lines = [line for record in records for line in _lines(record, 0, uris)] diff --git a/gedcom7/validator.py b/gedcom7/validator.py new file mode 100644 index 0000000..7781e57 --- /dev/null +++ b/gedcom7/validator.py @@ -0,0 +1,334 @@ +"""Report what is wrong with a dataset, using the tables the package already has. + +Covers the checks that need no data beyond ``const.payloads``, +``const.substructures`` and ``const.GEDCOM_MONTHS``. Cardinality and +enumeration vocabularies are not among them: nothing here knows that an +individual may have one SEX, or that its payload is drawn from a fixed list. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from . import cast, const, grammar, types + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + +_TAG = re.compile(grammar.tag) +_TAGDEF = re.compile(grammar.tagdef) + +# February is taken at its longest so the check does not turn on which calendar's +# leap rule applies; a day past these is wrong under either. +_DAYS_IN_MONTH = { + "JAN": 31, "FEB": 29, "MAR": 31, "APR": 30, "MAY": 31, "JUN": 30, + "JUL": 31, "AUG": 31, "SEP": 30, "OCT": 31, "NOV": 30, "DEC": 31, +} # fmt: skip + +# The months above name the Gregorian and Julian year. Other calendars have their +# own, which these tables do not carry, so their dates are left alone. +_MONTH_CALENDARS = (None, "GREGORIAN", "JULIAN") + + +@dataclass +class Error: + """One problem, and the path through the tree to where it was found.""" + + category: str + message: str + path: str + structure: types.GedcomStructure | None = field( + default=None, repr=False, compare=False + ) + + +def validate(records: Iterable[types.GedcomStructure]) -> list[Error]: + """Report every problem in a dataset, rather than raising at the first. + + :: + + for error in gedcom7.validate(records): + print(error.category, error.path, error.message) + + An empty list means the dataset passes the checks this can make; it does not + mean the dataset conforms, since cardinality and enumeration values are not + checked. See :mod:`gedcom7.validator` for what that leaves out. + """ + records = list(records) + errors: list[Error] = [] + _check_dataset(records, errors) + xrefs = {r.xref: r for r in records if r.xref} + declared = _check_schema(records, errors) + for record in records: + _check_structure(record, xrefs, declared, errors) + return errors + + +def _path(structure: types.GedcomStructure) -> str: + """Name a structure by the tags leading down to it from its record.""" + parts = [] + node: types.GedcomStructure | None = structure + while node is not None: + parts.append(f"{node.xref} {node.tag}" if node.xref else node.tag) + node = node.parent + return " > ".join(reversed(parts)) + + +def _report( + errors: list[Error], category: str, message: str, structure: types.GedcomStructure +) -> None: + errors.append(Error(category, message, _path(structure), structure)) + + +def _check_dataset(records: list[types.GedcomStructure], errors: list[Error]) -> None: + """Check the shape of the stream and the records at its top level.""" + if not records: + errors.append(Error("document-shape", "a dataset has no records", "")) + return + for tag, where, found in ( + (const.HEAD, "begin", records[0]), + (const.TRLR, "end", records[-1]), + ): + count = sum(1 for r in records if r.tag == tag) + if count != 1: + _report( + errors, + "document-shape", + f"a dataset has one {tag} pseudo-structure, not {count}", + found, + ) + elif found.tag != tag: + _report( + errors, + "document-shape", + f"a dataset must {where} with {tag}, not {found.tag}", + found, + ) + + seen: set[str] = set() + for record in records: + if record.tag in (const.HEAD, const.TRLR): + continue + if not record.xref: + _report( + errors, "document-shape", "a record needs a cross-reference", record + ) + elif record.xref in seen: + _report(errors, "duplicate-xref", f"{record.xref} is used twice", record) + else: + seen.add(record.xref) + if ( + _TAG.fullmatch(record.tag) + and not record.tag.startswith("_") + and record.tag not in const.substructures[""] + ): + _report( + errors, + "document-shape", + f"{record.tag} is not a record type", + record, + ) + + +def _check_schema( + records: list[types.GedcomStructure], errors: list[Error] +) -> set[str]: + """Check that the header maps each tag to one URI and back, and list the URIs.""" + by_uri: dict[str, set[str]] = {} + by_tag: dict[str, set[str]] = {} + origin: dict[str, types.GedcomStructure] = {} + for record in records: + if record.tag != const.HEAD: + continue + for schema in record.children: + if schema.tag != const.SCHMA: + continue + for definition in schema.children: + if definition.tag != const.TAG: + continue + match = _TAGDEF.fullmatch(definition.text) + if match is None: + _report( + errors, + "malformed-payload", + f"{definition.text!r} is not a tag definition", + definition, + ) + continue + tag, uri = match.group("exttag"), match.group("uri") + by_uri.setdefault(uri, set()).add(tag) + by_tag.setdefault(tag, set()).add(uri) + origin.setdefault(uri, definition) + origin.setdefault(tag, definition) + + for uri, tags in by_uri.items(): + if len(tags) > 1: + _report( + errors, + "schema-conflict", + f"{uri} is abbreviated by {' and '.join(sorted(tags))}, so " + "which one is written is arbitrary", + origin[uri], + ) + for tag, uris in by_tag.items(): + if len(uris) > 1: + _report( + errors, + "schema-conflict", + f"{tag} stands for {' and '.join(sorted(uris))}, so it resolves " + "to neither when read back", + origin[tag], + ) + return set(by_uri) + + +def _check_structure( + structure: types.GedcomStructure, + xrefs: dict[str, types.GedcomStructure], + declared: set[str], + errors: list[Error], +) -> None: + """Check one structure and everything below it.""" + type_id = structure.type_id + if _TAG.fullmatch(structure.tag) is None and structure.tag not in declared: + _report( + errors, + "undeclared-extension", + f"{structure.tag} has no HEAD.SCHMA.TAG declaration, so it cannot " + "be written", + structure, + ) + _check_substructure(structure, errors) + if type_id is not None: + payload = const.payloads.get(type_id) + if payload is not None: + _check_payload_kind(structure, payload, errors) + # Only one of these can say anything useful. Where the payload is of + # the wrong kind entirely, chasing its pointer or casting its text + # would report the same mistake a second time. + if payload.startswith("@<"): + _check_pointer(structure, payload, xrefs, errors) + else: + _check_payload_value(structure, type_id, errors) + for child in structure.children: + _check_structure(child, xrefs, declared, errors) + + +def _check_substructure(structure: types.GedcomStructure, errors: list[Error]) -> None: + """Check that a standard tag is one its superstructure may contain.""" + parent = structure.parent + if parent is None or parent.type_id is None: + # a record, or a substructure of an extension, whose content the + # extension defines rather than the specification + return + if structure.tag.startswith("_") or _TAG.fullmatch(structure.tag) is None: + return + if structure.tag not in const.substructures.get(parent.type_id, {}): + _report( + errors, + "unknown-substructure", + f"{parent.tag} has no {structure.tag} substructure", + structure, + ) + + +def _check_payload_kind( + structure: types.GedcomStructure, payload: str, errors: list[Error] +) -> None: + """Check that the payload is of the kind the structure type carries.""" + if payload.startswith("@<"): + if structure.text: + _report( + errors, + "misplaced-payload", + "this points at a record, so its text belongs in its pointer", + structure, + ) + elif not structure.pointer: + _report( + errors, + "misplaced-payload", + "this points at a record but has no pointer", + structure, + ) + elif payload == "": + if structure.text or structure.pointer: + _report(errors, "misplaced-payload", "this takes no payload", structure) + elif structure.pointer: + _report( + errors, + "misplaced-payload", + "this carries a value, so its pointer belongs in its text", + structure, + ) + + +def _check_pointer( + structure: types.GedcomStructure, + payload: str, + xrefs: dict[str, types.GedcomStructure], + errors: list[Error], +) -> None: + """Check that a pointer reaches a record, and one of the right type.""" + if not structure.pointer or structure.pointer == const.VOIDPTR: + return + target = xrefs.get(structure.pointer) + if target is None: + _report( + errors, + "dangling-pointer", + f"{structure.pointer} is not a record in this dataset", + structure, + ) + else: + wanted = payload[2:-2] + if target.type_id != wanted: + _report( + errors, + "pointer-target-type", + f"{structure.pointer} is a {target.tag} record, but this points " + f"at {wanted.rsplit('/', 1)[-1]}", + structure, + ) + + +def _dates(value: object) -> Iterator[types.Date | types.DateExact]: + """Yield every date inside a value, however it is wrapped.""" + if isinstance(value, types.Date | types.DateExact): + yield value + elif isinstance(value, types.DateApprox): + yield from _dates(value.date) + elif isinstance(value, types.DateRange): + for part in (value.start, value.end): + yield from _dates(part) + elif isinstance(value, types.DatePeriod): + for part in (value.from_, value.to): + yield from _dates(part) + + +def _check_payload_value( + structure: types.GedcomStructure, type_id: str, errors: list[Error] +) -> None: + """Check that the payload casts, and that any date in it could exist.""" + if not structure.text: + return + try: + value = cast.cast_value(structure.text, type_id) + except ValueError as exc: + _report(errors, "malformed-payload", str(exc), structure) + return + for date in _dates(value): + calendar = getattr(date, "calendar", None) + if calendar not in _MONTH_CALENDARS or date.month is None: + continue + if date.month not in _DAYS_IN_MONTH: + _report(errors, "invalid-date", f"{date.month} is not a month", structure) + elif date.day is not None and not 1 <= date.day <= _DAYS_IN_MONTH[date.month]: + _report( + errors, + "invalid-date", + f"{date.month} has no day {date.day}", + structure, + ) diff --git a/test/test_format.py b/test/test_formatter.py similarity index 89% rename from test/test_format.py rename to test/test_formatter.py index d89b443..ad12d23 100644 --- a/test/test_format.py +++ b/test/test_formatter.py @@ -5,7 +5,7 @@ import pytest import gedcom7 -from gedcom7 import GedcomSerializeError, const, format, types +from gedcom7 import GedcomSerializeError, const, formatter, types V7 = "https://gedcom.io/terms/v7/" LATI = "https://gedcom.io/terms/v7/LATI" @@ -44,7 +44,7 @@ ], ) def test_format_latitude(degrees: float, expected: str) -> None: - assert format._format_latitude(degrees) == expected + assert formatter._format_latitude(degrees) == expected @pytest.mark.parametrize( @@ -62,26 +62,26 @@ def test_format_latitude(degrees: float, expected: str) -> None: ], ) def test_format_longitude(degrees: float, expected: str) -> None: - assert format._format_longitude(degrees) == expected + assert formatter._format_longitude(degrees) == expected @pytest.mark.parametrize("degrees", [90.5, -90.5, 91, 1000, float("inf")]) def test_format_latitude_out_of_range(degrees: float) -> None: """The grammar admits "N90.5", so the range is enforced here instead.""" with pytest.raises(GedcomSerializeError): - format._format_latitude(degrees) + formatter._format_latitude(degrees) @pytest.mark.parametrize("degrees", [180.5, -180.5, 181]) def test_format_longitude_out_of_range(degrees: float) -> None: with pytest.raises(GedcomSerializeError): - format._format_longitude(degrees) + formatter._format_longitude(degrees) @pytest.mark.parametrize("value", ["18.15", True, None]) def test_format_latitude_rejects_non_numbers(value: object) -> None: with pytest.raises(GedcomSerializeError): - format._format_latitude(value) + formatter._format_latitude(value) # -------------------------------------------------------------------------- @@ -110,13 +110,13 @@ def test_format_latitude_rejects_non_numbers(value: object) -> None: ], ) def test_format_time(time: types.Time, expected: str) -> None: - assert format._format_time(time) == expected + assert formatter._format_time(time) == expected def test_format_time_fraction_without_seconds() -> None: """The grammar hangs the fraction off the seconds, so it cannot stand alone.""" with pytest.raises(GedcomSerializeError): - format._format_time(types.Time(hour=13, minute=15, fraction="5")) + formatter._format_time(types.Time(hour=13, minute=15, fraction="5")) @pytest.mark.parametrize( @@ -124,7 +124,7 @@ def test_format_time_fraction_without_seconds() -> None: ) def test_format_time_out_of_range(time: types.Time) -> None: with pytest.raises(GedcomSerializeError): - format._format_time(time) + formatter._format_time(time) # -------------------------------------------------------------------------- @@ -155,13 +155,13 @@ def test_format_time_out_of_range(time: types.Time) -> None: ], ) def test_format_personal_name(name: types.PersonalName, expected: str) -> None: - assert format._format_personal_name(name) == expected + assert formatter._format_personal_name(name) == expected def test_format_personal_name_prefers_parts_over_fullname() -> None: """fullname is the payload with its slashes removed, so it cannot place them.""" name = types.PersonalName(fullname="ignored entirely", given="John", surname="Doe") - assert format._format_personal_name(name) == "John /Doe/" + assert formatter._format_personal_name(name) == "John /Doe/" # -------------------------------------------------------------------------- @@ -181,13 +181,13 @@ def test_format_personal_name_prefers_parts_over_fullname() -> None: ], ) def test_format_age(age: types.Age, expected: str) -> None: - assert format._format_age(age) == expected + assert formatter._format_age(age) == expected @pytest.mark.parametrize("age", [types.Age(), types.Age(agebound=">")]) def test_format_age_without_duration(age: types.Age) -> None: with pytest.raises(GedcomSerializeError): - format._format_age(age) + formatter._format_age(age) # -------------------------------------------------------------------------- @@ -210,7 +210,7 @@ def test_format_age_without_duration(age: types.Age) -> None: ], ) def test_format_date(date: types.Date, expected: str) -> None: - assert format._format_date(date) == expected + assert formatter._format_date(date) == expected @pytest.mark.parametrize( @@ -219,12 +219,12 @@ def test_format_date(date: types.Date, expected: str) -> None: def test_format_date_incomplete(date: types.Date) -> None: """A date needs a year, and a day is meaningless without a month.""" with pytest.raises(GedcomSerializeError): - format._format_date(date) + formatter._format_date(date) def test_format_date_period_empty_is_a_legal_payload() -> None: """The grammar makes every part of a date period optional.""" - assert format._format_date_period(types.DatePeriod()) == "" + assert formatter._format_date_period(types.DatePeriod()) == "" @pytest.mark.parametrize( @@ -239,7 +239,7 @@ def test_format_date_period_empty_is_a_legal_payload() -> None: ], ) def test_format_date_period(period: types.DatePeriod, expected: str) -> None: - assert format._format_date_period(period) == expected + assert formatter._format_date_period(period) == expected @pytest.mark.parametrize( @@ -254,12 +254,12 @@ def test_format_date_period(period: types.DatePeriod, expected: str) -> None: ], ) def test_format_date_range(date_range: types.DateRange, expected: str) -> None: - assert format._format_date_range(date_range) == expected + assert formatter._format_date_range(date_range) == expected def test_format_date_range_empty() -> None: with pytest.raises(GedcomSerializeError): - format._format_date_range(types.DateRange()) + formatter._format_date_range(types.DateRange()) @pytest.mark.parametrize("qualifier", ["ABT", "CAL", "EST"]) @@ -267,33 +267,33 @@ def test_format_date_approx(qualifier: str) -> None: approx = types.DateApprox( date=types.Date(day=1, month="OCT", year=2023), approx=qualifier ) - assert format._format_date_approx(approx) == f"{qualifier} 1 OCT 2023" + assert formatter._format_date_approx(approx) == f"{qualifier} 1 OCT 2023" def test_format_date_approx_without_qualifier() -> None: with pytest.raises(GedcomSerializeError): - format._format_date_approx(types.DateApprox(date=types.Date(year=2023))) + formatter._format_date_approx(types.DateApprox(date=types.Date(year=2023))) def test_format_date_exact() -> None: - assert format._format_date_exact( + assert formatter._format_date_exact( types.DateExact(day=1, month="NOV", year=2022) ) == ("1 NOV 2022") def test_format_date_value_dispatches_on_the_form() -> None: """A date value is whichever of the four forms the value carries.""" - assert format._format_date_value(types.Date(year=1998)) == "1998" + assert formatter._format_date_value(types.Date(year=1998)) == "1998" assert ( - format._format_date_value(types.DatePeriod(to=types.Date(year=1800))) + formatter._format_date_value(types.DatePeriod(to=types.Date(year=1800))) == "TO 1800" ) assert ( - format._format_date_value(types.DateRange(end=types.Date(year=1800))) + formatter._format_date_value(types.DateRange(end=types.Date(year=1800))) == "BEF 1800" ) assert ( - format._format_date_value( + formatter._format_date_value( types.DateApprox(date=types.Date(year=1800), approx="ABT") ) == "ABT 1800" @@ -307,49 +307,50 @@ def test_format_date_value_dispatches_on_the_form() -> None: def test_format_bool() -> None: """A false boolean is written by leaving the structure out altogether.""" - assert format._format_bool(True) == "Y" - assert format._format_bool(False) is None + assert formatter._format_bool(True) == "Y" + assert formatter._format_bool(False) is None @pytest.mark.parametrize("value", [0, 1, "Y", None]) def test_format_bool_rejects_non_bools(value: object) -> None: with pytest.raises(GedcomSerializeError): - format._format_bool(value) + formatter._format_bool(value) def test_format_integer() -> None: - assert format._format_integer(0) == "0" - assert format._format_integer(100) == "100" + assert formatter._format_integer(0) == "0" + assert formatter._format_integer(100) == "100" @pytest.mark.parametrize("value", [-1, True, 1.5, "1"]) def test_format_integer_rejects(value: object) -> None: """The payload is a non-negative integer, and a bool is not an integer here.""" with pytest.raises(GedcomSerializeError): - format._format_integer(value) + formatter._format_integer(value) def test_format_list_text() -> None: assert ( - format._format_list_text(["City", "County", "State"]) == "City, County, State" + formatter._format_list_text(["City", "County", "State"]) + == "City, County, State" ) - assert format._format_list_text(["Somewhere"]) == "Somewhere" + assert formatter._format_list_text(["Somewhere"]) == "Somewhere" def test_format_list_text_item_containing_a_comma() -> None: """A list has no escaping, so a comma in an item would split it in two.""" with pytest.raises(GedcomSerializeError): - format._format_list_text(["Paris, France", "Europe"]) + formatter._format_list_text(["Paris, France", "Europe"]) def test_format_list_text_strips_space_around_items() -> None: """The grammar allows space either side of the separator, so it means nothing.""" - assert format._format_list_text(["City ", " County"]) == "City, County" - assert format._format_list_text([" Somewhere "]) == "Somewhere" + assert formatter._format_list_text(["City ", " County"]) == "City, County" + assert formatter._format_list_text([" Somewhere "]) == "Somewhere" def test_format_list_enum() -> None: - assert format._format_list_enum(["BIRT", "DEAT"]) == "BIRT, DEAT" + assert formatter._format_list_enum(["BIRT", "DEAT"]) == "BIRT, DEAT" def test_format_list_enum_does_not_strip_items() -> None: @@ -359,39 +360,39 @@ def test_format_list_enum_does_not_strip_items() -> None: padding around a delimiter. """ with pytest.raises(GedcomSerializeError): - format._format_list_enum([" BIRT", "DEAT"]) + formatter._format_list_enum([" BIRT", "DEAT"]) with pytest.raises(GedcomSerializeError): - format._format_enum(" BIRT") + formatter._format_enum(" BIRT") def test_format_enum() -> None: - assert format._format_enum("ADOPTED") == "ADOPTED" - assert format._format_enum("0") == "0" - assert format._format_enum("_CUSTOM") == "_CUSTOM" + assert formatter._format_enum("ADOPTED") == "ADOPTED" + assert formatter._format_enum("0") == "0" + assert formatter._format_enum("_CUSTOM") == "_CUSTOM" def test_format_enum_invalid() -> None: with pytest.raises(GedcomSerializeError): - format._format_enum("not an enum") + formatter._format_enum("not an enum") def test_format_mediatype() -> None: assert ( - format._format_mediatype(types.MediaType(media_type="text/plain")) + formatter._format_mediatype(types.MediaType(media_type="text/plain")) == "text/plain" ) def test_format_mediatype_invalid() -> None: with pytest.raises(GedcomSerializeError): - format._format_mediatype(types.MediaType(media_type="nonsense")) + formatter._format_mediatype(types.MediaType(media_type="nonsense")) def test_format_tag_definition() -> None: definition = types.TagDefinition( tag="_SKYPEID", uri="http://xmlns.com/foaf/0.1/skypeID" ) - assert format._format_tag_definition(definition) == ( + assert formatter._format_tag_definition(definition) == ( "_SKYPEID http://xmlns.com/foaf/0.1/skypeID" ) @@ -399,7 +400,7 @@ def test_format_tag_definition() -> None: def test_format_tag_definition_invalid_tag() -> None: """An extension tag begins with an underscore.""" with pytest.raises(GedcomSerializeError): - format._format_tag_definition( + formatter._format_tag_definition( types.TagDefinition(tag="SKYPEID", uri="http://x/") ) @@ -462,7 +463,7 @@ def test_every_payload_in_the_specification_is_accounted_for() -> None: for payload in set(const.payloads.values()): accounted_for = ( payload == "" - or payload in format.FORMAT_FUNCTIONS + or payload in formatter.FORMAT_FUNCTIONS or (payload.startswith("@<") and payload.endswith(">@")) ) assert accounted_for, f"{payload} would fall through to plain text" @@ -484,9 +485,9 @@ def test_format_functions_mirror_cast_functions() -> None: """Both tables key off the payload type, so they must cover the same set.""" from gedcom7 import cast - assert format.FORMAT_FUNCTIONS.keys() == cast.CAST_FUNCTIONS.keys() + assert formatter.FORMAT_FUNCTIONS.keys() == cast.CAST_FUNCTIONS.keys() for payload, cast_function in cast.CAST_FUNCTIONS.items(): - assert (cast_function is None) == (format.FORMAT_FUNCTIONS[payload] is None) + assert (cast_function is None) == (formatter.FORMAT_FUNCTIONS[payload] is None) # -------------------------------------------------------------------------- @@ -578,7 +579,7 @@ def test_round_trip_covers_every_data_type() -> None: """Every payload either table knows about must appear in the round trip.""" covered = {const.payloads[type_id] for type_id, _ in ROUND_TRIP} assert covered == set(gedcom7.cast.CAST_FUNCTIONS) - assert covered == set(format.FORMAT_FUNCTIONS) + assert covered == set(formatter.FORMAT_FUNCTIONS) # -------------------------------------------------------------------------- diff --git a/test/test_validator.py b/test/test_validator.py new file mode 100644 index 0000000..30ecb16 --- /dev/null +++ b/test/test_validator.py @@ -0,0 +1,307 @@ +"""Tests for reporting what is wrong with a dataset.""" + +import pathlib + +import pytest + +import gedcom7 +from gedcom7 import GedcomValidationError, types + +FOAF = "http://xmlns.com/foaf/0.1/skypeID" + + +def dataset(*records: types.GedcomStructure) -> list[types.GedcomStructure]: + """A minimal conforming dataset wrapped around the given records.""" + head = types.GedcomStructure(tag="HEAD") + gedc = types.GedcomStructure(tag="GEDC") + head.append_child(gedc) + gedc.append_child(types.GedcomStructure(tag="VERS", text="7.0")) + return [head, *records, types.GedcomStructure(tag="TRLR")] + + +def individual(*children: types.GedcomStructure) -> types.GedcomStructure: + record = types.GedcomStructure(tag="INDI", xref="@I1@") + for child in children: + record.append_child(child) + return record + + +def categories(records: list[types.GedcomStructure]) -> list[str]: + return sorted({error.category for error in gedcom7.validate(records)}) + + +# -------------------------------------------------------------------------- +# Nothing wrong +# -------------------------------------------------------------------------- + + +def test_official_file_validates_clean() -> None: + """The specification's own maximal file must report nothing.""" + filename = pathlib.Path(__file__).parent / "data" / "maximal70.ged" + records = gedcom7.loads(filename.read_text(encoding="utf-8")) + assert gedcom7.validate(records) == [] + + +def test_minimal_dataset_validates_clean() -> None: + assert gedcom7.validate(dataset(individual())) == [] + + +# -------------------------------------------------------------------------- +# Pointers +# -------------------------------------------------------------------------- + + +def test_dangling_pointer() -> None: + records = dataset(individual(types.GedcomStructure(tag="FAMS", pointer="@F9@"))) + assert categories(records) == ["dangling-pointer"] + + +def test_void_pointer_is_not_dangling() -> None: + """@VOID@ deliberately points at nothing.""" + records = dataset(individual(types.GedcomStructure(tag="FAMS", pointer="@VOID@"))) + assert gedcom7.validate(records) == [] + + +def test_pointer_at_the_wrong_record_type() -> None: + """FAMS names a family, so pointing it at an individual is wrong.""" + other = types.GedcomStructure(tag="INDI", xref="@I2@") + records = dataset( + individual(types.GedcomStructure(tag="FAMS", pointer="@I2@")), other + ) + assert categories(records) == ["pointer-target-type"] + + +def test_duplicate_xref() -> None: + records = dataset(individual(), types.GedcomStructure(tag="INDI", xref="@I1@")) + assert categories(records) == ["duplicate-xref"] + + +# -------------------------------------------------------------------------- +# Document shape +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "records", + [ + [types.GedcomStructure(tag="TRLR")], + [types.GedcomStructure(tag="HEAD")], + [ + types.GedcomStructure(tag="HEAD"), + types.GedcomStructure(tag="HEAD"), + types.GedcomStructure(tag="TRLR"), + ], + [ + types.GedcomStructure(tag="HEAD"), + types.GedcomStructure(tag="INDI"), + types.GedcomStructure(tag="TRLR"), + ], + ], + ids=["no head", "no trlr", "two heads", "record without xref"], +) +def test_document_shape(records: list[types.GedcomStructure]) -> None: + assert "document-shape" in categories(records) + + +def test_record_tag_that_is_not_a_record_type() -> None: + """NAME is a substructure, not something that stands at level zero.""" + records = dataset(types.GedcomStructure(tag="NAME", xref="@X1@", text="John")) + assert "document-shape" in categories(records) + + +def test_empty_dataset() -> None: + assert categories([]) == ["document-shape"] + + +# -------------------------------------------------------------------------- +# Payload kind +# -------------------------------------------------------------------------- + + +def test_text_where_a_pointer_belongs() -> None: + """The case format_value refuses for values, caught for hand-built trees.""" + records = dataset(individual(types.GedcomStructure(tag="FAMS", text="@F1@"))) + assert "misplaced-payload" in categories(records) + + +def test_pointer_where_text_belongs() -> None: + records = dataset(individual(types.GedcomStructure(tag="SEX", pointer="@I2@"))) + assert categories(records) == ["misplaced-payload"] + + +def test_pointer_structure_with_no_pointer() -> None: + records = dataset(individual(types.GedcomStructure(tag="FAMS"))) + assert categories(records) == ["misplaced-payload"] + + +def test_payload_on_a_structure_that_takes_none() -> None: + records = dataset(individual(types.GedcomStructure(tag="BAPL", text="x"))) + assert categories(records) == ["misplaced-payload"] + + +# -------------------------------------------------------------------------- +# Substructures +# -------------------------------------------------------------------------- + + +def test_unknown_substructure() -> None: + """A surname belongs under NAME, not under a birth.""" + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="SURN", text="Doe")) + assert categories(dataset(individual(birth))) == ["unknown-substructure"] + + +def test_extension_substructure_is_not_unknown() -> None: + """An undocumented extension tag is permitted anywhere.""" + records = dataset(individual(types.GedcomStructure(tag="_MINE", text="x"))) + assert gedcom7.validate(records) == [] + + +def test_substructure_of_an_extension_is_not_checked() -> None: + """Below an extension the specification says nothing, so neither does this.""" + extension = types.GedcomStructure(tag="_MINE") + extension.append_child(types.GedcomStructure(tag="SURN", text="Doe")) + assert gedcom7.validate(dataset(individual(extension))) == [] + + +# -------------------------------------------------------------------------- +# Payload values +# -------------------------------------------------------------------------- + + +def test_malformed_payload() -> None: + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="AGE", text="25x")) + assert categories(dataset(individual(birth))) == ["malformed-payload"] + + +@pytest.mark.parametrize("text", ["32 JAN 2000", "0 JAN 2000", "30 FEB 2000"]) +def test_day_outside_the_month(text: str) -> None: + """The grammar allows any digits for a day, so the range is checked here.""" + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="DATE", text=text)) + assert categories(dataset(individual(birth))) == ["invalid-date"] + + +def test_month_that_does_not_exist() -> None: + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="DATE", text="1 FOO 2000")) + assert categories(dataset(individual(birth))) == ["invalid-date"] + + +def test_dates_inside_a_range_are_checked() -> None: + birth = types.GedcomStructure(tag="BIRT") + birth.append_child( + types.GedcomStructure(tag="DATE", text="BET 1 JAN 2000 AND 32 JAN 2000") + ) + assert categories(dataset(individual(birth))) == ["invalid-date"] + + +def test_other_calendars_keep_their_own_months() -> None: + """TSH is a Hebrew month, and these tables only name the Gregorian ones.""" + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="DATE", text="HEBREW 1 TSH 5760")) + assert gedcom7.validate(dataset(individual(birth))) == [] + + +def test_enumeration_values_are_not_checked() -> None: + """A known gap: nothing here carries the vocabularies, so Q passes.""" + records = dataset(individual(types.GedcomStructure(tag="SEX", text="Q"))) + assert gedcom7.validate(records) == [] + + +# -------------------------------------------------------------------------- +# Schema +# -------------------------------------------------------------------------- + + +def schema_dataset(*definitions: str) -> list[types.GedcomStructure]: + records = dataset(individual(types.GedcomStructure(tag=FOAF, text="x"))) + schema = types.GedcomStructure(tag="SCHMA") + for text in definitions: + schema.append_child(types.GedcomStructure(tag="TAG", text=text)) + records[0].append_child(schema) + return records + + +def test_two_tags_for_one_uri() -> None: + """dumps would pick between them arbitrarily.""" + records = schema_dataset(f"_ONE {FOAF}", f"_TWO {FOAF}") + assert categories(records) == ["schema-conflict"] + + +def test_two_uris_for_one_tag() -> None: + """The parser refuses to resolve such a tag at all.""" + records = schema_dataset(f"_SAME {FOAF}", "_SAME http://example.com/other") + assert "schema-conflict" in categories(records) + + +def test_undeclared_extension_tag() -> None: + """dumps refuses this at write time; it is reported with everything else.""" + records = dataset(individual(types.GedcomStructure(tag=FOAF, text="x"))) + assert categories(records) == ["undeclared-extension"] + + +def test_declared_extension_tag_is_clean() -> None: + assert gedcom7.validate(schema_dataset(f"_SKYPEID {FOAF}")) == [] + + +# -------------------------------------------------------------------------- +# The reported errors +# -------------------------------------------------------------------------- + + +def test_error_says_where() -> None: + """A list of problems is only useful if each names its structure.""" + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="DATE", text="32 JAN 2000")) + (error,) = gedcom7.validate(dataset(individual(birth))) + assert error.path == "@I1@ INDI > BIRT > DATE" + assert error.structure is birth.children[0] + + +def test_every_problem_is_reported_not_just_the_first() -> None: + """The point of returning a list rather than raising.""" + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="DATE", text="32 JAN 2000")) + birth.append_child(types.GedcomStructure(tag="SURN", text="Doe")) + records = dataset( + individual(birth, types.GedcomStructure(tag="FAMS", pointer="@F9@")) + ) + assert categories(records) == [ + "dangling-pointer", + "invalid-date", + "unknown-substructure", + ] + + +# -------------------------------------------------------------------------- +# dumps(validate=True) +# -------------------------------------------------------------------------- + + +def test_dumps_does_not_validate_by_default() -> None: + """Validation costs a full walk, so it stays opt-in.""" + records = dataset(individual(types.GedcomStructure(tag="FAMS", pointer="@F9@"))) + assert "1 FAMS @F9@" in gedcom7.dumps(records) + + +def test_dumps_validate_raises_with_every_error() -> None: + birth = types.GedcomStructure(tag="BIRT") + birth.append_child(types.GedcomStructure(tag="DATE", text="32 JAN 2000")) + records = dataset( + individual(birth, types.GedcomStructure(tag="FAMS", pointer="@F9@")) + ) + with pytest.raises(GedcomValidationError) as caught: + gedcom7.dumps(records, validate=True) + assert sorted(e.category for e in caught.value.errors) == [ + "dangling-pointer", + "invalid-date", + ] + + +def test_dumps_validate_writes_a_clean_dataset() -> None: + records = dataset(individual()) + assert gedcom7.dumps(records, validate=True, byte_order_mark=False).startswith( + "0 HEAD" + )