diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f480993..c725acf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,15 +7,27 @@ jobs: if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest + env: + UV_PYTHON: ${{ matrix.python-version }} strategy: matrix: python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] - docutils-version: ['0.18', '0.19'] - pytest-version: ['7', '8', '9'] + docutils-version: ['0.20.1', '0.21.2'] + pytest-version: ['7.2.0', '8.4.2', '9.1.1'] + include: + - pytest-version: '7.2.0' + pytest-asyncio-version: '0.21.2' + pytest-rerunfailures-version: '13.0' + - pytest-version: '8.4.2' + pytest-asyncio-version: '1.4.0' + pytest-rerunfailures-version: '16.4' + - pytest-version: '9.1.1' + pytest-asyncio-version: '1.4.0' + pytest-rerunfailures-version: '16.4' exclude: - # Exclude pytest 7 from Python 3.14 to reduce matrix size + # Pytest 7 does not support Python 3.14. - python-version: '3.14' - pytest-version: '7' + pytest-version: '7.2.0' steps: - uses: actions/checkout@v7 @@ -30,26 +42,34 @@ jobs: - name: Install dependencies run: uv sync --all-extras --dev - - name: Install specific pytest version - run: uv pip install "pytest~=${{ matrix.pytest-version }}.0" + - name: Install matrix versions + run: >- + uv pip install + "docutils==${{ matrix.docutils-version }}" + "pytest==${{ matrix.pytest-version }}" + "pytest-asyncio==${{ matrix.pytest-asyncio-version }}" + "pytest-rerunfailures==${{ matrix.pytest-rerunfailures-version }}" + + - name: Check dependency consistency + run: uv pip check - name: Print python and pytest versions run: | python -V - uv run python -V - uv run pytest --version + uv run --no-sync python -V + uv run --no-sync pytest --version - name: Lint with ruff check - run: uv run ruff check . + run: uv run --no-sync ruff check . - name: Format with ruff format - run: uv run ruff format . --check + run: uv run --no-sync ruff format . --check - name: Lint with mypy - run: uv run mypy . + run: uv run --no-sync mypy . - name: Test with pytest - run: uv run py.test --cov=./ --cov-report=xml + run: uv run --no-sync py.test --cov=./ --cov-report=xml - uses: codecov/codecov-action@v7 with: diff --git a/README.md b/README.md index 850dfc9..b8d7c26 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,10 @@ git-pull projects, e.g. [cihai], [vcs-python], or [tmux-python]. Two components: -1. `doctest_docutils` module: Same specification as `doctest`, but can parse reStructuredText - and markdown -2. `pytest_doctest_docutils`: Pytest plugin, collects test items for pytest for reStructuredText and markdown files +1. `doctest_docutils`: a doctest-shaped direct API and CLI for reStructuredText + and Markdown +2. `pytest_doctest_docutils`: a pytest plugin that collects shared-state groups + from reStructuredText and Markdown files This means you can do: @@ -24,8 +25,8 @@ Two components: ### doctest module -This extends standard library `doctest` to support anything docutils can parse. -It can parse reStructuredText (.rst) and markdown (.md). +This uses standard-library `doctest` prompt and comparison conventions while +parsing reStructuredText (`.rst`) and Markdown (`.md`). See more: @@ -64,7 +65,7 @@ It supports two barebones directives: #### Usage -The `doctest_docutils` module preserves standard library's usage conventions: +The `doctest_docutils` module preserves the standard library's command shape: ##### reStructuredText @@ -84,10 +85,10 @@ $ python -m doctest_docutils README.md -v ### pytest plugin -_This plugin disables [pytest's standard `doctest` plugin]._ - -This plugin integrates `doctest_docutils` with pytest so documentation examples -run with the surrounding `conftest.py` setup. +This plugin runs documentation examples as pytest items. It composes with +[pytest's standard `doctest` plugin]: gp-libs owns matching documentation files, +while pytest continues to supply fixtures, checker and report options, and +Python-module doctest collection. ```console $ pytest docs/ @@ -154,7 +155,7 @@ You can test the unpublished version of g before its released. To lift the development burden of supporting legacy APIs, as this package is lightly used, minimum constraints have been pinned: -- docutils: 0.20.1+ +- docutils: >=0.20.1,<0.22 - myst-parser: 2.0.0+ If you have even passing interested in supporting legacy versions, file an diff --git a/docs/adrs/0001-typed-vanilla-doctest-core.md b/docs/adrs/0001-typed-vanilla-doctest-core.md index 44e813f..33988c4 100644 --- a/docs/adrs/0001-typed-vanilla-doctest-core.md +++ b/docs/adrs/0001-typed-vanilla-doctest-core.md @@ -50,7 +50,7 @@ pytest's `repr_failure` reconstruct locations by slicing that single string. So the blocks must be laid out on a synthetic page with blank-line padding and a clamp, and a wholly-skipped block must be lifted back out to report at all. -**Prompt-free `{testcode}` needs a CPython private.** The per-example loop +**Prompt-free `{testcode}` needs a second execution lane.** The per-example loop [`DocTestRunner.__run`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1344) hard-codes `"single"` ([`Lib/doctest.py:1400`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1400)), @@ -94,9 +94,9 @@ The decoupling is not "group versus block". It is **scheduling identity versus diagnostic identity**: pytest schedules the group, while each `DocTest` keeps its own source location, examples and failure gutter. -One {class}`pytest.Item` per (document, group). Inside it, a tuple of per-block -`DocTest`s run in phase order against one live `globs` mapping that never leaves -the item. +One {class}`pytest.Item` per (document, group). Inside it, immutable per-block +recipes materialize fresh `DocTest`s just before execution. They run in phase +order against one live `globs` mapping that never leaves the item. The execution shape is partly precedented. `sphinx.ext.doctest` runs several `DocTest`s against one shared group namespace — but only for the *test* phase. @@ -112,10 +112,11 @@ why `SphinxDocTestRunner` overrides a private stdlib method to swallow the resulting `IndexError`. Mapping the group onto one {class}`pytest.Item` while each block keeps its own identity is the contribution. -With the default checker that buys, for free and with no override of -`repr_failure` or `reportinfo`: per-block failure locations, per-block gutters, -and per-block "location unknown". -Meanwhile `-k`, `--lf`, `-x`, `--reruns` and every `--dist` mode are structurally +With the default checker that buys per-block failure locations, per-block +gutters, and per-block "location unknown" without synthetic merged source. The +adapter uses its narrow `repr_failure` renderer to retain the comparison-time +checker and does not override `reportinfo`. +Meanwhile `-k`, `--lf`, `-x`, `--reruns` and xdist scheduling are structurally incapable of splitting the shared state, because there is only one item to schedule. @@ -153,47 +154,27 @@ With one `DocTest` per block, every failure therefore carries its own `filename` and `lineno` for free. A block reached through `.. include::` reports the *included* file. A block docutils could not locate carries `lineno=None` and takes pytest's honest `EXAMPLE LOCATION UNKNOWN` branch **without poisoning its -siblings**. The default-checker path requires no override of `repr_failure` or -`reportinfo`. +siblings**. The adapter's narrow failure renderer retains those same per-failure +locations while reusing the checker instance that made each comparison. It does +not override `reportinfo`. This is what makes merging unnecessary: the synthetic page, its blank-line padding and its clamp exist only to reconstruct locations from a single spliced docstring, and there is no spliced docstring here. -**2. `_DocTestRunner__run` is an ordinary attribute override.** Name mangling -rewrites the *call site* at compile time, so the `self.__run(...)` lookup inside -`run()` -([`Lib/doctest.py:1571`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1571)) -resolves through the instance's MRO like any other attribute. A subclass that -defines `_DocTestRunner__run` takes over the loop, with `run()` untouched. - -This page is itself a doctest, so the claim is checked on every run: - -```{doctest} ->>> import doctest ->>> import io ->>> fired = [] ->>> class Runner(doctest.DocTestRunner): -... def _DocTestRunner__run(self, test, compileflags, out): -... fired.append(test.name) -... return super()._DocTestRunner__run(test, compileflags, out) ->>> example = doctest.Example("1 + 1\n", "2\n") ->>> test = doctest.DocTest([example], {}, "demo", "demo.py", 0, None) ->>> Runner().run(test, out=io.StringIO().write) -TestResults(failed=0, attempted=1) - -The subclass method ran, and `run()` was never overridden: - ->>> fired -['demo'] -``` - -Keeping `run()` as stdlib's matters: it owns the save-and-restore of +**2. The ordinary lane does not need to own CPython's loop.** A reporter +subclass can retain failures through `report_failure` and +`report_unexpected_exception` while inheriting the per-example loop unchanged +([`Lib/doctest.py:1286-1314`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1286-L1314)). +Keeping `run()` and its private loop as stdlib's matters: `run()` owns the +save-and-restore of `sys.stdout`, `pdb.set_trace`, `linecache.getlines`, `sys.displayhook`, `_colorize.can_colorize` and the `PYTHON_COLORS`/`FORCE_COLOR` environment variables, all in its own `finally` ([`Lib/doctest.py:1534-1573`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1534-L1573)). -That contract is not reproduced; it is inherited. +That contract is inherited for prompt blocks. Extended `exec` profiles use a +separate bounded runtime and make no claim to inherit it; see +{doc}`0002-runner-conformance-across-cpython`. **3. Making the item the sharing unit dissolves the *affinity* problem.** A live mapping never crosses a process boundary, so there is nothing for xdist to split, @@ -214,7 +195,7 @@ stated here rather than discovered later. | Signal | Granularity | Notes | |---|---|---| -| Failure location, `want`/`got`, gutter | **per block** | default `repr_failure` iterates failures and reads each one's own `DocTest`; custom checkers use the same locations in the narrow rendering branch | +| Failure location, `want`/`got`, gutter | **per block** | the adapter renderer iterates failures, reads each one's own `DocTest`, and uses its retained comparison-time checker | | `EXAMPLE LOCATION UNKNOWN` | **per block** | a block with `lineno=None` does not affect its siblings | | `passed` / `failed` / `skipped` | **per item** | `TestReport.outcome` is one scalar | | JUnit `` | **per item** | node reporters are keyed by node id | @@ -233,16 +214,20 @@ detail, not as a pytest outcome.** Setup and cleanup are infrastructure and do not contribute a passed or skipped test. No extra reports are synthesized. "Typed block detail" needs a channel, or implementers will re-invent skip lifting -or write to stderr. The channel is a `GroupResult` — one `BlockResult` per block, -each carrying phase, outcome, gate reason and location — attached to the item and -rendered in two places: the failure longrepr when the group fails, and a -terminal-summary line at `-rs`. It is **not** visible at default verbosity, and -it never becomes a JUnit `` entry. +or write to stderr. The channel begins with a `GroupResult` — one `BlockResult` +per block, each carrying phase, outcome, gate reason and location — attached to +the item. A versioned, JSON-safe projection must then cross the worker boundary +for terminal rendering. It never becomes a JUnit `` entry. That is a real product loss relative to lifting a gated block into its own item, and it is accepted deliberately: **a gated block inside a mixed group gives up its selectable skip row.** The information survives; the addressable unit does not. +The spike retains `GroupResult` worker-local and reports secondary cleanup +outcomes, but does not yet transport partial-skip detail to the controller or +terminal summary. That projection remains an acceptance gate rather than an +implemented claim. + `subtests` — a builtin pytest plugin since 9.0, exporting `pytest.Subtests` and `pytest.SubtestReport` — can emit per-block outcomes, and is the only sanctioned mechanism that can. It is not adopted here: pytest documents it as experimental, @@ -272,29 +257,29 @@ contracts settings model | Layer | Owns | Must not know | |---|---|---| | `contracts` | Public protocols and immutable contribution records: `DocumentParser`, `ExecutionProfile`, `ExecutionRuntime`, `CheckerFactory`, `Contributor` and `Registrar` | Sphinx, pytest, xdist and host lifecycle objects. Only stdlib and public parser types cross this boundary | -| `settings` | Three immutable facets — `ParseSettings`, `ProjectionSettings`, `RunSettings`. `None` sentinels at the resolve boundary make a future default change announceable | registries, pytest's `Config`, argparse, ini format, Sphinx's `app`. The host extracts; this resolves | -| `model` | `ParsedBlock`, `ParsedOutput`, `BlockKind`, `Phase`, `Diagnostic`, `ProjectedBlock`, `GroupPlan`, `RunContext` and the result types. **No stdlib subclasses.** | docutils, MyST, Sphinx, pytest, xdist, the filesystem. Stdlib imports only | +| `settings` | Three immutable facets — `ParseSettings`, `ProjectionSettings`, `RunSettings`. Hosts resolve their own input and instantiate final settings | registries, pytest's `Config`, argparse, ini format, Sphinx's `app` | +| `model` | `ParsedBlock`, `ParsedOutput`, `BlockKind`, `Phase`, `Diagnostic`, `ProjectedBlock`, `GroupPlan` and the result types. **No stdlib subclasses.** | docutils, MyST, Sphinx, pytest, xdist, the filesystem. Stdlib imports only | | `registry` | A private mutable builder and the public immutable `RegistrySnapshot` consumed by every later layer | host lifecycle objects after the snapshot is frozen | -| `markup/` | Text → `(blocks, diagnostics)`. **The whole docutils vocabulary**: which node classes each kind may arrive as, the `BlockAttributes` stamp, line-number recovery and its per-front-end meaning, `.. include::` attribution, `nodes.comment` traversal, reporter capture, idempotent directive registration, per-kind `option_spec` | Groups as a runtime concept, `DocTest`, pytest, pairing | +| `markup/` | Text → `(blocks, diagnostics)`. Recognized docutils and Sphinx node vocabulary: field-level stamp validation, line-number recovery, `.. include::` attribution, `nodes.comment` traversal, reporter capture, idempotent built-in directive registration, and preservation of custom stamped kind names | Groups as a runtime concept, `DocTest`, pytest, pairing | | `project` | The **only** place grouping exists: `*` expansion, anonymous naming, phase order, `testcode`/`testoutput` pairing, option defaults, name minting. A pure function | docutils, pytest, the filesystem, whether anything will run. Evaluates no user code | -| `runner` | `_DocTestRunner__run`; option merge, `SKIP`-after-merge, `FAIL_FAST`, `report_*` dispatch, version shims. `run_group()` owns phase sequencing, run-time `:skipif:`, the profile's context manager, and the `try`/`finally` guaranteeing `testcleanup` | docutils, markup, pytest. Never overrides `run()` | +| `runner` | Stock CPython execution for prompt blocks and a bounded independent loop for extended profiles. `run_group()` owns materialization, phase sequencing, run-time gates, profile lifetimes, and cleanup after block or gate failures | docutils, markup, pytest. Never overrides CPython's `run()` or private loop | | `pytest_doctest_docutils` | Options, `Document(pytest.Module)`, `DocutilsItem`, group `globs` lifetime, the outcome contract, built-in-plugin composition, surfacing diagnostics | docutils node classes, MyST configuration, grouping rules | -`pytest_doctest_docutils._compat` is the only module that imports +`_pytest_doctest_compat` is the only module that imports `_pytest.doctest`, behind a pinned support matrix. See {doc}`0006-pytest-private-api-compatibility`. ### Settings and the frozen registry -Settings have **lifetimes**, not just precedence. "Resolve exactly once at session -start" cannot be true of document front matter, which does not exist until that -document is parsed. Three scopes: +Settings have **lifetimes**, not just precedence. The spike establishes three +immutable facets and leaves document front matter for a later decision: -| Scope | Owns | Resolved | +| Facet | Owns | Resolved | |---|---|---| -| `SessionSettings` | defaults plus normalized host configuration | once, before collection or direct execution | -| `DocumentSettings` | the front-matter overlay a page is permitted to set | per document, after parsing | -| block / example policy | directive options, then inline `# doctest:` flags | per block, at projection and run | +| `ParseSettings` | diagnostic suppression | before parsing or extracting one document | +| `ProjectionSettings` | unlabelled-block grouping policy | before projecting one document | +| `RunSettings` | runner flags, failure continuation and checker selection | before executing one group attempt | +| block / example policy | directive options, gates, then inline `# doctest:` flags | per block, at projection and run | Not every field shares one ladder, so the precedence is stated per axis. For option flags it follows Sphinx: **runner defaults → directive or output @@ -307,11 +292,13 @@ not user-configurable knobs — exposing them would let a project produce node i no other project can read. `ProjectionSettings.ungrouped` defaults to `"default"`. An unlabelled runnable -block therefore joins the page's `default` group unless the user explicitly asks -for block isolation. This clean-slate default follows Sphinx's author vocabulary -and makes state-sharing opt-out rather than a project-specific surprise. +block therefore joins the page's `default` group unless the caller explicitly +asks for block isolation. This clean-slate core default follows Sphinx's author +vocabulary. The pytest adapter defaults to `"block"` to preserve gp-libs' +released per-block isolation; an explicit, unargumented Sphinx directive still +stamps `groups=["default"]` and shares under either adapter setting. -The **registry** is a separate input resolved after `SessionSettings`. "Frozen" +The **registry** is a separate input resolved before pipeline use. "Frozen" means the public `RegistrySnapshot` contains immutable mappings and records; the mutable builder is private and discarded. Registering after the host freezes its snapshot is an error. Keeping these values separate matters under xdist: settings @@ -325,9 +312,10 @@ builder and every consumer receives the same `RegistrySnapshot`. Host-specific registration timing is a separate decision; the core contract does not import a pytest hook or a Sphinx application. -The direct, pytest and Sphinx lifecycles, including the xdist consistency check, -are specified in {doc}`0007-host-plugin-registration-lifecycle` so this record -does not mistake a host bootstrap policy for a core dependency. +The direct and pytest lifecycles are specified in +{doc}`0007-host-plugin-registration-lifecycle`. Sphinx contribution timing and +an xdist registry manifest remain proposals there; the first spike proves only +resolved-doctree extraction and homogeneous-worker execution. ### Item lifecycle @@ -339,7 +327,7 @@ an implementer cannot get it wrong by omission: `setup()`, `reportinfo()` and `_check_all_skipped()`, so the subclass must define it even though a group holds many tests. `self.dtest` is a synthetic **zero-example** `DocTest` for the group, and its `globs` **is** the canonical - live mapping — the same object every `RunnableBlock`'s test is given. That + live mapping — the same object every freshly materialized test is given. That makes the inherited `setup()` inject fixtures into exactly the right place with no override of the injection itself. @@ -350,23 +338,24 @@ an implementer cannot get it wrong by omission: `super().setup()` so fixtures inject into that same object. Clearing in place rather than rebinding is what keeps `item.globs is run.globs` true for every block, and what stops attempt two of a `--reruns` run from reading attempt - one's mutations. Each `RunnableBlock` is materialized against this object — - its `DocTest.globs` assigned *after* construction, because `DocTest.__init__` - copies. + one's mutations. It does not materialize block tests; `run_group()` does that + immediately before each block runs, after its gates and paired output have + been resolved. 3. **`runtest()`** is overridden. It must not delegate to `DoctestItem.runtest`, which runs a single `dtest` with `clear_globs` defaulting to `True` — that would empty the shared mapping after the first block. It calls `run_group()`, which materializes and runs each - `RunnableBlock` in phase order with `clear_globs=False`, evaluates `:skipif:` + `ProjectedBlock` in phase order with `clear_globs=False`, evaluates `:skipif:` and `:pyversion:` against the live mapping and interpreter, - finalizes each paired `want` from its gated `ExpectedOutput`, and wraps the - body in a `try`/`finally` so cleanup runs whether or not the body raised. When - cleanup *also* fails, the body's failure is the one raised; cleanup's is - recorded in the `GroupResult`. - - The `OutcomeException` re-raise, the `bdb.BdbQuit` → `outcomes.exit` - conversion and `continue_on_failure` are reimplemented here, because - `PytestDoctestRunner` is nested inside a factory and cannot be imported. + finalizes each paired `want` from its gated `ExpectedOutput`, and preserves + cleanup after setup, test, or gate failure. When cleanup *also* fails, the + body's failure is the one raised; cleanup's is recorded in the `GroupResult`. + Profile context entry and exit failures still need the representation decision + in {doc}`0002-runner-conformance-across-cpython`. + + A host-neutral `ExceptionPolicy` classifies exceptions that must propagate. + The pytest adapter supplies pytest's outcome and debugger-exit policy; the + core does not import pytest or reproduce its private runner class. 4. **Outcome** follows [](#the-outcome-contract): only `Phase.TEST` blocks determine pass versus skip. A plan with no test block yields no item. Setup and cleanup are infrastructure: an error there may fail or abort the item, but a @@ -377,22 +366,20 @@ an implementer cannot get it wrong by omission: with grouping. 5. **Failure projection** flattens every `Failed.failures` tuple in block order - and raises pytest's `MultipleDoctestFailures`. With the default pytest checker, - `repr_failure` is inherited unchanged and therefore preserves pytest's exact - output. A contributed checker must supply both `check_output` and - `output_difference`; the item takes one small custom-rendering branch so the - checker that decided the failure also explains it. `repr_failure` does not - render `GroupResult`. Partial-skip detail goes to a report section and the - terminal summary; see [](#the-outcome-contract). - -6. **Reporting across processes.** The controller never sees the item — it - receives serialized `TestReport` dictionaries. So `pytest_runtest_makereport` - copies a **versioned, JSON-safe** block summary onto the report. The summary - contains only the schema version, group name, block name, phase, outcome, - counts and structured skip reason — never exceptions or live objects. pytest - serializes arbitrary report attributes and xdist reconstructs them - controller-side. The rich `GroupResult`, and every exception in it, stays - worker-local. + and raises pytest's `MultipleDoctestFailures`. The item uses one quarantined + renderer for both pytest's checker and contributed checkers so the instance + that decided each failure also explains it. The renderer preserves pytest's + report choice, gutter and per-failure location shape; it does not render + `GroupResult`. Secondary cleanup failures use a report section. The + partial-skip terminal channel described in [](#the-outcome-contract) remains + an acceptance gate. + +6. **Reporting across processes is deferred.** The controller never sees the + item; it receives serialized `TestReport` dictionaries. A complete adapter + therefore needs `pytest_runtest_makereport` to copy a versioned, JSON-safe + block summary onto the report. The spike keeps the rich `GroupResult` and its + exceptions worker-local, so partial-skip detail does not yet reach the + controller or terminal summary. ### Vocabulary @@ -411,7 +398,7 @@ referents. Each term below is decided once and used only that way. | directive | inline `# doctest: +FLAG` | — | a docutils directive with an `option_spec` | Reserved for the docutils meaning. doctest's form is "inline flags" | | optionflags | an int bitmask; `register_optionflag` | `doctest_optionflags` ini | `:options:` plus `doctest_default_flags` | Keep verbatim. `register_optionflag` is the one genuinely cross-library extension point | | setup / cleanup | `setUp`/`tearDown` on the suite builders | fixtures | `testsetup`/`testcleanup` directives | Author-facing names stay Sphinx's; `phase` is the internal ordering axis; a fixture is never "setup" | -| name | a dotted path; `__lt__` compares it as **text** | node id is `parent.nodeid + "::" + name` | the *group* name, shared by every block in it | `DocTest.name` is a unique machine-independent id. The absolute path lives only in `filename` | +| name | a dotted path; `__lt__` compares it as **text** | node id is `parent.nodeid + "::" + name` | the *group* name, shared by every block in it | `DocTest.name` is unique within one document plan; pytest's parent path makes the node id suite-wide. The source path lives in `filename` | ### Data model @@ -431,12 +418,20 @@ Failure: t.TypeAlias = doctest.DocTestFailure | doctest.UnexpectedException class RuntimeOutcome(t.NamedTuple): results: doctest.TestResults failures: tuple[Failure, ...] + skipped: int # explicit because Python 3.10 TestResults cannot carry it + + +class ExceptionPolicy(t.Protocol): + def should_propagate(self, error: BaseException) -> bool: ... + + def is_abort(self, error: BaseException) -> bool: ... class RuntimeSettings(t.NamedTuple): optionflags: int continue_on_failure: bool checker: doctest.OutputChecker + exception_policy: ExceptionPolicy class CheckerFactory(t.Protocol): @@ -458,7 +453,7 @@ class ExecutionProfile(t.Protocol): class ParsedBlock(t.NamedTuple): kind: str # registered BlockKind name - source: str # dedented body, verbatim author text + source: str # dedented, outer-newline-normalized extracted text path: pathlib.Path # the file the text lives in, not the collected document line: int | None # None when docutils could not recover one document_order: int # position among blocks AND outputs; the pairing key @@ -471,8 +466,9 @@ class ParsedBlock(t.NamedTuple): class ParsedOutput(t.NamedTuple): - """A `testoutput` body. Not a block: it never runs.""" + """An expected-output body. Not a block: it never runs.""" + kind: str # the BlockKind.pairs_with name stamped on the source node text: str path: pathlib.Path line: int | None @@ -487,7 +483,6 @@ class BlockKind(t.NamedTuple): phase: Phase profile_name: str # resolved against the frozen registry, not held here pairs_with: str | None - grouped: bool # --- projected: one per block, with everything the runner needs ------------ @@ -534,35 +529,17 @@ class GroupPlan(t.NamedTuple): seed: t.Mapping[str, t.Any] # initial namespace; copied per attempt -# --- run: one attempt materializes fresh stdlib objects -------------------- - - -class RunnableBlock(t.NamedTuple): - """Materialized per attempt. Its `DocTest` is never retained by a plan.""" - - recipe: ProjectedBlock - test: doctest.DocTest # fresh stock objects, built this attempt - runtime: ExecutionRuntime # from the profile factory, this attempt - - -class RunContext: - """One execution attempt. Owns the live mapping; a plan never does.""" - - plan: GroupPlan - globs: dict[str, t.Any] # cleared in place and reseeded per attempt - runtimes: dict[str, ExecutionRuntime] # one per profile used by the group - - # --- results: a discriminated union, so invalid states cannot be built ----- class Counts(t.NamedTuple): + failed: int # may exceed len(Failed.failures) under report-only-first attempted: int skipped: int # a PASSING block can still carry skipped examples class SkipReason(t.NamedTuple): - kind: t.Literal["skipif", "inline-flag", "pyversion", "profile"] + kind: t.Literal["skipif", "inline-flag", "pyversion"] detail: str # the gate expression, the flag, the specifier @@ -576,6 +553,7 @@ class Failed(t.NamedTuple): counts: Counts # PLURAL: continue_on_failure yields several from one block failures: tuple[Failure, ...] + checker: doctest.OutputChecker # the instance that made the comparison class Skipped(t.NamedTuple): @@ -601,9 +579,11 @@ class GroupResult(t.NamedTuple): `ParsedBlock` carries no `want`, because neither owner of a `want` is the parsed block: for a prompt-form block it is *inside* `source` and -{class}`doctest.DocTestParser` extracts it at projection, and for a -`testcode`/`testoutput` pair it is a separate `ParsedOutput`. Conflating the two -was what made "projection owns pairing" untrue. +{class}`doctest.DocTestParser` extracts it at projection, and for a paired block +it is a separate `ParsedOutput`. The output retains its stamped `kind`, so a +contributed `BlockKind.pairs_with` relationship survives extraction without +hard-coding `testoutput`. Conflating the two was what made "projection owns +pairing" untrue. `document_order` is one monotonic sequence shared by runnable blocks and `ParsedOutput` records. Pairing therefore follows the source stream even when an @@ -621,8 +601,9 @@ registration name resolve against the frozen registry. assigns `globs` to it after construction, and a run mutates that mapping — so a plan retaining one would not be a recipe, it would be last attempt's state. Under `--reruns` that is the false-green this design exists to prevent. `ProjectedBlock` -therefore carries the *ingredients*, and `RunContext` materializes fresh stock -`Example` and `DocTest` objects for every attempt. +therefore carries the *ingredients*, and `run_group()` materializes fresh stock +`Example` and `DocTest` objects for every block in every attempt. Attempt-local +runtimes remain local implementation state rather than a public context object. **The ingredients are per example, not per block.** One prompt block routinely yields several {class}`doctest.Example` objects, each with its own `source`, @@ -664,6 +645,9 @@ Four result details are load-bearing: - **`Failed.failures` is plural.** Under `continue_on_failure` one block reports several failures; a singular field silently keeps the first. +- **`Failed.checker` is the comparison-time instance.** A contributed checker + may carry configuration or state, so reconstructing one during pytest failure + rendering can explain the result differently from the object that decided it. - **`Passed` carries counts.** A block can pass *and* have skipped examples — `failed=0 attempted=2 skipped=1` — and a result type without counts loses the skip entirely, which is the same information ADR 0001's outcome contract @@ -672,8 +656,8 @@ Four result details are load-bearing: while an all-`SKIP` doctest has parsed examples and reports them skipped. The reason alone cannot distinguish those cases. - **`SkipReason` is typed.** A skip originates from `:skipif:`, an inline - `# doctest: +SKIP`, `:pyversion:`, or a profile declining to run — and "the gate - expression" describes only the first. + `# doctest: +SKIP`, or `:pyversion:` — and "the gate expression" describes only + the first. Profile decline is not in the initial runtime contract. **Exception precedence is phase-aware, not a single ladder.** Grouping {exc}`KeyboardInterrupt`, a debugger quit, {exc}`pytest.skip`, `xfail` and @@ -682,21 +666,23 @@ a real test failure, while a session exit must never be converted into block dat | Class | Examples | Rule | |---|---|---| -| process, debugger or session abort | {exc}`KeyboardInterrupt`, `SystemExit`, `bdb.BdbQuit`, `pytest.exit` | always propagates from every phase; cleanup still runs, but cannot replace it | +| process, debugger or session abort | core: {exc}`KeyboardInterrupt`; pytest policy also classifies `SystemExit`, `bdb.BdbQuit`, and `pytest.exit` | propagates from every block phase; cleanup runs and cannot replace it | | host outcome from setup or test | `pytest.skip`, `pytest.xfail` | propagates as the host outcome after cleanup | -| doctest mismatch from test | `DocTestFailure`, `UnexpectedException` | retained in source order and projected to the host after cleanup | -| runtime error from setup or test | a gate that raises, or a profile that cannot start | recorded as `Errored`; becomes the primary failure when no abort or host outcome exists | -| cleanup exception or cleanup host outcome | any exception, including `pytest.skip` or `pytest.xfail` | recorded as `secondary` when a primary exists; otherwise becomes the item failure, never a skip or xfail | +| doctest mismatch or ordinary executed exception | `DocTestFailure`, `UnexpectedException` in any phase | retained in source order and projected as doctest failures after cleanup | +| gate or host-owned error from setup or test | a gate that raises, `pytest.fail`, or another propagated host outcome | recorded as `Errored`; becomes the primary failure when no abort or host outcome exists | +| cleanup gate or host-owned error | a propagated error, including `pytest.skip` or `pytest.xfail` | recorded as `secondary` when a primary exists; otherwise becomes the item failure, never a skip or xfail | Profile runtimes are entered through {class}`contextlib.ExitStack`, so a partial -startup unwinds deterministically in reverse. Classifying which exceptions are -pytest outcomes is the pytest adapter's job; the core knows only the phase. +startup unwinds deterministically in reverse. The adapter supplies an +`ExceptionPolicy` that identifies host outcomes and the smaller set of aborts +that outrank every recorded result; the core remains host-neutral while +preserving their propagation semantics. `ParsedBlock.line` being nullable is load-bearing, not defensive. A bare `>>>` block nested in a `.. note::`, a list item or a block quote reports `line=None, source=None` from docutils, and an `.. include::`-ed block numbers against the -*included* file. Both propagate to `DocTest.lineno=None` and pytest's honest -"location unknown", rather than to a fabricated number. +*included* file. The first propagates to `DocTest.lineno=None` and pytest's +honest "location unknown"; the second retains the included path and line. `ProjectedBlock` carries phase and gate because `run_group()` owns phase sequencing, run-time `:skipif:` evaluation and a cleanup `finally` — and cannot do @@ -714,7 +700,7 @@ assertion. **A wildcard block is projected separately per group it joins.** Projection clones the recipe and mints a group-qualified name for each destination. Each -`RunContext` then builds its own `DocTest` from its own recipe. Reusing one +`run_group()` then builds its own `DocTest` from its own recipe. Reusing one `ProjectedBlock` would make its name ambiguous; sharing one materialized `DocTest` would be worse, because `DocTest.globs` is mutable and the second group's assignment would win. @@ -727,7 +713,7 @@ it is not what `sphinx-build` does. Recorded rather than hidden. **`ExecutionProfile` is an immutable factory; `ExecutionRuntime` is per attempt.** A group can mix prompt, `exec` and async blocks, so there is no single per-group -profile. The profile is chosen per block and names a factory; `RunContext` creates +profile. The profile is chosen per block and names a factory; `run_group()` creates one runtime *per distinct profile* the group uses, and blocks sharing a profile share its runtime. An async runtime therefore owns one event loop for the whole group, which is what lets awaited state cross block boundaries. @@ -739,9 +725,9 @@ event-loop lifetime a mode string cannot express. The runtime's context manager what `run_group()` enters, so that lifetime is served without overriding `run()` and stdlib's save-and-restore `finally` stays inherited. -**The ordinary lane does not use the owned loop at all.** For prompt-form blocks — +**The ordinary lane does not use an owned loop at all.** For prompt-form blocks — the overwhelming majority — the runner is a plain reporter subclass over CPython's -*untouched* per-example loop. `_DocTestRunner__run` is invoked only for extended +*untouched* per-example loop. A separate bounded runtime handles extended profiles: `exec` bodies, top-level await, and whatever comes next. Ordinary doctests are then compatible **by construction** rather than by differential testing, and {doc}`0002-runner-conformance-across-cpython`'s harness shrinks to @@ -749,11 +735,11 @@ guarding the extended lane. **A checker owns both comparison and explanation.** The default pytest registration constructs pytest's checker, preserving `ALLOW_UNICODE`, -`ALLOW_BYTES`, `NUMBER` and its inherited failure representation exactly. A -contributed `CheckerFactory` constructs a fresh checker for each runtime. The -same instance performs `check_output()` and `output_difference()`; using pytest's -private `_get_checker()` only at rendering time would let one checker reject the -example and another explain why. +`ALLOW_BYTES` and `NUMBER`. A contributed `CheckerFactory` constructs a fresh +checker for each runtime. The same instance performs `check_output()` and +`output_difference()` through the adapter's pytest-shaped renderer; using +pytest's private `_get_checker()` only at rendering time would let one checker +reject the example and another explain why. **Which docutils node classes a kind may arrive as is a front-end concern, not a `BlockKind` field.** `testsetup`, `testcleanup` and any `:hide:` block are @@ -785,7 +771,10 @@ layer that never changes what is constructed. def extract_blocks( - doctree: nodes.document, *, settings: ParseSettings + doctree: nodes.document, + *, + settings: ParseSettings, + registry: RegistrySnapshot, ) -> ParseResult: ... ``` @@ -793,7 +782,10 @@ layer that never changes what is constructed. one extractor is the *point* of the split, and a second implementation would reintroduce the standalone-versus-Sphinx divergence it exists to prevent. Standalone reST and MyST use both halves; a Sphinx extension calls only the - extractor, on the doctree it already resolved. + extractor, on the doctree it already resolved. Passing the same frozen + registry is load-bearing: extraction derives expected-output stamp names from + registered `BlockKind.pairs_with` relationships before projection resolves + them. **`DocumentParser` is not a {class}`doctest.DocTestParser`.** The two signatures are incompatible — stdlib's is `parse(self, string, name='')` @@ -836,19 +828,19 @@ layer that never changes what is constructed. One genuine nominal edge does exist: `DocTestSuite` sorts its results, and `DocTest.__lt__` returns `NotImplemented` for a non-`DocTest`, so a custom finder must return real `DocTest` objects. -- **`TypedDict` at the docutils boundary.** `BlockAttributes` types what a - directive stamps on a node, with one narrowing accessor that validates once. - This is where `Any` currently enters: `str(node.get("test") or ...)` and - `dict(node.get("options") or {})` are runtime coercions paid for a static hole, - and typeshed's docutils stub makes it worse — its `get(key, failobj: _T) -> _T` - claims `_T` even when the key is present holding something else. +- **Field-level narrowing at the docutils boundary.** External directives stamp + dynamically typed node attributes, so a `TypedDict` would falsely imply that + producers honor an owned schema. Small accessors validate each consumed field; + `ParsedBlock` and `ParsedOutput` are the first trusted typed boundary. - **`t.Literal` for closed vocabularies**, derived from one source of truth so a public signature and a config field cannot diverge. - **Plain `int` keys for optionflags.** {class}`enum.IntFlag` was considered and rejected; see [](#alternatives-rejected). -- **`py.typed` ships.** The project already runs mypy strict over `src` and - `tests`, and the marker file does not exist, so every consumer sees the package - as untyped. This is a packaging defect independent of the rest of this ADR. +- **`doctest_core/py.typed` ships.** The project already runs mypy strict over + `src` and `tests`; the wheel and sdist include the marker so consumers see the + core's public types. The legacy flat `doctest_docutils` and + `pytest_doctest_docutils` facades remain untyped compatibility surfaces unless + they later move behind typed packages or stubs. ### What "vanilla-compatible" promises @@ -859,22 +851,23 @@ several different strengths. |---|---| | `Example` / `DocTest` runtime types | **Exact.** Stock instances, never subclassed for metadata | | plain-text parsing | **Exact.** The stdlib lane uses `DocTestParser` unmodified | -| option flags and checkers | **Exact.** `register_optionflag` and the stdlib checker contract, with pytest's `ALLOW_UNICODE`/`ALLOW_BYTES`/`NUMBER` reachable | +| option flags and checkers | **Exact for the stdlib contract.** `register_optionflag` and `OutputChecker` remain stock; pytest's `ALLOW_UNICODE`/`ALLOW_BYTES`/`NUMBER` are available through its adapter | | prompt-block execution | **Exact.** CPython's own per-example loop, unmodified | -| `DocTestFinder`-shaped Python-object discovery | **Shaped**, as a separate adapter | -| `DocFileSuite` / `DocTestSuite` | **Façade only**, over the stdlib lane. A group plan cannot be expressed through an API that returns one `DocTest` per parser call | +| `DocTestFinder`-shaped Python-object discovery | **Deferred.** The spike implements document-text discovery only | +| `DocFileSuite` / `DocTestSuite` | **Not implemented by the spike.** A future stdlib-shaped façade can cover the plain lane, but cannot express one shared group through an API returning independent `DocTest`s | | `{testcode}`, async, groups, phases, Sphinx gates | **Deliberate extension.** No stdlib equivalent to be compatible with | | pytest collection, fixtures, reporting | pytest's own contracts, composed with rather than replaced | -| Sphinx **node** vocabulary | **Exact.** The `BlockAttributes` stamp is byte-compatible | +| Sphinx **node** vocabulary | **Extractor-compatible.** The core accepts Sphinx's stamps and may retain a metadata superset | | Sphinx **execution** | **Not promised.** See below | **The Sphinx promise is narrow, and this record narrows it deliberately.** What is offered is an *extractor over a Sphinx-resolved doctree* — a pure function from a doctree to blocks, callable from an extension. -{doc}`0007-host-plugin-registration-lifecycle` defines how Sphinx extensions -contribute capabilities, but not a Sphinx execution lifecycle or result channel. -Inventing those would be the builder that [](#alternatives-rejected) turns down. -The record promises doctree consumption and nothing more. +{doc}`0007-host-plugin-registration-lifecycle` proposes how Sphinx extensions +could contribute capabilities, but the spike implements neither that lifecycle +nor a Sphinx execution or result channel. Inventing a result channel would be the +builder that [](#alternatives-rejected) turns down. The implemented promise is +doctree consumption and nothing more. ## Constraints @@ -895,8 +888,9 @@ cited. The full derivation is in `notes/analyses/`. | Custom flag names must be registered at import; ints are `1 << len(OPTIONFLAGS_BY_NAME)` and an unregistered name makes a page fail to **parse** | [`doctest.py:153`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L153) | `report_skip` does not exist at v3.14.2 — the runner has only `report_start`, -`report_success`, `report_failure` and `report_unexpected_exception`. A runner -that owns the loop must probe for it rather than assume it. +`report_success`, `report_failure` and `report_unexpected_exception`. The prompt +lane inherits that surface. The extended runtime does not emulate reporter-hook +events. ### pytest (9.1.1) @@ -905,7 +899,7 @@ that owns the loop must probe for it rather than assume it. | `repr_failure` reads each failure's own `test` — the fact this design is built on | [`doctest.py:317-344`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L317-L344) | | `DoctestItem.setup()` does `self.dtest.globs.update(globs)`, so the mapping must be mutable and survive collection → setup → run | [`doctest.py:288-293`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L288-L293) | | `runtest()` calls `run(self.dtest, out=failures)` with `clear_globs` defaulting to `True` — which would empty a shared mapping after the first block | [`doctest.py:295-303`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L295-L303) | -| `PytestDoctestRunner` is defined *inside* `_init_runner_class()` and is not importable, so the `OutcomeException` re-raise, `BdbQuit` → `outcomes.exit` and `continue_on_failure` handling do **not** come for free | [`doctest.py:178-181`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L178-L181) | +| `PytestDoctestRunner` is defined *inside* `_init_runner_class()` and is not importable, so its outcome and continuation policy must be mapped at the adapter boundary rather than inherited | [`doctest.py:178-181`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L178-L181) | | A page must be a `pytest.Module` with `obj = None` as a **class** attribute, or the `Module` machinery tries to import the `.rst`/`.md` file | [`doctest.py:420-421`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L420-L421) | | Conftest autouse fixtures reach page items through `FixtureManager.pytest_plugin_registered`, **not** through a collector calling `parsefactories` — that call is `DoctestModule`-only, for fixtures defined in the collected `.py` itself | [`fixtures.py`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/fixtures.py), [`doctest.py:556`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L556) | | `_is_doctest` claims any `.txt`/`.rst` **initial path before consulting `--doctest-glob`**, so `pytest docs/page.rst` is claimed by the built-in plugin regardless of glob | [`doctest.py:148-152`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L148-L152) | @@ -960,7 +954,7 @@ an answer; the position taken and its price are recorded. **A vanilla `DocTest` cannot carry the front-end's metadata.** It has exactly `(examples, globs, name, filename, lineno, docstring)`. *Position:* all extension -metadata stays on `ProjectedBlock`, `RunContext` and the result records. Stock +metadata stays on `ProjectedBlock` and the result records. Stock `DocTest` and `Example` objects remain exact compatibility objects, not metadata carriers. *Price:* a consumer holding only the stdlib object sees only stdlib semantics; it must retain the core recipe to inspect groups, profiles or gates. @@ -985,9 +979,8 @@ True ``` A block's execution policy is **uniform across its examples**, so it belongs on -`ProjectedBlock`, not on the examples. The runner establishes an *active -execution request* immediately before delegating to stdlib `run()` and clears it -in a `finally`; `_DocTestRunner__run` reads it. Stock `Example` objects stay +`ProjectedBlock`, not on the examples. The selected execution profile receives a +fresh stock `DocTest` immediately before execution. Stock `Example` objects stay stock, and nothing in the compatibility kernel is subclassed for metadata at all. **Node-id granularity versus shared state.** *Position:* decouple them — N @@ -1024,22 +1017,27 @@ impossible. **Sphinx compatibility versus silent-loss behaviours.** Sphinx silently discards an orphan `testoutput`, silently discards a `testoutput` following a `doctest` block, silently overwrites a duplicate `testoutput`, and silently ignores -`:pyversion:` on a `testcode`. *Position:* keep the behaviour, add a diagnostic -with a stable code for each of the four. *Price:* a page warns under pytest and -is silent under `sphinx-build`; the results still match. - -**Guaranteed cleanup versus Sphinx's setup-failure short-circuit.** When setup +`:pyversion:` on a `testcode`. *Position:* enforce `:pyversion:` consistently on +extended blocks rather than preserve Sphinx's silent loss. The spike still +ignores orphan and misplaced outputs without diagnostics; duplicate outputs use +Sphinx's last-one-wins rule, also without a diagnostic. *Price:* +`testcode :pyversion:` can run differently from `sphinx-build`; diagnostics for +the silent cases remain an acceptance gap. + +**Guaranteed block cleanup versus Sphinx's setup-failure short-circuit.** When setup fails, Sphinx returns before the cleanup phase, skipping page `testcleanup` -blocks and `doctest_global_cleanup` alike. *Position:* run cleanup -unconditionally in a `try`/`finally`, because a page that spawns a server in -setup and fails mid-way should not leak it. *Price:* a page whose setup fails +blocks and `doctest_global_cleanup` alike. *Position:* once profile runtimes have +opened, run cleanup after block and gate failures because a page that spawns a +server in setup and fails mid-way should not leak it. Profile context entry and +exit failures remain open. *Price:* a page whose setup fails leaves different residue under pytest than under `sphinx-build`, and that is a deliberate divergence rather than an oversight. -**Owning the loop versus tracking CPython.** *Position:* own it, because -constraint 4 makes it the only way to control compile mode without a -process-global patch or a code-object clone. *Price:* a version shim and a -conformance harness. See {doc}`0002-runner-conformance-across-cpython`. +**Owning the extended loop versus tracking CPython.** *Position:* inherit the +prompt loop unchanged and own only the bounded extended runtime, because that is +the only way to control compile mode without a process-global patch or a +code-object clone. *Price:* each extended profile needs an explicit semantic +matrix. See {doc}`0002-runner-conformance-across-cpython`. ## What this avoids @@ -1148,24 +1146,31 @@ conformance test in CI. ### Positive - Failure locations are correct by construction, including through `.. include::`; - the default-checker path needs no `repr_failure` or `reportinfo` override. + the adapter's renderer needs no synthetic merged source or `reportinfo` + override. - A block docutils cannot locate degrades to an honest disclaimer instead of a fabricated line, and does not affect its siblings. -- Every `--dist` mode works, because there is no shared state to split. +- Shared groups require no affinity scheduler. The spike exercises xdist's + `load` and `worksteal` modes; other modes retain the same one-item boundary but + remain outside its evidence. - No CPython code-object clone, and no process-global rebinding of anything. -- Collection runs no author-supplied Python, so `--collect-only` has no side - effects and the largest source of worker divergence is removed. +- Collection does not execute collected doctest Python or evaluate gates, so + `--collect-only` removes the largest source of worker divergence. Parser + directives, includes, and plugin registration may still have side effects. - Grouping is one pure function with no docutils, pytest or filesystem dependency, and is testable without any of them. -- A new block kind is a registration, not an edit to a method branching on string - literals. -- Parse diagnostics become values with stable codes rather than stderr writes and - mid-parse aborts. +- A new block kind can provide projection policy and name a custom expected-output + stamp through registration. A new markup spelling also needs a parser or + stamped-node contribution; the first spike proves preservation and pairing of + custom stamps, not directive registration by name alone. +- Parse diagnostics become values rather than stderr writes and mid-parse + aborts. Project-owned codes are stable; docutils-originated classification is + provisional as recorded in {doc}`0004-diagnostics-as-data`. ### Tradeoffs -- The per-example loop is this project's to maintain across supported - interpreters, including two version-shaped divergences. +- The extended per-example loop is this project's to maintain across supported + interpreters. Prompt-form doctests continue to inherit CPython's loop. - A page containing Sphinx `{testcode}` blocks produces `DocTest`s a stock runner cannot run, because `compile("a = 1\nb = 2\n", "", "single")` raises. Prompt-form blocks — the overwhelming majority — run perfectly on an unmodified @@ -1176,20 +1181,21 @@ conformance test in CI. ### Risks -**Runner drift.** A CPython refactor that inlines the loop into `run()` would -silently route execution back to stdlib — invisible for prompt-form blocks, -immediately broken for `{testcode}`. Mitigated by the conformance harness in -{doc}`0002-runner-conformance-across-cpython`, gating on capability probes rather -than a `sys.version_info` ladder. +**Runner drift.** Prompt profiles inherit CPython changes directly. Extended +profiles deliberately reproduce only a bounded subset, so a new doctest behavior +must be considered explicitly rather than assumed. Mitigated by the conformance +matrix in {doc}`0002-runner-conformance-across-cpython` and capability probes for +version-shaped result objects. -**pytest private API.** Four private helpers and a subclassed item. Mitigated by +**pytest private API.** Collector, runner-option, failure and representation +helpers remain private. Mitigated by quarantining them in one module behind a pinned matrix; see {doc}`0006-pytest-private-api-compatibility`. **Foreign directive registration.** `Sphinx.add_directive` overrides existing registrations unconditionally, so `sphinx.ext.doctest` loaded in the same interpreter can replace these directive classes. Mitigated by reading -`BlockAttributes` off the node — byte-compatible with what Sphinx stamps — rather +extractor metadata off the node — compatible with what Sphinx stamps — rather than depending on this project's own classes having run. **Over-suppressed diagnostics.** Suppressing one code too many turns a broken page @@ -1199,8 +1205,8 @@ the narrow default set in {doc}`0004-diagnostics-as-data`. ## Relationship to other ADRs This ADR fixes the architecture. Six decisions it defers get their own records: -{doc}`0002-runner-conformance-across-cpython` (how the owned loop is proven -equivalent), {doc}`0003-rejecting-per-block-items` (why shared per-block items +{doc}`0002-runner-conformance-across-cpython` (the stock prompt lane and bounded +extended-runtime matrix), {doc}`0003-rejecting-per-block-items` (why shared per-block items are rejected), {doc}`0004-diagnostics-as-data` (what is reported and what is suppressed), {doc}`0005-line-recovery-for-nested-blocks` (the optional last step), {doc}`0006-pytest-private-api-compatibility` (the quarantine and its @@ -1213,10 +1219,12 @@ points). The core produces real {class}`doctest.DocTest` objects holding real {class}`doctest.Example` objects. `Example.source` is the stdlib-normalized executable body — prompts and indentation stripped, trailing newline added, the -stripped column recorded in `Example.indent` — not a synthesized wrapper and not -the author's verbatim text, which is what `ParsedBlock.source` holds. Everything -else — groups, phases, pairing, diagnostics, distribution — is a layer above that -fact, and no layer reaches around another. +stripped column recorded in `Example.indent` — not a synthesized wrapper. +`ParsedBlock.source` is the dedented, outer-newline-normalized body extracted +from markup. Prompt projection applies stdlib normalization to it; the exec lane +uses it as one indent-zero recipe. Everything else — groups, phases, pairing, +diagnostics, distribution — is a layer above that fact, and no layer reaches +around another. The unit that shares a `globs` mapping is the unit pytest schedules. That is the one invariant every other property in this document follows from, and it is not diff --git a/docs/adrs/0002-runner-conformance-across-cpython.md b/docs/adrs/0002-runner-conformance-across-cpython.md index afd1889..d699c7d 100644 --- a/docs/adrs/0002-runner-conformance-across-cpython.md +++ b/docs/adrs/0002-runner-conformance-across-cpython.md @@ -7,106 +7,141 @@ Date: 2026-08-02 ## Context -{doc}`0001-typed-vanilla-doctest-core` decides that the runner owns the -per-example loop by defining `_DocTestRunner__run` in a subclass, rather than -cloning CPython's code object or rebinding `doctest.compile` process-wide. - -Owning the loop means owning the private state it writes into, and that state has -changed shape inside this project's supported interpreter range. Three -divergences are known: - -**The outcome accumulator changed name and arity.** On 3.10 through 3.12 it is -`__record_outcome(self, test, f, t)` writing into `self._name2ft`; on 3.13 and -later it is `__record_outcome(self, test, failures, tries, skips)` writing into -`self._stats` -([`Lib/doctest.py:1485`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1485)). -A loop that calls the wrong one leaves `summarize()` reporting zeros for a -passing file — a silent, total failure of the reporting path. - -**`TestResults` gained a third value that is not a tuple field.** It carries -`skipped` as an extra instance attribute -([`Lib/doctest.py:114`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L114)), -so `TestResults(f, a, skipped=s)` works on 3.13+ and raises on earlier versions. - -**`report_skip` does not exist at v3.14.2.** The runner has only `report_start`, -`report_success`, `report_failure` and `report_unexpected_exception` -([`Lib/doctest.py:1286-1314`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1286-L1314)). -It appears in later prereleases, so a loop must probe rather than assume in -either direction. - -A fourth risk has no current instance but would be silent: a CPython refactor -that inlines the loop into `run()` would route execution back to stdlib. That is -invisible for prompt-form blocks and immediately broken for `{testcode}`. - -## Question - -How is an owned per-example loop proven equivalent to the interpreter's own, -continuously, without a `sys.version_info` ladder? - -## Direction - -A differential conformance harness, run in CI on every supported interpreter, -gating the build step that lands the runner. - -**Scoped to the extended lane.** {doc}`0001-typed-vanilla-doctest-core` runs -ordinary prompt blocks on CPython's untouched per-example loop, so those need no -differential proof — they *are* the reference. The owned `__run` is invoked only -for `exec` bodies, top-level await and future profiles, and that is what this -harness guards. It is a smaller obligation than an unconditionally owned loop, -and it is the reason owning the loop is affordable at all. - -A fixed case matrix — pass, fail, unexpected exception, `SyntaxError`, all -examples skipped, partially skipped, `FAIL_FAST`, `REPORT_ONLY_FIRST_FAILURE`, -`IGNORE_EXCEPTION_DETAIL`, and an exec-mode body — is run through both this -runner and a stock {class}`doctest.DocTestRunner`, asserting the captured -`report_*` text, `summarize()` output, the accumulator contents, and the result -as `(failed, attempted, skipped)`. - -**Assert the triple, not `TestResults` equality.** `TestResults` is a two-field -namedtuple carrying `skipped` off-tuple, so `==` compares only two of the three -values and a skip-count regression passes silently. `attempted` is also -incremented *before* the `SKIP` check, so a skip that wrongly executes moves -neither counter — it is invisible to both the tuple and to `summarize()` at zero -failures, and only the `report_*` text distinguishes it. - -The exec-mode case is the one the two runners are *meant* to disagree on, and it -still compares against stock. `compile()` raises on a multi-statement body, but -that call sits inside the loop's own `try` -([`Lib/doctest.py:1398-1408`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1398-L1408)), -so a stock {class}`doctest.DocTestRunner` catches the `SyntaxError` and records -it as an unexpected exception rather than propagating it: one -`report_unexpected_exception` call, `TestResults(failed=1, attempted=1)`, and -`_stats` at `(1, 1, 0)`. Only {class}`doctest.DebugRunner` — and pytest's runner -beneath it — converts that into a raise, as -{exc}`doctest.UnexpectedException`. So the case is asserted as a pair: stock -records the failure, this runner records a pass. A regression that silently -reverts to `"single"` mode shows up as the two converging. - -**What else belongs in the matrix, and what does not.** Add `report_*` hook -events — the only channel that distinguishes a skip which wrongly *executed*, -since `attempted` increments before the `SKIP` check and neither counter moves — -and repeated runs of one test, which exercise accumulator arithmetic across -calls. - -Cross-block `FAIL_FAST` and cleanup aggregation stay out. Both are properties of -`run_group()` rather than of the per-example loop, so a stock runner offers -nothing to compare them against; they belong to -{doc}`0001-typed-vanilla-doctest-core`'s item-lifecycle tests. A -{exc}`pytest.skip` raised inside an example and a debugger exit are likewise -pytest-layer concerns, testable only through a pytest session. - -Version handling is by capability probe, never by version comparison, so a -backport, a vendored interpreter or a fork behaves correctly rather than by -coincidence. {doc}`0001-typed-vanilla-doctest-core` rejects an import-time guard -that raises: a `pytest11` plugin that aborts at import takes down suites whose -majority of tests never touch a doctest. +{doc}`0001-typed-vanilla-doctest-core` has two execution lanes with different +compatibility claims. + +Prompt-form blocks are ordinary {class}`doctest.DocTest` objects executed by +CPython's own {class}`doctest.DocTestRunner` loop. The core subclasses only the +reporting hooks that retain failures for an embedding host. It does not override +`run()` or `_DocTestRunner__run`. + +Extended blocks such as Sphinx `testcode` cannot use that loop unchanged. CPython +compiles each example in `"single"` mode, while a `testcode` body may contain +several statements and requires `"exec"`. There is no stdlib execution mode to +select and therefore no exact-compatibility claim to make. + +The supported interpreters also expose different result semantics. Python 3.10 +increments `tries` only after an example passes its `SKIP` gate +([`Lib/doctest.py:1326-1337`](https://github.com/python/cpython/blob/v3.10.19/Lib/doctest.py#L1326-L1337)). +Python 3.14 increments `attempted` before the gate and records `skips` separately +([`Lib/doctest.py:1353-1379`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L1353-L1379)). +`TestResults` gained its `skipped` attribute with that newer shape +([`Lib/doctest.py:114-126`](https://github.com/python/cpython/blob/v3.14.2/Lib/doctest.py#L114-L126)). +The prompt lane must expose skipped examples without silently rewriting the +interpreter's own `attempted` value. The extended lane has no CPython count to +inherit and defines its own stable count below. + +## Decision + +Keep the two lanes structurally separate. + +The prompt runtime delegates to CPython's untouched per-example loop with +`clear_globs=False`. Its reporter subclass may retain +{class}`doctest.DocTestFailure` and {class}`doctest.UnexpectedException`, and may +propagate exceptions selected by the host's `ExceptionPolicy`; it does not own +compilation, option merging, comparison, debugger setup, display hooks, linecache +patching, or result accounting. + +An extended execution profile owns an independent, deliberately smaller runtime. +It accepts a fresh stock `DocTest` plus resolved `RuntimeSettings` and returns a +`RuntimeOutcome`. The initial `exec` runtime owns these semantics: + +- merge runner flags with per-example options, then honor `SKIP` and fail-fast; +- derive active future flags from the live `globs`, compile in `"exec"` mode, + and pass `dont_inherit=True` so the core module's future imports cannot leak; +- capture and restore stdout around every example; +- compare expected exceptions against the exception-only tail, including + `SyntaxError` normalization and `IGNORE_EXCEPTION_DETAIL`, while retaining + captured stdout for failure rendering; +- use the injected checker for comparison and retain stock failure objects; +- propagate host-owned outcomes through `ExceptionPolicy`; and +- leave group phase ordering, cleanup, and exception precedence to + `run_group()`. + +It does not update a `DocTestRunner` accumulator, call `report_*`, implement +`summarize()`, patch the debugger, or claim byte-for-byte output parity with the +prompt lane. A direct stdlib-shaped facade may translate `GroupResult` into the +version-specific accumulator needed by `summarize()`; that compatibility shim is +separate from execution. + +## Conformance gate + +The prompt lane is compatible by construction, but still runs on every supported +Python to catch subclass-state collisions and changes to reporter signatures. +Its tests assert stock object types, per-example option merging, fail-fast, +partial skips, repeated fresh materialization, and restoration of the shared +mapping contract. + +The extended lane has a behavioral matrix rather than a comparison against +CPython's `"single"` compiler mode. Before it is accepted, the matrix covers: + +| Behavior | Required assertion | +|---|---| +| pass and mismatch | stock failure objects, counts, and checker identity | +| future flags | no ambient inheritance; an explicitly imported feature persists through group `globs` | +| unexpected exception and `SyntaxError` | stock exception shape and stable traceback ownership | +| output before exception | defined capture and rendering behavior | +| all and partial skip | examples examined, including skips, plus an explicit skipped count on every interpreter | +| fail-fast and report-only-first | execution and reporting policies remain distinct | +| checker options | `IGNORE_EXCEPTION_DETAIL` and contributed checker behavior | +| process state | stdout is restored; debugger, display-hook, and linecache support is explicitly accepted or excluded | +| repeated calls | runtime-local state cannot leak between attempts | + +Group cleanup after failure, pytest outcomes, fixture injection, reruns, and xdist +belong to host and `run_group()` acceptance tests. They are not evidence about an +individual execution profile. + +Version handling uses capability probes, not `sys.version_info`. The prompt +runtime preserves CPython's own `attempted` value. The extended runtime counts +each example it examines, including an example skipped before compilation; on +interpreters whose `TestResults` cannot carry `skipped`, `run_group()` reconstructs +that value from the materialized test and stores it in `Counts`. + +## Alternative rejected + +Defining `_DocTestRunner__run` for extended profiles was rejected by the +implementation bakeoff. It couples a small `"exec"` requirement to private +accumulators, private outcome-recording arity, report-hook sequencing, debugger +machinery, and `summarize()` behavior that the host-neutral runtime does not use. +It is more code and a larger compatibility promise without making extended +syntax vanilla. + +Cloning and patching CPython's code object or rebinding `doctest.compile` +process-wide remain rejected. Both make unrelated doctest execution depend on +global mutable state. + +The spike still has two smaller CPython-private parser dependencies: +`DocTestParser._EXAMPLE_RE` recognizes prompt-form literal blocks and +`DocTestParser._EXCEPTION_RE` extracts the expected exception tail from paired +output. They do not couple execution to private runner state, but they are still +compatibility debt and need explicit probes across the supported Python matrix. +The legacy direct facade's use of `doctest._load_testfile` is outside the typed +core but belongs in the facade's own compatibility inventory. + +## Consequences + +- Ordinary doctests inherit CPython behavior directly rather than through a + differential approximation. +- Extended profiles state their semantic subset and can manage attempt-scoped + resources through their context manager. +- CPython's pre-3.13 and current prompt-lane skip counters remain observable; + extended profiles expose their separate version-independent count through + `Counts`. +- The direct compatibility facade needs a small version-shaped statistics shim + if it promises stdlib `summarize()` and `master.merge()` behavior. +- The direct facade cannot reproduce the complete verbose + `Trying`/`Expecting`/`ok` stream from `GroupResult`, because the core retains + failures but not successful per-example reporter events. Failure and summary + rendering remain stock-shaped. +- Each new execution profile owns its own behavioral matrix; adding async does + not expand the prompt lane's maintenance surface. ## Open -- Whether the harness asserts on `report_*` text verbatim, or on a normalized - form — verbatim is stricter and will churn when CPython adjusts wording. -- Whether a probe failure degrades to stdlib's loop with a diagnostic, or fails - the affected items loudly. Degrading is silent for prompt-form blocks, which is - the argument against it. -- The floor: whether supporting 3.10's `_name2ft` shape is worth its shim once - that version reaches end of life. +- Complete the extended matrix for report-only-first and repeated runtime calls. +- Probe the two private parser regex contracts on every supported Python. +- Decide whether extended runtimes should reproduce doctest's debugger, + display-hook, and linecache behavior or explicitly exclude interactive + debugging. +- Define how a profile context-manager entry or exit failure is represented while + still allowing an already-open cleanup profile to run. diff --git a/docs/adrs/0004-diagnostics-as-data.md b/docs/adrs/0004-diagnostics-as-data.md index 45ed820..173bc03 100644 --- a/docs/adrs/0004-diagnostics-as-data.md +++ b/docs/adrs/0004-diagnostics-as-data.md @@ -77,17 +77,35 @@ code to a docutils message is unsettled, which is why this record stays `Draft`. Every diagnostic raised by this project's own layers defaults to visible, and `level="error"` from those layers fails collection with the file and line named. -A page whose only block fails to parse, and a page with a malformed `:options:` -value, must both produce a collection error rather than collecting nothing and -passing. +A page whose only block fails to parse must produce a collection error rather +than collecting nothing and passing. A malformed `:options:` value follows +Sphinx's warning severity, but that warning must be visible rather than silently +discarded by the host. Expose promotion and suppression by code so a project can tune the set without a global on/off switch. +## Spike result + +The spike captures reporter messages as typed values, deduplicates messages seen +through both the observer and the doctree, and suppresses both messages emitted +for an unknown role. It provisionally classifies docutils messages by normalized +message substrings because docutils supplies no stable code. Unknown directives +remain visible in `ParseResult.diagnostics`. + +That proves capture and normalization, not host disposition. The pytest adapter +does not yet fail collection for an unsuppressed error or surface warnings with +source attribution. Until that policy and its reST/MyST wording matrix exist, a +malformed body-owning directive can still collect no tests without failing the +session, and malformed doctest options can warn only inside the retained parse +result. The direct facade also does not render those diagnostics. This record +therefore remains `Draft`. + ## Open - Whether diagnostics surface as {class}`pytest.PytestWarning` subclasses, giving - `-W error::` control for free, or as a dedicated report section. + `-W error::` control for free, or as collection errors and dedicated report + sections. Errors must not be silently ignored by the host. - **What classifies a code-less docutils message.** The options are an owned, version-pinned message-text table with a test that fails on upstream rewording (and which must handle two dialects — reST's `Unknown directive type "x".` at diff --git a/docs/adrs/0005-line-recovery-for-nested-blocks.md b/docs/adrs/0005-line-recovery-for-nested-blocks.md index 784f7c7..6397d5e 100644 --- a/docs/adrs/0005-line-recovery-for-nested-blocks.md +++ b/docs/adrs/0005-line-recovery-for-nested-blocks.md @@ -10,8 +10,8 @@ Date: 2026-08-02 docutils does not report a usable line for every node, and what it reports differs by front-end and by version. -At **docutils 0.21.2** — which this project does not pin but does resolve, via -its Sphinx and myst-parser constraints — a bare `>>>` block +At **docutils 0.21.2** — the newest line convention in this project's current +`docutils >= 0.20.1, < 0.22` range — a bare `>>>` block nested in a `.. note::`, a list item, a block quote or a `{tab}` directive reports `line=None, source=None`. A top-level reStructuredText `doctest_block` reports its **last** line. A MyST fence reports its **first** line. An @@ -79,7 +79,10 @@ Resolved versions per interpreter, with `docutils >= 0.22` requested: | 3.12–3.14 | 0.22.4 | 5.1.0 | 9.1.0 | Today's lock resolves Sphinx 8.1.3 on Python 3.10 and 8.2.3 elsewhere, and -neither permits docutils 0.22. +neither permits docutils 0.22. The package therefore caps docutils below 0.22 +until those support-policy steps can move together; silently accepting 0.22 +would apply the old last-line correction to its already-correct first-line +nodes. ## Consequences @@ -93,6 +96,12 @@ both reStructuredText and MyST report the first line — but the normalization layer stays, because the conventions still differ below the floor and a front-end is the right place to know which it is dealing with. +The standalone MyST parser recovers root-document body lines by scanning the +root source text. It deliberately does not apply that stamp to nodes whose +physical source is an included file: the root bytes cannot prove an included +line. Exact standalone MyST include-line fidelity remains open unless the parser +retains the included source text or supplies an absolute body line itself. + Every line-convention claim elsewhere in these records is version-qualified. A statement about "docutils" that does not name a version is a bug in the statement. diff --git a/docs/adrs/0006-pytest-private-api-compatibility.md b/docs/adrs/0006-pytest-private-api-compatibility.md index 175bba4..5aeb7d0 100644 --- a/docs/adrs/0006-pytest-private-api-compatibility.md +++ b/docs/adrs/0006-pytest-private-api-compatibility.md @@ -52,22 +52,29 @@ or already reported a collection error, which deselection cannot retract. What private surface is depended on, and how does a pytest release that changes it fail? -The current surface is `_get_checker`, `get_optionflags`, -`_get_continue_on_failure`, `_get_report_choice` and `MultipleDoctestFailures`. -Not everything in the quarantine is equally risky: {class}`pytest.DoctestItem` is -**public** — exported from `pytest` — so subclassing it is ordinary API use. The -collector class filtered out of the multicall result is private, and that filter -is the part that needs a version matrix. `_init_runner_class` is explicitly *not* usable: +The spike's private surface is `_get_checker`, `get_optionflags`, +`_get_continue_on_failure`, `_get_report_choice`, `DoctestTextfile`, +`MultipleDoctestFailures`, `ReprFailDoctest`, `_pytest._code` representation +classes, and the Darwin capture method on the public item. Not everything in the +quarantine is equally risky: +{class}`pytest.DoctestItem` is **public** from pytest 7.2 onward, so subclassing +it is ordinary API use and establishes the adapter's minimum pytest. The +collector class filtered out of the +multicall result and the representation helpers are private and need a version +matrix. `_init_runner_class` is explicitly not used: `PytestDoctestRunner` is defined inside it ([`_pytest/doctest.py:178-181`](https://github.com/pytest-dev/pytest/blob/9.1.1/src/_pytest/doctest.py#L178-L181)) -and is unreachable by name, which is why the `OutcomeException` re-raise, -`BdbQuit` → `outcomes.exit` and `continue_on_failure` handling must be -reimplemented rather than inherited. +and is unreachable by name. The carrier runner never executes; the adapter maps +the core's result records and host exception policy directly, while +`continue_on_failure` is read through the quarantined helper. ## Direction -Quarantine every private import in one module, `pytest_doctest_docutils._compat`, -with a pinned support matrix. +Quarantine every private import in one adapter-owned module with a pinned +support matrix. The clean-slate package spelling is +`pytest_doctest_docutils._compat`. The spike retains the released flat facade and +therefore uses the top-level `_pytest_doctest_compat`; that is an implementation +compromise, not the preferred namespace. **Filter the built-in's collector out of the `pytest_collect_file` result, in a hook wrapper.** The directory collector consumes the multicall result directly, @@ -82,11 +89,11 @@ under pytest 7 with a new enough pluggy. But pytest 7 declares only `pluggy>=0.12,<2.0`, so a resolver may legally install pluggy 1.0 or 1.1, where `wrapper=True` raises `TypeError` *while importing the plugin* — a session-wide abort, which this record and {doc}`0001-typed-vanilla-doctest-core` both forbid. -Old style needs no floor at all and was verified working on both pytest 7 and 9. +Old style adds no pluggy floor and was verified from pytest 7.2 through 9. -Whichever spelling is used, **name the minimum supported pytest**. The CI matrix -floor is 7 and the package declares no pytest dependency, so today the support -statement exists only in the workflow file. +Whichever spelling is used, **name the minimum supported pytest**. The package +declares `pytest>=7.2`, the first release exporting `pytest.DoctestItem`, and the +CI matrix pins that exact floor. **Fail on an unsupported pytest only when an affected document is collected**, not at plugin registration. A `pytest11` plugin that raises at import takes down @@ -96,23 +103,38 @@ sessions whose majority of tests never touch a doctest, and failures for the same reason. The error names the pytest version and the missing symbol, and it names the document that triggered it. -CI carries a job pinned to the minimum supported pytest and one tracking its -prerelease. +The acceptance matrix must carry a job pinned to the minimum supported pytest +and one tracking its prerelease. The spike implements the floor job; the +prerelease probe remains open. Registry construction is not a pytest-private-API concern. The host-neutral -contract, pytest hookspec, Sphinx adapter and xdist manifest are specified in +contract and host lifecycle proposals are specified in {doc}`0007-host-plugin-registration-lifecycle`. +## Spike result + +The old-style collection wrapper works across pytest 7.2, 8.4 and 9.1 and removes only +the built-in collector for documents this plugin claims. The built-in plugin is +required: collecting an affected documentation file, or a Python module through +this adapter's doctest-module mode, raises an actionable usage error when it has +been disabled. Fixture injection and pytest's checker/report options continue to +come from the built-in plugin. The adapter limits its claim to suffixes in the +frozen document-parser registry, so a separate `--doctest-glob=*.foo` remains +owned by pytest's text collector unless a contributor actually registers a +`.foo` parser. + +The quarantine is effective as an import boundary, but its symbol binding is +still eager. A supported or newer pytest release missing one of those private +names would fail while the plugin imports rather than when an affected document +is collected. Pytest below the declared 7.2 floor may likewise fail at the public +base-class import. The CI matrix covers released pytest 7.2, 8.4 and 9.1; it does +not yet include a prerelease probe. Those are remaining acceptance gaps, so this +record stays `Draft`. + ## Open -- Whether requiring the built-in plugin should be stated as a hard dependency. - `-p no:doctest` already fails today with a raw `ValueError` about an unknown - option, so this is not a regression — but the message should become actionable. - Whether the probe should accept a *newer* pytest it has not been tested against, or refuse it. Refusing is safer and more annoying; for a private-API quarantine with a small matrix, safer probably wins. -- What the filtering wrapper should do when the built-in's collector is the *only* - one for a path — that is the ordinary `--doctest-glob` case this plugin has no - business touching, so the filter must be scoped to paths it actually claims. - Whether any of these helpers can be promoted upstream, which would delete the quarantine entirely. diff --git a/docs/adrs/0007-host-plugin-registration-lifecycle.md b/docs/adrs/0007-host-plugin-registration-lifecycle.md index be3b515..122f02e 100644 --- a/docs/adrs/0007-host-plugin-registration-lifecycle.md +++ b/docs/adrs/0007-host-plugin-registration-lifecycle.md @@ -34,7 +34,8 @@ class Provider(t.NamedTuple): version: str | None -class Registration(t.NamedTuple, t.Generic[T]): +@dataclasses.dataclass(frozen=True, slots=True) +class Registration(t.Generic[T]): name: str value: T provider: Provider @@ -83,6 +84,11 @@ bound to that contributor's `Provider`; registrations cannot claim a different origin. A contributor retaining that registrar cannot retain a mutation path: every method raises `RegistryClosedError` after the snapshot is made. +`Registration` is a frozen, slotted dataclass rather than a generic +`NamedTuple`. The latter declaration fails while importing on Python 3.10, which +is inside the package's support range; immutability is the contract, not the +tuple representation. + ### Names, collisions and order Registration names are case-sensitive ASCII identifiers matching @@ -102,6 +108,11 @@ accepted only when the challenger uses the incumbent parser's name and passes `replace=True`; two differently named parsers cannot both win `.md` by incidental plugin load order. +Freeze also validates cross-references. Every block kind must name an existing +execution profile; a non-`None` expected-output kind must follow the registry +name grammar and cannot also be a runnable block kind. Errors identify the block +kind and provider before parsing begins. + ## Host adapters ### Direct API @@ -134,9 +145,24 @@ Nested conftests load during collection and are outside this lifecycle. A and raises `pytest.UsageError` naming that plugin and the closed registration phase. Fixtures and unrelated hooks in nested conftests remain valid. -### Sphinx +### Spike boundary + +The direct and pytest paths above are implemented. The pytest snapshot is frozen +once, custom checker contribution is exercised end to end with one +comparison-and-rendering instance, a custom block can pair with a custom output +stamp, and a late nested conftest contributor fails with an actionable usage +error. Low-level parse, extract, project, and run functions retain a convenience +`registry=None` default; registry identity across stages is guaranteed only when +a caller passes the same snapshot, as both host adapters do. + +The Sphinx contributor lifecycle and xdist manifest below were not needed to +test the core boundary and are deferred until an external contributor requires +them. The spike proves Sphinx-resolved doctree extraction and homogeneous xdist +execution, not these two bootstrap protocols. + +### Proposed Sphinx lifecycle -The Sphinx adapter exposes +The proposed Sphinx adapter exposes `add_doctest_core_contributor(app, contributor)`. Extensions call it from their `setup(app)` function. At `config-inited`, after extension setup and before any document is read, the adapter emits a `doctest-core-contributors` event, appends @@ -152,24 +178,24 @@ extension name. This lifecycle makes the extractor usable on Sphinx-resolved doctrees. It does not add a builder or claim parity with `sphinx-build -b doctest` execution. -## xdist consistency +## Proposed xdist consistency -The controller sends a JSON-safe manifest through `workerinput` from +The controller would send a JSON-safe manifest through `workerinput` from `pytest_configure_node`. Each worker builds its own snapshot during `pytest_configure` and compares before collection. The manifest has a schema version and contains: -- a JSON-safe projection of normalized `SessionSettings` +- JSON-safe projections of normalized parse, projection, and run settings - every registry category, name, provider and provider version in declared order - `doctest.OPTIONFLAGS_BY_NAME`, sorted by flag name -A mismatch aborts the session with the controller and worker manifests. This is +A mismatch would abort the session with the controller and worker manifests. This is an extension-set consistency check, not proof that two workers are semantically identical. Equal provider names and versions do not prove equal source code, and the manifest does not hash included documents, directive implementations or MyST plugins. -Version 1 therefore supports homogeneous worker environments. Equal source +The initial contract therefore supports homogeneous worker environments. Equal source closure and equal installed provider code are preconditions, while xdist's own identical-collection check remains authoritative for node ids. Stronger support for deliberately heterogeneous SSH or socket workers would require content or @@ -180,11 +206,13 @@ environment attestation and is deferred. - Core extension authors implement one `Contributor` regardless of host. - Settings remain serializable inputs; discovered objects remain in the registry. - Parse and execution code cannot mutate capabilities after collection starts. -- pytest and Sphinx own timing and diagnostics in their native idioms without - leaking their lifecycle types into the core. +- pytest owns registration timing in its native idioms without leaking lifecycle + types into the core. Sphinx can adopt the same contract when its lifecycle is + implemented. - Replacement is possible but visible, attributed and deterministic. -- Supporting heterogeneous xdist workers is explicitly outside the first - contract rather than implied by a weak manifest. +- Supporting heterogeneous xdist workers remains outside the first contract; + the proposed manifest would diagnose capability mismatches without pretending + to attest worker code or source closures. ## Open diff --git a/docs/modules/doctest_docutils/how-to.md b/docs/modules/doctest_docutils/how-to.md index da5be6f..fbbbfbf 100644 --- a/docs/modules/doctest_docutils/how-to.md +++ b/docs/modules/doctest_docutils/how-to.md @@ -18,9 +18,9 @@ Use the same command for `.rst` files: $ python -m doctest_docutils README.rst ``` -## See collected examples +## See the run summary -Pass `-v` for verbose standard-library doctest output: +Pass `-v` to list each tested group in the final summary: ```console $ python -m doctest_docutils README.md -v diff --git a/docs/modules/doctest_docutils/index.md b/docs/modules/doctest_docutils/index.md index e63866e..70d8f48 100644 --- a/docs/modules/doctest_docutils/index.md +++ b/docs/modules/doctest_docutils/index.md @@ -23,7 +23,7 @@ Run your first documentation doctest from a Markdown page. :::{grid-item-card} How-to :link: how-to :link-type: doc -Choose files, run verbose output, and map the command to stdlib doctest. +Choose files, inspect run summaries, and map the command to stdlib doctest. ::: :::{grid-item-card} Examples @@ -35,7 +35,7 @@ See the supported Markdown and reStructuredText example shapes. :::{grid-item-card} API Reference :link: reference :link-type: doc -Inspect finder, runner, directive, and CLI APIs. +Inspect finder, directive, and CLI APIs. ::: :::: @@ -48,8 +48,7 @@ Run a Markdown page: $ python -m doctest_docutils README.md ``` -No output means the examples passed. Add `-v` when you want the standard -doctest transcript. +No output means the examples passed. Add `-v` for a final group summary. ```{toctree} :hidden: diff --git a/docs/modules/pytest_doctest_docutils/how-to.md b/docs/modules/pytest_doctest_docutils/how-to.md index aaeef3c..6e2fe75 100644 --- a/docs/modules/pytest_doctest_docutils/how-to.md +++ b/docs/modules/pytest_doctest_docutils/how-to.md @@ -53,13 +53,13 @@ parses in `.rst`, `.md`, and Python-module doctests. The standalone `python -m doctest_docutils` command does not register it, so use the marker when you run examples through pytest. -## Keep pytest's built-in doctest plugin disabled +## Keep pytest's built-in doctest plugin enabled -The gp-libs plugin blocks pytest's built-in doctest plugin by default. Keep -`-p no:doctest` in local examples when you are demonstrating explicit pytest -configuration: +This plugin composes with pytest's built-in doctest plugin. The built-in plugin +supplies `doctest_namespace`, output-checker extensions, report options, and +Python-module collection, while gp-libs owns the documentation-file collector. -```ini -[pytest] -addopts = -p no:doctest -``` +Do not pass `-p no:doctest` when collecting documentation through gp-libs. If +the built-in plugin is disabled, collecting an affected documentation file or +using `--doctest-docutils-modules` raises a usage error instead of silently +collecting no tests. diff --git a/docs/modules/pytest_doctest_docutils/index.md b/docs/modules/pytest_doctest_docutils/index.md index 309c68d..53705dd 100644 --- a/docs/modules/pytest_doctest_docutils/index.md +++ b/docs/modules/pytest_doctest_docutils/index.md @@ -8,8 +8,10 @@ parses each page through {ref}`doctest_docutils` before pytest runs the examples. -The plugin blocks {ref}`pytest's standard doctest plugin ` by -default so the same examples are not collected twice. +The plugin composes with {ref}`pytest's standard doctest plugin `. +gp-libs owns matching documentation paths and filters pytest's duplicate text +collector before it parses the page; pytest continues to supply fixtures, +checker and report options, and Python-module doctest collection. ::::{grid} 1 1 2 2 :gutter: 2 2 3 3 diff --git a/docs/modules/pytest_doctest_docutils/tutorial.md b/docs/modules/pytest_doctest_docutils/tutorial.md index 3018fcc..8e6579c 100644 --- a/docs/modules/pytest_doctest_docutils/tutorial.md +++ b/docs/modules/pytest_doctest_docutils/tutorial.md @@ -16,8 +16,10 @@ $ pytest docs/ ``` {mod}`pytest_doctest_docutils` parses each matching documentation file with -{mod}`doctest_docutils`, then reports each collected doctest as a pytest item. -That gives documentation examples the same pass/fail surface as the rest of -your suite. +{mod}`doctest_docutils`, then reports each projected shared-state group as one +pytest item. Bare prompt blocks without a group stamp are isolated by default; +unargumented `doctest` directives join Sphinx's `default` group. Named blocks in +the same group share fixtures and Python globals, and execute together as one +schedulable item. [pytest]: https://docs.pytest.org/en/stable/ diff --git a/notes/analyses/24-implementation-bakeoff.md b/notes/analyses/24-implementation-bakeoff.md new file mode 100644 index 0000000..451fd5e --- /dev/null +++ b/notes/analyses/24-implementation-bakeoff.md @@ -0,0 +1,178 @@ +# Typed doctest core implementation bakeoff + +## Question + +Can the architecture in ADR 0001 be implemented as a typed, host-neutral core +while retaining stock doctest objects and composing with pytest, docutils, MyST, +Sphinx-resolved doctrees, reruns, and xdist? + +## Candidates + +### Extend the existing modules + +Keeping extraction, grouping, execution, pytest collection, and reporting in the +two existing modules minimizes import changes. It also preserves the current +coupling: projection cannot be tested without docutils, pytest policy leaks into +execution, and mutable `DocTest` instances are likely to survive across reruns. +This shape was rejected. + +### Typed core with compatibility adapters + +The successful shape is a new `doctest_core` package with thin direct and pytest +adapters: + +```text +contributors -> frozen registry + | +text/doctree -> extraction -> projection -> group runner + | | | + inert records recipes fresh DocTests + | + direct / pytest adapters +``` + +Extraction returns inert typed records and diagnostics. Projection is pure and +owns grouping, wildcard expansion, phase ordering, pairing, and names. The group +runner materializes fresh stock `doctest.Example` and `doctest.DocTest` objects +for every attempt and keeps the shared mapping inside one scheduled item. Host +adapters own collection, fixtures, exception policy, and presentation. + +This shape was selected. It preserves the one invariant that mattered under +reruns and xdist: the unit sharing mutable globals is also the unit the host +schedules. + +### Replace doctest semantics wholesale + +Owning parsing, examples, comparison, and reporting would make every extension +easy to express, but would discard the compatibility goal. It would also require +reimplementing pytest's checker extensions and CPython's process-state behavior. +The bakeoff found no benefit that justified that compatibility surface. + +## What implementation changed in the ADRs + +The ordinary prompt lane should delegate to CPython's untouched runner. An +extended `exec` lane should be a separate, bounded runtime. Rebinding +`doctest.compile`, cloning CPython's code object, or overriding its private loop +all attach extended syntax to global or private behavior that the core does not +otherwise need. ADR 0002 now records the two-lane contract. + +Practice also required contracts absent from the original data model: + +- `ExceptionPolicy` lets a host distinguish ordinary exceptions, host outcomes, + and aborts that must outrank prior failures without importing pytest into the + core. +- `Failed` retains the exact checker that compared output so a contributed + checker also explains its own failure. +- `Registration` is a frozen generic dataclass. A generic `NamedTuple` fails at + import on Python 3.10. +- A proposed `BlockAttributes` `TypedDict` was rejected as false precision over + third-party node stamps. Field-level validation narrows into `ParsedBlock` and + `ParsedOutput`, which are the first owned schema. +- Gates execute inside the group failure boundary, and cleanup runs after setup, + test, and gate failures. +- Extended compilation uses `dont_inherit=True` and only future flags explicitly + present in the live group mapping. +- An inline doctest `FAIL_FAST` flag stops the current runtime's example loop; + a runner-level flag also stops later group blocks despite the host's continue + policy. +- The core defaults unlabelled blocks to Sphinx's `default` group, while the + pytest adapter preserves gp-libs' released per-block isolation default. +- The core's failure-continuation default follows doctest and Sphinx; direct and + pytest hosts override it only for explicit fail-fast or debugger policy. +- The pytest adapter composes with the built-in doctest plugin and filters only + its duplicate documentation collector. It no longer unregisters the plugin + whose fixture, checker, options, and rendering it uses. +- The distribution declares its actual pytest 7.2 floor and direct `packaging` + dependency, and uses `pytest_doctest_docutils` as the pytest entry-point name + so the standard `-p no:pytest_doctest_docutils` spelling works. +- Sphinx compatibility is extractor compatibility over resolved doctrees, not + byte-identical directive stamps. +- Expected-output records retain their stamp name. A custom `pairs_with` + relationship therefore works through extraction and projection rather than + being nominal registry metadata. +- Freeze validates profile and expected-output references before parsing, and + anonymous group identities cannot collide with an author-written `block-N` + group. +- Prompt-free `doctest` directives project no group and produce no passing + carrier item. Collector filtering is limited to registered parser suffixes, + leaving unrelated `--doctest-glob` paths to pytest. + +## Evidence + +| Boundary | Result | +|---|---| +| Full repository suite on Python 3.14 and pytest 9 | 227 passed | +| Full repository suite on Python 3.12 and pytest 8.4 | 227 passed | +| Python 3.10, docutils 0.20.1, and pytest 7.2 floor suite | 224 passed, 3 skipped | +| Rerun isolation | a failed first attempt cannot pass from retained globals | +| xdist | stateful groups pass under `load` and `worksteal` without affinity | +| Sphinx | resolved doctrees retain hidden setup/cleanup nodes and include attribution | +| Extension seam | a contributed checker compares and renders with the same instance | +| pytest-asyncio | a 1.x async autouse fixture populates the doctest namespace | +| Packaging | the core package and `py.typed` are present in wheel-from-sdist validation | + +The xdist result proves the item boundary under two schedulers. It does not prove +heterogeneous workers or every distribution mode. The Sphinx result proves +extraction from its resolved tree, not a Sphinx execution lifecycle. + +## ADR shortcomings exposed by the bakeoff + +The architecture is usable, but these claims remain incomplete: + +- The diagnostics core captures, deduplicates, and suppresses known noise, but + the pytest adapter does not yet fail unsuppressed errors or render warnings. + Message-substring classification is provisional because docutils supplies no + stable diagnostic codes. The direct facade also drops the channel, so a + malformed option's Sphinx-compatible warning is not yet user-visible. +- Partial block skips remain worker-local in `GroupResult`; there is no versioned + JSON-safe pytest report projection or controller-side terminal summary. +- Sphinx contributor timing and the xdist registry/settings manifest are designs, + not implemented lifecycle contracts. +- The extended runtime matrix still lacks report-only-first, repeated-call, and + interactive debugger coverage. Expected-exception output, `SyntaxError`, + `IGNORE_EXCEPTION_DETAIL`, and inline fail-fast are covered. +- The core avoids CPython's private runner loop but still uses the private + `DocTestParser._EXAMPLE_RE` and `_EXCEPTION_RE` contracts. Their behavior is + exercised indirectly, not yet guarded by focused compatibility probes. +- The Python 3.10/Sphinx 8 stack constrains docutils to its pre-0.22 line + convention. The package now states `<0.22`; supporting docutils 0.22 requires + the coordinated Python/Sphinx policy change in ADR 0005. +- Standalone MyST root-line recovery cannot prove exact locations inside + included Markdown files. It refuses to stamp a root line onto an included + source and retains the parser's ambiguous fallback. +- Profile context-manager entry and exit failures do not yet have phase-aware + result semantics, and the initial runtime contract has no separate + profile-decline outcome. +- The direct facade cannot reproduce doctest's complete verbose + `Trying`/`Expecting`/`ok` stream because successful per-example events are not + retained. Failure and summary output remain stock-shaped. A cleanup error that + follows an ordinary doctest mismatch is also retained only in the core result; + the direct facade has no secondary-outcome rendering channel yet. +- The pytest private-API quarantine binds its symbols eagerly and has no + prerelease CI probe, so an unsupported pytest can still fail at plugin import. +- The pytest 7.2 floor also requires an older pytest-asyncio test dependency; + the matrix must pin those versions together rather than installing each + plugin's newest release independently. +- The legacy adapter is still a flat module, which leaves its private quarantine + as the top-level `_pytest_doctest_compat` module. A packaged adapter namespace + would contain that private surface more cleanly. +- A registered block kind and its custom expected-output stamp are preserved + through projection, but registration alone does not teach reST or MyST a new + directive. +- Orphan, misplaced, and duplicate `testoutput` records still need explicit + diagnostics. Pairing itself is group-local and duplicate output follows + Sphinx's last-one-wins rule. `testcode :pyversion:` is deliberately enforced + rather than silently ignored as Sphinx does. +- Async pytest fixtures compose with documentation items under pytest-asyncio + 1.x. Its pytest-7-compatible 0.21 line does not await an async autouse fixture + for this item shape. Async block execution is only represented by the + execution-profile seam; no async profile was implemented in this spike. +- Document front-matter settings were premature and are deferred. + +## Conclusion + +Keep the typed core and thin adapters. Do not return to the monolith and do not +base extended execution on CPython's private loop. The ADR's central +one-item-per-shared-group decision survived implementation; its overclaims were +mostly in conformance, diagnostics, host bootstrap, and reporting rather than in +the core boundary itself. diff --git a/notes/analyses/README.md b/notes/analyses/README.md index 72b5187..f6a679d 100644 --- a/notes/analyses/README.md +++ b/notes/analyses/README.md @@ -52,7 +52,8 @@ What exactly does each of them require, and where do they contradict each other? - Cross-cutting: [`20-data-structures.md`](20-data-structures.md), [`21-data-flows.md`](21-data-flows.md), [`22-extension-seams.md`](22-extension-seams.md), - [`23-namespace-scope-and-test-identity.md`](23-namespace-scope-and-test-identity.md). + [`23-namespace-scope-and-test-identity.md`](23-namespace-scope-and-test-identity.md), + and [`24-implementation-bakeoff.md`](24-implementation-bakeoff.md). - [`90-bibliography.md`](90-bibliography.md) — every pinned anchor cited by the ADRs, in one place. diff --git a/pyproject.toml b/pyproject.toml index e3426f0..4617f10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,8 +35,10 @@ readme = 'README.md' keywords = [] homepage = "https://gp-libs.git-pull.com" dependencies = [ - "myst_parser", - "docutils" + "docutils>=0.20.1,<0.22", + "myst-parser>=2.0.0", + "packaging", + "pytest>=7.2", ] [project.urls] @@ -55,6 +57,8 @@ dev = [ # Testing "gp-libs", "pytest", + "pytest-asyncio", + "pytest-xdist", "pytest-rerunfailures", "pytest-mock", "pytest-watcher", @@ -78,6 +82,8 @@ docs = [ testing = [ "gp-libs", "pytest", + "pytest-asyncio", + "pytest-xdist", "pytest-rerunfailures", "pytest-mock", "pytest-watcher", @@ -95,7 +101,7 @@ lint = [ ] [project.entry-points.pytest11] -sphinx = "pytest_doctest_docutils" +pytest_doctest_docutils = "pytest_doctest_docutils" [build-system] requires = ["hatchling"] @@ -127,11 +133,17 @@ sphinx-ux-autodoc-layout = false sphinx-ux-badges = false [tool.hatch.build.targets.sdist] -include = ["src/*.py"] +include = [ + "src/*.py", + "src/doctest_core/**", + "tests/**", +] [tool.hatch.build.targets.wheel] packages = [ + "src/_pytest_doctest_compat.py", "src/docutils_compat.py", + "src/doctest_core", "src/doctest_docutils.py", "src/gp_libs.py", "src/linkify_issues.py", diff --git a/src/_pytest_doctest_compat.py b/src/_pytest_doctest_compat.py new file mode 100644 index 0000000..f0728ee --- /dev/null +++ b/src/_pytest_doctest_compat.py @@ -0,0 +1,291 @@ +"""Quarantine pytest's private doctest APIs. + +The public functions in this module are the only boundary at which the +``pytest_doctest_docutils`` adapter should depend on ``_pytest.doctest``. +""" + +from __future__ import annotations + +import collections.abc +import doctest +import traceback +import typing as t + +import pytest +from _pytest import doctest as pytest_doctest +from _pytest._code import ExceptionInfo +from _pytest._code.code import ReprFileLocation, TerminalRepr + +DoctestTextfile = pytest_doctest.DoctestTextfile +MultipleDoctestFailures = pytest_doctest.MultipleDoctestFailures + + +class _OptionflagsContext: + """Present the option interface expected by pytest 7 through 9. + + Attributes + ---------- + config : pytest.Config + Configuration exposed for pytest 7's collector-like call shape. + """ + + def __init__(self, config: pytest.Config) -> None: + """Store the pytest configuration. + + Parameters + ---------- + config : pytest.Config + Configuration whose doctest flags are requested. + + Examples + -------- + The compatibility object retains the exact config object. + + >>> marker = object() + >>> context = _OptionflagsContext(t.cast(pytest.Config, marker)) + >>> context.config is marker + True + """ + self.config = config + + def getini(self, name: str) -> object: + """Delegate ini access for pytest 8 and newer. + + Parameters + ---------- + name : str + Ini option name. + + Returns + ------- + object + Parsed ini value. + + Examples + -------- + >>> class Config: + ... def getini(self, name: str) -> object: + ... return name + >>> context = _OptionflagsContext(t.cast(pytest.Config, Config())) + >>> context.getini("doctest_optionflags") + 'doctest_optionflags' + """ + return self.config.getini(name) + + +def get_checker() -> doctest.OutputChecker: + """Return pytest's extended doctest output checker. + + Returns + ------- + doctest.OutputChecker + Checker supporting pytest's ``ALLOW_*`` and ``NUMBER`` flags. + + Examples + -------- + >>> isinstance(get_checker(), doctest.OutputChecker) + True + """ + return pytest_doctest._get_checker() + + +def get_continue_on_failure(config: pytest.Config) -> bool: + """Return pytest's resolved continue-on-failure policy. + + Parameters + ---------- + config : pytest.Config + Active pytest configuration. + + Returns + ------- + bool + False when pdb requires stopping at the first failure. + + Examples + -------- + The result is always a concrete boolean for a configured session. + + >>> callable(get_continue_on_failure) + True + """ + return pytest_doctest._get_continue_on_failure(config) + + +def get_optionflags(config: pytest.Config) -> int: + """Return doctest option flags across pytest 7 through 9. + + Pytest 7 expects a collector-like object with ``.config``; pytest 8 and + newer expect ``Config`` directly. The compatibility context supports both. + + Parameters + ---------- + config : pytest.Config + Active pytest configuration. + + Returns + ------- + int + Bitwise combination of configured doctest flags. + + Examples + -------- + >>> callable(get_optionflags) + True + """ + context = _OptionflagsContext(config) + return pytest_doctest.get_optionflags(context) # type: ignore[arg-type] + + +def make_multiple_failures( + failures: collections.abc.Sequence[ + doctest.DocTestFailure | doctest.UnexpectedException + ], +) -> BaseException: + """Build pytest's aggregate doctest failure across its narrow annotation. + + Pytest's runner stores both ordinary and unexpected doctest failures, while + the private exception constructor is annotated for ordinary failures only. + + Parameters + ---------- + failures : sequence of doctest failures + Failures retained in source order. + + Returns + ------- + BaseException + Pytest's aggregate failure carrying the original sequence. + + Examples + -------- + >>> callable(make_multiple_failures) + True + """ + constructor = t.cast(t.Any, MultipleDoctestFailures) + return t.cast(BaseException, constructor(failures)) + + +def repr_failure_with_checkers( + item: pytest.DoctestItem, + excinfo: ExceptionInfo[BaseException], + checkers: t.Mapping[int, doctest.OutputChecker], +) -> str | TerminalRepr | None: + """Render doctest failures with their comparison-time checkers. + + Parameters + ---------- + item : pytest.DoctestItem + Item whose configuration selects pytest's report style. + excinfo : pytest.ExceptionInfo + Failure raised by the item. + checkers : mapping of int to doctest.OutputChecker + Checker instances indexed by ``id(failure)``. + + Returns + ------- + str, pytest.TerminalRepr, or None + Pytest's doctest representation, or ``None`` for non-doctest errors. + + Examples + -------- + >>> callable(repr_failure_with_checkers) + True + """ + failures: ( + collections.abc.Sequence[doctest.DocTestFailure | doctest.UnexpectedException] + | None + ) = None + if isinstance( + excinfo.value, + (doctest.DocTestFailure, doctest.UnexpectedException), + ): + failures = [excinfo.value] + elif isinstance(excinfo.value, MultipleDoctestFailures): + failures = t.cast( + collections.abc.Sequence[ + doctest.DocTestFailure | doctest.UnexpectedException + ], + excinfo.value.failures, + ) + if failures is None: + return None + + reprlocation_lines: list[tuple[t.Any, list[str]]] = [] + report_choice = pytest_doctest._get_report_choice( + item.config.getoption("doctestreport"), + ) + for failure in failures: + example = failure.example + test = failure.test + lineno = None if test.lineno is None else test.lineno + example.lineno + 1 + reprlocation = ReprFileLocation( + t.cast(str, test.filename), + lineno, # type: ignore[arg-type] + type(failure).__name__, + ) + if lineno is not None: + assert test.docstring is not None + assert test.lineno is not None + lines = [ + f"{index + test.lineno + 1:03d} {line}" + for index, line in enumerate(test.docstring.splitlines(False)) + ] + lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] + else: + lines = [ + "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example", + ] + indent = ">>>" + for line in example.source.splitlines(): + lines.append(f"??? {indent} {line}") + indent = "..." + + if isinstance(failure, doctest.DocTestFailure): + checker = checkers[id(failure)] + lines.extend( + checker.output_difference( + example, + failure.got, + report_choice, + ).split("\n"), + ) + else: + inner_excinfo = ExceptionInfo.from_exc_info( + failure.exc_info, + ) + lines.append(f"UNEXPECTED EXCEPTION: {inner_excinfo.value!r}") + lines.extend( + line.strip("\n") + for line in traceback.format_exception(*failure.exc_info) + ) + reprlocation_lines.append((reprlocation, lines)) + return pytest_doctest.ReprFailDoctest(reprlocation_lines) + + +def disable_output_capturing_for_darwin(item: pytest.DoctestItem) -> None: + """Apply pytest's Darwin doctest capture workaround to an item. + + Parameters + ---------- + item : pytest.DoctestItem + Item about to execute doctest examples. + + Examples + -------- + >>> callable(disable_output_capturing_for_darwin) + True + """ + item._disable_output_capturing_for_darwin() + + +__all__ = [ + "DoctestTextfile", + "MultipleDoctestFailures", + "disable_output_capturing_for_darwin", + "get_checker", + "get_continue_on_failure", + "get_optionflags", + "make_multiple_failures", + "repr_failure_with_checkers", +] diff --git a/src/doctest_core/__init__.py b/src/doctest_core/__init__.py new file mode 100644 index 0000000..82b24bc --- /dev/null +++ b/src/doctest_core/__init__.py @@ -0,0 +1,131 @@ +"""Typed, host-neutral doctest planning and execution.""" + +from __future__ import annotations + +from .contracts import ( + CheckerFactory, + Contributor, + DocumentParser, + ExceptionPolicy, + ExecutionProfile, + ExecutionRuntime, + Provider, + Registrar, + Registration, + RuntimeOutcome, + RuntimeSettings, +) +from .markup import ( + DoctestDirective, + MockTabDirective, + MystDocumentParser, + RstDocumentParser, + TestcleanupDirective, + TestcodeDirective, + TestoutputDirective, + TestsetupDirective, + ensure_directives_registered, + extract_blocks, + parse_document, +) +from .model import ( + BlockKind, + BlockResult, + Counts, + Diagnostic, + Errored, + ExampleRecipe, + ExpectedOutput, + Failed, + Failure, + GroupPlan, + GroupResult, + ParsedBlock, + ParsedOutput, + ParseResult, + Passed, + Phase, + ProjectedBlock, + Skipped, + SkipReason, +) +from .project import project +from .registry import ( + RegistryClosedError, + RegistryCollisionError, + RegistryError, + RegistrySnapshot, + build_registry, +) +from .runner import ( + DefaultExceptionPolicy, + ExecExecutionProfile, + ExecRuntime, + PromptExecutionProfile, + PromptRuntime, + materialize, + reset_globs, + run_group, +) +from .settings import ParseSettings, ProjectionSettings, RunSettings + +__all__ = [ + "BlockKind", + "BlockResult", + "CheckerFactory", + "Contributor", + "Counts", + "DefaultExceptionPolicy", + "Diagnostic", + "DoctestDirective", + "DocumentParser", + "Errored", + "ExampleRecipe", + "ExceptionPolicy", + "ExecExecutionProfile", + "ExecRuntime", + "ExecutionProfile", + "ExecutionRuntime", + "ExpectedOutput", + "Failed", + "Failure", + "GroupPlan", + "GroupResult", + "MockTabDirective", + "MystDocumentParser", + "ParseResult", + "ParseSettings", + "ParsedBlock", + "ParsedOutput", + "Passed", + "Phase", + "ProjectedBlock", + "ProjectionSettings", + "PromptExecutionProfile", + "PromptRuntime", + "Provider", + "Registrar", + "Registration", + "RegistryClosedError", + "RegistryCollisionError", + "RegistryError", + "RegistrySnapshot", + "RstDocumentParser", + "RunSettings", + "RuntimeOutcome", + "RuntimeSettings", + "SkipReason", + "Skipped", + "TestcleanupDirective", + "TestcodeDirective", + "TestoutputDirective", + "TestsetupDirective", + "build_registry", + "ensure_directives_registered", + "extract_blocks", + "materialize", + "parse_document", + "project", + "reset_globs", + "run_group", +] diff --git a/src/doctest_core/contracts.py b/src/doctest_core/contracts.py new file mode 100644 index 0000000..9873d9e --- /dev/null +++ b/src/doctest_core/contracts.py @@ -0,0 +1,247 @@ +"""Public structural contracts for doctest-core extensions.""" + +from __future__ import annotations + +import contextlib +import dataclasses +import doctest +import pathlib +import typing as t + +from docutils import nodes + +from .model import BlockKind, Diagnostic, Failure +from .settings import ParseSettings + + +class Provider(t.NamedTuple): + """Identity attached to every contributed capability. + + Attributes + ---------- + name : str + Stable provider name. + version : str or None + Provider version when one is available. + + >>> Provider("example", "1").name + 'example' + """ + + name: str + version: str | None + + +T = t.TypeVar("T") + + +@dataclasses.dataclass(frozen=True, slots=True) +class Registration(t.Generic[T]): + """One immutable, attributed registry entry. + + Attributes + ---------- + name : str + Case-sensitive registration name. + value : T + Registered capability. + provider : Provider + Contributor that supplied the value. + """ + + name: str + value: T + provider: Provider + + +class RuntimeOutcome(t.NamedTuple): + """Outcome returned by an execution runtime. + + Attributes + ---------- + results : doctest.TestResults + Standard attempted and failed totals. + failures : tuple of Failure + Failures retained for host-native reporting. + skipped : int + Examples reached and skipped by the runtime. + """ + + results: doctest.TestResults + failures: tuple[Failure, ...] + skipped: int + + +class ExceptionPolicy(t.Protocol): + """Classify exceptions that must escape a runtime's doctest loop.""" + + def should_propagate(self, error: BaseException) -> bool: + """Return whether ``error`` belongs to the embedding host. + + >>> isinstance(KeyboardInterrupt(), BaseException) + True + """ + ... + + def is_abort(self, error: BaseException) -> bool: + """Return whether ``error`` must outrank every recorded outcome. + + >>> isinstance(KeyboardInterrupt(), BaseException) + True + """ + ... + + +class RuntimeSettings(t.NamedTuple): + """Resolved objects and policy used by one execution runtime. + + Attributes + ---------- + optionflags : int + Runner-level doctest option bitmask. + continue_on_failure : bool + Continue after an example mismatch. + checker : doctest.OutputChecker + Fresh checker used for comparison and explanation. + exception_policy : ExceptionPolicy + Host-neutral classifier for exceptions that must propagate. + """ + + optionflags: int + continue_on_failure: bool + checker: doctest.OutputChecker + exception_policy: ExceptionPolicy + + +class CheckerFactory(t.Protocol): + """Construct a fresh output checker for an execution runtime.""" + + def __call__(self) -> doctest.OutputChecker: + r"""Return a checker used for comparison and failure explanation. + + >>> doctest.OutputChecker().check_output("42\n", "42\n", 0) + True + """ + ... + + +class ExecutionRuntime(t.Protocol): + """Attempt-local executor for materialized stock doctests.""" + + def run(self, test: doctest.DocTest) -> RuntimeOutcome: + """Execute ``test`` without clearing its shared globals. + + >>> test = doctest.DocTest([], {}, "example", "example.rst", 0, "") + >>> test.name + 'example' + """ + ... + + +class ExecutionProfile(t.Protocol): + """Immutable factory for attempt-local execution runtimes.""" + + def open( + self, + settings: RuntimeSettings, + ) -> contextlib.AbstractContextManager[ExecutionRuntime]: + """Open a runtime whose resources live for one group attempt. + + >>> issubclass(contextlib.AbstractContextManager, object) + True + """ + ... + + +class DocumentParser(t.Protocol): + """Parse one markup language into a docutils document.""" + + suffixes: t.ClassVar[frozenset[str]] + + def parse( + self, + text: str, + path: pathlib.Path, + *, + settings: ParseSettings, + ) -> tuple[nodes.document, tuple[Diagnostic, ...]]: + """Parse ``text`` while retaining normalized diagnostics. + + >>> pathlib.Path("guide.rst").suffix + '.rst' + """ + ... + + +class Registrar(t.Protocol): + """Provider-bound mutation surface available during contribution.""" + + def add_block_kind( + self, + name: str, + kind: BlockKind, + *, + replace: bool = False, + ) -> None: + """Register a block kind. + + >>> BlockKind.__name__ + 'BlockKind' + """ + ... + + def add_document_parser( + self, + name: str, + parser: DocumentParser, + *, + replace: bool = False, + ) -> None: + """Register a document parser and its suffix claims. + + >>> ".rst" in frozenset({".rst"}) + True + """ + ... + + def add_execution_profile( + self, + name: str, + profile: ExecutionProfile, + *, + replace: bool = False, + ) -> None: + """Register an execution-profile factory. + + >>> "prompt".islower() + True + """ + ... + + def add_output_checker( + self, + name: str, + factory: CheckerFactory, + *, + replace: bool = False, + ) -> None: + """Register an output-checker factory. + + >>> callable(doctest.OutputChecker) + True + """ + ... + + +class Contributor(t.Protocol): + """Host-neutral source of attributed registry entries.""" + + provider: Provider + + def contribute(self, registrar: Registrar) -> None: + """Add capabilities through the provider-bound ``registrar``. + + >>> Provider("example", None).version is None + True + """ + ... diff --git a/src/doctest_core/markup.py b/src/doctest_core/markup.py new file mode 100644 index 0000000..910bc5d --- /dev/null +++ b/src/doctest_core/markup.py @@ -0,0 +1,579 @@ +"""Docutils and MyST front ends for doctest core.""" + +from __future__ import annotations + +import doctest +import io +import pathlib +import re +import textwrap +import typing as t +import warnings + +from docutils import nodes +from docutils.frontend import OptionParser +from docutils.parsers.rst import Directive, Parser, directives +from docutils.utils import new_document + +from .model import Diagnostic, ParsedBlock, ParsedOutput, ParseResult +from .settings import ParseSettings + +if t.TYPE_CHECKING: + from .contracts import DocumentParser + from .registry import RegistrySnapshot + + +_BLANKLINE_RE = re.compile(r"^\s*", re.MULTILINE) +_DOCTEST_OPTION_RE = re.compile(r"[ \t]*#\s*doctest:.+$", re.MULTILINE) +_TEST_KINDS = frozenset( + {"doctest", "testsetup", "testcleanup", "testcode", "testoutput"}, +) +_REQUIRED_DIRECTIVES = (*sorted(_TEST_KINDS), "tab") + + +class _TestDirective(Directive): + """Create nodes carrying the Sphinx doctest attribute vocabulary.""" + + has_content = True + required_arguments = 0 + optional_arguments = 1 + final_argument_whitespace = True + + def run(self) -> list[nodes.Node]: + """Return one node stamped with inert doctest metadata.""" + code = "\n".join(self.content) + test = code + trim = "no-trim-doctest-flags" not in self.options + if self.name == "doctest" and trim: + display = _BLANKLINE_RE.sub("", code) + display = _DOCTEST_OPTION_RE.sub("", display) + else: + display = code + + node_type: type[nodes.TextElement] = nodes.literal_block + hidden = "hide" in self.options + if self.name in {"testsetup", "testcleanup"} or hidden: + node_type = nodes.comment + + groups = ( + [item.strip() for item in self.arguments[0].split(",")] + if self.arguments + else ["default"] + ) + node = node_type( + display, + display, + testnodetype=self.name, + groups=groups, + hidden=hidden, + ) + source, line = self.state_machine.get_source_and_line(self.lineno) + node.source = source + node.line = line + node["testline"] = self.content_offset + 1 + if test != display: + node["test"] = test + if self.name == "doctest": + node["language"] = "pycon3" + + node["options"] = self._parse_options() + for key in ("skipif", "pyversion"): + if key in self.options: + node[key] = self.options[key] + if "trim-doctest-flags" in self.options: + node["trim_flags"] = True + elif "no-trim-doctest-flags" in self.options: + node["trim_flags"] = False + return [node] + + def _parse_options(self) -> dict[int, bool]: + """Parse Sphinx ``:options:`` into doctest's integer flag mapping.""" + parsed: dict[int, bool] = {} + value = self.options.get("options") + if not isinstance(value, str): + return parsed + for option in value.replace(",", " ").split(): + if len(option) < 2 or option[0] not in "+-": + self.state.document.reporter.warning( + f"missing '+' or '-' in '{option}' option", + line=self.lineno, + ) + continue + flag = doctest.OPTIONFLAGS_BY_NAME.get(option[1:]) + if flag is None: + self.state.document.reporter.warning( + f"'{option[1:]}' is not a valid doctest option", + line=self.lineno, + ) + continue + parsed[flag] = option[0] == "+" + return parsed + + +class TestsetupDirective(_TestDirective): + """Parse a Sphinx-compatible ``testsetup`` directive.""" + + option_spec: t.ClassVar = { + "hide": directives.flag, + "skipif": directives.unchanged_required, + } + + +class TestcleanupDirective(_TestDirective): + """Parse a Sphinx-compatible ``testcleanup`` directive.""" + + option_spec: t.ClassVar = { + "hide": directives.flag, + "skipif": directives.unchanged_required, + } + + +class DoctestDirective(_TestDirective): + """Parse a Sphinx-compatible ``doctest`` directive.""" + + option_spec: t.ClassVar = { + "hide": directives.flag, + "no-trim-doctest-flags": directives.flag, + "options": directives.unchanged, + "pyversion": directives.unchanged_required, + "skipif": directives.unchanged_required, + "trim-doctest-flags": directives.flag, + } + + +class TestcodeDirective(_TestDirective): + """Parse a Sphinx-compatible ``testcode`` directive.""" + + option_spec: t.ClassVar = { + "hide": directives.flag, + "no-trim-doctest-flags": directives.flag, + "pyversion": directives.unchanged_required, + "skipif": directives.unchanged_required, + "trim-doctest-flags": directives.flag, + } + + +class TestoutputDirective(_TestDirective): + """Parse a Sphinx-compatible ``testoutput`` directive.""" + + option_spec: t.ClassVar = { + "hide": directives.flag, + "no-trim-doctest-flags": directives.flag, + "options": directives.unchanged, + "pyversion": directives.unchanged_required, + "skipif": directives.unchanged_required, + "trim-doctest-flags": directives.flag, + } + + +class MockTabDirective(Directive): + """Parse tab content when sphinx-inline-tabs is not installed.""" + + has_content = True + + def run(self) -> list[nodes.Node]: + """Return a transparent container around nested content.""" + self.assert_has_content() + content = nodes.container("", is_div=True, classes=["tab-content"]) + self.state.nested_parse(self.content, self.content_offset, content) + return [content] + + +_DIRECTIVE_TYPES: t.Mapping[str, type[Directive]] = { + "doctest": DoctestDirective, + "testsetup": TestsetupDirective, + "testcleanup": TestcleanupDirective, + "testcode": TestcodeDirective, + "testoutput": TestoutputDirective, + "tab": MockTabDirective, +} + + +def ensure_directives_registered() -> None: + """Register missing standalone directives without replacing Sphinx's. + + >>> ensure_directives_registered() + >>> all(name in directives._directives for name in _REQUIRED_DIRECTIVES) + True + """ + registry = t.cast(dict[str, t.Any], directives.__dict__["_directives"]) + for name, directive in _DIRECTIVE_TYPES.items(): + if name not in registry: + directives.register_directive(name, directive) + + +def _settings(parser_type: type[Parser]) -> t.Any: + """Build quiet docutils settings while retaining reporter messages.""" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + settings = OptionParser(components=(parser_type,)).get_default_values() + settings.report_level = 5 + settings.halt_level = 6 + settings.warning_stream = io.StringIO() + return settings + + +def _diagnostic_from_message(message: nodes.system_message) -> Diagnostic: + """Convert one docutils system message to a typed diagnostic.""" + level_number = int(message.get("level", 1)) + level: t.Literal["info", "warning", "error"] + if level_number >= 3: + level = "error" + elif level_number >= 2: + level = "warning" + else: + level = "info" + text = message.astext() + code = None + if "Unknown interpreted text role" in text or "No role entry for" in text: + code = "docutils.unknown-role" + elif "Unknown directive type" in text or "No directive entry for" in text: + code = "docutils.unknown-directive" + return Diagnostic( + level=level, + code=code, + message=text, + path=pathlib.Path(message.source or ""), + line=message.line, + ) + + +def _attach_diagnostics(document: nodes.document) -> list[Diagnostic]: + """Attach an observer and return its mutable capture list.""" + captured: list[Diagnostic] = [] + + def observe(message: nodes.system_message) -> None: + captured.append(_diagnostic_from_message(message)) + + document.reporter.attach_observer(observe) + return captured + + +class RstDocumentParser: + """Parse reStructuredText into a docutils document.""" + + suffixes: t.ClassVar = frozenset({".rst", ".txt"}) + + def parse( + self, + text: str, + path: pathlib.Path, + *, + settings: ParseSettings, + ) -> tuple[nodes.document, tuple[Diagnostic, ...]]: + """Parse text and return its doctree and diagnostics.""" + del settings + ensure_directives_registered() + parser = Parser() + document = new_document(str(path), settings=_settings(Parser)) + captured = _attach_diagnostics(document) + parser.parse(text, document) + return document, tuple(captured) + + +class MystDocumentParser: + """Parse MyST Markdown into a docutils document.""" + + suffixes: t.ClassVar = frozenset({".md"}) + + def parse( + self, + text: str, + path: pathlib.Path, + *, + settings: ParseSettings, + ) -> tuple[nodes.document, tuple[Diagnostic, ...]]: + """Parse text and return its doctree and diagnostics.""" + del settings + from myst_parser.config.main import MdParserConfig + from myst_parser.mdit_to_docutils.base import DocutilsRenderer + from myst_parser.parsers.docutils_ import Parser as MystParser + from myst_parser.parsers.mdit import create_md_parser + + ensure_directives_registered() + document = new_document(str(path), settings=_settings(MystParser)) + captured = _attach_diagnostics(document) + parser = create_md_parser( + MdParserConfig(commonmark_only=False), + DocutilsRenderer, + ) + parser.options["document"] = document + parser.render(text) + _stamp_myst_source_lines(document, text) + return document, tuple(captured) + + +def _normalize_body(value: str) -> str: + """Dedent a node body and preserve the executable trailing newline.""" + body = textwrap.dedent(value).strip("\n") + return f"{body}\n" if body else "" + + +def _groups(node: nodes.Element) -> tuple[str, ...]: + """Narrow a node's untyped group attribute.""" + value: object = node.get("groups", ()) + if isinstance(value, str): + return (value,) + if isinstance(value, (list, tuple)): + values = t.cast(list[object] | tuple[object, ...], value) + return tuple(str(item) for item in values) + return () + + +def _options(node: nodes.Element) -> t.Mapping[int, bool]: + """Narrow and copy a node's untyped option mapping.""" + value: object = node.get("options", {}) + if not isinstance(value, dict): + return {} + options = t.cast(dict[object, object], value) + return { + flag: bool(enabled) + for flag, enabled in options.items() + if isinstance(flag, int) + } + + +def _optional_text(node: nodes.Element, name: str) -> str | None: + """Validate an optional text attribute at the typed-model boundary. + + >>> node = nodes.literal_block("", "", skipif="enabled") + >>> _optional_text(node, "skipif") + 'enabled' + """ + value: object = node.get(name) + if value is None or isinstance(value, str): + return value + message = f"{name} node attribute must be str or None, got {type(value).__name__}" + raise TypeError(message) + + +def _node_kind(node: nodes.Node) -> str | None: + """Return the registered kind represented by a doctree node.""" + if isinstance(node, nodes.Element): + stamped = node.get("testnodetype") + if isinstance(stamped, str) and stamped: + return stamped + if isinstance(node, nodes.doctest_block): + return "doctest" + if isinstance(node, nodes.literal_block) and re.match( + doctest.DocTestParser._EXAMPLE_RE, # type: ignore[attr-defined] + node.astext(), + ): + return "doctest" + return None + + +def _stamp_myst_source_lines(doctree: nodes.document, text: str) -> None: + r"""Retain root-document body lines lost from MyST literal-block nodes. + + >>> tree = new_document("guide.md") + >>> node = nodes.literal_block(">>> 1 + 1\n2\n", ">>> 1 + 1\n2\n") + >>> node.source, node.line = "guide.md", 1 + >>> tree += node + >>> _stamp_myst_source_lines(tree, "```\n>>> 1 + 1\n2\n```\n") + >>> node["doctest_core_line"] + 2 + """ + lines = text.splitlines() + document_source = doctree.current_source or doctree.get("source") + for node in doctree.findall(nodes.literal_block): + if _node_kind(node) is None or node.line is None: + continue + if document_source and node.source != document_source: + continue + source = _normalize_body(str(node.get("test", node.astext()))) + if not source: + continue + first_line = source.splitlines()[0].strip() + for index in range(max(node.line - 1, 0), len(lines)): + if lines[index].strip() == first_line: + node["doctest_core_line"] = index + 1 + break + + +def _node_line(node: nodes.Element, source: str) -> int | None: + """Normalize parser-specific line conventions to the first body line.""" + if ":docstring of " in pathlib.Path(node.source or "").name: + return None + core_line = node.get("doctest_core_line") + if isinstance(core_line, int): + return core_line + suffix = pathlib.Path(node.source or "").suffix + if suffix == ".md" and isinstance(node, nodes.literal_block): + if node.line is None: + return None + local_testline = node.get("testline") + if isinstance(local_testline, int): + return node.line + local_testline + return node.line + 1 + testline = node.get("testline") + if isinstance(testline, int): + return testline + if node.line is None: + return None + if isinstance(node, nodes.doctest_block) and suffix != ".md": + return node.line - len(source.rstrip("\n").splitlines()) + 1 + return node.line + + +def extract_blocks( + doctree: nodes.document, + *, + settings: ParseSettings | None = None, + registry: RegistrySnapshot | None = None, +) -> ParseResult: + """Extract typed doctest records from an existing resolved doctree. + + >>> from docutils import nodes + >>> tree = nodes.document("", "") + >>> extract_blocks(tree).blocks + () + """ + if registry is None: + from .registry import build_registry + + registry = build_registry() + settings = settings or ParseSettings() + output_kinds = frozenset( + registration.value.pairs_with + for registration in registry.block_kinds.values() + if registration.value.pairs_with is not None + ) + blocks: list[ParsedBlock] = [] + outputs: list[ParsedOutput] = [] + block_ordinal = 0 + document_order = 0 + for node in doctree.findall(): + kind = _node_kind(node) + if kind is None or not isinstance(node, nodes.Element): + continue + source = _normalize_body(str(node.get("test", node.astext()))) + path = pathlib.Path(node.source or doctree.source or "") + line = _node_line(node, source) + groups = _groups(node) + options = _options(node) + skipif = _optional_text(node, "skipif") + pyversion = _optional_text(node, "pyversion") + if kind in output_kinds: + outputs.append( + ParsedOutput( + kind=kind, + text=source, + path=path, + line=line, + document_order=document_order, + groups=groups, + options=options, + skipif=skipif, + pyversion=pyversion, + ), + ) + elif kind in registry.block_kinds: + blocks.append( + ParsedBlock( + kind=kind, + source=source, + path=path, + line=line, + document_order=document_order, + block_ordinal=block_ordinal, + groups=groups, + options=options, + skipif=skipif, + pyversion=pyversion, + hidden=isinstance(node, nodes.comment) + or bool(node.get("hidden", False)), + ), + ) + block_ordinal += 1 + document_order += 1 + + diagnostics = tuple( + diagnostic + for node in doctree.findall(nodes.system_message) + if (diagnostic := _diagnostic_from_message(node)).code + not in settings.suppressed_diagnostics + ) + return ParseResult(tuple(blocks), tuple(outputs), diagnostics) + + +def _parser_for_path( + path: pathlib.Path, + registry: RegistrySnapshot, +) -> DocumentParser: + """Select one parser by its declared suffix.""" + matches = [ + registration.value + for registration in registry.document_parsers.values() + if path.suffix in registration.value.suffixes + ] + if len(matches) != 1: + message = f"expected one document parser for suffix {path.suffix!r}" + raise ValueError(message) + return matches[0] + + +def _merge_diagnostics( + parser_diagnostics: t.Iterable[Diagnostic], + tree_diagnostics: t.Iterable[Diagnostic], + settings: ParseSettings, +) -> tuple[Diagnostic, ...]: + """Merge parser channels and prefer tree copies with source provenance. + + >>> diagnostic = Diagnostic("error", "example", "bad", pathlib.Path("x"), 1) + >>> merged = _merge_diagnostics((diagnostic,), (diagnostic,), ParseSettings()) + >>> (len(merged), merged[0].line) + (1, 1) + """ + tree = [ + diagnostic + for diagnostic in tree_diagnostics + if diagnostic.code not in settings.suppressed_diagnostics + ] + unmatched_tree = [ + (diagnostic.level, diagnostic.code, diagnostic.message) for diagnostic in tree + ] + merged: list[Diagnostic] = [] + for diagnostic in parser_diagnostics: + if diagnostic.code in settings.suppressed_diagnostics: + continue + key = (diagnostic.level, diagnostic.code, diagnostic.message) + try: + matched_index = unmatched_tree.index(key) + except ValueError: + merged.append(diagnostic) + else: + unmatched_tree.pop(matched_index) + merged.extend(tree) + return tuple(merged) + + +def parse_document( + text: str, + path: pathlib.Path, + *, + settings: ParseSettings | None = None, + registry: RegistrySnapshot | None = None, +) -> ParseResult: + r"""Parse and extract a documentation page through the frozen registry. + + >>> parse_document('>>> 1 + 1\n2\n', pathlib.Path('x.rst')).blocks[0].kind + 'doctest' + """ + if registry is None: + from .registry import build_registry + + registry = build_registry() + settings = settings or ParseSettings() + parser = _parser_for_path(path, registry) + doctree, parser_diagnostics = parser.parse(text, path, settings=settings) + extracted = extract_blocks(doctree, settings=settings, registry=registry) + return ParseResult( + blocks=extracted.blocks, + outputs=extracted.outputs, + diagnostics=_merge_diagnostics( + parser_diagnostics, + extracted.diagnostics, + settings, + ), + ) diff --git a/src/doctest_core/model.py b/src/doctest_core/model.py new file mode 100644 index 0000000..23eb542 --- /dev/null +++ b/src/doctest_core/model.py @@ -0,0 +1,401 @@ +"""Host-neutral values carried through the doctest pipeline.""" + +from __future__ import annotations + +import doctest +import enum +import pathlib +import typing as t + + +class Phase(enum.IntEnum): + """Execution phase for a projected block. + + Setup and cleanup surround test blocks while retaining source order within + each phase. + + >>> Phase.SETUP < Phase.TEST < Phase.CLEANUP + True + """ + + SETUP = 0 + TEST = 1 + CLEANUP = 2 + + +class Diagnostic(t.NamedTuple): + """A parser diagnostic that can cross host boundaries. + + Attributes + ---------- + level : {"info", "warning", "error"} + Normalized severity. + code : str or None + Stable classifier when the parser provides one. + message : str + Human-readable explanation. + path : pathlib.Path + Source containing the diagnostic. + line : int or None + One-based source line when available. + """ + + level: t.Literal["info", "warning", "error"] + code: str | None + message: str + path: pathlib.Path + line: int | None + + +class ParsedBlock(t.NamedTuple): + """An inert runnable block extracted from markup. + + Attributes + ---------- + kind : str + Registered block-kind name. + source : str + Dedented author text. + path : pathlib.Path + File containing the block. + line : int or None + One-based source line when available. + document_order : int + Position in the shared block-and-output stream. + block_ordinal : int + Stable position among runnable blocks. + groups : tuple of str + Author-declared Sphinx group names. + options : mapping of int to bool + Doctest option overrides. + skipif : str or None + Unevaluated skip expression. + pyversion : str or None + Unevaluated PEP 440 version specifier. + hidden : bool + Whether the markup hides the block from rendered output. + """ + + kind: str + source: str + path: pathlib.Path + line: int | None + document_order: int + block_ordinal: int + groups: tuple[str, ...] + options: t.Mapping[int, bool] + skipif: str | None + pyversion: str | None + hidden: bool + + +class ParsedOutput(t.NamedTuple): + """An inert expected-output body. + + Attributes + ---------- + kind : str + Output-kind name referenced by a registered block kind. + text : str + Expected output text. + path : pathlib.Path + File containing the output. + line : int or None + One-based source line when available. + document_order : int + Position in the shared block-and-output stream. + groups : tuple of str + Author-declared Sphinx group names. + options : mapping of int to bool + Doctest option overrides. + skipif : str or None + Unevaluated skip expression. + pyversion : str or None + Unevaluated PEP 440 version specifier. + """ + + kind: str + text: str + path: pathlib.Path + line: int | None + document_order: int + groups: tuple[str, ...] + options: t.Mapping[int, bool] + skipif: str | None + pyversion: str | None + + +class ParseResult(t.NamedTuple): + """Complete typed output of parsing one document. + + Attributes + ---------- + blocks : tuple of ParsedBlock + Runnable blocks in source order. + outputs : tuple of ParsedOutput + Expected-output records in source order. + diagnostics : tuple of Diagnostic + Parser diagnostics retained as data. + """ + + blocks: tuple[ParsedBlock, ...] + outputs: tuple[ParsedOutput, ...] + diagnostics: tuple[Diagnostic, ...] + + +class BlockKind(t.NamedTuple): + """Projection policy registered for one markup block kind. + + Attributes + ---------- + phase : Phase + Phase in which the block executes. + profile_name : str + Registered execution-profile name. + pairs_with : str or None + Expected-output kind paired with the block, if any. + """ + + phase: Phase + profile_name: str + pairs_with: str | None + + +class ExpectedOutput(t.NamedTuple): + """Expected output paired with an executable block. + + Attributes + ---------- + text : str + Expected output text. + options : mapping of int to bool + Output-specific doctest option overrides. + skipif : str or None + Unevaluated skip expression. + pyversion : str or None + Unevaluated PEP 440 version specifier. + """ + + text: str + options: t.Mapping[int, bool] + skipif: str | None + pyversion: str | None + + +class ExampleRecipe(t.NamedTuple): + """Fields needed to rebuild one stock :class:`doctest.Example`. + + Attributes + ---------- + source : str + Python source ending in a newline. + want : str + Expected output ending in a newline when non-empty. + exc_msg : str or None + Expected exception detail. + lineno : int + Zero-based line relative to the block. + indent : int + Prompt indentation. + options : mapping of int to bool + Inline doctest option overrides. + """ + + source: str + want: str + exc_msg: str | None + lineno: int + indent: int + options: t.Mapping[int, bool] + + +class ProjectedBlock(t.NamedTuple): + """Immutable recipe for one runnable source block. + + Attributes + ---------- + phase : Phase + Execution phase. + name : str + Unique, machine-independent doctest name. + block_ordinal : int + Stable position among runnable source blocks. + examples : tuple of ExampleRecipe + Recipes materialized into stock examples per attempt. + docstring : str + Source used by failure renderers. + filename : str + Source filename shown by doctest and host adapters. + lineno : int or None + One-based block line when known. + options : mapping of int to bool + Block-level doctest option overrides. + profile_name : str + Registered execution-profile name. + skipif : str or None + Unevaluated skip expression. + pyversion : str or None + Unevaluated PEP 440 version specifier. + expected : ExpectedOutput or None + Paired expected output for an executable-code block. + """ + + phase: Phase + name: str + block_ordinal: int + examples: tuple[ExampleRecipe, ...] + docstring: str + filename: str + lineno: int | None + options: t.Mapping[int, bool] + profile_name: str + skipif: str | None + pyversion: str | None + expected: ExpectedOutput | None + + +class GroupPlan(t.NamedTuple): + """Structurally immutable execution plan for one Sphinx group. + + Attributes + ---------- + group : str + Author-facing group name. + blocks : tuple of ProjectedBlock + Block recipes in execution order. + seed : mapping of str to Any + Initial names copied into each attempt's live mapping. + """ + + group: str + blocks: tuple[ProjectedBlock, ...] + seed: t.Mapping[str, t.Any] + + +Failure: t.TypeAlias = doctest.DocTestFailure | doctest.UnexpectedException + + +class Counts(t.NamedTuple): + """Failure, attempt, and skip counts for one block. + + Attributes + ---------- + failed : int + Number of mismatches, including failures hidden from detailed reports. + attempted : int + Number of examples doctest attempted. + skipped : int + Number of examples skipped by inline or block policy. + """ + + failed: int + attempted: int + skipped: int + + +class SkipReason(t.NamedTuple): + """Structured explanation for a skipped block. + + Attributes + ---------- + kind : {"skipif", "inline-flag", "pyversion"} + Policy that skipped the block. + detail : str + Gate expression, option name, specifier, or profile explanation. + """ + + kind: t.Literal["skipif", "inline-flag", "pyversion"] + detail: str + + +class Passed(t.NamedTuple): + """Successful block result. + + Attributes + ---------- + block : ProjectedBlock + Block that ran. + counts : Counts + Failure, attempt, and skip totals. + """ + + block: ProjectedBlock + counts: Counts + + +class Failed(t.NamedTuple): + """Doctest-comparison failures from one block. + + Attributes + ---------- + block : ProjectedBlock + Block that ran. + counts : Counts + Failure, attempt, and skip totals. + failures : tuple of Failure + Failures retained in example order. + checker : doctest.OutputChecker + Exact checker instance that compared and must explain the failures. + """ + + block: ProjectedBlock + counts: Counts + failures: tuple[Failure, ...] + checker: doctest.OutputChecker + + +class Skipped(t.NamedTuple): + """Block skipped by a run-time policy. + + Attributes + ---------- + block : ProjectedBlock + Block that did not run. + counts : Counts + Failure, attempt, and skip totals. + reason : SkipReason + Structured skip explanation. + """ + + block: ProjectedBlock + counts: Counts + reason: SkipReason + + +class Errored(t.NamedTuple): + """Infrastructure or gate error from one block. + + Attributes + ---------- + block : ProjectedBlock + Block whose execution failed outside doctest comparison. + error : BaseException + Original exception retained for the host adapter. + """ + + block: ProjectedBlock + error: BaseException + + +BlockResult: t.TypeAlias = Passed | Failed | Skipped | Errored + + +class GroupResult(t.NamedTuple): + """Result of one group attempt. + + Attributes + ---------- + group : str + Author-facing group name. + blocks : tuple of BlockResult + Results in execution order. + primary : BaseException or None + Body exception a host should re-raise. + secondary : tuple of BaseException + Additional exceptions, such as cleanup errors after body failure. + """ + + group: str + blocks: tuple[BlockResult, ...] + primary: BaseException | None + secondary: tuple[BaseException, ...] diff --git a/src/doctest_core/project.py b/src/doctest_core/project.py new file mode 100644 index 0000000..dd6200d --- /dev/null +++ b/src/doctest_core/project.py @@ -0,0 +1,313 @@ +"""Pure projection from parsed records to immutable group plans.""" + +from __future__ import annotations + +import doctest +import pathlib +import types +import typing as t + +from .model import ( + ExampleRecipe, + ExpectedOutput, + GroupPlan, + ParsedBlock, + ParsedOutput, + ParseResult, + Phase, + ProjectedBlock, +) +from .settings import ProjectionSettings + +if t.TYPE_CHECKING: + from .registry import RegistrySnapshot + + +class _GroupKey(t.NamedTuple): + """Collision-free identity for one projected group. + + Attributes + ---------- + author_name : str or None + Declared group name, or ``None`` for a generated block group. + block_ordinal : int or None + Runnable-block ordinal for a generated group, or ``None`` for a + declared group. + """ + + author_name: str | None + block_ordinal: int | None + + +def _read_only(values: t.Mapping[t.Any, t.Any]) -> t.Mapping[t.Any, t.Any]: + """Copy a mapping behind a read-only view.""" + return types.MappingProxyType(dict(values)) + + +def _groups_for_block( + block: ParsedBlock, + settings: ProjectionSettings, +) -> tuple[_GroupKey, ...]: + """Resolve an ordinary block's non-wildcard group names.""" + if block.groups: + return tuple(_GroupKey(group, None) for group in block.groups) + if settings.ungrouped == "default": + return (_GroupKey("default", None),) + return (_GroupKey(None, block.block_ordinal),) + + +def _test_groups( + parsed: ParseResult, + settings: ProjectionSettings, + registry: RegistrySnapshot, +) -> tuple[_GroupKey, ...]: + """Return executable group names in first declaration order.""" + groups: list[_GroupKey] = [] + has_wildcard = False + for block in parsed.blocks: + registration = registry.block_kinds.get(block.kind) + if registration is None or registration.value.phase is not Phase.TEST: + continue + for group in _groups_for_block(block, settings): + if group.author_name == "*": + has_wildcard = True + elif group not in groups: + groups.append(group) + if has_wildcard and not groups: + groups.append(_GroupKey("default", None)) + return tuple(groups) + + +def _group_labels(groups: tuple[_GroupKey, ...]) -> dict[_GroupKey, str]: + """Derive concise unique display labels without changing group identity.""" + reserved = {group.author_name for group in groups if group.author_name is not None} + labels: dict[_GroupKey, str] = {} + used: set[str] = set() + for group in groups: + if group.author_name is not None: + label = group.author_name + else: + base = f"block-{group.block_ordinal}" + label = base + suffix = 1 + while label in reserved or label in used: + marker = "anonymous" if suffix == 1 else f"anonymous-{suffix}" + label = f"{base}[{marker}]" + suffix += 1 + labels[group] = label + used.add(label) + return labels + + +def _destinations( + block: ParsedBlock, + *, + settings: ProjectionSettings, + groups: tuple[_GroupKey, ...], +) -> tuple[_GroupKey, ...]: + """Expand one block's declared groups against document groups.""" + declared = _groups_for_block(block, settings) + if any(group.author_name == "*" for group in declared): + return groups + return tuple(group for group in declared if group in groups) + + +def _output_matches(output: ParsedOutput, group: _GroupKey) -> bool: + """Return whether an expected-output record belongs to ``group``.""" + return "*" in output.groups or ( + group.author_name is not None and group.author_name in output.groups + ) + + +def _paired_output( + block: ParsedBlock, + output_kind: str, + group: _GroupKey, + parsed: ParseResult, + settings: ProjectionSettings, + groups: tuple[_GroupKey, ...], + registry: RegistrySnapshot, +) -> ParsedOutput | None: + """Find the latest output before the next test block in this group.""" + later_blocks = [ + candidate.document_order + for candidate in parsed.blocks + if candidate.document_order > block.document_order + and candidate.kind in registry.block_kinds + and registry.block_kinds[candidate.kind].value.phase is Phase.TEST + and group + in _destinations( + candidate, + settings=settings, + groups=groups, + ) + ] + boundary = min(later_blocks, default=2**63 - 1) + matches = [ + output + for output in parsed.outputs + if output.kind == output_kind + and block.document_order < output.document_order < boundary + and _output_matches(output, group) + ] + return matches[-1] if matches else None + + +def _prompt_recipes(block: ParsedBlock) -> tuple[ExampleRecipe, ...]: + """Project through the unmodified standard-library parser.""" + test = doctest.DocTestParser().get_doctest( + block.source, + {}, + "", + str(block.path), + 0, + ) + return tuple( + ExampleRecipe( + source=example.source, + want=example.want, + exc_msg=example.exc_msg, + lineno=example.lineno, + indent=example.indent, + options=_read_only(example.options), + ) + for example in test.examples + ) + + +def _exec_recipe(block: ParsedBlock) -> tuple[ExampleRecipe, ...]: + """Represent one prompt-free body as a stock example recipe.""" + return ( + ExampleRecipe( + source=block.source, + want="", + exc_msg=None, + lineno=0, + indent=0, + options=_read_only({}), + ), + ) + + +def _project_block( + block: ParsedBlock, + *, + group: _GroupKey, + group_label: str, + document_name: str, + parsed: ParseResult, + settings: ProjectionSettings, + groups: tuple[_GroupKey, ...], + registry: RegistrySnapshot, +) -> ProjectedBlock: + """Create a fresh group-qualified recipe for one parsed block.""" + kind = registry.block_kinds[block.kind].value + output = ( + _paired_output( + block, + kind.pairs_with, + group, + parsed, + settings, + groups, + registry, + ) + if kind.pairs_with + else None + ) + expected = ( + ExpectedOutput( + text=output.text, + options=_read_only(output.options), + skipif=output.skipif, + pyversion=output.pyversion, + ) + if output is not None + else None + ) + examples = ( + _prompt_recipes(block) if kind.profile_name == "prompt" else _exec_recipe(block) + ) + stem = pathlib.PurePath(document_name).stem + return ProjectedBlock( + phase=kind.phase, + name=f"{stem}::{group_label}[{block.block_ordinal}]", + block_ordinal=block.block_ordinal, + examples=examples, + docstring=block.source, + filename=str(block.path), + lineno=None if block.line is None else max(block.line - 1, 0), + options=_read_only(block.options), + profile_name=kind.profile_name, + skipif=block.skipif, + pyversion=block.pyversion, + expected=expected, + ) + + +def project( + parsed: ParseResult, + *, + document_name: str, + settings: ProjectionSettings | None = None, + registry: RegistrySnapshot | None = None, + seed: t.Mapping[str, t.Any] | None = None, +) -> tuple[GroupPlan, ...]: + """Project inert records into one immutable plan per shared-state group. + + Grouping and pairing are pure: no user code, filesystem access, docutils, + or host lifecycle object crosses this boundary. + + >>> from .model import ParseResult + >>> project(ParseResult((), (), ()), document_name="empty.rst") + () + """ + if registry is None: + from .registry import build_registry + + registry = build_registry() + settings = settings or ProjectionSettings() + groups = _test_groups(parsed, settings, registry) + labels = _group_labels(groups) + by_group: dict[_GroupKey, list[tuple[int, ProjectedBlock]]] = { + group: [] for group in groups + } + for block in parsed.blocks: + if block.kind not in registry.block_kinds: + continue + for group in _destinations(block, settings=settings, groups=groups): + projected = _project_block( + block, + group=group, + group_label=labels[group], + document_name=document_name, + parsed=parsed, + settings=settings, + groups=groups, + registry=registry, + ) + if projected.phase is Phase.TEST and not projected.examples: + continue + by_group[group].append( + ( + block.document_order, + projected, + ), + ) + + frozen_seed = _read_only(seed or {}) + return tuple( + GroupPlan( + group=labels[group], + blocks=tuple( + block + for _, block in sorted( + by_group[group], + key=lambda entry: (entry[1].phase, entry[0]), + ) + ), + seed=frozen_seed, + ) + for group in groups + if any(block.phase is Phase.TEST for _, block in by_group[group]) + ) diff --git a/src/doctest_core/py.typed b/src/doctest_core/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/doctest_core/py.typed @@ -0,0 +1 @@ + diff --git a/src/doctest_core/registry.py b/src/doctest_core/registry.py new file mode 100644 index 0000000..6ec014b --- /dev/null +++ b/src/doctest_core/registry.py @@ -0,0 +1,313 @@ +"""Deterministic construction of immutable doctest-core registries.""" + +from __future__ import annotations + +import doctest +import re +import types +import typing as t + +from .contracts import ( + CheckerFactory, + Contributor, + DocumentParser, + ExecutionProfile, + Provider, + Registrar, + Registration, +) +from .model import BlockKind, Phase + + +class RegistryError(ValueError): + """Base error for invalid registry construction.""" + + +class RegistryClosedError(RegistryError): + """Raised when a retained registrar is used after registry freeze.""" + + +class RegistryCollisionError(RegistryError): + """Raised when a registration would implicitly replace another.""" + + +class RegistrySnapshot(t.NamedTuple): + """Read-only capability set consumed by pipeline stages. + + Attributes + ---------- + block_kinds : mapping of str to Registration[BlockKind] + Projection policies in declaration order. + document_parsers : mapping of str to Registration[DocumentParser] + Markup parsers in declaration order. + execution_profiles : mapping of str to Registration[ExecutionProfile] + Runtime factories in declaration order. + output_checkers : mapping of str to Registration[CheckerFactory] + Checker factories in declaration order. + """ + + block_kinds: t.Mapping[str, Registration[BlockKind]] + document_parsers: t.Mapping[str, Registration[DocumentParser]] + execution_profiles: t.Mapping[str, Registration[ExecutionProfile]] + output_checkers: t.Mapping[str, Registration[CheckerFactory]] + + +_NAME_PATTERN = re.compile(r"[a-z][a-z0-9_.-]*\Z", flags=re.ASCII) +_BUILTIN_PROVIDER = Provider(name="builtin", version=None) +U = t.TypeVar("U") + + +def _read_only( + entries: dict[str, Registration[U]], +) -> t.Mapping[str, Registration[U]]: + return types.MappingProxyType(dict(entries)) + + +class _RegistryBuilder: + def __init__(self) -> None: + self.block_kinds: dict[str, Registration[BlockKind]] = {} + self.document_parsers: dict[str, Registration[DocumentParser]] = {} + self.execution_profiles: dict[str, Registration[ExecutionProfile]] = {} + self.output_checkers: dict[str, Registration[CheckerFactory]] = {} + self.closed = False + + def registrar(self, provider: Provider) -> _BoundRegistrar: + self._require_open() + return _BoundRegistrar(self, provider) + + def close(self) -> None: + self.closed = True + + def freeze(self) -> RegistrySnapshot: + self._require_open() + self._validate_references() + self.close() + return RegistrySnapshot( + block_kinds=_read_only(self.block_kinds), + document_parsers=_read_only(self.document_parsers), + execution_profiles=_read_only(self.execution_profiles), + output_checkers=_read_only(self.output_checkers), + ) + + def _validate_references(self) -> None: + """Reject block policies that cannot resolve unambiguously.""" + for name, registration in self.block_kinds.items(): + kind = registration.value + if kind.profile_name not in self.execution_profiles: + message = ( + f"block kind {name!r} from provider " + f"{registration.provider.name!r} references missing execution " + f"profile {kind.profile_name!r}" + ) + raise RegistryError(message) + if kind.pairs_with is None: + continue + self._validate_name(kind.pairs_with) + output_collision = self.block_kinds.get(kind.pairs_with) + if output_collision is not None: + message = ( + f"block kind {name!r} from provider " + f"{registration.provider.name!r} pairs with {kind.pairs_with!r}, " + "which is also a runnable block kind from provider " + f"{output_collision.provider.name!r}" + ) + raise RegistryCollisionError(message) + + def add( + self, + category: str, + entries: dict[str, Registration[U]], + name: str, + value: U, + provider: Provider, + *, + replace: bool, + ) -> None: + self._require_open() + self._validate_name(name) + incumbent = entries.get(name) + if incumbent is not None and not replace: + msg = ( + f"{category} {name!r} from provider " + f"{incumbent.provider.name!r} already exists; provider " + f"{provider.name!r} must pass replace=True" + ) + raise RegistryCollisionError(msg) + entries[name] = Registration(name, value, provider) + + def add_document_parser( + self, + name: str, + parser: DocumentParser, + provider: Provider, + *, + replace: bool, + ) -> None: + self._require_open() + self._validate_name(name) + for incumbent_name, incumbent in self.document_parsers.items(): + if incumbent_name == name: + continue + overlap = parser.suffixes & incumbent.value.suffixes + if overlap: + suffixes = ", ".join(sorted(overlap)) + msg = ( + f"document parser {name!r} from provider " + f"{provider.name!r} overlaps {incumbent_name!r} from " + f"provider {incumbent.provider.name!r} for {suffixes}" + ) + raise RegistryCollisionError(msg) + self.add( + "document parser", + self.document_parsers, + name, + parser, + provider, + replace=replace, + ) + + def _require_open(self) -> None: + if self.closed: + msg = "registry registration is closed" + raise RegistryClosedError(msg) + + @staticmethod + def _validate_name(name: str) -> None: + if _NAME_PATTERN.fullmatch(name) is None: + msg = f"invalid registry name {name!r}; expected [a-z][a-z0-9_.-]*" + raise RegistryError(msg) + + +class _BoundRegistrar: + def __init__(self, builder: _RegistryBuilder, provider: Provider) -> None: + self._builder = builder + self._provider = provider + + def add_block_kind( + self, + name: str, + kind: BlockKind, + *, + replace: bool = False, + ) -> None: + self._builder.add( + "block kind", + self._builder.block_kinds, + name, + kind, + self._provider, + replace=replace, + ) + + def add_document_parser( + self, + name: str, + parser: DocumentParser, + *, + replace: bool = False, + ) -> None: + self._builder.add_document_parser( + name, + parser, + self._provider, + replace=replace, + ) + + def add_execution_profile( + self, + name: str, + profile: ExecutionProfile, + *, + replace: bool = False, + ) -> None: + self._builder.add( + "execution profile", + self._builder.execution_profiles, + name, + profile, + self._provider, + replace=replace, + ) + + def add_output_checker( + self, + name: str, + factory: CheckerFactory, + *, + replace: bool = False, + ) -> None: + self._builder.add( + "output checker", + self._builder.output_checkers, + name, + factory, + self._provider, + replace=replace, + ) + + +def _register_builtins(registrar: Registrar) -> None: + # Import implementations only while constructing a registry. This keeps the + # foundational contracts independent of parsing and execution modules. + from .markup import MystDocumentParser, RstDocumentParser + from .runner import ExecExecutionProfile, PromptExecutionProfile + + registrar.add_block_kind( + "doctest", + BlockKind(Phase.TEST, "prompt", None), + ) + registrar.add_block_kind( + "testsetup", + BlockKind(Phase.SETUP, "exec", None), + ) + registrar.add_block_kind( + "testcleanup", + BlockKind(Phase.CLEANUP, "exec", None), + ) + registrar.add_block_kind( + "testcode", + BlockKind(Phase.TEST, "exec", "testoutput"), + ) + registrar.add_document_parser("rst", RstDocumentParser()) + registrar.add_document_parser("myst", MystDocumentParser()) + registrar.add_execution_profile("prompt", PromptExecutionProfile()) + registrar.add_execution_profile("exec", ExecExecutionProfile()) + registrar.add_output_checker("stdlib", doctest.OutputChecker) + + +def build_registry( + contributors: t.Iterable[Contributor] = (), +) -> RegistrySnapshot: + """Build and freeze a deterministic capability snapshot. + + Built-ins retain their declaration order and contributors are applied once + in the order supplied by the host. + + Parameters + ---------- + contributors : iterable of Contributor + Explicit host-discovered contributions. + + Returns + ------- + RegistrySnapshot + Immutable registry mappings and attributed records. + + Raises + ------ + RegistryCollisionError + If a contribution replaces a capability without explicit permission. + + Examples + -------- + >>> tuple(build_registry().output_checkers) + ('stdlib',) + """ + builder = _RegistryBuilder() + try: + _register_builtins(builder.registrar(_BUILTIN_PROVIDER)) + for contributor in contributors: + contributor.contribute(builder.registrar(contributor.provider)) + return builder.freeze() + finally: + builder.close() diff --git a/src/doctest_core/runner.py b/src/doctest_core/runner.py new file mode 100644 index 0000000..a7bc4a8 --- /dev/null +++ b/src/doctest_core/runner.py @@ -0,0 +1,601 @@ +"""Fresh materialization and host-neutral group execution.""" + +from __future__ import annotations +import __future__ + +import contextlib +import doctest +import io +import sys +import traceback +import types +import typing as t + +from packaging.specifiers import SpecifierSet +from packaging.version import Version + +from .contracts import ( + ExceptionPolicy, + ExecutionRuntime, + RuntimeOutcome, + RuntimeSettings, +) +from .model import ( + BlockResult, + Counts, + Errored, + Failed, + Failure, + GroupPlan, + GroupResult, + Passed, + Phase, + ProjectedBlock, + Skipped, + SkipReason, +) +from .settings import RunSettings + +if t.TYPE_CHECKING: + from doctest import _Out + + from .registry import RegistrySnapshot + + +class DefaultExceptionPolicy: + """Preserve the standard-library doctest exception boundary.""" + + def should_propagate(self, error: BaseException) -> bool: + """Return whether ``error`` must escape doctest handling. + + >>> DefaultExceptionPolicy().should_propagate(KeyboardInterrupt()) + True + >>> DefaultExceptionPolicy().should_propagate(ValueError()) + False + """ + return isinstance(error, KeyboardInterrupt) + + def is_abort(self, error: BaseException) -> bool: + """Return whether ``error`` must outrank block and cleanup results. + + >>> DefaultExceptionPolicy().is_abort(SystemExit()) + False + >>> DefaultExceptionPolicy().is_abort(ValueError()) + False + """ + return isinstance(error, KeyboardInterrupt) + + +def _results(failed: int, attempted: int, skipped: int) -> doctest.TestResults: + """Construct ``TestResults`` across CPython's supported shapes.""" + try: + constructor = t.cast(t.Any, doctest.TestResults) + return t.cast( + doctest.TestResults, + constructor(failed, attempted, skipped=skipped), + ) + except TypeError: + return doctest.TestResults(failed, attempted) + + +def _effective_flags(defaults: int, options: t.Mapping[int, bool]) -> int: + """Apply per-example boolean overrides to an option bitmask.""" + flags = defaults + for flag, enabled in options.items(): + if enabled: + flags |= flag + else: + flags &= ~flag + return flags + + +def _compile_flags(globs: t.Mapping[str, t.Any]) -> int: + """Return future-feature compiler flags already active in ``globs``. + + >>> _compile_flags({}) + 0 + """ + flags = 0 + for name in __future__.all_feature_names: + feature: t.Any = getattr(__future__, name) + compiler_flag: int = feature.compiler_flag + if globs.get(name) is feature: + flags |= compiler_flag + return flags + + +def _captured_output(stream: io.StringIO) -> str: + r"""Return captured stdout with doctest's implied trailing newline. + + >>> stream = io.StringIO("partial") + >>> _captured_output(stream) + 'partial\n' + """ + output = stream.getvalue() + if output and not output.endswith("\n"): + return f"{output}\n" + return output + + +class _CollectingRunner(doctest.DocTestRunner): + """Stock prompt runner with pytest-neutral failure collection.""" + + def __init__(self, settings: RuntimeSettings) -> None: + super().__init__( + checker=settings.checker, + optionflags=settings.optionflags, + ) + self.original_optionflags = settings.optionflags + self.continue_on_failure = settings.continue_on_failure + self.exception_policy = settings.exception_policy + self.recorded_failures: list[Failure] = [] + + def report_failure( + self, + out: _Out, + test: doctest.DocTest, + example: doctest.Example, + got: str, + ) -> None: + """Retain a comparison failure for the embedding host.""" + del out + self.recorded_failures.append(doctest.DocTestFailure(test, example, got)) + if not self.continue_on_failure: + self.optionflags |= doctest.FAIL_FAST + + def report_unexpected_exception( + self, + out: _Out, + test: doctest.DocTest, + example: doctest.Example, + exc_info: tuple[ + type[BaseException], + BaseException, + types.TracebackType, + ], + ) -> None: + """Retain Python failures while propagating host-owned exceptions.""" + del out + if self.exception_policy.should_propagate(exc_info[1]): + raise exc_info[1] + self.recorded_failures.append( + doctest.UnexpectedException(test, example, exc_info), + ) + if not self.continue_on_failure: + self.optionflags |= doctest.FAIL_FAST + + +class PromptRuntime: + """Execute prompt-form examples on CPython's untouched example loop.""" + + def __init__(self, settings: RuntimeSettings) -> None: + self.runner = _CollectingRunner(settings) + + def run(self, test: doctest.DocTest) -> RuntimeOutcome: + """Run a stock doctest without clearing its shared globals.""" + self.runner.recorded_failures.clear() + results = self.runner.run( + test, + out=lambda _: None, + clear_globs=False, + ) + failures = tuple(self.runner.recorded_failures) + skipped = getattr(results, "skipped", None) + if skipped is None: + skipped = _prompt_skipped(test, failures, self.runner) + return RuntimeOutcome(results, failures, skipped) + + +def _prompt_skipped( + test: doctest.DocTest, + failures: tuple[Failure, ...], + runner: _CollectingRunner, +) -> int: + """Reconstruct reached skips on CPython versions that do not report them.""" + stop_index: int | None = None + if failures: + first_failure = test.examples.index(failures[0].example) + if not runner.continue_on_failure: + stop_index = first_failure + else: + for index in range(first_failure, len(test.examples)): + example = test.examples[index] + flags = _effective_flags( + runner.original_optionflags, + example.options, + ) + if flags & doctest.SKIP: + continue + if flags & doctest.FAIL_FAST: + stop_index = index + break + reached = test.examples if stop_index is None else test.examples[: stop_index + 1] + return sum( + bool( + _effective_flags(runner.original_optionflags, example.options) + & doctest.SKIP + ) + for example in reached + ) + + +class ExecRuntime: + """Execute prompt-free Sphinx blocks with doctest comparison semantics.""" + + def __init__(self, settings: RuntimeSettings) -> None: + self.settings = settings + + def run(self, test: doctest.DocTest) -> RuntimeOutcome: + """Run examples in ``exec`` mode against the test's live mapping.""" + failures: list[Failure] = [] + attempted = 0 + skipped = 0 + for index, example in enumerate(test.examples): + attempted += 1 + flags = _effective_flags(self.settings.optionflags, example.options) + if flags & doctest.SKIP: + skipped += 1 + continue + got_stream = io.StringIO() + exc_info: ( + tuple[ + type[BaseException], + BaseException, + types.TracebackType | None, + ] + | None + ) = None + try: + code = compile( + example.source, + f"", + "exec", + _compile_flags(test.globs), + dont_inherit=True, + ) + with contextlib.redirect_stdout(got_stream): + # Doctests execute author-provided Python by definition. + exec(code, test.globs) # noqa: S102 + except BaseException as error: + if self.settings.exception_policy.should_propagate(error): + raise + traceback_head = error.__traceback__ + exc_info = ( + type(error), + error, + None if traceback_head is None else traceback_head.tb_next, + ) + + got = _captured_output(got_stream) + failure = self._compare(test, example, got, exc_info, flags) + if failure is not None: + failures.append(failure) + if not self.settings.continue_on_failure or flags & doctest.FAIL_FAST: + break + return RuntimeOutcome( + _results(len(failures), attempted, skipped), + tuple(failures), + skipped, + ) + + def _compare( + self, + test: doctest.DocTest, + example: doctest.Example, + got: str, + exc_info: tuple[ + type[BaseException], + BaseException, + types.TracebackType | None, + ] + | None, + flags: int, + ) -> Failure | None: + """Return a stock failure object when one example does not match.""" + if exc_info is not None: + if example.exc_msg is None: + return doctest.UnexpectedException( + test, + example, + t.cast(t.Any, exc_info), + ) + formatted = traceback.format_exception_only(exc_info[0], exc_info[1]) + if issubclass(exc_info[0], SyntaxError): + prefixes = ( + f"{exc_info[0].__qualname__}:", + f"{exc_info[0].__module__}.{exc_info[0].__qualname__}:", + ) + message_index = next( + index + for index, line in enumerate(formatted) + if line.startswith(prefixes) + ) + formatted = formatted[message_index:] + exc_msg = "".join(formatted) + if self.settings.checker.check_output(example.exc_msg, exc_msg, flags): + return None + if flags & doctest.IGNORE_EXCEPTION_DETAIL: + expected = _strip_exception_details(example.exc_msg) + actual = _strip_exception_details(exc_msg) + if self.settings.checker.check_output(expected, actual, flags): + return None + traceback_text = "".join(traceback.format_exception(*exc_info)) + return doctest.DocTestFailure(test, example, got + traceback_text) + if example.exc_msg is not None: + return doctest.DocTestFailure(test, example, got) + if self.settings.checker.check_output(example.want, got, flags): + return None + return doctest.DocTestFailure(test, example, got) + + +def _strip_exception_details(message: str) -> str: + r"""Retain only the exception name for detail-insensitive comparison. + + >>> _strip_exception_details("package.Error: detail\n") + 'Error' + """ + line = message.split("\n", 1)[0] + name = line.split(":", 1)[0] + return name.rsplit(".", 1)[-1] + + +class PromptExecutionProfile: + """Factory for the vanilla prompt runtime.""" + + def open( + self, + settings: RuntimeSettings, + ) -> contextlib.AbstractContextManager[ExecutionRuntime]: + """Return an attempt-local prompt runtime.""" + return contextlib.nullcontext(PromptRuntime(settings)) + + +class ExecExecutionProfile: + """Factory for prompt-free ``testcode`` and phase blocks.""" + + def open( + self, + settings: RuntimeSettings, + ) -> contextlib.AbstractContextManager[ExecutionRuntime]: + """Return an attempt-local exec runtime.""" + return contextlib.nullcontext(ExecRuntime(settings)) + + +def _expected_enabled( + block: ProjectedBlock, + globs: dict[str, t.Any], +) -> bool: + """Evaluate the paired output's gates against the live group mapping.""" + expected = block.expected + if expected is None: + return False + if expected.skipif is not None and bool(eval(expected.skipif, globs)): + return False + return expected.pyversion is None or _version_allowed(expected.pyversion) + + +def _exception_message(want: str) -> str | None: + r"""Extract doctest's expected exception tail from paired output. + + >>> _exception_message( + ... 'Traceback (most recent call last):\n...\nValueError: bad\n' + ... ) + 'ValueError: bad\n' + >>> _exception_message('ordinary output\n') is None + True + """ + match = doctest.DocTestParser._EXCEPTION_RE.match(want) # type: ignore[attr-defined] + return match.group("msg") if match is not None else None + + +def materialize( + block: ProjectedBlock, + globs: dict[str, t.Any], + *, + expected_enabled: bool = True, +) -> doctest.DocTest: + r"""Build fresh stock ``Example`` and ``DocTest`` objects for an attempt. + + >>> import doctest + >>> type(doctest.Example("pass\n", "")) is doctest.Example + True + """ + examples: list[doctest.Example] = [] + for index, recipe in enumerate(block.examples): + options = dict(block.options) + want = recipe.want + exc_msg = recipe.exc_msg + if block.expected is not None: + if expected_enabled: + options.update(block.expected.options) + options[doctest.DONT_ACCEPT_BLANKLINE] = True + want = block.expected.text + exc_msg = _exception_message(want) + else: + want = "" + exc_msg = None + options.update(recipe.options) + examples.append( + doctest.Example( + source=recipe.source, + want=want, + exc_msg=exc_msg, + lineno=recipe.lineno, + indent=recipe.indent, + options=options, + ), + ) + if block.expected is not None and index == 0: + break + test = doctest.DocTest( + examples, + globs, + block.name, + block.filename, + block.lineno, + block.docstring, + ) + test.globs = globs + return test + + +def reset_globs( + plan: GroupPlan, + globs: dict[str, t.Any], + *, + extraglobs: t.Mapping[str, t.Any] | None = None, +) -> None: + """Clear and reseed one canonical group mapping in place. + + >>> mapping = {"old": True} + >>> reset_globs(GroupPlan("default", (), {"seed": 1}), mapping) + >>> mapping + {'seed': 1, '__name__': '__main__'} + """ + globs.clear() + globs.update(plan.seed) + if extraglobs is not None: + globs.update(extraglobs) + globs.setdefault("__name__", "__main__") + + +def _version_allowed(specifier: str) -> bool: + """Return whether the current interpreter satisfies a PEP 440 specifier.""" + version = Version(".".join(str(part) for part in sys.version_info[:3])) + return version in SpecifierSet(specifier) + + +def _block_gate( + block: ProjectedBlock, + globs: dict[str, t.Any], +) -> SkipReason | None: + """Evaluate one block gate at the execution boundary.""" + if block.skipif is not None and bool(eval(block.skipif, globs)): + return SkipReason("skipif", block.skipif) + if block.pyversion is not None and not _version_allowed(block.pyversion): + return SkipReason("pyversion", block.pyversion) + return None + + +def _run_block( + block: ProjectedBlock, + globs: dict[str, t.Any], + runtime: ExecutionRuntime, + checker: doctest.OutputChecker, + settings: RunSettings, +) -> BlockResult: + """Gate, materialize, and run one projected block.""" + try: + gate = _block_gate(block, globs) + if gate is not None: + return Skipped(block, Counts(0, 0, 0), gate) + expected_enabled = _expected_enabled(block, globs) + test = materialize(block, globs, expected_enabled=expected_enabled) + outcome = runtime.run(test) + # The exception policy decides which host and process outcomes propagate. + except BaseException as error: # noqa: BLE001 + return Errored(block, error) + counts = Counts( + outcome.results.failed, + outcome.results.attempted, + outcome.skipped, + ) + if outcome.failures: + return Failed(block, counts, outcome.failures, checker) + if test.examples and outcome.skipped == len(test.examples): + return Skipped( + block, + counts, + SkipReason("inline-flag", "SKIP"), + ) + return Passed(block, counts) + + +def run_group( + plan: GroupPlan, + globs: dict[str, t.Any], + *, + settings: RunSettings | None = None, + registry: RegistrySnapshot | None = None, + exception_policy: ExceptionPolicy | None = None, +) -> GroupResult: + """Run one group attempt with setup/test/cleanup phase semantics.""" + if registry is None: + from .registry import build_registry + + registry = build_registry() + settings = settings or RunSettings() + exception_policy = exception_policy or DefaultExceptionPolicy() + checker_registration = registry.output_checkers[settings.checker_name] + results: list[BlockResult] = [] + primary: BaseException | None = None + secondary: list[BaseException] = [] + body_failed = False + stop_after_failure = not settings.continue_on_failure or bool( + settings.optionflags & doctest.FAIL_FAST + ) + + with contextlib.ExitStack() as stack: + runtimes: dict[str, ExecutionRuntime] = {} + checkers: dict[str, doctest.OutputChecker] = {} + for profile_name in dict.fromkeys(block.profile_name for block in plan.blocks): + profile = registry.execution_profiles[profile_name].value + checker = checker_registration.value() + runtime_settings = RuntimeSettings( + optionflags=settings.optionflags, + continue_on_failure=settings.continue_on_failure, + checker=checker, + exception_policy=exception_policy, + ) + checkers[profile_name] = checker + runtimes[profile_name] = stack.enter_context(profile.open(runtime_settings)) + + setup_failed = False + for phase in (Phase.SETUP, Phase.TEST): + if phase is Phase.TEST and setup_failed: + break + for block in (item for item in plan.blocks if item.phase is phase): + result = _run_block( + block, + globs, + runtimes[block.profile_name], + checkers[block.profile_name], + settings, + ) + results.append(result) + if isinstance(result, Errored): + primary = result.error + body_failed = True + setup_failed = phase is Phase.SETUP + break + if isinstance(result, Failed): + body_failed = True + setup_failed = phase is Phase.SETUP + if setup_failed or stop_after_failure: + break + if primary is not None or setup_failed: + break + + for block in (item for item in plan.blocks if item.phase is Phase.CLEANUP): + result = _run_block( + block, + globs, + runtimes[block.profile_name], + checkers[block.profile_name], + settings, + ) + results.append(result) + if isinstance(result, Errored): + if exception_policy.is_abort(result.error): + if primary is None or not exception_policy.is_abort(primary): + if primary is not None: + secondary.append(primary) + primary = result.error + else: + secondary.append(result.error) + elif primary is None and not body_failed: + primary = result.error + else: + secondary.append(result.error) + + return GroupResult(plan.group, tuple(results), primary, tuple(secondary)) diff --git a/src/doctest_core/settings.py b/src/doctest_core/settings.py new file mode 100644 index 0000000..7ddb6a0 --- /dev/null +++ b/src/doctest_core/settings.py @@ -0,0 +1,56 @@ +"""Immutable settings resolved before doctest-core pipeline stages.""" + +from __future__ import annotations + +import typing as t + + +class ParseSettings(t.NamedTuple): + """Settings for markup parsing and extraction. + + Attributes + ---------- + suppressed_diagnostics : frozenset of str + Diagnostic codes omitted from the returned parse result. + + >>> ParseSettings().suppressed_diagnostics + frozenset({'docutils.unknown-role'}) + """ + + suppressed_diagnostics: frozenset[str] = frozenset({"docutils.unknown-role"}) + + +class ProjectionSettings(t.NamedTuple): + """Settings for pure block-to-group projection. + + Attributes + ---------- + ungrouped : {"default", "block"} + Put unlabelled blocks in the shared ``default`` group or isolate them. + + >>> ProjectionSettings().ungrouped + 'default' + """ + + ungrouped: t.Literal["default", "block"] = "default" + + +class RunSettings(t.NamedTuple): + """Serializable policy for one doctest run. + + Attributes + ---------- + optionflags : int + Runner-level doctest option bitmask. + continue_on_failure : bool + Retain later failures from the same block after a mismatch. + checker_name : str + Output-checker registration selected for the run. + + >>> (RunSettings().continue_on_failure, RunSettings().checker_name) + (True, 'stdlib') + """ + + optionflags: int = 0 + continue_on_failure: bool = True + checker_name: str = "stdlib" diff --git a/src/doctest_docutils.py b/src/doctest_docutils.py index f8f2cde..50bea8b 100644 --- a/src/doctest_docutils.py +++ b/src/doctest_docutils.py @@ -3,27 +3,30 @@ from __future__ import annotations import doctest -import linecache import logging import os import pathlib -import pprint import re import sys +import types import typing as t import docutils -from docutils import nodes -from docutils.parsers.rst import Directive, directives +from docutils.parsers.rst import directives from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.version import Version -from docutils_compat import findall +import doctest_core +from doctest_core.markup import ( + DoctestDirective as _CoreDoctestDirective, + MockTabDirective as _CoreMockTabDirective, + TestcleanupDirective as _CoreTestcleanupDirective, + TestsetupDirective as _CoreTestsetupDirective, + _TestDirective as _CoreTestDirective, +) if t.TYPE_CHECKING: - import types - - from docutils.nodes import Node, TextElement + from docutils.nodes import Node logger = logging.getLogger(__name__) @@ -53,13 +56,10 @@ def is_allowed_version(version: str, spec: str) -> bool: return Version(version) in SpecifierSet(spec) -class TestDirective(Directive): - """Base class for doctest-related directives.""" +class TestDirective(_CoreTestDirective): + """Compatibility base for doctest-related directives.""" - has_content = True - required_arguments = 0 - optional_arguments = 1 - final_argument_whitespace = True + __test__ = False def get_source_info(self) -> tuple[str, int]: """Get source and line number.""" @@ -69,115 +69,21 @@ def set_source_info(self, node: Node) -> None: """Set source and line number to the node.""" node.source, node.line = self.get_source_info() - def run(self) -> list[Node]: - """Run docutils test directive.""" - # use ordinary docutils nodes for test code: they get special attributes - # so that our builder recognizes them, and the other builders are happy. - code = "\n".join(self.content) - test = None - - logger.debug(f"directive run: self.name {self.name}") - if self.name == "doctest": - if "" in code: - # convert s to ordinary blank lines for presentation - test = code - code = blankline_re.sub("", code) - if ( - doctestopt_re.search(code) - and "no-trim-doctest-flags" not in self.options - ): - if not test: - test = code - code = doctestopt_re.sub("", code) - nodetype: type[TextElement] = nodes.literal_block - if self.name in {"testsetup", "testcleanup"} or "hide" in self.options: - nodetype = nodes.comment - if self.arguments: - groups = [x.strip() for x in self.arguments[0].split(",")] - else: - groups = ["default"] - node = nodetype(code, code, testnodetype=self.name, groups=groups) - self.set_source_info(node) - if test is not None: - # only save if it differs from code - node["test"] = test - if self.name == "doctest": - node["language"] = "pycon3" - node["options"] = {} - if self.name in ("doctest") and "options" in self.options: - # parse doctest-like output comparison flags - option_strings = self.options["options"].replace(",", " ").split() - for option in option_strings: - prefix, option_name = option[0], option[1:] - if prefix not in "+-": - self.state.document.reporter.warning( - f"missing '+' or '-' in '{option}' option.", - line=self.lineno, - ) - continue - if option_name not in doctest.OPTIONFLAGS_BY_NAME: - self.state.document.reporter.warning( - f"'{option_name}' is not a valid option.", - line=self.lineno, - ) - continue - flag = doctest.OPTIONFLAGS_BY_NAME[option[1:]] - node["options"][flag] = option[0] == "+" - if self.name == "doctest" and "pyversion" in self.options: - try: - spec = self.options["pyversion"] - python_version = ".".join([str(v) for v in sys.version_info[:3]]) - if not is_allowed_version(spec, python_version): - flag = doctest.OPTIONFLAGS_BY_NAME["SKIP"] - node["options"][flag] = True # Skip the test - except InvalidSpecifier: - self.state.document.reporter.warning( - f"'{spec}' is not a valid pyversion option", - line=self.lineno, - ) - if "skipif" in self.options: - node["skipif"] = self.options["skipif"] - if "trim-doctest-flags" in self.options: - node["trim_flags"] = True - elif "no-trim-doctest-flags" in self.options: - node["trim_flags"] = False - return [node] - - -class TestsetupDirective(TestDirective): - """Test setup directive.""" - - option_spec: t.ClassVar = {"skipif": directives.unchanged_required} - -class TestcleanupDirective(TestDirective): - """Test cleanup directive.""" +class TestsetupDirective(_CoreTestsetupDirective, TestDirective): + """Compatibility name for the core ``testsetup`` directive.""" - option_spec: t.ClassVar = {"skipif": directives.unchanged_required} +class TestcleanupDirective(_CoreTestcleanupDirective, TestDirective): + """Compatibility name for the core ``testcleanup`` directive.""" -class DoctestDirective(TestDirective): - """Doctest directive.""" - option_spec: t.ClassVar = { - "no-trim-doctest-flags": directives.flag, - "options": directives.unchanged, - "pyversion": directives.unchanged_required, - "skipif": directives.unchanged_required, - "trim-doctest-flags": directives.flag, - } +class DoctestDirective(_CoreDoctestDirective, TestDirective): + """Compatibility name for the core ``doctest`` directive.""" -class MockTabDirective(TestDirective): - """Mock tab directive.""" - - def run(self) -> list[Node]: - """Parse a mock-tabs directive.""" - self.assert_has_content() - - content = nodes.container("", is_div=True, classes=["tab-content"]) - self.state.nested_parse(self.content, self.content_offset, content) - return [content] +class MockTabDirective(_CoreMockTabDirective, TestDirective): + """Compatibility name for the core mock tab directive.""" def setup() -> dict[str, t.Any]: @@ -188,32 +94,19 @@ def setup() -> dict[str, t.Any]: # Third party mock directive: sphinx-inline-tabs @ 2022.01.02.beta11 directives.register_directive("tab", MockTabDirective) + doctest_core.ensure_directives_registered() return {"version": docutils.__version__, "parallel_read_safe": True} -# For backward compatibility, a global instance of a DocTestRunner -# class, updated by testmod. -master = None +# For backward compatibility, a global runner updated by ``testdocutils``. +master: doctest.DocTestRunner | None = None parser = doctest.DocTestParser() -_DIRECTIVES_READY = False -_REQUIRED_DIRECTIVES = ("doctest", "testsetup", "testcleanup", "tab") - - -def _directive_registry() -> dict[str, t.Any]: - """Return docutils directive registry with typing info.""" - return t.cast(dict[str, t.Any], directives.__dict__["_directives"]) def _ensure_directives_registered() -> None: - """Register doctest-related directives once per interpreter.""" - global _DIRECTIVES_READY - registry = _directive_registry() - missing = any(name not in registry for name in _REQUIRED_DIRECTIVES) - if _DIRECTIVES_READY and not missing: - return - setup() - _DIRECTIVES_READY = True + """Register missing core directives without replacing another owner.""" + doctest_core.ensure_directives_registered() class DocTestFinderNameDoesNotExist(ValueError): @@ -271,12 +164,6 @@ def find( if name is None: raise DocTestFinderNameDoesNotExist(string=string) - # No access to a loader, so assume it's a normal - # filesystem path - source_lines = linecache.getlines(name) or None - if not source_lines: - source_lines = None - # Initialize globals, and merge in extraglobs. globs = {} if globs is None else globs.copy() if extraglobs is not None: @@ -288,12 +175,7 @@ def find( source_path: pathlib.Path | None = ( pathlib.Path(name) if name is not None else None ) - self._find(tests, string, name, source_lines, globs, {}, source_path) - # Sort the tests by alpha order of names, for consistency in - # verbose-mode output. This was a feature of doctest in Pythons - # <= 2.3 that got lost by accident in 2.4. It was repaired in - # 2.4.4 and 2.5. - tests.sort() + self._find(tests, string, name, None, globs, {}, source_path) return tests def _find( @@ -308,98 +190,60 @@ def _find( ) -> None: """Find tests for the given string, and add them to `tests`.""" if self._verbose: - logger.info(f"Finding tests in {name}") + logger.info("finding tests in %s", name) # If we've already processed this string, then ignore it. if id(string) in seen: return seen[id(string)] = 1 - - # Find a test for this string, and add it to the list of tests. - logger.debug( - "_find({})".format( - pprint.pformat( - { - "tests": tests, - "string": string, - "name": name, - "source_lines": source_lines, - "globs": globs, - "seen": seen, - }, - ), - ), - ) - ext = pathlib.Path(name).suffix - logger.debug(f"parse, ext: {ext}") - if ext == ".md": - import myst_parser.parsers.docutils_ - from myst_parser.config.main import MdParserConfig - from myst_parser.mdit_to_docutils.base import ( - DocutilsRenderer, - make_document, - ) - from myst_parser.parsers.mdit import create_md_parser - - DocutilsParser = myst_parser.parsers.docutils_.Parser - config: MdParserConfig = MdParserConfig(commonmark_only=False) - md_parser = create_md_parser(config, DocutilsRenderer) - - doc = make_document( - source_path=str(source_path), - parser_cls=DocutilsParser, - ) - md_parser.options["document"] = doc - md_parser.render(string) - else: - import docutils.utils - from docutils.frontend import OptionParser - from docutils.parsers.rst import Parser - - parser = Parser() - settings = OptionParser(components=(Parser,)).get_default_values() - - doc = docutils.utils.new_document( - source_path=str(source_path), - settings=settings, - ) - parser.parse(string, doc) - - def condition(node: Node) -> bool: - return ( - ( - isinstance(node, (nodes.literal_block, nodes.comment)) - and "testnodetype" in node - ) - or ( - isinstance(node, nodes.literal_block) - and re.match( - doctest.DocTestParser._EXAMPLE_RE, # type:ignore - node.astext(), - ) - is not None - ) - or isinstance(node, nodes.doctest_block) - ) - - for idx, node in enumerate(findall(doc)(condition)): - logger.debug(f"() node: {node.astext()}") - assert isinstance(node, nodes.Element) - test_name = node.get("groups") - if isinstance(test_name, list): - test_name = test_name[0] - if test_name is None or test_name == "default": - test_name = f"{name}[{idx}]" - logger.debug(f"() node: {test_name}") + del source_lines + parse_path = source_path or pathlib.Path(name) + if parse_path.suffix not in {".md", ".rst", ".txt"}: + parse_path = parse_path.with_suffix(".rst") + parsed = doctest_core.parse_document(string, parse_path) + for block in parsed.blocks: + test_name = self._compatibility_name(block, name) test = self._get_test( - string=node.astext(), + string=block.source, name=test_name, - filename=name, + filename=str(block.path), globs=globs, - source_lines=[str(node.line)], + source_lines=[ + str(0 if block.line is None else max(block.line - 1, 0)), + ], ) - if test is not None: - tests.append(test) + self._apply_block_options(test, block) + tests.append(test) + + @staticmethod + def _compatibility_name(block: doctest_core.ParsedBlock, name: str) -> str: + """Reproduce the legacy first-group and anonymous naming scheme.""" + group = block.groups[0] if block.groups else None + if group is None or group == "default": + return f"{name}[{block.block_ordinal}]" + return group + + @staticmethod + def _apply_block_options( + test: doctest.DocTest, + block: doctest_core.ParsedBlock, + ) -> None: + """Merge directive policy into each stock example's inline options.""" + block_options = dict(block.options) + if block.pyversion is not None: + version = ".".join(str(value) for value in sys.version_info[:3]) + try: + if not is_allowed_version(version, block.pyversion): + block_options[doctest.SKIP] = True + except InvalidSpecifier: + logger.warning( + "invalid pyversion option", + extra={"doctest_source_file": test.filename}, + ) + for example in test.examples: + options = block_options.copy() + options.update(example.options) + example.options = options def _get_test( self, @@ -416,9 +260,137 @@ def _get_test( return self._parser.get_doctest(string, globs, name, filename, lineno) +def _direct_plan( + plan: doctest_core.GroupPlan, + *, + filename: str, + parser: doctest.DocTestParser, +) -> doctest_core.GroupPlan: + """Adapt a typed plan to the compatibility facade's names and parser.""" + blocks: list[doctest_core.ProjectedBlock] = [] + for block in plan.blocks: + block_name = ( + plan.group + if plan.group != "default" + else f"{filename}[{block.block_ordinal}]" + ) + examples = block.examples + if block.profile_name == "prompt": + parsed_test = parser.get_doctest( + block.docstring, + {}, + block_name, + block.filename, + 0, + ) + examples = tuple( + doctest_core.ExampleRecipe( + source=example.source, + want=example.want, + exc_msg=example.exc_msg, + lineno=example.lineno, + indent=example.indent, + options=types.MappingProxyType(dict(example.options)), + ) + for example in parsed_test.examples + ) + blocks.append(block._replace(name=block_name, examples=examples)) + return plan._replace(blocks=tuple(blocks)) + + +def _report_failure( + runner: doctest.DocTestRunner, + failure: doctest.DocTestFailure | doctest.UnexpectedException, +) -> None: + """Render a core failure through the stock direct runner hooks.""" + if isinstance(failure, doctest.DocTestFailure): + runner.report_failure( + sys.stdout.write, + failure.test, + failure.example, + failure.got, + ) + return + runner.report_unexpected_exception( + sys.stdout.write, + failure.test, + failure.example, + failure.exc_info, + ) + + +def _record_statistics( + runner: doctest.DocTestRunner, + *, + name: str, + failures: int, + attempted: int, + skipped: int, +) -> None: + """Populate CPython's version-specific summary bookkeeping.""" + runner.failures += failures + runner.tries += attempted + stats = getattr(runner, "_stats", None) + if isinstance(stats, dict): + typed_stats = t.cast(dict[str, tuple[int, int, int]], stats) + old_failures, old_attempted, old_skipped = typed_stats.get( + name, + (0, 0, 0), + ) + typed_stats[name] = ( + old_failures + failures, + old_attempted + attempted, + old_skipped + skipped, + ) + runner.skips += skipped # type: ignore[attr-defined] + return + name_to_counts = t.cast( + dict[str, tuple[int, int]], + runner.__dict__["_name2ft"], + ) + old_failures, old_attempted = name_to_counts.get(name, (0, 0)) + name_to_counts[name] = ( + old_failures + failures, + old_attempted + attempted, + ) + + +def _consume_result( + runner: doctest.DocTestRunner, + result: doctest_core.GroupResult, +) -> None: + """Project one core group result onto the direct doctest runner.""" + for block_result in result.blocks: + failures = 0 + attempted = 0 + skipped = 0 + if isinstance(block_result, doctest_core.Failed): + failures = block_result.counts.failed + attempted = block_result.counts.attempted + skipped = block_result.counts.skipped + for failure in block_result.failures: + _report_failure(runner, failure) + elif block_result.block.phase is doctest_core.Phase.TEST and isinstance( + block_result, (doctest_core.Passed, doctest_core.Skipped) + ): + attempted = block_result.counts.attempted + skipped = block_result.counts.skipped + _record_statistics( + runner, + name=block_result.block.name, + failures=failures, + attempted=attempted, + skipped=skipped, + ) + if result.primary is not None: + raise result.primary + + class TestDocutilsPackageRelativeError(Exception): """Raise when doctest_docutils is called for package not relative to module.""" + __test__ = False + def __init__(self) -> None: super().__init__( "Package may only be specified for module-relative paths.", @@ -451,7 +423,7 @@ def testdocutils( # Keep the absolute file paths. This is needed for Include directies to work. # The absolute path will be applied to source_path when creating the docutils doc. _ensure_directives_registered() - text, _ = doctest._load_testfile( # type: ignore + text, source_filename = doctest._load_testfile( # type: ignore filename, package, module_relative, @@ -469,9 +441,6 @@ def testdocutils( if "__name__" not in globs: globs["__name__"] = "__main__" - # Find, parse, and run all tests in the given module. - finder = DocutilsDocTestFinder() - runner: doctest.DebugRunner | doctest.DocTestRunner if raise_on_error: @@ -479,8 +448,38 @@ def testdocutils( else: runner = doctest.DocTestRunner(verbose=verbose, optionflags=optionflags) - for test in finder.find(text, filename, globs=globs, extraglobs=extraglobs): - runner.run(test) + source_path = pathlib.Path(source_filename) + if source_path.suffix not in {".md", ".rst", ".txt"}: + source_path = source_path.with_suffix(".rst") + registry = doctest_core.build_registry() + parsed = doctest_core.parse_document( + text, + source_path, + registry=registry, + ) + plans = doctest_core.project( + parsed, + document_name=name, + registry=registry, + seed=globs, + ) + settings = doctest_core.RunSettings( + optionflags=optionflags, + continue_on_failure=( + not raise_on_error and not bool(optionflags & doctest.FAIL_FAST) + ), + ) + for plan in plans: + direct_plan = _direct_plan(plan, filename=filename, parser=parser) + live_globs: dict[str, t.Any] = {} + doctest_core.reset_globs(direct_plan, live_globs) + result = doctest_core.run_group( + direct_plan, + live_globs, + settings=settings, + registry=registry, + ) + _consume_result(runner, result) if report: runner.summarize() @@ -490,16 +489,24 @@ def testdocutils( else: master.merge(runner) + if hasattr(runner, "skips"): + constructor = t.cast(t.Any, doctest.TestResults) + return t.cast( + doctest.TestResults, + constructor( + runner.failures, + runner.tries, + skipped=runner.skips, + ), + ) return doctest.TestResults(runner.failures, runner.tries) -def _test() -> int: - """Execute doctest module via CLI. +testdocutils.__test__ = False # type: ignore[attr-defined] - Port changes from standard library at 3.10: - - Sets up logging.basicLogging(level=logging.DEBUG) w/ args.verbose - """ +def _test() -> int: + """Execute doctest module via CLI.""" import argparse p = argparse.ArgumentParser(description="doctest runner") @@ -508,7 +515,7 @@ def _test() -> int: "--verbose", action="store_true", default=False, - help="logger.debug very verbose output for all tests", + help="list tested groups in the final summary", ) p.add_argument( "--log-level", diff --git a/src/pytest_doctest_docutils.py b/src/pytest_doctest_docutils.py index 13c2db0..34cf1ad 100644 --- a/src/pytest_doctest_docutils.py +++ b/src/pytest_doctest_docutils.py @@ -1,121 +1,188 @@ -"""pytest plugin for doctest w/ reStructuredText and markdown. - -.. seealso:: - - - http://www.sphinx-doc.org/en/stable/ext/doctest.html - - https://github.com/sphinx-doc/sphinx/blob/master/sphinx/ext/doctest.py - - This is a derivative of my PR https://github.com/thisch/pytest-sphinx/pull/38 to - pytest-sphinx (BSD 3-clause), 2022-09-03. -""" +"""Pytest host adapter for the typed doctest core.""" from __future__ import annotations import bdb +import collections.abc import doctest -import io -import logging -import sys +import pathlib +import traceback import typing as t +import weakref -import _pytest import pytest -from _pytest import outcomes -from _pytest.outcomes import OutcomeException -from doctest_docutils import DocutilsDocTestFinder, _ensure_directives_registered +import _pytest_doctest_compat as compat +from doctest_core import ( + Contributor, + Errored, + Failed, + GroupPlan, + GroupResult, + Phase, + ProjectionSettings, + Provider, + Registrar, + RegistrySnapshot, + RunSettings, + Skipped, + build_registry, + parse_document, + project, + reset_globs, + run_group, +) +from doctest_core.markup import ensure_directives_registered if t.TYPE_CHECKING: - import pathlib - import types - from collections.abc import Iterable - from doctest import _Out - + from _pytest._code import ExceptionInfo + from _pytest._code.code import TerminalRepr + from _pytest.config import PytestPluginManager from _pytest.config.argparsing import Parser - from _pytest.doctest import DoctestItem -logger = logging.getLogger(__name__) +PYTEST_VERSION = tuple(int(part) for part in pytest.__version__.split(".")[:2]) +_REGISTRY_KEY: pytest.StashKey[RegistrySnapshot] = pytest.StashKey() +_FROZEN_PLUGIN_MANAGERS: weakref.WeakSet[t.Any] = weakref.WeakSet() + + +class DoctestCoreHooks: + """Hooks published by the doctest-core pytest adapter.""" + + @pytest.hookspec + def pytest_doctest_core_contributors( + self, + ) -> Contributor | collections.abc.Iterable[Contributor] | None: + """Return host-neutral contributors before collection.""" + + +class _PytestContributor: + """Use pytest's checker for the core's default checker registration.""" + + provider = Provider(name="pytest", version=pytest.__version__) + + def contribute(self, registrar: Registrar) -> None: + """Replace the stdlib checker with pytest's compatible extension.""" + registrar.add_output_checker( + "stdlib", + compat.get_checker, + replace=True, + ) + + +class _PytestExceptionPolicy: + """Classify pytest outcomes and process aborts for the core runtime.""" + + def should_propagate(self, error: BaseException) -> bool: + """Return whether pytest, rather than doctest, owns ``error``.""" + outcome_types = ( + pytest.skip.Exception, + pytest.xfail.Exception, + pytest.fail.Exception, + pytest.exit.Exception, + ) + return isinstance( + error, + (*outcome_types, KeyboardInterrupt, SystemExit, bdb.BdbQuit), + ) + + def is_abort(self, error: BaseException) -> bool: + """Return whether ``error`` must abort despite prior block outcomes. + + >>> _PytestExceptionPolicy().is_abort(KeyboardInterrupt()) + True + >>> _PytestExceptionPolicy().is_abort(ValueError()) + False + """ + return isinstance( + error, + (pytest.exit.Exception, KeyboardInterrupt, SystemExit, bdb.BdbQuit), + ) + + +def pytest_addhooks(pluginmanager: PytestPluginManager) -> None: + """Publish the contributor hook before pytest loads initial conftests.""" + pluginmanager.add_hookspecs(DoctestCoreHooks) + -# Parse pytest version for version-specific features -PYTEST_VERSION = tuple(int(x) for x in pytest.__version__.split(".")[:2]) +def pytest_plugin_registered( + plugin: object, + manager: PytestPluginManager, +) -> None: + """Reject contributor hooks registered after the host snapshot freezes. -# Lazy definition of runner class -RUNNER_CLASS = None + >>> callable(pytest_plugin_registered) + True + """ + if manager not in _FROZEN_PLUGIN_MANAGERS: + return + contributor_hook = getattr(plugin, "pytest_doctest_core_contributors", None) + if callable(contributor_hook): + plugin_name = manager.get_name(plugin) or type(plugin).__name__ + message = ( + f"pytest plugin {plugin_name!r} registered a doctest-core contributor " + "after the contribution phase closed" + ) + raise pytest.UsageError(message) def pytest_addoption(parser: Parser) -> None: - """Add options to py.test for doctest_docutils.""" + """Add doctest-docutils host options.""" group = parser.getgroup("collect") group.addoption( "--doctest-docutils-modules", action="store_true", default=False, - help="run doctest-doctests in .py modules (pass-through to pytest-doctest)", + help="run doctests in Python modules through pytest's doctest plugin", dest="doctestmodules", ) group.addoption( "--no-doctest-docutils-modules", action="store_false", - help="disable doctest-doctests in .py modules (pass-through to pytest-doctest)", + help="disable doctests in Python modules", dest="doctestmodules", ) + parser.addini( + "doctest_docutils_ungrouped", + "sharing policy for bare documentation blocks: block or default", + default="block", + ) -def pytest_configure(config: pytest.Config) -> None: - """Disable pytest.doctest to prevent running tests twice. +def _flatten_contributors(results: t.Iterable[object]) -> list[Contributor]: + """Flatten pluggy's per-implementation return values in hook order.""" + contributors: list[Contributor] = [] + for result in results: + if result is None: + continue + if hasattr(result, "contribute") and hasattr(result, "provider"): + contributors.append(t.cast(Contributor, result)) + continue + if isinstance(result, collections.abc.Iterable): + contributors.extend(t.cast(collections.abc.Iterable[Contributor], result)) + return contributors - Todo: Find a way to make these plugins cooperate without collecting twice. - """ - # Register HIDE eagerly, before collection parses any docstring. The .py - # path delegates to pytest's own DoctestModule (which never calls our - # _get_flag_lookup), so registering it here is what lets a docstring carry - # ``# doctest: +HIDE`` without raising ``invalid option`` at parse time. - _get_hide_flag() - if config.pluginmanager.has_plugin("doctest"): - config.pluginmanager.set_blocked("doctest") - - -def _unblock_doctest(config: pytest.Config) -> bool: - """Unblock doctest plugin (pytest 8.1+ only). - - Re-enables the built-in doctest plugin after it was blocked by - pytest_configure. Uses the public unblock() API introduced in pytest 8.1.0. - - Parameters - ---------- - config : pytest.Config - The pytest configuration object - - Returns - ------- - bool - True if unblocked successfully, False if API not available - """ - pm = config.pluginmanager - if PYTEST_VERSION >= (8, 1) and hasattr(pm, "unblock"): - return pm.unblock("doctest") - return False - -def pytest_unconfigure() -> None: - """Unconfigure hook for pytest-doctest-docutils.""" - global RUNNER_CLASS - - RUNNER_CLASS = None +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + """Freeze host contributions without unregistering pytest's doctest plugin.""" + doctest.register_optionflag("HIDE") + raw_hook = t.cast(t.Any, config.hook).pytest_doctest_core_contributors() + contributors = [_PytestContributor(), *_flatten_contributors(raw_hook)] + config.stash[_REGISTRY_KEY] = build_registry(contributors) + _FROZEN_PLUGIN_MANAGERS.add(config.pluginmanager) + value = config.getini("doctest_docutils_ungrouped") + if value not in {"block", "default"}: + message = "doctest_docutils_ungrouped must be 'block' or 'default'" + raise pytest.UsageError(message) def pytest_ignore_collect(collection_path: pathlib.Path) -> bool | None: - """Skip Sphinx ``_build/`` output during collection. + """Skip generated Sphinx ``_build`` trees. - pytest's default ``norecursedirs`` excludes ``build`` but not ``_build``, - so Sphinx output (which mirrors sources, broken relative includes and all) - would otherwise be collected and abort the session. - - >>> import pathlib - >>> pytest_ignore_collect(pathlib.Path("docs/_build/html/history.md")) + >>> pytest_ignore_collect(pathlib.Path("docs/_build/html/page.md")) True - >>> pytest_ignore_collect(pathlib.Path("docs/history.md")) is None + >>> pytest_ignore_collect(pathlib.Path("docs/page.md")) is None True """ if "_build" in collection_path.parts: @@ -123,276 +190,258 @@ def pytest_ignore_collect(collection_path: pathlib.Path) -> bool | None: return None -def pytest_collect_file( - file_path: pathlib.Path, - parent: pytest.Collector, -) -> DocTestDocutilsFile | _pytest.doctest.DoctestModule | None: - """Test collector for pytest-doctest-docutils.""" - config = parent.config - if file_path.suffix == ".py": - if config.option.doctestmodules and not any( - # if not any( - ( - _pytest.doctest._is_setup_py(file_path), - _pytest.doctest._is_main_py(file_path), - ), - ): - mod: DocTestDocutilsFile | _pytest.doctest.DoctestModule = ( - _pytest.doctest.DoctestModule.from_parent(parent, path=file_path) - ) - return mod - elif _is_doctest(config, file_path, parent): - return DocTestDocutilsFile.from_parent(parent, path=file_path) - return None - - def _is_doctest( config: pytest.Config, path: pathlib.Path, parent: pytest.Collector, ) -> bool: - if path.suffix in {".rst", ".md"} and parent.session.isinitpath(path): + """Return whether this adapter claims a documentation path.""" + registry = config.stash.get(_REGISTRY_KEY, None) + supported_suffixes = ( + { + suffix + for registration in registry.document_parsers.values() + for suffix in registration.value.suffixes + } + if registry is not None + else {".rst", ".md"} + ) + if path.suffix not in supported_suffixes: + return False + if parent.session.isinitpath(path): return True - globs = config.getoption("doctestglob") or ["*.rst", "*.md"] - return any(path.match(path_pattern=glob) for glob in globs) - + patterns = config.getoption("doctestglob", default=None) or ["*.rst", "*.md"] + return any(path.match(pattern) for pattern in patterns) -def _init_runner_class() -> type[doctest.DocTestRunner]: - import doctest - - class PytestDoctestRunner(doctest.DebugRunner): - """Runner to collect failures. - - Note that the out variable in this case is a list instead of a - stdout-like object. - """ - - def __init__( - self, - checker: doctest.OutputChecker | None = None, - verbose: bool | None = None, - optionflags: int = 0, - continue_on_failure: bool = True, - ) -> None: - super().__init__(checker=checker, verbose=verbose, optionflags=optionflags) - self.continue_on_failure = continue_on_failure - - def report_failure( - self, - out: _Out, - test: doctest.DocTest, - example: doctest.Example, - got: str, - ) -> None: - failure = doctest.DocTestFailure(test, example, got) - if self.continue_on_failure: - assert isinstance(out, list) - out.append(failure) - else: - raise failure - - def report_unexpected_exception( - self, - out: _Out, - test: doctest.DocTest, - example: doctest.Example, - exc_info: tuple[ - type[BaseException], - BaseException, - types.TracebackType, - ], - ) -> None: - if isinstance(exc_info[1], OutcomeException): - raise exc_info[1] - if isinstance(exc_info[1], bdb.BdbQuit): - outcomes.exit("Quitting debugger") - failure = doctest.UnexpectedException(test, example, exc_info) - if self.continue_on_failure: - assert isinstance(out, list) - out.append(failure) - else: - raise failure - - return PytestDoctestRunner - - -def _get_allow_unicode_flag() -> int: - """Register and return the ALLOW_UNICODE flag.""" - import doctest - - return doctest.register_optionflag("ALLOW_UNICODE") - - -def _get_allow_bytes_flag() -> int: - """Register and return the ALLOW_BYTES flag.""" - import doctest - - return doctest.register_optionflag("ALLOW_BYTES") - - -def _get_number_flag() -> int: - """Register and return the NUMBER flag.""" - import doctest - - return doctest.register_optionflag("NUMBER") - - -def _get_hide_flag() -> int: - """Register and return the HIDE flag. - - ``HIDE`` is a no-op for execution: the output checker never consults it. - It marks a doctest example that documentation tooling should drop from the - rendered output while still running it as a test. Registering it here means - ``# doctest: +HIDE`` parses instead of raising ``ValueError: invalid - option`` at collection time. - """ - import doctest - - return doctest.register_optionflag("HIDE") +@pytest.hookimpl(hookwrapper=True, tryfirst=True, specname="pytest_collect_file") +def pytest_collect_file_filter( + file_path: pathlib.Path, + parent: pytest.Collector, +) -> t.Generator[None, object, None]: + """Remove pytest's duplicate textfile collector before it parses the file.""" + outcome = yield + if not _is_doctest(parent.config, file_path, parent): + return + hook_result = t.cast(t.Any, outcome).get_result() + filtered = [ + collector + for collector in hook_result + if not isinstance(collector, compat.DoctestTextfile) + ] + t.cast(t.Any, outcome).force_result(filtered) -def _get_flag_lookup() -> dict[str, int]: - import doctest - return { - "DONT_ACCEPT_TRUE_FOR_1": doctest.DONT_ACCEPT_TRUE_FOR_1, - "DONT_ACCEPT_BLANKLINE": doctest.DONT_ACCEPT_BLANKLINE, - "NORMALIZE_WHITESPACE": doctest.NORMALIZE_WHITESPACE, - "ELLIPSIS": doctest.ELLIPSIS, - "IGNORE_EXCEPTION_DETAIL": doctest.IGNORE_EXCEPTION_DETAIL, - "COMPARISON_FLAGS": doctest.COMPARISON_FLAGS, - "ALLOW_UNICODE": _get_allow_unicode_flag(), - "ALLOW_BYTES": _get_allow_bytes_flag(), - "NUMBER": _get_number_flag(), - "HIDE": _get_hide_flag(), - } +def pytest_collect_file( + file_path: pathlib.Path, + parent: pytest.Collector, +) -> DocTestDocutilsFile | pytest.Collector | None: + """Collect documentation here and delegate Python modules to pytest.""" + config = parent.config + if file_path.suffix == ".py": + if config.option.doctestmodules and not config.pluginmanager.has_plugin( + "doctest", + ): + message = ( + f"{file_path}: --doctest-docutils-modules requires pytest's " + "built-in doctest plugin" + ) + raise pytest.UsageError(message) + return None + if _is_doctest(config, file_path, parent): + return DocTestDocutilsFile.from_parent(parent, path=file_path) + return None def get_optionflags(config: pytest.Config) -> int: - """Fetch optionflags from pytest configuration. - - Extracted from pytest.doctest 8.0 (license: MIT). - """ - optionflags = config.getini("doctest_optionflags") - # It takes this rocket surgery to satisfy mypy - optionflags_str = ( - [str(i) for i in optionflags] - if isinstance(optionflags, list) - and all( - isinstance( - item, - str, - ) - for item in optionflags + """Return pytest's resolved doctest option flags.""" + return compat.get_optionflags(config) + + +class DocutilsItem(pytest.DoctestItem): + """One pytest item owning one shared-state doctest group.""" + + @classmethod + def from_parent( # type: ignore[override] + cls, + parent: pytest.Collector, + *, + name: str, + runner: doctest.DocTestRunner, + dtest: doctest.DocTest, + plan: GroupPlan, + registry: RegistrySnapshot, + run_settings: RunSettings, + ) -> DocutilsItem: + """Construct through pytest's cooperative item factory.""" + item = super(pytest.DoctestItem, cls).from_parent( + parent=parent, + name=name, + runner=runner, + dtest=dtest, + plan=plan, + registry=registry, + run_settings=run_settings, ) - else [] - ) - - flag_lookup_table = _get_flag_lookup() - flag_acc = 0 - for flag in optionflags_str: - flag_acc |= flag_lookup_table[flag] - return flag_acc - - -def _get_runner( - checker: doctest.OutputChecker | None = None, - verbose: bool | None = None, - optionflags: int = 0, - continue_on_failure: bool = True, -) -> doctest.DocTestRunner: - # We need this in order to do a lazy import on doctest - global RUNNER_CLASS - if RUNNER_CLASS is None: - RUNNER_CLASS = _init_runner_class() - # Type ignored because the continue_on_failure argument is only defined on - # PytestDoctestRunner, which is lazily defined so can't be used as a type. - return RUNNER_CLASS( # type: ignore - checker=checker, - verbose=verbose, - optionflags=optionflags, - continue_on_failure=continue_on_failure, - ) + return item - -class DocutilsDocTestRunner(doctest.DocTestRunner): - """DocTestRunner for doctest_docutils.""" - - def summarize( # type: ignore + def __init__( self, - out: _Out, - verbose: bool | None = None, - ) -> tuple[int, int]: - """Summarize the test runs.""" - string_io = io.StringIO() - old_stdout = sys.stdout - sys.stdout = string_io - try: - res = super().summarize(verbose) - finally: - sys.stdout = old_stdout - out(string_io.getvalue()) - return res # type:ignore[return-value,unused-ignore] - - def _DocTestRunner__patched_linecache_getlines( + *, + plan: GroupPlan, + registry: RegistrySnapshot, + run_settings: RunSettings, + **kwargs: t.Any, + ) -> None: + super().__init__(**kwargs) + self.plan = plan + self.registry = registry + self.run_settings = run_settings + self.group_result: GroupResult | None = None + self._failure_checkers: dict[int, doctest.OutputChecker] = {} + + def setup(self) -> None: + """Reset attempt state, then let pytest inject fixtures in place.""" + reset_globs(self.plan, self.dtest.globs) + super().setup() + + def runtest(self) -> None: + """Run all block doctests in phase order against the carrier mapping.""" + compat.disable_output_capturing_for_darwin(self) + result = run_group( + self.plan, + self.dtest.globs, + settings=self.run_settings, + registry=self.registry, + exception_policy=_PytestExceptionPolicy(), + ) + self.group_result = result + self._failure_checkers = { + id(failure): block.checker + for block in result.blocks + if isinstance(block, Failed) + for failure in block.failures + } + if result.secondary: + details = "\n\n".join( + "".join( + traceback.format_exception( + type(error), + error, + error.__traceback__, + ), + ) + for error in result.secondary + ) + self.add_report_section("call", "doctest cleanup", details) + if result.primary is not None: + if isinstance(result.primary, bdb.BdbQuit): + pytest.exit("Quitting debugger") + cleanup_outcome = any( + isinstance(block, Errored) + and block.block.phase is Phase.CLEANUP + and block.error is result.primary + for block in result.blocks + ) + if cleanup_outcome and isinstance( + result.primary, + (pytest.skip.Exception, pytest.xfail.Exception), + ): + message = ( + "doctest cleanup raised " + f"{type(result.primary).__name__}: {result.primary}" + ) + raise RuntimeError(message) from result.primary + raise result.primary + + failures = [ + failure + for block in result.blocks + if isinstance(block, Failed) + for failure in block.failures + ] + if failures: + raise compat.make_multiple_failures(failures) + + test_results = [ + block + for block in result.blocks + if block.block.phase is Phase.TEST and not isinstance(block, Errored) + ] + if test_results and all(isinstance(block, Skipped) for block in test_results): + pytest.skip("all examples were skipped") + + def repr_failure( # type: ignore[override] self, - filename: str, - module_globals: t.Any = None, - ) -> t.Any: - # this is overridden from DocTestRunner adding the try-except below - m = self._DocTestRunner__LINECACHE_FILENAME_RE.match(filename) # type: ignore - if m and m.group("name") == self.test.name: - try: - example = self.test.examples[int(m.group("examplenum"))] - # because we compile multiple doctest blocks with the same name - # (viz. the group name) this might, for outer stack frames in a - # traceback, get the wrong test which might not have enough examples - except IndexError: - pass - else: - return example.source.splitlines(True) - return self.save_linecache_getlines(filename, module_globals) # type: ignore + excinfo: ExceptionInfo[BaseException], + ) -> str | TerminalRepr: + """Use comparison-time checkers for contributed output semantics.""" + rendered = compat.repr_failure_with_checkers( + self, + excinfo, + self._failure_checkers, + ) + if rendered is not None: + return rendered + return super().repr_failure(excinfo) class DocTestDocutilsFile(pytest.Module): - """Pytest module for doctest_docutils.""" - - obj = None # Fix pytest-asyncio issue. #46, pytest-asyncio#872 + """Documentation module projecting one item per doctest group.""" - def collect(self) -> Iterable[DoctestItem]: - """Collect tests for pytest module.""" - _ensure_directives_registered() + obj = None + def collect(self) -> collections.abc.Iterable[DocutilsItem]: + """Parse once, project pure plans, and build synthetic carriers.""" + if not self.config.pluginmanager.has_plugin("doctest"): + message = ( + f"{self.path}: documentation collection requires pytest's " + "built-in doctest plugin" + ) + raise pytest.UsageError(message) + ensure_directives_registered() encoding = self.config.getini("doctest_encoding") - text = self.path.read_text(encoding) - - # Uses internal doctest module parsing mechanism. - finder = DocutilsDocTestFinder() + text = self.path.read_text(encoding=encoding) + registry = self.config.stash[_REGISTRY_KEY] + parsed = parse_document(text, self.path, registry=registry) - # While doctests in .rst/.md files don't support fixtures directly, - # we still need to pick up autouse fixtures. - # Backported from pytest commit 9cd14b4ff (2024-02-06). - # https://github.com/pytest-dev/pytest/commit/9cd14b4ff - self.session._fixturemanager.parsefactories(self) - - optionflags = get_optionflags(self.config) - - runner = _get_runner( - verbose=False, - optionflags=optionflags, - checker=_pytest.doctest._get_checker(), - continue_on_failure=_pytest.doctest._get_continue_on_failure(self.config), + ungrouped = t.cast( + t.Literal["block", "default"], + self.config.getini("doctest_docutils_ungrouped"), ) - from _pytest.doctest import DoctestItem - - for test in finder.find( - text, - str(self.path), - ): - if test.examples: # skip empty doctests - yield DoctestItem.from_parent( - self, # type: ignore - name=test.name, - runner=runner, - dtest=test, - ) + plans = project( + parsed, + document_name=self.path.name, + settings=ProjectionSettings(ungrouped=ungrouped), + registry=registry, + ) + optionflags = get_optionflags(self.config) + continue_on_failure = compat.get_continue_on_failure(self.config) + for plan in plans: + globs: dict[str, t.Any] = {} + carrier = doctest.DocTest( + [], + globs, + plan.group, + str(self.path), + 0, + "", + ) + carrier.globs = globs + runner = doctest.DocTestRunner( + checker=compat.get_checker(), + optionflags=optionflags, + ) + yield DocutilsItem.from_parent( + self, + name=plan.group, + runner=runner, + dtest=carrier, + plan=plan, + registry=registry, + run_settings=RunSettings( + optionflags=optionflags, + continue_on_failure=continue_on_failure, + checker_name="stdlib", + ), + ) diff --git a/tests/regressions/test_autouse_fixtures.py b/tests/regressions/test_autouse_fixtures.py index 1058619..1e0b876 100644 --- a/tests/regressions/test_autouse_fixtures.py +++ b/tests/regressions/test_autouse_fixtures.py @@ -99,7 +99,7 @@ def test_autouse_fixtures_with_doctest_files( pytest=textwrap.dedent( """ [pytest] -addopts=-p no:doctest -vv +addopts=-vv """.strip(), ), ) diff --git a/tests/test_doctest_core_host_boundaries.py b/tests/test_doctest_core_host_boundaries.py new file mode 100644 index 0000000..9da3f3d --- /dev/null +++ b/tests/test_doctest_core_host_boundaries.py @@ -0,0 +1,130 @@ +"""Host-boundary acceptance tests for grouped doctest execution.""" + +from __future__ import annotations + +import textwrap + +import _pytest.pytester +import pytest + + +def test_rerun_reseeds_group_globs( + pytester: _pytest.pytester.Pytester, +) -> None: + """A rerun cannot pass by observing mutations from its first attempt.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: shared + + >>> attempt = globals().get("attempt", 0) + 1 + >>> attempt + 2 + """, + ), + ) + + result = pytester.runpytest("guide.rst", "--reruns", "1", "-q") + + result.assert_outcomes(failed=1) + assert result.parseoutcomes()["rerun"] == 1 + + +@pytest.mark.parametrize("distribution", ["load", "worksteal"]) +def test_xdist_runs_stateful_groups_without_affinity( + pytester: _pytest.pytester.Pytester, + distribution: str, +) -> None: + """Each xdist scheduler may move groups without splitting group state.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makeconftest( + textwrap.dedent( + """ + import pytest + + @pytest.fixture(autouse=True) + def inject_worker(doctest_namespace, worker_id): + doctest_namespace["worker_id"] = worker_id + """, + ), + ) + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: first + + >>> state = [worker_id, "first"] + + .. doctest:: first + + >>> state[0].startswith("gw") + True + >>> state[1] + 'first' + + .. doctest:: second + + >>> state = [worker_id, "second"] + + .. doctest:: second + + >>> state[0].startswith("gw") + True + >>> state[1] + 'second' + """, + ), + ) + + result = pytester.runpytest( + "guide.rst", + "-n", + "2", + "--dist", + distribution, + "-q", + ) + + result.assert_outcomes(passed=2) + assert result.ret is pytest.ExitCode.OK + + +def test_pytest_asyncio_fixture_composes_with_document_item( + pytester: _pytest.pytester.Pytester, +) -> None: + """An async autouse fixture can populate the doctest namespace.""" + pytest.importorskip("pytest_asyncio", minversion="1.0") + pytester.plugins = ["pytest_doctest_docutils", "pytest_asyncio.plugin"] + pytester.makeini("[pytest]\nasyncio_mode = auto\n") + pytester.makeconftest( + textwrap.dedent( + """ + import asyncio + + import pytest_asyncio + + @pytest_asyncio.fixture(autouse=True) + async def inject_value(doctest_namespace): + await asyncio.sleep(0) + doctest_namespace["value"] = 42 + """, + ), + ) + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: + + >>> value + 42 + """, + ), + ) + + result = pytester.runpytest("guide.rst", "-q") + + result.assert_outcomes(passed=1) diff --git a/tests/test_doctest_core_projection.py b/tests/test_doctest_core_projection.py new file mode 100644 index 0000000..770645b --- /dev/null +++ b/tests/test_doctest_core_projection.py @@ -0,0 +1,538 @@ +"""Tests for doctree extraction and pure group projection.""" + +from __future__ import annotations + +import doctest +import pathlib +import typing as t + +import pytest +from docutils import nodes +from docutils.utils import new_document + +from doctest_core import ( + BlockKind, + ParseSettings, + Phase, + ProjectionSettings, + Provider, + build_registry, + extract_blocks, + parse_document, + project, +) +from doctest_core.markup import _stamp_myst_source_lines + + +@pytest.fixture(params=["rst", "myst"]) +def grouped_document(request: pytest.FixtureRequest) -> tuple[pathlib.Path, str]: + """Return equivalent reStructuredText and MyST documents.""" + if request.param == "rst": + return ( + pathlib.Path("guide.rst"), + """ +.. testsetup:: alpha, beta + + value = 40 + +.. doctest:: alpha, beta + :options: +ELLIPSIS + :skipif: False + :pyversion: >=3.10 + + >>> value + 2 + 42 + +.. testcode:: alpha + + print(value + 3) + +.. testoutput:: alpha + :options: +NORMALIZE_WHITESPACE + + 43 + +.. testcleanup:: alpha, beta + + del value +""", + ) + return ( + pathlib.Path("guide.md"), + """ +```{testsetup} alpha, beta +value = 40 +``` + +```{doctest} alpha, beta +:options: +ELLIPSIS +:skipif: False +:pyversion: ">=3.10" + +>>> value + 2 +42 +``` + +```{testcode} alpha +print(value + 3) +``` + +```{testoutput} alpha +:options: +NORMALIZE_WHITESPACE + +43 +``` + +```{testcleanup} alpha, beta +del value +``` +""", + ) + + +def test_extracts_sphinx_vocabulary_as_typed_data( + grouped_document: tuple[pathlib.Path, str], +) -> None: + """Both front ends preserve groups, gates, options, and source order.""" + path, source = grouped_document + + result = parse_document( + source, + path, + settings=ParseSettings(), + registry=build_registry(), + ) + + assert [block.kind for block in result.blocks] == [ + "testsetup", + "doctest", + "testcode", + "testcleanup", + ] + assert [block.document_order for block in result.blocks] == [0, 1, 2, 4] + assert [block.block_ordinal for block in result.blocks] == [0, 1, 2, 3] + assert result.blocks[1].groups == ("alpha", "beta") + assert result.blocks[1].options == {doctest.ELLIPSIS: True} + assert result.blocks[1].skipif == "False" + assert result.blocks[1].pyversion == ">=3.10" + assert result.outputs[0].document_order == 3 + assert result.outputs[0].options == {doctest.NORMALIZE_WHITESPACE: True} + assert not [item for item in result.diagnostics if item.level == "error"] + + +def test_projection_groups_without_merging_blocks( + grouped_document: tuple[pathlib.Path, str], +) -> None: + """One plan owns shared state while every source block keeps a recipe.""" + path, source = grouped_document + parsed = parse_document(source, path, registry=build_registry()) + + plans = project( + parsed, + document_name=path.name, + settings=ProjectionSettings(), + registry=build_registry(), + ) + + assert [plan.group for plan in plans] == ["alpha", "beta"] + assert [block.phase for block in plans[0].blocks] == [ + Phase.SETUP, + Phase.TEST, + Phase.TEST, + Phase.CLEANUP, + ] + assert [block.phase for block in plans[1].blocks] == [ + Phase.SETUP, + Phase.TEST, + Phase.CLEANUP, + ] + assert plans[0].blocks[1] is not plans[1].blocks[1] + assert plans[0].blocks[1].name == "guide::alpha[1]" + assert plans[1].blocks[1].name == "guide::beta[1]" + assert plans[0].blocks[2].expected is not None + assert plans[0].blocks[2].expected.text == "43\n" + + +def test_prompt_recipe_is_exactly_stdlib_normalized() -> None: + """Projection preserves every field emitted by ``DocTestParser``.""" + source = """ +>>> value = 1 +>>> value + 1 +2 +>>> int('bad') +Traceback (most recent call last): +ValueError: invalid literal... +""" + parsed = parse_document(source, pathlib.Path("guide.rst")) + + plan = project(parsed, document_name="guide")[0] + recipes = plan.blocks[0].examples + expected = ( + doctest.DocTestParser() + .get_doctest( + parsed.blocks[0].source, + {}, + "guide", + "guide.rst", + 0, + ) + .examples + ) + + assert [tuple(recipe) for recipe in recipes] == [ + ( + example.source, + example.want, + example.exc_msg, + example.lineno, + example.indent, + example.options, + ) + for example in expected + ] + + +def test_ungrouped_policy_is_explicit() -> None: + """The projection setting chooses sharing without changing block identity.""" + parsed = parse_document( + ">>> one = 1\n\nSome prose.\n\n>>> one + 1\n2\n", + pathlib.Path("guide.rst"), + ) + + shared = project( + parsed, + document_name="guide", + settings=ProjectionSettings(ungrouped="default"), + ) + isolated = project( + parsed, + document_name="guide", + settings=ProjectionSettings(ungrouped="block"), + ) + + assert [(plan.group, len(plan.blocks)) for plan in shared] == [("default", 2)] + assert [(plan.group, len(plan.blocks)) for plan in isolated] == [ + ("block-0", 1), + ("block-1", 1), + ] + + +def test_anonymous_group_does_not_collide_with_named_group() -> None: + """A generated block group cannot alias an author-declared group.""" + parsed = parse_document( + """>>> anonymous = True + +.. doctest:: block-0 + + >>> named = True +""", + pathlib.Path("guide.rst"), + ) + + plans = project( + parsed, + document_name="guide", + settings=ProjectionSettings(ungrouped="block"), + ) + + assert len(plans) == 2 + assert len({plan.group for plan in plans}) == 2 + assert [[block.block_ordinal for block in plan.blocks] for plan in plans] == [ + [0], + [1], + ] + + +def test_parse_deduplicates_reporter_and_doctree_diagnostics() -> None: + """One parser problem produces one typed diagnostic with its best line.""" + parsed = parse_document( + ".. unknown-directive::\n", + pathlib.Path("guide.rst"), + settings=ParseSettings(suppressed_diagnostics=frozenset()), + ) + + errors = [ + diagnostic for diagnostic in parsed.diagnostics if diagnostic.level == "error" + ] + + assert len(errors) == 1 + assert errors[0].code == "docutils.unknown-directive" + assert errors[0].line == 1 + + +def test_default_unknown_role_suppression_includes_lookup_companion() -> None: + """Suppressing an unknown role removes both docutils messages.""" + parsed = parse_document( + ":missing-role:`value`\n", + pathlib.Path("guide.rst"), + ) + + assert not [ + diagnostic + for diagnostic in parsed.diagnostics + if "role" in diagnostic.message.lower() + ] + + +def test_registered_block_kind_survives_generic_node_extraction() -> None: + """Extraction preserves stamps while projection resolves their policy.""" + tree = new_document("guide.rst") + node = nodes.literal_block( + ">>> 6 * 7\n42\n", + ">>> 6 * 7\n42\n", + testnodetype="example", + groups=["shared"], + ) + node.source = "guide.rst" + node.line = 1 + tree += node + + class Contributor: + provider = Provider("example", "1") + + def contribute(self, registrar: t.Any) -> None: + """Register the node stamp's projection policy.""" + registrar.add_block_kind( + "example", + BlockKind(Phase.TEST, "prompt", None), + ) + + registry = build_registry([Contributor()]) + parsed = extract_blocks(tree, registry=registry) + plans = project(parsed, document_name="guide.rst", registry=registry) + + assert parsed.blocks[0].kind == "example" + assert plans[0].blocks[0].examples[0].source == "6 * 7\n" + + +def test_unregistered_stamp_does_not_rename_runnable_blocks() -> None: + """Foreign node metadata cannot consume a runnable identity ordinal.""" + tree = new_document("guide.rst") + tree += nodes.literal_block( + "foreign", + "foreign", + testnodetype="foreign", + ) + tree += nodes.doctest_block(">>> 6 * 7\n42\n", ">>> 6 * 7\n42\n") + + parsed = extract_blocks(tree) + + assert [(block.kind, block.block_ordinal) for block in parsed.blocks] == [ + ("doctest", 0), + ] + + +def test_extraction_rejects_malformed_typed_text_stamps() -> None: + """Dynamic node metadata cannot violate the public parsed-record types.""" + tree = new_document("guide.rst") + node = nodes.literal_block( + ">>> 6 * 7\n42\n", + ">>> 6 * 7\n42\n", + testnodetype="doctest", + groups=["shared"], + skipif=123, + ) + tree += node + + with pytest.raises(TypeError, match="skipif node attribute"): + extract_blocks(tree) + + +def test_registered_block_kind_pairs_with_custom_output_stamp() -> None: + """A block kind can name an output stamp without parser changes.""" + tree = new_document("guide.rst") + code = nodes.literal_block( + 'print("answer")', + 'print("answer")', + testnodetype="example", + groups=["shared"], + ) + code.source = "guide.rst" + code.line = 1 + output = nodes.literal_block( + "answer", + "answer", + testnodetype="expected", + groups=["shared"], + ) + output.source = "guide.rst" + output.line = 3 + tree += code + tree += output + + class Contributor: + provider = Provider("example", "1") + + def contribute(self, registrar: t.Any) -> None: + """Register the custom executable and output relationship.""" + registrar.add_block_kind( + "example", + BlockKind(Phase.TEST, "exec", "expected"), + ) + + registry = build_registry([Contributor()]) + parsed = extract_blocks(tree, registry=registry) + plans = project(parsed, document_name="guide.rst", registry=registry) + + assert parsed.outputs[0].kind == "expected" + assert plans[0].blocks[0].expected is not None + assert plans[0].blocks[0].expected.text == "answer\n" + + +@pytest.mark.parametrize("body", ["", "value = 42"]) +def test_prompt_free_doctest_does_not_project_a_group(body: str) -> None: + """A doctest directive without examples cannot become a host item.""" + parsed = parse_document( + f".. doctest::\n\n {body}\n", + pathlib.Path("guide.rst"), + ) + + assert project(parsed, document_name="guide.rst") == () + + +@pytest.mark.parametrize( + ("path", "source"), + [ + ( + pathlib.Path("guide.rst"), + """ +.. testcode:: + :trim-doctest-flags: + + print("answer") + +.. testoutput:: + :no-trim-doctest-flags: + + answer +""", + ), + ( + pathlib.Path("guide.md"), + """ +```{testcode} +:trim-doctest-flags: +print("answer") +``` + +```{testoutput} +:no-trim-doctest-flags: +answer +``` +""", + ), + ], +) +def test_testcode_output_accept_sphinx_trim_options( + path: pathlib.Path, + source: str, +) -> None: + """Standalone parsers accept Sphinx's full trim-option vocabulary.""" + parsed = parse_document(source, path) + + assert [block.kind for block in parsed.blocks] == ["testcode"] + assert [output.kind for output in parsed.outputs] == ["testoutput"] + assert not [item for item in parsed.diagnostics if item.level == "error"] + + +def test_myst_directive_line_is_document_absolute() -> None: + """MyST-local content offsets do not replace the fence's document line.""" + parsed = parse_document( + """Heading +======= +```{doctest} +>>> 1 + 1 +3 +``` +""", + pathlib.Path("guide.md"), + ) + + assert parsed.blocks[0].line == 4 + plan = project(parsed, document_name="guide.md")[0] + assert plan.blocks[0].lineno == 3 + + with_options = parse_document( + """Heading +======= +```{doctest} +:options: +ELLIPSIS + +>>> 1 + 1 +3 +``` +""", + pathlib.Path("guide.md"), + ) + + assert with_options.blocks[0].line == 6 + + +@pytest.mark.parametrize( + ("source", "expected_line"), + [ + ("Text\n\n >>> 1 + 1\n 2\n", 3), + ("Text\n\n```\n>>> 1 + 1\n2\n```\n", 4), + ], +) +def test_myst_bare_prompt_line_distinguishes_indent_and_fence( + source: str, + expected_line: int, +) -> None: + """Standalone MyST stamps the first executable source line.""" + parsed = parse_document(source, pathlib.Path("guide.md")) + + assert parsed.blocks[0].line == expected_line + + +def test_myst_line_stamper_does_not_attribute_included_source_to_root() -> None: + """Root text cannot supply an absolute line for an included node.""" + tree = new_document("guide.md") + node = nodes.literal_block( + ">>> 1 + 1\n2\n", + ">>> 1 + 1\n2\n", + testnodetype="doctest", + ) + node.source = "included.md" + node.line = 1 + tree += node + + _stamp_myst_source_lines(tree, "Text\n\n>>> 1 + 1\n2\n") + parsed = extract_blocks(tree) + + assert parsed.blocks[0].path == pathlib.Path("included.md") + assert parsed.blocks[0].line == 2 + + +def test_output_pairing_is_group_local_and_latest_wins() -> None: + """Other groups do not break pairing and later output replaces earlier.""" + source = """ +.. testcode:: alpha + + print("alpha") + +.. testcode:: beta + + print("beta") + +.. testoutput:: alpha + + stale + +.. testoutput:: alpha + + alpha + +.. testoutput:: beta + + beta +""" + parsed = parse_document(source, pathlib.Path("guide.rst")) + + plans = project(parsed, document_name="guide") + + assert [plan.group for plan in plans] == ["alpha", "beta"] + assert plans[0].blocks[0].expected is not None + assert plans[0].blocks[0].expected.text == "alpha\n" + assert plans[1].blocks[0].expected is not None + assert plans[1].blocks[0].expected.text == "beta\n" diff --git a/tests/test_doctest_core_pytest.py b/tests/test_doctest_core_pytest.py new file mode 100644 index 0000000..3f63d5d --- /dev/null +++ b/tests/test_doctest_core_pytest.py @@ -0,0 +1,424 @@ +"""End-to-end tests for the typed core's pytest host adapter.""" + +from __future__ import annotations + +import textwrap + +import _pytest.pytester +import pytest + + +def test_pytest_composes_with_builtin_and_collects_one_group( + pytester: _pytest.pytester.Pytester, +) -> None: + """The adapter keeps pytest doctest active and owns one grouped item.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makeconftest( + textwrap.dedent( + """ + import pytest + + def pytest_sessionstart(session): + assert session.config.pluginmanager.has_plugin("doctest") + + @pytest.fixture(autouse=True) + def inject(doctest_namespace): + doctest_namespace["fixture_value"] = 40 + """, + ), + ) + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: shared + + >>> value = fixture_value + + .. doctest:: shared + + >>> value + 2 + 42 + """, + ), + ) + pytester.makepyfile( + test_module=textwrap.dedent( + ''' + def answer(): + """Return the answer. + + >>> answer() + 42 + """ + return 42 + ''', + ), + ) + + result = pytester.runpytest( + "guide.rst", + "test_module.py", + "--doctest-docutils-modules", + "-q", + ) + + result.assert_outcomes(passed=2) + + +def test_pytest_outcome_escapes_and_cleanup_runs( + pytester: _pytest.pytester.Pytester, +) -> None: + """A host skip remains a skip and cannot bypass group cleanup.""" + pytester.plugins = ["pytest_doctest_docutils"] + marker = pytester.path / "cleaned" + pytester.makeconftest( + textwrap.dedent( + f""" + import pathlib + import pytest + + @pytest.fixture(autouse=True) + def inject(doctest_namespace): + doctest_namespace.update( + pytest=pytest, + marker=pathlib.Path({str(marker)!r}), + ) + """, + ), + ) + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: shared + + >>> pytest.skip("not available") + + .. testcleanup:: shared + + marker.write_text("yes") + """, + ), + ) + + result = pytester.runpytest("guide.rst", "-q", "-rs") + + result.assert_outcomes(skipped=1) + result.stdout.fnmatch_lines(["*SKIPPED*not available*"]) + assert marker.read_text(encoding="utf-8") == "yes" + + +def test_cleanup_outcome_is_a_failure( + pytester: _pytest.pytester.Pytester, +) -> None: + """A cleanup skip cannot relabel a completed test as skipped.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makeconftest( + textwrap.dedent( + """ + import pytest + + @pytest.fixture(autouse=True) + def inject(doctest_namespace): + doctest_namespace["pytest"] = pytest + """, + ), + ) + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: shared + + >>> 6 * 7 + 42 + + .. testcleanup:: shared + + pytest.skip("cleanup refused") + """, + ), + ) + + result = pytester.runpytest("guide.rst", "-q") + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines(["*cleanup*cleanup refused*"]) + + +def test_cleanup_outcome_is_reported_beside_primary_failure( + pytester: _pytest.pytester.Pytester, +) -> None: + """A secondary cleanup outcome remains visible beside the primary failure.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makeconftest( + textwrap.dedent( + """ + import pytest + + @pytest.fixture(autouse=True) + def inject(doctest_namespace): + doctest_namespace["pytest"] = pytest + """, + ), + ) + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. doctest:: shared + + >>> 1 + 1 + 3 + + .. testcleanup:: shared + + pytest.skip("cleanup refused after failure") + """, + ), + ) + + result = pytester.runpytest("guide.rst", "-q") + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + ["*doctest cleanup*", "*cleanup refused after failure*"], + ) + + +def test_pytest_direct_path_has_no_builtin_duplicate( + pytester: _pytest.pytester.Pytester, +) -> None: + """The collector wrapper removes pytest's textfile collector before parse.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile( + ".rst", + guide=">>> 6 * 7\n42\n", + ) + + result = pytester.runpytest("guide.rst", "--collect-only", "-q") + + result.assert_outcomes(errors=0) + result.stdout.fnmatch_lines(["*1 test collected*"]) + + +def test_anonymous_item_does_not_collide_with_named_group( + pytester: _pytest.pytester.Pytester, +) -> None: + """A bare block and a same-named author group get separate items and state.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + >>> marker = "anonymous" + + .. doctest:: block-0 + + >>> "marker" in globals() + False + """, + ), + ) + + result = pytester.runpytest("guide.rst", "-q") + + result.assert_outcomes(passed=2) + + +@pytest.mark.parametrize("body", ["", "value = 42"]) +def test_pytest_does_not_collect_prompt_free_doctest( + pytester: _pytest.pytester.Pytester, + body: str, +) -> None: + """An empty stock DocTest does not become a passing carrier item.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile(".rst", guide=f".. doctest::\n\n {body}\n") + + result = pytester.runpytest("guide.rst", "--collect-only", "-q") + + result.assert_outcomes(errors=0) + result.stdout.fnmatch_lines(["*no tests collected*"]) + + +def test_custom_doctest_glob_remains_owned_by_pytest( + pytester: _pytest.pytester.Pytester, +) -> None: + """Unsupported parser suffixes remain the built-in plugin's concern.""" + pytester.plugins = ["pytest_doctest_docutils"] + path = pytester.path / "guide.foo" + path.write_text(">>> 6 * 7\n42\n", encoding="utf-8") + + result = pytester.runpytest(str(path), "--doctest-glob=*.foo", "-q") + + result.assert_outcomes(passed=1) + + +def test_pytest_preserves_per_block_failure_locations( + pytester: _pytest.pytester.Pytester, +) -> None: + """One group item retains each failed block's source line.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile( + ".rst", + guide=""".. doctest:: shared + + >>> 1 + 1 + 3 + +.. doctest:: shared + + >>> 2 + 2 + 5 +""", + ) + + result = pytester.runpytest( + "guide.rst", + "--doctest-continue-on-failure", + "-q", + ) + + result.assert_outcomes(failed=1) + output = result.stdout.str() + assert "guide.rst:3" in output + assert "guide.rst:8" in output + + +def test_contributed_checker_compares_and_explains( + pytester: _pytest.pytester.Pytester, +) -> None: + """The checker that rejects output also renders the resulting failure.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makeconftest( + textwrap.dedent( + """ + import doctest + + from doctest_core import Provider + + class Checker(doctest.OutputChecker): + def __init__(self): + self.compared = False + + def check_output(self, want, got, optionflags): + self.compared = True + return False + + def output_difference(self, example, got, optionflags): + assert self.compared + return "CUSTOM CHECKER DIFFERENCE" + + class Contributor: + provider = Provider("pytest", "probe") + + def contribute(self, registrar): + registrar.add_output_checker( + "stdlib", Checker, replace=True + ) + + def pytest_doctest_core_contributors(): + return Contributor() + """, + ), + ) + pytester.makefile(".rst", guide=">>> 6 * 7\n42\n") + + result = pytester.runpytest("guide.rst", "-q") + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines(["*CUSTOM CHECKER DIFFERENCE*"]) + + +def test_late_nested_contributor_is_rejected( + pytester: _pytest.pytester.Pytester, +) -> None: + """A nested conftest cannot silently miss the frozen registry.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makeini("[pytest]\ntestpaths = nested\n") + nested = pytester.path / "nested" + nested.mkdir() + (nested / "conftest.py").write_text( + textwrap.dedent( + """ + def pytest_doctest_core_contributors(): + return None + """, + ), + encoding="utf-8", + ) + (nested / "guide.rst").write_text(">>> 6 * 7\n42\n", encoding="utf-8") + + result = pytester.runpytest(".", "-q") + + assert result.ret is pytest.ExitCode.INTERRUPTED + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + ["*nested/conftest.py*doctest-core contributor*phase closed*"], + ) + + +def test_document_collection_requires_builtin_doctest( + pytester: _pytest.pytester.Pytester, +) -> None: + """Disabling the composed host fails only an affected documentation path.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile(".rst", guide=">>> 6 * 7\n42\n") + + result = pytester.runpytest("guide.rst", "-p", "no:doctest", "-q") + + assert result.ret is pytest.ExitCode.INTERRUPTED + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + ["*guide.rst*requires pytest's built-in doctest plugin*"], + ) + + +def test_discovered_document_requires_builtin_doctest( + pytester: _pytest.pytester.Pytester, +) -> None: + """Directory discovery reaches the same actionable disabled-host error.""" + pytester.plugins = ["pytest_doctest_docutils"] + docs = pytester.path / "docs" + docs.mkdir() + (docs / "guide.rst").write_text(">>> 6 * 7\n42\n", encoding="utf-8") + + result = pytester.runpytest("docs", "-p", "no:doctest", "-q") + + assert result.ret is pytest.ExitCode.INTERRUPTED + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines( + ["*guide.rst*requires pytest's built-in doctest plugin*"], + ) + + +def test_cleanup_exit_outranks_doctest_failure( + pytester: _pytest.pytester.Pytester, +) -> None: + """A cleanup session exit cannot be reduced to a secondary report.""" + pytester.plugins = ["pytest_doctest_docutils"] + pytester.makefile( + ".rst", + guide=textwrap.dedent( + """ + .. testcode:: shared + + print(1) + + .. testoutput:: shared + + 2 + + .. testcleanup:: shared + + import pytest + pytest.exit("cleanup requested") + """, + ), + ) + + result = pytester.runpytest("guide.rst", "-q") + + assert result.ret is pytest.ExitCode.INTERRUPTED + result.stdout.fnmatch_lines(["*Exit: cleanup requested*"]) diff --git a/tests/test_doctest_core_registry.py b/tests/test_doctest_core_registry.py new file mode 100644 index 0000000..18ff7b0 --- /dev/null +++ b/tests/test_doctest_core_registry.py @@ -0,0 +1,144 @@ +"""Tests for the doctest core registry.""" + +from __future__ import annotations + +import doctest +import typing as t + +import pytest + +from doctest_core import ( + BlockKind, + Phase, + Provider, + RegistryClosedError, + RegistryCollisionError, + RegistryError, + build_registry, +) + + +class RecordingContributor: + """Register one checker and retain the registrar for the freeze test.""" + + provider = Provider(name="tests", version="1") + + def __init__(self, *, replace: bool = False) -> None: + self.registrar: t.Any = None + self.replace = replace + + def contribute(self, registrar: t.Any) -> None: + """Register a checker factory. + + >>> contributor = RecordingContributor() + >>> contributor.provider.name + 'tests' + """ + self.registrar = registrar + registrar.add_output_checker( + "stdlib", + doctest.OutputChecker, + replace=self.replace, + ) + + +def test_registry_snapshot_is_ordered_and_immutable() -> None: + """Built-ins freeze in declaration order behind read-only mappings.""" + snapshot = build_registry() + + assert tuple(snapshot.block_kinds) == ( + "doctest", + "testsetup", + "testcleanup", + "testcode", + ) + assert tuple(snapshot.document_parsers) == ("rst", "myst") + assert tuple(snapshot.execution_profiles) == ("prompt", "exec") + assert tuple(snapshot.output_checkers) == ("stdlib",) + + with pytest.raises(TypeError): + snapshot.output_checkers["other"] = snapshot.output_checkers["stdlib"] # type: ignore[index] + + +def test_registry_rejects_implicit_collision() -> None: + """A contributor cannot silently replace another provider's capability.""" + with pytest.raises(RegistryCollisionError, match=r"stdlib.*builtin.*tests"): + build_registry([RecordingContributor()]) + + +def test_registry_explicit_replacement_preserves_position() -> None: + """An explicit replacement retains the incumbent's precedence.""" + contributor = RecordingContributor(replace=True) + + snapshot = build_registry([contributor]) + + assert tuple(snapshot.output_checkers) == ("stdlib",) + assert snapshot.output_checkers["stdlib"].provider == contributor.provider + + +def test_registry_retained_registrar_closes_after_freeze() -> None: + """A retained registrar cannot mutate a frozen snapshot.""" + contributor = RecordingContributor(replace=True) + build_registry([contributor]) + + with pytest.raises(RegistryClosedError): + contributor.registrar.add_output_checker("late", doctest.OutputChecker) + + +def test_registry_rejects_missing_execution_profile_reference() -> None: + """A frozen block kind cannot defer a missing-profile KeyError to runtime.""" + + class Contributor: + provider = Provider("broken", "1") + + def contribute(self, registrar: t.Any) -> None: + """Register a block kind with no executable profile.""" + registrar.add_block_kind( + "example", + BlockKind(Phase.TEST, "missing", None), + ) + + with pytest.raises(RegistryError, match=r"example.*broken.*missing"): + build_registry([Contributor()]) + + +def test_registry_rejects_runnable_output_kind_collision() -> None: + """One node stamp cannot be both executable and expected output.""" + + class Contributor: + provider = Provider("ambiguous", "1") + + def contribute(self, registrar: t.Any) -> None: + """Register contradictory executable and output roles.""" + registrar.add_block_kind( + "example", + BlockKind(Phase.TEST, "exec", "expected"), + ) + registrar.add_block_kind( + "expected", + BlockKind(Phase.TEST, "exec", None), + ) + + with pytest.raises( + RegistryCollisionError, + match=r"example.*ambiguous.*expected.*ambiguous", + ): + build_registry([Contributor()]) + + +@pytest.mark.parametrize("output_kind", ["", "Expected", "bad name"]) +def test_registry_rejects_invalid_output_kind_reference(output_kind: str) -> None: + """Cross-references obey the same name grammar as registrations.""" + + class Contributor: + provider = Provider("broken", "1") + + def contribute(self, registrar: t.Any) -> None: + """Register a block kind with a malformed output reference.""" + registrar.add_block_kind( + "example", + BlockKind(Phase.TEST, "exec", output_kind), + ) + + with pytest.raises(RegistryError, match=r"invalid registry name"): + build_registry([Contributor()]) diff --git a/tests/test_doctest_core_runner.py b/tests/test_doctest_core_runner.py new file mode 100644 index 0000000..9ffe5a4 --- /dev/null +++ b/tests/test_doctest_core_runner.py @@ -0,0 +1,717 @@ +"""Tests for fresh materialization and group execution.""" + +from __future__ import annotations + +import doctest +import pathlib +import sys +import traceback + +from doctest_core import ( + Counts, + ExampleRecipe, + Failed, + GroupPlan, + Passed, + Phase, + ProjectedBlock, + RunSettings, + build_registry, + materialize, + parse_document, + project, + reset_globs, + run_group, +) + + +def test_materializes_fresh_stock_objects_against_one_mapping() -> None: + """Plans retain recipes while attempts receive ordinary fresh objects.""" + parsed = parse_document(">>> 1 + 1\n2\n", pathlib.Path("guide.rst")) + block = project(parsed, document_name="guide")[0].blocks[0] + globs: dict[str, object] = {} + + first = materialize(block, globs) + second = materialize(block, globs) + + assert type(first) is doctest.DocTest + assert type(first.examples[0]) is doctest.Example + assert first is not second + assert first.examples[0] is not second.examples[0] + assert first.globs is globs + assert second.globs is globs + assert first.examples[0].source == "1 + 1\n" + assert first.examples[0].want == "2\n" + + +def test_group_runner_shares_state_and_always_runs_cleanup() -> None: + """Separate block tests share one mapping owned by their group attempt.""" + source = """ +.. testsetup:: example + + value = 40 + +.. doctest:: example + + >>> value + 2 + 42 + +.. doctest:: example + + >>> value + 3 + 99 + +.. testcleanup:: example + + cleaned = True +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {"residue": "old"} + identity = id(globs) + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings(continue_on_failure=False), + registry=build_registry(), + ) + + assert id(globs) == identity + assert "residue" not in globs + assert globs["cleaned"] is True + assert isinstance(result.blocks[0], Passed) + assert isinstance(result.blocks[1], Passed) + assert isinstance(result.blocks[2], Failed) + failure = result.blocks[2].failures[0] + assert failure.test.name == "guide::example[2]" + assert failure.test.globs is globs + assert result.primary is None + + +def test_exec_profile_pairs_testcode_output() -> None: + """Prompt-free Sphinx code uses its paired output and shared globals.""" + source = """ +.. testsetup:: example + + value = 41 + +.. testcode:: example + + print(value + 1) + +.. testoutput:: example + + 42 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Passed, Passed] + test_result = result.blocks[-1] + assert isinstance(test_result, Passed) + assert test_result.counts == Counts(failed=0, attempted=1, skipped=0) + + +def test_reset_globs_reseeds_each_attempt() -> None: + """A second attempt cannot observe mutations left by its predecessor.""" + parsed = parse_document(">>> token\n'fresh'\n", pathlib.Path("guide.rst")) + plan = project(parsed, document_name="guide", seed={"token": "fresh"})[0] + globs: dict[str, object] = {} + + reset_globs(plan, globs) + globs["token"] = "mutated" + reset_globs(plan, globs, extraglobs={"fixture": 42}) + + assert globs == {"token": "fresh", "fixture": 42, "__name__": "__main__"} + + +def test_gate_error_is_recorded_and_cleanup_still_runs() -> None: + """An author-controlled gate cannot escape the group cleanup boundary.""" + source = """ +.. doctest:: shared + :skipif: 1 / 0 + + >>> value = 42 + +.. testcleanup:: shared + + cleaned = True +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert isinstance(result.primary, ZeroDivisionError) + assert globs["cleaned"] is True + assert [type(block).__name__ for block in result.blocks] == [ + "Errored", + "Passed", + ] + + +def test_exec_profile_does_not_inherit_core_future_flags() -> None: + """Exec bodies inherit document state, not this module's future imports.""" + source = """ +.. testcode:: shared + + def identity(value: int) -> int: + return value + print(identity.__annotations__) + +.. testoutput:: shared + + {'value': , 'return': } +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block).__name__ for block in result.blocks] == ["Passed"] + + +def test_exec_profile_restores_stdout_and_records_unexpected_exception() -> None: + """The extended lane restores process state and emits stock failures.""" + source = """ +.. testcode:: shared + + print("before") + raise ValueError("boom") +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + stdout = sys.stdout + + result = run_group(plan, globs) + + assert sys.stdout is stdout + block = result.blocks[0] + assert isinstance(block, Failed) + assert isinstance(block.failures[0], doctest.UnexpectedException) + assert block.counts == Counts(failed=1, attempted=1, skipped=0) + rendered = "".join(traceback.format_exception(*block.failures[0].exc_info)) + assert "src/doctest_core/runner.py" not in rendered + assert "" in rendered + + +def test_default_exception_policy_retains_system_exit() -> None: + """The host-neutral default matches CPython's unexpected-exception rule.""" + plan = project( + parse_document(">>> raise SystemExit(7)\n", pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + block = result.blocks[0] + assert isinstance(block, Failed) + assert isinstance(block.failures[0], doctest.UnexpectedException) + assert result.primary is None + + +def test_prompt_profile_uses_stock_fail_fast_and_skip_accounting() -> None: + """Prompt execution retains CPython's option merge and attempt counts.""" + source = """ +.. doctest:: shared + + >>> 1 + 1 # doctest: +SKIP + 99 + >>> 2 + 2 + 5 + >>> 3 + 3 + 6 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings(continue_on_failure=False), + ) + + block = result.blocks[0] + assert isinstance(block, Failed) + stock_has_skip_count = hasattr(doctest.TestResults(0, 0), "skipped") + expected_attempts = 2 if stock_has_skip_count else 1 + assert block.counts == Counts(failed=1, attempted=expected_attempts, skipped=1) + assert len(block.failures) == 1 + + +def test_prompt_skip_count_excludes_unreached_examples() -> None: + """Old CPython fallback counts only skips reached before fail-fast.""" + source = """ +.. doctest:: shared + + >>> 1 + 1 + 3 + >>> 2 + 2 # doctest: +SKIP + 4 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings( + optionflags=doctest.FAIL_FAST, + continue_on_failure=True, + ), + ) + + block = result.blocks[0] + assert isinstance(block, Failed) + assert block.counts == Counts(failed=1, attempted=1, skipped=0) + + +def test_report_only_first_retains_total_failure_count() -> None: + """Report suppression does not reduce typed or summary failure totals.""" + source = """ +.. doctest:: shared + + >>> 1 + 1 + 3 + >>> 2 + 2 + 5 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings( + optionflags=doctest.REPORT_ONLY_FIRST_FAILURE, + continue_on_failure=True, + ), + ) + + block = result.blocks[0] + assert isinstance(block, Failed) + assert block.counts == Counts(failed=2, attempted=2, skipped=0) + assert len(block.failures) == 1 + + +def test_report_only_hidden_fail_fast_bounds_old_skip_fallback() -> None: + """A quiet stopping failure does not make later skips look examined.""" + source = """ +.. doctest:: shared + + >>> 1 + 1 + 3 + >>> 2 + 2 # doctest: +FAIL_FAST + 5 + >>> 3 + 3 # doctest: +SKIP + 6 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings( + optionflags=doctest.REPORT_ONLY_FIRST_FAILURE, + continue_on_failure=True, + ), + ) + + block = result.blocks[0] + assert isinstance(block, Failed) + assert block.counts == Counts(failed=2, attempted=2, skipped=0) + assert len(block.failures) == 1 + + +def test_inline_report_only_preserves_cpython_sequencing() -> None: + """Prompt reporting retains CPython's previous-example option timing.""" + source = """ +.. doctest:: shared + + >>> 1 + 1 # doctest: +REPORT_ONLY_FIRST_FAILURE + 3 + >>> 2 + 2 + 5 + >>> 3 + 3 + 7 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + block = result.blocks[0] + assert isinstance(block, Failed) + assert block.counts == Counts(failed=3, attempted=3, skipped=0) + assert [failure.example.source for failure in block.failures] == [ + "1 + 1 # doctest: +REPORT_ONLY_FIRST_FAILURE\n", + "3 + 3\n", + ] + + +def test_exec_profile_honors_explicit_fail_fast_option() -> None: + """The extended lane keeps runner flags distinct from host continuation.""" + source = """ +.. testcode:: shared + + print(1) + +.. testoutput:: shared + + 2 + +.. testcode:: shared + + print(3) + +.. testoutput:: shared + + 4 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings( + optionflags=doctest.FAIL_FAST, + continue_on_failure=True, + ), + ) + + assert len(result.blocks) == 1 + assert isinstance(result.blocks[0], Failed) + + +def test_default_run_settings_continue_across_failed_blocks() -> None: + """The host-neutral default follows doctest and Sphinx continuation.""" + source = """ +.. doctest:: shared + + >>> 1 + 1 + 3 + +.. doctest:: shared + + >>> 2 + 2 + 5 +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Failed, Failed] + + +def test_host_stop_policy_cannot_be_disabled_inline() -> None: + """An example cannot override the host's debugger-style stop policy.""" + source = """ +.. doctest:: shared + + >>> state = [] + >>> 1 + 1 # doctest: -FAIL_FAST + 3 + >>> state.append("ran") +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings(continue_on_failure=False), + ) + + assert isinstance(result.blocks[0], Failed) + assert result.blocks[0].counts == Counts(failed=1, attempted=2, skipped=0) + assert globs["state"] == [] + + +def test_cleanup_abort_is_not_demoted_after_test_failure() -> None: + """A process abort from cleanup remains stronger than block failures.""" + source = """ +.. testcode:: shared + + print(1) + +.. testoutput:: shared + + 2 + +.. testcleanup:: shared + + raise KeyboardInterrupt() +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert isinstance(result.blocks[0], Failed) + assert isinstance(result.primary, KeyboardInterrupt) + + +def test_exec_profile_accepts_expected_exception() -> None: + """A Sphinx testoutput traceback supplies the stock expected exception.""" + source = """ +.. testcode:: shared + + raise ValueError("boom") + +.. testoutput:: shared + + Traceback (most recent call last): + ... + ValueError: boom +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Passed] + + +def test_exec_profile_ignores_stdout_before_expected_exception() -> None: + """Expected exceptions compare the exception tail, as CPython does.""" + source = """ +.. testcode:: shared + + print("before") + raise ValueError("boom") + +.. testoutput:: shared + + Traceback (most recent call last): + ... + ValueError: boom +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Passed] + + +def test_exec_profile_honors_ignore_exception_detail() -> None: + """Expected exceptions retain CPython's detail-insensitive fallback.""" + source = """ +.. testcode:: shared + + raise ValueError("actual detail") + +.. testoutput:: shared + :options: +IGNORE_EXCEPTION_DETAIL + + Traceback (most recent call last): + ... + ValueError: expected detail +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Passed] + + +def test_expected_exception_mismatch_hides_runtime_frame() -> None: + """Failure traceback ownership starts at the author's compiled block.""" + source = """ +.. testcode:: shared + + raise ValueError("actual") + +.. testoutput:: shared + + Traceback (most recent call last): + ... + TypeError: expected +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + block = result.blocks[0] + assert isinstance(block, Failed) + failure = block.failures[0] + assert isinstance(failure, doctest.DocTestFailure) + assert "src/doctest_core/runner.py" not in failure.got + assert "" in failure.got + + +def test_exec_profile_normalizes_missing_stdout_newline() -> None: + """Extended output keeps doctest's unrepresentable-newline convention.""" + source = """ +.. testcode:: shared + + import sys + sys.stdout.write("answer") + +.. testoutput:: shared + + answer +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Passed] + + +def test_exec_profile_normalizes_syntax_error_details() -> None: + """Syntax errors compare from the exception line across Python versions.""" + source = """ +.. testcode:: shared + + compile("if:", "bad.py", "exec") + +.. testoutput:: shared + + Traceback (most recent call last): + ... + SyntaxError: invalid syntax +""" + plan = project( + parse_document(source, pathlib.Path("guide.rst")), + document_name="guide", + )[0] + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group(plan, globs) + + assert [type(block) for block in result.blocks] == [Passed] + + +def test_exec_profile_honors_inline_fail_fast() -> None: + """The current example's effective flags control extended execution.""" + block = ProjectedBlock( + phase=Phase.TEST, + name="guide::shared[0]", + block_ordinal=0, + examples=( + ExampleRecipe( + source='print("first")\n', + want="wrong\n", + exc_msg=None, + lineno=0, + indent=0, + options={doctest.FAIL_FAST: True}, + ), + ExampleRecipe( + source='print("second")\n', + want="wrong\n", + exc_msg=None, + lineno=1, + indent=0, + options={}, + ), + ), + docstring="", + filename="guide.rst", + lineno=0, + options={}, + profile_name="exec", + skipif=None, + pyversion=None, + expected=None, + ) + plan = GroupPlan("shared", (block,), {}) + globs: dict[str, object] = {} + reset_globs(plan, globs) + + result = run_group( + plan, + globs, + settings=RunSettings(continue_on_failure=True), + ) + + assert isinstance(result.blocks[0], Failed) + assert result.blocks[0].counts == Counts(failed=1, attempted=1, skipped=0) diff --git a/tests/test_doctest_core_sphinx.py b/tests/test_doctest_core_sphinx.py new file mode 100644 index 0000000..a07525d --- /dev/null +++ b/tests/test_doctest_core_sphinx.py @@ -0,0 +1,88 @@ +"""Acceptance tests for consuming Sphinx-resolved doctrees.""" + +from __future__ import annotations + +import pathlib +import typing as t + +from docutils import nodes +from docutils.utils import new_document + +from doctest_core import extract_blocks + +if t.TYPE_CHECKING: + from sphinx.testing.util import SphinxTestApp + + from .conftest import MakeAppParams + + +def test_extracts_hidden_blocks_from_sphinx_resolved_doctree( + make_app: t.Callable[..., SphinxTestApp], + make_app_params: MakeAppParams, + tmp_path: pathlib.Path, +) -> None: + """Resolved includes retain Sphinx comment nodes and source ownership.""" + included_path = tmp_path / "examples.rst" + included_path.write_text( + """.. testsetup:: shared + + value = 40 + +.. doctest:: shared + + >>> value + 2 + 42 + +.. testcleanup:: shared + + del value +""", + encoding="utf8", + ) + args, kwargs = make_app_params( + index="""Resolved page +============= + +.. include:: examples.rst +""", + confoverrides={"extensions": ["sphinx.ext.doctest"]}, + ) + app = make_app(*args, **kwargs) + app.build() + doctree = app.env.get_and_resolve_doctree("index", app.builder) + + sphinx_hidden_kinds = [ + str(node["testnodetype"]) + for node in doctree.findall(nodes.comment) + if node.get("testnodetype") in {"testsetup", "testcleanup"} + ] + assert sphinx_hidden_kinds == ["testsetup", "testcleanup"] + + result = extract_blocks(doctree) + + assert [block.kind for block in result.blocks] == [ + "testsetup", + "doctest", + "testcleanup", + ] + assert {block.path for block in result.blocks} == {included_path} + assert [block.hidden for block in result.blocks] == [True, False, True] + assert all(block.line is not None for block in result.blocks) + + +def test_sphinx_docstring_source_has_unknown_file_line() -> None: + """Docstring-relative node lines are not fabricated as file locations.""" + tree = new_document("module.rst") + node = nodes.literal_block( + ">>> 6 * 7\n42\n", + ">>> 6 * 7\n42\n", + testnodetype="doctest", + groups=["shared"], + ) + node.source = "/tmp/:docstring of package.module" + node.line = 7 + tree += node + + result = extract_blocks(tree) + + assert result.blocks[0].line is None diff --git a/tests/test_doctest_docutils.py b/tests/test_doctest_docutils.py index 42807e1..9b34ee8 100644 --- a/tests/test_doctest_docutils.py +++ b/tests/test_doctest_docutils.py @@ -336,3 +336,130 @@ def test_docutils_package_relative_error_message() -> None: exc = doctest_docutils.TestDocutilsPackageRelativeError() assert str(exc) == "Package may only be specified for module-relative paths." + + +def test_finder_keeps_stock_per_block_results() -> None: + """The compatibility finder returns independent stock doctest objects.""" + source = """.. testsetup:: shared + + value = 40 + +.. doctest:: shared + + >>> value + 2 + 42 + +.. testcleanup:: shared + + del value +""" + + tests = doctest_docutils.DocutilsDocTestFinder().find(source, "guide.rst") + + assert len(tests) == 3 + assert all(type(test) is doctest.DocTest for test in tests) + assert len({id(test.globs) for test in tests}) == 3 + + +def test_finder_preserves_source_order_past_single_digit_ordinals() -> None: + """Compatibility results retain source order instead of sorting names.""" + source = "\n".join(f">>> {ordinal}\n{ordinal}\n" for ordinal in range(12)) + + tests = doctest_docutils.DocutilsDocTestFinder().find(source, "guide.rst") + + assert [test.name for test in tests] == [ + f"guide.rst[{ordinal}]" for ordinal in range(12) + ] + + +def test_finder_preserves_zero_based_line_and_include_source( + tmp_path: pathlib.Path, +) -> None: + """Stock finder results report the block's physical source location.""" + included = tmp_path / "included.rst" + included.write_text(">>> 1 + 1\n3\n", encoding="utf-8") + root = tmp_path / "guide.rst" + root.write_text(".. include:: included.rst\n", encoding="utf-8") + + tests = doctest_docutils.DocutilsDocTestFinder().find( + root.read_text(encoding="utf-8"), + str(root), + ) + + assert len(tests) == 1 + assert tests[0].filename == str(included) + assert tests[0].lineno == 0 + + +def test_testdocutils_owns_group_lifecycle(tmp_path: pathlib.Path) -> None: + """The direct runner shares setup state and cleans up after failure.""" + source_path = tmp_path / "guide.rst" + source_path.write_text( + """.. testsetup:: shared + + value = 40 + +.. doctest:: shared + + >>> value + 2 + 99 + +.. testcleanup:: shared + + cleaned.append(value) +""", + encoding="utf-8", + ) + cleaned: list[int] = [] + + result = doctest_docutils.testdocutils( + str(source_path), + module_relative=False, + globs={"cleaned": cleaned}, + report=False, + ) + + assert result.failed == 1 + assert result.attempted == 1 + assert cleaned == [40] + + +def test_testdocutils_retains_report_suppressed_failure_total( + tmp_path: pathlib.Path, +) -> None: + """The direct summary count is independent of detailed failure reports.""" + source_path = tmp_path / "guide.rst" + source_path.write_text( + ">>> 1 + 1\n3\n>>> 2 + 2\n5\n", + encoding="utf-8", + ) + + result = doctest_docutils.testdocutils( + str(source_path), + module_relative=False, + report=False, + optionflags=doctest.REPORT_ONLY_FIRST_FAILURE, + ) + + assert result.failed == 2 + assert result.attempted == 2 + + +def test_testdocutils_returns_modern_skip_statistics( + tmp_path: pathlib.Path, +) -> None: + """The direct facade retains CPython's version-shaped skip total.""" + source_path = tmp_path / "guide.rst" + source_path.write_text( + ">>> 1 + 1 # doctest: +SKIP\n99\n>>> 2 + 2\n4\n", + encoding="utf-8", + ) + + result = doctest_docutils.testdocutils( + str(source_path), + module_relative=False, + report=False, + ) + + if hasattr(result, "skipped"): + assert t.cast(t.Any, result).skipped == 1 diff --git a/tests/test_doctest_options.py b/tests/test_doctest_options.py index aa9cd68..d2537d6 100644 --- a/tests/test_doctest_options.py +++ b/tests/test_doctest_options.py @@ -204,7 +204,7 @@ def test_doctest_options( pytester.plugins = ["pytest_doctest_docutils"] # Build pytest.ini content - ini_lines = ["[pytest]", "addopts=-p no:doctest -vv"] + ini_lines = ["[pytest]", "addopts=-vv"] if ini_options: ini_lines.append(ini_options) ini_content = "\n".join(ini_lines) @@ -300,7 +300,7 @@ def test_continue_on_failure( When enabled, all doctest failures should be reported, not just the first. """ pytester.plugins = ["pytest_doctest_docutils"] - pytester.makefile(".ini", pytest="[pytest]\naddopts=-p no:doctest -vv") + pytester.makefile(".ini", pytest="[pytest]\naddopts=-vv") # Create the test file filename = f"test_doc{file_ext}" @@ -380,7 +380,7 @@ def test_custom_flags( """ pytester.plugins = ["pytest_doctest_docutils"] - ini_lines = ["[pytest]", "addopts=-p no:doctest -vv"] + ini_lines = ["[pytest]", "addopts=-vv"] if ini_options: ini_lines.append(ini_options) ini_content = "\n".join(ini_lines) @@ -484,7 +484,7 @@ def test_edge_cases( Tests empty files and files without doctests. """ pytester.plugins = ["pytest_doctest_docutils"] - pytester.makefile(".ini", pytest="[pytest]\naddopts=-p no:doctest -vv") + pytester.makefile(".ini", pytest="[pytest]\naddopts=-vv") # Create the test file filename = f"test_doc{file_ext}" @@ -494,8 +494,8 @@ def test_edge_cases( result = pytester.runpytest(str(file_path), "-v") if expected_outcome == "no_tests": - # Should collect 0 tests (file may be collected but no items) + result.assert_outcomes(errors=0) stdout = result.stdout.str() - assert "0 items" in stdout or "no tests ran" in stdout or expected_tests == 0 + assert "0 items" in stdout or "no tests ran" in stdout elif expected_outcome == "passed": result.assert_outcomes(passed=expected_tests) diff --git a/tests/test_plugin_suppression.py b/tests/test_plugin_suppression.py index 09b14cb..7042fbd 100644 --- a/tests/test_plugin_suppression.py +++ b/tests/test_plugin_suppression.py @@ -1,13 +1,14 @@ -"""Test pytest plugin suppression and precedence. +"""Test pytest doctest collector composition and precedence. -Tests for pytest plugin blocking behavior in pytest_doctest_docutils. -Ensures plugin suppression works correctly across pytest 7.x/8.x/9.x. +Documentation paths have one owner while pytest's doctest plugin remains +available for Python modules and fixture injection. Ref: pytest's test_pluginmanager.py patterns for plugin blocking tests. """ from __future__ import annotations +import importlib.metadata import re import textwrap import typing as t @@ -19,6 +20,17 @@ PYTEST_VERSION = tuple(int(x) for x in pytest.__version__.split(".")[:2]) +def test_pytest_entry_point_uses_adapter_name() -> None: + """The installed plugin can be disabled by its module-shaped name.""" + entries = [ + entry + for entry in importlib.metadata.entry_points(group="pytest11") + if entry.value == "pytest_doctest_docutils" + ] + + assert [entry.name for entry in entries] == ["pytest_doctest_docutils"] + + def requires_pytest_version( min_version: tuple[int, int], reason: str, @@ -55,18 +67,11 @@ class PluginSuppressionCase(t.NamedTuple): PLUGIN_SUPPRESSION_CASES = [ PluginSuppressionCase( - test_id="auto-blocks-builtin-doctest", + test_id="composes-with-builtin-doctest", cli_args=["--collect-only", "-q"], ini_content="", expected_tests_collected=1, - description="pytest_doctest_docutils auto-blocks built-in doctest", - ), - PluginSuppressionCase( - test_id="ini-addopts-no-doctest", - cli_args=["--collect-only", "-q"], - ini_content="addopts = -p no:doctest", - expected_tests_collected=1, - description="addopts=-p no:doctest in pytest.ini works", + description="the adapter filters only the duplicate collector", ), ] @@ -76,7 +81,7 @@ class PluginSuppressionCase(t.NamedTuple): PLUGIN_SUPPRESSION_CASES, ids=[c.test_id for c in PLUGIN_SUPPRESSION_CASES], ) -def test_plugin_suppression( +def test_collector_composition( pytester: _pytest.pytester.Pytester, test_id: str, cli_args: list[str], @@ -84,11 +89,7 @@ def test_plugin_suppression( expected_tests_collected: int, description: str, ) -> None: - """Test plugin suppression behavior. - - Verifies that pytest_doctest_docutils correctly blocks the built-in - doctest plugin to prevent duplicate test collection. - """ + """Verify documentation paths collect exactly once.""" pytester.plugins = ["pytest_doctest_docutils"] # Create pytest.ini if content provided @@ -252,18 +253,12 @@ def hello(): result.assert_outcomes(passed=expected_passed) -def test_pytest_configure_blocks_doctest( +def test_pytest_configure_keeps_doctest( pytester: _pytest.pytester.Pytester, ) -> None: - """Test that pytest_configure automatically blocks the doctest plugin. - - This tests the core behavior in pytest_doctest_docutils.pytest_configure: - if config.pluginmanager.has_plugin("doctest"): - config.pluginmanager.set_blocked("doctest") - """ + """The adapter keeps the built-in doctest plugin registered.""" pytester.plugins = ["pytest_doctest_docutils"] - # Create conftest that checks plugin state after configuration pytester.makeconftest( textwrap.dedent( """ @@ -271,31 +266,27 @@ def test_pytest_configure_blocks_doctest( @pytest.hookimpl(trylast=True) def pytest_configure(config): - # After all pytest_configure hooks run, doctest should be blocked pm = config.pluginmanager - # is_blocked exists in pytest 7+ - if hasattr(pm, 'is_blocked'): - # Store result for test to check - config._doctest_was_blocked = pm.is_blocked('doctest') - else: - config._doctest_was_blocked = None + config._doctest_was_blocked = pm.is_blocked('doctest') + config._doctest_is_loaded = pm.has_plugin('doctest') @pytest.fixture - def doctest_blocked_status(request): - return getattr(request.config, '_doctest_was_blocked', None) + def doctest_plugin_status(request): + return ( + request.config._doctest_was_blocked, + request.config._doctest_is_loaded, + ) """, ), ) - # Create test that verifies the blocking happened pytester.makepyfile( test_verify=textwrap.dedent( """ - def test_doctest_was_blocked(doctest_blocked_status): - if doctest_blocked_status is not None: - assert doctest_blocked_status is True, ( - "doctest plugin should be blocked by pytest_doctest_docutils" - ) + def test_doctest_is_composed(doctest_plugin_status): + blocked, loaded = doctest_plugin_status + assert blocked is False + assert loaded is True """, ), ) @@ -382,7 +373,7 @@ def test_collector_routing( pytester.plugins = ["pytest_doctest_docutils"] pytester.makefile( ".ini", - pytest="[pytest]\naddopts=-p no:doctest -vv", + pytest="[pytest]\naddopts=-vv", ) # Create the test file @@ -404,79 +395,6 @@ def test_collector_routing( ) -# pytest 8.1+ version-specific tests - - -@requires_pytest_version((8, 1), "pluginmanager.unblock() API") -def test_unblock_api_available( - pytester: _pytest.pytester.Pytester, -) -> None: - """Test pluginmanager.unblock() API available in pytest 8.1+. - - Verifies that the unblock() method exists and can be used to - re-enable a previously blocked plugin. - - Ref: pytest 8.1.0 changelog - pluginmanager.unblock() public API - """ - pytester.plugins = ["pytest_doctest_docutils"] - - # Create conftest that tests unblock API - pytester.makeconftest( - textwrap.dedent( - """ - import pytest - - @pytest.hookimpl(trylast=True) - def pytest_configure(config): - pm = config.pluginmanager - - # Verify unblock method exists - assert hasattr(pm, 'unblock'), "unblock() API not found" - - # doctest should be blocked by pytest_doctest_docutils - assert pm.is_blocked('doctest'), "doctest should be blocked" - - # Test unblock API - result = pm.unblock('doctest') - - # Store results for test verification - config._unblock_api_exists = True - config._unblock_result = result - config._doctest_unblocked = not pm.is_blocked('doctest') - - @pytest.fixture - def unblock_test_results(request): - return { - 'api_exists': getattr(request.config, '_unblock_api_exists', False), - 'unblock_result': getattr(request.config, '_unblock_result', None), - 'doctest_unblocked': getattr( - request.config, '_doctest_unblocked', False - ), - } - """, - ), - ) - - # Create test that verifies unblock worked - pytester.makepyfile( - test_verify=textwrap.dedent( - """ - def test_unblock_api_works(unblock_test_results): - assert unblock_test_results['api_exists'], "unblock() API should exist" - assert unblock_test_results['unblock_result'] is True, ( - "unblock() should return True when successful" - ) - assert unblock_test_results['doctest_unblocked'], ( - "doctest should be unblocked after calling unblock()" - ) - """, - ), - ) - - result = pytester.runpytest("test_verify.py", "-v") - result.assert_outcomes(passed=1) - - # pytest 8.4+ version-specific tests diff --git a/tests/test_pytest_doctest_docutils.py b/tests/test_pytest_doctest_docutils.py index 70fbd5a..577ab2d 100644 --- a/tests/test_pytest_doctest_docutils.py +++ b/tests/test_pytest_doctest_docutils.py @@ -216,7 +216,7 @@ def test_pluginDocutilsDocTestFinder( pytest=textwrap.dedent( """ [pytest] -addopts=-p no:doctest -vv +addopts=-vv """.strip(), ), @@ -250,7 +250,7 @@ def test_conftest_py( pytest=textwrap.dedent( """ [pytest] -addopts=-p no:doctest -vv +addopts=-vv """.strip(), ), @@ -321,7 +321,7 @@ def test_conftest_md( pytest=textwrap.dedent( """ [pytest] -addopts=-p no:doctest -vv +addopts=-vv """.strip(), ), @@ -443,7 +443,7 @@ def test_ignore_build_artifacts( pytest=textwrap.dedent( """ [pytest] -addopts=-p no:doctest -vv +addopts=-vv """.strip(), ), @@ -492,15 +492,6 @@ def test_hide_optionflag_py_docstring( ``ValueError: ... invalid option: '+HIDE'``. Here it must simply run. """ pytester.plugins = ["pytest_doctest_docutils"] - pytester.makefile( - ".ini", - pytest=textwrap.dedent( - """ -[pytest] -addopts=-p no:doctest - """.strip(), - ), - ) example = pytester.path / "example.py" example.write_text( textwrap.dedent( diff --git a/uv.lock b/uv.lock index a5aa06a..8a51590 100644 --- a/uv.lock +++ b/uv.lock @@ -115,6 +115,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] +[[package]] +name = "backports-asyncio-runner" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/ff/70dca7d7cb1cbc0edb2c6cc0c38b65cba36cccc491eca64cabd5fe7f8670/backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162", size = 69893, upload-time = "2025-07-02T02:27:15.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.15.0" @@ -382,6 +391,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "gp-furo-theme" version = "0.1.0a37" @@ -407,6 +425,8 @@ dependencies = [ { name = "docutils" }, { name = "myst-parser", version = "4.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "myst-parser", version = "5.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, + { name = "pytest" }, ] [package.dev-dependencies] @@ -422,10 +442,12 @@ dev = [ { name = "gp-sphinx" }, { name = "mypy" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "pytest-rerunfailures" }, { name = "pytest-watcher" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "sphinx-autobuild", version = "2024.10.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "sphinx-autobuild", version = "2025.8.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -449,15 +471,19 @@ lint = [ testing = [ { name = "gp-libs" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-mock" }, { name = "pytest-rerunfailures" }, { name = "pytest-watcher" }, + { name = "pytest-xdist" }, ] [package.metadata] requires-dist = [ - { name = "docutils" }, - { name = "myst-parser" }, + { name = "docutils", specifier = ">=0.20.1,<0.22" }, + { name = "myst-parser", specifier = ">=2.0.0" }, + { name = "packaging" }, + { name = "pytest", specifier = ">=7.2" }, ] [package.metadata.requires-dev] @@ -473,10 +499,12 @@ dev = [ { name = "gp-sphinx", specifier = "==0.1.0a37" }, { name = "mypy" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "pytest-rerunfailures" }, { name = "pytest-watcher" }, + { name = "pytest-xdist" }, { name = "ruff", specifier = ">=0.16.0" }, { name = "sphinx-autobuild" }, { name = "sphinx-autodoc-api-style", specifier = "==0.1.0a37" }, @@ -498,9 +526,11 @@ lint = [ testing = [ { name = "gp-libs" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-mock" }, { name = "pytest-rerunfailures" }, { name = "pytest-watcher" }, + { name = "pytest-xdist" }, ] [[package]] @@ -981,6 +1011,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-asyncio-runner", marker = "python_full_version < '3.11'" }, + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + [[package]] name = "pytest-cov" version = "7.1.0" @@ -1033,6 +1077,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/3f/172d73600ad2771774cda108efb813fc724fc345e5240a81a1085f1ade5d/pytest_watcher-0.6.3-py3-none-any.whl", hash = "sha256:83e7748c933087e8276edb6078663e6afa9926434b4fd8b85cf6b32b1d5bec89", size = 12431, upload-time = "2026-01-10T23:28:17.64Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3"