From 9cf376d7f24b87a0f7d7e4c990c682d6435ebcb5 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:03:57 -0300 Subject: [PATCH 01/30] test: migrate Python binding checks to pytest --- .../task-1-report.md | 42 ++ native-lib/build.gradle | 19 +- native-lib/python/pyproject.toml | 9 + native-lib/python/pytest.ini | 7 + native-lib/python/tests/conftest.py | 27 + .../tests/integration/test_callbacks.py | 84 +++ .../tests/integration/test_execution.py | 63 ++ .../tests/integration/test_lifecycle.py | 10 + .../tests/integration/test_streaming.py | 115 ++++ .../python/tests/test_dataweave_module.py | 539 ------------------ 10 files changed, 375 insertions(+), 540 deletions(-) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md create mode 100644 native-lib/python/pytest.ini create mode 100644 native-lib/python/tests/conftest.py create mode 100644 native-lib/python/tests/integration/test_callbacks.py create mode 100644 native-lib/python/tests/integration/test_execution.py create mode 100644 native-lib/python/tests/integration/test_lifecycle.py create mode 100644 native-lib/python/tests/integration/test_streaming.py delete mode 100755 native-lib/python/tests/test_dataweave_module.py diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md new file mode 100644 index 00000000..9e4e2ddc --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md @@ -0,0 +1,42 @@ +# Task 1 Report: Pytest Lanes And Existing Behavior + +## Files Changed + +- `native-lib/python/pytest.ini`: registers `unit`, `integration`, and `tck` markers; excludes TCK tests by default. +- `native-lib/python/pyproject.toml`: declares the `test` optional dependency group with `pytest` and `pytest-cov`. +- `native-lib/python/tests/conftest.py`: adds source-path setup, automatic module-global runtime cleanup, and a streaming collection fixture. +- `native-lib/python/tests/integration/test_execution.py`: migrates five execution and input-conversion scenarios. +- `native-lib/python/tests/integration/test_streaming.py`: migrates eight streaming and transform scenarios. +- `native-lib/python/tests/integration/test_callbacks.py`: migrates four callback streaming scenarios. +- `native-lib/python/tests/integration/test_lifecycle.py`: migrates the explicit lifecycle scenario. +- `native-lib/python/tests/test_dataweave_module.py`: removed the superseded hand-run script. +- `native-lib/build.gradle`: makes `pythonTest` run normal pytest lanes and write JUnit plus coverage XML under `native-lib/build`. + +## Design Choices + +- Retained all 18 legacy scenarios as separately named `@pytest.mark.integration` tests, grouped by execution mode. +- Used an autouse fixture to call `dataweave.cleanup()` before and after every test, isolating the module-level native runtime without changing public runtime behavior or the C ABI. +- Kept TCK excluded from default pytest collection and made Gradle explicitly run only `unit or integration` markers. +- Wrote test reports to `native-lib/build/test-results/python/junit.xml` and coverage to `native-lib/build/reports/coverage/python/coverage.xml`. + +## Tests Run + +1. `/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python -m pip install '.[test]'` + - Passed. Installed `pytest 8.4.2` and `pytest-cov 7.1.0` in an isolated temporary virtual environment. +2. `/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python -m pytest tests/integration -m integration -v` + - Passed: `18 passed in 0.39s`. +3. `./gradlew native-lib:pythonTest -PpythonExe=/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python` + - Passed: native `dwlib` compiled and staged; pytest reported `18 passed in 1.04s`; JUnit and coverage XML were written to the configured build paths. +4. `pytest -m 'not tck' --collect-only -q` using the temporary virtual environment + - Passed: 18 tests collected, confirming default TCK exclusion. +5. `git diff --check` + - Passed. + +## Commit + +- Pending creation: `test: migrate Python binding checks to pytest` + +## Concerns + +- The system Python does not have pytest and cannot write its global site-packages. Verification used an isolated temporary virtual environment supplied through Gradle's existing `-PpythonExe` override. +- Native-image and Gradle emit existing Java/native-image deprecation warnings during the native build; the task itself completed successfully. diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 436f6066..7ecf3df2 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -155,7 +155,24 @@ tasks.register('pythonTest', Exec) { dependsOn tasks.named('stagePythonNativeLib') workingDir("${projectDir}/python") - commandLine(pythonExe, 'tests/test_dataweave_module.py') + def testReportsDir = layout.buildDirectory.dir('test-results/python') + def coverageReportsDir = layout.buildDirectory.dir('reports/coverage/python') + outputs.file(testReportsDir.map { it.file('junit.xml') }) + outputs.file(coverageReportsDir.map { it.file('coverage.xml') }) + doFirst { + testReportsDir.get().asFile.mkdirs() + coverageReportsDir.get().asFile.mkdirs() + } + commandLine( + pythonExe, + '-m', + 'pytest', + '-m', + 'unit or integration', + '--junitxml', testReportsDir.map { it.file('junit.xml') }.get().asFile, + '--cov=dataweave', + '--cov-report=xml:' + coverageReportsDir.map { it.file('coverage.xml') }.get().asFile, + ) } // --- Node.js native package tasks --- diff --git a/native-lib/python/pyproject.toml b/native-lib/python/pyproject.toml index 642ab3ce..03a366ae 100644 --- a/native-lib/python/pyproject.toml +++ b/native-lib/python/pyproject.toml @@ -1,3 +1,12 @@ [build-system] requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" + +[project] +name = "dataweave-native" +version = "0.0.1" +description = "Python bindings for the DataWeave native library" +requires-python = ">=3.9" + +[project.optional-dependencies] +test = ["pytest", "pytest-cov"] diff --git a/native-lib/python/pytest.ini b/native-lib/python/pytest.ini new file mode 100644 index 00000000..09f6009a --- /dev/null +++ b/native-lib/python/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +testpaths = tests +addopts = -m "not tck" +markers = + unit: fast tests that do not require the native DataWeave library + integration: tests that execute the staged native DataWeave library + tck: DataWeave conformance tests excluded from normal test runs diff --git a/native-lib/python/tests/conftest.py b/native-lib/python/tests/conftest.py new file mode 100644 index 00000000..7eac9ca2 --- /dev/null +++ b/native-lib/python/tests/conftest.py @@ -0,0 +1,27 @@ +import sys +from pathlib import Path + +import pytest + + +PYTHON_SRC_DIR = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(PYTHON_SRC_DIR)) + +import dataweave + + +@pytest.fixture(autouse=True) +def clean_dataweave_runtime(): + """Keep module-level isolate state from leaking between integration tests.""" + dataweave.cleanup() + yield + dataweave.cleanup() + + +@pytest.fixture +def collect_stream(): + def collect(stream): + chunks = list(stream) + return b"".join(chunks), stream.metadata + + return collect diff --git a/native-lib/python/tests/integration/test_callbacks.py b/native-lib/python/tests/integration/test_callbacks.py new file mode 100644 index 00000000..c9b11157 --- /dev/null +++ b/native-lib/python/tests/integration/test_callbacks.py @@ -0,0 +1,84 @@ +import io + +import pytest + +import dataweave + + +@pytest.mark.integration +def test_callback_streams_basic_output(): + chunks = [] + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_callback("2 + 2", on_write) + + assert result.success is True + assert b"".join(chunks).decode(result.charset or "utf-8") == "4" + + +@pytest.mark.integration +def test_callback_streams_output_with_inputs(): + chunks = [] + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_callback("num1 + num2", on_write, inputs={"num1": 25, "num2": 17}) + + assert result.success is True + assert b"".join(chunks).decode(result.charset or "utf-8") == "42" + + +@pytest.mark.integration +def test_callback_transforms_streamed_input_and_output(): + source = io.BytesIO(b"[10, 20, 30, 40, 50]") + chunks = [] + + def on_read(buffer_size: int) -> bytes: + return source.read(buffer_size) + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_input_output_callback( + "output application/json\n---\npayload map ($ * 2)", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + output = b"".join(chunks).decode(result.charset or "utf-8") + assert result.success is True + assert "20" in output + assert "100" in output + + +@pytest.mark.integration +def test_callback_accepts_large_streamed_input(): + records = b"[" + b",".join(f'{{"id":{index}}}'.encode() for index in range(1, 1001)) + b"]" + source = io.BytesIO(records) + chunks = [] + + def on_read(buffer_size: int) -> bytes: + return source.read(buffer_size) + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_input_output_callback( + "output application/json\n---\nsizeOf(payload)", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + assert result.success is True + assert b"".join(chunks).decode(result.charset or "utf-8") == "1000" diff --git a/native-lib/python/tests/integration/test_execution.py b/native-lib/python/tests/integration/test_execution.py new file mode 100644 index 00000000..3768163d --- /dev/null +++ b/native-lib/python/tests/integration/test_execution.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import pytest + +import dataweave + + +@pytest.mark.integration +def test_input_value_accepts_public_mime_type_constructor_keyword(): + value = dataweave.InputValue( + content="1234567", + mime_type="application/csv", + properties={"header": False, "separator": "4"}, + ) + + assert value.mime_type == "application/csv" + + +@pytest.mark.integration +def test_runs_basic_script(): + result = dataweave.run("2 + 2", {}) + + assert result.get_string() == "4" + + +@pytest.mark.integration +def test_runs_script_with_inputs(): + result = dataweave.run("num1 + num2", {"num1": 25, "num2": 17}) + + assert result.get_string() == "42" + + +@pytest.mark.integration +def test_converts_utf16_xml_input_to_csv(): + xml_path = Path(__file__).resolve().parents[1] / "person.xml" + script = """output application/csv header=true +--- +[payload.person] +""" + + result = dataweave.run( + script, + { + "payload": { + "content": xml_path.read_bytes(), + "mimeType": "application/xml", + "charset": "UTF-16", + } + }, + ) + + output = result.get_string() or "" + assert result.success is True + assert "name" in output and "age" in output + assert "Billy" in output + assert "31" in output + + +@pytest.mark.integration +def test_converts_python_list_input_automatically(): + result = dataweave.run("numbers[0]", {"numbers": [1, 2, 3]}) + + assert result.get_string() == "1" diff --git a/native-lib/python/tests/integration/test_lifecycle.py b/native-lib/python/tests/integration/test_lifecycle.py new file mode 100644 index 00000000..a91f1049 --- /dev/null +++ b/native-lib/python/tests/integration/test_lifecycle.py @@ -0,0 +1,10 @@ +import pytest + +import dataweave + + +@pytest.mark.integration +def test_context_manager_runs_multiple_scripts(): + with dataweave.DataWeave() as dw: + assert dw.run("sqrt(144)").get_string() == "12" + assert dw.run("sqrt(10000)").get_string() == "100" diff --git a/native-lib/python/tests/integration/test_streaming.py b/native-lib/python/tests/integration/test_streaming.py new file mode 100644 index 00000000..dc92c272 --- /dev/null +++ b/native-lib/python/tests/integration/test_streaming.py @@ -0,0 +1,115 @@ +import json +from pathlib import Path + +import pytest + +import dataweave + + +@pytest.mark.integration +def test_run_streaming_returns_chunks_and_metadata(collect_stream): + output, metadata = collect_stream(dataweave.run_streaming("output application/json --- {a: 1, b: 2}")) + + text = output.decode(metadata.charset or "utf-8") + assert output + assert '"a": 1' in text or '"a":1' in text + assert metadata.success is True + assert metadata.mime_type == "application/json" + + +@pytest.mark.integration +def test_run_streaming_splits_large_output_into_multiple_chunks(collect_stream): + stream = dataweave.run_streaming('output application/json --- (1 to 5000) map {id: $, name: "item_" ++ $}') + chunks = list(stream) + output = b"".join(chunks) + metadata = stream.metadata + + text = output.decode(metadata.charset or "utf-8") + assert metadata.success is True + assert len(chunks) > 1 + assert '"id": 5000' in text or '"id":5000' in text + + +@pytest.mark.integration +def test_run_streaming_returns_error_metadata(collect_stream): + output, metadata = collect_stream(dataweave.run_streaming("output application/json --- invalid_var")) + + assert metadata.success is False + assert metadata.error is not None + assert output == b"" + + +@pytest.mark.integration +def test_run_streaming_accepts_input_bindings(collect_stream): + output, metadata = collect_stream(dataweave.run_streaming("num1 + num2", {"num1": 25, "num2": 17})) + + assert metadata.success is True + assert output.decode(metadata.charset or "utf-8").strip() == "42" + + +@pytest.mark.integration +def test_run_transform_streams_iterable_input(collect_stream): + stream = dataweave.run_transform( + "output application/json\n---\npayload map ($ * 2)", + input_stream=[b"[10, 20, 30, 40, 50]"], + input_mime_type="application/json", + ) + output, metadata = collect_stream(stream) + + text = output.decode(metadata.charset or "utf-8") + assert metadata.success is True + assert "20" in text + assert "100" in text + + +@pytest.mark.integration +def test_run_transform_reads_chunked_input(collect_stream): + input_data = b"[" + b",".join(f'{{"id":{index}}}'.encode() for index in range(1, 1001)) + b"]" + + def chunked(): + for index in range(0, len(input_data), 4096): + yield input_data[index:index + 4096] + + stream = dataweave.run_transform( + "output application/json\n---\nsizeOf(payload)", + input_stream=chunked(), + input_mime_type="application/json", + ) + output, metadata = collect_stream(stream) + + assert metadata.success is True + assert output.decode(metadata.charset or "utf-8") == "1000" + + +@pytest.mark.integration +def test_run_transform_preserves_large_single_input_chunk(collect_stream): + payload = json.dumps([{"id": index, "name": f"item_{index}", "value": index * 3} for index in range(1, 2001)]).encode() + assert len(payload) > 8192 + + stream = dataweave.run_transform( + "output application/json\n---\nsizeOf(payload)", + input_stream=iter([payload]), + input_mime_type="application/json", + ) + output, metadata = collect_stream(stream) + + assert metadata.success is True + assert output.decode(metadata.charset or "utf-8") == "2000" + + +@pytest.mark.integration +def test_run_transform_reads_file_input(collect_stream): + xml_path = Path(__file__).resolve().parents[1] / "person.xml" + with xml_path.open("rb") as source: + stream = dataweave.run_transform( + "output application/csv header=true\n---\n[payload.person]", + input_stream=iter(lambda: source.read(4096), b""), + input_mime_type="application/xml", + input_charset="UTF-16", + ) + output, metadata = collect_stream(stream) + + text = output.decode(metadata.charset or "utf-8") + assert metadata.success is True + assert "Billy" in text + assert "31" in text diff --git a/native-lib/python/tests/test_dataweave_module.py b/native-lib/python/tests/test_dataweave_module.py deleted file mode 100755 index 959834b3..00000000 --- a/native-lib/python/tests/test_dataweave_module.py +++ /dev/null @@ -1,539 +0,0 @@ -#!/usr/bin/env python3 -""" -Quick test script for the DataWeave Python module. -""" - -import sys -from pathlib import Path - -_PYTHON_SRC_DIR = Path(__file__).resolve().parents[1] / "src" -sys.path.insert(0, str(_PYTHON_SRC_DIR)) - -import dataweave - -def test_input_value_mime_type_constructor(): - """Test InputValue accepts the public mime_type constructor keyword.""" - print("Testing InputValue mime_type constructor...") - try: - value = dataweave.InputValue( - content="1234567", - mime_type="application/csv", - properties={"header": False, "separator": "4"}, - ) - assert value.mime_type == "application/csv" - print("[OK] InputValue mime_type constructor works") - return True - except Exception as e: - print(f"[FAIL] InputValue mime_type constructor failed: {e}") - return False - -def test_basic(): - """Test basic functionality""" - print("Testing basic script execution...") - try: - result = dataweave.run("2 + 2", {}) - assert result.get_string() == "4", f"Expected '4', got '{result.get_string()}'" - print("[OK] Basic script execution works") - return True - except Exception as e: - print(f"[FAIL] Basic script execution failed: {e}") - return False - -def test_with_inputs(): - """Test script with inputs""" - print("\nTesting script with inputs...") - try: - result = dataweave.run("num1 + num2", {"num1": 25, "num2": 17}) - assert result.get_string() == "42", f"Expected '42', got '{result.get_string()}'" - print("[OK] Script with inputs works") - return True - except Exception as e: - print(f"[FAIL] Script with inputs failed: {e}") - return False - -def test_context_manager(): - """Test context manager""" - print("\nTesting with context manager...") - try: - with dataweave.DataWeave() as dw: - - result = dw.run("sqrt(144)") - assert result.get_string() == "12", f"Expected '12', got '{result.get_string()}'" - result = dw.run("sqrt(10000)") - assert result.get_string() == "100", f"Expected '100', got '{result.get_string()}'" - print("[OK] Script execution witch context manager works") - return True - except Exception as e: - print(f"[FAIL] Script execution witch context manager failed: {e}") - return False - -def test_encoding(): - """Test reading UTF-16 XML input and producing CSV output""" - print("\nTesting encoding (UTF-16 XML -> CSV)...") - try: - xml_path = ( - Path(__file__).resolve().parent / "person.xml" - ) - xml_bytes = xml_path.read_bytes() - - script = """output application/csv header=true ---- -[payload.person] -""" - - result = dataweave.run( - script, - { - "payload": { - "content": xml_bytes, - "mimeType": "application/xml", - "charset": "UTF-16", - } - }, - ) - - out = result.get_string() or "" - print(f"out: \n{out}") - assert result.success is True, f"Expected success=true, got: {result}" - assert "name" in out and "age" in out, f"CSV header missing, got: {out!r}" - assert "Billy" in out, f"Expected name 'Billy' in CSV, got: {out!r}" - assert "31" in out, f"Expected age '31' in CSV, got: {out!r}" - - print("[OK] Encoding conversion works") - return True - except Exception as e: - print(f"[FAIL] Encoding conversion failed: {e}") - return False - -def test_auto_conversion(): - """Test auto-conversion of different types""" - print("\nTesting auto-conversion...") - try: - - # Test array - result = dataweave.run( - "numbers[0]", - {"numbers": [1, 2, 3]} - ) - assert result.get_string() == "1", f"Expected '1', got '{result.get_string()}'" - - print("[OK] Auto-conversion works") - return True - except Exception as e: - print(f"[FAIL] Auto-conversion failed: {e}") - return False - -def test_callback_output_basic(): - """Test callback-based output streaming""" - print("\nTesting callback output basic...") - try: - chunks = [] - - def on_write(data: bytes) -> int: - chunks.append(data) - return 0 - - result = dataweave.run_callback("2 + 2", on_write) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(chunks) - text = full.decode(result.charset or "utf-8") - assert text == "4", f"Expected '4', got '{text}'" - print(f"[OK] Callback output basic works (chunks={len(chunks)}, result='{text}')") - return True - except Exception as e: - print(f"[FAIL] Callback output basic failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_callback_output_with_inputs(): - """Test callback-based output streaming with inputs""" - print("\nTesting callback output with inputs...") - try: - chunks = [] - - def on_write(data: bytes) -> int: - chunks.append(data) - return 0 - - result = dataweave.run_callback("num1 + num2", on_write, inputs={"num1": 25, "num2": 17}) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(chunks) - text = full.decode(result.charset or "utf-8") - assert text == "42", f"Expected '42', got '{text}'" - print(f"[OK] Callback output with inputs works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] Callback output with inputs failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_callback_input_output(): - """Test callback-based input and output streaming""" - print("\nTesting callback input+output...") - try: - import io as _io - - source = _io.BytesIO(b'[10, 20, 30, 40, 50]') - output_chunks = [] - - def on_read(buf_size: int) -> bytes: - return source.read(buf_size) - - def on_write(data: bytes) -> int: - output_chunks.append(data) - return 0 - - script = "output application/json\n---\npayload map ($ * 2)" - result = dataweave.run_input_output_callback( - script, - input_name="payload", - input_mime_type="application/json", - read_callback=on_read, - write_callback=on_write, - ) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(output_chunks) - text = full.decode(result.charset or "utf-8") - assert "20" in text, f"Expected 20 in result (10*2), got: {text}" - assert "100" in text, f"Expected 100 in result (50*2), got: {text}" - print(f"[OK] Callback input+output works (chunks={len(output_chunks)}, result={text.strip()[:80]}...)") - return True - except Exception as e: - print(f"[FAIL] Callback input+output failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_basic(): - """Test run_streaming yields chunks and returns metadata""" - print("\nTesting run_streaming basic...") - try: - stream = dataweave.run_streaming("output application/json --- {a: 1, b: 2}") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - assert len(full) > 0, "Expected non-empty output" - text = full.decode(metadata.charset or "utf-8") - assert '"a": 1' in text or '"a":1' in text, f"Expected key 'a' in JSON, got: {text}" - assert metadata.success is True, f"Expected success, got: {metadata}" - assert metadata.mime_type == "application/json", f"Expected json mime, got: {metadata.mime_type}" - print(f"[OK] run_streaming basic works (chunks={len(chunks)}, result={text.strip()[:60]})") - return True - except Exception as e: - print(f"[FAIL] run_streaming basic failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_large(): - """Test run_streaming with large output to verify true streaming (multiple chunks)""" - print("\nTesting run_streaming large...") - try: - script = 'output application/json --- (1 to 5000) map {id: $, name: "item_" ++ $}' - stream = dataweave.run_streaming(script) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert len(chunks) > 1, f"Expected multiple chunks for large output, got {len(chunks)}" - assert '"id": 5000' in text or '"id":5000' in text, f"Expected last item in output" - print(f"[OK] run_streaming large works (chunks={len(chunks)}, bytes={len(full)})") - return True - except Exception as e: - print(f"[FAIL] run_streaming large failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_error(): - """Test run_streaming with an invalid script returns error metadata""" - print("\nTesting run_streaming error...") - try: - stream = dataweave.run_streaming("output application/json --- invalid_var") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - assert metadata.success is False, f"Expected failure, got: {metadata}" - assert metadata.error is not None, "Expected error message" - assert len(chunks) == 0, f"Expected no chunks on error, got {len(chunks)}" - print(f"[OK] run_streaming error works (error={metadata.error[:60]}...)") - return True - except Exception as e: - print(f"[FAIL] run_streaming error failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_with_inputs(): - """Test run_streaming with input bindings""" - print("\nTesting run_streaming with inputs...") - try: - stream = dataweave.run_streaming("num1 + num2", {"num1": 25, "num2": 17}) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert text.strip() == "42", f"Expected '42', got '{text.strip()}'" - print(f"[OK] run_streaming with inputs works (result='{text.strip()}')") - return True - except Exception as e: - print(f"[FAIL] run_streaming with inputs failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_callback_input_output_large(): - """Test callback-based input+output streaming with large data""" - print("\nTesting callback input+output large...") - try: - import io as _io - - # Build a large JSON array - parts = [b"["] - for i in range(1, 1001): - if i > 1: - parts.append(b",") - parts.append(f'{{"id":{i}}}'.encode()) - parts.append(b"]") - source = _io.BytesIO(b"".join(parts)) - output_chunks = [] - - def on_read(buf_size: int) -> bytes: - return source.read(buf_size) - - def on_write(data: bytes) -> int: - output_chunks.append(data) - return 0 - - result = dataweave.run_input_output_callback( - "output application/json\n---\nsizeOf(payload)", - input_name="payload", - input_mime_type="application/json", - read_callback=on_read, - write_callback=on_write, - ) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(output_chunks) - text = full.decode(result.charset or "utf-8") - assert text == "1000", f"Expected '1000', got '{text}'" - print(f"[OK] Callback input+output large works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] Callback input+output large failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_basic(): - """Test run_transform with an iterable input and streaming output""" - print("\nTesting run_transform basic...") - try: - input_data = [b'[10, 20, 30, 40, 50]'] - script = "output application/json\n---\npayload map ($ * 2)" - stream = dataweave.run_transform(script, input_stream=input_data, input_mime_type="application/json") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert "20" in text, f"Expected 20 in result (10*2), got: {text}" - assert "100" in text, f"Expected 100 in result (50*2), got: {text}" - print(f"[OK] run_transform basic works (chunks={len(chunks)}, result={text.strip()[:60]})") - return True - except Exception as e: - print(f"[FAIL] run_transform basic failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_large(): - """Test run_transform with large chunked input to verify streaming both directions""" - print("\nTesting run_transform large...") - try: - import io as _io - - # Build a large JSON array as chunked input - parts = [b"["] - for i in range(1, 1001): - if i > 1: - parts.append(b",") - parts.append(f'{{"id":{i}}}'.encode()) - parts.append(b"]") - full_input = b"".join(parts) - - # Feed in 4KB chunks (simulating a file/network read) - def chunked(data, size=4096): - for i in range(0, len(data), size): - yield data[i:i+size] - - script = "output application/json\n---\nsizeOf(payload)" - stream = dataweave.run_transform( - script, - input_stream=chunked(full_input), - input_mime_type="application/json", - ) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert text == "1000", f"Expected '1000', got '{text}'" - print(f"[OK] run_transform large works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] run_transform large failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_large_single_chunk(): - """Regression: a single input chunk > 8KB native buffer must not be truncated.""" - print("\nTesting run_transform large single chunk (>8KB, regression)...") - try: - import json as _json - records = [{"id": i, "name": f"item_{i}", "value": i*3} for i in range(1, 2001)] - payload = _json.dumps(records).encode() # ~97KB, one chunk - assert len(payload) > 8192, f"Expected payload > 8192 bytes, got {len(payload)}" - script = "output application/json\n---\nsizeOf(payload)" - stream = dataweave.run_transform(script, input_stream=iter([payload]), input_mime_type="application/json") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - text = b"".join(chunks).decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert text == "2000", f"Expected '2000', got '{text}'" - print(f"[OK] run_transform large single chunk works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] run_transform large single chunk failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_with_file(): - """Test run_transform reading from a file-like object""" - print("\nTesting run_transform with file...") - try: - from pathlib import Path - - xml_path = Path(__file__).resolve().parent / "person.xml" - - with open(xml_path, "rb") as f: - stream = dataweave.run_transform( - "output application/csv header=true\n---\n[payload.person]", - input_stream=iter(lambda: f.read(4096), b""), - input_mime_type="application/xml", - input_charset="UTF-16", - ) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert "Billy" in text, f"Expected 'Billy' in CSV, got: {text}" - assert "31" in text, f"Expected '31' in CSV, got: {text}" - print(f"[OK] run_transform with file works (result={text.strip()[:60]})") - return True - except Exception as e: - print(f"[FAIL] run_transform with file failed: {e}") - import traceback - traceback.print_exc() - return False - -def main(): - """Run all tests""" - print("="*70) - print("DataWeave Python Module - Test Suite") - print("="*70) - - try: - results = [] - results.append(test_input_value_mime_type_constructor()) - results.append(test_basic()) - results.append(test_with_inputs()) - results.append(test_context_manager()) - results.append(test_encoding()) - results.append(test_auto_conversion()) - results.append(test_callback_output_basic()) - results.append(test_callback_output_with_inputs()) - results.append(test_callback_input_output()) - results.append(test_callback_input_output_large()) - results.append(test_run_streaming_basic()) - results.append(test_run_streaming_large()) - results.append(test_run_streaming_error()) - results.append(test_run_streaming_with_inputs()) - results.append(test_run_transform_basic()) - results.append(test_run_transform_large()) - results.append(test_run_transform_large_single_chunk()) - results.append(test_run_transform_with_file()) - - # Cleanup - dataweave.cleanup() - - print("\n" + "="*70) - passed = sum(results) - total = len(results) - print(f"Results: {passed}/{total} tests passed") - print("="*70) - - if passed == total: - print("\n[OK] All tests passed!") - sys.exit(0) - else: - print(f"\n[FAIL] {total - passed} test(s) failed") - sys.exit(1) - - except dataweave.DataWeaveLibraryNotFoundError as e: - print(f"\n[ERROR] {e}") - print("\nPlease build the native library first:") - print(" ./gradlew nativeCompile") - sys.exit(2) - except Exception as e: - print(f"\n[ERROR] Unexpected error: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - -if __name__ == "__main__": - main() From be3f479c7b804528ddb8e7e32107bf4811551b1f Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:04:11 -0300 Subject: [PATCH 02/30] docs: record Python pytest migration results --- .../2026-08-19-python-binding-modernization/task-1-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md index 9e4e2ddc..503f4310 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md @@ -34,7 +34,7 @@ ## Commit -- Pending creation: `test: migrate Python binding checks to pytest` +- `9cf376d test: migrate Python binding checks to pytest` ## Concerns From b14f58beed0bb0f78e02884e262487805f3a0fe4 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:12:39 -0300 Subject: [PATCH 03/30] test: cover Python callback failures --- .github/actions/build-foundation/action.yml | 5 ++ .../task-1-report.md | 27 ++++++++++ native-lib/build.gradle | 3 ++ native-lib/python/README.md | 15 +++++- .../tests/integration/test_callbacks.py | 51 +++++++++++++++++++ 5 files changed, 100 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-foundation/action.yml b/.github/actions/build-foundation/action.yml index c36f9863..3b47b58f 100644 --- a/.github/actions/build-foundation/action.yml +++ b/.github/actions/build-foundation/action.yml @@ -28,6 +28,11 @@ runs: distribution: 'graalvm-community' github-token: ${{ inputs.github-token }} + - name: Install Python test dependencies + run: python3 -m pip install '.[test]' + shell: bash + working-directory: native-lib/python + - name: Run Build run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md index 503f4310..56a97ac3 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md @@ -40,3 +40,30 @@ - The system Python does not have pytest and cannot write its global site-packages. Verification used an isolated temporary virtual environment supplied through Gradle's existing `-PpythonExe` override. - Native-image and Gradle emit existing Java/native-image deprecation warnings during the native build; the task itself completed successfully. + +## Review Fixes + +### Files Changed + +- `.github/actions/build-foundation/action.yml`: installs `native-lib/python`'s `test` optional dependency group before the foundation build invokes `pythonTest`. +- `native-lib/build.gradle`: declares Python tests and pytest configuration as `pythonTest` inputs so test additions invalidate Gradle's up-to-date state. +- `native-lib/python/tests/integration/test_callbacks.py`: adds three integration scenarios covering exceptions in output-only write callbacks and input/output read and write callbacks. +- `native-lib/python/README.md`: documents installing `.[test]`, direct pytest integration execution, and lane marker behavior. + +### Design Choices + +- Callback implementations already catch Python exceptions and return `-1`, which the native APIs translate into unsuccessful `StreamingResult` metadata. The new tests characterize that established public behavior without changing runtime code or the native ABI. +- The CI dependency installation belongs in `build-foundation` because its `build` command can trigger `native-lib:pythonTest`; per the review ruling, no caller-level provisioning is required. +- The Gradle input declaration was added after observing that `pythonTest` remained up-to-date after a test-only edit. This prevents future test changes from being silently skipped. + +### Tests Run + +1. `/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python -m pytest tests/integration/test_callbacks.py -m integration -v` + - Passed: `7 passed in 1.16s`, including callback exception containment cases. +2. `./gradlew native-lib:pythonTest -PpythonExe=/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python` + - Passed: native library compiled and staged; pytest reported `21 passed in 2.08s`; JUnit and coverage XML were regenerated beneath `native-lib/build`. + +### Concerns + +- The CI action change was reviewed structurally but not executed in GitHub Actions from this local worktree. +- Native-image emits existing Java/native-image deprecation warnings during local Gradle execution. diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 7ecf3df2..62fd585a 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -155,6 +155,9 @@ tasks.register('pythonTest', Exec) { dependsOn tasks.named('stagePythonNativeLib') workingDir("${projectDir}/python") + inputs.dir("${projectDir}/python/tests") + inputs.file("${projectDir}/python/pytest.ini") + inputs.file("${projectDir}/python/pyproject.toml") def testReportsDir = layout.buildDirectory.dir('test-results/python') def coverageReportsDir = layout.buildDirectory.dir('reports/coverage/python') outputs.file(testReportsDir.map { it.file('junit.xml') }) diff --git a/native-lib/python/README.md b/native-lib/python/README.md index 76da00b8..d0576571 100644 --- a/native-lib/python/README.md +++ b/native-lib/python/README.md @@ -32,6 +32,15 @@ python3 -m pip install native-lib/python/dist/dataweave_native-0.0.1-*.whl python3 -m pip install -e native-lib/python ``` +### Test dependencies + +Install the pytest test extra before running the Python test lanes locally: + +```bash +cd native-lib/python +python3 -m pip install '.[test]' +``` + ### Option C: Use externally-built library via environment variable ```bash @@ -249,7 +258,7 @@ print(b"".join(chunks)) # b'[1,4,9,16,25]' ```bash cd native-lib/python -python3 tests/test_dataweave_module.py +python3 -m pytest tests/integration -m integration -v ``` Or via Gradle: @@ -258,6 +267,10 @@ Or via Gradle: ./gradlew :native-lib:pythonTest ``` +`pytest.ini` registers `unit`, `integration`, and `tck` markers. Normal pytest +runs exclude `tck`; use `-m "unit or integration"` to run the lanes used by +`pythonTest`. + ## Running Examples ```bash diff --git a/native-lib/python/tests/integration/test_callbacks.py b/native-lib/python/tests/integration/test_callbacks.py index c9b11157..b7ee1c53 100644 --- a/native-lib/python/tests/integration/test_callbacks.py +++ b/native-lib/python/tests/integration/test_callbacks.py @@ -33,6 +33,17 @@ def on_write(data: bytes) -> int: assert b"".join(chunks).decode(result.charset or "utf-8") == "42" +@pytest.mark.integration +def test_callback_translates_write_callback_exception_to_unsuccessful_result(): + def on_write(_data: bytes) -> int: + raise RuntimeError("write callback failed") + + result = dataweave.run_callback("2 + 2", on_write) + + assert result.success is False + assert result.error is not None + + @pytest.mark.integration def test_callback_transforms_streamed_input_and_output(): source = io.BytesIO(b"[10, 20, 30, 40, 50]") @@ -82,3 +93,43 @@ def on_write(data: bytes) -> int: assert result.success is True assert b"".join(chunks).decode(result.charset or "utf-8") == "1000" + + +@pytest.mark.integration +def test_input_output_callback_translates_read_callback_exception_to_unsuccessful_result(): + def on_read(_buffer_size: int) -> bytes: + raise RuntimeError("read callback failed") + + def on_write(_data: bytes) -> int: + return 0 + + result = dataweave.run_input_output_callback( + "output application/json\n---\npayload", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + assert result.success is False + assert result.error is not None + + +@pytest.mark.integration +def test_input_output_callback_translates_write_callback_exception_to_unsuccessful_result(): + def on_read(_buffer_size: int) -> bytes: + return b"[1]" + + def on_write(_data: bytes) -> int: + raise RuntimeError("write callback failed") + + result = dataweave.run_input_output_callback( + "output application/json\n---\npayload", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + assert result.success is False + assert result.error is not None From f624a6de61a009896bb8ed71aea7573715f526d3 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:15:32 -0300 Subject: [PATCH 04/30] ci: support PEP 668 Python test setup --- .github/actions/build-foundation/action.yml | 2 +- .../task-1-report.md | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-foundation/action.yml b/.github/actions/build-foundation/action.yml index 3b47b58f..6e512469 100644 --- a/.github/actions/build-foundation/action.yml +++ b/.github/actions/build-foundation/action.yml @@ -29,7 +29,7 @@ runs: github-token: ${{ inputs.github-token }} - name: Install Python test dependencies - run: python3 -m pip install '.[test]' + run: python3 -m pip install ${{ runner.environment == 'github-hosted' && '--break-system-packages' || '' }} '.[test]' shell: bash working-directory: native-lib/python diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md index 56a97ac3..ce69818c 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md @@ -67,3 +67,26 @@ - The CI action change was reviewed structurally but not executed in GitHub Actions from this local worktree. - Native-image emits existing Java/native-image deprecation warnings during local Gradle execution. + +## Review Fix Round 2 + +### Files Changed + +- `.github/actions/build-foundation/action.yml`: conditionally adds `--break-system-packages` when `runner.environment` is `github-hosted`, while retaining the unmodified pip invocation for self-hosted runners. + +### Design Choice + +- Matched the existing Python artifact action's conditional flag pattern. This addresses PEP 668 on GitHub-hosted macOS without imposing `--break-system-packages` on self-hosted MuleSoft runners. + +### Checks Run + +1. `ruby -e "require 'yaml'; YAML.load_file('.github/actions/build-foundation/action.yml'); puts 'YAML valid'"` + - Passed: YAML parsed successfully. +2. `git diff --check` + - Passed: no whitespace errors. +3. Compared the condition against `.github/actions/python/action.yml`. + - Confirmed the existing action uses the same GitHub Actions expression form for conditional pip flags. + +### Concerns + +- The GitHub-hosted runner expression cannot be executed locally; validation is structural and follows the repository's established Python action convention. From 2f6a788d9403988debed4747025ac9d69f60de38 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:27:18 -0300 Subject: [PATCH 05/30] test: add Python binding unit characterization coverage --- .../task-2-report.md | 72 ++++++++++ native-lib/python/src/dataweave/__init__.py | 3 +- native-lib/python/tests/unit/test_encoding.py | 53 +++++++ native-lib/python/tests/unit/test_facade.py | 40 ++++++ native-lib/python/tests/unit/test_models.py | 51 +++++++ native-lib/python/tests/unit/test_native.py | 54 +++++++ .../python/tests/unit/test_streaming.py | 136 ++++++++++++++++++ 7 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md create mode 100644 native-lib/python/tests/unit/test_encoding.py create mode 100644 native-lib/python/tests/unit/test_facade.py create mode 100644 native-lib/python/tests/unit/test_models.py create mode 100644 native-lib/python/tests/unit/test_native.py create mode 100644 native-lib/python/tests/unit/test_streaming.py diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md new file mode 100644 index 00000000..de25f81a --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md @@ -0,0 +1,72 @@ +# Task 2 Report: Python Binding Unit Characterization Coverage + +## Changed Files + +- `native-lib/python/src/dataweave/__init__.py` + - Replaced dynamic `__import__("os")` environment access with a normal private module import. Public API and native ABI are unchanged. +- `native-lib/python/tests/unit/test_models.py` + - Characterizes `InputValue` encoding and `ExecutionResult` decoding/error behavior. +- `native-lib/python/tests/unit/test_encoding.py` + - Characterizes implicit text encoding and explicit-input validation/metadata preservation. +- `native-lib/python/tests/unit/test_native.py` + - Characterizes native response decoding, malformed input handling, candidate library path priority, and native-string cleanup on decoding failure. +- `native-lib/python/tests/unit/test_streaming.py` + - Characterizes callback exception-to-abort conversion, large input chunk remainders, empty native metadata, attach failure, worker detachment behavior, and early stream closure with a fake native collaborator. +- `native-lib/python/tests/unit/test_facade.py` + - Characterizes module singleton initialization/recreation and global cleanup. + +## Implementation Details + +- Added 24 `@pytest.mark.unit` tests that import the Python package from source and never instantiate a staged `dwlib`. +- Fake native collaborators exercise ctypes callback boundaries without a native shared library. They verify callback exceptions return the documented nonzero abort status rather than escaping C callbacks. +- Tests preserve the existing stream contract: early generator closure leaves `Stream.metadata` unset because no terminal metadata was consumed. +- `DataWeave._decode_and_free` is exercised through a failing UTF-8 decode to verify native strings are freed from its existing `finally` block. +- The only source adjustment replaces dynamic standard-library import resolution with a normal `os` import; no public names, signatures, or ABI fields changed. + +## Commands And Output + +1. Initial required unit command before pytest was installed: + + ```text + python3 -m pytest tests/unit -m unit -v + /Library/Developer/CommandLineTools/usr/bin/python3: No module named pytest + ``` + +2. After installing local test dependencies, the initial test-first run collected 25 tests: 21 passed and 4 failed. The expected failures identified the existing dict-explicit-input behavior, callback abort status, empty-native-response metadata, and missing `Stream.close` API. Tests were refined to characterize the existing behavior rather than alter public API. + +3. Final unit verification: + + ```text + python3 -m pytest tests/unit -m unit -v + 24 passed in 0.02s + ``` + +4. Syntax verification: + + ```text + python3 -m compileall -q src tests/unit + exit 0 + ``` + +5. Diff whitespace verification: + + ```text + git diff --check + exit 0 + ``` + +6. Gradle task configuration check: + + ```text + ./gradlew native-lib:pythonTest --dry-run + BUILD SUCCESSFUL + ``` + +## Commit + +- Pending at report creation: `test: add Python binding unit characterization coverage` + +## Concerns + +- None for Task 2 scope. +- The local Python installation initially lacked pytest. It was installed in the user site to execute the required unit lane; this did not alter tracked project files. diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index 15b50211..f4e9e186 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -46,6 +46,7 @@ import base64 import ctypes import json +import os from dataclasses import dataclass from pathlib import Path from queue import Queue @@ -221,7 +222,7 @@ def _parse_streaming_result(meta: dict) -> StreamingResult: def _candidate_library_paths() -> list[Path]: paths: list[Path] = [] - env_value = (__import__("os").environ.get(_ENV_NATIVE_LIB) or "").strip() + env_value = (os.environ.get(_ENV_NATIVE_LIB) or "").strip() if env_value: paths.append(Path(env_value)) diff --git a/native-lib/python/tests/unit/test_encoding.py b/native-lib/python/tests/unit/test_encoding.py new file mode 100644 index 00000000..2b1fb7b9 --- /dev/null +++ b/native-lib/python/tests/unit/test_encoding.py @@ -0,0 +1,53 @@ +import base64 + +import pytest + +import dataweave + + +@pytest.mark.unit +def test_normalize_plain_text_uses_text_mime_type(): + normalized = dataweave._normalize_input_value("hello") + + assert normalized == { + "content": base64.b64encode(b"hello").decode("ascii"), + "mimeType": "text/plain", + "charset": "utf-8", + } + + +@pytest.mark.unit +def test_normalize_explicit_bytes_preserves_mime_charset_and_properties(): + normalized = dataweave._normalize_input_value({ + "content": b"caf\xe9", + "mimeType": "text/plain", + "charset": "latin-1", + "properties": {"header": False}, + }) + + assert normalized == { + "content": base64.b64encode(b"caf\xe9").decode("ascii"), + "mimeType": "text/plain", + "charset": "latin-1", + "properties": {"header": False}, + } + + +@pytest.mark.unit +@pytest.mark.parametrize("value", [ + {"content": "body"}, + {"mimeType": "text/plain"}, +]) +def test_normalize_explicit_input_requires_content_and_mime_type(value): + with pytest.raises(dataweave.DataWeaveError, match="must include both 'content' and 'mimeType'"): + dataweave._normalize_input_value(value) + + +@pytest.mark.unit +def test_normalize_explicit_input_rejects_unsupported_keys(): + with pytest.raises(dataweave.DataWeaveError, match="unsupported keys: unexpected"): + dataweave._normalize_input_value({ + "content": "body", + "mimeType": "text/plain", + "unexpected": True, + }) diff --git a/native-lib/python/tests/unit/test_facade.py b/native-lib/python/tests/unit/test_facade.py new file mode 100644 index 00000000..f0c628f3 --- /dev/null +++ b/native-lib/python/tests/unit/test_facade.py @@ -0,0 +1,40 @@ +import pytest + +import dataweave + + +@pytest.mark.unit +def test_global_facade_initializes_once_and_cleanup_allows_recreation(monkeypatch): + created = [] + registered = [] + + class FakeRuntime: + def __init__(self): + self.cleaned = False + created.append(self) + + def initialize(self): + pass + + def cleanup(self): + self.cleaned = True + + monkeypatch.setattr(dataweave, "DataWeave", FakeRuntime) + monkeypatch.setattr("atexit.register", registered.append) + + first = dataweave._get_global_instance() + second = dataweave._get_global_instance() + dataweave.cleanup() + third = dataweave._get_global_instance() + + assert first is second + assert first.cleaned is True + assert third is not first + assert registered == [dataweave.cleanup, dataweave.cleanup] + + +@pytest.mark.unit +def test_cleanup_is_noop_without_global_runtime(): + dataweave.cleanup() + + assert dataweave._global_instance is None diff --git a/native-lib/python/tests/unit/test_models.py b/native-lib/python/tests/unit/test_models.py new file mode 100644 index 00000000..c523be58 --- /dev/null +++ b/native-lib/python/tests/unit/test_models.py @@ -0,0 +1,51 @@ +import base64 + +import pytest + +import dataweave + + +@pytest.mark.unit +def test_input_value_encodes_text_with_its_charset(): + value = dataweave.InputValue("caf\u00e9", charset="latin-1") + + assert value.encode_content() == base64.b64encode(b"caf\xe9").decode("ascii") + + +@pytest.mark.unit +def test_execution_result_decodes_text_payload(): + result = dataweave.ExecutionResult( + success=True, + result=base64.b64encode(b"hello").decode("ascii"), + error=None, + binary=False, + mime_type="text/plain", + charset="utf-8", + ) + + assert result.get_bytes() == b"hello" + assert result.get_string() == "hello" + + +@pytest.mark.unit +def test_execution_result_keeps_binary_payload_as_base64_text(): + payload = base64.b64encode(b"\x00\xff").decode("ascii") + result = dataweave.ExecutionResult(True, payload, None, True, "application/octet-stream", None) + + assert result.get_bytes() == b"\x00\xff" + assert result.get_string() == payload + + +@pytest.mark.unit +def test_execution_result_returns_none_for_unsuccessful_execution(): + result = dataweave.ExecutionResult(False, None, "failed", False, None, None) + + assert result.get_bytes() is None + assert result.get_string() is None + + +@pytest.mark.unit +def test_parse_native_response_preserves_error_result(): + result = dataweave._parse_native_encoded_response('{"success": false, "error": "script failed"}') + + assert result == dataweave.ExecutionResult(False, None, "script failed", False, None, None) diff --git a/native-lib/python/tests/unit/test_native.py b/native-lib/python/tests/unit/test_native.py new file mode 100644 index 00000000..a31bc9c1 --- /dev/null +++ b/native-lib/python/tests/unit/test_native.py @@ -0,0 +1,54 @@ +from pathlib import Path + +import pytest + +import dataweave + + +@pytest.mark.unit +def test_parse_native_response_rejects_malformed_json(): + result = dataweave._parse_native_encoded_response("not json") + + assert result.success is False + assert result.error.startswith("Failed to parse native JSON response:") + + +@pytest.mark.unit +@pytest.mark.parametrize("raw, expected_error", [ + (None, "Native returned null"), + ("", "Native returned empty response"), + ("[]", "Native response JSON is not an object"), +]) +def test_parse_native_response_rejects_invalid_native_values(raw, expected_error): + result = dataweave._parse_native_encoded_response(raw) + + assert result == dataweave.ExecutionResult(False, None, expected_error, False, None, None) + + +@pytest.mark.unit +def test_candidate_paths_prioritize_environment_override(monkeypatch, tmp_path): + override = tmp_path / "custom-dwlib" + monkeypatch.setenv("DATAWEAVE_NATIVE_LIB", str(override)) + + paths = dataweave._candidate_library_paths() + + assert paths[0] == override + assert paths[1:4] == [ + Path(dataweave.__file__).resolve().parent / "native" / "dwlib.dylib", + Path(dataweave.__file__).resolve().parent / "native" / "dwlib.so", + Path(dataweave.__file__).resolve().parent / "native" / "dwlib.dll", + ] + + +@pytest.mark.unit +def test_decode_and_free_releases_native_string_when_decoding_fails(monkeypatch): + freed = [] + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + runtime._thread = "thread" + runtime._lib = type("Native", (), {"free_cstring": lambda _self, thread, ptr: freed.append((thread, ptr))})() + monkeypatch.setattr(dataweave.ctypes, "string_at", lambda _ptr: b"\xff") + + with pytest.raises(UnicodeDecodeError): + runtime._decode_and_free(123) + + assert freed == [("thread", 123)] diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py new file mode 100644 index 00000000..cbb4033d --- /dev/null +++ b/native-lib/python/tests/unit/test_streaming.py @@ -0,0 +1,136 @@ +import ctypes + +import pytest + +import dataweave + + +class FakeNative: + def __init__(self, metadata=None, attach_code=0, emit=b"", consume_input=False): + self.metadata = metadata + self.attach_code = attach_code + self.emit = emit + self.consume_input = consume_input + self.detached = [] + self.freed = [] + self._buffers = [] + + def graal_attach_thread(self, _isolate, _thread): + return self.attach_code + + def graal_detach_thread(self, thread): + self.detached.append(thread) + return 0 + + def free_cstring(self, thread, ptr): + self.freed.append((thread, ptr)) + + def _response_pointer(self): + if self.metadata is None: + return None + buffer = ctypes.create_string_buffer(self.metadata.encode("utf-8")) + self._buffers.append(buffer) + return ctypes.addressof(buffer) + + def run_script_callback(self, _thread, _script, _inputs, write_callback, _context): + if self.emit: + buffer = ctypes.create_string_buffer(self.emit) + self.write_status = write_callback(None, ctypes.addressof(buffer), len(self.emit)) + return self._response_pointer() + + def run_script_input_output_callback( + self, _thread, _script, _inputs, _input_name, _mime_type, _charset, read_callback, write_callback, _context, + ): + if self.consume_input: + buffer = ctypes.create_string_buffer(3) + read = [] + while True: + size = read_callback(None, ctypes.addressof(buffer), len(buffer)) + if size == 0: + break + assert size > 0 + read.append(bytes(buffer.raw[:size])) + self.read_input = b"".join(read) + if self.emit: + buffer = ctypes.create_string_buffer(self.emit) + assert write_callback(None, ctypes.addressof(buffer), len(self.emit)) == 0 + return self._response_pointer() + + +def configured_runtime(native): + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + runtime._setup_graal_structures() + runtime._initialized = True + runtime._has_callback_streaming = True + runtime._has_callback_input_output = True + runtime._lib = native + runtime._isolate = object() + runtime._thread = object() + return runtime + + +@pytest.mark.unit +def test_run_callback_converts_write_callback_exception_to_abort_result(): + native = FakeNative('{"success": false, "error": "callback aborted"}', emit=b"chunk") + runtime = configured_runtime(native) + + result = runtime.run_callback("script", lambda _chunk: (_ for _ in ()).throw(RuntimeError("stop"))) + + assert result == dataweave.StreamingResult(False, "callback aborted", None, None, False) + assert native.write_status == -1 + + +@pytest.mark.unit +def test_run_input_output_callback_converts_read_exception_to_abort_result(): + runtime = configured_runtime(FakeNative('{"success": false, "error": "read aborted"}')) + + result = runtime.run_input_output_callback( + "script", "payload", "application/json", lambda _size: (_ for _ in ()).throw(RuntimeError("stop")), lambda _data: 0, + ) + + assert result == dataweave.StreamingResult(False, "read aborted", None, None, False) + + +@pytest.mark.unit +def test_run_transform_preserves_remainder_of_large_input_chunk(): + native = FakeNative('{"success": true, "mimeType": "application/json", "charset": "utf-8"}', consume_input=True) + runtime = configured_runtime(native) + stream = runtime.run_transform("script", [b"abcdefgh"], input_mime_type="application/json") + + assert list(stream) == [] + assert native.read_input == b"abcdefgh" + assert stream.metadata == dataweave.StreamingResult(True, None, "application/json", "utf-8", False) + + +@pytest.mark.unit +def test_run_streaming_returns_failure_metadata_when_native_response_is_empty(): + runtime = configured_runtime(FakeNative()) + stream = runtime.run_streaming("script") + + assert list(stream) == [] + assert stream.metadata == dataweave.StreamingResult(False, "Empty response", None, None, False) + + +@pytest.mark.unit +def test_run_streaming_returns_attach_failure_without_detaching_unattached_thread(): + native = FakeNative(attach_code=9) + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + + assert list(stream) == [] + assert stream.metadata == dataweave.StreamingResult(False, "Failed to attach worker thread to isolate (code 9)", None, None, False) + assert native.detached == [] + + +@pytest.mark.unit +def test_stream_keeps_metadata_unset_when_consumer_closes_generator_early(): + def generate(): + yield b"first" + return dataweave.StreamingResult(True, None, "text/plain", "utf-8", False) + + stream = dataweave.Stream(generate()) + + assert next(stream) == b"first" + stream._gen.close() + + assert stream.metadata is None From 82444013b31588bfca60500a599a5b5356ff9653 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:27:24 -0300 Subject: [PATCH 06/30] docs: record Python unit test coverage --- .../2026-08-19-python-binding-modernization/task-2-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md index de25f81a..2bd06d59 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md @@ -64,7 +64,7 @@ ## Commit -- Pending at report creation: `test: add Python binding unit characterization coverage` +- `2f6a788 test: add Python binding unit characterization coverage` ## Concerns From 2f169ee6c06c4e781b77f5cf688f7b84b9908492 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:32:54 -0300 Subject: [PATCH 07/30] test: harden Python streaming unit coverage --- .../task-2-report.md | 24 +++++++- native-lib/python/src/dataweave/__init__.py | 44 +++++++++----- .../python/tests/unit/test_streaming.py | 57 ++++++++++++++----- 3 files changed, 95 insertions(+), 30 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md index 2bd06d59..6704c776 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md @@ -3,7 +3,7 @@ ## Changed Files - `native-lib/python/src/dataweave/__init__.py` - - Replaced dynamic `__import__("os")` environment access with a normal private module import. Public API and native ABI are unchanged. + - Replaced dynamic `__import__("os")` environment access with a normal private module import. Added private streaming cancellation plumbing so abandoned streaming consumers cause future native write callbacks to abort and the worker is joined/detached. Public API and native ABI are unchanged. - `native-lib/python/tests/unit/test_models.py` - Characterizes `InputValue` encoding and `ExecutionResult` decoding/error behavior. - `native-lib/python/tests/unit/test_encoding.py` @@ -11,7 +11,7 @@ - `native-lib/python/tests/unit/test_native.py` - Characterizes native response decoding, malformed input handling, candidate library path priority, and native-string cleanup on decoding failure. - `native-lib/python/tests/unit/test_streaming.py` - - Characterizes callback exception-to-abort conversion, large input chunk remainders, empty native metadata, attach failure, worker detachment behavior, and early stream closure with a fake native collaborator. + - Characterizes callback exception-to-abort conversion, including observed read callback abort status; large input chunk remainders; true worker metadata absence; attach failure; and safe early consumer abandonment with prompt worker detachment using fake native collaborators. - `native-lib/python/tests/unit/test_facade.py` - Characterizes module singleton initialization/recreation and global cleanup. @@ -19,7 +19,7 @@ - Added 24 `@pytest.mark.unit` tests that import the Python package from source and never instantiate a staged `dwlib`. - Fake native collaborators exercise ctypes callback boundaries without a native shared library. They verify callback exceptions return the documented nonzero abort status rather than escaping C callbacks. -- Tests preserve the existing stream contract: early generator closure leaves `Stream.metadata` unset because no terminal metadata was consumed. +- Streaming generators now use an internal cancellation event. The private `_close()` test seam sets cancellation before closing the generator; cancellation makes subsequent native write callbacks return `-1`, and generator cleanup joins the worker while the worker's `finally` detaches its isolate thread. - `DataWeave._decode_and_free` is exercised through a failing UTF-8 decode to verify native strings are freed from its existing `finally` block. - The only source adjustment replaces dynamic standard-library import resolution with a normal `os` import; no public names, signatures, or ABI fields changed. @@ -62,9 +62,27 @@ BUILD SUCCESSFUL ``` +7. Fix round 1 focused verification: + + ```text + python3 -m pytest tests/unit/test_streaming.py -m unit -v + 6 passed in 0.02s + ``` + +8. Fix round 1 full unit verification: + + ```text + python3 -m pytest tests/unit -m unit -v + 24 passed in 0.03s + python3 -m compileall -q src tests/unit + git diff --check + exit 0 + ``` + ## Commit - `2f6a788 test: add Python binding unit characterization coverage` +- Pending at report update: `test: harden Python streaming unit coverage` ## Concerns diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index f4e9e186..4f0c3edd 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -50,7 +50,7 @@ from dataclasses import dataclass from pathlib import Path from queue import Queue -from threading import Thread +from threading import Event, Thread from typing import Any, Callable, Dict, Generator, Iterable, Optional, Union # Bound for streaming output queues: limits memory under slow/stalled consumers @@ -170,6 +170,12 @@ def __next__(self) -> bytes: def metadata(self) -> Optional[StreamingResult]: return self._metadata + def _close(self) -> None: + on_close = getattr(self, "_on_close", None) + if on_close is not None: + on_close() + self._gen.close() + def _parse_native_encoded_response(raw: str) -> ExecutionResult: if raw is None: @@ -551,12 +557,17 @@ def run_streaming( :raises DataWeaveError: if the runtime is not initialized or the callback API is not available """ - return Stream(self._run_streaming_gen(script, inputs)) + cancelled = Event() + stream = Stream(self._run_streaming_gen(script, inputs, cancelled)) + stream._on_close = cancelled.set + stream._cancelled = cancelled + return stream def _run_streaming_gen( self, script: str, inputs: Optional[Dict[str, Any]] = None, + cancelled: Optional[Event] = None, ) -> Generator[bytes, None, StreamingResult]: if not self._initialized: raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") @@ -577,6 +588,8 @@ def _run_streaming_gen( @WRITE_CALLBACK def _write_cb(_ctx, buf, length): try: + if cancelled is not None and cancelled.is_set(): + return -1 # With maxsize set, put() blocks when the queue is full, exerting # backpressure onto the native producer. Timeout prevents indefinite # blocking if the consumer abandons the generator. @@ -619,18 +632,21 @@ def _run_native(): worker.start() meta = None - while True: - item = q.get() - if item is _SENTINEL: - break - if isinstance(item, dict): - meta = item - else: - yield item - - worker.join(timeout=30) - if worker.is_alive(): - raise DataWeaveError("Worker thread timeout after 30 seconds") + try: + while True: + item = q.get() + if item is _SENTINEL: + break + if isinstance(item, dict): + meta = item + else: + yield item + finally: + if cancelled is not None: + cancelled.set() + worker.join(timeout=30) + if worker.is_alive(): + raise DataWeaveError("Worker thread timeout after 30 seconds") if meta is None: meta = {"success": False, "error": "No metadata received from native call"} diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index cbb4033d..74e3cba1 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -1,4 +1,6 @@ import ctypes +from queue import Queue +from threading import Event import pytest @@ -14,12 +16,14 @@ def __init__(self, metadata=None, attach_code=0, emit=b"", consume_input=False): self.detached = [] self.freed = [] self._buffers = [] + self.detached_event = Event() def graal_attach_thread(self, _isolate, _thread): return self.attach_code def graal_detach_thread(self, thread): self.detached.append(thread) + self.detached_event.set() return 0 def free_cstring(self, thread, ptr): @@ -46,9 +50,11 @@ def run_script_input_output_callback( read = [] while True: size = read_callback(None, ctypes.addressof(buffer), len(buffer)) + self.read_status = size if size == 0: break - assert size > 0 + if size < 0: + return self._response_pointer() read.append(bytes(buffer.raw[:size])) self.read_input = b"".join(read) if self.emit: @@ -82,13 +88,15 @@ def test_run_callback_converts_write_callback_exception_to_abort_result(): @pytest.mark.unit def test_run_input_output_callback_converts_read_exception_to_abort_result(): - runtime = configured_runtime(FakeNative('{"success": false, "error": "read aborted"}')) + native = FakeNative('{"success": false, "error": "read aborted"}', consume_input=True) + runtime = configured_runtime(native) result = runtime.run_input_output_callback( "script", "payload", "application/json", lambda _size: (_ for _ in ()).throw(RuntimeError("stop")), lambda _data: 0, ) assert result == dataweave.StreamingResult(False, "read aborted", None, None, False) + assert native.read_status == -1 @pytest.mark.unit @@ -103,12 +111,19 @@ def test_run_transform_preserves_remainder_of_large_input_chunk(): @pytest.mark.unit -def test_run_streaming_returns_failure_metadata_when_native_response_is_empty(): - runtime = configured_runtime(FakeNative()) +def test_run_streaming_returns_failure_metadata_when_worker_produces_no_metadata(monkeypatch): + class MetadataDroppingQueue(Queue): + def put(self, item, *args, **kwargs): + if isinstance(item, dict): + return None + return super().put(item, *args, **kwargs) + + monkeypatch.setattr(dataweave, "Queue", MetadataDroppingQueue) + runtime = configured_runtime(FakeNative('{"success": true}')) stream = runtime.run_streaming("script") assert list(stream) == [] - assert stream.metadata == dataweave.StreamingResult(False, "Empty response", None, None, False) + assert stream.metadata == dataweave.StreamingResult(False, "No metadata received from native call", None, None, False) @pytest.mark.unit @@ -123,14 +138,30 @@ def test_run_streaming_returns_attach_failure_without_detaching_unattached_threa @pytest.mark.unit -def test_stream_keeps_metadata_unset_when_consumer_closes_generator_early(): - def generate(): - yield b"first" - return dataweave.StreamingResult(True, None, "text/plain", "utf-8", False) - - stream = dataweave.Stream(generate()) +def test_stream_private_close_aborts_worker_and_detaches_after_consumer_abandons_output(): + class BlockingFakeNative(FakeNative): + def __init__(self): + super().__init__('{"success": false, "error": "aborted"}') + self.first_chunk_written = Event() + self.cancelled = None + + def run_script_callback(self, _thread, _script, _inputs, write_callback, _context): + first = ctypes.create_string_buffer(b"first") + assert write_callback(None, ctypes.addressof(first), 5) == 0 + self.first_chunk_written.set() + assert self.cancelled.wait(1) + second = ctypes.create_string_buffer(b"second") + self.write_status = write_callback(None, ctypes.addressof(second), 6) + return self._response_pointer() + + native = BlockingFakeNative() + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + native.cancelled = stream._cancelled assert next(stream) == b"first" - stream._gen.close() + assert native.first_chunk_written.wait(1) + stream._close() - assert stream.metadata is None + assert native.detached_event.wait(1) + assert native.write_status == -1 From 3b2694b470175a45720706c238e4b32d7a15f061 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:33:57 -0300 Subject: [PATCH 08/30] docs: update Python streaming test report --- .../2026-08-19-python-binding-modernization/task-2-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md index 6704c776..62b7e781 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md @@ -82,7 +82,7 @@ ## Commit - `2f6a788 test: add Python binding unit characterization coverage` -- Pending at report update: `test: harden Python streaming unit coverage` +- `2f169ee test: harden Python streaming unit coverage` ## Concerns From bcf6ccd49e64ed0d13d82e630cda02c256fed75a Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:39:38 -0300 Subject: [PATCH 09/30] fix: prevent Python streaming worker shutdown stalls --- .../task-2-report.md | 18 ++++++- native-lib/python/src/dataweave/__init__.py | 20 +++++--- .../python/tests/unit/test_streaming.py | 49 ++++++++++++++++++- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md index 62b7e781..436c11e3 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md @@ -3,7 +3,7 @@ ## Changed Files - `native-lib/python/src/dataweave/__init__.py` - - Replaced dynamic `__import__("os")` environment access with a normal private module import. Added private streaming cancellation plumbing so abandoned streaming consumers cause future native write callbacks to abort and the worker is joined/detached. Public API and native ABI are unchanged. + - Replaced dynamic `__import__("os")` environment access with a normal private module import. Added private streaming cancellation plumbing so abandoned streaming consumers cause future native write callbacks to abort and the worker is joined/detached. Terminal metadata and sentinel publication now stop retrying after cancellation, preventing a full abandoned output queue from blocking worker shutdown. Public API and native ABI are unchanged. - `native-lib/python/tests/unit/test_models.py` - Characterizes `InputValue` encoding and `ExecutionResult` decoding/error behavior. - `native-lib/python/tests/unit/test_encoding.py` @@ -11,7 +11,7 @@ - `native-lib/python/tests/unit/test_native.py` - Characterizes native response decoding, malformed input handling, candidate library path priority, and native-string cleanup on decoding failure. - `native-lib/python/tests/unit/test_streaming.py` - - Characterizes callback exception-to-abort conversion, including observed read callback abort status; large input chunk remainders; true worker metadata absence; attach failure; and safe early consumer abandonment with prompt worker detachment using fake native collaborators. + - Characterizes callback exception-to-abort conversion, including observed read callback abort status; large input chunk remainders; true worker metadata absence; attach failure; safe early consumer abandonment with prompt worker detachment; and full-queue terminal shutdown using fake native collaborators. - `native-lib/python/tests/unit/test_facade.py` - Characterizes module singleton initialization/recreation and global cleanup. @@ -20,6 +20,7 @@ - Added 24 `@pytest.mark.unit` tests that import the Python package from source and never instantiate a staged `dwlib`. - Fake native collaborators exercise ctypes callback boundaries without a native shared library. They verify callback exceptions return the documented nonzero abort status rather than escaping C callbacks. - Streaming generators now use an internal cancellation event. The private `_close()` test seam sets cancellation before closing the generator; cancellation makes subsequent native write callbacks return `-1`, and generator cleanup joins the worker while the worker's `finally` detaches its isolate thread. +- Terminal queue publication retries with a short timeout only while a consumer remains active. Once cancellation is set, it stops instead of blocking forever on a full queue, allowing the native worker to reach its detach `finally` block. - `DataWeave._decode_and_free` is exercised through a failing UTF-8 decode to verify native strings are freed from its existing `finally` block. - The only source adjustment replaces dynamic standard-library import resolution with a normal `os` import; no public names, signatures, or ABI fields changed. @@ -79,10 +80,23 @@ exit 0 ``` +9. Fix round 2 focused and full unit verification: + + ```text + python3 -m pytest tests/unit/test_streaming.py -m unit -v + 7 passed in 0.01s + python3 -m pytest tests/unit -m unit -v + 25 passed in 0.02s + python3 -m compileall -q src tests/unit + git diff --check + exit 0 + ``` + ## Commit - `2f6a788 test: add Python binding unit characterization coverage` - `2f169ee test: harden Python streaming unit coverage` +- Pending at report update: `fix: prevent Python streaming worker shutdown stalls` ## Concerns diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index 4f0c3edd..bfed7096 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -49,7 +49,7 @@ import os from dataclasses import dataclass from pathlib import Path -from queue import Queue +from queue import Full, Queue from threading import Event, Thread from typing import Any, Callable, Dict, Generator, Iterable, Optional, Union @@ -585,6 +585,14 @@ def _run_streaming_gen( _SENTINEL = object() q: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) + def _publish_terminal(item: Any) -> None: + while cancelled is None or not cancelled.is_set(): + try: + q.put(item, timeout=0.1) + return + except Full: + pass + @WRITE_CALLBACK def _write_cb(_ctx, buf, length): try: @@ -603,8 +611,8 @@ def _run_native(): worker_thread = self._graal_isolatethread_t_ptr() rc = self._lib.graal_attach_thread(self._isolate, ctypes.byref(worker_thread)) if rc != 0: - q.put({"success": False, "error": f"Failed to attach worker thread to isolate (code {rc})"}) - q.put(_SENTINEL) + _publish_terminal({"success": False, "error": f"Failed to attach worker thread to isolate (code {rc})"}) + _publish_terminal(_SENTINEL) return try: result_ptr = self._lib.run_script_callback( @@ -621,12 +629,12 @@ def _run_native(): else: raw = "" meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - q.put(meta) + _publish_terminal(meta) except Exception as e: - q.put({"success": False, "error": str(e)}) + _publish_terminal({"success": False, "error": str(e)}) finally: self._lib.graal_detach_thread(worker_thread) - q.put(_SENTINEL) + _publish_terminal(_SENTINEL) worker = Thread(target=_run_native, name="dw-streaming-worker", daemon=False) worker.start() diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index 74e3cba1..3561e81e 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -1,5 +1,5 @@ import ctypes -from queue import Queue +from queue import Full, Queue from threading import Event import pytest @@ -165,3 +165,50 @@ def run_script_callback(self, _thread, _script, _inputs, write_callback, _contex assert native.detached_event.wait(1) assert native.write_status == -1 + + +@pytest.mark.unit +def test_stream_early_close_does_not_block_terminal_publication_on_full_queue(monkeypatch): + cancelled = [] + terminal_blocked = Event() + + class CancellationAwareQueue(Queue): + def put(self, item, block=True, timeout=None): + if not isinstance(item, bytes) and cancelled[0].is_set(): + if timeout is None: + terminal_blocked.set() + raise Full + return super().put(item, block, timeout) + + class FullQueueFakeNative(FakeNative): + def __init__(self): + super().__init__('{"success": false, "error": "aborted"}') + self.queue_full = Event() + self.cancelled = None + + def run_script_callback(self, _thread, _script, _inputs, write_callback, _context): + first = ctypes.create_string_buffer(b"first") + assert write_callback(None, ctypes.addressof(first), 5) == 0 + second = ctypes.create_string_buffer(b"second") + assert write_callback(None, ctypes.addressof(second), 6) == 0 + self.queue_full.set() + assert self.cancelled.wait(1) + third = ctypes.create_string_buffer(b"third") + self.write_status = write_callback(None, ctypes.addressof(third), 5) + return self._response_pointer() + + monkeypatch.setattr(dataweave, "Queue", CancellationAwareQueue) + monkeypatch.setattr(dataweave, "_OUTPUT_QUEUE_MAXSIZE", 1) + native = FullQueueFakeNative() + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + cancelled.append(stream._cancelled) + native.cancelled = stream._cancelled + + assert next(stream) == b"first" + assert native.queue_full.wait(1) + stream._close() + + assert native.write_status == -1 + assert native.detached_event.wait(1) + assert terminal_blocked.is_set() is False From 9e8182d2e7f0581cdded0a8e7ce18c9a7c3e7573 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:39:46 -0300 Subject: [PATCH 10/30] docs: update Python streaming test report --- .../2026-08-19-python-binding-modernization/task-2-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md index 436c11e3..cf70742e 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md @@ -96,7 +96,7 @@ - `2f6a788 test: add Python binding unit characterization coverage` - `2f169ee test: harden Python streaming unit coverage` -- Pending at report update: `fix: prevent Python streaming worker shutdown stalls` +- `bcf6ccd fix: prevent Python streaming worker shutdown stalls` ## Concerns From 1276d12ee178f080204ff0037839392321d339bc Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:44:34 -0300 Subject: [PATCH 11/30] refactor: separate Python binding models and encoding --- .../task-3-report.md | 49 +++ native-lib/python/pytest.ini | 2 +- native-lib/python/src/dataweave/__init__.py | 305 ++---------------- native-lib/python/src/dataweave/encoding.py | 123 +++++++ native-lib/python/src/dataweave/models.py | 121 +++++++ native-lib/python/tests/unit/test_encoding.py | 8 + native-lib/python/tests/unit/test_facade.py | 6 + native-lib/python/tests/unit/test_models.py | 13 + 8 files changed, 350 insertions(+), 277 deletions(-) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md create mode 100644 native-lib/python/src/dataweave/encoding.py create mode 100644 native-lib/python/src/dataweave/models.py diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md new file mode 100644 index 00000000..0089c4fc --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md @@ -0,0 +1,49 @@ +# Task 3 Report: Extract Models And Encoding + +## Files Changed + +- Added `native-lib/python/src/dataweave/models.py` for public models, exceptions, callback aliases, ctypes callback signatures, and `Stream`. +- Added `native-lib/python/src/dataweave/encoding.py` for input normalization and native response parsing. +- Updated `native-lib/python/src/dataweave/__init__.py` to re-export the legacy public API and delegate internal model/encoding operations to the new modules. +- Updated `native-lib/python/tests/unit/test_facade.py`, `test_models.py`, and `test_encoding.py` with facade/export compatibility coverage. +- Updated `native-lib/python/pytest.ini` to use pytest importlib import mode, avoiding same-basename test-module collection collisions between unit and integration lanes. + +## Behavior Preservation + +- `dataweave.__all__`, module-level public names, and existing call signatures remain unchanged. +- `ExecutionResult.get_bytes()` and `ExecutionResult.get_string()` retain their prior base64, binary, charset, and unsuccessful-result behavior. +- The public models, exceptions, callback type aliases, and ctypes callback signatures are now also available from `dataweave.models`. +- `normalize_input_value`, `parse_native_encoded_response`, and `parse_streaming_result` are now public from `dataweave.encoding`; legacy private facade aliases remain for existing internal callers and tests. +- Native wire keys remain unchanged: `mimeType`, `charset`, `binary`, `result`, and `error`. +- No native/runtime extraction was performed. + +## Verification + +Command: + +```bash +cd native-lib/python +python3 -m pytest tests/unit tests/integration -m "unit or integration" -v +``` + +Output: + +```text +49 passed in 1.46s +``` + +Additional check: + +```bash +git diff --check +``` + +Output: no whitespace errors. + +## Commit + +Pending at report creation. + +## Concerns + +- The test tree has same-basename unit and integration modules (`test_streaming.py`). Pytest’s default prepend import mode causes collection to fail; `--import-mode=importlib` is now configured so the required combined test command runs consistently. diff --git a/native-lib/python/pytest.ini b/native-lib/python/pytest.ini index 09f6009a..33c8c8d4 100644 --- a/native-lib/python/pytest.ini +++ b/native-lib/python/pytest.ini @@ -1,6 +1,6 @@ [pytest] testpaths = tests -addopts = -m "not tck" +addopts = --import-mode=importlib -m "not tck" markers = unit: fast tests that do not require the native DataWeave library integration: tests that execute the staged native DataWeave library diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index bfed7096..0479fb5e 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -43,186 +43,38 @@ Call dataweave.cleanup() to release them earlier if needed. """ -import base64 import ctypes import json import os -from dataclasses import dataclass from pathlib import Path from queue import Full, Queue from threading import Event, Thread -from typing import Any, Callable, Dict, Generator, Iterable, Optional, Union +from typing import Any, Dict, Generator, Iterable, Optional + +from .encoding import normalize_input_value, parse_native_encoded_response, parse_streaming_result +from .models import ( + READ_CALLBACK, + WRITE_CALLBACK, + DataWeaveError, + DataWeaveLibraryNotFoundError, + DataWeaveScriptError, + ExecutionResult, + InputValue, + ReadCallback, + Stream, + StreamingResult, + WriteCallback, +) # Bound for streaming output queues: limits memory under slow/stalled consumers # by exerting backpressure onto the native producer. _OUTPUT_QUEUE_MAXSIZE = 512 -class DataWeaveError(Exception): - pass - - -class DataWeaveScriptError(DataWeaveError): - """Raised when a DataWeave script fails (compile or runtime error). - - Carries the full result object so callers can inspect details. - """ - - def __init__(self, result): - self.result = result - super().__init__(result.error or "Script execution failed") - - -class DataWeaveLibraryNotFoundError(Exception): - pass - - -# ctypes callback signatures matching NativeCallbacks.WriteCallback / ReadCallback. -# Buffer parameters use c_void_p (not c_char_p) because ctypes gives c_char_p -# special treatment that prevents writing into the buffer. -# int (*WriteCallback)(void *ctx, const char *buffer, int length) -WRITE_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) -# int (*ReadCallback)(void *ctx, char *buffer, int bufferSize) -READ_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) - - -WriteCallback = Callable[[bytes], int] -ReadCallback = Callable[[int], bytes] - _ENV_NATIVE_LIB = "DATAWEAVE_NATIVE_LIB" - -@dataclass -class InputValue: - content: Union[str, bytes] - mime_type: Optional[str] = None - charset: Optional[str] = None - properties: Optional[Dict[str, Union[str, int, bool]]] = None - - def encode_content(self) -> str: - if isinstance(self.content, bytes): - raw = self.content - else: - raw = self.content.encode(self.charset or "utf-8") - return base64.b64encode(raw).decode("ascii") - - -@dataclass(repr=False) -class ExecutionResult: - success: bool - result: Optional[str] - error: Optional[str] - binary: bool - mime_type: Optional[str] - charset: Optional[str] - - def __repr__(self): - if not self.success: - return f"ExecutionResult(success=False, error={self.error!r})" - preview = (self.result[:50] + "...") if self.result and len(self.result) > 50 else self.result - return f"ExecutionResult(success=True, mime_type={self.mime_type!r}, charset={self.charset!r}, result={preview!r})" - - def get_bytes(self) -> Optional[bytes]: - if not self.success or self.result is None: - return None - return base64.b64decode(self.result) - - def get_string(self) -> Optional[str]: - if not self.success or self.result is None: - return None - if self.binary: - return self.result - return self.get_bytes().decode(self.charset or "utf-8") - - -@dataclass -class StreamingResult: - """Metadata returned after a streaming execution completes.""" - success: bool - error: Optional[str] - mime_type: Optional[str] - charset: Optional[str] - binary: bool - - -class Stream: - """Wrapper around a streaming generator that captures metadata. - - Iterate to consume output chunks. After iteration completes, - access ``.metadata`` for the :class:`StreamingResult`. - """ - - def __init__(self, gen: Generator[bytes, None, StreamingResult]): - self._gen = gen - self._metadata: Optional[StreamingResult] = None - - def __iter__(self): - return self - - def __next__(self) -> bytes: - try: - return next(self._gen) - except StopIteration as e: - self._metadata = e.value - raise - - @property - def metadata(self) -> Optional[StreamingResult]: - return self._metadata - - def _close(self) -> None: - on_close = getattr(self, "_on_close", None) - if on_close is not None: - on_close() - self._gen.close() - - -def _parse_native_encoded_response(raw: str) -> ExecutionResult: - if raw is None: - return ExecutionResult(False, None, "Native returned null", False, None, None) - - if raw == "": - return ExecutionResult(False, None, "Native returned empty response", False, None, None) - - try: - parsed = json.loads(raw) - except Exception as e: - return ExecutionResult(False, None, f"Failed to parse native JSON response: {e}", False, None, None) - - if not isinstance(parsed, dict): - return ExecutionResult(False, None, "Native response JSON is not an object", False, None, None) - - success = bool(parsed.get("success", False)) - if not success: - return ExecutionResult(False, None, parsed.get("error"), False, None, None) - - return ExecutionResult( - success=True, - result=parsed.get("result"), - error=None, - binary=bool(parsed.get("binary", False)), - mime_type=parsed.get("mimeType"), - charset=parsed.get("charset"), - ) - - -def _parse_streaming_result(meta: dict) -> StreamingResult: - success = meta.get("success", False) - if not success: - return StreamingResult( - success=False, - error=meta.get("error"), - mime_type=None, - charset=None, - binary=False, - ) - return StreamingResult( - success=True, - error=None, - mime_type=meta.get("mimeType"), - charset=meta.get("charset"), - binary=meta.get("binary", False), - ) +_parse_native_encoded_response = parse_native_encoded_response +_parse_streaming_result = parse_streaming_result def _candidate_library_paths() -> list[Path]: @@ -267,74 +119,7 @@ def _find_library() -> str: ) -def _normalize_input_value(value: Any, mime_type: Optional[str] = None) -> Dict[str, Any]: - if isinstance(value, dict): - allowed_keys = {"content", "mimeType", "charset", "properties"} - extra_keys = set(value.keys()) - allowed_keys - if extra_keys: - raise DataWeaveError( - "Explicit input dict contains unsupported keys: " + ", ".join(sorted(extra_keys)) - ) - - if "content" in value or "mimeType" in value: - if "content" not in value or "mimeType" not in value: - raise DataWeaveError( - "Explicit input dict must include both 'content' and 'mimeType'" - ) - - raw_content = value.get("content") - charset = value.get("charset") or "utf-8" - if isinstance(raw_content, bytes): - encoded_content = base64.b64encode(raw_content).decode("ascii") - else: - encoded_content = base64.b64encode(str(raw_content).encode(charset)).decode("ascii") - - normalized: Dict[str, Any] = { - "content": encoded_content, - "mimeType": value.get("mimeType"), - } - if "charset" in value: - normalized["charset"] = value.get("charset") - if "properties" in value: - normalized["properties"] = value.get("properties") - return normalized - - if isinstance(value, InputValue): - out: Dict[str, Any] = { - "content": value.encode_content(), - "mimeType": value.mime_type or mime_type, - } - if value.charset is not None: - out["charset"] = value.charset - if value.properties is not None: - out["properties"] = value.properties - return out - - if isinstance(value, str): - content = value - default_mime = "text/plain" - elif isinstance(value, (int, float, bool)): - content = json.dumps(value) - default_mime = "application/json" - elif value is None: - content = "null" - default_mime = "application/json" - else: - try: - content = json.dumps(value) - default_mime = "application/json" - except (TypeError, ValueError): - content = str(value) - default_mime = "text/plain" - - charset = "utf-8" - encoded_content = base64.b64encode(content.encode(charset)).decode("ascii") - - return { - "content": encoded_content, - "mimeType": mime_type or default_mime, - "charset": charset, - } +_normalize_input_value = normalize_input_value class DataWeave: @@ -505,7 +290,7 @@ def run_callback( if inputs is None: inputs = {} - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} + normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} inputs_json = json.dumps(normalized_inputs) @WRITE_CALLBACK @@ -529,7 +314,7 @@ def _write_cb(_ctx, buf, length): except Exception as e: raise DataWeaveError(f"Failed to execute callback streaming: {e}") - return _parse_streaming_result(meta) + return parse_streaming_result(meta) def run_streaming( self, @@ -579,7 +364,7 @@ def _run_streaming_gen( if inputs is None: inputs = {} - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} + normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} inputs_json = json.dumps(normalized_inputs) _SENTINEL = object() @@ -659,23 +444,7 @@ def _run_native(): if meta is None: meta = {"success": False, "error": "No metadata received from native call"} - success = meta.get("success", False) - if not success: - return StreamingResult( - success=False, - error=meta.get("error"), - mime_type=None, - charset=None, - binary=False, - ) - - return StreamingResult( - success=True, - error=None, - mime_type=meta.get("mimeType"), - charset=meta.get("charset"), - binary=meta.get("binary", False), - ) + return parse_streaming_result(meta) def run_transform( self, @@ -739,7 +508,7 @@ def _run_transform_gen( if inputs is None: inputs = {} - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} + normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} inputs_json = json.dumps(normalized_inputs) _SENTINEL = object() @@ -842,23 +611,7 @@ def _run_native(): if meta is None: meta = {"success": False, "error": "No metadata received from native call"} - success = meta.get("success", False) - if not success: - return StreamingResult( - success=False, - error=meta.get("error"), - mime_type=None, - charset=None, - binary=False, - ) - - return StreamingResult( - success=True, - error=None, - mime_type=meta.get("mimeType"), - charset=meta.get("charset"), - binary=meta.get("binary", False), - ) + return parse_streaming_result(meta) def run_input_output_callback( self, @@ -899,7 +652,7 @@ def run_input_output_callback( if inputs is None: inputs = {} - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} + normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} inputs_json = json.dumps(normalized_inputs) @READ_CALLBACK @@ -939,7 +692,7 @@ def _write_cb(_ctx, buf, length): except Exception as e: raise DataWeaveError(f"Failed to execute callback input/output streaming: {e}") - return _parse_streaming_result(meta) + return parse_streaming_result(meta) def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bool = False) -> ExecutionResult: if not self._initialized: @@ -948,7 +701,7 @@ def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_err if inputs is None: inputs = {} - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} + normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} inputs_json = json.dumps(normalized_inputs) try: @@ -958,7 +711,7 @@ def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_err inputs_json.encode("utf-8"), ) raw = self._decode_and_free(result_ptr) - result = _parse_native_encoded_response(raw) + result = parse_native_encoded_response(raw) except Exception as e: raise DataWeaveError(f"Failed to execute script: {e}") diff --git a/native-lib/python/src/dataweave/encoding.py b/native-lib/python/src/dataweave/encoding.py new file mode 100644 index 00000000..0f3c0881 --- /dev/null +++ b/native-lib/python/src/dataweave/encoding.py @@ -0,0 +1,123 @@ +import base64 +import json +from typing import Any, Dict, Optional + +from .models import DataWeaveError, ExecutionResult, InputValue, StreamingResult + + +def parse_native_encoded_response(raw: str) -> ExecutionResult: + if raw is None: + return ExecutionResult(False, None, "Native returned null", False, None, None) + + if raw == "": + return ExecutionResult(False, None, "Native returned empty response", False, None, None) + + try: + parsed = json.loads(raw) + except Exception as e: + return ExecutionResult(False, None, f"Failed to parse native JSON response: {e}", False, None, None) + + if not isinstance(parsed, dict): + return ExecutionResult(False, None, "Native response JSON is not an object", False, None, None) + + success = bool(parsed.get("success", False)) + if not success: + return ExecutionResult(False, None, parsed.get("error"), False, None, None) + + return ExecutionResult( + success=True, + result=parsed.get("result"), + error=None, + binary=bool(parsed.get("binary", False)), + mime_type=parsed.get("mimeType"), + charset=parsed.get("charset"), + ) + + +def parse_streaming_result(meta: dict) -> StreamingResult: + success = meta.get("success", False) + if not success: + return StreamingResult( + success=False, + error=meta.get("error"), + mime_type=None, + charset=None, + binary=False, + ) + return StreamingResult( + success=True, + error=None, + mime_type=meta.get("mimeType"), + charset=meta.get("charset"), + binary=meta.get("binary", False), + ) + + +def normalize_input_value(value: Any, mime_type: Optional[str] = None) -> Dict[str, Any]: + if isinstance(value, dict): + allowed_keys = {"content", "mimeType", "charset", "properties"} + extra_keys = set(value.keys()) - allowed_keys + if extra_keys: + raise DataWeaveError( + "Explicit input dict contains unsupported keys: " + ", ".join(sorted(extra_keys)) + ) + + if "content" in value or "mimeType" in value: + if "content" not in value or "mimeType" not in value: + raise DataWeaveError( + "Explicit input dict must include both 'content' and 'mimeType'" + ) + + raw_content = value.get("content") + charset = value.get("charset") or "utf-8" + if isinstance(raw_content, bytes): + encoded_content = base64.b64encode(raw_content).decode("ascii") + else: + encoded_content = base64.b64encode(str(raw_content).encode(charset)).decode("ascii") + + normalized: Dict[str, Any] = { + "content": encoded_content, + "mimeType": value.get("mimeType"), + } + if "charset" in value: + normalized["charset"] = value.get("charset") + if "properties" in value: + normalized["properties"] = value.get("properties") + return normalized + + if isinstance(value, InputValue): + out: Dict[str, Any] = { + "content": value.encode_content(), + "mimeType": value.mime_type or mime_type, + } + if value.charset is not None: + out["charset"] = value.charset + if value.properties is not None: + out["properties"] = value.properties + return out + + if isinstance(value, str): + content = value + default_mime = "text/plain" + elif isinstance(value, (int, float, bool)): + content = json.dumps(value) + default_mime = "application/json" + elif value is None: + content = "null" + default_mime = "application/json" + else: + try: + content = json.dumps(value) + default_mime = "application/json" + except (TypeError, ValueError): + content = str(value) + default_mime = "text/plain" + + charset = "utf-8" + encoded_content = base64.b64encode(content.encode(charset)).decode("ascii") + + return { + "content": encoded_content, + "mimeType": mime_type or default_mime, + "charset": charset, + } diff --git a/native-lib/python/src/dataweave/models.py b/native-lib/python/src/dataweave/models.py new file mode 100644 index 00000000..fd932261 --- /dev/null +++ b/native-lib/python/src/dataweave/models.py @@ -0,0 +1,121 @@ +import base64 +import ctypes +from dataclasses import dataclass +from typing import Callable, Dict, Generator, Optional, Union + + +class DataWeaveError(Exception): + pass + + +class DataWeaveScriptError(DataWeaveError): + """Raised when a DataWeave script fails (compile or runtime error). + + Carries the full result object so callers can inspect details. + """ + + def __init__(self, result): + self.result = result + super().__init__(result.error or "Script execution failed") + + +class DataWeaveLibraryNotFoundError(Exception): + pass + + +# ctypes callback signatures matching NativeCallbacks.WriteCallback / ReadCallback. +# Buffer parameters use c_void_p (not c_char_p) because ctypes gives c_char_p +# special treatment that prevents writing into the buffer. +# int (*WriteCallback)(void *ctx, const char *buffer, int length) +WRITE_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) +# int (*ReadCallback)(void *ctx, char *buffer, int bufferSize) +READ_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) + + +WriteCallback = Callable[[bytes], int] +ReadCallback = Callable[[int], bytes] + + +@dataclass +class InputValue: + content: Union[str, bytes] + mime_type: Optional[str] = None + charset: Optional[str] = None + properties: Optional[Dict[str, Union[str, int, bool]]] = None + + def encode_content(self) -> str: + if isinstance(self.content, bytes): + raw = self.content + else: + raw = self.content.encode(self.charset or "utf-8") + return base64.b64encode(raw).decode("ascii") + + +@dataclass(repr=False) +class ExecutionResult: + success: bool + result: Optional[str] + error: Optional[str] + binary: bool + mime_type: Optional[str] + charset: Optional[str] + + def __repr__(self): + if not self.success: + return f"ExecutionResult(success=False, error={self.error!r})" + preview = (self.result[:50] + "...") if self.result and len(self.result) > 50 else self.result + return f"ExecutionResult(success=True, mime_type={self.mime_type!r}, charset={self.charset!r}, result={preview!r})" + + def get_bytes(self) -> Optional[bytes]: + if not self.success or self.result is None: + return None + return base64.b64decode(self.result) + + def get_string(self) -> Optional[str]: + if not self.success or self.result is None: + return None + if self.binary: + return self.result + return self.get_bytes().decode(self.charset or "utf-8") + + +@dataclass +class StreamingResult: + """Metadata returned after a streaming execution completes.""" + success: bool + error: Optional[str] + mime_type: Optional[str] + charset: Optional[str] + binary: bool + + +class Stream: + """Wrapper around a streaming generator that captures metadata. + + Iterate to consume output chunks. After iteration completes, + access ``.metadata`` for the :class:`StreamingResult`. + """ + + def __init__(self, gen: Generator[bytes, None, StreamingResult]): + self._gen = gen + self._metadata: Optional[StreamingResult] = None + + def __iter__(self): + return self + + def __next__(self) -> bytes: + try: + return next(self._gen) + except StopIteration as e: + self._metadata = e.value + raise + + @property + def metadata(self) -> Optional[StreamingResult]: + return self._metadata + + def _close(self) -> None: + on_close = getattr(self, "_on_close", None) + if on_close is not None: + on_close() + self._gen.close() diff --git a/native-lib/python/tests/unit/test_encoding.py b/native-lib/python/tests/unit/test_encoding.py index 2b1fb7b9..d16bd357 100644 --- a/native-lib/python/tests/unit/test_encoding.py +++ b/native-lib/python/tests/unit/test_encoding.py @@ -3,6 +3,14 @@ import pytest import dataweave +from dataweave import encoding + + +@pytest.mark.unit +def test_public_encoding_functions_are_used_by_compatibility_aliases(): + assert dataweave._normalize_input_value is encoding.normalize_input_value + assert dataweave._parse_native_encoded_response is encoding.parse_native_encoded_response + assert dataweave._parse_streaming_result is encoding.parse_streaming_result @pytest.mark.unit diff --git a/native-lib/python/tests/unit/test_facade.py b/native-lib/python/tests/unit/test_facade.py index f0c628f3..d3f76e8a 100644 --- a/native-lib/python/tests/unit/test_facade.py +++ b/native-lib/python/tests/unit/test_facade.py @@ -3,6 +3,12 @@ import dataweave +@pytest.mark.unit +def test_facade_exports_all_declared_public_names(): + for name in dataweave.__all__: + assert hasattr(dataweave, name), name + + @pytest.mark.unit def test_global_facade_initializes_once_and_cleanup_allows_recreation(monkeypatch): created = [] diff --git a/native-lib/python/tests/unit/test_models.py b/native-lib/python/tests/unit/test_models.py index c523be58..2e678e2b 100644 --- a/native-lib/python/tests/unit/test_models.py +++ b/native-lib/python/tests/unit/test_models.py @@ -3,6 +3,19 @@ import pytest import dataweave +from dataweave import models + + +@pytest.mark.unit +def test_public_models_are_exported_from_models_module(): + assert models.ExecutionResult is dataweave.ExecutionResult + assert models.InputValue is dataweave.InputValue + assert models.StreamingResult is dataweave.StreamingResult + assert models.DataWeaveError is dataweave.DataWeaveError + assert models.DataWeaveScriptError is dataweave.DataWeaveScriptError + assert models.DataWeaveLibraryNotFoundError is dataweave.DataWeaveLibraryNotFoundError + assert models.READ_CALLBACK is dataweave.READ_CALLBACK + assert models.WRITE_CALLBACK is dataweave.WRITE_CALLBACK @pytest.mark.unit From d551dd246795bbd8325e636fe7586d1426992096 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:44:45 -0300 Subject: [PATCH 12/30] docs: finalize Python binding task 3 report --- .../2026-08-19-python-binding-modernization/task-3-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md index 0089c4fc..22da0e66 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md @@ -42,7 +42,7 @@ Output: no whitespace errors. ## Commit -Pending at report creation. +`1276d12 refactor: separate Python binding models and encoding` ## Concerns From 6530b1d8357f3e7fba8fbed86b4529ee3056eff3 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:46:58 -0300 Subject: [PATCH 13/30] test: lock Python binding facade exports --- .../task-3-report.md | 7 +++++ native-lib/python/tests/unit/test_facade.py | 28 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md index 22da0e66..aa71369e 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md @@ -47,3 +47,10 @@ Output: no whitespace errors. ## Concerns - The test tree has same-basename unit and integration modules (`test_streaming.py`). Pytest’s default prepend import mode causes collection to fail; `--import-mode=importlib` is now configured so the required combined test command runs consistently. + +## Fix Round 1 + +- Replaced the dynamic facade export assertion with a fixed, pre-Task-3 list of all 18 legacy public names. +- The test now verifies every legacy name is explicitly present in `dataweave.__all__` and resolves through `getattr(dataweave, name)`. +- Focused verification: `python3 -m pytest tests/unit/test_facade.py -m unit -v` reported `3 passed in 0.01s`. +- Combined verification and fix-round commit are recorded in the follow-up commit for this round. diff --git a/native-lib/python/tests/unit/test_facade.py b/native-lib/python/tests/unit/test_facade.py index d3f76e8a..c61f8bd0 100644 --- a/native-lib/python/tests/unit/test_facade.py +++ b/native-lib/python/tests/unit/test_facade.py @@ -4,9 +4,31 @@ @pytest.mark.unit -def test_facade_exports_all_declared_public_names(): - for name in dataweave.__all__: - assert hasattr(dataweave, name), name +def test_facade_preserves_fixed_legacy_public_exports(): + legacy_exports = [ + "DataWeave", + "DataWeaveError", + "DataWeaveLibraryNotFoundError", + "DataWeaveScriptError", + "ExecutionResult", + "InputValue", + "ReadCallback", + "Stream", + "StreamingResult", + "WriteCallback", + "READ_CALLBACK", + "WRITE_CALLBACK", + "run", + "run_callback", + "run_input_output_callback", + "run_streaming", + "run_transform", + "cleanup", + ] + + for name in legacy_exports: + assert name in dataweave.__all__ + getattr(dataweave, name) @pytest.mark.unit From 9930f5bac6a0057e7adfb57f2b261510db4584d2 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 15:58:59 -0300 Subject: [PATCH 14/30] refactor: isolate Python native runtime and stream worker --- .../task-4-report.md | 24 + native-lib/python/src/dataweave/__init__.py | 780 +----------------- native-lib/python/src/dataweave/native.py | 161 ++++ native-lib/python/src/dataweave/runtime.py | 257 ++++++ native-lib/python/tests/unit/test_native.py | 50 ++ .../python/tests/unit/test_streaming.py | 29 +- 6 files changed, 536 insertions(+), 765 deletions(-) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md create mode 100644 native-lib/python/src/dataweave/native.py create mode 100644 native-lib/python/src/dataweave/runtime.py diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md new file mode 100644 index 00000000..80aa22f1 --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md @@ -0,0 +1,24 @@ +# Task 4 Report: Native Adapter And Runtime Streaming + +## Status + +Complete. The Python binding now separates native ctypes/isolate ownership from the public `DataWeave` orchestration API. + +## Changes + +- Added `dataweave.native.NativeRuntime` for native library discovery/loading, opaque Graal pointer types, ABI registration, isolate/thread lifecycle, native invocation, and C-string release. +- Added `dataweave.runtime.DataWeave` for buffered execution, direct callbacks, and both streaming APIs. +- Consolidated output-only and duplex streaming onto one worker implementation with bounded queue backpressure, cancellation, timed consumer waits, worker join timeout, metadata propagation, and detach in `finally`. +- Preserved oversized input chunk remainders and Python callback failure translation to the native `-1` abort status. +- Reduced `dataweave.__init__` to the public facade, singleton lifecycle, re-exports, and legacy private helper aliases. +- Added unit coverage for native ABI registration, load failure, idempotent cleanup, module ownership, and worker timeout behavior. + +## Verification + +- `python3 -m pytest tests/unit tests/integration` passed: 53 tests. +- `./gradlew native-lib:pythonTest` passed: native image build and 53 Python tests. + +## Concerns + +- The Gradle native-image invocation emits existing Graal/Gradle deprecation and restricted-native-access warnings; it still completed successfully. +- No Task 5, documentation, or CI work was included. diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index 0479fb5e..f1db4641 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -1,57 +1,12 @@ -""" -DataWeave Python Module - -Execute DataWeave scripts from Python via a GraalVM native shared library. -Supports buffered execution, output streaming, and bidirectional streaming -with constant memory overhead. - -Basic usage: - import dataweave - - result = dataweave.run("2 + 2") - print(result.get_string()) # "4" - -Output streaming (yields chunks as produced): - stream = dataweave.run_streaming("output json --- (1 to 10000) map {id: $}") - for chunk in stream: - sys.stdout.buffer.write(chunk) - -Bidirectional streaming (iterable in, generator out): - with open("large.json", "rb") as f: - stream = dataweave.run_transform( - "output csv --- payload", - input_stream=iter(lambda: f.read(8192), b""), - input_mime_type="application/json", - ) - for chunk in stream: - process(chunk) - -Context manager (explicit lifecycle control): - from dataweave import DataWeave - - with DataWeave() as dw: - result = dw.run("2 + 2") - print(result.get_string()) - -Error handling: - try: - result = dataweave.run("invalid", raise_on_error=True) - except dataweave.DataWeaveScriptError as e: - print(e.result.error) - -Native resources are released automatically at interpreter exit via atexit. -Call dataweave.cleanup() to release them earlier if needed. -""" +"""Public facade for the DataWeave Python native binding.""" import ctypes -import json -import os -from pathlib import Path -from queue import Full, Queue -from threading import Event, Thread -from typing import Any, Dict, Generator, Iterable, Optional -from .encoding import normalize_input_value, parse_native_encoded_response, parse_streaming_result +from typing import Any, Dict, Iterable, Optional + +from .encoding import normalize_input_value as _normalize_input_value +from .encoding import parse_native_encoded_response as _parse_native_encoded_response +from .encoding import parse_streaming_result as _parse_streaming_result from .models import ( READ_CALLBACK, WRITE_CALLBACK, @@ -65,667 +20,9 @@ StreamingResult, WriteCallback, ) - -# Bound for streaming output queues: limits memory under slow/stalled consumers -# by exerting backpressure onto the native producer. -_OUTPUT_QUEUE_MAXSIZE = 512 - - -_ENV_NATIVE_LIB = "DATAWEAVE_NATIVE_LIB" - -_parse_native_encoded_response = parse_native_encoded_response -_parse_streaming_result = parse_streaming_result - - -def _candidate_library_paths() -> list[Path]: - paths: list[Path] = [] - - env_value = (os.environ.get(_ENV_NATIVE_LIB) or "").strip() - if env_value: - paths.append(Path(env_value)) - - pkg_dir = Path(__file__).resolve().parent - native_dir = pkg_dir / "native" - paths.append(native_dir / "dwlib.dylib") - paths.append(native_dir / "dwlib.so") - paths.append(native_dir / "dwlib.dll") - - # Dev fallback: if this package is being used from the data-weave-cli repo - # tree, locate native-lib/build/native/nativeCompile. - for parent in pkg_dir.parents: - build_dir = parent / "build" / "native" / "nativeCompile" - if build_dir.exists(): - paths.append(build_dir / "dwlib.dylib") - paths.append(build_dir / "dwlib.so") - paths.append(build_dir / "dwlib.dll") - break - - # CWD fallback - paths.append(Path("dwlib.dylib")) - paths.append(Path("dwlib.so")) - paths.append(Path("dwlib.dll")) - - return paths - - -def _find_library() -> str: - for p in _candidate_library_paths(): - if p.exists() and p.is_file(): - return str(p) - - raise DataWeaveLibraryNotFoundError( - "Could not find DataWeave native library (dwlib). " - f"Set {_ENV_NATIVE_LIB} to an absolute path or install a wheel that bundles the native library." - ) - - -_normalize_input_value = normalize_input_value - - -class DataWeave: - def __init__(self, lib_path: Optional[str] = None): - self._lib_path = lib_path or _find_library() - self._lib = None - self._isolate = None - self._thread = None - self._initialized = False - - def _load_library(self): - try: - self._lib = ctypes.CDLL(self._lib_path) - except OSError as e: - raise DataWeaveError(f"Failed to load library from {self._lib_path}: {e}") - - def _setup_graal_structures(self): - class graal_isolate_t(ctypes.Structure): - pass - - class graal_isolatethread_t(ctypes.Structure): - pass - - self._graal_isolate_t_ptr = ctypes.POINTER(graal_isolate_t) - self._graal_isolatethread_t_ptr = ctypes.POINTER(graal_isolatethread_t) - - def _create_isolate(self): - self._lib.graal_create_isolate.argtypes = [ - ctypes.c_void_p, - ctypes.POINTER(self._graal_isolate_t_ptr), - ctypes.POINTER(self._graal_isolatethread_t_ptr), - ] - self._lib.graal_create_isolate.restype = ctypes.c_int - - self._isolate = self._graal_isolate_t_ptr() - self._thread = self._graal_isolatethread_t_ptr() - - result = self._lib.graal_create_isolate(None, ctypes.byref(self._isolate), ctypes.byref(self._thread)) - if result != 0: - raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") - - def _setup_functions(self): - if not hasattr(self._lib, "run_script"): - raise DataWeaveError("Native library does not export run_script") - - self._lib.run_script.argtypes = [ - self._graal_isolatethread_t_ptr, - ctypes.c_char_p, - ctypes.c_char_p, - ] - self._lib.run_script.restype = ctypes.c_void_p - - if hasattr(self._lib, "free_cstring"): - self._lib.free_cstring.argtypes = [self._graal_isolatethread_t_ptr, ctypes.c_void_p] - self._lib.free_cstring.restype = None - - # Thread attachment for background threads - if hasattr(self._lib, "graal_attach_thread"): - self._lib.graal_attach_thread.argtypes = [self._graal_isolate_t_ptr, ctypes.POINTER(self._graal_isolatethread_t_ptr)] - self._lib.graal_attach_thread.restype = ctypes.c_int - if hasattr(self._lib, "graal_detach_thread"): - self._lib.graal_detach_thread.argtypes = [self._graal_isolatethread_t_ptr] - self._lib.graal_detach_thread.restype = ctypes.c_int - if hasattr(self._lib, "graal_tear_down_isolate"): - self._lib.graal_tear_down_isolate.argtypes = [self._graal_isolatethread_t_ptr] - self._lib.graal_tear_down_isolate.restype = ctypes.c_int - - # Callback-based Streaming API - if hasattr(self._lib, "run_script_callback"): - self._lib.run_script_callback.argtypes = [ - self._graal_isolatethread_t_ptr, - ctypes.c_char_p, - ctypes.c_char_p, - WRITE_CALLBACK, - ctypes.c_void_p, - ] - self._lib.run_script_callback.restype = ctypes.c_void_p - - self._has_callback_streaming = True - else: - self._has_callback_streaming = False - - if hasattr(self._lib, "run_script_input_output_callback"): - self._lib.run_script_input_output_callback.argtypes = [ - self._graal_isolatethread_t_ptr, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_char_p, - READ_CALLBACK, - WRITE_CALLBACK, - ctypes.c_void_p, - ] - self._lib.run_script_input_output_callback.restype = ctypes.c_void_p - - self._has_callback_input_output = True - else: - self._has_callback_input_output = False - - def _decode_and_free(self, ptr: Optional[int]) -> str: - if not ptr: - return "" - - try: - result_bytes = ctypes.string_at(ptr) - return result_bytes.decode("utf-8") - finally: - if self._lib is not None and hasattr(self._lib, "free_cstring"): - self._lib.free_cstring(self._thread, ptr) - - def initialize(self): - if self._initialized: - return - - self._load_library() - self._setup_graal_structures() - self._create_isolate() - self._setup_functions() - self._initialized = True - - def cleanup(self): - if not self._initialized: - return - - if hasattr(self._lib, "graal_tear_down_isolate") and self._thread: - try: - self._lib.graal_tear_down_isolate(self._thread) - except Exception: - pass - elif hasattr(self._lib, "graal_detach_thread") and self._thread: - try: - self._lib.graal_detach_thread(self._thread) - except Exception: - pass - - self._initialized = False - self._thread = None - self._isolate = None - self._lib = None - - def run_callback( - self, - script: str, - write_callback: WriteCallback, - inputs: Optional[Dict[str, Any]] = None, - ) -> StreamingResult: - """Execute a DataWeave script and stream the output via a write callback. - - The native side reads the output internally and invokes *write_callback* - for each chunk. - - :param script: the DataWeave script source - :param write_callback: callable ``(data: bytes) -> int`` invoked with each - output chunk. Must return ``0`` on success or non-zero to abort. - :param inputs: optional input bindings (same format as :meth:`run`) - :return: a :class:`StreamingResult` with metadata - :raises DataWeaveError: if the runtime is not initialized or the callback API - is not available - """ - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_streaming: - raise DataWeaveError( - "Native library does not support callback streaming API (run_script_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - data = ctypes.string_at(buf, length) - return write_callback(data) - except Exception: - return -1 - - try: - result_ptr = self._lib.run_script_callback( - self._thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - _write_cb, - None, - ) - raw = self._decode_and_free(result_ptr) - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - except Exception as e: - raise DataWeaveError(f"Failed to execute callback streaming: {e}") - - return parse_streaming_result(meta) - - def run_streaming( - self, - script: str, - inputs: Optional[Dict[str, Any]] = None, - ) -> Stream: - """Execute a DataWeave script and yield output chunks as they arrive. - - Chunks are yielded in real-time as the native engine produces them, - using a background thread and queue. The caller sees data before the - script finishes executing. - - Usage:: - - with DataWeave() as dw: - stream = dw.run_streaming("output json --- {items: (1 to 100)}") - for chunk in stream: - sys.stdout.buffer.write(chunk) - metadata = stream.metadata # StreamingResult with mime_type, charset, etc. - - :param script: the DataWeave script source - :param inputs: optional input bindings (same format as :meth:`run`) - :return: a :class:`Stream` yielding ``bytes`` chunks; after iteration, - ``.metadata`` holds a :class:`StreamingResult` - :raises DataWeaveError: if the runtime is not initialized or the callback API - is not available - """ - cancelled = Event() - stream = Stream(self._run_streaming_gen(script, inputs, cancelled)) - stream._on_close = cancelled.set - stream._cancelled = cancelled - return stream - - def _run_streaming_gen( - self, - script: str, - inputs: Optional[Dict[str, Any]] = None, - cancelled: Optional[Event] = None, - ) -> Generator[bytes, None, StreamingResult]: - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_streaming: - raise DataWeaveError( - "Native library does not support callback streaming API (run_script_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - _SENTINEL = object() - q: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) - - def _publish_terminal(item: Any) -> None: - while cancelled is None or not cancelled.is_set(): - try: - q.put(item, timeout=0.1) - return - except Full: - pass - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - if cancelled is not None and cancelled.is_set(): - return -1 - # With maxsize set, put() blocks when the queue is full, exerting - # backpressure onto the native producer. Timeout prevents indefinite - # blocking if the consumer abandons the generator. - q.put(ctypes.string_at(buf, length), timeout=30) - return 0 - except Exception: - # Timeout or other failure: signal the native side to abort. - return -1 - - def _run_native(): - worker_thread = self._graal_isolatethread_t_ptr() - rc = self._lib.graal_attach_thread(self._isolate, ctypes.byref(worker_thread)) - if rc != 0: - _publish_terminal({"success": False, "error": f"Failed to attach worker thread to isolate (code {rc})"}) - _publish_terminal(_SENTINEL) - return - try: - result_ptr = self._lib.run_script_callback( - worker_thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - _write_cb, - None, - ) - raw_ptr = result_ptr - if raw_ptr: - raw = ctypes.string_at(raw_ptr).decode("utf-8") - self._lib.free_cstring(worker_thread, raw_ptr) - else: - raw = "" - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - _publish_terminal(meta) - except Exception as e: - _publish_terminal({"success": False, "error": str(e)}) - finally: - self._lib.graal_detach_thread(worker_thread) - _publish_terminal(_SENTINEL) - - worker = Thread(target=_run_native, name="dw-streaming-worker", daemon=False) - worker.start() - - meta = None - try: - while True: - item = q.get() - if item is _SENTINEL: - break - if isinstance(item, dict): - meta = item - else: - yield item - finally: - if cancelled is not None: - cancelled.set() - worker.join(timeout=30) - if worker.is_alive(): - raise DataWeaveError("Worker thread timeout after 30 seconds") - - if meta is None: - meta = {"success": False, "error": "No metadata received from native call"} - - return parse_streaming_result(meta) - - def run_transform( - self, - script: str, - input_stream: Iterable[bytes], - input_name: str = "payload", - input_mime_type: str = "application/json", - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, - ) -> Stream: - """Execute a DataWeave script with streaming input and output. - - Input data is pulled from *input_stream* (any iterable of bytes) and - output chunks are yielded as they are produced — fully streaming in - both directions with constant memory overhead. - - Usage:: - - with DataWeave() as dw: - with open("large.json", "rb") as f: - stream = dw.run_transform( - "output application/csv --- payload", - input_stream=iter(lambda: f.read(8192), b""), - input_mime_type="application/json", - ) - for chunk in stream: - sys.stdout.buffer.write(chunk) - metadata = stream.metadata - - :param script: the DataWeave script source - :param input_stream: iterable yielding ``bytes`` chunks for the input binding - :param input_name: binding name for the streamed input (default ``"payload"``) - :param input_mime_type: MIME type of the streamed input - :param input_charset: charset of the streamed input (default UTF-8) - :param inputs: optional additional input bindings (same format as :meth:`run`) - :return: a :class:`Stream` yielding ``bytes`` output chunks; after iteration, - ``.metadata`` holds a :class:`StreamingResult` - :raises DataWeaveError: if the runtime is not initialized or the API is missing - """ - return Stream(self._run_transform_gen( - script, input_stream, input_name, input_mime_type, input_charset, inputs, - )) - - def _run_transform_gen( - self, - script: str, - input_stream: Iterable[bytes], - input_name: str = "payload", - input_mime_type: str = "application/json", - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, - ) -> Generator[bytes, None, StreamingResult]: - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_input_output: - raise DataWeaveError( - "Native library does not support callback input/output API " - "(run_script_input_output_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - _SENTINEL = object() - q: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - # With maxsize set, put() blocks when the queue is full, exerting - # backpressure onto the native producer. Timeout prevents indefinite - # blocking if the consumer abandons the generator. - q.put(ctypes.string_at(buf, length), timeout=30) - return 0 - except Exception: - # Timeout or other failure: signal the native side to abort. - return -1 - - input_iter = iter(input_stream) - # Stateful reader to handle chunks larger than native buffer size. - # Mirrors Node binding's createChunkReader: maintains current chunk + offset, - # returns at most buf_size bytes per call, advances to next chunk only when - # current chunk is fully consumed. - read_state = {"current_chunk": b"", "offset": 0, "done": False} - - @READ_CALLBACK - def _read_cb(_ctx, buf, buf_size): - try: - while True: - # If we have buffered data, return what we can - if read_state["offset"] < len(read_state["current_chunk"]): - chunk = read_state["current_chunk"] - offset = read_state["offset"] - n = min(len(chunk) - offset, buf_size) - ctypes.memmove(buf, chunk[offset:offset+n], n) - read_state["offset"] += n - return n - # Current chunk exhausted and iterator done -> EOF - if read_state["done"]: - return 0 - # Pull next chunk - data = next(input_iter, None) - if data is None or not data: - read_state["done"] = True - return 0 - read_state["current_chunk"] = data - read_state["offset"] = 0 - # Loop back to serve from the new chunk - except Exception: - return -1 - - def _run_native(): - worker_thread = self._graal_isolatethread_t_ptr() - rc = self._lib.graal_attach_thread(self._isolate, ctypes.byref(worker_thread)) - if rc != 0: - q.put({"success": False, "error": f"Failed to attach worker thread to isolate (code {rc})"}) - q.put(_SENTINEL) - return - try: - result_ptr = self._lib.run_script_input_output_callback( - worker_thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - input_name.encode("utf-8"), - input_mime_type.encode("utf-8"), - input_charset.encode("utf-8") if input_charset else None, - _read_cb, - _write_cb, - None, - ) - if result_ptr: - raw = ctypes.string_at(result_ptr).decode("utf-8") - self._lib.free_cstring(worker_thread, result_ptr) - else: - raw = "" - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - q.put(meta) - except Exception as e: - q.put({"success": False, "error": str(e)}) - finally: - self._lib.graal_detach_thread(worker_thread) - q.put(_SENTINEL) - - worker = Thread(target=_run_native, name="dw-transform-worker", daemon=False) - worker.start() - - meta = None - while True: - item = q.get() - if item is _SENTINEL: - break - if isinstance(item, dict): - meta = item - else: - yield item - - worker.join(timeout=30) - if worker.is_alive(): - raise DataWeaveError("Worker thread timeout after 30 seconds") - - if meta is None: - meta = {"success": False, "error": "No metadata received from native call"} - - return parse_streaming_result(meta) - - def run_input_output_callback( - self, - script: str, - input_name: str, - input_mime_type: str, - read_callback: ReadCallback, - write_callback: WriteCallback, - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, - ) -> StreamingResult: - """Execute a DataWeave script with callback-driven input *and* output streaming. - - The native side calls *read_callback* on a background thread to pull input - data for the binding named *input_name*, and calls *write_callback* on the - calling thread to push output chunks. - - :param script: the DataWeave script source - :param input_name: the binding name for the callback-supplied input - :param input_mime_type: MIME type of the callback-supplied input - :param read_callback: callable ``(buf_size: int) -> bytes`` returning the - next chunk, empty bytes ``b""`` on EOF, or raising on error - :param write_callback: callable ``(data: bytes) -> int`` returning ``0`` on - success or non-zero to abort - :param input_charset: charset of the callback-supplied input (default UTF-8) - :param inputs: optional additional input bindings - :return: a :class:`StreamingResult` with metadata - :raises DataWeaveError: if the runtime is not initialized or the API is missing - """ - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_input_output: - raise DataWeaveError( - "Native library does not support callback input/output API " - "(run_script_input_output_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - @READ_CALLBACK - def _read_cb(_ctx, buf, buf_size): - try: - data = read_callback(buf_size) - if not data: - return 0 # EOF - n = min(len(data), buf_size) - ctypes.memmove(buf, data, n) - return n - except Exception: - return -1 - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - data = ctypes.string_at(buf, length) - return write_callback(data) - except Exception: - return -1 - - try: - result_ptr = self._lib.run_script_input_output_callback( - self._thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - input_name.encode("utf-8"), - input_mime_type.encode("utf-8"), - input_charset.encode("utf-8") if input_charset else None, - _read_cb, - _write_cb, - None, - ) - raw = self._decode_and_free(result_ptr) - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - except Exception as e: - raise DataWeaveError(f"Failed to execute callback input/output streaming: {e}") - - return parse_streaming_result(meta) - - def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bool = False) -> ExecutionResult: - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - - if inputs is None: - inputs = {} - - normalized_inputs = {key: normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - try: - result_ptr = self._lib.run_script( - self._thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - ) - raw = self._decode_and_free(result_ptr) - result = parse_native_encoded_response(raw) - except Exception as e: - raise DataWeaveError(f"Failed to execute script: {e}") - - if raise_on_error and not result.success: - raise DataWeaveScriptError(result) - return result - - def __enter__(self): - self.initialize() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.cleanup() - return False +from .native import candidate_library_paths as _candidate_library_paths +from .native import find_library as _find_library +from .runtime import DataWeave _global_instance: Optional[DataWeave] = None @@ -745,48 +42,23 @@ def run(script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bo return _get_global_instance().run(script, inputs, raise_on_error=raise_on_error) -def run_streaming( - script: str, inputs: Optional[Dict[str, Any]] = None, -) -> Stream: - """Execute a script and yield output chunks. See :meth:`DataWeave.run_streaming`.""" +def run_streaming(script: str, inputs: Optional[Dict[str, Any]] = None) -> Stream: return _get_global_instance().run_streaming(script, inputs) def run_callback(script: str, write_callback: WriteCallback, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: - """Execute a script and stream output via a write callback. See :meth:`DataWeave.run_callback`.""" return _get_global_instance().run_callback(script, write_callback, inputs) -def run_transform( - script: str, - input_stream: Iterable[bytes], - input_name: str = "payload", - input_mime_type: str = "application/json", - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, -) -> Stream: - """Execute a script with streaming input and output. See :meth:`DataWeave.run_transform`.""" - return _get_global_instance().run_transform( - script, input_stream, input_name, input_mime_type, input_charset, inputs, - ) +def run_transform(script: str, input_stream: Iterable[bytes], input_name: str = "payload", input_mime_type: str = "application/json", input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> Stream: + return _get_global_instance().run_transform(script, input_stream, input_name, input_mime_type, input_charset, inputs) -def run_input_output_callback( - script: str, - input_name: str, - input_mime_type: str, - read_callback: ReadCallback, - write_callback: WriteCallback, - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, -) -> StreamingResult: - """Execute a script with callback-driven input and output. See :meth:`DataWeave.run_input_output_callback`.""" - return _get_global_instance().run_input_output_callback( - script, input_name, input_mime_type, read_callback, write_callback, input_charset, inputs, - ) +def run_input_output_callback(script: str, input_name: str, input_mime_type: str, read_callback: ReadCallback, write_callback: WriteCallback, input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: + return _get_global_instance().run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset, inputs) -def cleanup(): +def cleanup() -> None: global _global_instance if _global_instance is not None: _global_instance.cleanup() @@ -794,22 +66,8 @@ def cleanup(): __all__ = [ - "DataWeave", - "DataWeaveError", - "DataWeaveLibraryNotFoundError", - "DataWeaveScriptError", - "ExecutionResult", - "InputValue", - "ReadCallback", - "Stream", - "StreamingResult", - "WriteCallback", - "READ_CALLBACK", - "WRITE_CALLBACK", - "run", - "run_callback", - "run_input_output_callback", - "run_streaming", - "run_transform", - "cleanup", + "DataWeave", "DataWeaveError", "DataWeaveLibraryNotFoundError", "DataWeaveScriptError", + "ExecutionResult", "InputValue", "ReadCallback", "Stream", "StreamingResult", "WriteCallback", + "READ_CALLBACK", "WRITE_CALLBACK", "run", "run_callback", "run_input_output_callback", + "run_streaming", "run_transform", "cleanup", ] diff --git a/native-lib/python/src/dataweave/native.py b/native-lib/python/src/dataweave/native.py new file mode 100644 index 00000000..470de200 --- /dev/null +++ b/native-lib/python/src/dataweave/native.py @@ -0,0 +1,161 @@ +import ctypes +import os +from pathlib import Path +from typing import Optional + +from .models import DataWeaveError, DataWeaveLibraryNotFoundError, READ_CALLBACK, WRITE_CALLBACK + + +_ENV_NATIVE_LIB = "DATAWEAVE_NATIVE_LIB" + + +class graal_isolate_t(ctypes.Structure): + pass + + +class graal_isolatethread_t(ctypes.Structure): + pass + + +GraalIsolatePointer = ctypes.POINTER(graal_isolate_t) +GraalIsolateThreadPointer = ctypes.POINTER(graal_isolatethread_t) + + +def candidate_library_paths() -> list[Path]: + paths: list[Path] = [] + env_value = (os.environ.get(_ENV_NATIVE_LIB) or "").strip() + if env_value: + paths.append(Path(env_value)) + + pkg_dir = Path(__file__).resolve().parent + native_dir = pkg_dir / "native" + paths.extend(native_dir / name for name in ("dwlib.dylib", "dwlib.so", "dwlib.dll")) + + for parent in pkg_dir.parents: + build_dir = parent / "build" / "native" / "nativeCompile" + if build_dir.exists(): + paths.extend(build_dir / name for name in ("dwlib.dylib", "dwlib.so", "dwlib.dll")) + break + + paths.extend(Path(name) for name in ("dwlib.dylib", "dwlib.so", "dwlib.dll")) + return paths + + +def find_library() -> str: + for path in candidate_library_paths(): + if path.exists() and path.is_file(): + return str(path) + raise DataWeaveLibraryNotFoundError( + "Could not find DataWeave native library (dwlib). " + f"Set {_ENV_NATIVE_LIB} to an absolute path or install a wheel that bundles the native library." + ) + + +class NativeRuntime: + """Owns the native library handle, isolate lifecycle, and ctypes ABI.""" + + def __init__(self, lib_path: Optional[str] = None): + self.lib_path = lib_path or find_library() + self.lib = None + self.isolate = None + self.thread = None + self.initialized = False + self.has_callback_streaming = False + self.has_callback_input_output = False + + def initialize(self) -> None: + if self.initialized: + return + try: + self.lib = ctypes.CDLL(self.lib_path) + except OSError as error: + raise DataWeaveError(f"Failed to load library from {self.lib_path}: {error}") + + self._create_isolate() + self._setup_functions() + self.initialized = True + + def _create_isolate(self) -> None: + self.lib.graal_create_isolate.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(GraalIsolatePointer), + ctypes.POINTER(GraalIsolateThreadPointer), + ] + self.lib.graal_create_isolate.restype = ctypes.c_int + self.isolate = GraalIsolatePointer() + self.thread = GraalIsolateThreadPointer() + result = self.lib.graal_create_isolate(None, ctypes.byref(self.isolate), ctypes.byref(self.thread)) + if result != 0: + raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") + + def _setup_functions(self) -> None: + if not hasattr(self.lib, "run_script"): + raise DataWeaveError("Native library does not export run_script") + self.lib.run_script.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p] + self.lib.run_script.restype = ctypes.c_void_p + if hasattr(self.lib, "free_cstring"): + self.lib.free_cstring.argtypes = [GraalIsolateThreadPointer, ctypes.c_void_p] + self.lib.free_cstring.restype = None + if hasattr(self.lib, "graal_attach_thread"): + self.lib.graal_attach_thread.argtypes = [GraalIsolatePointer, ctypes.POINTER(GraalIsolateThreadPointer)] + self.lib.graal_attach_thread.restype = ctypes.c_int + if hasattr(self.lib, "graal_detach_thread"): + self.lib.graal_detach_thread.argtypes = [GraalIsolateThreadPointer] + self.lib.graal_detach_thread.restype = ctypes.c_int + if hasattr(self.lib, "graal_tear_down_isolate"): + self.lib.graal_tear_down_isolate.argtypes = [GraalIsolateThreadPointer] + self.lib.graal_tear_down_isolate.restype = ctypes.c_int + if hasattr(self.lib, "run_script_callback"): + self.lib.run_script_callback.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p, WRITE_CALLBACK, ctypes.c_void_p] + self.lib.run_script_callback.restype = ctypes.c_void_p + self.has_callback_streaming = True + if hasattr(self.lib, "run_script_input_output_callback"): + self.lib.run_script_input_output_callback.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, READ_CALLBACK, WRITE_CALLBACK, ctypes.c_void_p] + self.lib.run_script_input_output_callback.restype = ctypes.c_void_p + self.has_callback_input_output = True + + def attach_thread(self): + worker_thread = GraalIsolateThreadPointer() + result = self.lib.graal_attach_thread(self.isolate, ctypes.byref(worker_thread)) + if result != 0: + raise DataWeaveError(f"Failed to attach worker thread to isolate (code {result})") + return worker_thread + + def detach_thread(self, thread) -> None: + self.lib.graal_detach_thread(thread) + + def decode_and_free(self, ptr, thread=None) -> str: + if not ptr: + return "" + try: + return ctypes.string_at(ptr).decode("utf-8") + finally: + if self.lib is not None and hasattr(self.lib, "free_cstring"): + self.lib.free_cstring(thread or self.thread, ptr) + + def run_script(self, thread, script: bytes, inputs: bytes): + return self.lib.run_script(thread, script, inputs) + + def run_script_callback(self, thread, script: bytes, inputs: bytes, write_callback): + return self.lib.run_script_callback(thread, script, inputs, write_callback, None) + + def run_script_input_output_callback(self, thread, script: bytes, inputs: bytes, input_name: bytes, input_mime_type: bytes, input_charset: Optional[bytes], read_callback, write_callback): + return self.lib.run_script_input_output_callback( + thread, script, inputs, input_name, input_mime_type, input_charset, read_callback, write_callback, None, + ) + + def cleanup(self) -> None: + if not self.initialized: + return + try: + if hasattr(self.lib, "graal_tear_down_isolate") and self.thread is not None: + self.lib.graal_tear_down_isolate(self.thread) + elif hasattr(self.lib, "graal_detach_thread") and self.thread is not None: + self.lib.graal_detach_thread(self.thread) + except Exception: + pass + finally: + self.initialized = False + self.thread = None + self.isolate = None + self.lib = None diff --git a/native-lib/python/src/dataweave/runtime.py b/native-lib/python/src/dataweave/runtime.py new file mode 100644 index 00000000..9db2e75f --- /dev/null +++ b/native-lib/python/src/dataweave/runtime.py @@ -0,0 +1,257 @@ +import ctypes +import json +from queue import Empty, Full, Queue +from threading import Event, Thread +from typing import Any, Dict, Generator, Iterable, Optional + +from .encoding import normalize_input_value, parse_native_encoded_response, parse_streaming_result +from .models import ( + READ_CALLBACK, + WRITE_CALLBACK, + DataWeaveError, + DataWeaveScriptError, + ExecutionResult, + ReadCallback, + Stream, + StreamingResult, + WriteCallback, +) +from .native import NativeRuntime + + +_OUTPUT_QUEUE_MAXSIZE = 512 +_WORKER_TIMEOUT_SECONDS = 30 + + +class DataWeave: + """High-level execution API backed by a :class:`NativeRuntime`.""" + + def __init__(self, lib_path: Optional[str] = None): + self._native = NativeRuntime(lib_path) + + def __getattr__(self, name): + # Preserve the private attributes used by existing embedders and tests. + aliases = { + "_lib": "lib", "_isolate": "isolate", "_thread": "thread", + "_initialized": "initialized", "_has_callback_streaming": "has_callback_streaming", + "_has_callback_input_output": "has_callback_input_output", + } + if name in aliases: + return getattr(self._native, aliases[name]) + raise AttributeError(name) + + def __setattr__(self, name, value): + aliases = { + "_lib": "lib", "_isolate": "isolate", "_thread": "thread", + "_initialized": "initialized", "_has_callback_streaming": "has_callback_streaming", + "_has_callback_input_output": "has_callback_input_output", + } + if name != "_native" and name in aliases and "_native" in self.__dict__: + setattr(self._native, aliases[name], value) + else: + super().__setattr__(name, value) + + def _setup_graal_structures(self): + # Compatibility shim for the private characterization tests from Tasks 1-3. + from .native import GraalIsolatePointer, GraalIsolateThreadPointer + if "_native" not in self.__dict__: + native = NativeRuntime.__new__(NativeRuntime) + native.lib = self.__dict__.pop("_lib", None) + native.isolate = self.__dict__.pop("_isolate", None) + native.thread = self.__dict__.pop("_thread", None) + native.initialized = self.__dict__.pop("_initialized", False) + native.has_callback_streaming = self.__dict__.pop("_has_callback_streaming", False) + native.has_callback_input_output = self.__dict__.pop("_has_callback_input_output", False) + self._native = native + self._graal_isolate_t_ptr = GraalIsolatePointer + self._graal_isolatethread_t_ptr = GraalIsolateThreadPointer + + def _decode_and_free(self, ptr): + if "_native" not in self.__dict__: + self._setup_graal_structures() + return self._native.decode_and_free(ptr) + + def initialize(self): + self._native.initialize() + + def cleanup(self): + self._native.cleanup() + + def _require_initialized(self, supported: bool, api_name: str) -> None: + if not self._native.initialized: + raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") + if not supported: + raise DataWeaveError(f"Native library does not support {api_name}.") + + @staticmethod + def _inputs_json(inputs: Optional[Dict[str, Any]]) -> bytes: + return json.dumps({key: normalize_input_value(value) for key, value in (inputs or {}).items()}).encode("utf-8") + + def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bool = False) -> ExecutionResult: + self._require_initialized(True, "script execution") + try: + raw = self._native.decode_and_free(self._native.run_script(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs))) + result = parse_native_encoded_response(raw) + except Exception as error: + raise DataWeaveError(f"Failed to execute script: {error}") + if raise_on_error and not result.success: + raise DataWeaveScriptError(result) + return result + + def run_callback(self, script: str, write_callback: WriteCallback, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: + self._require_initialized(self._native.has_callback_streaming, "callback streaming API (run_script_callback not found)") + @WRITE_CALLBACK + def write_cb(_context, buffer, length): + try: + return write_callback(ctypes.string_at(buffer, length)) + except Exception: + return -1 + try: + ptr = self._native.run_script_callback(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs), write_cb) + raw = self._native.decode_and_free(ptr) + except Exception as error: + raise DataWeaveError(f"Failed to execute callback streaming: {error}") + return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) + + def _stream_worker(self, invoke, cancelled: Event) -> Generator[bytes, None, StreamingResult]: + sentinel = object() + queue: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) + + def publish(item) -> None: + while not cancelled.is_set(): + try: + queue.put(item, timeout=0.1) + return + except Full: + pass + + @WRITE_CALLBACK + def write_cb(_context, buffer, length): + try: + if cancelled.is_set(): + return -1 + queue.put(ctypes.string_at(buffer, length), timeout=_WORKER_TIMEOUT_SECONDS) + return 0 + except Exception: + return -1 + + def worker_main(): + worker_thread = None + try: + worker_thread = self._native.attach_thread() + raw = self._native.decode_and_free(invoke(worker_thread, write_cb), worker_thread) + publish(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) + except Exception as error: + publish({"success": False, "error": str(error)}) + finally: + if worker_thread is not None: + try: + self._native.detach_thread(worker_thread) + except Exception: + pass + publish(sentinel) + + worker = Thread(target=worker_main, name="dw-streaming-worker", daemon=False) + worker.start() + metadata = None + try: + while True: + try: + item = queue.get(timeout=_WORKER_TIMEOUT_SECONDS) + except Empty: + cancelled.set() + worker.join(timeout=_WORKER_TIMEOUT_SECONDS) + raise DataWeaveError(f"Worker thread timeout after {_WORKER_TIMEOUT_SECONDS} seconds") + if item is sentinel: + break + if isinstance(item, dict): + metadata = item + else: + yield item + finally: + cancelled.set() + worker.join(timeout=_WORKER_TIMEOUT_SECONDS) + if worker.is_alive(): + raise DataWeaveError(f"Worker thread timeout after {_WORKER_TIMEOUT_SECONDS} seconds") + return parse_streaming_result(metadata or {"success": False, "error": "No metadata received from native call"}) + + def run_streaming(self, script: str, inputs: Optional[Dict[str, Any]] = None) -> Stream: + self._require_initialized(self._native.has_callback_streaming, "callback streaming API (run_script_callback not found)") + cancelled = Event() + encoded_inputs = self._inputs_json(inputs) + stream = Stream(self._stream_worker(lambda thread, write_cb: self._native.run_script_callback(thread, script.encode("utf-8"), encoded_inputs, write_cb), cancelled)) + stream._on_close = cancelled.set + stream._cancelled = cancelled + return stream + + @staticmethod + def _chunk_reader(input_stream: Iterable[bytes]): + iterator = iter(input_stream) + state = {"chunk": b"", "offset": 0, "done": False} + @READ_CALLBACK + def read_cb(_context, buffer, buffer_size): + try: + while True: + chunk = state["chunk"] + if state["offset"] < len(chunk): + size = min(len(chunk) - state["offset"], buffer_size) + ctypes.memmove(buffer, chunk[state["offset"]:state["offset"] + size], size) + state["offset"] += size + return size + if state["done"]: + return 0 + chunk = next(iterator, None) + if not chunk: + state["done"] = True + return 0 + state["chunk"] = chunk + state["offset"] = 0 + except Exception: + return -1 + return read_cb + + def run_transform(self, script: str, input_stream: Iterable[bytes], input_name: str = "payload", input_mime_type: str = "application/json", input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> Stream: + self._require_initialized(self._native.has_callback_input_output, "callback input/output API (run_script_input_output_callback not found)") + cancelled = Event() + read_cb = self._chunk_reader(input_stream) + encoded_inputs = self._inputs_json(inputs) + def invoke(thread, write_cb): + return self._native.run_script_input_output_callback(thread, script.encode("utf-8"), encoded_inputs, input_name.encode("utf-8"), input_mime_type.encode("utf-8"), input_charset.encode("utf-8") if input_charset else None, read_cb, write_cb) + stream = Stream(self._stream_worker(invoke, cancelled)) + stream._on_close = cancelled.set + stream._cancelled = cancelled + return stream + + def run_input_output_callback(self, script: str, input_name: str, input_mime_type: str, read_callback: ReadCallback, write_callback: WriteCallback, input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: + self._require_initialized(self._native.has_callback_input_output, "callback input/output API (run_script_input_output_callback not found)") + @READ_CALLBACK + def read_cb(_context, buffer, buffer_size): + try: + data = read_callback(buffer_size) + if not data: + return 0 + size = min(len(data), buffer_size) + ctypes.memmove(buffer, data, size) + return size + except Exception: + return -1 + @WRITE_CALLBACK + def write_cb(_context, buffer, length): + try: + return write_callback(ctypes.string_at(buffer, length)) + except Exception: + return -1 + try: + ptr = self._native.run_script_input_output_callback(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs), input_name.encode("utf-8"), input_mime_type.encode("utf-8"), input_charset.encode("utf-8") if input_charset else None, read_cb, write_cb) + raw = self._native.decode_and_free(ptr) + except Exception as error: + raise DataWeaveError(f"Failed to execute callback input/output streaming: {error}") + return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) + + def __enter__(self): + self.initialize() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + return False diff --git a/native-lib/python/tests/unit/test_native.py b/native-lib/python/tests/unit/test_native.py index a31bc9c1..dc26949e 100644 --- a/native-lib/python/tests/unit/test_native.py +++ b/native-lib/python/tests/unit/test_native.py @@ -3,6 +3,7 @@ import pytest import dataweave +from dataweave import native @pytest.mark.unit @@ -52,3 +53,52 @@ def test_decode_and_free_releases_native_string_when_decoding_fails(monkeypatch) runtime._decode_and_free(123) assert freed == [("thread", 123)] + + +@pytest.mark.unit +def test_native_runtime_registers_abi_and_cleans_up_idempotently(monkeypatch): + class Function: + pass + + class FakeLibrary: + run_script = Function() + free_cstring = Function() + graal_attach_thread = Function() + graal_detach_thread = Function() + + def __init__(self): + self.tear_down_threads = [] + self.graal_create_isolate = Function() + self.graal_create_isolate.__call__ = lambda _params, _isolate, _thread: 0 + self.graal_tear_down_isolate = Function() + self.graal_tear_down_isolate.__call__ = lambda thread: self.tear_down_threads.append(thread) or 0 + + class CallableFunction(Function): + def __init__(self, callback): + self.callback = callback + + def __call__(self, *args): + return self.callback(*args) + + library = FakeLibrary() + library.graal_create_isolate = CallableFunction(lambda _params, _isolate, _thread: 0) + library.graal_tear_down_isolate = CallableFunction(lambda thread: library.tear_down_threads.append(thread) or 0) + + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + runtime = native.NativeRuntime("/tmp/dwlib") + runtime.initialize() + runtime.cleanup() + runtime.cleanup() + + assert library.run_script.argtypes[1:] == [native.ctypes.c_char_p, native.ctypes.c_char_p] + assert library.free_cstring.argtypes[1] is native.ctypes.c_void_p + assert len(library.tear_down_threads) == 1 + assert runtime.initialized is False + + +@pytest.mark.unit +def test_native_runtime_wraps_library_load_errors(monkeypatch): + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: (_ for _ in ()).throw(OSError("bad image"))) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to load library from /tmp/dwlib: bad image"): + native.NativeRuntime("/tmp/dwlib").initialize() diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index 3561e81e..5be598cf 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -1,10 +1,12 @@ import ctypes from queue import Full, Queue -from threading import Event +from threading import Event, Thread +from time import sleep import pytest import dataweave +from dataweave import runtime as runtime_module class FakeNative: @@ -118,7 +120,7 @@ def put(self, item, *args, **kwargs): return None return super().put(item, *args, **kwargs) - monkeypatch.setattr(dataweave, "Queue", MetadataDroppingQueue) + monkeypatch.setattr(runtime_module, "Queue", MetadataDroppingQueue) runtime = configured_runtime(FakeNative('{"success": true}')) stream = runtime.run_streaming("script") @@ -197,8 +199,8 @@ def run_script_callback(self, _thread, _script, _inputs, write_callback, _contex self.write_status = write_callback(None, ctypes.addressof(third), 5) return self._response_pointer() - monkeypatch.setattr(dataweave, "Queue", CancellationAwareQueue) - monkeypatch.setattr(dataweave, "_OUTPUT_QUEUE_MAXSIZE", 1) + monkeypatch.setattr(runtime_module, "Queue", CancellationAwareQueue) + monkeypatch.setattr(runtime_module, "_OUTPUT_QUEUE_MAXSIZE", 1) native = FullQueueFakeNative() runtime = configured_runtime(native) stream = runtime.run_streaming("script") @@ -212,3 +214,22 @@ def run_script_callback(self, _thread, _script, _inputs, write_callback, _contex assert native.write_status == -1 assert native.detached_event.wait(1) assert terminal_blocked.is_set() is False + + +@pytest.mark.unit +def test_runtime_module_owns_dataweave_orchestration(): + assert dataweave.DataWeave is runtime_module.DataWeave + + +@pytest.mark.unit +def test_run_streaming_reports_worker_timeout_when_native_call_produces_no_output(monkeypatch): + class BlockingFakeNative(FakeNative): + def run_script_callback(self, _thread, _script, _inputs, _write_callback, _context): + sleep(0.05) + return self._response_pointer() + + monkeypatch.setattr(runtime_module, "_WORKER_TIMEOUT_SECONDS", 0.01) + runtime = configured_runtime(BlockingFakeNative('{"success": true}')) + + with pytest.raises(dataweave.DataWeaveError, match="Worker thread timeout after 0.01 seconds"): + list(runtime.run_streaming("script")) From d614f10db7c184c6f90c2c1a1d1ac57a53a20c7f Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 16:10:12 -0300 Subject: [PATCH 15/30] fix: harden Python native runtime lifecycle --- .../task-4-report.md | 13 ++ native-lib/python/src/dataweave/native.py | 98 +++++++++----- native-lib/python/src/dataweave/runtime.py | 55 ++------ native-lib/python/tests/unit/test_native.py | 126 +++++++++++++++++- .../python/tests/unit/test_streaming.py | 28 +++- 5 files changed, 233 insertions(+), 87 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md index 80aa22f1..252cb861 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md @@ -22,3 +22,16 @@ Complete. The Python binding now separates native ctypes/isolate ownership from - The Gradle native-image invocation emits existing Graal/Gradle deprecation and restricted-native-access warnings; it still completed successfully. - No Task 5, documentation, or CI work was included. + +## Fix Round 1 + +- Made initialization transactional: failed isolate creation or ABI validation now resets all native state; after isolate creation, validation failure attempts isolate teardown before reset and reraises the primary failure. +- Required `run_script`, `free_cstring`, and isolate teardown exports at initialization. Streaming callback exports additionally require attach/detach exports before the runtime is considered initialized. +- Validated isolate teardown and worker detach return codes. Cleanup failures now surface as `DataWeaveError`; worker detach failures surface only when there was no preceding execution failure. +- Removed obsolete `DataWeave` private-state forwarding and test-only Graal setup compatibility machinery. Tests now configure `NativeRuntime` directly. +- Added focused coverage for failed isolate creation, partial-initialization cleanup, missing required exports, teardown return codes, and detach return codes. + +### Fix Round 1 Verification + +- Final focused and full Python verification passed: 25 focused lifecycle/streaming tests and 61 unit/integration tests. +- `./gradlew native-lib:pythonTest` passed: native image build and all 61 Python tests. diff --git a/native-lib/python/src/dataweave/native.py b/native-lib/python/src/dataweave/native.py index 470de200..f20b1471 100644 --- a/native-lib/python/src/dataweave/native.py +++ b/native-lib/python/src/dataweave/native.py @@ -70,10 +70,17 @@ def initialize(self) -> None: self.lib = ctypes.CDLL(self.lib_path) except OSError as error: raise DataWeaveError(f"Failed to load library from {self.lib_path}: {error}") - - self._create_isolate() - self._setup_functions() - self.initialized = True + isolate_created = False + try: + self._create_isolate() + isolate_created = True + self._setup_functions() + self.initialized = True + except Exception: + if isolate_created: + self._tear_down_isolate(suppress_errors=True) + self._reset() + raise def _create_isolate(self) -> None: self.lib.graal_create_isolate.argtypes = [ @@ -89,31 +96,39 @@ def _create_isolate(self) -> None: raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") def _setup_functions(self) -> None: - if not hasattr(self.lib, "run_script"): - raise DataWeaveError("Native library does not export run_script") + self._require_export("run_script") + self._require_export("free_cstring") + self._require_export("graal_tear_down_isolate") self.lib.run_script.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p] self.lib.run_script.restype = ctypes.c_void_p - if hasattr(self.lib, "free_cstring"): - self.lib.free_cstring.argtypes = [GraalIsolateThreadPointer, ctypes.c_void_p] - self.lib.free_cstring.restype = None - if hasattr(self.lib, "graal_attach_thread"): - self.lib.graal_attach_thread.argtypes = [GraalIsolatePointer, ctypes.POINTER(GraalIsolateThreadPointer)] - self.lib.graal_attach_thread.restype = ctypes.c_int - if hasattr(self.lib, "graal_detach_thread"): - self.lib.graal_detach_thread.argtypes = [GraalIsolateThreadPointer] - self.lib.graal_detach_thread.restype = ctypes.c_int - if hasattr(self.lib, "graal_tear_down_isolate"): - self.lib.graal_tear_down_isolate.argtypes = [GraalIsolateThreadPointer] - self.lib.graal_tear_down_isolate.restype = ctypes.c_int + self.lib.free_cstring.argtypes = [GraalIsolateThreadPointer, ctypes.c_void_p] + self.lib.free_cstring.restype = None + self.lib.graal_tear_down_isolate.argtypes = [GraalIsolateThreadPointer] + self.lib.graal_tear_down_isolate.restype = ctypes.c_int if hasattr(self.lib, "run_script_callback"): + self._require_streaming_lifecycle_exports("run_script_callback") self.lib.run_script_callback.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p, WRITE_CALLBACK, ctypes.c_void_p] self.lib.run_script_callback.restype = ctypes.c_void_p self.has_callback_streaming = True if hasattr(self.lib, "run_script_input_output_callback"): + self._require_streaming_lifecycle_exports("run_script_input_output_callback") self.lib.run_script_input_output_callback.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, READ_CALLBACK, WRITE_CALLBACK, ctypes.c_void_p] self.lib.run_script_input_output_callback.restype = ctypes.c_void_p self.has_callback_input_output = True + def _require_export(self, name: str) -> None: + if not hasattr(self.lib, name): + raise DataWeaveError(f"Native library does not export {name}") + + def _require_streaming_lifecycle_exports(self, callback_name: str) -> None: + for name in ("free_cstring", "graal_attach_thread", "graal_detach_thread"): + if not hasattr(self.lib, name): + raise DataWeaveError(f"{callback_name} requires native export {name}") + self.lib.graal_attach_thread.argtypes = [GraalIsolatePointer, ctypes.POINTER(GraalIsolateThreadPointer)] + self.lib.graal_attach_thread.restype = ctypes.c_int + self.lib.graal_detach_thread.argtypes = [GraalIsolateThreadPointer] + self.lib.graal_detach_thread.restype = ctypes.c_int + def attach_thread(self): worker_thread = GraalIsolateThreadPointer() result = self.lib.graal_attach_thread(self.isolate, ctypes.byref(worker_thread)) @@ -122,16 +137,26 @@ def attach_thread(self): return worker_thread def detach_thread(self, thread) -> None: - self.lib.graal_detach_thread(thread) + result = self.lib.graal_detach_thread(thread) + if result != 0: + raise DataWeaveError(f"Failed to detach worker thread from isolate. Error code: {result}") def decode_and_free(self, ptr, thread=None) -> str: if not ptr: return "" + primary_error = None try: return ctypes.string_at(ptr).decode("utf-8") + except Exception as error: + primary_error = error + raise finally: - if self.lib is not None and hasattr(self.lib, "free_cstring"): - self.lib.free_cstring(thread or self.thread, ptr) + try: + if self.lib is not None: + self.lib.free_cstring(thread or self.thread, ptr) + except Exception: + if primary_error is None: + raise def run_script(self, thread, script: bytes, inputs: bytes): return self.lib.run_script(thread, script, inputs) @@ -148,14 +173,25 @@ def cleanup(self) -> None: if not self.initialized: return try: - if hasattr(self.lib, "graal_tear_down_isolate") and self.thread is not None: - self.lib.graal_tear_down_isolate(self.thread) - elif hasattr(self.lib, "graal_detach_thread") and self.thread is not None: - self.lib.graal_detach_thread(self.thread) - except Exception: - pass + self._tear_down_isolate() finally: - self.initialized = False - self.thread = None - self.isolate = None - self.lib = None + self._reset() + + def _tear_down_isolate(self, suppress_errors: bool = False) -> None: + if self.thread is None: + return + try: + result = self.lib.graal_tear_down_isolate(self.thread) + if result != 0: + raise DataWeaveError(f"Failed to tear down GraalVM isolate. Error code: {result}") + except Exception: + if not suppress_errors: + raise + + def _reset(self) -> None: + self.initialized = False + self.thread = None + self.isolate = None + self.lib = None + self.has_callback_streaming = False + self.has_callback_input_output = False diff --git a/native-lib/python/src/dataweave/runtime.py b/native-lib/python/src/dataweave/runtime.py index 9db2e75f..8f88a79a 100644 --- a/native-lib/python/src/dataweave/runtime.py +++ b/native-lib/python/src/dataweave/runtime.py @@ -29,48 +29,6 @@ class DataWeave: def __init__(self, lib_path: Optional[str] = None): self._native = NativeRuntime(lib_path) - def __getattr__(self, name): - # Preserve the private attributes used by existing embedders and tests. - aliases = { - "_lib": "lib", "_isolate": "isolate", "_thread": "thread", - "_initialized": "initialized", "_has_callback_streaming": "has_callback_streaming", - "_has_callback_input_output": "has_callback_input_output", - } - if name in aliases: - return getattr(self._native, aliases[name]) - raise AttributeError(name) - - def __setattr__(self, name, value): - aliases = { - "_lib": "lib", "_isolate": "isolate", "_thread": "thread", - "_initialized": "initialized", "_has_callback_streaming": "has_callback_streaming", - "_has_callback_input_output": "has_callback_input_output", - } - if name != "_native" and name in aliases and "_native" in self.__dict__: - setattr(self._native, aliases[name], value) - else: - super().__setattr__(name, value) - - def _setup_graal_structures(self): - # Compatibility shim for the private characterization tests from Tasks 1-3. - from .native import GraalIsolatePointer, GraalIsolateThreadPointer - if "_native" not in self.__dict__: - native = NativeRuntime.__new__(NativeRuntime) - native.lib = self.__dict__.pop("_lib", None) - native.isolate = self.__dict__.pop("_isolate", None) - native.thread = self.__dict__.pop("_thread", None) - native.initialized = self.__dict__.pop("_initialized", False) - native.has_callback_streaming = self.__dict__.pop("_has_callback_streaming", False) - native.has_callback_input_output = self.__dict__.pop("_has_callback_input_output", False) - self._native = native - self._graal_isolate_t_ptr = GraalIsolatePointer - self._graal_isolatethread_t_ptr = GraalIsolateThreadPointer - - def _decode_and_free(self, ptr): - if "_native" not in self.__dict__: - self._setup_graal_structures() - return self._native.decode_and_free(ptr) - def initialize(self): self._native.initialize() @@ -117,6 +75,10 @@ def _stream_worker(self, invoke, cancelled: Event) -> Generator[bytes, None, Str sentinel = object() queue: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) + class CleanupFailure: + def __init__(self, error: Exception): + self.error = error + def publish(item) -> None: while not cancelled.is_set(): try: @@ -137,18 +99,21 @@ def write_cb(_context, buffer, length): def worker_main(): worker_thread = None + primary_error = None try: worker_thread = self._native.attach_thread() raw = self._native.decode_and_free(invoke(worker_thread, write_cb), worker_thread) publish(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) except Exception as error: + primary_error = error publish({"success": False, "error": str(error)}) finally: if worker_thread is not None: try: self._native.detach_thread(worker_thread) - except Exception: - pass + except Exception as error: + if primary_error is None: + publish(CleanupFailure(error)) publish(sentinel) worker = Thread(target=worker_main, name="dw-streaming-worker", daemon=False) @@ -164,6 +129,8 @@ def worker_main(): raise DataWeaveError(f"Worker thread timeout after {_WORKER_TIMEOUT_SECONDS} seconds") if item is sentinel: break + if isinstance(item, CleanupFailure): + raise item.error if isinstance(item, dict): metadata = item else: diff --git a/native-lib/python/tests/unit/test_native.py b/native-lib/python/tests/unit/test_native.py index dc26949e..71153aec 100644 --- a/native-lib/python/tests/unit/test_native.py +++ b/native-lib/python/tests/unit/test_native.py @@ -44,17 +44,28 @@ def test_candidate_paths_prioritize_environment_override(monkeypatch, tmp_path): @pytest.mark.unit def test_decode_and_free_releases_native_string_when_decoding_fails(monkeypatch): freed = [] - runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) - runtime._thread = "thread" - runtime._lib = type("Native", (), {"free_cstring": lambda _self, thread, ptr: freed.append((thread, ptr))})() - monkeypatch.setattr(dataweave.ctypes, "string_at", lambda _ptr: b"\xff") + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.thread = "thread" + runtime.lib = type("Native", (), {"free_cstring": lambda _self, thread, ptr: freed.append((thread, ptr))})() + monkeypatch.setattr(native.ctypes, "string_at", lambda _ptr: b"\xff") with pytest.raises(UnicodeDecodeError): - runtime._decode_and_free(123) + runtime.decode_and_free(123) assert freed == [("thread", 123)] +@pytest.mark.unit +def test_decode_and_free_preserves_decode_failure_when_free_also_fails(monkeypatch): + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.thread = "thread" + runtime.lib = type("Native", (), {"free_cstring": lambda _self, _thread, _ptr: (_ for _ in ()).throw(RuntimeError("free failed"))})() + monkeypatch.setattr(native.ctypes, "string_at", lambda _ptr: b"\xff") + + with pytest.raises(UnicodeDecodeError): + runtime.decode_and_free(123) + + @pytest.mark.unit def test_native_runtime_registers_abi_and_cleans_up_idempotently(monkeypatch): class Function: @@ -102,3 +113,108 @@ def test_native_runtime_wraps_library_load_errors(monkeypatch): with pytest.raises(dataweave.DataWeaveError, match="Failed to load library from /tmp/dwlib: bad image"): native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_initialize_resets_state_when_isolate_creation_fails(monkeypatch): + class Function: + def __call__(self, *_args): + return 9 + + library = type("Native", (), {"graal_create_isolate": Function()})() + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + runtime = native.NativeRuntime("/tmp/dwlib") + + with pytest.raises(dataweave.DataWeaveError, match="Failed to create GraalVM isolate. Error code: 9"): + runtime.initialize() + + assert runtime.lib is None + assert runtime.isolate is None + assert runtime.thread is None + assert runtime.initialized is False + + +@pytest.mark.unit +def test_initialize_tears_down_isolate_when_required_export_is_missing(monkeypatch): + class Function: + def __init__(self, callback): + self.callback = callback + + def __call__(self, *args): + return self.callback(*args) + + torn_down = [] + library = type("Native", (), {})() + library.graal_create_isolate = Function(lambda _params, _isolate, _thread: 0) + library.graal_tear_down_isolate = Function(lambda thread: torn_down.append(thread) or 0) + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + runtime = native.NativeRuntime("/tmp/dwlib") + + with pytest.raises(dataweave.DataWeaveError, match="Native library does not export run_script"): + runtime.initialize() + + assert len(torn_down) == 1 + assert runtime.lib is None + assert runtime.isolate is None + assert runtime.thread is None + + +@pytest.mark.unit +def test_initialize_rejects_streaming_export_without_required_lifecycle_symbols(monkeypatch): + class Function: + def __init__(self, callback=lambda *_args: 0): + self.callback = callback + + def __call__(self, *args): + return self.callback(*args) + + torn_down = [] + library = type("Native", (), {})() + library.graal_create_isolate = Function() + library.graal_tear_down_isolate = Function(lambda thread: torn_down.append(thread) or 0) + library.run_script = Function() + library.run_script_callback = Function() + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + + with pytest.raises(dataweave.DataWeaveError, match="Native library does not export free_cstring"): + native.NativeRuntime("/tmp/dwlib").initialize() + + assert len(torn_down) == 1 + + +@pytest.mark.unit +@pytest.mark.parametrize("missing_symbol", ["graal_attach_thread", "graal_detach_thread"]) +def test_initialize_rejects_streaming_export_without_thread_lifecycle_symbols(monkeypatch, missing_symbol): + class Function: + def __call__(self, *_args): + return 0 + + library = type("Native", (), {})() + library.graal_create_isolate = Function() + library.graal_tear_down_isolate = Function() + library.run_script = Function() + library.free_cstring = Function() + library.run_script_callback = Function() + for symbol in ("graal_attach_thread", "graal_detach_thread"): + if symbol != missing_symbol: + setattr(library, symbol, Function()) + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + + with pytest.raises(dataweave.DataWeaveError, match=f"run_script_callback requires native export {missing_symbol}"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_cleanup_surfaces_native_teardown_error_code(monkeypatch): + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.initialized = True + runtime.thread = object() + runtime.isolate = object() + runtime.lib = type("Native", (), {"graal_tear_down_isolate": lambda _self, _thread: 7})() + + with pytest.raises(dataweave.DataWeaveError, match="Failed to tear down GraalVM isolate. Error code: 7"): + runtime.cleanup() + + assert runtime.lib is None + assert runtime.thread is None + assert runtime.isolate is None diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index 5be598cf..fc91af03 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -67,13 +67,14 @@ def run_script_input_output_callback( def configured_runtime(native): runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) - runtime._setup_graal_structures() - runtime._initialized = True - runtime._has_callback_streaming = True - runtime._has_callback_input_output = True - runtime._lib = native - runtime._isolate = object() - runtime._thread = object() + native_runtime = runtime_module.NativeRuntime.__new__(runtime_module.NativeRuntime) + native_runtime.initialized = True + native_runtime.has_callback_streaming = True + native_runtime.has_callback_input_output = True + native_runtime.lib = native + native_runtime.isolate = object() + native_runtime.thread = object() + runtime._native = native_runtime return runtime @@ -233,3 +234,16 @@ def run_script_callback(self, _thread, _script, _inputs, _write_callback, _conte with pytest.raises(dataweave.DataWeaveError, match="Worker thread timeout after 0.01 seconds"): list(runtime.run_streaming("script")) + + +@pytest.mark.unit +def test_run_streaming_surfaces_detach_failure_without_primary_execution_failure(): + class DetachFailingNative(FakeNative): + def graal_detach_thread(self, thread): + super().graal_detach_thread(thread) + return 7 + + runtime = configured_runtime(DetachFailingNative('{"success": true}')) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to detach worker thread from isolate. Error code: 7"): + list(runtime.run_streaming("script")) From 6c0a7202232d7b6436bc739bfbd1cc8586fc21e1 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 19:47:21 -0300 Subject: [PATCH 16/30] fix: preserve Python runtime primary failures --- .../task-4-report.md | 19 ++++++ native-lib/python/src/dataweave/__init__.py | 6 +- native-lib/python/src/dataweave/native.py | 21 +++++-- native-lib/python/src/dataweave/runtime.py | 13 +++- .../tests/integration/test_lifecycle.py | 20 +++++++ native-lib/python/tests/unit/test_facade.py | 25 ++++++++ native-lib/python/tests/unit/test_native.py | 60 +++++++++++++++++++ .../python/tests/unit/test_streaming.py | 14 +++++ 8 files changed, 169 insertions(+), 9 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md index 252cb861..a68398b4 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md @@ -35,3 +35,22 @@ Complete. The Python binding now separates native ctypes/isolate ownership from - Final focused and full Python verification passed: 25 focused lifecycle/streaming tests and 61 unit/integration tests. - `./gradlew native-lib:pythonTest` passed: native image build and all 61 Python tests. + +## Fix Round 2 + +- Treated unsuccessful native streaming metadata as the primary execution outcome, so worker detach failures do not replace a script failure. +- Preserved an active context-manager body exception when cleanup also fails; cleanup failures still surface when no body exception is active. +- Cleared the facade singleton in a `finally` block, allowing reinitialization after native cleanup raises. +- Validated `graal_create_isolate` is exported and wrapped its invocation failure as contextual `DataWeaveError`. +- Wrapped `graal_attach_thread` and `graal_detach_thread` invocation failures as contextual `DataWeaveError`; the stream worker suppresses detach failures whenever attach, execution, decode, or native failure metadata is primary. +- Added focused tests for missing/throwing isolate creation, throwing attach/detach lifecycle calls, failed singleton cleanup, context cleanup precedence, and unsuccessful streaming metadata plus detach failure. + +### Fix Round 2 Verification + +- Focused lifecycle/native/streaming/facade tests passed: 38 tests. +- Full Python unit and integration suite passed: 70 tests. +- `./gradlew native-lib:pythonTest` passed: native image build and all 70 Python tests. + +### Fix Round 2 Concerns + +- The Gradle native-image build continues to emit pre-existing Graal/Gradle deprecation and restricted-native-access warnings, but the build and Python suite completed successfully. diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index f1db4641..77070380 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -61,8 +61,10 @@ def run_input_output_callback(script: str, input_name: str, input_mime_type: str def cleanup() -> None: global _global_instance if _global_instance is not None: - _global_instance.cleanup() - _global_instance = None + try: + _global_instance.cleanup() + finally: + _global_instance = None __all__ = [ diff --git a/native-lib/python/src/dataweave/native.py b/native-lib/python/src/dataweave/native.py index f20b1471..98f771a1 100644 --- a/native-lib/python/src/dataweave/native.py +++ b/native-lib/python/src/dataweave/native.py @@ -83,6 +83,7 @@ def initialize(self) -> None: raise def _create_isolate(self) -> None: + self._require_export("graal_create_isolate") self.lib.graal_create_isolate.argtypes = [ ctypes.c_void_p, ctypes.POINTER(GraalIsolatePointer), @@ -91,7 +92,10 @@ def _create_isolate(self) -> None: self.lib.graal_create_isolate.restype = ctypes.c_int self.isolate = GraalIsolatePointer() self.thread = GraalIsolateThreadPointer() - result = self.lib.graal_create_isolate(None, ctypes.byref(self.isolate), ctypes.byref(self.thread)) + try: + result = self.lib.graal_create_isolate(None, ctypes.byref(self.isolate), ctypes.byref(self.thread)) + except Exception as error: + raise DataWeaveError(f"Failed to create GraalVM isolate: {error}") from error if result != 0: raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") @@ -131,13 +135,19 @@ def _require_streaming_lifecycle_exports(self, callback_name: str) -> None: def attach_thread(self): worker_thread = GraalIsolateThreadPointer() - result = self.lib.graal_attach_thread(self.isolate, ctypes.byref(worker_thread)) + try: + result = self.lib.graal_attach_thread(self.isolate, ctypes.byref(worker_thread)) + except Exception as error: + raise DataWeaveError(f"Failed to attach worker thread to isolate: {error}") from error if result != 0: raise DataWeaveError(f"Failed to attach worker thread to isolate (code {result})") return worker_thread def detach_thread(self, thread) -> None: - result = self.lib.graal_detach_thread(thread) + try: + result = self.lib.graal_detach_thread(thread) + except Exception as error: + raise DataWeaveError(f"Failed to detach worker thread from isolate: {error}") from error if result != 0: raise DataWeaveError(f"Failed to detach worker thread from isolate. Error code: {result}") @@ -184,9 +194,12 @@ def _tear_down_isolate(self, suppress_errors: bool = False) -> None: result = self.lib.graal_tear_down_isolate(self.thread) if result != 0: raise DataWeaveError(f"Failed to tear down GraalVM isolate. Error code: {result}") - except Exception: + except DataWeaveError: if not suppress_errors: raise + except Exception as error: + if not suppress_errors: + raise DataWeaveError(f"Failed to tear down GraalVM isolate: {error}") from error def _reset(self) -> None: self.initialized = False diff --git a/native-lib/python/src/dataweave/runtime.py b/native-lib/python/src/dataweave/runtime.py index 8f88a79a..119c94d7 100644 --- a/native-lib/python/src/dataweave/runtime.py +++ b/native-lib/python/src/dataweave/runtime.py @@ -100,10 +100,13 @@ def write_cb(_context, buffer, length): def worker_main(): worker_thread = None primary_error = None + primary_outcome = False try: worker_thread = self._native.attach_thread() raw = self._native.decode_and_free(invoke(worker_thread, write_cb), worker_thread) - publish(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) + metadata = json.loads(raw) if raw else {"success": False, "error": "Empty response"} + primary_outcome = not metadata.get("success", False) + publish(metadata) except Exception as error: primary_error = error publish({"success": False, "error": str(error)}) @@ -112,7 +115,7 @@ def worker_main(): try: self._native.detach_thread(worker_thread) except Exception as error: - if primary_error is None: + if primary_error is None and not primary_outcome: publish(CleanupFailure(error)) publish(sentinel) @@ -220,5 +223,9 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.cleanup() + try: + self.cleanup() + except Exception: + if exc_type is None: + raise return False diff --git a/native-lib/python/tests/integration/test_lifecycle.py b/native-lib/python/tests/integration/test_lifecycle.py index a91f1049..cfabda1a 100644 --- a/native-lib/python/tests/integration/test_lifecycle.py +++ b/native-lib/python/tests/integration/test_lifecycle.py @@ -8,3 +8,23 @@ def test_context_manager_runs_multiple_scripts(): with dataweave.DataWeave() as dw: assert dw.run("sqrt(144)").get_string() == "12" assert dw.run("sqrt(10000)").get_string() == "100" + + +@pytest.mark.unit +def test_context_exit_preserves_body_exception_when_cleanup_fails(monkeypatch): + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + monkeypatch.setattr(runtime, "initialize", lambda: None) + monkeypatch.setattr(runtime, "cleanup", lambda: (_ for _ in ()).throw(dataweave.DataWeaveError("cleanup failed"))) + + with pytest.raises(ValueError, match="body failed"): + with runtime: + raise ValueError("body failed") + + +@pytest.mark.unit +def test_context_exit_surfaces_cleanup_failure_without_body_exception(monkeypatch): + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + monkeypatch.setattr(runtime, "cleanup", lambda: (_ for _ in ()).throw(dataweave.DataWeaveError("cleanup failed"))) + + with pytest.raises(dataweave.DataWeaveError, match="cleanup failed"): + runtime.__exit__(None, None, None) diff --git a/native-lib/python/tests/unit/test_facade.py b/native-lib/python/tests/unit/test_facade.py index c61f8bd0..f4ea4d2c 100644 --- a/native-lib/python/tests/unit/test_facade.py +++ b/native-lib/python/tests/unit/test_facade.py @@ -66,3 +66,28 @@ def test_cleanup_is_noop_without_global_runtime(): dataweave.cleanup() assert dataweave._global_instance is None + + +@pytest.mark.unit +def test_global_cleanup_clears_failed_runtime_and_allows_recreation(monkeypatch): + created = [] + + class FakeRuntime: + def __init__(self): + created.append(self) + + def initialize(self): + pass + + def cleanup(self): + raise dataweave.DataWeaveError("teardown failed") + + monkeypatch.setattr(dataweave, "DataWeave", FakeRuntime) + first = dataweave._get_global_instance() + + with pytest.raises(dataweave.DataWeaveError, match="teardown failed"): + dataweave.cleanup() + + second = dataweave._get_global_instance() + assert second is not first + dataweave._global_instance = None diff --git a/native-lib/python/tests/unit/test_native.py b/native-lib/python/tests/unit/test_native.py index 71153aec..edaeb5f0 100644 --- a/native-lib/python/tests/unit/test_native.py +++ b/native-lib/python/tests/unit/test_native.py @@ -134,6 +134,39 @@ def __call__(self, *_args): assert runtime.initialized is False +@pytest.mark.unit +def test_initialize_requires_create_isolate_export(monkeypatch): + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: object()) + + with pytest.raises(dataweave.DataWeaveError, match="Native library does not export graal_create_isolate"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_initialize_wraps_create_isolate_exception(monkeypatch): + class Function: + def __call__(self, *_args): + raise RuntimeError("native create failure") + + library = type("Native", (), {"graal_create_isolate": Function()})() + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to create GraalVM isolate: native create failure"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_cleanup_wraps_teardown_exception(): + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.initialized = True + runtime.thread = object() + runtime.isolate = object() + runtime.lib = type("Native", (), {"graal_tear_down_isolate": lambda _self, _thread: (_ for _ in ()).throw(RuntimeError("native teardown failure"))})() + + with pytest.raises(dataweave.DataWeaveError, match="Failed to tear down GraalVM isolate: native teardown failure"): + runtime.cleanup() + + @pytest.mark.unit def test_initialize_tears_down_isolate_when_required_export_is_missing(monkeypatch): class Function: @@ -218,3 +251,30 @@ def test_cleanup_surfaces_native_teardown_error_code(monkeypatch): assert runtime.lib is None assert runtime.thread is None assert runtime.isolate is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("method_name", "error_message"), + [ + ("attach_thread", "Failed to attach worker thread to isolate: native attach failure"), + ("detach_thread", "Failed to detach worker thread from isolate: native detach failure"), + ], +) +def test_thread_lifecycle_wraps_native_invocation_errors(method_name, error_message): + class Native: + def graal_attach_thread(self, _isolate, _thread): + raise RuntimeError("native attach failure") + + def graal_detach_thread(self, _thread): + raise RuntimeError("native detach failure") + + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.lib = Native() + runtime.isolate = object() + + with pytest.raises(dataweave.DataWeaveError, match=error_message): + if method_name == "attach_thread": + runtime.attach_thread() + else: + runtime.detach_thread(object()) diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index fc91af03..405e7b78 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -247,3 +247,17 @@ def graal_detach_thread(self, thread): with pytest.raises(dataweave.DataWeaveError, match="Failed to detach worker thread from isolate. Error code: 7"): list(runtime.run_streaming("script")) + + +@pytest.mark.unit +def test_run_streaming_preserves_unsuccessful_metadata_when_detach_fails(): + class DetachFailingNative(FakeNative): + def graal_detach_thread(self, thread): + super().graal_detach_thread(thread) + return 7 + + runtime = configured_runtime(DetachFailingNative('{"success": false, "error": "script failed"}')) + stream = runtime.run_streaming("script") + + assert list(stream) == [] + assert stream.metadata == dataweave.StreamingResult(False, "script failed", None, None, False) From b285d2172484c6f549691a2ef737f0377df7f806 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 21:06:01 -0300 Subject: [PATCH 17/30] test: add Python TCK conformance lane --- .../task-5-report.md | 42 ++++ native-lib/build.gradle | 12 ++ native-lib/python/tests/__init__.py | 0 native-lib/python/tests/conftest.py | 67 ++++++ native-lib/python/tests/tck/__init__.py | 0 native-lib/python/tests/tck/case_loader.py | 118 +++++++++++ native-lib/python/tests/tck/compare.py | 105 +++++++++ native-lib/python/tests/tck/ignore_list.py | 83 ++++++++ .../python/tests/tck/test_conformance.py | 199 ++++++++++++++++++ 9 files changed, 626 insertions(+) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md create mode 100644 native-lib/python/tests/__init__.py create mode 100644 native-lib/python/tests/tck/__init__.py create mode 100644 native-lib/python/tests/tck/case_loader.py create mode 100644 native-lib/python/tests/tck/compare.py create mode 100644 native-lib/python/tests/tck/ignore_list.py create mode 100644 native-lib/python/tests/tck/test_conformance.py diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md new file mode 100644 index 00000000..e60b756d --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md @@ -0,0 +1,42 @@ +# Task 5 Report: Python TCK Harness + +## Files + +- `native-lib/build.gradle` +- `native-lib/python/tests/__init__.py` +- `native-lib/python/tests/conftest.py` +- `native-lib/python/tests/tck/__init__.py` +- `native-lib/python/tests/tck/case_loader.py` +- `native-lib/python/tests/tck/compare.py` +- `native-lib/python/tests/tck/ignore_list.py` +- `native-lib/python/tests/tck/test_conformance.py` + +## TDD And Test Commands + +1. RED: `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'module_resolution_exclusions_only_skip_importing_cases' -vv` + Result: failed as expected because `validate_exclusions` did not accept or validate staged scenarios. +2. GREEN: `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'discover or compare or exclusion or tck_summary' -vv` + Result: 14 passed, 732 deselected. +3. Exclusion visibility: `python3 -m pytest -m tck -k 'import-lib-out or import-star-out' -vv` + Result: 2 skipped, with header `discovered=731`, `structural-skips=191`, and `categorized-exclusions=6 (module-resolution-not-supported=6)`. +4. Non-excluded failure guard: `python3 -m pytest -m tck -x -vv` + Result: 59 passed, then `core-modules/csv-invalid-utf8-out.csv` failed with `text mismatch`; the harness exited nonzero. The terminal report showed `passed=59, failed=1`. +5. Corpus staging: `./gradlew native-lib:stageTckSuites` + Result: passed; staged the resolved `runtime` and `core-modules` artifacts under the existing Node TCK corpus directory. +6. Gradle lane: `./gradlew native-lib:stageTckSuites native-lib:pythonTck` + Result: invoked native staging, the shared corpus staging task, and pytest. The initial complete run exceeded the local command time limit after reaching the corpus tests. A subsequent direct pytest run established the required non-excluded failure behavior above. + +## Exclusions Discovered + +- 19 staged runtime corpus cases import test-only DW modules. +- Every exclusion is keyed by full suite/case identifier and categorized as `module-resolution-not-supported` with the explicit reason that the Python binding has no module resolver. +- Registry validation rejects missing category/reason and rejects this category for a discovered case whose transform does not contain a DW `import` directive. + +## Commit + +Pending commit: `test: add Python TCK conformance lane` + +## Concerns + +- The current staged corpus has a real, non-excluded failure in `core-modules/csv-invalid-utf8-out.csv`: the runtime emits a replacement character where the fixture expects an empty CSV value. It is intentionally not excluded so `pythonTck` remains a failing conformance gate. +- Full corpus execution can exceed the local command timeout because each parameterized scenario initializes the native runtime through the existing autouse cleanup fixture. diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 62fd585a..3742b188 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -228,6 +228,18 @@ tasks.register('stageTckSuites') { } } +// Python consumes the corpus already staged for the Node TCK lane above. This +// shares the resolved artifacts and extraction, avoiding a second download. +tasks.register('pythonTck', Exec) { + dependsOn tasks.named('stagePythonNativeLib') + dependsOn tasks.named('stageTckSuites') + workingDir("${projectDir}/python") + inputs.dir("${projectDir}/python/tests/tck") + inputs.dir(tckSuitesDir) + inputs.file("${projectDir}/python/pytest.ini") + commandLine(pythonExe, '-m', 'pytest', '-m', 'tck') +} + tasks.register('buildNodePackage', Exec) { dependsOn tasks.named('stageNodeNativeLib') workingDir("${projectDir}/node") diff --git a/native-lib/python/tests/__init__.py b/native-lib/python/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/native-lib/python/tests/conftest.py b/native-lib/python/tests/conftest.py index 7eac9ca2..78d634e5 100644 --- a/native-lib/python/tests/conftest.py +++ b/native-lib/python/tests/conftest.py @@ -6,10 +6,77 @@ PYTHON_SRC_DIR = Path(__file__).resolve().parents[1] / "src" sys.path.insert(0, str(PYTHON_SRC_DIR)) +sys.path.insert(0, str(Path(__file__).parent)) import dataweave +def _tck_discovery(): + from tck.case_loader import discover_cases + from tck.ignore_list import exclusion_for + + suites_dir = Path(__file__).resolve().parents[2] / "node" / "tests" / "tck" / "suites" + discovery = discover_cases(suites_dir) + scenarios = [ + scenario + for discovered_case in discovery.cases + for scenario in discovered_case.scenarios + ] + exclusions = [ + scenario + for scenario in scenarios + if exclusion_for(scenario.identifier.rsplit(":", 1)[0]) + ] + return discovery, scenarios, exclusions + + +def pytest_configure(config): + if config.option.markexpr == "tck": + config._tck_discovery = _tck_discovery() + + +def pytest_report_header(config): + if not hasattr(config, "_tck_discovery"): + return None + discovery, scenarios, exclusions = config._tck_discovery + categories = {} + from tck.ignore_list import exclusion_for + + for scenario in exclusions: + exclusion = exclusion_for(scenario.identifier.rsplit(":", 1)[0]) + categories[exclusion.category] = categories.get(exclusion.category, 0) + 1 + category_totals = ", ".join( + f"{category}={count}" for category, count in sorted(categories.items()) + ) or "none" + return ( + f"TCK: discovered={len(scenarios)}, structural-skips={discovery.structural_skips}, " + f"categorized-exclusions={len(exclusions)} ({category_totals})" + ) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + if not hasattr(config, "_tck_discovery"): + return + discovery, scenarios, exclusions = config._tck_discovery + reports = [ + report + for reports in terminalreporter.stats.values() + for report in reports + if getattr(report, "when", None) == "call" + and "::test_tck_scenario[" in report.nodeid + ] + totals = { + outcome: sum(report.outcome == outcome for report in reports) + for outcome in ("passed", "failed") + } + terminalreporter.write_line( + "TCK totals: " + f"discovered={len(scenarios)}, structural-skips={discovery.structural_skips}, " + f"categorized-exclusions={len(exclusions)}, passed={totals['passed']}, " + f"failed={totals['failed']}" + ) + + @pytest.fixture(autouse=True) def clean_dataweave_runtime(): """Keep module-level isolate state from leaking between integration tests.""" diff --git a/native-lib/python/tests/tck/__init__.py b/native-lib/python/tests/tck/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/native-lib/python/tests/tck/case_loader.py b/native-lib/python/tests/tck/case_loader.py new file mode 100644 index 00000000..88a32ed9 --- /dev/null +++ b/native-lib/python/tests/tck/case_loader.py @@ -0,0 +1,118 @@ +"""Filesystem loader for the Gradle-staged DataWeave TCK corpus.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional + +from dataweave import InputValue + + +EXTENSION_TO_MIME = { + "bin": "application/octet-stream", + "csv": "application/csv", + "dwl": "application/dw", + "json": "application/json", + "multipart": "multipart/form-data", + "properties": "text/x-java-properties", + "txt": "text/plain", + "urlencoded": "application/x-www-form-urlencoded", + "xml": "application/xml", +} + + +@dataclass(frozen=True) +class TckScenario: + identifier: str + transform: str + inputs: Dict[str, InputValue] + expected: bytes + output_extension: str + charset: Optional[str] + + +@dataclass(frozen=True) +class DiscoveredCase: + identifier: str + scenarios: List[TckScenario] + + +@dataclass(frozen=True) +class Discovery: + cases: List[DiscoveredCase] + structural_skips: int + + +def extension_of(name: str) -> str: + return Path(name).suffix.removeprefix(".").lower() + + +def discover_cases(suites_dir: Path) -> Discovery: + cases: List[DiscoveredCase] = [] + structural_skips = 0 + if not suites_dir.exists(): + return Discovery(cases, structural_skips) + + for suite_dir in sorted(path for path in suites_dir.iterdir() if path.is_dir()): + for case_dir in sorted(path for path in suite_dir.iterdir() if path.is_dir()): + if not case_dir.exists(): + structural_skips += 1 + continue + scenarios = _load_case(suite_dir.name, case_dir) + if scenarios is None: + structural_skips += 1 + else: + cases.append(DiscoveredCase(f"{suite_dir.name}/{case_dir.name}", scenarios)) + return Discovery(cases, structural_skips) + + +def _load_case(suite_name: str, case_dir: Path) -> Optional[List[TckScenario]]: + files = {path.name: path for path in case_dir.iterdir() if path.is_file()} + case_name = case_dir.name + if case_name.endswith("_wip") or case_name.endswith("wip"): + return None + if "config.properties" in files or any( + name.startswith(("in", "out")) and name.endswith("-config.properties") for name in files + ): + return None + if any(extension_of(name) == "groovy" for name in files): + return None + + transforms = [ + path for name, path in files.items() + if extension_of(name) == "dwl" and not name.startswith("in") and not name.startswith("out") + ] + if len(transforms) != 1 or "transform.dwl" not in files: + return None + + input_paths = sorted( + (path for name, path in files.items() if name.startswith("in") and name[2:].split(".", 1)[0].isdigit()), + key=lambda path: path.name, + ) + if any(extension_of(path.name) not in EXTENSION_TO_MIME for path in input_paths): + return None + output_paths = sorted( + (path for name, path in files.items() if name.startswith("out.") and extension_of(name) in EXTENSION_TO_MIME), + key=lambda path: path.name, + ) + if not output_paths: + return None + + inputs = { + path.stem: InputValue(path.read_bytes(), mime_type=EXTENSION_TO_MIME[extension_of(path.name)]) + for path in input_paths + } + transform = files["transform.dwl"].read_text(encoding="utf-8") + encoding = files.get("encoding") + charset = encoding.read_text(encoding="utf-8").strip() if encoding else None + case_id = f"{suite_name}/{case_name}" + return [ + TckScenario( + identifier=f"{case_id}:{path.name}", + transform=transform, + inputs=inputs, + expected=path.read_bytes(), + output_extension=extension_of(path.name), + charset=charset, + ) + for path in output_paths + ] diff --git a/native-lib/python/tests/tck/compare.py b/native-lib/python/tests/tck/compare.py new file mode 100644 index 00000000..b2671e00 --- /dev/null +++ b/native-lib/python/tests/tck/compare.py @@ -0,0 +1,105 @@ +"""Output comparators for the formats emitted by the staged TCK corpus.""" + +import json +import xml.etree.ElementTree as ElementTree +from dataclasses import dataclass +from typing import Any, Optional + + +@dataclass(frozen=True) +class CompareResult: + match: bool + detail: str = "" + + +def compare_output( + extension: str, actual: bytes, expected: bytes, charset: Optional[str] = None +) -> CompareResult: + extension = extension.removeprefix(".").lower() + if extension == "bin": + return _result(actual == expected, "binary mismatch") + if extension not in { + "json", "xml", "csv", "txt", "dwl", "properties", "urlencoded", "multipart" + }: + return CompareResult(False, f"unknown output extension: {extension}") + + encoding = _python_encoding(charset) + try: + actual_text = actual.decode(encoding) + expected_text = expected.decode(encoding) + except UnicodeDecodeError as error: + return CompareResult(False, f"cannot decode output as {encoding}: {error}") + if extension == "json": + return _compare_json(actual_text, expected_text) + if extension == "xml": + return _compare_xml(actual_text, expected_text) + if extension == "dwl": + return _result(_strip_whitespace(actual_text) == _strip_whitespace(expected_text), "DWL mismatch") + return _result(_normalize_text(actual_text) == _normalize_text(expected_text), "text mismatch") + + +def _compare_json(actual: str, expected: str) -> CompareResult: + try: + actual_value = json.loads(actual) + except json.JSONDecodeError as error: + return CompareResult(False, f"actual is not valid JSON: {error}") + try: + expected_value = json.loads(expected) + except json.JSONDecodeError as error: + return CompareResult(False, f"expected is not valid JSON: {error}") + return _result(_json_equal(actual_value, expected_value), "JSON mismatch") + + +def _compare_xml(actual: str, expected: str) -> CompareResult: + try: + actual_value = _xml_value(ElementTree.fromstring(actual)) + except ElementTree.ParseError as error: + return CompareResult(False, f"actual is not valid XML: {error}") + try: + expected_value = _xml_value(ElementTree.fromstring(expected)) + except ElementTree.ParseError as error: + return CompareResult(False, f"expected is not valid XML: {error}") + return _result(actual_value == expected_value, "XML mismatch") + + +def _xml_value(element: ElementTree.Element) -> Any: + return ( + element.tag, + tuple(sorted(element.attrib.items())), + (element.text or "").strip(), + tuple(_xml_value(child) for child in element), + ) + + +def _json_equal(actual: Any, expected: Any) -> bool: + if isinstance(actual, str) and isinstance(expected, str): + return _normalize_eol(actual) == _normalize_eol(expected) + if isinstance(actual, list) and isinstance(expected, list): + return len(actual) == len(expected) and all( + _json_equal(left, right) for left, right in zip(actual, expected) + ) + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all( + _json_equal(actual[key], expected[key]) for key in actual + ) + return actual == expected + + +def _python_encoding(charset: Optional[str]) -> str: + return (charset or "utf-8").replace("UTF-16", "utf-16") + + +def _normalize_eol(value: str) -> str: + return value.replace("\r\n", "\n") + + +def _normalize_text(value: str) -> str: + return _normalize_eol(value).strip() + + +def _strip_whitespace(value: str) -> str: + return "".join(value.split()) + + +def _result(match: bool, detail: str) -> CompareResult: + return CompareResult(match, "" if match else detail) diff --git a/native-lib/python/tests/tck/ignore_list.py b/native-lib/python/tests/tck/ignore_list.py new file mode 100644 index 00000000..b5480af2 --- /dev/null +++ b/native-lib/python/tests/tck/ignore_list.py @@ -0,0 +1,83 @@ +"""Auditable exclusions for TCK cases Python cannot execute.""" + +from dataclasses import dataclass +from typing import Dict, Iterable, List, Mapping, Optional + + +MODULE_RESOLUTION_NOT_SUPPORTED = "module-resolution-not-supported" + + +@dataclass(frozen=True) +class Exclusion: + category: str + reason: str + + +# The binding exposes no module resolver. These cases import test-only DW +# modules, so they cannot be executed until the Python API gains that feature. +_MODULE_CASES = ( + "runtime/implicit_type_parameters-out.json", + "runtime/import-component-alias-lib-out.json", + "runtime/import-lib-out.json", + "runtime/import-lib-with-alias-out.json", + "runtime/import-named-lib-out.json", + "runtime/import-star-out.json", + "runtime/import_mapping-out.json", + "runtime/import_mapping_with_functions-out.json", + "runtime/import_mapping_with_implicit_input-out.json", + "runtime/import_namespace-out.xml", + "runtime/infinit_list-out.json", + "runtime/interceptor_functions-out.json", + "runtime/lazy_metadata_definition-out.json", + "runtime/location-out.json", + "runtime/locationString-out.json", + "runtime/logwith_function-out.json", + "runtime/module-singleton-out.json", + "runtime/type_selector_materialize-out.json", + "runtime/weave_multiple_namespace-out.dwl", +) + +EXCLUDED_CASES: Dict[str, Exclusion] = { + case: Exclusion( + MODULE_RESOLUTION_NOT_SUPPORTED, + "imports a test-only DW module; Python binding has no module resolver", + ) + for case in _MODULE_CASES +} + + +def exclusion_for(case_identifier: str) -> Optional[Exclusion]: + return EXCLUDED_CASES.get(case_identifier) + + +def validate_exclusions( + entries: Mapping[str, object], scenarios: Optional[Iterable[object]] = None +) -> List[str]: + errors = [] + for identifier, entry in entries.items(): + category = entry.category if isinstance(entry, Exclusion) else entry.get("category") + reason = entry.reason if isinstance(entry, Exclusion) else entry.get("reason") + if not category: + errors.append(f"{identifier}: missing category") + if not reason: + errors.append(f"{identifier}: missing reason") + if scenarios is not None: + transforms = { + scenario.identifier.rsplit(":", 1)[0]: scenario.transform + for scenario in scenarios + } + for identifier, entry in entries.items(): + category = entry.category if isinstance(entry, Exclusion) else entry.get("category") + if ( + category == MODULE_RESOLUTION_NOT_SUPPORTED + and identifier in transforms + and not _imports_module(transforms[identifier]) + ): + errors.append( + f"{identifier}: module-resolution-not-supported requires a DW import" + ) + return errors + + +def _imports_module(transform: str) -> bool: + return any(line.lstrip().startswith("import ") for line in transform.splitlines()) diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py new file mode 100644 index 00000000..4dc5785b --- /dev/null +++ b/native-lib/python/tests/tck/test_conformance.py @@ -0,0 +1,199 @@ +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, Union + +import pytest + +import dataweave + +sys.path.insert(0, str(Path(__file__).parent)) + +from case_loader import TckScenario, discover_cases +from compare import compare_output +from ignore_list import ( + EXCLUDED_CASES, + Exclusion, + MODULE_RESOLUTION_NOT_SUPPORTED, + exclusion_for, + validate_exclusions, +) +from conftest import pytest_terminal_summary + + +pytestmark = pytest.mark.tck + + +SUITES_DIR = Path(__file__).resolve().parents[3] / "node" / "tests" / "tck" / "suites" +DISCOVERY = discover_cases(SUITES_DIR) +SCENARIOS = [ + scenario + for discovered_case in DISCOVERY.cases + for scenario in discovered_case.scenarios +] +@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda scenario: scenario.identifier) +def test_tck_scenario(scenario): + """Runs each non-excluded staged corpus scenario against the Python binding.""" + exclusion = exclusion_for(scenario.identifier.rsplit(":", 1)[0]) + if exclusion: + pytest.skip(f"{exclusion.category}: {exclusion.reason}") + + result = dataweave.run(scenario.transform, scenario.inputs) + assert result.success, result.error + comparison = compare_output( + scenario.output_extension, + result.get_bytes(), + scenario.expected, + scenario.charset, + ) + assert comparison.match, comparison.detail + + +def write_case(root: Path, name: str, files: Dict[str, Union[bytes, str]]) -> Path: + case = root / "runtime" / name + case.mkdir(parents=True) + for file_name, content in files.items(): + path = case / file_name + if isinstance(content, bytes): + path.write_bytes(content) + else: + path.write_text(content, encoding="utf-8") + return case + + +def test_discover_cases_loads_transform_input_output_and_scenarios(tmp_path: Path): + """Catches a loader that ignores TCK inputs or expected-output scenarios.""" + write_case( + tmp_path, + "maps-out.json", + { + "transform.dwl": "%dw 2.0\noutput application/json\n---\nin0", + "in0.json": '{"answer": 42}', + "out.json": '{"answer": 42}', + "out.xml": "42", + }, + ) + + discovery = discover_cases(tmp_path) + + assert discovery.structural_skips == 0 + assert len(discovery.cases) == 1 + scenarios = discovery.cases[0].scenarios + assert [scenario.identifier for scenario in scenarios] == [ + "runtime/maps-out.json:out.json", + "runtime/maps-out.json:out.xml", + ] + assert scenarios[0].transform == "%dw 2.0\noutput application/json\n---\nin0" + assert scenarios[0].inputs["in0"].mime_type == "application/json" + assert scenarios[0].inputs["in0"].content == b'{"answer": 42}' + assert scenarios[0].expected == b'{"answer": 42}' + + +@pytest.mark.parametrize( + ("extension", "actual", "expected"), + [ + ("json", b'{"a": 1, "b": 2}', b'{"b":2,"a":1}'), + ("xml", b"\n 1\n", b"1"), + ("csv", b"a,b\r\n1,2\r\n", b"a,b\n1,2"), + ("txt", b"value\r\n", b"value\n"), + ("dwl", b"fun f(x) = x + 1", b"fun f(x)=x+1"), + ("properties", b"a=b\r\n", b"a=b\n"), + ("urlencoded", b"a=1&b=2\r\n", b"a=1&b=2"), + ("multipart", b"--boundary\r\nbody", b"--boundary\nbody"), + ("bin", b"\x00\x01", b"\x00\x01"), + ], +) +def test_compare_output_matches_required_corpus_extensions( + extension: str, actual: bytes, expected: bytes +): + """Catches extension dispatch that compares TCK writer output too strictly.""" + assert compare_output(extension, actual, expected).match + + +def test_compare_output_rejects_unknown_extension(): + """Catches silent fallbacks that hide unsupported corpus output formats.""" + result = compare_output("unknown", b"actual", b"expected") + + assert not result.match + assert "unknown output extension" in result.detail + + +def test_exclusion_registry_requires_category_and_reason(): + """Catches exclusions that cannot be audited by category and rationale.""" + errors = validate_exclusions( + { + "missing-category": {"reason": "needs a module"}, + "missing-reason": {"category": "module-resolution-not-supported"}, + } + ) + + assert errors == [ + "missing-category: missing category", + "missing-reason: missing reason", + ] + + +def test_only_declared_case_identifiers_are_excluded(): + """Catches broad exclusion matching that can skip unrelated failures.""" + assert validate_exclusions(EXCLUDED_CASES, SCENARIOS) == [] + assert exclusion_for("unknown-case") is None + exclusion = exclusion_for("runtime/import-lib-out.json") + assert exclusion.category == "module-resolution-not-supported" + + +def test_module_resolution_exclusions_only_skip_importing_cases(): + """Catches a module-resolution exclusion that would hide an unrelated failure.""" + imported = TckScenario( + "runtime/imports:out.json", + "%dw 2.0\nimport sample from test::module\n--- sample", + {}, + b"null", + "json", + None, + ) + plain = TckScenario( + "runtime/plain:out.json", + "%dw 2.0\noutput application/json\n--- 1", + {}, + b"1", + "json", + None, + ) + errors = validate_exclusions( + { + "runtime/imports": Exclusion( + MODULE_RESOLUTION_NOT_SUPPORTED, "imports test module" + ), + "runtime/plain": Exclusion( + MODULE_RESOLUTION_NOT_SUPPORTED, "incorrectly broad" + ), + }, + [imported, plain], + ) + + assert errors == [ + "runtime/plain: module-resolution-not-supported requires a DW import" + ] + + +def test_tck_summary_ignores_collection_nodes(): + """Catches pytest collection nodes being counted as test reports.""" + output = [] + terminalreporter = SimpleNamespace( + stats={ + "passed": [ + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/plain:out.json]", + outcome="passed", + ) + ], + "": [SimpleNamespace(nodeid="tests/tck/test_conformance.py")], + }, + write_line=output.append, + ) + config = SimpleNamespace(_tck_discovery=(DISCOVERY, SCENARIOS, [])) + + pytest_terminal_summary(terminalreporter, 0, config) + + assert output[-1].endswith("passed=1, failed=0") From 1f6023ff5bc01c6787951f6bcd83c74c2772a69f Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 23:17:52 -0300 Subject: [PATCH 18/30] fix: stabilize Python TCK runtime lifecycle --- .../task-5-report.md | 62 +++++++++++++- native-lib/python/tests/conftest.py | 41 +++++++--- native-lib/python/tests/tck/case_loader.py | 8 +- native-lib/python/tests/tck/compare.py | 5 +- native-lib/python/tests/tck/ignore_list.py | 51 ++++++++---- .../python/tests/tck/test_conformance.py | 82 +++++++++++++++++-- 6 files changed, 213 insertions(+), 36 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md index e60b756d..0bd3cc05 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md @@ -34,9 +34,67 @@ ## Commit -Pending commit: `test: add Python TCK conformance lane` +- `b285d21 test: add Python TCK conformance lane` ## Concerns - The current staged corpus has a real, non-excluded failure in `core-modules/csv-invalid-utf8-out.csv`: the runtime emits a replacement character where the fixture expects an empty CSV value. It is intentionally not excluded so `pythonTck` remains a failing conformance gate. -- Full corpus execution can exceed the local command timeout because each parameterized scenario initializes the native runtime through the existing autouse cleanup fixture. +- The original full run did not reach terminal reporting because the generic + per-test cleanup could block after a successful deferred writer. This is a + lifecycle stall, not aggregate runtime-initialization cost; Fix Round 1 + replaces that lifecycle for TCK scenarios. + +## Fix Round 1 + +### Lifecycle And Regression Coverage + +- The TCK lane now owns one session-scoped `DataWeave` instance. The generic + module-level runtime cleanup fixture is bypassed for all `tck`-marked tests, + so a deferred writer is not followed by per-scenario isolate destruction. +- The session runtime deliberately remains process-scoped after pytest's + terminal summary. Direct isolate teardown after + `deferred-write-should-terminate-out.json` blocks in Graal; attempting it at + session teardown would again prevent pytest from reaching its report. Python + process exit owns final release of this TCK-only isolated runtime. +- RED: the deferred-write regression failed with `fixture 'tck_runtime' not + found`. GREEN: it runs the actual deferred-write corpus transform and then a + second transform successfully using the same runtime. + +### Exclusions And Reporting + +- The active registry contains exactly 18 runnable module-import scenarios: + the original six plus the 12 observed unresolved `dw::Client`/`dw::Natives` + cases. Registry validation now rejects stale or structurally unreachable + entries. +- The 13 former registry entries which are loader `transform-shape` structural + skips are not active exclusions. They are reported separately as + `structural-module-cases=13`. +- The terminal report now reconciles scenario-only totals: + `selected=731`, `executed=713`, `active-exclusions=18`, `passed=673`, and + `failed=40`. Thus `executed + active-exclusions == selected` and + `passed + failed == executed`. +- XML comparison retains child tail text, so structural comparison no longer + treats `actual` and `expected` as equivalent. + +### Verification + +1. `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'deferred-write-should-terminate or is-empty-using-empty-stream or streaming_binary_inside_value or try-handle' -vv` + Result: 2 passed, 12 skipped. The deferred-write case completed and each of + the 12 newly active unresolved-module exclusions skipped as categorized. +2. `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'discover or compare or exclusion or tck_summary or tck_session_runtime' -vv` + Result: 18 passed. +3. `./gradlew native-lib:stageTckSuites native-lib:pythonTck` + Result: terminal pytest report was reached in 34.87 seconds. It reported + `selected=731, structural-skips=191, structural-module-cases=13, + executed=713, active-exclusions=18, passed=673, failed=40`, then Gradle + failed as expected because non-excluded conformance mismatches remain. + `core-modules/csv-invalid-utf8-out.csv` remains a reported text mismatch and + was not excluded. The stage task continues to reuse the Node TCK artifacts; + no duplicate suite download was introduced. + +### Updated Concern + +- `pythonTck` is intentionally still a failing conformance gate for the 40 + active, non-excluded mismatches. This round fixes the deferred-writer + teardown stall and reporting attribution; it does not claim to resolve those + runtime/output compatibility failures. diff --git a/native-lib/python/tests/conftest.py b/native-lib/python/tests/conftest.py index 78d634e5..aac62c53 100644 --- a/native-lib/python/tests/conftest.py +++ b/native-lib/python/tests/conftest.py @@ -13,7 +13,7 @@ def _tck_discovery(): from tck.case_loader import discover_cases - from tck.ignore_list import exclusion_for + from tck.ignore_list import EXCLUDED_CASES, STRUCTURAL_MODULE_CASES, validate_exclusions suites_dir = Path(__file__).resolve().parents[2] / "node" / "tests" / "tck" / "suites" discovery = discover_cases(suites_dir) @@ -22,12 +22,15 @@ def _tck_discovery(): for discovered_case in discovery.cases for scenario in discovered_case.scenarios ] + errors = validate_exclusions(EXCLUDED_CASES, scenarios) + if errors: + raise pytest.UsageError("Invalid active TCK exclusions: " + "; ".join(errors)) exclusions = [ - scenario - for scenario in scenarios - if exclusion_for(scenario.identifier.rsplit(":", 1)[0]) + scenario for scenario in scenarios + if scenario.identifier.rsplit(":", 1)[0] in EXCLUDED_CASES ] - return discovery, scenarios, exclusions + structural_modules = set(discovery.structural_case_identifiers) & STRUCTURAL_MODULE_CASES + return discovery, scenarios, exclusions, structural_modules def pytest_configure(config): @@ -38,7 +41,7 @@ def pytest_configure(config): def pytest_report_header(config): if not hasattr(config, "_tck_discovery"): return None - discovery, scenarios, exclusions = config._tck_discovery + discovery, scenarios, exclusions, structural_modules = config._tck_discovery categories = {} from tck.ignore_list import exclusion_for @@ -50,14 +53,15 @@ def pytest_report_header(config): ) or "none" return ( f"TCK: discovered={len(scenarios)}, structural-skips={discovery.structural_skips}, " - f"categorized-exclusions={len(exclusions)} ({category_totals})" + f"structural-module-cases={len(structural_modules)}, " + f"active-exclusions={len(exclusions)} ({category_totals})" ) def pytest_terminal_summary(terminalreporter, exitstatus, config): if not hasattr(config, "_tck_discovery"): return - discovery, scenarios, exclusions = config._tck_discovery + discovery, scenarios, exclusions, structural_modules = config._tck_discovery reports = [ report for reports in terminalreporter.stats.values() @@ -67,24 +71,37 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): ] totals = { outcome: sum(report.outcome == outcome for report in reports) - for outcome in ("passed", "failed") + for outcome in ("passed", "failed", "skipped") } + executed = totals["passed"] + totals["failed"] terminalreporter.write_line( "TCK totals: " - f"discovered={len(scenarios)}, structural-skips={discovery.structural_skips}, " - f"categorized-exclusions={len(exclusions)}, passed={totals['passed']}, " + f"selected={len(scenarios)}, structural-skips={discovery.structural_skips}, " + f"structural-module-cases={len(structural_modules)}, " + f"executed={executed}, active-exclusions={totals['skipped']}, passed={totals['passed']}, " f"failed={totals['failed']}" ) @pytest.fixture(autouse=True) -def clean_dataweave_runtime(): +def clean_dataweave_runtime(request): """Keep module-level isolate state from leaking between integration tests.""" + if request.node.get_closest_marker("tck"): + yield + return dataweave.cleanup() yield dataweave.cleanup() +@pytest.fixture(scope="session") +def tck_runtime(): + """Own one isolate for the TCK session; deferred writers cannot be torn down per case.""" + runtime = dataweave.DataWeave() + runtime.initialize() + yield runtime + + @pytest.fixture def collect_stream(): def collect(stream): diff --git a/native-lib/python/tests/tck/case_loader.py b/native-lib/python/tests/tck/case_loader.py index 88a32ed9..d042e7f6 100644 --- a/native-lib/python/tests/tck/case_loader.py +++ b/native-lib/python/tests/tck/case_loader.py @@ -40,6 +40,7 @@ class DiscoveredCase: class Discovery: cases: List[DiscoveredCase] structural_skips: int + structural_case_identifiers: List[str] def extension_of(name: str) -> str: @@ -49,20 +50,23 @@ def extension_of(name: str) -> str: def discover_cases(suites_dir: Path) -> Discovery: cases: List[DiscoveredCase] = [] structural_skips = 0 + structural_case_identifiers: List[str] = [] if not suites_dir.exists(): - return Discovery(cases, structural_skips) + return Discovery(cases, structural_skips, structural_case_identifiers) for suite_dir in sorted(path for path in suites_dir.iterdir() if path.is_dir()): for case_dir in sorted(path for path in suite_dir.iterdir() if path.is_dir()): if not case_dir.exists(): structural_skips += 1 + structural_case_identifiers.append(f"{suite_dir.name}/{case_dir.name}") continue scenarios = _load_case(suite_dir.name, case_dir) if scenarios is None: structural_skips += 1 + structural_case_identifiers.append(f"{suite_dir.name}/{case_dir.name}") else: cases.append(DiscoveredCase(f"{suite_dir.name}/{case_dir.name}", scenarios)) - return Discovery(cases, structural_skips) + return Discovery(cases, structural_skips, structural_case_identifiers) def _load_case(suite_name: str, case_dir: Path) -> Optional[List[TckScenario]]: diff --git a/native-lib/python/tests/tck/compare.py b/native-lib/python/tests/tck/compare.py index b2671e00..33ec5d45 100644 --- a/native-lib/python/tests/tck/compare.py +++ b/native-lib/python/tests/tck/compare.py @@ -67,7 +67,10 @@ def _xml_value(element: ElementTree.Element) -> Any: element.tag, tuple(sorted(element.attrib.items())), (element.text or "").strip(), - tuple(_xml_value(child) for child in element), + tuple( + (_xml_value(child), (child.tail or "").strip()) + for child in element + ), ) diff --git a/native-lib/python/tests/tck/ignore_list.py b/native-lib/python/tests/tck/ignore_list.py index b5480af2..c6a68bdd 100644 --- a/native-lib/python/tests/tck/ignore_list.py +++ b/native-lib/python/tests/tck/ignore_list.py @@ -7,6 +7,28 @@ MODULE_RESOLUTION_NOT_SUPPORTED = "module-resolution-not-supported" +# These cases are transform-shape structural skips because each bundles its +# imported module beside transform.dwl. They are reported separately and are +# deliberately not active exclusions, so they cannot hide runnable failures. +STRUCTURAL_MODULE_CASES = frozenset( + ( + "runtime/implicit_type_parameters-out.json", + "runtime/import_mapping-out.json", + "runtime/import_mapping_with_functions-out.json", + "runtime/import_mapping_with_implicit_input-out.json", + "runtime/import_namespace-out.xml", + "runtime/infinit_list-out.json", + "runtime/interceptor_functions-out.json", + "runtime/lazy_metadata_definition-out.json", + "runtime/location-out.json", + "runtime/locationString-out.json", + "runtime/logwith_function-out.json", + "runtime/type_selector_materialize-out.json", + "runtime/weave_multiple_namespace-out.dwl", + ) +) + + @dataclass(frozen=True) class Exclusion: category: str @@ -16,25 +38,24 @@ class Exclusion: # The binding exposes no module resolver. These cases import test-only DW # modules, so they cannot be executed until the Python API gains that feature. _MODULE_CASES = ( - "runtime/implicit_type_parameters-out.json", "runtime/import-component-alias-lib-out.json", "runtime/import-lib-out.json", "runtime/import-lib-with-alias-out.json", "runtime/import-named-lib-out.json", "runtime/import-star-out.json", - "runtime/import_mapping-out.json", - "runtime/import_mapping_with_functions-out.json", - "runtime/import_mapping_with_implicit_input-out.json", - "runtime/import_namespace-out.xml", - "runtime/infinit_list-out.json", - "runtime/interceptor_functions-out.json", - "runtime/lazy_metadata_definition-out.json", - "runtime/location-out.json", - "runtime/locationString-out.json", - "runtime/logwith_function-out.json", "runtime/module-singleton-out.json", - "runtime/type_selector_materialize-out.json", - "runtime/weave_multiple_namespace-out.dwl", + "runtime/is-empty-using-empty-stream-out.json", + "runtime/streaming_binary_inside_value-out.json", + "runtime/try-handle-array-value-with-failures-out.json", + "runtime/try-handle-attribute-delegate-with-failures-out.json", + "runtime/try-handle-attributes-value-with-failures-out.json", + "runtime/try-handle-binary-value-with-failures-out.json", + "runtime/try-handle-delegate-value-with-failures-out.json", + "runtime/try-handle-key-value-pair-value-with-failures-out.json", + "runtime/try-handle-materialized-object-with-failures-out.json", + "runtime/try-handle-name-value-pair-value-with-failures-out.json", + "runtime/try-handle-schema-property-value-with-failures-out.json", + "runtime/try-handle-schema-value-with-failures-out.json", ) EXCLUDED_CASES: Dict[str, Exclusion] = { @@ -68,9 +89,11 @@ def validate_exclusions( } for identifier, entry in entries.items(): category = entry.category if isinstance(entry, Exclusion) else entry.get("category") + if identifier not in transforms: + errors.append(f"{identifier}: not a discovered runnable case") + continue if ( category == MODULE_RESOLUTION_NOT_SUPPORTED - and identifier in transforms and not _imports_module(transforms[identifier]) ): errors.append( diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py index 4dc5785b..9cbaea0e 100644 --- a/native-lib/python/tests/tck/test_conformance.py +++ b/native-lib/python/tests/tck/test_conformance.py @@ -31,14 +31,16 @@ for discovered_case in DISCOVERY.cases for scenario in discovered_case.scenarios ] + + @pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda scenario: scenario.identifier) -def test_tck_scenario(scenario): +def test_tck_scenario(scenario, tck_runtime): """Runs each non-excluded staged corpus scenario against the Python binding.""" exclusion = exclusion_for(scenario.identifier.rsplit(":", 1)[0]) if exclusion: pytest.skip(f"{exclusion.category}: {exclusion.reason}") - result = dataweave.run(scenario.transform, scenario.inputs) + result = tck_runtime.run(scenario.transform, scenario.inputs) assert result.success, result.error comparison = compare_output( scenario.output_extension, @@ -49,6 +51,23 @@ def test_tck_scenario(scenario): assert comparison.match, comparison.detail +def test_tck_session_runtime_runs_after_deferred_write_without_teardown(tck_runtime): + """Catches per-scenario isolate cleanup after a deferred writer stalls the lane.""" + deferred = next( + scenario + for scenario in SCENARIOS + if scenario.identifier + == "core-modules/deferred-write-should-terminate-out.json:out.json" + ) + + deferred_result = tck_runtime.run(deferred.transform, deferred.inputs) + following_result = tck_runtime.run("%dw 2.0\noutput application/json\n--- 1") + + assert deferred_result.success, deferred_result.error + assert following_result.success, following_result.error + assert following_result.get_bytes() == b"1" + + def write_case(root: Path, name: str, files: Dict[str, Union[bytes, str]]) -> Path: case = root / "runtime" / name case.mkdir(parents=True) @@ -77,6 +96,7 @@ def test_discover_cases_loads_transform_input_output_and_scenarios(tmp_path: Pat discovery = discover_cases(tmp_path) assert discovery.structural_skips == 0 + assert discovery.structural_case_identifiers == [] assert len(discovery.cases) == 1 scenarios = discovery.cases[0].scenarios assert [scenario.identifier for scenario in scenarios] == [ @@ -94,6 +114,7 @@ def test_discover_cases_loads_transform_input_output_and_scenarios(tmp_path: Pat [ ("json", b'{"a": 1, "b": 2}', b'{"b":2,"a":1}'), ("xml", b"\n 1\n", b"1"), + ("xml", b"1tail", b"1tail"), ("csv", b"a,b\r\n1,2\r\n", b"a,b\n1,2"), ("txt", b"value\r\n", b"value\n"), ("dwl", b"fun f(x) = x + 1", b"fun f(x)=x+1"), @@ -118,6 +139,17 @@ def test_compare_output_rejects_unknown_extension(): assert "unknown output extension" in result.detail +def test_compare_output_rejects_different_xml_tail_text(): + """Catches structural XML comparison that discards text after child elements.""" + result = compare_output( + "xml", + b"1actual tail", + b"1expected tail", + ) + + assert not result.match + + def test_exclusion_registry_requires_category_and_reason(): """Catches exclusions that cannot be audited by category and rationale.""" errors = validate_exclusions( @@ -139,6 +171,7 @@ def test_only_declared_case_identifiers_are_excluded(): assert exclusion_for("unknown-case") is None exclusion = exclusion_for("runtime/import-lib-out.json") assert exclusion.category == "module-resolution-not-supported" + assert len(EXCLUDED_CASES) == 18 def test_module_resolution_exclusions_only_skip_importing_cases(): @@ -176,6 +209,32 @@ def test_module_resolution_exclusions_only_skip_importing_cases(): ] +def test_exclusion_registry_rejects_unreachable_active_entries(): + """Catches active exclusions that cannot affect any discovered runnable scenario.""" + scenario = TckScenario( + "runtime/imports:out.json", + "%dw 2.0\nimport sample from test::module\n--- sample", + {}, + b"null", + "json", + None, + ) + + errors = validate_exclusions( + { + "runtime/imports": Exclusion( + MODULE_RESOLUTION_NOT_SUPPORTED, "imports test module" + ), + "runtime/not-discovered": Exclusion( + MODULE_RESOLUTION_NOT_SUPPORTED, "stale entry" + ), + }, + [scenario], + ) + + assert errors == ["runtime/not-discovered: not a discovered runnable case"] + + def test_tck_summary_ignores_collection_nodes(): """Catches pytest collection nodes being counted as test reports.""" output = [] @@ -186,14 +245,27 @@ def test_tck_summary_ignores_collection_nodes(): when="call", nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/plain:out.json]", outcome="passed", - ) + ), + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/excluded:out.json]", + outcome="skipped", + ), + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/failing:out.json]", + outcome="failed", + ), ], "": [SimpleNamespace(nodeid="tests/tck/test_conformance.py")], }, write_line=output.append, ) - config = SimpleNamespace(_tck_discovery=(DISCOVERY, SCENARIOS, [])) + config = SimpleNamespace(_tck_discovery=(DISCOVERY, SCENARIOS, [SCENARIOS[0]], set())) pytest_terminal_summary(terminalreporter, 0, config) - assert output[-1].endswith("passed=1, failed=0") + assert output[-1] == ( + "TCK totals: selected=731, structural-skips=191, structural-module-cases=0, executed=2, " + "active-exclusions=1, passed=1, failed=1" + ) From d5ff747e706920c854766081a36b8834f9ee938e Mon Sep 17 00:00:00 2001 From: mlischetti Date: Wed, 19 Aug 2026 23:43:30 -0300 Subject: [PATCH 19/30] test: classify Python TCK exclusions --- .../task-5-report.md | 49 +++ native-lib/python/tests/tck/ignore_list.py | 370 +++++++++++++++--- .../python/tests/tck/test_conformance.py | 111 ++++-- 3 files changed, 442 insertions(+), 88 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md index 0bd3cc05..cc262caa 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md @@ -98,3 +98,52 @@ active, non-excluded mismatches. This round fixes the deferred-writer teardown stall and reporting attribution; it does not claim to resolve those runtime/output compatibility failures. + +## Fix Round 2 + +### Evidence-Backed Exclusions + +- Read the reviewed `task-5-failure-inventory.md` before changing the registry. + Its 40 observed failures contained 37 unsupported environment/runtime cases + and three retained conformance mismatches. +- Replaced the generic module-only registry shape with per-case `Exclusion` + records. Every active record carries its full case identifier, a reason with + the direct observed limitation, and one of the nine approved categories. +- Added 37 case-specific entries for the inventory evidence. Together with the + pre-existing 18 unresolved-import exclusions, the active registry now has 55 + entries: + `unsupported-dw-module-resolution=24`, `unavailable-java-module=11`, + `unavailable-classpath-test-resource=4`, + `nondeterministic-properties-output=2`, `multipart-runtime-compatibility=6`, + `coercion-runtime-compatibility=3`, `dw-runtime-compatibility=2`, + `locale-dependent-output=1`, and `source-location-dependent-output=2`. +- Registry validation now rejects missing or mismatched case identity, blank + category/reason fields, unsupported categories, and entries that cannot + affect a discovered runnable scenario. The category-count test locks the + inventory reconciliation and prevents a later category collapse. + +### Retained Conformance Mismatches + +- `core-modules/csv-invalid-utf8-out.csv:out.csv` remains active: the runtime + emits a replacement character where the fixture requires an empty CSV value. +- `core-modules/number-addition-out.json:out.json` remains active: numeric + serialization differs from the fixture's exact integer representation. +- `core-modules/number-subtraction-out.json:out.json` remains active: numeric + serialization produces `0` where the fixture requires `0.0`. + +### Verification + +1. RED: `python3 -m pytest tests/tck/test_conformance.py -m tck -k + exclusion_registry_requires_case_identity_supported_category_and_reason -vv` + Result: failed as expected before the registry change because it did not + validate case identity, approved category, or nonblank reason. +2. Focused registry validation: `python3 -m pytest + tests/tck/test_conformance.py -m tck -k exclusion -vv` + Result: 4 passed. The header reported all 55 active exclusions by category. +3. Full terminal TCK: `python3 -m pytest tests/tck/test_conformance.py -m tck + -vv --maxfail=0` + Result: 3 failed, 693 passed, 55 skipped in 32.38s. The reconciled terminal + totals were `selected=731`, `structural-skips=191`, + `structural-module-cases=13`, `executed=676`, `active-exclusions=55`, + `passed=673`, and `failed=3`. Both invariants hold: + `676 + 55 == 731` and `673 + 3 == 676`. diff --git a/native-lib/python/tests/tck/ignore_list.py b/native-lib/python/tests/tck/ignore_list.py index c6a68bdd..2a001547 100644 --- a/native-lib/python/tests/tck/ignore_list.py +++ b/native-lib/python/tests/tck/ignore_list.py @@ -1,10 +1,32 @@ -"""Auditable exclusions for TCK cases Python cannot execute.""" +"""Auditable exclusions for TCK cases Python cannot execute deterministically.""" from dataclasses import dataclass from typing import Dict, Iterable, List, Mapping, Optional -MODULE_RESOLUTION_NOT_SUPPORTED = "module-resolution-not-supported" +UNSUPPORTED_DW_MODULE_RESOLUTION = "unsupported-dw-module-resolution" +UNAVAILABLE_JAVA_MODULE = "unavailable-java-module" +UNAVAILABLE_CLASSPATH_TEST_RESOURCE = "unavailable-classpath-test-resource" +NONDETERMINISTIC_PROPERTIES_OUTPUT = "nondeterministic-properties-output" +MULTIPART_RUNTIME_COMPATIBILITY = "multipart-runtime-compatibility" +COERCION_RUNTIME_COMPATIBILITY = "coercion-runtime-compatibility" +DW_RUNTIME_COMPATIBILITY = "dw-runtime-compatibility" +LOCALE_DEPENDENT_OUTPUT = "locale-dependent-output" +SOURCE_LOCATION_DEPENDENT_OUTPUT = "source-location-dependent-output" + +SUPPORTED_CATEGORIES = frozenset( + ( + UNSUPPORTED_DW_MODULE_RESOLUTION, + UNAVAILABLE_JAVA_MODULE, + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + NONDETERMINISTIC_PROPERTIES_OUTPUT, + MULTIPART_RUNTIME_COMPATIBILITY, + COERCION_RUNTIME_COMPATIBILITY, + DW_RUNTIME_COMPATIBILITY, + LOCALE_DEPENDENT_OUTPUT, + SOURCE_LOCATION_DEPENDENT_OUTPUT, + ) +) # These cases are transform-shape structural skips because each bundles its @@ -31,39 +53,293 @@ @dataclass(frozen=True) class Exclusion: + case_identifier: str category: str reason: str -# The binding exposes no module resolver. These cases import test-only DW -# modules, so they cannot be executed until the Python API gains that feature. -_MODULE_CASES = ( - "runtime/import-component-alias-lib-out.json", - "runtime/import-lib-out.json", - "runtime/import-lib-with-alias-out.json", - "runtime/import-named-lib-out.json", - "runtime/import-star-out.json", - "runtime/module-singleton-out.json", - "runtime/is-empty-using-empty-stream-out.json", - "runtime/streaming_binary_inside_value-out.json", - "runtime/try-handle-array-value-with-failures-out.json", - "runtime/try-handle-attribute-delegate-with-failures-out.json", - "runtime/try-handle-attributes-value-with-failures-out.json", - "runtime/try-handle-binary-value-with-failures-out.json", - "runtime/try-handle-delegate-value-with-failures-out.json", - "runtime/try-handle-key-value-pair-value-with-failures-out.json", - "runtime/try-handle-materialized-object-with-failures-out.json", - "runtime/try-handle-name-value-pair-value-with-failures-out.json", - "runtime/try-handle-schema-property-value-with-failures-out.json", - "runtime/try-handle-schema-value-with-failures-out.json", -) +def _exclusion(case_identifier: str, category: str, reason: str) -> Exclusion: + return Exclusion(case_identifier, category, reason) + +# Each entry has a full case identifier and direct runtime evidence. Categories +# describe only an observed, unsupported limitation; they never match patterns. EXCLUDED_CASES: Dict[str, Exclusion] = { - case: Exclusion( - MODULE_RESOLUTION_NOT_SUPPORTED, - "imports a test-only DW module; Python binding has no module resolver", - ) - for case in _MODULE_CASES + "runtime/import-component-alias-lib-out.json": _exclusion( + "runtime/import-component-alias-lib-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-lib-out.json": _exclusion( + "runtime/import-lib-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-lib-with-alias-out.json": _exclusion( + "runtime/import-lib-with-alias-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-named-lib-out.json": _exclusion( + "runtime/import-named-lib-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-star-out.json": _exclusion( + "runtime/import-star-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/module-singleton-out.json": _exclusion( + "runtime/module-singleton-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/is-empty-using-empty-stream-out.json": _exclusion( + "runtime/is-empty-using-empty-stream-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Client, which is not resolved by the Python binding", + ), + "runtime/streaming_binary_inside_value-out.json": _exclusion( + "runtime/streaming_binary_inside_value-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-array-value-with-failures-out.json": _exclusion( + "runtime/try-handle-array-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-attribute-delegate-with-failures-out.json": _exclusion( + "runtime/try-handle-attribute-delegate-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-attributes-value-with-failures-out.json": _exclusion( + "runtime/try-handle-attributes-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-binary-value-with-failures-out.json": _exclusion( + "runtime/try-handle-binary-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-delegate-value-with-failures-out.json": _exclusion( + "runtime/try-handle-delegate-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-key-value-pair-value-with-failures-out.json": _exclusion( + "runtime/try-handle-key-value-pair-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-materialized-object-with-failures-out.json": _exclusion( + "runtime/try-handle-materialized-object-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-name-value-pair-value-with-failures-out.json": _exclusion( + "runtime/try-handle-name-value-pair-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-schema-property-value-with-failures-out.json": _exclusion( + "runtime/try-handle-schema-property-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-schema-value-with-failures-out.json": _exclusion( + "runtime/try-handle-schema-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "core-modules/multipart-write-binary-out.json": _exclusion( + "core-modules/multipart-write-binary-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions before exercising multipart binary output", + ), + "core-modules/read-binary-files-out.bin": _exclusion( + "core-modules/read-binary-files-out.bin", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions before reading the binary fixture", + ), + "runtime/full-qualified-name-ref-out.json": _exclusion( + "runtime/full-qualified-name-ref-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve org::mule::weave::v2::libs::lib test modules", + ), + "runtime/private_scope_directives-out.xml": _exclusion( + "runtime/private_scope_directives-out.xml", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::Module", + ), + "runtime/try-out.json": _exclusion( + "runtime/try-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions", + ), + "runtime/urlEncodeDecode-out.json": _exclusion( + "runtime/urlEncodeDecode-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions", + ), + "runtime/java-big-decimal-out.xml": _exclusion( + "runtime/java-big-decimal-out.xml", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve java::lang::String::valueOf", + ), + "runtime/java-field-ref-out.json": _exclusion( + "runtime/java-field-ref-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve test POJO Constants or java::lang::String", + ), + "runtime/java-interop-enum-out.json": _exclusion( + "runtime/java-interop-enum-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve test POJO GenderEnum or java::lang::String", + ), + "runtime/java-interop-function-call-out.json": _exclusion( + "runtime/java-interop-function-call-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve test POJO MyCompanyUtils or java::lang::String", + ), + "runtime/java_epoch_bridge-out.json": _exclusion( + "runtime/java_epoch_bridge-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve java::time::Instant members", + ), + "runtime/runtime_run_coercionException-out.json": _exclusion( + "runtime/runtime_run_coercionException-out.json", + UNAVAILABLE_JAVA_MODULE, + "dw::Runtime.run cannot resolve application/java and returns UnknownContentTypeException", + ), + "runtime/runtime_run_fibo-out.json": _exclusion( + "runtime/runtime_run_fibo-out.json", + UNAVAILABLE_JAVA_MODULE, + "dw::Runtime.run cannot resolve application/java and returns UnknownContentTypeException", + ), + "runtime/runtime_run_null_java-out.json": _exclusion( + "runtime/runtime_run_null_java-out.json", + UNAVAILABLE_JAVA_MODULE, + "application/java returns UnknownContentTypeException", + ), + "runtime/sql_date_mapping-out.json": _exclusion( + "runtime/sql_date_mapping-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve Java test class org::mule::weave::v2::pojo::SqlDateTest", + ), + "runtime/underflow-out.json": _exclusion( + "runtime/underflow-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve java::lang::Long::{MIN_VALUE,MAX_VALUE}", + ), + "runtime/write-function-with-null-out.xml": _exclusion( + "runtime/write-function-with-null-out.xml", + UNAVAILABLE_JAVA_MODULE, + "write(null, application/java) reports unknown content type", + ), + "runtime/dw-binary-out.dwl": _exclusion( + "runtime/dw-binary-out.dwl", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://dw-binary/in0.bin", + ), + "runtime/read-function-by-id-out.json": _exclusion( + "runtime/read-function-by-id-out.json", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://read-function-by-id/include.dwl", + ), + "runtime/read-function-out.json": _exclusion( + "runtime/read-function-out.json", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://read-function/include.dwl", + ), + "runtime/read_lines-out.json": _exclusion( + "runtime/read_lines-out.json", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://read_lines/test.txt", + ), + "core-modules/properties-passthrough-out.properties": _exclusion( + "core-modules/properties-passthrough-out.properties", + NONDETERMINISTIC_PROPERTIES_OUTPUT, + "runtime prepends a current-date properties comment absent from the fixture", + ), + "runtime/properties-writer-out.properties": _exclusion( + "runtime/properties-writer-out.properties", + NONDETERMINISTIC_PROPERTIES_OUTPUT, + "runtime prepends a current-date properties comment and changes fixture order", + ), + "core-modules/multipart-binary-out.multipart": _exclusion( + "core-modules/multipart-binary-out.multipart", + MULTIPART_RUNTIME_COMPATIBILITY, + "multipart comparison cannot decode binary ZIP body and transfer encoding differs", + ), + "core-modules/multipart-class-cast-issue-out.multipart": _exclusion( + "core-modules/multipart-class-cast-issue-out.multipart", + MULTIPART_RUNTIME_COMPATIBILITY, + "runtime emits a multipart boundary different from the fixture", + ), + "core-modules/multipart-empty-part-out.multipart": _exclusion( + "core-modules/multipart-empty-part-out.multipart", + MULTIPART_RUNTIME_COMPATIBILITY, + "runtime emits a multipart boundary different from the fixture", + ), + "core-modules/multipart-mixed-message-out.multipart": _exclusion( + "core-modules/multipart-mixed-message-out.multipart", + MULTIPART_RUNTIME_COMPATIBILITY, + "writer rejects in0 because Multipart Object has empty parts", + ), + "core-modules/multipart-write-message-out.multipart": _exclusion( + "core-modules/multipart-write-message-out.multipart", + MULTIPART_RUNTIME_COMPATIBILITY, + "writer rejects empty multipart parts", + ), + "core-modules/multipart-write-subtype-override-out.multipart": _exclusion( + "core-modules/multipart-write-subtype-override-out.multipart", + MULTIPART_RUNTIME_COMPATIBILITY, + "writer rejects empty multipart parts", + ), + "runtime/access_raw_value-out.json": _exclusion( + "runtime/access_raw_value-out.json", + COERCION_RUNTIME_COMPATIBILITY, + "in0.^raw cannot coerce Null to String", + ), + "runtime/read-concat-out.json": _exclusion( + "runtime/read-concat-out.json", + COERCION_RUNTIME_COMPATIBILITY, + "in0.^raw cannot coerce Null to String", + ), + "runtime/update-op-out.dwl": _exclusion( + "runtime/update-op-out.dwl", + COERCION_RUNTIME_COMPATIBILITY, + "update selector attempts to coerce Null to Number", + ), + "runtime/runtime_dataFormatsDescriptors-out.json": _exclusion( + "runtime/runtime_dataFormatsDescriptors-out.json", + DW_RUNTIME_COMPATIBILITY, + "dw::Runtime reports 9 data-format descriptors while the fixture expects 10", + ), + "runtime/runtime_run-out.json": _exclusion( + "runtime/runtime_run-out.json", + DW_RUNTIME_COMPATIBILITY, + "dw::Runtime.run omits the fixture Java stream class metadata for Binary", + ), + "runtime/coerciones_toString-out.json": _exclusion( + "runtime/coerciones_toString-out.json", + LOCALE_DEPENDENT_OUTPUT, + "runtime emits locale-sensitive p. m. while the fixture requires PM", + ), + "runtime/runtime_orElseTry-out.json": _exclusion( + "runtime/runtime_orElseTry-out.json", + SOURCE_LOCATION_DEPENDENT_OUTPUT, + "dw::Runtime.orElseTry reports line 8 while the fixture reports line 9", + ), + "runtime/try-recursive-call-out.json": _exclusion( + "runtime/try-recursive-call-out.json", + SOURCE_LOCATION_DEPENDENT_OUTPUT, + "error stack embeds anonymous:15:7 instead of fixture runtime class coordinates", + ), } @@ -76,31 +352,31 @@ def validate_exclusions( ) -> List[str]: errors = [] for identifier, entry in entries.items(): - category = entry.category if isinstance(entry, Exclusion) else entry.get("category") - reason = entry.reason if isinstance(entry, Exclusion) else entry.get("reason") + case_identifier = _entry_field(entry, "case_identifier") + category = _entry_field(entry, "category") + reason = _entry_field(entry, "reason") + if not case_identifier: + errors.append(f"{identifier}: missing case identity") + elif case_identifier != identifier: + errors.append(f"{identifier}: case identity must match registry key") if not category: errors.append(f"{identifier}: missing category") - if not reason: + elif category not in SUPPORTED_CATEGORIES: + errors.append(f"{identifier}: unsupported category {category}") + if not reason or not reason.strip(): errors.append(f"{identifier}: missing reason") if scenarios is not None: - transforms = { - scenario.identifier.rsplit(":", 1)[0]: scenario.transform + discovered = { + scenario.identifier.rsplit(":", 1)[0] for scenario in scenarios } - for identifier, entry in entries.items(): - category = entry.category if isinstance(entry, Exclusion) else entry.get("category") - if identifier not in transforms: + for identifier in entries: + if identifier not in discovered: errors.append(f"{identifier}: not a discovered runnable case") - continue - if ( - category == MODULE_RESOLUTION_NOT_SUPPORTED - and not _imports_module(transforms[identifier]) - ): - errors.append( - f"{identifier}: module-resolution-not-supported requires a DW import" - ) return errors -def _imports_module(transform: str) -> bool: - return any(line.lstrip().startswith("import ") for line in transform.splitlines()) +def _entry_field(entry: object, field: str) -> Optional[str]: + if isinstance(entry, Exclusion): + return getattr(entry, field) + return entry.get(field) diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py index 9cbaea0e..a85866bd 100644 --- a/native-lib/python/tests/tck/test_conformance.py +++ b/native-lib/python/tests/tck/test_conformance.py @@ -14,7 +14,7 @@ from ignore_list import ( EXCLUDED_CASES, Exclusion, - MODULE_RESOLUTION_NOT_SUPPORTED, + UNSUPPORTED_DW_MODULE_RESOLUTION, exclusion_for, validate_exclusions, ) @@ -154,8 +154,14 @@ def test_exclusion_registry_requires_category_and_reason(): """Catches exclusions that cannot be audited by category and rationale.""" errors = validate_exclusions( { - "missing-category": {"reason": "needs a module"}, - "missing-reason": {"category": "module-resolution-not-supported"}, + "missing-category": { + "case_identifier": "missing-category", + "reason": "needs a module", + }, + "missing-reason": { + "case_identifier": "missing-reason", + "category": "unsupported-dw-module-resolution", + }, } ) @@ -165,50 +171,69 @@ def test_exclusion_registry_requires_category_and_reason(): ] -def test_only_declared_case_identifiers_are_excluded(): - """Catches broad exclusion matching that can skip unrelated failures.""" - assert validate_exclusions(EXCLUDED_CASES, SCENARIOS) == [] - assert exclusion_for("unknown-case") is None - exclusion = exclusion_for("runtime/import-lib-out.json") - assert exclusion.category == "module-resolution-not-supported" - assert len(EXCLUDED_CASES) == 18 - - -def test_module_resolution_exclusions_only_skip_importing_cases(): - """Catches a module-resolution exclusion that would hide an unrelated failure.""" - imported = TckScenario( - "runtime/imports:out.json", - "%dw 2.0\nimport sample from test::module\n--- sample", - {}, - b"null", - "json", - None, - ) - plain = TckScenario( - "runtime/plain:out.json", - "%dw 2.0\noutput application/json\n--- 1", - {}, - b"1", - "json", - None, - ) +def test_exclusion_registry_requires_case_identity_supported_category_and_reason(): + """Catches exclusions that cannot be traced to one approved runtime limitation.""" errors = validate_exclusions( { - "runtime/imports": Exclusion( - MODULE_RESOLUTION_NOT_SUPPORTED, "imports test module" - ), - "runtime/plain": Exclusion( - MODULE_RESOLUTION_NOT_SUPPORTED, "incorrectly broad" - ), - }, - [imported, plain], + "runtime/missing-identity": { + "category": "unsupported-dw-module-resolution", + "reason": "Cannot resolve dw::core::Assertions", + }, + "runtime/mismatched-identity": { + "case_identifier": "runtime/another-case", + "category": "unsupported-dw-module-resolution", + "reason": "Cannot resolve dw::core::Assertions", + }, + "runtime/unsupported-category": { + "case_identifier": "runtime/unsupported-category", + "category": "broad-runtime-exception", + "reason": "runtime failure", + }, + "runtime/blank-reason": { + "case_identifier": "runtime/blank-reason", + "category": "unsupported-dw-module-resolution", + "reason": " ", + }, + } ) assert errors == [ - "runtime/plain: module-resolution-not-supported requires a DW import" + "runtime/missing-identity: missing case identity", + "runtime/mismatched-identity: case identity must match registry key", + "runtime/unsupported-category: unsupported category broad-runtime-exception", + "runtime/blank-reason: missing reason", ] +def test_only_declared_case_identifiers_are_excluded(): + """Catches broad exclusion matching that can skip unrelated failures.""" + assert validate_exclusions(EXCLUDED_CASES, SCENARIOS) == [] + assert exclusion_for("unknown-case") is None + exclusion = exclusion_for("runtime/import-lib-out.json") + assert exclusion.case_identifier == "runtime/import-lib-out.json" + assert exclusion.category == "unsupported-dw-module-resolution" + assert len(EXCLUDED_CASES) == 55 + + +def test_exclusion_registry_uses_the_inventory_categories(): + """Catches category collapse that would conceal the unsupported boundary.""" + categories = {} + for exclusion in EXCLUDED_CASES.values(): + categories[exclusion.category] = categories.get(exclusion.category, 0) + 1 + + assert categories == { + "coercion-runtime-compatibility": 3, + "dw-runtime-compatibility": 2, + "locale-dependent-output": 1, + "multipart-runtime-compatibility": 6, + "nondeterministic-properties-output": 2, + "source-location-dependent-output": 2, + "unavailable-classpath-test-resource": 4, + "unavailable-java-module": 11, + "unsupported-dw-module-resolution": 24, + } + + def test_exclusion_registry_rejects_unreachable_active_entries(): """Catches active exclusions that cannot affect any discovered runnable scenario.""" scenario = TckScenario( @@ -223,10 +248,14 @@ def test_exclusion_registry_rejects_unreachable_active_entries(): errors = validate_exclusions( { "runtime/imports": Exclusion( - MODULE_RESOLUTION_NOT_SUPPORTED, "imports test module" + "runtime/imports", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports test module", ), "runtime/not-discovered": Exclusion( - MODULE_RESOLUTION_NOT_SUPPORTED, "stale entry" + "runtime/not-discovered", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "stale entry", ), }, [scenario], From 83ef2c6f983fc1a521e38e67d586d173bc373a57 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 09:27:56 -0300 Subject: [PATCH 20/30] ci: validate Python binding tests and TCK --- .github/actions/python/action.yml | 24 ++++++++-- .github/workflows/main.yml | 1 + .../task-6-report.md | 46 +++++++++++++++++++ native-lib/python/README.md | 41 +++++++++++++---- native-lib/python/examples/streaming_demo.py | 3 +- .../python/tests/unit/test_ci_structure.py | 22 +++++++++ 6 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md create mode 100644 native-lib/python/tests/unit/test_ci_structure.py diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index f8864ba3..251031f6 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -1,9 +1,10 @@ name: Python artifact description: >- Installs Python build dependencies and builds the DataWeave Python wheel - (which embeds dwlib); optionally publishes the wheel as a CI artifact (main) - or a release asset (release). No TCK phase today. Requires build-foundation to - have run earlier in the same job. + (which embeds dwlib), runs the Python unit/integration tests, and optionally + runs the master-only Python TCK conformance lane before publishing the wheel + as a CI artifact (main) or a release asset (release). Requires + build-foundation to have run earlier in the same job. inputs: native-version: description: -PnativeVersion value; empty omits the flag. @@ -16,6 +17,10 @@ inputs: runners). required: false default: 'false' + run-tck: + description: When 'true', run the master-only Python TCK conformance lane. + required: false + default: 'false' publish: description: "'none' | 'artifact' | 'release'." required: false @@ -31,12 +36,21 @@ inputs: runs: using: composite steps: - - name: Install Python build dependencies + - name: Install Python build and test dependencies # --ignore-installed: on macOS runners, Homebrew's own setuptools/wheel have no # pip RECORD file, so a plain --upgrade fails trying to uninstall them first # (pip error "uninstall-no-record-file"). --ignore-installed installs pip's # copy on top without needing to remove the untracked brew one. - run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade --ignore-installed setuptools wheel + run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade --ignore-installed setuptools wheel 'native-lib/python[test]' + shell: bash + + - name: Run Python Tests + run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTest ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + shell: bash + + - name: Run Python TCK Conformance + if: inputs.run-tck == 'true' + run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck shell: bash - name: Create Native Lib Python Wheel diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7941eccd..c4babf66 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,6 +63,7 @@ jobs: with: native-version: ${{ env.NATIVE_VERSION }} break-system-packages: 'true' + run-tck: ${{ github.ref == 'refs/heads/master' }} publish: 'artifact' - name: Node diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md new file mode 100644 index 00000000..984f6b1d --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -0,0 +1,46 @@ +# Task 6 Report: Gate Artifacts And Document Actual Behavior + +## Changes + +- `.github/actions/python/action.yml` installs the Python `test` extra, runs + `native-lib:pythonTest` before `native-lib:buildPythonWheel`, and exposes a + `run-tck` input for the Python conformance lane. +- `.github/workflows/main.yml` passes `run-tck` only when the ref is `master`, + alongside the existing Node TCK gate. `pythonTck` stages and reuses the same + corpus as Node through its existing Gradle dependency. +- `native-lib/python/tests/unit/test_ci_structure.py` asserts the artifact-test + ordering and the master-only TCK wiring. +- `native-lib/python/README.md` now documents pytest normal and TCK commands, + bounded queue/chunk behavior, callback abort semantics, post-completion + stream metadata, module-resolution exclusions, and the remaining TCK + failures. +- `native-lib/python/examples/streaming_demo.py` no longer presents the + unsupported `input_properties` argument. + +## TDD And Verification + +1. RED: `python3 -m pytest tests/unit/test_ci_structure.py -m unit -v` + initially failed because the artifact action did not invoke + `native-lib:pythonTest`. +2. GREEN: the same focused command passed with `2 passed` after the action and + workflow wiring were added. +3. `./gradlew native-lib:pythonTest` + passed with `72 passed, 751 deselected` and regenerated the configured JUnit + and coverage XML reports. +4. `./gradlew native-lib:stageTckSuites native-lib:pythonTck` + completed its terminal report and failed as expected: `693 passed, 55 + skipped, 3 failed`. The three intentional non-excluded mismatches are + `csv-invalid-utf8-out.csv`, `number-addition-out.json`, and + `number-subtraction-out.json`. +5. YAML parsing and `git diff --check` passed. + +## Commit + +- Pending: `ci: validate Python binding tests and TCK` + +## Concerns + +- The master-only `pythonTck` command is intentionally terminally failing + while the three genuine runtime conformance mismatches remain. This task + reports them and does not alter core TCK scope or hide them with exclusions. +- Native-image emits existing GraalVM deprecation warnings during local runs. diff --git a/native-lib/python/README.md b/native-lib/python/README.md index d0576571..23068a3b 100644 --- a/native-lib/python/README.md +++ b/native-lib/python/README.md @@ -177,7 +177,9 @@ with dataweave.DataWeave() as dw: ### Bidirectional Streaming (Input + Output) -Stream both input and output for constant memory usage with large files: +Stream both input and output with bounded Python-side queueing. Memory usage is +bounded by the queue capacity plus the input and output chunk sizes; DataWeave +itself can still buffer while parsing or evaluating a transform. ```python # Stream a file through DataWeave @@ -254,11 +256,18 @@ print(result) # StreamingResult(success=True, ...) print(b"".join(chunks)) # b'[1,4,9,16,25]' ``` +Read callbacks return bytes and are called with the native buffer size. Return +`b""` for EOF; an exception is translated to `-1`, which aborts the native +operation. Write callbacks return `0` on success. Any nonzero return value, or +an exception, aborts the operation and returns unsuccessful `StreamingResult` +metadata rather than unwinding a Python exception through the native callback. + ## Running Tests ```bash cd native-lib/python -python3 -m pytest tests/integration -m integration -v +python3 -m pip install '.[test]' +python3 -m pytest -m "unit or integration" -v ``` Or via Gradle: @@ -271,6 +280,18 @@ Or via Gradle: runs exclude `tck`; use `-m "unit or integration"` to run the lanes used by `pythonTest`. +To stage and run the Python conformance suite, use: + +```bash +./gradlew :native-lib:stageTckSuites :native-lib:pythonTck +``` + +`pythonTck` is intentionally separate from normal testing and runs only in the +master-only CI lane. It reuses the corpus staged for Node TCK. It excludes +cases requiring unsupported DataWeave module resolution and other documented +environment limitations. The remaining three known conformance mismatches are +reported as failures, so this lane currently exits nonzero by design. + ## Running Examples ```bash @@ -299,18 +320,19 @@ Execute a script and stream the output. **Returns:** `Stream` iterator yielding chunks, with `.metadata` attribute -#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, input_properties=None) -> Stream` +#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, inputs=None) -> Stream` Execute a script with streaming input and output. **Parameters:** - `input_stream`: Iterable of bytes (file, generator, list) - `input_mime_type`: MIME type of the input stream -- Other parameters configure input handling +- `input_charset`: Optional charset for the streamed input +- `inputs`: Optional additional DataWeave input bindings **Returns:** `Stream` iterator yielding output chunks -#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, input_properties=None, inputs=None) -> StreamingResult` +#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, inputs=None) -> StreamingResult` Low-level callback API for advanced use cases. @@ -352,10 +374,11 @@ class ExecutionResult: ### `Stream` -Iterator that yields output chunks. +Iterator that yields output chunks through a bounded queue. **Attributes:** -- `metadata: StreamingResult` - Available after iteration completes +- `metadata: StreamingResult` - Available only after the iterator completes; + check `success` and `error` after consuming the stream ### `StreamingResult` @@ -474,7 +497,9 @@ if not stream.metadata.success: - **Buffered execution** (`run()`) - Best for small outputs (<1MB) - **Output streaming** (`run_streaming()`) - Use for large outputs (>10MB) - **Bidirectional streaming** (`run_transform()`) - Use for large inputs AND outputs -- **Memory usage**: Streaming uses constant memory (~64KB buffer) +- **Memory usage**: Streaming uses a bounded queue and native callback-sized + chunks. It avoids accumulating output in the Python binding, but does not + guarantee fixed or constant memory for the DataWeave runtime or transform. ## Environment Variables diff --git a/native-lib/python/examples/streaming_demo.py b/native-lib/python/examples/streaming_demo.py index aa5bab93..c581bc47 100755 --- a/native-lib/python/examples/streaming_demo.py +++ b/native-lib/python/examples/streaming_demo.py @@ -90,8 +90,7 @@ def generate_json_chunks(): stream = dataweave.run_transform( 'output application/json --- payload map { name: $.name, age: $.age }', input_stream=iter(lambda: input_file.read(20), b""), # Read 20 bytes at a time - input_mime_type="application/csv", - input_properties={"header": True} + input_mime_type="application/csv" ) output = b"".join(stream).decode('utf-8') diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py new file mode 100644 index 00000000..dcab225d --- /dev/null +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -0,0 +1,22 @@ +from pathlib import Path + +import pytest + + +@pytest.mark.unit +def test_python_artifact_runs_python_test_before_building_wheel(): + action = (Path(__file__).resolve().parents[4] / ".github/actions/python/action.yml").read_text() + + assert "native-lib:pythonTest" in action + assert action.index("native-lib:pythonTest") < action.index("native-lib:buildPythonWheel") + + +@pytest.mark.unit +def test_python_tck_is_gated_by_the_master_only_workflow_input(): + root = Path(__file__).resolve().parents[4] + action = (root / ".github/actions/python/action.yml").read_text() + workflow = (root / ".github/workflows/main.yml").read_text() + + assert "if: inputs.run-tck == 'true'" in action + assert "native-lib:pythonTck" in action + assert "run-tck: ${{ github.ref == 'refs/heads/master' }}" in workflow From 7c286ebe5af3dcc590e38d6c80abdfb7d5ab1d62 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 09:28:06 -0300 Subject: [PATCH 21/30] docs: finalize Python binding task 6 report --- .../2026-08-19-python-binding-modernization/task-6-report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md index 984f6b1d..baaa4aa5 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -36,7 +36,7 @@ ## Commit -- Pending: `ci: validate Python binding tests and TCK` +- `83ef2c6 ci: validate Python binding tests and TCK` ## Concerns From 24732c9b6ba4f3913bfffb9f06042e39955409aa Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 09:48:22 -0300 Subject: [PATCH 22/30] ci: make Python TCK mismatches strict xfails --- .github/actions/build-foundation/action.yml | 5 -- .github/actions/node/action.yml | 1 - .github/actions/python/action.yml | 8 ++ .github/workflows/main.yml | 5 ++ .../task-6-report.md | 53 +++++++----- native-lib/build.gradle | 4 +- native-lib/python/tests/conftest.py | 9 ++- .../python/tests/tck/test_conformance.py | 81 ++++++++++++++++++- .../python/tests/unit/test_ci_structure.py | 27 +++++++ 9 files changed, 163 insertions(+), 30 deletions(-) diff --git a/.github/actions/build-foundation/action.yml b/.github/actions/build-foundation/action.yml index 6e512469..c36f9863 100644 --- a/.github/actions/build-foundation/action.yml +++ b/.github/actions/build-foundation/action.yml @@ -28,11 +28,6 @@ runs: distribution: 'graalvm-community' github-token: ${{ inputs.github-token }} - - name: Install Python test dependencies - run: python3 -m pip install ${{ runner.environment == 'github-hosted' && '--break-system-packages' || '' }} '.[test]' - shell: bash - working-directory: native-lib/python - - name: Run Build run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash diff --git a/.github/actions/node/action.yml b/.github/actions/node/action.yml index d3ca9a8e..baae7cb8 100644 --- a/.github/actions/node/action.yml +++ b/.github/actions/node/action.yml @@ -53,7 +53,6 @@ runs: - name: Run Node.js TCK Conformance if: inputs.run-tck == 'true' run: | - ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites cd native-lib/node && npm run test:tck shell: bash diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index 251031f6..26071354 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -53,6 +53,14 @@ runs: run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck shell: bash + - name: Upload Python TCK JUnit + if: always() && inputs.run-tck == 'true' + uses: actions/upload-artifact@v7.0.1 + with: + name: python-tck-junit + path: native-lib/python/build/test-results/pythonTck.xml + if-no-files-found: error + - name: Create Native Lib Python Wheel run: ./gradlew --stacktrace --no-problems-report native-lib:buildPythonWheel ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4babf66..d4e539cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,11 @@ jobs: script-name: ${{ matrix.script_name }} distro-os: ${{ matrix.distro_os }} + - name: Stage TCK corpus + if: github.ref == 'refs/heads/master' + run: ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites + shell: bash + - name: Python uses: ./.github/actions/python with: diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md index baaa4aa5..2d94b9f4 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -1,5 +1,22 @@ # Task 6 Report: Gate Artifacts And Document Actual Behavior +## Fix Round 1 + +- The three accepted baseline conformance mismatches are now visible strict + pytest xfails. Each uses its full scenario identifier and an explicit reason: + `core-modules/csv-invalid-utf8-out.csv`, + `core-modules/number-addition-out.json`, and + `core-modules/number-subtraction-out.json`. +- No other conformance failure is excluded or xfailed. A new unexpected + mismatch still fails `pythonTck`; an XPASS also fails because the xfails are + strict. +- Python test dependency installation now belongs solely to the Python artifact + action, rather than the shared build foundation. The action publishes the + Python TCK JUnit report with `always()` whenever its master-only TCK lane ran. +- The master workflow stages the runtime/core-modules corpus once before the + Python and Node artifact actions. Neither binding action restages it; local + `pythonTck` remains usable after an explicit `stageTckSuites` invocation. + ## Changes - `.github/actions/python/action.yml` installs the Python `test` extra, runs @@ -19,28 +36,28 @@ ## TDD And Verification -1. RED: `python3 -m pytest tests/unit/test_ci_structure.py -m unit -v` - initially failed because the artifact action did not invoke - `native-lib:pythonTest`. -2. GREEN: the same focused command passed with `2 passed` after the action and - workflow wiring were added. -3. `./gradlew native-lib:pythonTest` - passed with `72 passed, 751 deselected` and regenerated the configured JUnit - and coverage XML reports. -4. `./gradlew native-lib:stageTckSuites native-lib:pythonTck` - completed its terminal report and failed as expected: `693 passed, 55 - skipped, 3 failed`. The three intentional non-excluded mismatches are - `csv-invalid-utf8-out.csv`, `number-addition-out.json`, and - `number-subtraction-out.json`. -5. YAML parsing and `git diff --check` passed. +1. RED: focused CI structure and strict-xfail tests failed before moving + dependency ownership, staging, report upload, and mismatch marks. The + failures identified foundation-owned dependencies, duplicate corpus staging, + missing upload wiring, and absent xfail parameters. +2. GREEN: `python3 -m pytest tests/unit/test_ci_structure.py + tests/tck/test_conformance.py -m unit -k 'strict_xfails or artifact_owns or + stages_the_shared' -vv` passed: `3 passed`. +3. `./gradlew native-lib:pythonTest` passed with `76 passed, 751 deselected` + and regenerated normal JUnit and coverage reports. +4. `./gradlew native-lib:stageTckSuites` stages the shared corpus once; the + terminal `./gradlew native-lib:pythonTck` run passed with `695 passed, 55 + skipped, 3 xfailed`. Its terminal report records `xfail=3`; all other + selected scenarios passed or used independently categorized exclusions. +5. YAML parsing and `git diff --check` are recorded with the final change. ## Commit -- `83ef2c6 ci: validate Python binding tests and TCK` +- Pending Fix Round 1 commit: `ci: make Python TCK mismatches strict xfails` ## Concerns -- The master-only `pythonTck` command is intentionally terminally failing - while the three genuine runtime conformance mismatches remain. This task - reports them and does not alter core TCK scope or hide them with exclusions. +- The accepted baseline is deliberately narrow: only the three named strict + xfails are tolerated. Any new mismatch remains a blocking `pythonTck` + failure; a repaired accepted mismatch becomes an XPASS and also fails. - Native-image emits existing GraalVM deprecation warnings during local runs. diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 3742b188..2a6e2a74 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -232,12 +232,12 @@ tasks.register('stageTckSuites') { // shares the resolved artifacts and extraction, avoiding a second download. tasks.register('pythonTck', Exec) { dependsOn tasks.named('stagePythonNativeLib') - dependsOn tasks.named('stageTckSuites') workingDir("${projectDir}/python") inputs.dir("${projectDir}/python/tests/tck") inputs.dir(tckSuitesDir) inputs.file("${projectDir}/python/pytest.ini") - commandLine(pythonExe, '-m', 'pytest', '-m', 'tck') + commandLine(pythonExe, '-m', 'pytest', '-m', 'tck', + '--junitxml', "${layout.buildDirectory.get().asFile}/test-results/pythonTck.xml") } tasks.register('buildNodePackage', Exec) { diff --git a/native-lib/python/tests/conftest.py b/native-lib/python/tests/conftest.py index aac62c53..d2efc3de 100644 --- a/native-lib/python/tests/conftest.py +++ b/native-lib/python/tests/conftest.py @@ -70,16 +70,21 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): and "::test_tck_scenario[" in report.nodeid ] totals = { - outcome: sum(report.outcome == outcome for report in reports) + outcome: sum( + report.outcome == outcome + and (outcome != "skipped" or not getattr(report, "wasxfail", None)) + for report in reports + ) for outcome in ("passed", "failed", "skipped") } + xfailed = sum(bool(getattr(report, "wasxfail", None)) for report in reports) executed = totals["passed"] + totals["failed"] terminalreporter.write_line( "TCK totals: " f"selected={len(scenarios)}, structural-skips={discovery.structural_skips}, " f"structural-module-cases={len(structural_modules)}, " f"executed={executed}, active-exclusions={totals['skipped']}, passed={totals['passed']}, " - f"failed={totals['failed']}" + f"failed={totals['failed']}, xfail={xfailed}" ) diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py index a85866bd..99a2ea91 100644 --- a/native-lib/python/tests/tck/test_conformance.py +++ b/native-lib/python/tests/tck/test_conformance.py @@ -33,7 +33,84 @@ ] -@pytest.mark.parametrize("scenario", SCENARIOS, ids=lambda scenario: scenario.identifier) +ACCEPTED_BASELINE_MISMATCHES = { + "core-modules/csv-invalid-utf8-out.csv:out.csv": ( + "runtime emits a replacement character where the fixture expects an empty CSV value" + ), + "core-modules/number-addition-out.json:out.json": ( + "runtime emits a numeric result that differs from the accepted baseline fixture" + ), + "core-modules/number-subtraction-out.json:out.json": ( + "runtime emits a numeric result that differs from the accepted baseline fixture" + ), +} + + +def tck_params(): + return [ + pytest.param( + scenario, + marks=pytest.mark.xfail(strict=True, reason=reason), + id=scenario.identifier, + ) + if (reason := ACCEPTED_BASELINE_MISMATCHES.get(scenario.identifier)) + else pytest.param(scenario, id=scenario.identifier) + for scenario in SCENARIOS + ] + + +@pytest.mark.unit +def test_accepted_baseline_mismatches_are_strict_xfails_with_reasons(): + params = tck_params() + expected = { + "core-modules/csv-invalid-utf8-out.csv:out.csv": ( + "runtime emits a replacement character where the fixture expects an empty CSV value" + ), + "core-modules/number-addition-out.json:out.json": ( + "runtime emits a numeric result that differs from the accepted baseline fixture" + ), + "core-modules/number-subtraction-out.json:out.json": ( + "runtime emits a numeric result that differs from the accepted baseline fixture" + ), + } + + xfails = { + parameter.id: next(mark for mark in parameter.marks if mark.name == "xfail") + for parameter in params + if any(mark.name == "xfail" for mark in parameter.marks) + } + + assert set(xfails) == set(expected) + for identifier, reason in expected.items(): + assert xfails[identifier].kwargs["strict"] is True + assert xfails[identifier].kwargs["reason"] == reason + + +@pytest.mark.unit +def test_tck_summary_counts_xfails_as_visible_expected_mismatches(): + output = [] + terminalreporter = SimpleNamespace( + stats={ + "xfailed": [ + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[core-modules/csv-invalid-utf8-out.csv:out.csv]", + outcome="skipped", + wasxfail="accepted mismatch", + ) + ] + }, + write_line=output.append, + ) + config = SimpleNamespace(_tck_discovery=(DISCOVERY, SCENARIOS, [], set())) + + pytest_terminal_summary(terminalreporter, 0, config) + + assert "active-exclusions=0" in output[-1] + assert "xfail=1" in output[-1] + + +@pytest.mark.parametrize("scenario", tck_params()) def test_tck_scenario(scenario, tck_runtime): """Runs each non-excluded staged corpus scenario against the Python binding.""" exclusion = exclusion_for(scenario.identifier.rsplit(":", 1)[0]) @@ -296,5 +373,5 @@ def test_tck_summary_ignores_collection_nodes(): assert output[-1] == ( "TCK totals: selected=731, structural-skips=191, structural-module-cases=0, executed=2, " - "active-exclusions=1, passed=1, failed=1" + "active-exclusions=1, passed=1, failed=1, xfail=0" ) diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index dcab225d..4514f7c3 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -20,3 +20,30 @@ def test_python_tck_is_gated_by_the_master_only_workflow_input(): assert "if: inputs.run-tck == 'true'" in action assert "native-lib:pythonTck" in action assert "run-tck: ${{ github.ref == 'refs/heads/master' }}" in workflow + + +@pytest.mark.unit +def test_python_artifact_owns_test_dependencies_and_tck_junit_upload(): + root = Path(__file__).resolve().parents[4] + foundation = (root / ".github/actions/build-foundation/action.yml").read_text() + action = (root / ".github/actions/python/action.yml").read_text() + + assert "Install Python test dependencies" not in foundation + assert "native-lib/python[test]" in action + assert action.index("Run Python TCK Conformance") < action.index("Upload Python TCK JUnit") + assert "if: always() && inputs.run-tck == 'true'" in action + assert "native-lib/python/build/test-results/pythonTck.xml" in action + + +@pytest.mark.unit +def test_master_tck_stages_the_shared_corpus_once_before_python_and_node(): + root = Path(__file__).resolve().parents[4] + gradle = (root / "native-lib/build.gradle").read_text() + workflow = (root / ".github/workflows/main.yml").read_text() + node_action = (root / ".github/actions/node/action.yml").read_text() + + python_tck = gradle[gradle.index("tasks.register('pythonTck'"):gradle.index("tasks.register('buildNodePackage'")] + assert "dependsOn tasks.named('stageTckSuites')" not in python_tck + assert "native-lib:stageTckSuites" not in node_action + assert workflow.index("Stage TCK corpus") < workflow.index("- name: Python") + assert workflow.index("Stage TCK corpus") < workflow.index("- name: Node") From 55d7b09ac57ec96ce8395ddc52d247d6e9e4de68 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 09:57:19 -0300 Subject: [PATCH 23/30] ci: harden binding artifact sequencing --- .github/actions/build-foundation/action.yml | 2 +- .github/actions/python/action.yml | 30 ++++++++++-------- .github/workflows/main.yml | 10 ++++++ .../task-6-report.md | 31 ++++++++++++++++++- .../python/tests/unit/test_ci_structure.py | 21 +++++++++++++ 5 files changed, 79 insertions(+), 15 deletions(-) diff --git a/.github/actions/build-foundation/action.yml b/.github/actions/build-foundation/action.yml index c36f9863..c806730c 100644 --- a/.github/actions/build-foundation/action.yml +++ b/.github/actions/build-foundation/action.yml @@ -29,5 +29,5 @@ runs: github-token: ${{ inputs.github-token }} - name: Run Build - run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipPythonTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index 26071354..26fada60 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -21,6 +21,10 @@ inputs: description: When 'true', run the master-only Python TCK conformance lane. required: false default: 'false' + platform: + description: Platform token for matrix-qualified TCK JUnit artifact names. + required: false + default: '' publish: description: "'none' | 'artifact' | 'release'." required: false @@ -48,19 +52,6 @@ runs: run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTest ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash - - name: Run Python TCK Conformance - if: inputs.run-tck == 'true' - run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck - shell: bash - - - name: Upload Python TCK JUnit - if: always() && inputs.run-tck == 'true' - uses: actions/upload-artifact@v7.0.1 - with: - name: python-tck-junit - path: native-lib/python/build/test-results/pythonTck.xml - if-no-files-found: error - - name: Create Native Lib Python Wheel run: ./gradlew --stacktrace --no-problems-report native-lib:buildPythonWheel ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash @@ -81,3 +72,16 @@ runs: file_glob: true tag: ${{ inputs.tag }} overwrite: true + + - name: Run Python TCK Conformance + if: inputs.run-tck == 'true' + run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck + shell: bash + + - name: Upload Python TCK JUnit + if: always() && inputs.run-tck == 'true' + uses: actions/upload-artifact@v7.0.1 + with: + name: python-tck-junit-${{ inputs.platform }} + path: native-lib/python/build/test-results/pythonTck.xml + if-no-files-found: error diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4e539cb..0620098a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -64,15 +64,20 @@ jobs: shell: bash - name: Python + id: python uses: ./.github/actions/python + continue-on-error: true with: native-version: ${{ env.NATIVE_VERSION }} break-system-packages: 'true' run-tck: ${{ github.ref == 'refs/heads/master' }} + platform: ${{ matrix.script_name }} publish: 'artifact' - name: Node + id: node uses: ./.github/actions/node + continue-on-error: true with: native-version: ${{ env.NATIVE_VERSION }} run-tck: ${{ github.ref == 'refs/heads/master' }} @@ -80,6 +85,11 @@ jobs: arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} + - name: Fail if binding artifacts failed + if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure') + run: exit 1 + shell: bash + - name: Native library uses: ./.github/actions/native-lib with: diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md index 2d94b9f4..155fbd00 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -17,6 +17,23 @@ Python and Node artifact actions. Neither binding action restages it; local `pythonTck` remains usable after an explicit `stageTckSuites` invocation. +## Fix Round 2 + +- The foundation Gradle build now passes `-PskipPythonTests=true`, leaving the + Python artifact action as the only CI owner that installs Python dependencies + and runs `native-lib:pythonTest`. +- The Python wheel is built and uploaded before the optional Python TCK. This + preserves the package artifact when TCK conformance fails, while the TCK + result still determines the final job outcome. +- The Python and Node artifact steps use `continue-on-error: true`, so a failed + binding lane does not prevent the other lane from executing. A following + `always()` aggregation step fails the job when either binding lane failed. +- Python TCK JUnit artifacts include the workflow matrix platform token: + `python-tck-junit-${{ inputs.platform }}`, preventing cross-platform upload + name collisions. +- The strict xfail baseline remains unchanged: only the three accepted named + mismatches are strict xfails; any new mismatch and any XPASS fail the TCK. + ## Changes - `.github/actions/python/action.yml` installs the Python `test` extra, runs @@ -51,9 +68,21 @@ selected scenarios passed or used independently categorized exclusions. 5. YAML parsing and `git diff --check` are recorded with the final change. +### Fix Round 2 Verification + +1. RED: `python3 -m pytest tests/unit/test_ci_structure.py -m unit -v` failed + with the missing platform input/artifact name and missing + `-PskipPythonTests=true` foundation flag. +2. GREEN: the same focused CI structure test passed with `5 passed` after the + workflow, action, and structure-test updates. +3. YAML parsing for the changed workflow/actions and `git diff --check` passed. +4. Foundation-equivalent dry run/build command passed: + `./gradlew --stacktrace --no-problems-report -PskipNodeTests=true + -PskipPythonTests=true -PskipTCKTests=true build`. + ## Commit -- Pending Fix Round 1 commit: `ci: make Python TCK mismatches strict xfails` +- Pending Fix Round 2 commit: `ci: harden binding artifact sequencing` ## Concerns diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index 4514f7c3..b707f76a 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -30,6 +30,11 @@ def test_python_artifact_owns_test_dependencies_and_tck_junit_upload(): assert "Install Python test dependencies" not in foundation assert "native-lib/python[test]" in action + assert "platform:" in action + assert "python-tck-junit-${{ inputs.platform }}" in action + assert action.index("Create Native Lib Python Wheel") < action.index("Run Python TCK Conformance") + assert action.index("Upload Python wheel (artifact)") < action.index("Run Python TCK Conformance") + assert action.index("Upload Python wheel to release") < action.index("Run Python TCK Conformance") assert action.index("Run Python TCK Conformance") < action.index("Upload Python TCK JUnit") assert "if: always() && inputs.run-tck == 'true'" in action assert "native-lib/python/build/test-results/pythonTck.xml" in action @@ -47,3 +52,19 @@ def test_master_tck_stages_the_shared_corpus_once_before_python_and_node(): assert "native-lib:stageTckSuites" not in node_action assert workflow.index("Stage TCK corpus") < workflow.index("- name: Python") assert workflow.index("Stage TCK corpus") < workflow.index("- name: Node") + + +@pytest.mark.unit +def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): + root = Path(__file__).resolve().parents[4] + foundation = (root / ".github/actions/build-foundation/action.yml").read_text() + workflow = (root / ".github/workflows/main.yml").read_text() + + assert "-PskipPythonTests=true" in foundation + assert "id: python" in workflow + assert "id: node" in workflow + assert workflow.count("continue-on-error: true") >= 2 + assert "Fail if binding artifacts failed" in workflow + assert "steps.python.outcome == 'failure'" in workflow + assert "steps.node.outcome == 'failure'" in workflow + assert "platform: ${{ matrix.script_name }}" in workflow From 0408d7e82db7d023b0b8be20a00d71d2eb4f5ba9 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 10:01:00 -0300 Subject: [PATCH 24/30] ci: run binding TCKs after failures --- .github/actions/node/action.yml | 2 +- .github/actions/python/action.yml | 2 +- .github/workflows/main.yml | 10 ++++----- .../task-6-report.md | 22 ++++++++++++++++++- .../python/tests/unit/test_ci_structure.py | 7 +++++- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/.github/actions/node/action.yml b/.github/actions/node/action.yml index baae7cb8..706dbe19 100644 --- a/.github/actions/node/action.yml +++ b/.github/actions/node/action.yml @@ -51,7 +51,7 @@ runs: shell: bash - name: Run Node.js TCK Conformance - if: inputs.run-tck == 'true' + if: always() && inputs.run-tck == 'true' run: | cd native-lib/node && npm run test:tck shell: bash diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index 26fada60..f3e38c3d 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -74,7 +74,7 @@ runs: overwrite: true - name: Run Python TCK Conformance - if: inputs.run-tck == 'true' + if: always() && inputs.run-tck == 'true' run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck shell: bash diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0620098a..1985ad22 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,11 +85,6 @@ jobs: arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} - - name: Fail if binding artifacts failed - if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure') - run: exit 1 - shell: bash - - name: Native library uses: ./.github/actions/native-lib with: @@ -97,3 +92,8 @@ jobs: publish: 'artifact' arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} + + - name: Fail if binding artifacts failed + if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure') + run: exit 1 + shell: bash diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md index 155fbd00..409ac0a3 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -34,6 +34,17 @@ - The strict xfail baseline remains unchanged: only the three accepted named mismatches are strict xfails; any new mismatch and any XPASS fail the TCK. +## Fix Round 3 + +- The Python and Node master-only TCK conformance steps now use + `always() && inputs.run-tck == 'true'`, so they run even if an earlier step + in the same composite action failed. +- The binding failure aggregation step remains guarded by `always()` and the + Python/Node step outcomes, but now follows the Native library artifact step. + It is therefore the workflow's final binding-failure verdict. +- CI structure tests assert both exact TCK conditions and that Native library + precedes the aggregation step. Strict Python TCK xfails remain unchanged. + ## Changes - `.github/actions/python/action.yml` installs the Python `test` extra, runs @@ -80,9 +91,18 @@ `./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipPythonTests=true -PskipTCKTests=true build`. +### Fix Round 3 Verification + +1. RED: `python3 -m pytest tests/unit/test_ci_structure.py -m unit -v` failed + because the Node TCK condition lacked `always()` and the binding aggregation + preceded Native library. +2. GREEN: the same focused test passed with `5 passed` after updating both + composite action TCK guards and moving final aggregation. +3. YAML parsing for the changed workflow/actions and `git diff --check` passed. + ## Commit -- Pending Fix Round 2 commit: `ci: harden binding artifact sequencing` +- Pending Fix Round 3 commit: `ci: run binding TCKs after failures` ## Concerns diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index b707f76a..d7dba43e 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -17,7 +17,7 @@ def test_python_tck_is_gated_by_the_master_only_workflow_input(): action = (root / ".github/actions/python/action.yml").read_text() workflow = (root / ".github/workflows/main.yml").read_text() - assert "if: inputs.run-tck == 'true'" in action + assert "if: always() && inputs.run-tck == 'true'" in action assert "native-lib:pythonTck" in action assert "run-tck: ${{ github.ref == 'refs/heads/master' }}" in workflow @@ -59,6 +59,8 @@ def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): root = Path(__file__).resolve().parents[4] foundation = (root / ".github/actions/build-foundation/action.yml").read_text() workflow = (root / ".github/workflows/main.yml").read_text() + python_action = (root / ".github/actions/python/action.yml").read_text() + node_action = (root / ".github/actions/node/action.yml").read_text() assert "-PskipPythonTests=true" in foundation assert "id: python" in workflow @@ -68,3 +70,6 @@ def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): assert "steps.python.outcome == 'failure'" in workflow assert "steps.node.outcome == 'failure'" in workflow assert "platform: ${{ matrix.script_name }}" in workflow + assert "if: always() && inputs.run-tck == 'true'" in python_action + assert "if: always() && inputs.run-tck == 'true'" in node_action + assert workflow.index("- name: Native library") < workflow.index("- name: Fail if binding artifacts failed") From e1bd3375043ffda2ee5d19d53e05821578058f4d Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 10:03:31 -0300 Subject: [PATCH 25/30] test: scope CI workflow guard assertions --- .../task-6-report.md | 20 ++++++++++++++- .../python/tests/unit/test_ci_structure.py | 25 ++++++++++++++++--- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md index 409ac0a3..fc9b4103 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -45,6 +45,16 @@ - CI structure tests assert both exact TCK conditions and that Native library precedes the aggregation step. Strict Python TCK xfails remain unchanged. +## Fix Round 4 + +- CI structure tests no longer use global `if` substring checks for TCK or + binding failure aggregation. A scoped `named_step_if` extractor selects the + named YAML step and reads only that step's `if` field. +- The test asserts the exact guard for `Run Python TCK Conformance`, `Run + Node.js TCK Conformance`, and `Fail if binding artifacts failed`. Thus the + Python TCK assertion cannot be satisfied by the similarly guarded JUnit + upload step, and changes to any required `always()` or outcome clause fail. + ## Changes - `.github/actions/python/action.yml` installs the Python `test` extra, runs @@ -100,9 +110,17 @@ composite action TCK guards and moving final aggregation. 3. YAML parsing for the changed workflow/actions and `git diff --check` passed. +### Fix Round 4 Verification + +1. RED: the focused CI structure test failed after replacing global checks with + calls to the not-yet-defined scoped extractor. +2. GREEN: after adding `named_step_if`, the focused test passed with `5 + passed`. +3. YAML parsing for changed workflow/actions and `git diff --check` passed. + ## Commit -- Pending Fix Round 3 commit: `ci: run binding TCKs after failures` +- Pending Fix Round 4 commit: `test: scope CI workflow guard assertions` ## Concerns diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index d7dba43e..770a113b 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -1,8 +1,22 @@ from pathlib import Path +import re import pytest +def named_step_if(document: str, name: str) -> str: + step = re.search( + rf"^\s*- name: {re.escape(name)}\n(?P(?:^\s{{6}}.*\n?)*)", + document, + re.MULTILINE, + ) + assert step, f"missing step {name!r}" + + guard = re.search(r"^\s+if: (?P.+)$", step.group("body"), re.MULTILINE) + assert guard, f"missing if guard for step {name!r}" + return guard.group("guard") + + @pytest.mark.unit def test_python_artifact_runs_python_test_before_building_wheel(): action = (Path(__file__).resolve().parents[4] / ".github/actions/python/action.yml").read_text() @@ -17,7 +31,7 @@ def test_python_tck_is_gated_by_the_master_only_workflow_input(): action = (root / ".github/actions/python/action.yml").read_text() workflow = (root / ".github/workflows/main.yml").read_text() - assert "if: always() && inputs.run-tck == 'true'" in action + assert named_step_if(action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" assert "native-lib:pythonTck" in action assert "run-tck: ${{ github.ref == 'refs/heads/master' }}" in workflow @@ -36,7 +50,7 @@ def test_python_artifact_owns_test_dependencies_and_tck_junit_upload(): assert action.index("Upload Python wheel (artifact)") < action.index("Run Python TCK Conformance") assert action.index("Upload Python wheel to release") < action.index("Run Python TCK Conformance") assert action.index("Run Python TCK Conformance") < action.index("Upload Python TCK JUnit") - assert "if: always() && inputs.run-tck == 'true'" in action + assert named_step_if(action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" assert "native-lib/python/build/test-results/pythonTck.xml" in action @@ -70,6 +84,9 @@ def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): assert "steps.python.outcome == 'failure'" in workflow assert "steps.node.outcome == 'failure'" in workflow assert "platform: ${{ matrix.script_name }}" in workflow - assert "if: always() && inputs.run-tck == 'true'" in python_action - assert "if: always() && inputs.run-tck == 'true'" in node_action + assert named_step_if(python_action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" + assert named_step_if(node_action, "Run Node.js TCK Conformance") == "always() && inputs.run-tck == 'true'" + assert named_step_if(workflow, "Fail if binding artifacts failed") == ( + "always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure')" + ) assert workflow.index("- name: Native library") < workflow.index("- name: Fail if binding artifacts failed") From da02ebfc8a919a591a7dd81009f17f379b810e12 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 10:15:53 -0300 Subject: [PATCH 26/30] test: bound CI workflow guard step extraction --- .../task-6-report.md | 19 ++++++++++++++++++- .../python/tests/unit/test_ci_structure.py | 18 ++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md index fc9b4103..1078470a 100644 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md @@ -118,9 +118,26 @@ passed`. 3. YAML parsing for changed workflow/actions and `git diff --check` passed. +### Fix Round 5 Verification + +1. Root cause: the prior `named_step_if` extractor used `\s` for indentation. + Because `\s` includes newlines, the body capture could cross a same-level + step boundary and read that later step's `if` guard. +2. RED: `test_named_step_if_does_not_read_a_later_step_guard` failed against + the prior extractor: after removing the aggregation step guard and placing + it on a later same-level step, the helper incorrectly returned the later + guard instead of raising `missing if guard`. +3. GREEN: the extractor now captures the named step's leading horizontal + whitespace and consumes only lines with additional horizontal whitespace. + The new mutation regression and the focused CI structure suite passed with + `6 passed`. +4. Python YAML parsing of the changed workflow/action files and `git diff + --check` passed. + ## Commit -- Pending Fix Round 4 commit: `test: scope CI workflow guard assertions` +- Fix Round 4 commit: `test: scope CI workflow guard assertions` +- Pending Fix Round 5 commit: `test: bound CI workflow guard step extraction` ## Concerns diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index 770a113b..ae19f5bd 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -6,13 +6,14 @@ def named_step_if(document: str, name: str) -> str: step = re.search( - rf"^\s*- name: {re.escape(name)}\n(?P(?:^\s{{6}}.*\n?)*)", + rf"^(?P[ \t]*)- name: {re.escape(name)}\n" + rf"(?P(?:^(?P=indent)[ \t]+.*\n?)*)", document, re.MULTILINE, ) assert step, f"missing step {name!r}" - guard = re.search(r"^\s+if: (?P.+)$", step.group("body"), re.MULTILINE) + guard = re.search(r"^[ \t]+if: (?P.+)$", step.group("body"), re.MULTILINE) assert guard, f"missing if guard for step {name!r}" return guard.group("guard") @@ -90,3 +91,16 @@ def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): "always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure')" ) assert workflow.index("- name: Native library") < workflow.index("- name: Fail if binding artifacts failed") + + +@pytest.mark.unit +def test_named_step_if_does_not_read_a_later_step_guard(): + mutated_workflow = """\ + - name: Fail if binding artifacts failed + run: exit 1 + - name: Later step + if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure') +""" + + with pytest.raises(AssertionError, match="missing if guard"): + named_step_if(mutated_workflow, "Fail if binding artifacts failed") From 979f512c389875f740bc6417bf6225da3a855541 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 11:04:51 -0300 Subject: [PATCH 27/30] fix: remediate Python binding final review --- .../final-fix-report.md | 57 +++++++++ ...-19-python-binding-modernization-design.md | 53 ++++++++ native-lib/python/README.md | 27 +++- native-lib/python/src/dataweave/models.py | 22 +++- native-lib/python/src/dataweave/runtime.py | 18 +-- native-lib/python/tests/conftest.py | 7 +- native-lib/python/tests/tck/compare.py | 32 +++-- native-lib/python/tests/tck/ignore_list.py | 92 ------------- .../python/tests/tck/test_conformance.py | 121 +++++++++++++----- native-lib/python/tests/unit/test_models.py | 20 +++ .../python/tests/unit/test_streaming.py | 41 +++++- 11 files changed, 338 insertions(+), 152 deletions(-) create mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md create mode 100644 docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md new file mode 100644 index 00000000..56de06fe --- /dev/null +++ b/.superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md @@ -0,0 +1,57 @@ +# Final Review Remediation Report + +## Resolutions + +1. `Stream.close()` is now public and `Stream` supports `with`. Closing sets + cancellation and generator cleanup is non-raising. Native workers use a + bounded 0.1-second join and are daemon threads because Python cannot cancel + an in-progress native call. Public early-close and uncancellable-worker + finalization regression tests cover this policy. +2. The low-level read callback now rejects data larger than the native buffer + with `-1`; it no longer silently truncates callback input. The iterable + transform path continues preserving chunk remainders. +3. The TCK session fixture now always calls `runtime.cleanup()`. Only the + deferred-writer scenario executes in a subprocess, preventing its known + isolate-teardown stall from leaving the session runtime unmanaged. +4. TCK skips now contain only 39 binding/environment capability cases: 24 + module-resolution, 11 unavailable Java-module, and 4 unavailable classpath + resource cases. The 19 runtime/output differences are exact strict xfails; + an XPASS or an unlisted mismatch fails the lane. +5. XML comparison now preserves namespace prefixes while ignoring `xmlns` + declaration placement, aligning with the Node comparator. Tail text remains + significant. +6. README now documents strict xfails, deferred-writer subprocess isolation, + `Stream.close()`/context-manager behavior, bounded native-worker policy, + oversized callback input rejection, and the correct `DataWeave(lib_path)` + parameter name. +7. Restored the tracked approved design at + `docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md`. + +## TDD Evidence + +- Red: public close/context manager, oversized callback input, bounded + finalization, strict-xfail expansion, and namespace-prefix tests initially + failed against the prior implementation. +- Green: focused streaming tests passed `4 passed`; strict-xfail characterization + passed; complete unit suite passed `56 passed`. + +## Verification + +1. `./gradlew native-lib:pythonTest` + - Passed: `81 passed, 752 deselected`. +2. `./gradlew native-lib:stageTckSuites native-lib:pythonTck` + - Passed: `696 passed, 39 skipped, 79 deselected, 19 xfailed`. + - TCK totals: `selected=731`, `executed=673`, `passed=673`, `failed=0`, + `active-exclusions=39`, `xfail=19`. +3. `git diff --check` + - Passed before final report creation; rerun before commit. + +## Remaining Limitations + +- Python cannot forcibly cancel a native call. A cancelled worker that never + returns is daemonized after the bounded join, so it can retain native resources + until process exit. +- The deferred-writer TCK scenario is isolated in a subprocess because its + native isolate teardown can block. The main TCK session runtime is managed. +- Native-image continues to emit existing GraalVM deprecation and experimental + option warnings during builds. diff --git a/docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md b/docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md new file mode 100644 index 00000000..e37bb1fe --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md @@ -0,0 +1,53 @@ +# Python Binding Modernization Design + +## Goal + +Modernize the Python `dataweave` binding without changing its supported public +facade or the `dwlib` C ABI. The binding gains isolated test lanes, explicit +native lifecycle ownership, streaming support, and a master-only conformance +lane. + +## Architecture + +The `dataweave` package remains the stable facade. Public models and callback +types live in `models.py`; input/output wire conversion lives in `encoding.py`; +`native.py` owns ctypes library loading, isolate lifecycle, ABI signatures, and +native string release; `runtime.py` owns `DataWeave` orchestration. + +`DataWeave` composes one `NativeRuntime`. Module-level functions retain the +existing lazy singleton behavior. Explicit callers can use `DataWeave` as a +context manager. Native failures raise `DataWeaveError`; script failures remain +result envelopes unless the caller selects `raise_on_error`. + +## Streaming + +Both output-only and input/output streaming use one bounded queue worker. Each +native worker attaches and detaches its own isolate thread. Callback exceptions +return `-1` and never unwind across the C ABI. Stream input retains remainders +when the iterable source provides chunks larger than the native buffer. + +`Stream.close()` and its context manager request cancellation. Python cannot +forcibly interrupt a native call, so cleanup uses a short bounded join and an +unresponsive worker is daemonized; finalization never raises. Low-level callback +input larger than the supplied native buffer is rejected rather than truncated. + +## Testing And TCK + +Pytest has `unit`, `integration`, and `tck` lanes. Unit tests use fake native +collaborators; integration tests use staged `dwlib`; the TCK is on-demand and +master-only. The TCK comparator follows the Node policy, including structural +XML comparison with namespace declaration placement ignored while prefixes and +content remain significant. + +TCK skips are reserved for concrete binding/environment capabilities. Known +runtime/output deviations are case-specific strict xfails, so new mismatches +and repaired baselines are visible failures. The one deferred-writer case runs +in a subprocess because its isolate teardown may block; the shared TCK runtime +remains managed and is cleaned up at session end. + +## CI + +The Python artifact action installs test dependencies, runs normal Python tests, +builds the wheel, and runs TCK on master. The staged TCK corpus is shared with +the Node lane. Generated native libraries, wheels, corpus files, and reports are +not committed. diff --git a/native-lib/python/README.md b/native-lib/python/README.md index 23068a3b..09e3dfd2 100644 --- a/native-lib/python/README.md +++ b/native-lib/python/README.md @@ -159,13 +159,20 @@ Stream output chunks as they're produced, without buffering the entire result: import sys stream = dataweave.run_streaming("output application/json --- (1 to 10000) map {id: $}") -for chunk in stream: - sys.stdout.buffer.write(chunk) +with stream: + for chunk in stream: + sys.stdout.buffer.write(chunk) metadata = stream.metadata print(f"\nDone: {metadata.mime_type}, {metadata.charset}") ``` +Call `stream.close()` when stopping consumption early. `Stream` also supports a +context manager, as above. Closing requests cancellation and waits only briefly +for the native worker. A native call cannot be forcibly cancelled by Python, so +an unresponsive call is left to finish in a daemon worker rather than delaying +application shutdown or raising during finalization. + Or with explicit context: ```python @@ -261,6 +268,8 @@ Read callbacks return bytes and are called with the native buffer size. Return operation. Write callbacks return `0` on success. Any nonzero return value, or an exception, aborts the operation and returns unsuccessful `StreamingResult` metadata rather than unwinding a Python exception through the native callback. +Low-level read callbacks must return no more than the requested buffer size; +oversized callback data is rejected with `-1` rather than silently truncated. ## Running Tests @@ -288,9 +297,12 @@ To stage and run the Python conformance suite, use: `pythonTck` is intentionally separate from normal testing and runs only in the master-only CI lane. It reuses the corpus staged for Node TCK. It excludes -cases requiring unsupported DataWeave module resolution and other documented -environment limitations. The remaining three known conformance mismatches are -reported as failures, so this lane currently exits nonzero by design. +only binding/environment capability gaps, such as unavailable module resolution, +Java modules, and classpath test resources. Accepted runtime/output baseline +mismatches are strict xfails: a new mismatch fails the lane and a repaired +baseline mismatch XPASSes and also fails. The deferred-writer TCK scenario runs +in a subprocess because that runtime's isolate teardown may block; the main TCK +session runtime is always cleaned up. ## Running Examples @@ -340,7 +352,7 @@ Low-level callback API for advanced use cases. ### `DataWeave` Class -#### `DataWeave(library_path=None)` +#### `DataWeave(lib_path=None)` Context manager for explicit lifecycle control. @@ -380,6 +392,9 @@ Iterator that yields output chunks through a bounded queue. - `metadata: StreamingResult` - Available only after the iterator completes; check `success` and `error` after consuming the stream +**Methods:** +- `close() -> None` - Stop consuming early and request bounded worker cleanup + ### `StreamingResult` ```python diff --git a/native-lib/python/src/dataweave/models.py b/native-lib/python/src/dataweave/models.py index fd932261..ae33ceac 100644 --- a/native-lib/python/src/dataweave/models.py +++ b/native-lib/python/src/dataweave/models.py @@ -114,8 +114,26 @@ def __next__(self) -> bytes: def metadata(self) -> Optional[StreamingResult]: return self._metadata - def _close(self) -> None: + def close(self) -> None: + """Stop consuming output and request bounded worker cleanup.""" on_close = getattr(self, "_on_close", None) if on_close is not None: on_close() - self._gen.close() + try: + self._gen.close() + except Exception: + # Stream cancellation is best-effort; native calls cannot be forcibly + # interrupted from Python and cleanup must not escape finalization. + pass + + def __enter__(self): + return self + + def __exit__(self, _exc_type, _exc_val, _exc_tb): + self.close() + return False + + def __del__(self): + self.close() + + _close = close diff --git a/native-lib/python/src/dataweave/runtime.py b/native-lib/python/src/dataweave/runtime.py index 119c94d7..d7d54c4e 100644 --- a/native-lib/python/src/dataweave/runtime.py +++ b/native-lib/python/src/dataweave/runtime.py @@ -21,6 +21,7 @@ _OUTPUT_QUEUE_MAXSIZE = 512 _WORKER_TIMEOUT_SECONDS = 30 +_WORKER_JOIN_TIMEOUT_SECONDS = 0.1 class DataWeave: @@ -119,7 +120,9 @@ def worker_main(): publish(CleanupFailure(error)) publish(sentinel) - worker = Thread(target=worker_main, name="dw-streaming-worker", daemon=False) + # Python cannot cancel a native call. Daemon workers keep an abandoned + # call from extending interpreter lifetime after bounded cancellation. + worker = Thread(target=worker_main, name="dw-streaming-worker", daemon=True) worker.start() metadata = None try: @@ -128,7 +131,7 @@ def worker_main(): item = queue.get(timeout=_WORKER_TIMEOUT_SECONDS) except Empty: cancelled.set() - worker.join(timeout=_WORKER_TIMEOUT_SECONDS) + worker.join(timeout=_WORKER_JOIN_TIMEOUT_SECONDS) raise DataWeaveError(f"Worker thread timeout after {_WORKER_TIMEOUT_SECONDS} seconds") if item is sentinel: break @@ -140,9 +143,7 @@ def worker_main(): yield item finally: cancelled.set() - worker.join(timeout=_WORKER_TIMEOUT_SECONDS) - if worker.is_alive(): - raise DataWeaveError(f"Worker thread timeout after {_WORKER_TIMEOUT_SECONDS} seconds") + worker.join(timeout=_WORKER_JOIN_TIMEOUT_SECONDS) return parse_streaming_result(metadata or {"success": False, "error": "No metadata received from native call"}) def run_streaming(self, script: str, inputs: Optional[Dict[str, Any]] = None) -> Stream: @@ -200,9 +201,10 @@ def read_cb(_context, buffer, buffer_size): data = read_callback(buffer_size) if not data: return 0 - size = min(len(data), buffer_size) - ctypes.memmove(buffer, data, size) - return size + if len(data) > buffer_size: + return -1 + ctypes.memmove(buffer, data, len(data)) + return len(data) except Exception: return -1 @WRITE_CALLBACK diff --git a/native-lib/python/tests/conftest.py b/native-lib/python/tests/conftest.py index d2efc3de..91acf043 100644 --- a/native-lib/python/tests/conftest.py +++ b/native-lib/python/tests/conftest.py @@ -101,10 +101,13 @@ def clean_dataweave_runtime(request): @pytest.fixture(scope="session") def tck_runtime(): - """Own one isolate for the TCK session; deferred writers cannot be torn down per case.""" + """Own one isolate for the TCK session and release it after the lane.""" runtime = dataweave.DataWeave() runtime.initialize() - yield runtime + try: + yield runtime + finally: + runtime.cleanup() @pytest.fixture diff --git a/native-lib/python/tests/tck/compare.py b/native-lib/python/tests/tck/compare.py index 33ec5d45..0f30612d 100644 --- a/native-lib/python/tests/tck/compare.py +++ b/native-lib/python/tests/tck/compare.py @@ -1,7 +1,7 @@ """Output comparators for the formats emitted by the staged TCK corpus.""" import json -import xml.etree.ElementTree as ElementTree +from xml.dom import Node, minidom from dataclasses import dataclass from typing import Any, Optional @@ -52,28 +52,38 @@ def _compare_json(actual: str, expected: str) -> CompareResult: def _compare_xml(actual: str, expected: str) -> CompareResult: try: - actual_value = _xml_value(ElementTree.fromstring(actual)) - except ElementTree.ParseError as error: + actual_value = _xml_value(minidom.parseString(actual).documentElement) + except Exception as error: return CompareResult(False, f"actual is not valid XML: {error}") try: - expected_value = _xml_value(ElementTree.fromstring(expected)) - except ElementTree.ParseError as error: + expected_value = _xml_value(minidom.parseString(expected).documentElement) + except Exception as error: return CompareResult(False, f"expected is not valid XML: {error}") return _result(actual_value == expected_value, "XML mismatch") -def _xml_value(element: ElementTree.Element) -> Any: +def _xml_value(element) -> Any: return ( - element.tag, - tuple(sorted(element.attrib.items())), - (element.text or "").strip(), + element.tagName, + tuple(sorted( + (attribute.name, attribute.value) + for attribute in element.attributes.values() + if not attribute.name.startswith("xmlns") + )), tuple( - (_xml_value(child), (child.tail or "").strip()) - for child in element + _xml_child_value(child) + for child in element.childNodes + if child.nodeType == Node.ELEMENT_NODE or child.data.strip() ), ) +def _xml_child_value(node) -> Any: + if node.nodeType == Node.ELEMENT_NODE: + return _xml_value(node) + return ("#text", node.data.strip()) + + def _json_equal(actual: Any, expected: Any) -> bool: if isinstance(actual, str) and isinstance(expected, str): return _normalize_eol(actual) == _normalize_eol(expected) diff --git a/native-lib/python/tests/tck/ignore_list.py b/native-lib/python/tests/tck/ignore_list.py index 2a001547..9edb7cca 100644 --- a/native-lib/python/tests/tck/ignore_list.py +++ b/native-lib/python/tests/tck/ignore_list.py @@ -7,24 +7,12 @@ UNSUPPORTED_DW_MODULE_RESOLUTION = "unsupported-dw-module-resolution" UNAVAILABLE_JAVA_MODULE = "unavailable-java-module" UNAVAILABLE_CLASSPATH_TEST_RESOURCE = "unavailable-classpath-test-resource" -NONDETERMINISTIC_PROPERTIES_OUTPUT = "nondeterministic-properties-output" -MULTIPART_RUNTIME_COMPATIBILITY = "multipart-runtime-compatibility" -COERCION_RUNTIME_COMPATIBILITY = "coercion-runtime-compatibility" -DW_RUNTIME_COMPATIBILITY = "dw-runtime-compatibility" -LOCALE_DEPENDENT_OUTPUT = "locale-dependent-output" -SOURCE_LOCATION_DEPENDENT_OUTPUT = "source-location-dependent-output" SUPPORTED_CATEGORIES = frozenset( ( UNSUPPORTED_DW_MODULE_RESOLUTION, UNAVAILABLE_JAVA_MODULE, UNAVAILABLE_CLASSPATH_TEST_RESOURCE, - NONDETERMINISTIC_PROPERTIES_OUTPUT, - MULTIPART_RUNTIME_COMPATIBILITY, - COERCION_RUNTIME_COMPATIBILITY, - DW_RUNTIME_COMPATIBILITY, - LOCALE_DEPENDENT_OUTPUT, - SOURCE_LOCATION_DEPENDENT_OUTPUT, ) ) @@ -260,86 +248,6 @@ def _exclusion(case_identifier: str, category: str, reason: str) -> Exclusion: UNAVAILABLE_CLASSPATH_TEST_RESOURCE, "readUrl cannot find classpath://read_lines/test.txt", ), - "core-modules/properties-passthrough-out.properties": _exclusion( - "core-modules/properties-passthrough-out.properties", - NONDETERMINISTIC_PROPERTIES_OUTPUT, - "runtime prepends a current-date properties comment absent from the fixture", - ), - "runtime/properties-writer-out.properties": _exclusion( - "runtime/properties-writer-out.properties", - NONDETERMINISTIC_PROPERTIES_OUTPUT, - "runtime prepends a current-date properties comment and changes fixture order", - ), - "core-modules/multipart-binary-out.multipart": _exclusion( - "core-modules/multipart-binary-out.multipart", - MULTIPART_RUNTIME_COMPATIBILITY, - "multipart comparison cannot decode binary ZIP body and transfer encoding differs", - ), - "core-modules/multipart-class-cast-issue-out.multipart": _exclusion( - "core-modules/multipart-class-cast-issue-out.multipart", - MULTIPART_RUNTIME_COMPATIBILITY, - "runtime emits a multipart boundary different from the fixture", - ), - "core-modules/multipart-empty-part-out.multipart": _exclusion( - "core-modules/multipart-empty-part-out.multipart", - MULTIPART_RUNTIME_COMPATIBILITY, - "runtime emits a multipart boundary different from the fixture", - ), - "core-modules/multipart-mixed-message-out.multipart": _exclusion( - "core-modules/multipart-mixed-message-out.multipart", - MULTIPART_RUNTIME_COMPATIBILITY, - "writer rejects in0 because Multipart Object has empty parts", - ), - "core-modules/multipart-write-message-out.multipart": _exclusion( - "core-modules/multipart-write-message-out.multipart", - MULTIPART_RUNTIME_COMPATIBILITY, - "writer rejects empty multipart parts", - ), - "core-modules/multipart-write-subtype-override-out.multipart": _exclusion( - "core-modules/multipart-write-subtype-override-out.multipart", - MULTIPART_RUNTIME_COMPATIBILITY, - "writer rejects empty multipart parts", - ), - "runtime/access_raw_value-out.json": _exclusion( - "runtime/access_raw_value-out.json", - COERCION_RUNTIME_COMPATIBILITY, - "in0.^raw cannot coerce Null to String", - ), - "runtime/read-concat-out.json": _exclusion( - "runtime/read-concat-out.json", - COERCION_RUNTIME_COMPATIBILITY, - "in0.^raw cannot coerce Null to String", - ), - "runtime/update-op-out.dwl": _exclusion( - "runtime/update-op-out.dwl", - COERCION_RUNTIME_COMPATIBILITY, - "update selector attempts to coerce Null to Number", - ), - "runtime/runtime_dataFormatsDescriptors-out.json": _exclusion( - "runtime/runtime_dataFormatsDescriptors-out.json", - DW_RUNTIME_COMPATIBILITY, - "dw::Runtime reports 9 data-format descriptors while the fixture expects 10", - ), - "runtime/runtime_run-out.json": _exclusion( - "runtime/runtime_run-out.json", - DW_RUNTIME_COMPATIBILITY, - "dw::Runtime.run omits the fixture Java stream class metadata for Binary", - ), - "runtime/coerciones_toString-out.json": _exclusion( - "runtime/coerciones_toString-out.json", - LOCALE_DEPENDENT_OUTPUT, - "runtime emits locale-sensitive p. m. while the fixture requires PM", - ), - "runtime/runtime_orElseTry-out.json": _exclusion( - "runtime/runtime_orElseTry-out.json", - SOURCE_LOCATION_DEPENDENT_OUTPUT, - "dw::Runtime.orElseTry reports line 8 while the fixture reports line 9", - ), - "runtime/try-recursive-call-out.json": _exclusion( - "runtime/try-recursive-call-out.json", - SOURCE_LOCATION_DEPENDENT_OUTPUT, - "error stack embeds anonymous:15:7 instead of fixture runtime class coordinates", - ), } diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py index 99a2ea91..8cb16d33 100644 --- a/native-lib/python/tests/tck/test_conformance.py +++ b/native-lib/python/tests/tck/test_conformance.py @@ -1,3 +1,7 @@ +import base64 +import json +import os +import subprocess import sys from pathlib import Path from types import SimpleNamespace @@ -43,8 +47,26 @@ "core-modules/number-subtraction-out.json:out.json": ( "runtime emits a numeric result that differs from the accepted baseline fixture" ), + "core-modules/multipart-binary-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-class-cast-issue-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-empty-part-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-mixed-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-write-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-write-subtype-override-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/properties-passthrough-out.properties:out.properties": "properties writer output differs from the baseline fixture", + "runtime/access_raw_value-out.json:out.json": "runtime coercion output differs from the baseline fixture", + "runtime/coerciones_toString-out.json:out.json": "locale-sensitive runtime output differs from the baseline fixture", + "runtime/properties-writer-out.properties:out.properties": "properties writer output differs from the baseline fixture", + "runtime/read-concat-out.json:out.json": "runtime coercion output differs from the baseline fixture", + "runtime/runtime_dataFormatsDescriptors-out.json:out.json": "dw::Runtime output differs from the baseline fixture", + "runtime/runtime_orElseTry-out.json:out.json": "source-location runtime output differs from the baseline fixture", + "runtime/runtime_run-out.json:out.json": "dw::Runtime output differs from the baseline fixture", + "runtime/try-recursive-call-out.json:out.json": "source-location runtime output differs from the baseline fixture", + "runtime/update-op-out.dwl:out.dwl": "runtime coercion output differs from the baseline fixture", } +DEFERRED_WRITER_CASE = "core-modules/deferred-write-should-terminate-out.json:out.json" + def tck_params(): return [ @@ -62,28 +84,16 @@ def tck_params(): @pytest.mark.unit def test_accepted_baseline_mismatches_are_strict_xfails_with_reasons(): params = tck_params() - expected = { - "core-modules/csv-invalid-utf8-out.csv:out.csv": ( - "runtime emits a replacement character where the fixture expects an empty CSV value" - ), - "core-modules/number-addition-out.json:out.json": ( - "runtime emits a numeric result that differs from the accepted baseline fixture" - ), - "core-modules/number-subtraction-out.json:out.json": ( - "runtime emits a numeric result that differs from the accepted baseline fixture" - ), - } - xfails = { parameter.id: next(mark for mark in parameter.marks if mark.name == "xfail") for parameter in params if any(mark.name == "xfail" for mark in parameter.marks) } - assert set(xfails) == set(expected) - for identifier, reason in expected.items(): + assert set(xfails) == set(ACCEPTED_BASELINE_MISMATCHES) + for identifier, mark in xfails.items(): assert xfails[identifier].kwargs["strict"] is True - assert xfails[identifier].kwargs["reason"] == reason + assert xfails[identifier].kwargs["reason"] == ACCEPTED_BASELINE_MISMATCHES[identifier] @pytest.mark.unit @@ -117,19 +127,24 @@ def test_tck_scenario(scenario, tck_runtime): if exclusion: pytest.skip(f"{exclusion.category}: {exclusion.reason}") - result = tck_runtime.run(scenario.transform, scenario.inputs) - assert result.success, result.error + if scenario.identifier == DEFERRED_WRITER_CASE: + success, error, output = _run_deferred_writer_in_subprocess(scenario) + assert success, error + else: + result = tck_runtime.run(scenario.transform, scenario.inputs) + assert result.success, result.error + output = result.get_bytes() comparison = compare_output( scenario.output_extension, - result.get_bytes(), + output, scenario.expected, scenario.charset, ) assert comparison.match, comparison.detail -def test_tck_session_runtime_runs_after_deferred_write_without_teardown(tck_runtime): - """Catches per-scenario isolate cleanup after a deferred writer stalls the lane.""" +def test_tck_session_runtime_runs_after_subprocess_deferred_write(tck_runtime): + """Deferred writers must not prevent the managed TCK runtime from continuing.""" deferred = next( scenario for scenario in SCENARIOS @@ -137,14 +152,57 @@ def test_tck_session_runtime_runs_after_deferred_write_without_teardown(tck_runt == "core-modules/deferred-write-should-terminate-out.json:out.json" ) - deferred_result = tck_runtime.run(deferred.transform, deferred.inputs) + success, error, _output = _run_deferred_writer_in_subprocess(deferred) following_result = tck_runtime.run("%dw 2.0\noutput application/json\n--- 1") - assert deferred_result.success, deferred_result.error + assert success, error assert following_result.success, following_result.error assert following_result.get_bytes() == b"1" +def _run_deferred_writer_in_subprocess(scenario): + """Contain the native deferred writer whose isolate teardown can block.""" + source_dir = Path(__file__).resolve().parents[2] / "src" + inputs = { + name: { + "content": base64.b64encode(value.content if isinstance(value.content, bytes) else value.content.encode()).decode(), + "mime_type": value.mime_type, + "charset": value.charset, + "properties": value.properties, + } + for name, value in scenario.inputs.items() + } + code = """ +import base64 +import json +import sys +from dataweave import DataWeave, InputValue + +script, inputs_json = sys.argv[1:] +inputs = { + name: InputValue(base64.b64decode(value['content']), value['mime_type'], value['charset'], value['properties']) + for name, value in json.loads(inputs_json).items() +} +runtime = DataWeave() +runtime.initialize() +result = runtime.run(script, inputs) +print(json.dumps({'success': result.success, 'error': result.error, 'result': result.result})) +""" + environment = os.environ.copy() + environment["PYTHONPATH"] = str(source_dir) + os.pathsep + environment.get("PYTHONPATH", "") + completed = subprocess.run( + [sys.executable, "-c", code, scenario.transform, json.dumps(inputs)], + capture_output=True, + check=False, + env=environment, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr + response = json.loads(completed.stdout) + return response["success"], response["error"], base64.b64decode(response["result"] or "") + + def write_case(root: Path, name: str, files: Dict[str, Union[bytes, str]]) -> Path: case = root / "runtime" / name case.mkdir(parents=True) @@ -227,6 +285,17 @@ def test_compare_output_rejects_different_xml_tail_text(): assert not result.match +def test_compare_output_rejects_different_namespace_prefixes(): + """Matches Node's policy: declaration placement is ignored, prefixes are not.""" + result = compare_output( + "xml", + b'1', + b'1', + ) + + assert not result.match + + def test_exclusion_registry_requires_category_and_reason(): """Catches exclusions that cannot be audited by category and rationale.""" errors = validate_exclusions( @@ -289,7 +358,7 @@ def test_only_declared_case_identifiers_are_excluded(): exclusion = exclusion_for("runtime/import-lib-out.json") assert exclusion.case_identifier == "runtime/import-lib-out.json" assert exclusion.category == "unsupported-dw-module-resolution" - assert len(EXCLUDED_CASES) == 55 + assert len(EXCLUDED_CASES) == 39 def test_exclusion_registry_uses_the_inventory_categories(): @@ -299,12 +368,6 @@ def test_exclusion_registry_uses_the_inventory_categories(): categories[exclusion.category] = categories.get(exclusion.category, 0) + 1 assert categories == { - "coercion-runtime-compatibility": 3, - "dw-runtime-compatibility": 2, - "locale-dependent-output": 1, - "multipart-runtime-compatibility": 6, - "nondeterministic-properties-output": 2, - "source-location-dependent-output": 2, "unavailable-classpath-test-resource": 4, "unavailable-java-module": 11, "unsupported-dw-module-resolution": 24, diff --git a/native-lib/python/tests/unit/test_models.py b/native-lib/python/tests/unit/test_models.py index 2e678e2b..b337896d 100644 --- a/native-lib/python/tests/unit/test_models.py +++ b/native-lib/python/tests/unit/test_models.py @@ -62,3 +62,23 @@ def test_parse_native_response_preserves_error_result(): result = dataweave._parse_native_encoded_response('{"success": false, "error": "script failed"}') assert result == dataweave.ExecutionResult(False, None, "script failed", False, None, None) + + +@pytest.mark.unit +def test_stream_public_close_and_context_manager_close_the_underlying_generator(): + closed = [] + + def generate(): + try: + yield b"chunk" + finally: + closed.append(True) + + stream = dataweave.Stream(generate()) + + with stream as managed: + assert next(managed) == b"chunk" + + stream.close() + + assert closed == [True] diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index 405e7b78..d60a9b95 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -141,7 +141,7 @@ def test_run_streaming_returns_attach_failure_without_detaching_unattached_threa @pytest.mark.unit -def test_stream_private_close_aborts_worker_and_detaches_after_consumer_abandons_output(): +def test_stream_public_close_aborts_worker_and_detaches_after_consumer_abandons_output(): class BlockingFakeNative(FakeNative): def __init__(self): super().__init__('{"success": false, "error": "aborted"}') @@ -164,12 +164,35 @@ def run_script_callback(self, _thread, _script, _inputs, write_callback, _contex assert next(stream) == b"first" assert native.first_chunk_written.wait(1) - stream._close() + stream.close() assert native.detached_event.wait(1) assert native.write_status == -1 +@pytest.mark.unit +def test_run_input_output_callback_rejects_oversized_read_chunk_without_truncating(): + class OversizedInputNative(FakeNative): + def run_script_input_output_callback( + self, _thread, _script, _inputs, _input_name, _mime_type, _charset, read_callback, _write_callback, _context, + ): + buffer = ctypes.create_string_buffer(3) + self.read_status = read_callback(None, ctypes.addressof(buffer), len(buffer)) + self.read_input = bytes(buffer.raw[:max(self.read_status, 0)]) + return self._response_pointer() + + native = OversizedInputNative('{"success": false, "error": "read aborted"}') + runtime = configured_runtime(native) + + result = runtime.run_input_output_callback( + "script", "payload", "application/json", lambda _size: b"oversized", lambda _data: 0, + ) + + assert result == dataweave.StreamingResult(False, "read aborted", None, None, False) + assert native.read_status == -1 + assert native.read_input == b"" + + @pytest.mark.unit def test_stream_early_close_does_not_block_terminal_publication_on_full_queue(monkeypatch): cancelled = [] @@ -236,6 +259,20 @@ def run_script_callback(self, _thread, _script, _inputs, _write_callback, _conte list(runtime.run_streaming("script")) +@pytest.mark.unit +def test_stream_finalization_does_not_raise_when_a_native_worker_cannot_cancel(monkeypatch): + class UncancellableNative(FakeNative): + def run_script_callback(self, _thread, _script, _inputs, _write_callback, _context): + sleep(0.1) + return self._response_pointer() + + monkeypatch.setattr(runtime_module, "_WORKER_JOIN_TIMEOUT_SECONDS", 0.001) + runtime = configured_runtime(UncancellableNative('{"success": true}')) + stream = runtime.run_streaming("script") + + stream.close() + + @pytest.mark.unit def test_run_streaming_surfaces_detach_failure_without_primary_execution_failure(): class DetachFailingNative(FakeNative): From 9cc94bbc90df559498e3006392b11f76e034c1b0 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 11:43:15 -0300 Subject: [PATCH 28/30] fix: keep TCK metadata validation in TCK lane --- native-lib/python/tests/tck/test_conformance.py | 1 - native-lib/python/tests/unit/test_ci_structure.py | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py index 8cb16d33..b6db68cc 100644 --- a/native-lib/python/tests/tck/test_conformance.py +++ b/native-lib/python/tests/tck/test_conformance.py @@ -81,7 +81,6 @@ def tck_params(): ] -@pytest.mark.unit def test_accepted_baseline_mismatches_are_strict_xfails_with_reasons(): params = tck_params() xfails = { diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index ae19f5bd..3137e977 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -69,6 +69,14 @@ def test_master_tck_stages_the_shared_corpus_once_before_python_and_node(): assert workflow.index("Stage TCK corpus") < workflow.index("- name: Node") +@pytest.mark.unit +def test_tck_metadata_validation_is_not_selected_by_the_pr_python_test_lane(): + root = Path(__file__).resolve().parents[4] + conformance = (root / "native-lib/python/tests/tck/test_conformance.py").read_text() + + assert "@pytest.mark.unit\ndef test_accepted_baseline_mismatches" not in conformance + + @pytest.mark.unit def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): root = Path(__file__).resolve().parents[4] From 730d5ded3529b8765dc1285a0e06a886217b76a7 Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 11:48:15 -0300 Subject: [PATCH 29/30] chore: remove tracked superpowers reports --- .../final-fix-report.md | 57 ------- .../task-1-report.md | 92 ----------- .../task-2-report.md | 104 ------------ .../task-3-report.md | 56 ------- .../task-4-report.md | 56 ------- .../task-5-report.md | 149 ------------------ .../task-6-report.md | 147 ----------------- 7 files changed, 661 deletions(-) delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md delete mode 100644 .superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md deleted file mode 100644 index 56de06fe..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/final-fix-report.md +++ /dev/null @@ -1,57 +0,0 @@ -# Final Review Remediation Report - -## Resolutions - -1. `Stream.close()` is now public and `Stream` supports `with`. Closing sets - cancellation and generator cleanup is non-raising. Native workers use a - bounded 0.1-second join and are daemon threads because Python cannot cancel - an in-progress native call. Public early-close and uncancellable-worker - finalization regression tests cover this policy. -2. The low-level read callback now rejects data larger than the native buffer - with `-1`; it no longer silently truncates callback input. The iterable - transform path continues preserving chunk remainders. -3. The TCK session fixture now always calls `runtime.cleanup()`. Only the - deferred-writer scenario executes in a subprocess, preventing its known - isolate-teardown stall from leaving the session runtime unmanaged. -4. TCK skips now contain only 39 binding/environment capability cases: 24 - module-resolution, 11 unavailable Java-module, and 4 unavailable classpath - resource cases. The 19 runtime/output differences are exact strict xfails; - an XPASS or an unlisted mismatch fails the lane. -5. XML comparison now preserves namespace prefixes while ignoring `xmlns` - declaration placement, aligning with the Node comparator. Tail text remains - significant. -6. README now documents strict xfails, deferred-writer subprocess isolation, - `Stream.close()`/context-manager behavior, bounded native-worker policy, - oversized callback input rejection, and the correct `DataWeave(lib_path)` - parameter name. -7. Restored the tracked approved design at - `docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md`. - -## TDD Evidence - -- Red: public close/context manager, oversized callback input, bounded - finalization, strict-xfail expansion, and namespace-prefix tests initially - failed against the prior implementation. -- Green: focused streaming tests passed `4 passed`; strict-xfail characterization - passed; complete unit suite passed `56 passed`. - -## Verification - -1. `./gradlew native-lib:pythonTest` - - Passed: `81 passed, 752 deselected`. -2. `./gradlew native-lib:stageTckSuites native-lib:pythonTck` - - Passed: `696 passed, 39 skipped, 79 deselected, 19 xfailed`. - - TCK totals: `selected=731`, `executed=673`, `passed=673`, `failed=0`, - `active-exclusions=39`, `xfail=19`. -3. `git diff --check` - - Passed before final report creation; rerun before commit. - -## Remaining Limitations - -- Python cannot forcibly cancel a native call. A cancelled worker that never - returns is daemonized after the bounded join, so it can retain native resources - until process exit. -- The deferred-writer TCK scenario is isolated in a subprocess because its - native isolate teardown can block. The main TCK session runtime is managed. -- Native-image continues to emit existing GraalVM deprecation and experimental - option warnings during builds. diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md deleted file mode 100644 index ce69818c..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-1-report.md +++ /dev/null @@ -1,92 +0,0 @@ -# Task 1 Report: Pytest Lanes And Existing Behavior - -## Files Changed - -- `native-lib/python/pytest.ini`: registers `unit`, `integration`, and `tck` markers; excludes TCK tests by default. -- `native-lib/python/pyproject.toml`: declares the `test` optional dependency group with `pytest` and `pytest-cov`. -- `native-lib/python/tests/conftest.py`: adds source-path setup, automatic module-global runtime cleanup, and a streaming collection fixture. -- `native-lib/python/tests/integration/test_execution.py`: migrates five execution and input-conversion scenarios. -- `native-lib/python/tests/integration/test_streaming.py`: migrates eight streaming and transform scenarios. -- `native-lib/python/tests/integration/test_callbacks.py`: migrates four callback streaming scenarios. -- `native-lib/python/tests/integration/test_lifecycle.py`: migrates the explicit lifecycle scenario. -- `native-lib/python/tests/test_dataweave_module.py`: removed the superseded hand-run script. -- `native-lib/build.gradle`: makes `pythonTest` run normal pytest lanes and write JUnit plus coverage XML under `native-lib/build`. - -## Design Choices - -- Retained all 18 legacy scenarios as separately named `@pytest.mark.integration` tests, grouped by execution mode. -- Used an autouse fixture to call `dataweave.cleanup()` before and after every test, isolating the module-level native runtime without changing public runtime behavior or the C ABI. -- Kept TCK excluded from default pytest collection and made Gradle explicitly run only `unit or integration` markers. -- Wrote test reports to `native-lib/build/test-results/python/junit.xml` and coverage to `native-lib/build/reports/coverage/python/coverage.xml`. - -## Tests Run - -1. `/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python -m pip install '.[test]'` - - Passed. Installed `pytest 8.4.2` and `pytest-cov 7.1.0` in an isolated temporary virtual environment. -2. `/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python -m pytest tests/integration -m integration -v` - - Passed: `18 passed in 0.39s`. -3. `./gradlew native-lib:pythonTest -PpythonExe=/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python` - - Passed: native `dwlib` compiled and staged; pytest reported `18 passed in 1.04s`; JUnit and coverage XML were written to the configured build paths. -4. `pytest -m 'not tck' --collect-only -q` using the temporary virtual environment - - Passed: 18 tests collected, confirming default TCK exclusion. -5. `git diff --check` - - Passed. - -## Commit - -- `9cf376d test: migrate Python binding checks to pytest` - -## Concerns - -- The system Python does not have pytest and cannot write its global site-packages. Verification used an isolated temporary virtual environment supplied through Gradle's existing `-PpythonExe` override. -- Native-image and Gradle emit existing Java/native-image deprecation warnings during the native build; the task itself completed successfully. - -## Review Fixes - -### Files Changed - -- `.github/actions/build-foundation/action.yml`: installs `native-lib/python`'s `test` optional dependency group before the foundation build invokes `pythonTest`. -- `native-lib/build.gradle`: declares Python tests and pytest configuration as `pythonTest` inputs so test additions invalidate Gradle's up-to-date state. -- `native-lib/python/tests/integration/test_callbacks.py`: adds three integration scenarios covering exceptions in output-only write callbacks and input/output read and write callbacks. -- `native-lib/python/README.md`: documents installing `.[test]`, direct pytest integration execution, and lane marker behavior. - -### Design Choices - -- Callback implementations already catch Python exceptions and return `-1`, which the native APIs translate into unsuccessful `StreamingResult` metadata. The new tests characterize that established public behavior without changing runtime code or the native ABI. -- The CI dependency installation belongs in `build-foundation` because its `build` command can trigger `native-lib:pythonTest`; per the review ruling, no caller-level provisioning is required. -- The Gradle input declaration was added after observing that `pythonTest` remained up-to-date after a test-only edit. This prevents future test changes from being silently skipped. - -### Tests Run - -1. `/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python -m pytest tests/integration/test_callbacks.py -m integration -v` - - Passed: `7 passed in 1.16s`, including callback exception containment cases. -2. `./gradlew native-lib:pythonTest -PpythonExe=/var/folders/n2/069kfxz14k3dg0dctt0gblm80000gn/T/opencode/dataweave-python-test/bin/python` - - Passed: native library compiled and staged; pytest reported `21 passed in 2.08s`; JUnit and coverage XML were regenerated beneath `native-lib/build`. - -### Concerns - -- The CI action change was reviewed structurally but not executed in GitHub Actions from this local worktree. -- Native-image emits existing Java/native-image deprecation warnings during local Gradle execution. - -## Review Fix Round 2 - -### Files Changed - -- `.github/actions/build-foundation/action.yml`: conditionally adds `--break-system-packages` when `runner.environment` is `github-hosted`, while retaining the unmodified pip invocation for self-hosted runners. - -### Design Choice - -- Matched the existing Python artifact action's conditional flag pattern. This addresses PEP 668 on GitHub-hosted macOS without imposing `--break-system-packages` on self-hosted MuleSoft runners. - -### Checks Run - -1. `ruby -e "require 'yaml'; YAML.load_file('.github/actions/build-foundation/action.yml'); puts 'YAML valid'"` - - Passed: YAML parsed successfully. -2. `git diff --check` - - Passed: no whitespace errors. -3. Compared the condition against `.github/actions/python/action.yml`. - - Confirmed the existing action uses the same GitHub Actions expression form for conditional pip flags. - -### Concerns - -- The GitHub-hosted runner expression cannot be executed locally; validation is structural and follows the repository's established Python action convention. diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md deleted file mode 100644 index cf70742e..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-2-report.md +++ /dev/null @@ -1,104 +0,0 @@ -# Task 2 Report: Python Binding Unit Characterization Coverage - -## Changed Files - -- `native-lib/python/src/dataweave/__init__.py` - - Replaced dynamic `__import__("os")` environment access with a normal private module import. Added private streaming cancellation plumbing so abandoned streaming consumers cause future native write callbacks to abort and the worker is joined/detached. Terminal metadata and sentinel publication now stop retrying after cancellation, preventing a full abandoned output queue from blocking worker shutdown. Public API and native ABI are unchanged. -- `native-lib/python/tests/unit/test_models.py` - - Characterizes `InputValue` encoding and `ExecutionResult` decoding/error behavior. -- `native-lib/python/tests/unit/test_encoding.py` - - Characterizes implicit text encoding and explicit-input validation/metadata preservation. -- `native-lib/python/tests/unit/test_native.py` - - Characterizes native response decoding, malformed input handling, candidate library path priority, and native-string cleanup on decoding failure. -- `native-lib/python/tests/unit/test_streaming.py` - - Characterizes callback exception-to-abort conversion, including observed read callback abort status; large input chunk remainders; true worker metadata absence; attach failure; safe early consumer abandonment with prompt worker detachment; and full-queue terminal shutdown using fake native collaborators. -- `native-lib/python/tests/unit/test_facade.py` - - Characterizes module singleton initialization/recreation and global cleanup. - -## Implementation Details - -- Added 24 `@pytest.mark.unit` tests that import the Python package from source and never instantiate a staged `dwlib`. -- Fake native collaborators exercise ctypes callback boundaries without a native shared library. They verify callback exceptions return the documented nonzero abort status rather than escaping C callbacks. -- Streaming generators now use an internal cancellation event. The private `_close()` test seam sets cancellation before closing the generator; cancellation makes subsequent native write callbacks return `-1`, and generator cleanup joins the worker while the worker's `finally` detaches its isolate thread. -- Terminal queue publication retries with a short timeout only while a consumer remains active. Once cancellation is set, it stops instead of blocking forever on a full queue, allowing the native worker to reach its detach `finally` block. -- `DataWeave._decode_and_free` is exercised through a failing UTF-8 decode to verify native strings are freed from its existing `finally` block. -- The only source adjustment replaces dynamic standard-library import resolution with a normal `os` import; no public names, signatures, or ABI fields changed. - -## Commands And Output - -1. Initial required unit command before pytest was installed: - - ```text - python3 -m pytest tests/unit -m unit -v - /Library/Developer/CommandLineTools/usr/bin/python3: No module named pytest - ``` - -2. After installing local test dependencies, the initial test-first run collected 25 tests: 21 passed and 4 failed. The expected failures identified the existing dict-explicit-input behavior, callback abort status, empty-native-response metadata, and missing `Stream.close` API. Tests were refined to characterize the existing behavior rather than alter public API. - -3. Final unit verification: - - ```text - python3 -m pytest tests/unit -m unit -v - 24 passed in 0.02s - ``` - -4. Syntax verification: - - ```text - python3 -m compileall -q src tests/unit - exit 0 - ``` - -5. Diff whitespace verification: - - ```text - git diff --check - exit 0 - ``` - -6. Gradle task configuration check: - - ```text - ./gradlew native-lib:pythonTest --dry-run - BUILD SUCCESSFUL - ``` - -7. Fix round 1 focused verification: - - ```text - python3 -m pytest tests/unit/test_streaming.py -m unit -v - 6 passed in 0.02s - ``` - -8. Fix round 1 full unit verification: - - ```text - python3 -m pytest tests/unit -m unit -v - 24 passed in 0.03s - python3 -m compileall -q src tests/unit - git diff --check - exit 0 - ``` - -9. Fix round 2 focused and full unit verification: - - ```text - python3 -m pytest tests/unit/test_streaming.py -m unit -v - 7 passed in 0.01s - python3 -m pytest tests/unit -m unit -v - 25 passed in 0.02s - python3 -m compileall -q src tests/unit - git diff --check - exit 0 - ``` - -## Commit - -- `2f6a788 test: add Python binding unit characterization coverage` -- `2f169ee test: harden Python streaming unit coverage` -- `bcf6ccd fix: prevent Python streaming worker shutdown stalls` - -## Concerns - -- None for Task 2 scope. -- The local Python installation initially lacked pytest. It was installed in the user site to execute the required unit lane; this did not alter tracked project files. diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md deleted file mode 100644 index aa71369e..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-3-report.md +++ /dev/null @@ -1,56 +0,0 @@ -# Task 3 Report: Extract Models And Encoding - -## Files Changed - -- Added `native-lib/python/src/dataweave/models.py` for public models, exceptions, callback aliases, ctypes callback signatures, and `Stream`. -- Added `native-lib/python/src/dataweave/encoding.py` for input normalization and native response parsing. -- Updated `native-lib/python/src/dataweave/__init__.py` to re-export the legacy public API and delegate internal model/encoding operations to the new modules. -- Updated `native-lib/python/tests/unit/test_facade.py`, `test_models.py`, and `test_encoding.py` with facade/export compatibility coverage. -- Updated `native-lib/python/pytest.ini` to use pytest importlib import mode, avoiding same-basename test-module collection collisions between unit and integration lanes. - -## Behavior Preservation - -- `dataweave.__all__`, module-level public names, and existing call signatures remain unchanged. -- `ExecutionResult.get_bytes()` and `ExecutionResult.get_string()` retain their prior base64, binary, charset, and unsuccessful-result behavior. -- The public models, exceptions, callback type aliases, and ctypes callback signatures are now also available from `dataweave.models`. -- `normalize_input_value`, `parse_native_encoded_response`, and `parse_streaming_result` are now public from `dataweave.encoding`; legacy private facade aliases remain for existing internal callers and tests. -- Native wire keys remain unchanged: `mimeType`, `charset`, `binary`, `result`, and `error`. -- No native/runtime extraction was performed. - -## Verification - -Command: - -```bash -cd native-lib/python -python3 -m pytest tests/unit tests/integration -m "unit or integration" -v -``` - -Output: - -```text -49 passed in 1.46s -``` - -Additional check: - -```bash -git diff --check -``` - -Output: no whitespace errors. - -## Commit - -`1276d12 refactor: separate Python binding models and encoding` - -## Concerns - -- The test tree has same-basename unit and integration modules (`test_streaming.py`). Pytest’s default prepend import mode causes collection to fail; `--import-mode=importlib` is now configured so the required combined test command runs consistently. - -## Fix Round 1 - -- Replaced the dynamic facade export assertion with a fixed, pre-Task-3 list of all 18 legacy public names. -- The test now verifies every legacy name is explicitly present in `dataweave.__all__` and resolves through `getattr(dataweave, name)`. -- Focused verification: `python3 -m pytest tests/unit/test_facade.py -m unit -v` reported `3 passed in 0.01s`. -- Combined verification and fix-round commit are recorded in the follow-up commit for this round. diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md deleted file mode 100644 index a68398b4..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-4-report.md +++ /dev/null @@ -1,56 +0,0 @@ -# Task 4 Report: Native Adapter And Runtime Streaming - -## Status - -Complete. The Python binding now separates native ctypes/isolate ownership from the public `DataWeave` orchestration API. - -## Changes - -- Added `dataweave.native.NativeRuntime` for native library discovery/loading, opaque Graal pointer types, ABI registration, isolate/thread lifecycle, native invocation, and C-string release. -- Added `dataweave.runtime.DataWeave` for buffered execution, direct callbacks, and both streaming APIs. -- Consolidated output-only and duplex streaming onto one worker implementation with bounded queue backpressure, cancellation, timed consumer waits, worker join timeout, metadata propagation, and detach in `finally`. -- Preserved oversized input chunk remainders and Python callback failure translation to the native `-1` abort status. -- Reduced `dataweave.__init__` to the public facade, singleton lifecycle, re-exports, and legacy private helper aliases. -- Added unit coverage for native ABI registration, load failure, idempotent cleanup, module ownership, and worker timeout behavior. - -## Verification - -- `python3 -m pytest tests/unit tests/integration` passed: 53 tests. -- `./gradlew native-lib:pythonTest` passed: native image build and 53 Python tests. - -## Concerns - -- The Gradle native-image invocation emits existing Graal/Gradle deprecation and restricted-native-access warnings; it still completed successfully. -- No Task 5, documentation, or CI work was included. - -## Fix Round 1 - -- Made initialization transactional: failed isolate creation or ABI validation now resets all native state; after isolate creation, validation failure attempts isolate teardown before reset and reraises the primary failure. -- Required `run_script`, `free_cstring`, and isolate teardown exports at initialization. Streaming callback exports additionally require attach/detach exports before the runtime is considered initialized. -- Validated isolate teardown and worker detach return codes. Cleanup failures now surface as `DataWeaveError`; worker detach failures surface only when there was no preceding execution failure. -- Removed obsolete `DataWeave` private-state forwarding and test-only Graal setup compatibility machinery. Tests now configure `NativeRuntime` directly. -- Added focused coverage for failed isolate creation, partial-initialization cleanup, missing required exports, teardown return codes, and detach return codes. - -### Fix Round 1 Verification - -- Final focused and full Python verification passed: 25 focused lifecycle/streaming tests and 61 unit/integration tests. -- `./gradlew native-lib:pythonTest` passed: native image build and all 61 Python tests. - -## Fix Round 2 - -- Treated unsuccessful native streaming metadata as the primary execution outcome, so worker detach failures do not replace a script failure. -- Preserved an active context-manager body exception when cleanup also fails; cleanup failures still surface when no body exception is active. -- Cleared the facade singleton in a `finally` block, allowing reinitialization after native cleanup raises. -- Validated `graal_create_isolate` is exported and wrapped its invocation failure as contextual `DataWeaveError`. -- Wrapped `graal_attach_thread` and `graal_detach_thread` invocation failures as contextual `DataWeaveError`; the stream worker suppresses detach failures whenever attach, execution, decode, or native failure metadata is primary. -- Added focused tests for missing/throwing isolate creation, throwing attach/detach lifecycle calls, failed singleton cleanup, context cleanup precedence, and unsuccessful streaming metadata plus detach failure. - -### Fix Round 2 Verification - -- Focused lifecycle/native/streaming/facade tests passed: 38 tests. -- Full Python unit and integration suite passed: 70 tests. -- `./gradlew native-lib:pythonTest` passed: native image build and all 70 Python tests. - -### Fix Round 2 Concerns - -- The Gradle native-image build continues to emit pre-existing Graal/Gradle deprecation and restricted-native-access warnings, but the build and Python suite completed successfully. diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md deleted file mode 100644 index cc262caa..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-5-report.md +++ /dev/null @@ -1,149 +0,0 @@ -# Task 5 Report: Python TCK Harness - -## Files - -- `native-lib/build.gradle` -- `native-lib/python/tests/__init__.py` -- `native-lib/python/tests/conftest.py` -- `native-lib/python/tests/tck/__init__.py` -- `native-lib/python/tests/tck/case_loader.py` -- `native-lib/python/tests/tck/compare.py` -- `native-lib/python/tests/tck/ignore_list.py` -- `native-lib/python/tests/tck/test_conformance.py` - -## TDD And Test Commands - -1. RED: `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'module_resolution_exclusions_only_skip_importing_cases' -vv` - Result: failed as expected because `validate_exclusions` did not accept or validate staged scenarios. -2. GREEN: `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'discover or compare or exclusion or tck_summary' -vv` - Result: 14 passed, 732 deselected. -3. Exclusion visibility: `python3 -m pytest -m tck -k 'import-lib-out or import-star-out' -vv` - Result: 2 skipped, with header `discovered=731`, `structural-skips=191`, and `categorized-exclusions=6 (module-resolution-not-supported=6)`. -4. Non-excluded failure guard: `python3 -m pytest -m tck -x -vv` - Result: 59 passed, then `core-modules/csv-invalid-utf8-out.csv` failed with `text mismatch`; the harness exited nonzero. The terminal report showed `passed=59, failed=1`. -5. Corpus staging: `./gradlew native-lib:stageTckSuites` - Result: passed; staged the resolved `runtime` and `core-modules` artifacts under the existing Node TCK corpus directory. -6. Gradle lane: `./gradlew native-lib:stageTckSuites native-lib:pythonTck` - Result: invoked native staging, the shared corpus staging task, and pytest. The initial complete run exceeded the local command time limit after reaching the corpus tests. A subsequent direct pytest run established the required non-excluded failure behavior above. - -## Exclusions Discovered - -- 19 staged runtime corpus cases import test-only DW modules. -- Every exclusion is keyed by full suite/case identifier and categorized as `module-resolution-not-supported` with the explicit reason that the Python binding has no module resolver. -- Registry validation rejects missing category/reason and rejects this category for a discovered case whose transform does not contain a DW `import` directive. - -## Commit - -- `b285d21 test: add Python TCK conformance lane` - -## Concerns - -- The current staged corpus has a real, non-excluded failure in `core-modules/csv-invalid-utf8-out.csv`: the runtime emits a replacement character where the fixture expects an empty CSV value. It is intentionally not excluded so `pythonTck` remains a failing conformance gate. -- The original full run did not reach terminal reporting because the generic - per-test cleanup could block after a successful deferred writer. This is a - lifecycle stall, not aggregate runtime-initialization cost; Fix Round 1 - replaces that lifecycle for TCK scenarios. - -## Fix Round 1 - -### Lifecycle And Regression Coverage - -- The TCK lane now owns one session-scoped `DataWeave` instance. The generic - module-level runtime cleanup fixture is bypassed for all `tck`-marked tests, - so a deferred writer is not followed by per-scenario isolate destruction. -- The session runtime deliberately remains process-scoped after pytest's - terminal summary. Direct isolate teardown after - `deferred-write-should-terminate-out.json` blocks in Graal; attempting it at - session teardown would again prevent pytest from reaching its report. Python - process exit owns final release of this TCK-only isolated runtime. -- RED: the deferred-write regression failed with `fixture 'tck_runtime' not - found`. GREEN: it runs the actual deferred-write corpus transform and then a - second transform successfully using the same runtime. - -### Exclusions And Reporting - -- The active registry contains exactly 18 runnable module-import scenarios: - the original six plus the 12 observed unresolved `dw::Client`/`dw::Natives` - cases. Registry validation now rejects stale or structurally unreachable - entries. -- The 13 former registry entries which are loader `transform-shape` structural - skips are not active exclusions. They are reported separately as - `structural-module-cases=13`. -- The terminal report now reconciles scenario-only totals: - `selected=731`, `executed=713`, `active-exclusions=18`, `passed=673`, and - `failed=40`. Thus `executed + active-exclusions == selected` and - `passed + failed == executed`. -- XML comparison retains child tail text, so structural comparison no longer - treats `actual` and `expected` as equivalent. - -### Verification - -1. `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'deferred-write-should-terminate or is-empty-using-empty-stream or streaming_binary_inside_value or try-handle' -vv` - Result: 2 passed, 12 skipped. The deferred-write case completed and each of - the 12 newly active unresolved-module exclusions skipped as categorized. -2. `python3 -m pytest tests/tck/test_conformance.py -m tck -k 'discover or compare or exclusion or tck_summary or tck_session_runtime' -vv` - Result: 18 passed. -3. `./gradlew native-lib:stageTckSuites native-lib:pythonTck` - Result: terminal pytest report was reached in 34.87 seconds. It reported - `selected=731, structural-skips=191, structural-module-cases=13, - executed=713, active-exclusions=18, passed=673, failed=40`, then Gradle - failed as expected because non-excluded conformance mismatches remain. - `core-modules/csv-invalid-utf8-out.csv` remains a reported text mismatch and - was not excluded. The stage task continues to reuse the Node TCK artifacts; - no duplicate suite download was introduced. - -### Updated Concern - -- `pythonTck` is intentionally still a failing conformance gate for the 40 - active, non-excluded mismatches. This round fixes the deferred-writer - teardown stall and reporting attribution; it does not claim to resolve those - runtime/output compatibility failures. - -## Fix Round 2 - -### Evidence-Backed Exclusions - -- Read the reviewed `task-5-failure-inventory.md` before changing the registry. - Its 40 observed failures contained 37 unsupported environment/runtime cases - and three retained conformance mismatches. -- Replaced the generic module-only registry shape with per-case `Exclusion` - records. Every active record carries its full case identifier, a reason with - the direct observed limitation, and one of the nine approved categories. -- Added 37 case-specific entries for the inventory evidence. Together with the - pre-existing 18 unresolved-import exclusions, the active registry now has 55 - entries: - `unsupported-dw-module-resolution=24`, `unavailable-java-module=11`, - `unavailable-classpath-test-resource=4`, - `nondeterministic-properties-output=2`, `multipart-runtime-compatibility=6`, - `coercion-runtime-compatibility=3`, `dw-runtime-compatibility=2`, - `locale-dependent-output=1`, and `source-location-dependent-output=2`. -- Registry validation now rejects missing or mismatched case identity, blank - category/reason fields, unsupported categories, and entries that cannot - affect a discovered runnable scenario. The category-count test locks the - inventory reconciliation and prevents a later category collapse. - -### Retained Conformance Mismatches - -- `core-modules/csv-invalid-utf8-out.csv:out.csv` remains active: the runtime - emits a replacement character where the fixture requires an empty CSV value. -- `core-modules/number-addition-out.json:out.json` remains active: numeric - serialization differs from the fixture's exact integer representation. -- `core-modules/number-subtraction-out.json:out.json` remains active: numeric - serialization produces `0` where the fixture requires `0.0`. - -### Verification - -1. RED: `python3 -m pytest tests/tck/test_conformance.py -m tck -k - exclusion_registry_requires_case_identity_supported_category_and_reason -vv` - Result: failed as expected before the registry change because it did not - validate case identity, approved category, or nonblank reason. -2. Focused registry validation: `python3 -m pytest - tests/tck/test_conformance.py -m tck -k exclusion -vv` - Result: 4 passed. The header reported all 55 active exclusions by category. -3. Full terminal TCK: `python3 -m pytest tests/tck/test_conformance.py -m tck - -vv --maxfail=0` - Result: 3 failed, 693 passed, 55 skipped in 32.38s. The reconciled terminal - totals were `selected=731`, `structural-skips=191`, - `structural-module-cases=13`, `executed=676`, `active-exclusions=55`, - `passed=673`, and `failed=3`. Both invariants hold: - `676 + 55 == 731` and `673 + 3 == 676`. diff --git a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md b/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md deleted file mode 100644 index 1078470a..00000000 --- a/.superpowers/sdd/2026-08-19-python-binding-modernization/task-6-report.md +++ /dev/null @@ -1,147 +0,0 @@ -# Task 6 Report: Gate Artifacts And Document Actual Behavior - -## Fix Round 1 - -- The three accepted baseline conformance mismatches are now visible strict - pytest xfails. Each uses its full scenario identifier and an explicit reason: - `core-modules/csv-invalid-utf8-out.csv`, - `core-modules/number-addition-out.json`, and - `core-modules/number-subtraction-out.json`. -- No other conformance failure is excluded or xfailed. A new unexpected - mismatch still fails `pythonTck`; an XPASS also fails because the xfails are - strict. -- Python test dependency installation now belongs solely to the Python artifact - action, rather than the shared build foundation. The action publishes the - Python TCK JUnit report with `always()` whenever its master-only TCK lane ran. -- The master workflow stages the runtime/core-modules corpus once before the - Python and Node artifact actions. Neither binding action restages it; local - `pythonTck` remains usable after an explicit `stageTckSuites` invocation. - -## Fix Round 2 - -- The foundation Gradle build now passes `-PskipPythonTests=true`, leaving the - Python artifact action as the only CI owner that installs Python dependencies - and runs `native-lib:pythonTest`. -- The Python wheel is built and uploaded before the optional Python TCK. This - preserves the package artifact when TCK conformance fails, while the TCK - result still determines the final job outcome. -- The Python and Node artifact steps use `continue-on-error: true`, so a failed - binding lane does not prevent the other lane from executing. A following - `always()` aggregation step fails the job when either binding lane failed. -- Python TCK JUnit artifacts include the workflow matrix platform token: - `python-tck-junit-${{ inputs.platform }}`, preventing cross-platform upload - name collisions. -- The strict xfail baseline remains unchanged: only the three accepted named - mismatches are strict xfails; any new mismatch and any XPASS fail the TCK. - -## Fix Round 3 - -- The Python and Node master-only TCK conformance steps now use - `always() && inputs.run-tck == 'true'`, so they run even if an earlier step - in the same composite action failed. -- The binding failure aggregation step remains guarded by `always()` and the - Python/Node step outcomes, but now follows the Native library artifact step. - It is therefore the workflow's final binding-failure verdict. -- CI structure tests assert both exact TCK conditions and that Native library - precedes the aggregation step. Strict Python TCK xfails remain unchanged. - -## Fix Round 4 - -- CI structure tests no longer use global `if` substring checks for TCK or - binding failure aggregation. A scoped `named_step_if` extractor selects the - named YAML step and reads only that step's `if` field. -- The test asserts the exact guard for `Run Python TCK Conformance`, `Run - Node.js TCK Conformance`, and `Fail if binding artifacts failed`. Thus the - Python TCK assertion cannot be satisfied by the similarly guarded JUnit - upload step, and changes to any required `always()` or outcome clause fail. - -## Changes - -- `.github/actions/python/action.yml` installs the Python `test` extra, runs - `native-lib:pythonTest` before `native-lib:buildPythonWheel`, and exposes a - `run-tck` input for the Python conformance lane. -- `.github/workflows/main.yml` passes `run-tck` only when the ref is `master`, - alongside the existing Node TCK gate. `pythonTck` stages and reuses the same - corpus as Node through its existing Gradle dependency. -- `native-lib/python/tests/unit/test_ci_structure.py` asserts the artifact-test - ordering and the master-only TCK wiring. -- `native-lib/python/README.md` now documents pytest normal and TCK commands, - bounded queue/chunk behavior, callback abort semantics, post-completion - stream metadata, module-resolution exclusions, and the remaining TCK - failures. -- `native-lib/python/examples/streaming_demo.py` no longer presents the - unsupported `input_properties` argument. - -## TDD And Verification - -1. RED: focused CI structure and strict-xfail tests failed before moving - dependency ownership, staging, report upload, and mismatch marks. The - failures identified foundation-owned dependencies, duplicate corpus staging, - missing upload wiring, and absent xfail parameters. -2. GREEN: `python3 -m pytest tests/unit/test_ci_structure.py - tests/tck/test_conformance.py -m unit -k 'strict_xfails or artifact_owns or - stages_the_shared' -vv` passed: `3 passed`. -3. `./gradlew native-lib:pythonTest` passed with `76 passed, 751 deselected` - and regenerated normal JUnit and coverage reports. -4. `./gradlew native-lib:stageTckSuites` stages the shared corpus once; the - terminal `./gradlew native-lib:pythonTck` run passed with `695 passed, 55 - skipped, 3 xfailed`. Its terminal report records `xfail=3`; all other - selected scenarios passed or used independently categorized exclusions. -5. YAML parsing and `git diff --check` are recorded with the final change. - -### Fix Round 2 Verification - -1. RED: `python3 -m pytest tests/unit/test_ci_structure.py -m unit -v` failed - with the missing platform input/artifact name and missing - `-PskipPythonTests=true` foundation flag. -2. GREEN: the same focused CI structure test passed with `5 passed` after the - workflow, action, and structure-test updates. -3. YAML parsing for the changed workflow/actions and `git diff --check` passed. -4. Foundation-equivalent dry run/build command passed: - `./gradlew --stacktrace --no-problems-report -PskipNodeTests=true - -PskipPythonTests=true -PskipTCKTests=true build`. - -### Fix Round 3 Verification - -1. RED: `python3 -m pytest tests/unit/test_ci_structure.py -m unit -v` failed - because the Node TCK condition lacked `always()` and the binding aggregation - preceded Native library. -2. GREEN: the same focused test passed with `5 passed` after updating both - composite action TCK guards and moving final aggregation. -3. YAML parsing for the changed workflow/actions and `git diff --check` passed. - -### Fix Round 4 Verification - -1. RED: the focused CI structure test failed after replacing global checks with - calls to the not-yet-defined scoped extractor. -2. GREEN: after adding `named_step_if`, the focused test passed with `5 - passed`. -3. YAML parsing for changed workflow/actions and `git diff --check` passed. - -### Fix Round 5 Verification - -1. Root cause: the prior `named_step_if` extractor used `\s` for indentation. - Because `\s` includes newlines, the body capture could cross a same-level - step boundary and read that later step's `if` guard. -2. RED: `test_named_step_if_does_not_read_a_later_step_guard` failed against - the prior extractor: after removing the aggregation step guard and placing - it on a later same-level step, the helper incorrectly returned the later - guard instead of raising `missing if guard`. -3. GREEN: the extractor now captures the named step's leading horizontal - whitespace and consumes only lines with additional horizontal whitespace. - The new mutation regression and the focused CI structure suite passed with - `6 passed`. -4. Python YAML parsing of the changed workflow/action files and `git diff - --check` passed. - -## Commit - -- Fix Round 4 commit: `test: scope CI workflow guard assertions` -- Pending Fix Round 5 commit: `test: bound CI workflow guard step extraction` - -## Concerns - -- The accepted baseline is deliberately narrow: only the three named strict - xfails are tolerated. Any new mismatch remains a blocking `pythonTck` - failure; a repaired accepted mismatch becomes an XPASS and also fails. -- Native-image emits existing GraalVM deprecation warnings during local runs. From cb0bbef5d7c3d31ac886be8f7de83451acb327fa Mon Sep 17 00:00:00 2001 From: mlischetti Date: Thu, 20 Aug 2026 18:27:51 -0300 Subject: [PATCH 30/30] fix: address Python binding review findings --- .github/actions/python/action.yml | 2 +- native-lib/python/src/dataweave/__init__.py | 6 +- native-lib/python/src/dataweave/runtime.py | 46 +++++++- native-lib/python/tests/tck/compare.py | 10 ++ .../python/tests/tck/test_conformance.py | 5 + .../python/tests/unit/test_ci_structure.py | 2 +- native-lib/python/tests/unit/test_facade.py | 4 +- .../python/tests/unit/test_streaming.py | 106 ++++++++++++++++++ 8 files changed, 168 insertions(+), 13 deletions(-) diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index f3e38c3d..345062ec 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -83,5 +83,5 @@ runs: uses: actions/upload-artifact@v7.0.1 with: name: python-tck-junit-${{ inputs.platform }} - path: native-lib/python/build/test-results/pythonTck.xml + path: native-lib/build/test-results/pythonTck.xml if-no-files-found: error diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index 77070380..f1db4641 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -61,10 +61,8 @@ def run_input_output_callback(script: str, input_name: str, input_mime_type: str def cleanup() -> None: global _global_instance if _global_instance is not None: - try: - _global_instance.cleanup() - finally: - _global_instance = None + _global_instance.cleanup() + _global_instance = None __all__ = [ diff --git a/native-lib/python/src/dataweave/runtime.py b/native-lib/python/src/dataweave/runtime.py index d7d54c4e..8d57d918 100644 --- a/native-lib/python/src/dataweave/runtime.py +++ b/native-lib/python/src/dataweave/runtime.py @@ -1,7 +1,7 @@ import ctypes import json from queue import Empty, Full, Queue -from threading import Event, Thread +from threading import Event, Lock, Thread, current_thread from typing import Any, Dict, Generator, Iterable, Optional from .encoding import normalize_input_value, parse_native_encoded_response, parse_streaming_result @@ -29,12 +29,42 @@ class DataWeave: def __init__(self, lib_path: Optional[str] = None): self._native = NativeRuntime(lib_path) + self._stream_workers = set() + self._stream_workers_lock = Lock() + self._cleaning_up = False def initialize(self): self._native.initialize() def cleanup(self): - self._native.cleanup() + workers, lock = self._worker_registry() + with lock: + if workers: + raise DataWeaveError("Cannot clean up DataWeave runtime while an active streaming worker is attached.") + self._cleaning_up = True + try: + self._native.cleanup() + finally: + with lock: + self._cleaning_up = False + + def _worker_registry(self): + if not hasattr(self, "_stream_workers"): + self._stream_workers = set() + self._stream_workers_lock = Lock() + return self._stream_workers, self._stream_workers_lock + + def _register_stream_worker(self, worker: Thread) -> None: + workers, lock = self._worker_registry() + with lock: + if getattr(self, "_cleaning_up", False): + raise DataWeaveError("Cannot start a streaming worker while the DataWeave runtime is being cleaned up.") + workers.add(worker) + + def _unregister_stream_worker(self, worker: Optional[Thread] = None) -> None: + workers, lock = self._worker_registry() + with lock: + workers.discard(worker or current_thread()) def _require_initialized(self, supported: bool, api_name: str) -> None: if not self._native.initialized: @@ -68,9 +98,9 @@ def write_cb(_context, buffer, length): try: ptr = self._native.run_script_callback(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs), write_cb) raw = self._native.decode_and_free(ptr) + return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) except Exception as error: raise DataWeaveError(f"Failed to execute callback streaming: {error}") - return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) def _stream_worker(self, invoke, cancelled: Event) -> Generator[bytes, None, StreamingResult]: sentinel = object() @@ -119,11 +149,17 @@ def worker_main(): if primary_error is None and not primary_outcome: publish(CleanupFailure(error)) publish(sentinel) + self._unregister_stream_worker() # Python cannot cancel a native call. Daemon workers keep an abandoned # call from extending interpreter lifetime after bounded cancellation. worker = Thread(target=worker_main, name="dw-streaming-worker", daemon=True) - worker.start() + self._register_stream_worker(worker) + try: + worker.start() + except Exception: + self._unregister_stream_worker(worker) + raise metadata = None try: while True: @@ -216,9 +252,9 @@ def write_cb(_context, buffer, length): try: ptr = self._native.run_script_input_output_callback(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs), input_name.encode("utf-8"), input_mime_type.encode("utf-8"), input_charset.encode("utf-8") if input_charset else None, read_cb, write_cb) raw = self._native.decode_and_free(ptr) + return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) except Exception as error: raise DataWeaveError(f"Failed to execute callback input/output streaming: {error}") - return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) def __enter__(self): self.initialize() diff --git a/native-lib/python/tests/tck/compare.py b/native-lib/python/tests/tck/compare.py index 0f30612d..16adc3fe 100644 --- a/native-lib/python/tests/tck/compare.py +++ b/native-lib/python/tests/tck/compare.py @@ -85,6 +85,12 @@ def _xml_child_value(node) -> Any: def _json_equal(actual: Any, expected: Any) -> bool: + if isinstance(actual, bool) or isinstance(expected, bool): + return type(actual) is type(expected) and actual == expected + if _is_number(actual) and _is_number(expected): + return actual == expected + if type(actual) is not type(expected): + return False if isinstance(actual, str) and isinstance(expected, str): return _normalize_eol(actual) == _normalize_eol(expected) if isinstance(actual, list) and isinstance(expected, list): @@ -98,6 +104,10 @@ def _json_equal(actual: Any, expected: Any) -> bool: return actual == expected +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + def _python_encoding(charset: Optional[str]) -> str: return (charset or "utf-8").replace("UTF-16", "utf-16") diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py index b6db68cc..e1aaaed9 100644 --- a/native-lib/python/tests/tck/test_conformance.py +++ b/native-lib/python/tests/tck/test_conformance.py @@ -273,6 +273,11 @@ def test_compare_output_rejects_unknown_extension(): assert "unknown output extension" in result.detail +@pytest.mark.parametrize(("actual", "expected"), [(b"true", b"1"), (b"false", b"0")]) +def test_compare_output_rejects_json_booleans_as_numbers(actual: bytes, expected: bytes): + assert not compare_output("json", actual, expected).match + + def test_compare_output_rejects_different_xml_tail_text(): """Catches structural XML comparison that discards text after child elements.""" result = compare_output( diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index 3137e977..560f8687 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -52,7 +52,7 @@ def test_python_artifact_owns_test_dependencies_and_tck_junit_upload(): assert action.index("Upload Python wheel to release") < action.index("Run Python TCK Conformance") assert action.index("Run Python TCK Conformance") < action.index("Upload Python TCK JUnit") assert named_step_if(action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" - assert "native-lib/python/build/test-results/pythonTck.xml" in action + assert "native-lib/build/test-results/pythonTck.xml" in action @pytest.mark.unit diff --git a/native-lib/python/tests/unit/test_facade.py b/native-lib/python/tests/unit/test_facade.py index f4ea4d2c..c281495e 100644 --- a/native-lib/python/tests/unit/test_facade.py +++ b/native-lib/python/tests/unit/test_facade.py @@ -69,7 +69,7 @@ def test_cleanup_is_noop_without_global_runtime(): @pytest.mark.unit -def test_global_cleanup_clears_failed_runtime_and_allows_recreation(monkeypatch): +def test_global_cleanup_retains_failed_runtime_for_retry(monkeypatch): created = [] class FakeRuntime: @@ -89,5 +89,5 @@ def cleanup(self): dataweave.cleanup() second = dataweave._get_global_instance() - assert second is not first + assert second is first dataweave._global_instance = None diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py index d60a9b95..b489da0e 100644 --- a/native-lib/python/tests/unit/test_streaming.py +++ b/native-lib/python/tests/unit/test_streaming.py @@ -102,6 +102,23 @@ def test_run_input_output_callback_converts_read_exception_to_abort_result(): assert native.read_status == -1 +@pytest.mark.unit +@pytest.mark.parametrize( + "invoke", + [ + lambda runtime: runtime.run_callback("script", lambda _data: 0), + lambda runtime: runtime.run_input_output_callback( + "script", "payload", "application/json", lambda _size: b"", lambda _data: 0, + ), + ], +) +def test_callback_apis_translate_malformed_native_metadata(invoke): + runtime = configured_runtime(FakeNative("not-json")) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to execute callback"): + invoke(runtime) + + @pytest.mark.unit def test_run_transform_preserves_remainder_of_large_input_chunk(): native = FakeNative('{"success": true, "mimeType": "application/json", "charset": "utf-8"}', consume_input=True) @@ -259,6 +276,22 @@ def run_script_callback(self, _thread, _script, _inputs, _write_callback, _conte list(runtime.run_streaming("script")) +@pytest.mark.unit +def test_stream_worker_start_failure_does_not_block_cleanup(monkeypatch): + native = FakeNative('{"success": true}') + native.graal_tear_down_isolate = lambda _thread: 0 + runtime = configured_runtime(native) + + def fail_start(_worker): + raise RuntimeError("cannot start") + + monkeypatch.setattr(runtime_module.Thread, "start", fail_start) + + with pytest.raises(RuntimeError, match="cannot start"): + list(runtime.run_streaming("script")) + runtime.cleanup() + + @pytest.mark.unit def test_stream_finalization_does_not_raise_when_a_native_worker_cannot_cancel(monkeypatch): class UncancellableNative(FakeNative): @@ -273,6 +306,79 @@ def run_script_callback(self, _thread, _script, _inputs, _write_callback, _conte stream.close() +@pytest.mark.unit +def test_cleanup_refuses_to_tear_down_isolate_while_stream_worker_is_active(monkeypatch): + class BlockingNative(FakeNative): + def __init__(self): + super().__init__('{"success": true}') + self.started = Event() + self.release = Event() + self.torn_down = False + + def run_script_callback(self, _thread, _script, _inputs, _write_callback, _context): + self.started.set() + self.release.wait() + return self._response_pointer() + + def graal_tear_down_isolate(self, _thread): + self.torn_down = True + return 0 + + monkeypatch.setattr(runtime_module, "_WORKER_JOIN_TIMEOUT_SECONDS", 0.001) + native = BlockingNative() + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + consumer_error = [] + + def consume(): + try: + next(stream, None) + except dataweave.DataWeaveError as error: + consumer_error.append(error) + + consumer = Thread(target=consume) + consumer.start() + assert native.started.wait(timeout=1) + stream.close() + + with pytest.raises(dataweave.DataWeaveError, match="active streaming worker"): + runtime.cleanup() + assert native.torn_down is False + + native.release.set() + consumer.join(timeout=1) + assert consumer_error == [] + assert native.detached_event.wait(timeout=1) + runtime.cleanup() + assert native.torn_down is True + + +@pytest.mark.unit +def test_stream_worker_cannot_register_after_isolate_teardown_starts(): + class BlockingCleanupNative(FakeNative): + def __init__(self): + super().__init__('{"success": true}') + self.cleanup_started = Event() + self.release_cleanup = Event() + + def graal_tear_down_isolate(self, _thread): + self.cleanup_started.set() + self.release_cleanup.wait() + return 0 + + native = BlockingCleanupNative() + runtime = configured_runtime(native) + cleanup = Thread(target=runtime.cleanup) + cleanup.start() + assert native.cleanup_started.wait(timeout=1) + + with pytest.raises(dataweave.DataWeaveError, match="being cleaned up"): + runtime._register_stream_worker(Thread()) + + native.release_cleanup.set() + cleanup.join(timeout=1) + + @pytest.mark.unit def test_run_streaming_surfaces_detach_failure_without_primary_execution_failure(): class DetachFailingNative(FakeNative):