@W-23751646: Modernize Python native binding tests and runtime

.github/actions/build-foundation/action.yml

@@ -29,5 +29,5 @@ runs:
         github-token: ${{ inputs.github-token }}
 
     - name: Run Build
-      run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }}
+      run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipPythonTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }}
       shell: bash

.github/actions/node/action.yml

@@ -51,9 +51,8 @@ runs:
       shell: bash
 
     - name: Run Node.js TCK Conformance
-      if: inputs.run-tck == 'true'
+      if: always() && inputs.run-tck == 'true'
       run: |
-        ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites
         cd native-lib/node && npm run test:tck
       shell: bash
 

.github/actions/python/action.yml

@@ -1,9 +1,10 @@
 name: Python artifact
 description: >-
   Installs Python build dependencies and builds the DataWeave Python wheel
-  (which embeds dwlib); optionally publishes the wheel as a CI artifact (main)
-  or a release asset (release). No TCK phase today. Requires build-foundation to
-  have run earlier in the same job.
+  (which embeds dwlib), runs the Python unit/integration tests, and optionally
+  runs the master-only Python TCK conformance lane before publishing the wheel
+  as a CI artifact (main) or a release asset (release). Requires
+  build-foundation to have run earlier in the same job.
 inputs:
   native-version:
     description: -PnativeVersion value; empty omits the flag.
@@ -16,6 +17,14 @@ inputs:
       runners).
     required: false
     default: 'false'
+  run-tck:
+    description: When 'true', run the master-only Python TCK conformance lane.
+    required: false
+    default: 'false'
+  platform:
+    description: Platform token for matrix-qualified TCK JUnit artifact names.
+    required: false
+    default: ''
   publish:
     description: "'none' | 'artifact' | 'release'."
     required: false
@@ -31,12 +40,16 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Install Python build dependencies
+    - name: Install Python build and test dependencies
       # --ignore-installed: on macOS runners, Homebrew's own setuptools/wheel have no
       # pip RECORD file, so a plain --upgrade fails trying to uninstall them first
       # (pip error "uninstall-no-record-file"). --ignore-installed installs pip's
       # copy on top without needing to remove the untracked brew one.
-      run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade --ignore-installed setuptools wheel
+      run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade --ignore-installed setuptools wheel 'native-lib/python[test]'
+      shell: bash
+
+    - name: Run Python Tests
+      run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTest ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }}
       shell: bash
 
     - name: Create Native Lib Python Wheel
@@ -59,3 +72,16 @@ runs:
         file_glob: true
         tag: ${{ inputs.tag }}
         overwrite: true
+
+    - name: Run Python TCK Conformance
+      if: always() && inputs.run-tck == 'true'
+      run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck
+      shell: bash
+
+    - name: Upload Python TCK JUnit
+      if: always() && inputs.run-tck == 'true'
+      uses: actions/upload-artifact@v7.0.1
+      with:
+        name: python-tck-junit-${{ inputs.platform }}
+        path: native-lib/build/test-results/pythonTck.xml
+        if-no-files-found: error

.github/workflows/main.yml

@@ -58,15 +58,26 @@ jobs:
           script-name: ${{ matrix.script_name }}
           distro-os: ${{ matrix.distro_os }}
 
+      - name: Stage TCK corpus
+        if: github.ref == 'refs/heads/master'
+        run: ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites
+        shell: bash
+
       - name: Python
+        id: python
         uses: ./.github/actions/python
+        continue-on-error: true
         with:
           native-version: ${{ env.NATIVE_VERSION }}
           break-system-packages: 'true'
+          run-tck: ${{ github.ref == 'refs/heads/master' }}
+          platform: ${{ matrix.script_name }}
           publish: 'artifact'
 
       - name: Node
+        id: node
         uses: ./.github/actions/node
+        continue-on-error: true
         with:
           native-version: ${{ env.NATIVE_VERSION }}
           run-tck: ${{ github.ref == 'refs/heads/master' }}
@@ -81,3 +92,8 @@ jobs:
           publish: 'artifact'
           arch: ${{ env.ARCH }}
           script-name: ${{ matrix.script_name }}
+
+      - name: Fail if binding artifacts failed
+        if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure')
+        run: exit 1
+        shell: bash

docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md

@@ -0,0 +1,53 @@
+# Python Binding Modernization Design
+
+## Goal
+
+Modernize the Python `dataweave` binding without changing its supported public
+facade or the `dwlib` C ABI. The binding gains isolated test lanes, explicit
+native lifecycle ownership, streaming support, and a master-only conformance
+lane.
+
+## Architecture
+
+The `dataweave` package remains the stable facade. Public models and callback
+types live in `models.py`; input/output wire conversion lives in `encoding.py`;
+`native.py` owns ctypes library loading, isolate lifecycle, ABI signatures, and
+native string release; `runtime.py` owns `DataWeave` orchestration.
+
+`DataWeave` composes one `NativeRuntime`. Module-level functions retain the
+existing lazy singleton behavior. Explicit callers can use `DataWeave` as a
+context manager. Native failures raise `DataWeaveError`; script failures remain
+result envelopes unless the caller selects `raise_on_error`.
+
+## Streaming
+
+Both output-only and input/output streaming use one bounded queue worker. Each
+native worker attaches and detaches its own isolate thread. Callback exceptions
+return `-1` and never unwind across the C ABI. Stream input retains remainders
+when the iterable source provides chunks larger than the native buffer.
+
+`Stream.close()` and its context manager request cancellation. Python cannot
+forcibly interrupt a native call, so cleanup uses a short bounded join and an
+unresponsive worker is daemonized; finalization never raises. Low-level callback
+input larger than the supplied native buffer is rejected rather than truncated.
+
+## Testing And TCK
+
+Pytest has `unit`, `integration`, and `tck` lanes. Unit tests use fake native
+collaborators; integration tests use staged `dwlib`; the TCK is on-demand and
+master-only. The TCK comparator follows the Node policy, including structural
+XML comparison with namespace declaration placement ignored while prefixes and
+content remain significant.
+
+TCK skips are reserved for concrete binding/environment capabilities. Known
+runtime/output deviations are case-specific strict xfails, so new mismatches
+and repaired baselines are visible failures. The one deferred-writer case runs
+in a subprocess because its isolate teardown may block; the shared TCK runtime
+remains managed and is cleaned up at session end.
+
+## CI
+
+The Python artifact action installs test dependencies, runs normal Python tests,
+builds the wheel, and runs TCK on master. The staged TCK corpus is shared with
+the Node lane. Generated native libraries, wheels, corpus files, and reports are
+not committed.

native-lib/build.gradle

@@ -155,7 +155,27 @@ tasks.register('pythonTest', Exec) {
 
   dependsOn tasks.named('stagePythonNativeLib')
   workingDir("${projectDir}/python")
-  commandLine(pythonExe, 'tests/test_dataweave_module.py')
+  inputs.dir("${projectDir}/python/tests")
+  inputs.file("${projectDir}/python/pytest.ini")
+  inputs.file("${projectDir}/python/pyproject.toml")
+  def testReportsDir = layout.buildDirectory.dir('test-results/python')
+  def coverageReportsDir = layout.buildDirectory.dir('reports/coverage/python')
+  outputs.file(testReportsDir.map { it.file('junit.xml') })
+  outputs.file(coverageReportsDir.map { it.file('coverage.xml') })
+  doFirst {
+    testReportsDir.get().asFile.mkdirs()
+    coverageReportsDir.get().asFile.mkdirs()
+  }
+  commandLine(
+    pythonExe,
+    '-m',
+    'pytest',
+    '-m',
+    'unit or integration',
+    '--junitxml', testReportsDir.map { it.file('junit.xml') }.get().asFile,
+    '--cov=dataweave',
+    '--cov-report=xml:' + coverageReportsDir.map { it.file('coverage.xml') }.get().asFile,
+  )
 }
 
 // --- Node.js native package tasks ---
@@ -208,6 +228,18 @@ tasks.register('stageTckSuites') {
   }
 }
 
+// Python consumes the corpus already staged for the Node TCK lane above. This
+// shares the resolved artifacts and extraction, avoiding a second download.
+tasks.register('pythonTck', Exec) {
+  dependsOn tasks.named('stagePythonNativeLib')
+  workingDir("${projectDir}/python")
+  inputs.dir("${projectDir}/python/tests/tck")
+  inputs.dir(tckSuitesDir)
+  inputs.file("${projectDir}/python/pytest.ini")
+  commandLine(pythonExe, '-m', 'pytest', '-m', 'tck',
+    '--junitxml', "${layout.buildDirectory.get().asFile}/test-results/pythonTck.xml")
+}
+
 tasks.register('buildNodePackage', Exec) {
   dependsOn tasks.named('stageNodeNativeLib')
   workingDir("${projectDir}/node")

native-lib/python/README.md

@@ -32,6 +32,15 @@ python3 -m pip install native-lib/python/dist/dataweave_native-0.0.1-*.whl
 python3 -m pip install -e native-lib/python
 ```
 
+### Test dependencies
+
+Install the pytest test extra before running the Python test lanes locally:
+
+```bash
+cd native-lib/python
+python3 -m pip install '.[test]'
+```
+
 ### Option C: Use externally-built library via environment variable
 
 ```bash
@@ -150,13 +159,20 @@ Stream output chunks as they're produced, without buffering the entire result:
 import sys
 
 stream = dataweave.run_streaming("output application/json --- (1 to 10000) map {id: $}")
-for chunk in stream:
-    sys.stdout.buffer.write(chunk)
+with stream:
+    for chunk in stream:
+        sys.stdout.buffer.write(chunk)
 
 metadata = stream.metadata
 print(f"\nDone: {metadata.mime_type}, {metadata.charset}")
 ```
 
+Call `stream.close()` when stopping consumption early. `Stream` also supports a
+context manager, as above. Closing requests cancellation and waits only briefly
+for the native worker. A native call cannot be forcibly cancelled by Python, so
+an unresponsive call is left to finish in a daemon worker rather than delaying
+application shutdown or raising during finalization.
+
 Or with explicit context:
 
 ```python
@@ -168,7 +184,9 @@ with dataweave.DataWeave() as dw:
 
 ### Bidirectional Streaming (Input + Output)
 
-Stream both input and output for constant memory usage with large files:
+Stream both input and output with bounded Python-side queueing. Memory usage is
+bounded by the queue capacity plus the input and output chunk sizes; DataWeave
+itself can still buffer while parsing or evaluating a transform.
 
 ```python
 # Stream a file through DataWeave
@@ -245,11 +263,20 @@ print(result)            # StreamingResult(success=True, ...)
 print(b"".join(chunks))  # b'[1,4,9,16,25]'
 ```
 
+Read callbacks return bytes and are called with the native buffer size. Return
+`b""` for EOF; an exception is translated to `-1`, which aborts the native
+operation. Write callbacks return `0` on success. Any nonzero return value, or
+an exception, aborts the operation and returns unsuccessful `StreamingResult`
+metadata rather than unwinding a Python exception through the native callback.
+Low-level read callbacks must return no more than the requested buffer size;
+oversized callback data is rejected with `-1` rather than silently truncated.
+
 ## Running Tests
 
 ```bash
 cd native-lib/python
-python3 tests/test_dataweave_module.py
+python3 -m pip install '.[test]'
+python3 -m pytest -m "unit or integration" -v
 ```
 
 Or via Gradle:
@@ -258,6 +285,25 @@ Or via Gradle:
 ./gradlew :native-lib:pythonTest
 ```
 
+`pytest.ini` registers `unit`, `integration`, and `tck` markers. Normal pytest
+runs exclude `tck`; use `-m "unit or integration"` to run the lanes used by
+`pythonTest`.
+
+To stage and run the Python conformance suite, use:
+
+```bash
+./gradlew :native-lib:stageTckSuites :native-lib:pythonTck
+```
+
+`pythonTck` is intentionally separate from normal testing and runs only in the
+master-only CI lane. It reuses the corpus staged for Node TCK. It excludes
+only binding/environment capability gaps, such as unavailable module resolution,
+Java modules, and classpath test resources. Accepted runtime/output baseline
+mismatches are strict xfails: a new mismatch fails the lane and a repaired
+baseline mismatch XPASSes and also fails. The deferred-writer TCK scenario runs
+in a subprocess because that runtime's isolate teardown may block; the main TCK
+session runtime is always cleaned up.
+
 ## Running Examples
 
 ```bash
@@ -286,26 +332,27 @@ Execute a script and stream the output.
 
 **Returns:** `Stream` iterator yielding chunks, with `.metadata` attribute
 
-#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, input_properties=None) -> Stream`
+#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, inputs=None) -> Stream`
 
 Execute a script with streaming input and output.
 
 **Parameters:**
 - `input_stream`: Iterable of bytes (file, generator, list)
 - `input_mime_type`: MIME type of the input stream
-- Other parameters configure input handling
+- `input_charset`: Optional charset for the streamed input
+- `inputs`: Optional additional DataWeave input bindings
 
 **Returns:** `Stream` iterator yielding output chunks
 
-#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, input_properties=None, inputs=None) -> StreamingResult`
+#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, inputs=None) -> StreamingResult`
 
 Low-level callback API for advanced use cases.
 
 **Returns:** `StreamingResult` with success/error/metadata
 
 ### `DataWeave` Class
 
-#### `DataWeave(library_path=None)`
+#### `DataWeave(lib_path=None)`
 
 Context manager for explicit lifecycle control.
 
@@ -339,10 +386,14 @@ class ExecutionResult:
 
 ### `Stream`
 
-Iterator that yields output chunks.
+Iterator that yields output chunks through a bounded queue.
 
 **Attributes:**
-- `metadata: StreamingResult` - Available after iteration completes
+- `metadata: StreamingResult` - Available only after the iterator completes;
+  check `success` and `error` after consuming the stream
+
+**Methods:**
+- `close() -> None` - Stop consuming early and request bounded worker cleanup
 
 ### `StreamingResult`
 
@@ -461,7 +512,9 @@ if not stream.metadata.success:
 - **Buffered execution** (`run()`) - Best for small outputs (<1MB)
 - **Output streaming** (`run_streaming()`) - Use for large outputs (>10MB)
 - **Bidirectional streaming** (`run_transform()`) - Use for large inputs AND outputs
-- **Memory usage**: Streaming uses constant memory (~64KB buffer)
+- **Memory usage**: Streaming uses a bounded queue and native callback-sized
+  chunks. It avoids accumulating output in the Python binding, but does not
+  guarantee fixed or constant memory for the DataWeave runtime or transform.
 
 ## Environment Variables
 

native-lib/python/examples/streaming_demo.py

@@ -90,8 +90,7 @@ def generate_json_chunks():
     stream = dataweave.run_transform(
         'output application/json --- payload map { name: $.name, age: $.age }',
         input_stream=iter(lambda: input_file.read(20), b""),  # Read 20 bytes at a time
-        input_mime_type="application/csv",
-        input_properties={"header": True}
+        input_mime_type="application/csv"
     )
 
     output = b"".join(stream).decode('utf-8')