diff --git a/.github/actions/build-foundation/action.yml b/.github/actions/build-foundation/action.yml index c36f9863..c806730c 100644 --- a/.github/actions/build-foundation/action.yml +++ b/.github/actions/build-foundation/action.yml @@ -29,5 +29,5 @@ runs: github-token: ${{ inputs.github-token }} - name: Run Build - run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} + run: ./gradlew --stacktrace --no-problems-report -PskipNodeTests=true -PskipPythonTests=true -PskipTCKTests=true build ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash diff --git a/.github/actions/node/action.yml b/.github/actions/node/action.yml index d3ca9a8e..706dbe19 100644 --- a/.github/actions/node/action.yml +++ b/.github/actions/node/action.yml @@ -51,9 +51,8 @@ runs: shell: bash - name: Run Node.js TCK Conformance - if: inputs.run-tck == 'true' + if: always() && inputs.run-tck == 'true' run: | - ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites cd native-lib/node && npm run test:tck shell: bash diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index f8864ba3..345062ec 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -1,9 +1,10 @@ name: Python artifact description: >- Installs Python build dependencies and builds the DataWeave Python wheel - (which embeds dwlib); optionally publishes the wheel as a CI artifact (main) - or a release asset (release). No TCK phase today. Requires build-foundation to - have run earlier in the same job. + (which embeds dwlib), runs the Python unit/integration tests, and optionally + runs the master-only Python TCK conformance lane before publishing the wheel + as a CI artifact (main) or a release asset (release). Requires + build-foundation to have run earlier in the same job. inputs: native-version: description: -PnativeVersion value; empty omits the flag. @@ -16,6 +17,14 @@ inputs: runners). required: false default: 'false' + run-tck: + description: When 'true', run the master-only Python TCK conformance lane. + required: false + default: 'false' + platform: + description: Platform token for matrix-qualified TCK JUnit artifact names. + required: false + default: '' publish: description: "'none' | 'artifact' | 'release'." required: false @@ -31,12 +40,16 @@ inputs: runs: using: composite steps: - - name: Install Python build dependencies + - name: Install Python build and test dependencies # --ignore-installed: on macOS runners, Homebrew's own setuptools/wheel have no # pip RECORD file, so a plain --upgrade fails trying to uninstall them first # (pip error "uninstall-no-record-file"). --ignore-installed installs pip's # copy on top without needing to remove the untracked brew one. - run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade --ignore-installed setuptools wheel + run: python3 -m pip install ${{ inputs.break-system-packages == 'true' && '--break-system-packages' || '' }} --upgrade --ignore-installed setuptools wheel 'native-lib/python[test]' + shell: bash + + - name: Run Python Tests + run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTest ${{ inputs.native-version != '' && format('-PnativeVersion={0}', inputs.native-version) || '' }} shell: bash - name: Create Native Lib Python Wheel @@ -59,3 +72,16 @@ runs: file_glob: true tag: ${{ inputs.tag }} overwrite: true + + - name: Run Python TCK Conformance + if: always() && inputs.run-tck == 'true' + run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck + shell: bash + + - name: Upload Python TCK JUnit + if: always() && inputs.run-tck == 'true' + uses: actions/upload-artifact@v7.0.1 + with: + name: python-tck-junit-${{ inputs.platform }} + path: native-lib/build/test-results/pythonTck.xml + if-no-files-found: error diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7941eccd..1985ad22 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,15 +58,26 @@ jobs: script-name: ${{ matrix.script_name }} distro-os: ${{ matrix.distro_os }} + - name: Stage TCK corpus + if: github.ref == 'refs/heads/master' + run: ./gradlew --stacktrace --no-problems-report native-lib:stageTckSuites + shell: bash + - name: Python + id: python uses: ./.github/actions/python + continue-on-error: true with: native-version: ${{ env.NATIVE_VERSION }} break-system-packages: 'true' + run-tck: ${{ github.ref == 'refs/heads/master' }} + platform: ${{ matrix.script_name }} publish: 'artifact' - name: Node + id: node uses: ./.github/actions/node + continue-on-error: true with: native-version: ${{ env.NATIVE_VERSION }} run-tck: ${{ github.ref == 'refs/heads/master' }} @@ -81,3 +92,8 @@ jobs: publish: 'artifact' arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} + + - name: Fail if binding artifacts failed + if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure') + run: exit 1 + shell: bash diff --git a/docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md b/docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md new file mode 100644 index 00000000..e37bb1fe --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-python-binding-modernization-design.md @@ -0,0 +1,53 @@ +# Python Binding Modernization Design + +## Goal + +Modernize the Python `dataweave` binding without changing its supported public +facade or the `dwlib` C ABI. The binding gains isolated test lanes, explicit +native lifecycle ownership, streaming support, and a master-only conformance +lane. + +## Architecture + +The `dataweave` package remains the stable facade. Public models and callback +types live in `models.py`; input/output wire conversion lives in `encoding.py`; +`native.py` owns ctypes library loading, isolate lifecycle, ABI signatures, and +native string release; `runtime.py` owns `DataWeave` orchestration. + +`DataWeave` composes one `NativeRuntime`. Module-level functions retain the +existing lazy singleton behavior. Explicit callers can use `DataWeave` as a +context manager. Native failures raise `DataWeaveError`; script failures remain +result envelopes unless the caller selects `raise_on_error`. + +## Streaming + +Both output-only and input/output streaming use one bounded queue worker. Each +native worker attaches and detaches its own isolate thread. Callback exceptions +return `-1` and never unwind across the C ABI. Stream input retains remainders +when the iterable source provides chunks larger than the native buffer. + +`Stream.close()` and its context manager request cancellation. Python cannot +forcibly interrupt a native call, so cleanup uses a short bounded join and an +unresponsive worker is daemonized; finalization never raises. Low-level callback +input larger than the supplied native buffer is rejected rather than truncated. + +## Testing And TCK + +Pytest has `unit`, `integration`, and `tck` lanes. Unit tests use fake native +collaborators; integration tests use staged `dwlib`; the TCK is on-demand and +master-only. The TCK comparator follows the Node policy, including structural +XML comparison with namespace declaration placement ignored while prefixes and +content remain significant. + +TCK skips are reserved for concrete binding/environment capabilities. Known +runtime/output deviations are case-specific strict xfails, so new mismatches +and repaired baselines are visible failures. The one deferred-writer case runs +in a subprocess because its isolate teardown may block; the shared TCK runtime +remains managed and is cleaned up at session end. + +## CI + +The Python artifact action installs test dependencies, runs normal Python tests, +builds the wheel, and runs TCK on master. The staged TCK corpus is shared with +the Node lane. Generated native libraries, wheels, corpus files, and reports are +not committed. diff --git a/native-lib/build.gradle b/native-lib/build.gradle index 436f6066..2a6e2a74 100644 --- a/native-lib/build.gradle +++ b/native-lib/build.gradle @@ -155,7 +155,27 @@ tasks.register('pythonTest', Exec) { dependsOn tasks.named('stagePythonNativeLib') workingDir("${projectDir}/python") - commandLine(pythonExe, 'tests/test_dataweave_module.py') + inputs.dir("${projectDir}/python/tests") + inputs.file("${projectDir}/python/pytest.ini") + inputs.file("${projectDir}/python/pyproject.toml") + def testReportsDir = layout.buildDirectory.dir('test-results/python') + def coverageReportsDir = layout.buildDirectory.dir('reports/coverage/python') + outputs.file(testReportsDir.map { it.file('junit.xml') }) + outputs.file(coverageReportsDir.map { it.file('coverage.xml') }) + doFirst { + testReportsDir.get().asFile.mkdirs() + coverageReportsDir.get().asFile.mkdirs() + } + commandLine( + pythonExe, + '-m', + 'pytest', + '-m', + 'unit or integration', + '--junitxml', testReportsDir.map { it.file('junit.xml') }.get().asFile, + '--cov=dataweave', + '--cov-report=xml:' + coverageReportsDir.map { it.file('coverage.xml') }.get().asFile, + ) } // --- Node.js native package tasks --- @@ -208,6 +228,18 @@ tasks.register('stageTckSuites') { } } +// Python consumes the corpus already staged for the Node TCK lane above. This +// shares the resolved artifacts and extraction, avoiding a second download. +tasks.register('pythonTck', Exec) { + dependsOn tasks.named('stagePythonNativeLib') + workingDir("${projectDir}/python") + inputs.dir("${projectDir}/python/tests/tck") + inputs.dir(tckSuitesDir) + inputs.file("${projectDir}/python/pytest.ini") + commandLine(pythonExe, '-m', 'pytest', '-m', 'tck', + '--junitxml', "${layout.buildDirectory.get().asFile}/test-results/pythonTck.xml") +} + tasks.register('buildNodePackage', Exec) { dependsOn tasks.named('stageNodeNativeLib') workingDir("${projectDir}/node") diff --git a/native-lib/python/README.md b/native-lib/python/README.md index 76da00b8..09e3dfd2 100644 --- a/native-lib/python/README.md +++ b/native-lib/python/README.md @@ -32,6 +32,15 @@ python3 -m pip install native-lib/python/dist/dataweave_native-0.0.1-*.whl python3 -m pip install -e native-lib/python ``` +### Test dependencies + +Install the pytest test extra before running the Python test lanes locally: + +```bash +cd native-lib/python +python3 -m pip install '.[test]' +``` + ### Option C: Use externally-built library via environment variable ```bash @@ -150,13 +159,20 @@ Stream output chunks as they're produced, without buffering the entire result: import sys stream = dataweave.run_streaming("output application/json --- (1 to 10000) map {id: $}") -for chunk in stream: - sys.stdout.buffer.write(chunk) +with stream: + for chunk in stream: + sys.stdout.buffer.write(chunk) metadata = stream.metadata print(f"\nDone: {metadata.mime_type}, {metadata.charset}") ``` +Call `stream.close()` when stopping consumption early. `Stream` also supports a +context manager, as above. Closing requests cancellation and waits only briefly +for the native worker. A native call cannot be forcibly cancelled by Python, so +an unresponsive call is left to finish in a daemon worker rather than delaying +application shutdown or raising during finalization. + Or with explicit context: ```python @@ -168,7 +184,9 @@ with dataweave.DataWeave() as dw: ### Bidirectional Streaming (Input + Output) -Stream both input and output for constant memory usage with large files: +Stream both input and output with bounded Python-side queueing. Memory usage is +bounded by the queue capacity plus the input and output chunk sizes; DataWeave +itself can still buffer while parsing or evaluating a transform. ```python # Stream a file through DataWeave @@ -245,11 +263,20 @@ print(result) # StreamingResult(success=True, ...) print(b"".join(chunks)) # b'[1,4,9,16,25]' ``` +Read callbacks return bytes and are called with the native buffer size. Return +`b""` for EOF; an exception is translated to `-1`, which aborts the native +operation. Write callbacks return `0` on success. Any nonzero return value, or +an exception, aborts the operation and returns unsuccessful `StreamingResult` +metadata rather than unwinding a Python exception through the native callback. +Low-level read callbacks must return no more than the requested buffer size; +oversized callback data is rejected with `-1` rather than silently truncated. + ## Running Tests ```bash cd native-lib/python -python3 tests/test_dataweave_module.py +python3 -m pip install '.[test]' +python3 -m pytest -m "unit or integration" -v ``` Or via Gradle: @@ -258,6 +285,25 @@ Or via Gradle: ./gradlew :native-lib:pythonTest ``` +`pytest.ini` registers `unit`, `integration`, and `tck` markers. Normal pytest +runs exclude `tck`; use `-m "unit or integration"` to run the lanes used by +`pythonTest`. + +To stage and run the Python conformance suite, use: + +```bash +./gradlew :native-lib:stageTckSuites :native-lib:pythonTck +``` + +`pythonTck` is intentionally separate from normal testing and runs only in the +master-only CI lane. It reuses the corpus staged for Node TCK. It excludes +only binding/environment capability gaps, such as unavailable module resolution, +Java modules, and classpath test resources. Accepted runtime/output baseline +mismatches are strict xfails: a new mismatch fails the lane and a repaired +baseline mismatch XPASSes and also fails. The deferred-writer TCK scenario runs +in a subprocess because that runtime's isolate teardown may block; the main TCK +session runtime is always cleaned up. + ## Running Examples ```bash @@ -286,18 +332,19 @@ Execute a script and stream the output. **Returns:** `Stream` iterator yielding chunks, with `.metadata` attribute -#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, input_properties=None) -> Stream` +#### `run_transform(script, input_stream, input_name="payload", input_mime_type="application/json", input_charset=None, inputs=None) -> Stream` Execute a script with streaming input and output. **Parameters:** - `input_stream`: Iterable of bytes (file, generator, list) - `input_mime_type`: MIME type of the input stream -- Other parameters configure input handling +- `input_charset`: Optional charset for the streamed input +- `inputs`: Optional additional DataWeave input bindings **Returns:** `Stream` iterator yielding output chunks -#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, input_properties=None, inputs=None) -> StreamingResult` +#### `run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset=None, inputs=None) -> StreamingResult` Low-level callback API for advanced use cases. @@ -305,7 +352,7 @@ Low-level callback API for advanced use cases. ### `DataWeave` Class -#### `DataWeave(library_path=None)` +#### `DataWeave(lib_path=None)` Context manager for explicit lifecycle control. @@ -339,10 +386,14 @@ class ExecutionResult: ### `Stream` -Iterator that yields output chunks. +Iterator that yields output chunks through a bounded queue. **Attributes:** -- `metadata: StreamingResult` - Available after iteration completes +- `metadata: StreamingResult` - Available only after the iterator completes; + check `success` and `error` after consuming the stream + +**Methods:** +- `close() -> None` - Stop consuming early and request bounded worker cleanup ### `StreamingResult` @@ -461,7 +512,9 @@ if not stream.metadata.success: - **Buffered execution** (`run()`) - Best for small outputs (<1MB) - **Output streaming** (`run_streaming()`) - Use for large outputs (>10MB) - **Bidirectional streaming** (`run_transform()`) - Use for large inputs AND outputs -- **Memory usage**: Streaming uses constant memory (~64KB buffer) +- **Memory usage**: Streaming uses a bounded queue and native callback-sized + chunks. It avoids accumulating output in the Python binding, but does not + guarantee fixed or constant memory for the DataWeave runtime or transform. ## Environment Variables diff --git a/native-lib/python/examples/streaming_demo.py b/native-lib/python/examples/streaming_demo.py index aa5bab93..c581bc47 100755 --- a/native-lib/python/examples/streaming_demo.py +++ b/native-lib/python/examples/streaming_demo.py @@ -90,8 +90,7 @@ def generate_json_chunks(): stream = dataweave.run_transform( 'output application/json --- payload map { name: $.name, age: $.age }', input_stream=iter(lambda: input_file.read(20), b""), # Read 20 bytes at a time - input_mime_type="application/csv", - input_properties={"header": True} + input_mime_type="application/csv" ) output = b"".join(stream).decode('utf-8') diff --git a/native-lib/python/pyproject.toml b/native-lib/python/pyproject.toml index 642ab3ce..03a366ae 100644 --- a/native-lib/python/pyproject.toml +++ b/native-lib/python/pyproject.toml @@ -1,3 +1,12 @@ [build-system] requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" + +[project] +name = "dataweave-native" +version = "0.0.1" +description = "Python bindings for the DataWeave native library" +requires-python = ">=3.9" + +[project.optional-dependencies] +test = ["pytest", "pytest-cov"] diff --git a/native-lib/python/pytest.ini b/native-lib/python/pytest.ini new file mode 100644 index 00000000..33c8c8d4 --- /dev/null +++ b/native-lib/python/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +testpaths = tests +addopts = --import-mode=importlib -m "not tck" +markers = + unit: fast tests that do not require the native DataWeave library + integration: tests that execute the staged native DataWeave library + tck: DataWeave conformance tests excluded from normal test runs diff --git a/native-lib/python/src/dataweave/__init__.py b/native-lib/python/src/dataweave/__init__.py index 15b50211..f1db4641 100644 --- a/native-lib/python/src/dataweave/__init__.py +++ b/native-lib/python/src/dataweave/__init__.py @@ -1,953 +1,28 @@ -""" -DataWeave Python Module +"""Public facade for the DataWeave Python native binding.""" -Execute DataWeave scripts from Python via a GraalVM native shared library. -Supports buffered execution, output streaming, and bidirectional streaming -with constant memory overhead. - -Basic usage: - import dataweave - - result = dataweave.run("2 + 2") - print(result.get_string()) # "4" - -Output streaming (yields chunks as produced): - stream = dataweave.run_streaming("output json --- (1 to 10000) map {id: $}") - for chunk in stream: - sys.stdout.buffer.write(chunk) - -Bidirectional streaming (iterable in, generator out): - with open("large.json", "rb") as f: - stream = dataweave.run_transform( - "output csv --- payload", - input_stream=iter(lambda: f.read(8192), b""), - input_mime_type="application/json", - ) - for chunk in stream: - process(chunk) - -Context manager (explicit lifecycle control): - from dataweave import DataWeave - - with DataWeave() as dw: - result = dw.run("2 + 2") - print(result.get_string()) - -Error handling: - try: - result = dataweave.run("invalid", raise_on_error=True) - except dataweave.DataWeaveScriptError as e: - print(e.result.error) - -Native resources are released automatically at interpreter exit via atexit. -Call dataweave.cleanup() to release them earlier if needed. -""" - -import base64 import ctypes -import json -from dataclasses import dataclass -from pathlib import Path -from queue import Queue -from threading import Thread -from typing import Any, Callable, Dict, Generator, Iterable, Optional, Union - -# Bound for streaming output queues: limits memory under slow/stalled consumers -# by exerting backpressure onto the native producer. -_OUTPUT_QUEUE_MAXSIZE = 512 - - -class DataWeaveError(Exception): - pass - - -class DataWeaveScriptError(DataWeaveError): - """Raised when a DataWeave script fails (compile or runtime error). - - Carries the full result object so callers can inspect details. - """ - - def __init__(self, result): - self.result = result - super().__init__(result.error or "Script execution failed") - - -class DataWeaveLibraryNotFoundError(Exception): - pass - - -# ctypes callback signatures matching NativeCallbacks.WriteCallback / ReadCallback. -# Buffer parameters use c_void_p (not c_char_p) because ctypes gives c_char_p -# special treatment that prevents writing into the buffer. -# int (*WriteCallback)(void *ctx, const char *buffer, int length) -WRITE_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) -# int (*ReadCallback)(void *ctx, char *buffer, int bufferSize) -READ_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) - - -WriteCallback = Callable[[bytes], int] -ReadCallback = Callable[[int], bytes] - -_ENV_NATIVE_LIB = "DATAWEAVE_NATIVE_LIB" - - -@dataclass -class InputValue: - content: Union[str, bytes] - mime_type: Optional[str] = None - charset: Optional[str] = None - properties: Optional[Dict[str, Union[str, int, bool]]] = None - - def encode_content(self) -> str: - if isinstance(self.content, bytes): - raw = self.content - else: - raw = self.content.encode(self.charset or "utf-8") - return base64.b64encode(raw).decode("ascii") - - -@dataclass(repr=False) -class ExecutionResult: - success: bool - result: Optional[str] - error: Optional[str] - binary: bool - mime_type: Optional[str] - charset: Optional[str] - - def __repr__(self): - if not self.success: - return f"ExecutionResult(success=False, error={self.error!r})" - preview = (self.result[:50] + "...") if self.result and len(self.result) > 50 else self.result - return f"ExecutionResult(success=True, mime_type={self.mime_type!r}, charset={self.charset!r}, result={preview!r})" - - def get_bytes(self) -> Optional[bytes]: - if not self.success or self.result is None: - return None - return base64.b64decode(self.result) - - def get_string(self) -> Optional[str]: - if not self.success or self.result is None: - return None - if self.binary: - return self.result - return self.get_bytes().decode(self.charset or "utf-8") - - -@dataclass -class StreamingResult: - """Metadata returned after a streaming execution completes.""" - success: bool - error: Optional[str] - mime_type: Optional[str] - charset: Optional[str] - binary: bool - - -class Stream: - """Wrapper around a streaming generator that captures metadata. - - Iterate to consume output chunks. After iteration completes, - access ``.metadata`` for the :class:`StreamingResult`. - """ - - def __init__(self, gen: Generator[bytes, None, StreamingResult]): - self._gen = gen - self._metadata: Optional[StreamingResult] = None - - def __iter__(self): - return self - - def __next__(self) -> bytes: - try: - return next(self._gen) - except StopIteration as e: - self._metadata = e.value - raise - - @property - def metadata(self) -> Optional[StreamingResult]: - return self._metadata - - -def _parse_native_encoded_response(raw: str) -> ExecutionResult: - if raw is None: - return ExecutionResult(False, None, "Native returned null", False, None, None) - - if raw == "": - return ExecutionResult(False, None, "Native returned empty response", False, None, None) - - try: - parsed = json.loads(raw) - except Exception as e: - return ExecutionResult(False, None, f"Failed to parse native JSON response: {e}", False, None, None) - - if not isinstance(parsed, dict): - return ExecutionResult(False, None, "Native response JSON is not an object", False, None, None) - - success = bool(parsed.get("success", False)) - if not success: - return ExecutionResult(False, None, parsed.get("error"), False, None, None) - - return ExecutionResult( - success=True, - result=parsed.get("result"), - error=None, - binary=bool(parsed.get("binary", False)), - mime_type=parsed.get("mimeType"), - charset=parsed.get("charset"), - ) - - -def _parse_streaming_result(meta: dict) -> StreamingResult: - success = meta.get("success", False) - if not success: - return StreamingResult( - success=False, - error=meta.get("error"), - mime_type=None, - charset=None, - binary=False, - ) - return StreamingResult( - success=True, - error=None, - mime_type=meta.get("mimeType"), - charset=meta.get("charset"), - binary=meta.get("binary", False), - ) - - -def _candidate_library_paths() -> list[Path]: - paths: list[Path] = [] - - env_value = (__import__("os").environ.get(_ENV_NATIVE_LIB) or "").strip() - if env_value: - paths.append(Path(env_value)) - - pkg_dir = Path(__file__).resolve().parent - native_dir = pkg_dir / "native" - paths.append(native_dir / "dwlib.dylib") - paths.append(native_dir / "dwlib.so") - paths.append(native_dir / "dwlib.dll") - - # Dev fallback: if this package is being used from the data-weave-cli repo - # tree, locate native-lib/build/native/nativeCompile. - for parent in pkg_dir.parents: - build_dir = parent / "build" / "native" / "nativeCompile" - if build_dir.exists(): - paths.append(build_dir / "dwlib.dylib") - paths.append(build_dir / "dwlib.so") - paths.append(build_dir / "dwlib.dll") - break - - # CWD fallback - paths.append(Path("dwlib.dylib")) - paths.append(Path("dwlib.so")) - paths.append(Path("dwlib.dll")) - - return paths - - -def _find_library() -> str: - for p in _candidate_library_paths(): - if p.exists() and p.is_file(): - return str(p) - - raise DataWeaveLibraryNotFoundError( - "Could not find DataWeave native library (dwlib). " - f"Set {_ENV_NATIVE_LIB} to an absolute path or install a wheel that bundles the native library." - ) - - -def _normalize_input_value(value: Any, mime_type: Optional[str] = None) -> Dict[str, Any]: - if isinstance(value, dict): - allowed_keys = {"content", "mimeType", "charset", "properties"} - extra_keys = set(value.keys()) - allowed_keys - if extra_keys: - raise DataWeaveError( - "Explicit input dict contains unsupported keys: " + ", ".join(sorted(extra_keys)) - ) - - if "content" in value or "mimeType" in value: - if "content" not in value or "mimeType" not in value: - raise DataWeaveError( - "Explicit input dict must include both 'content' and 'mimeType'" - ) - - raw_content = value.get("content") - charset = value.get("charset") or "utf-8" - if isinstance(raw_content, bytes): - encoded_content = base64.b64encode(raw_content).decode("ascii") - else: - encoded_content = base64.b64encode(str(raw_content).encode(charset)).decode("ascii") - - normalized: Dict[str, Any] = { - "content": encoded_content, - "mimeType": value.get("mimeType"), - } - if "charset" in value: - normalized["charset"] = value.get("charset") - if "properties" in value: - normalized["properties"] = value.get("properties") - return normalized - - if isinstance(value, InputValue): - out: Dict[str, Any] = { - "content": value.encode_content(), - "mimeType": value.mime_type or mime_type, - } - if value.charset is not None: - out["charset"] = value.charset - if value.properties is not None: - out["properties"] = value.properties - return out - - if isinstance(value, str): - content = value - default_mime = "text/plain" - elif isinstance(value, (int, float, bool)): - content = json.dumps(value) - default_mime = "application/json" - elif value is None: - content = "null" - default_mime = "application/json" - else: - try: - content = json.dumps(value) - default_mime = "application/json" - except (TypeError, ValueError): - content = str(value) - default_mime = "text/plain" - - charset = "utf-8" - encoded_content = base64.b64encode(content.encode(charset)).decode("ascii") - - return { - "content": encoded_content, - "mimeType": mime_type or default_mime, - "charset": charset, - } - - -class DataWeave: - def __init__(self, lib_path: Optional[str] = None): - self._lib_path = lib_path or _find_library() - self._lib = None - self._isolate = None - self._thread = None - self._initialized = False - - def _load_library(self): - try: - self._lib = ctypes.CDLL(self._lib_path) - except OSError as e: - raise DataWeaveError(f"Failed to load library from {self._lib_path}: {e}") - - def _setup_graal_structures(self): - class graal_isolate_t(ctypes.Structure): - pass - - class graal_isolatethread_t(ctypes.Structure): - pass - - self._graal_isolate_t_ptr = ctypes.POINTER(graal_isolate_t) - self._graal_isolatethread_t_ptr = ctypes.POINTER(graal_isolatethread_t) - - def _create_isolate(self): - self._lib.graal_create_isolate.argtypes = [ - ctypes.c_void_p, - ctypes.POINTER(self._graal_isolate_t_ptr), - ctypes.POINTER(self._graal_isolatethread_t_ptr), - ] - self._lib.graal_create_isolate.restype = ctypes.c_int - - self._isolate = self._graal_isolate_t_ptr() - self._thread = self._graal_isolatethread_t_ptr() - - result = self._lib.graal_create_isolate(None, ctypes.byref(self._isolate), ctypes.byref(self._thread)) - if result != 0: - raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") - - def _setup_functions(self): - if not hasattr(self._lib, "run_script"): - raise DataWeaveError("Native library does not export run_script") - - self._lib.run_script.argtypes = [ - self._graal_isolatethread_t_ptr, - ctypes.c_char_p, - ctypes.c_char_p, - ] - self._lib.run_script.restype = ctypes.c_void_p - - if hasattr(self._lib, "free_cstring"): - self._lib.free_cstring.argtypes = [self._graal_isolatethread_t_ptr, ctypes.c_void_p] - self._lib.free_cstring.restype = None - - # Thread attachment for background threads - if hasattr(self._lib, "graal_attach_thread"): - self._lib.graal_attach_thread.argtypes = [self._graal_isolate_t_ptr, ctypes.POINTER(self._graal_isolatethread_t_ptr)] - self._lib.graal_attach_thread.restype = ctypes.c_int - if hasattr(self._lib, "graal_detach_thread"): - self._lib.graal_detach_thread.argtypes = [self._graal_isolatethread_t_ptr] - self._lib.graal_detach_thread.restype = ctypes.c_int - if hasattr(self._lib, "graal_tear_down_isolate"): - self._lib.graal_tear_down_isolate.argtypes = [self._graal_isolatethread_t_ptr] - self._lib.graal_tear_down_isolate.restype = ctypes.c_int - - # Callback-based Streaming API - if hasattr(self._lib, "run_script_callback"): - self._lib.run_script_callback.argtypes = [ - self._graal_isolatethread_t_ptr, - ctypes.c_char_p, - ctypes.c_char_p, - WRITE_CALLBACK, - ctypes.c_void_p, - ] - self._lib.run_script_callback.restype = ctypes.c_void_p - - self._has_callback_streaming = True - else: - self._has_callback_streaming = False - - if hasattr(self._lib, "run_script_input_output_callback"): - self._lib.run_script_input_output_callback.argtypes = [ - self._graal_isolatethread_t_ptr, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_char_p, - READ_CALLBACK, - WRITE_CALLBACK, - ctypes.c_void_p, - ] - self._lib.run_script_input_output_callback.restype = ctypes.c_void_p - - self._has_callback_input_output = True - else: - self._has_callback_input_output = False - - def _decode_and_free(self, ptr: Optional[int]) -> str: - if not ptr: - return "" - - try: - result_bytes = ctypes.string_at(ptr) - return result_bytes.decode("utf-8") - finally: - if self._lib is not None and hasattr(self._lib, "free_cstring"): - self._lib.free_cstring(self._thread, ptr) - - def initialize(self): - if self._initialized: - return - - self._load_library() - self._setup_graal_structures() - self._create_isolate() - self._setup_functions() - self._initialized = True - - def cleanup(self): - if not self._initialized: - return - - if hasattr(self._lib, "graal_tear_down_isolate") and self._thread: - try: - self._lib.graal_tear_down_isolate(self._thread) - except Exception: - pass - elif hasattr(self._lib, "graal_detach_thread") and self._thread: - try: - self._lib.graal_detach_thread(self._thread) - except Exception: - pass - - self._initialized = False - self._thread = None - self._isolate = None - self._lib = None - - def run_callback( - self, - script: str, - write_callback: WriteCallback, - inputs: Optional[Dict[str, Any]] = None, - ) -> StreamingResult: - """Execute a DataWeave script and stream the output via a write callback. - - The native side reads the output internally and invokes *write_callback* - for each chunk. - - :param script: the DataWeave script source - :param write_callback: callable ``(data: bytes) -> int`` invoked with each - output chunk. Must return ``0`` on success or non-zero to abort. - :param inputs: optional input bindings (same format as :meth:`run`) - :return: a :class:`StreamingResult` with metadata - :raises DataWeaveError: if the runtime is not initialized or the callback API - is not available - """ - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_streaming: - raise DataWeaveError( - "Native library does not support callback streaming API (run_script_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - data = ctypes.string_at(buf, length) - return write_callback(data) - except Exception: - return -1 - - try: - result_ptr = self._lib.run_script_callback( - self._thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - _write_cb, - None, - ) - raw = self._decode_and_free(result_ptr) - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - except Exception as e: - raise DataWeaveError(f"Failed to execute callback streaming: {e}") - - return _parse_streaming_result(meta) - - def run_streaming( - self, - script: str, - inputs: Optional[Dict[str, Any]] = None, - ) -> Stream: - """Execute a DataWeave script and yield output chunks as they arrive. - - Chunks are yielded in real-time as the native engine produces them, - using a background thread and queue. The caller sees data before the - script finishes executing. - - Usage:: - - with DataWeave() as dw: - stream = dw.run_streaming("output json --- {items: (1 to 100)}") - for chunk in stream: - sys.stdout.buffer.write(chunk) - metadata = stream.metadata # StreamingResult with mime_type, charset, etc. - - :param script: the DataWeave script source - :param inputs: optional input bindings (same format as :meth:`run`) - :return: a :class:`Stream` yielding ``bytes`` chunks; after iteration, - ``.metadata`` holds a :class:`StreamingResult` - :raises DataWeaveError: if the runtime is not initialized or the callback API - is not available - """ - return Stream(self._run_streaming_gen(script, inputs)) - - def _run_streaming_gen( - self, - script: str, - inputs: Optional[Dict[str, Any]] = None, - ) -> Generator[bytes, None, StreamingResult]: - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_streaming: - raise DataWeaveError( - "Native library does not support callback streaming API (run_script_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - _SENTINEL = object() - q: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - # With maxsize set, put() blocks when the queue is full, exerting - # backpressure onto the native producer. Timeout prevents indefinite - # blocking if the consumer abandons the generator. - q.put(ctypes.string_at(buf, length), timeout=30) - return 0 - except Exception: - # Timeout or other failure: signal the native side to abort. - return -1 - - def _run_native(): - worker_thread = self._graal_isolatethread_t_ptr() - rc = self._lib.graal_attach_thread(self._isolate, ctypes.byref(worker_thread)) - if rc != 0: - q.put({"success": False, "error": f"Failed to attach worker thread to isolate (code {rc})"}) - q.put(_SENTINEL) - return - try: - result_ptr = self._lib.run_script_callback( - worker_thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - _write_cb, - None, - ) - raw_ptr = result_ptr - if raw_ptr: - raw = ctypes.string_at(raw_ptr).decode("utf-8") - self._lib.free_cstring(worker_thread, raw_ptr) - else: - raw = "" - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - q.put(meta) - except Exception as e: - q.put({"success": False, "error": str(e)}) - finally: - self._lib.graal_detach_thread(worker_thread) - q.put(_SENTINEL) - - worker = Thread(target=_run_native, name="dw-streaming-worker", daemon=False) - worker.start() - - meta = None - while True: - item = q.get() - if item is _SENTINEL: - break - if isinstance(item, dict): - meta = item - else: - yield item - - worker.join(timeout=30) - if worker.is_alive(): - raise DataWeaveError("Worker thread timeout after 30 seconds") - - if meta is None: - meta = {"success": False, "error": "No metadata received from native call"} - - success = meta.get("success", False) - if not success: - return StreamingResult( - success=False, - error=meta.get("error"), - mime_type=None, - charset=None, - binary=False, - ) - - return StreamingResult( - success=True, - error=None, - mime_type=meta.get("mimeType"), - charset=meta.get("charset"), - binary=meta.get("binary", False), - ) - - def run_transform( - self, - script: str, - input_stream: Iterable[bytes], - input_name: str = "payload", - input_mime_type: str = "application/json", - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, - ) -> Stream: - """Execute a DataWeave script with streaming input and output. - - Input data is pulled from *input_stream* (any iterable of bytes) and - output chunks are yielded as they are produced — fully streaming in - both directions with constant memory overhead. - - Usage:: - - with DataWeave() as dw: - with open("large.json", "rb") as f: - stream = dw.run_transform( - "output application/csv --- payload", - input_stream=iter(lambda: f.read(8192), b""), - input_mime_type="application/json", - ) - for chunk in stream: - sys.stdout.buffer.write(chunk) - metadata = stream.metadata - - :param script: the DataWeave script source - :param input_stream: iterable yielding ``bytes`` chunks for the input binding - :param input_name: binding name for the streamed input (default ``"payload"``) - :param input_mime_type: MIME type of the streamed input - :param input_charset: charset of the streamed input (default UTF-8) - :param inputs: optional additional input bindings (same format as :meth:`run`) - :return: a :class:`Stream` yielding ``bytes`` output chunks; after iteration, - ``.metadata`` holds a :class:`StreamingResult` - :raises DataWeaveError: if the runtime is not initialized or the API is missing - """ - return Stream(self._run_transform_gen( - script, input_stream, input_name, input_mime_type, input_charset, inputs, - )) - - def _run_transform_gen( - self, - script: str, - input_stream: Iterable[bytes], - input_name: str = "payload", - input_mime_type: str = "application/json", - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, - ) -> Generator[bytes, None, StreamingResult]: - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_input_output: - raise DataWeaveError( - "Native library does not support callback input/output API " - "(run_script_input_output_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - _SENTINEL = object() - q: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - # With maxsize set, put() blocks when the queue is full, exerting - # backpressure onto the native producer. Timeout prevents indefinite - # blocking if the consumer abandons the generator. - q.put(ctypes.string_at(buf, length), timeout=30) - return 0 - except Exception: - # Timeout or other failure: signal the native side to abort. - return -1 - - input_iter = iter(input_stream) - # Stateful reader to handle chunks larger than native buffer size. - # Mirrors Node binding's createChunkReader: maintains current chunk + offset, - # returns at most buf_size bytes per call, advances to next chunk only when - # current chunk is fully consumed. - read_state = {"current_chunk": b"", "offset": 0, "done": False} - - @READ_CALLBACK - def _read_cb(_ctx, buf, buf_size): - try: - while True: - # If we have buffered data, return what we can - if read_state["offset"] < len(read_state["current_chunk"]): - chunk = read_state["current_chunk"] - offset = read_state["offset"] - n = min(len(chunk) - offset, buf_size) - ctypes.memmove(buf, chunk[offset:offset+n], n) - read_state["offset"] += n - return n - # Current chunk exhausted and iterator done -> EOF - if read_state["done"]: - return 0 - # Pull next chunk - data = next(input_iter, None) - if data is None or not data: - read_state["done"] = True - return 0 - read_state["current_chunk"] = data - read_state["offset"] = 0 - # Loop back to serve from the new chunk - except Exception: - return -1 - - def _run_native(): - worker_thread = self._graal_isolatethread_t_ptr() - rc = self._lib.graal_attach_thread(self._isolate, ctypes.byref(worker_thread)) - if rc != 0: - q.put({"success": False, "error": f"Failed to attach worker thread to isolate (code {rc})"}) - q.put(_SENTINEL) - return - try: - result_ptr = self._lib.run_script_input_output_callback( - worker_thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - input_name.encode("utf-8"), - input_mime_type.encode("utf-8"), - input_charset.encode("utf-8") if input_charset else None, - _read_cb, - _write_cb, - None, - ) - if result_ptr: - raw = ctypes.string_at(result_ptr).decode("utf-8") - self._lib.free_cstring(worker_thread, result_ptr) - else: - raw = "" - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - q.put(meta) - except Exception as e: - q.put({"success": False, "error": str(e)}) - finally: - self._lib.graal_detach_thread(worker_thread) - q.put(_SENTINEL) - - worker = Thread(target=_run_native, name="dw-transform-worker", daemon=False) - worker.start() - - meta = None - while True: - item = q.get() - if item is _SENTINEL: - break - if isinstance(item, dict): - meta = item - else: - yield item - - worker.join(timeout=30) - if worker.is_alive(): - raise DataWeaveError("Worker thread timeout after 30 seconds") - - if meta is None: - meta = {"success": False, "error": "No metadata received from native call"} - - success = meta.get("success", False) - if not success: - return StreamingResult( - success=False, - error=meta.get("error"), - mime_type=None, - charset=None, - binary=False, - ) - - return StreamingResult( - success=True, - error=None, - mime_type=meta.get("mimeType"), - charset=meta.get("charset"), - binary=meta.get("binary", False), - ) - - def run_input_output_callback( - self, - script: str, - input_name: str, - input_mime_type: str, - read_callback: ReadCallback, - write_callback: WriteCallback, - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, - ) -> StreamingResult: - """Execute a DataWeave script with callback-driven input *and* output streaming. - - The native side calls *read_callback* on a background thread to pull input - data for the binding named *input_name*, and calls *write_callback* on the - calling thread to push output chunks. - - :param script: the DataWeave script source - :param input_name: the binding name for the callback-supplied input - :param input_mime_type: MIME type of the callback-supplied input - :param read_callback: callable ``(buf_size: int) -> bytes`` returning the - next chunk, empty bytes ``b""`` on EOF, or raising on error - :param write_callback: callable ``(data: bytes) -> int`` returning ``0`` on - success or non-zero to abort - :param input_charset: charset of the callback-supplied input (default UTF-8) - :param inputs: optional additional input bindings - :return: a :class:`StreamingResult` with metadata - :raises DataWeaveError: if the runtime is not initialized or the API is missing - """ - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - if not self._has_callback_input_output: - raise DataWeaveError( - "Native library does not support callback input/output API " - "(run_script_input_output_callback not found)." - ) - - if inputs is None: - inputs = {} - - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - @READ_CALLBACK - def _read_cb(_ctx, buf, buf_size): - try: - data = read_callback(buf_size) - if not data: - return 0 # EOF - n = min(len(data), buf_size) - ctypes.memmove(buf, data, n) - return n - except Exception: - return -1 - - @WRITE_CALLBACK - def _write_cb(_ctx, buf, length): - try: - data = ctypes.string_at(buf, length) - return write_callback(data) - except Exception: - return -1 - - try: - result_ptr = self._lib.run_script_input_output_callback( - self._thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - input_name.encode("utf-8"), - input_mime_type.encode("utf-8"), - input_charset.encode("utf-8") if input_charset else None, - _read_cb, - _write_cb, - None, - ) - raw = self._decode_and_free(result_ptr) - meta = json.loads(raw) if raw else {"success": False, "error": "Empty response"} - except Exception as e: - raise DataWeaveError(f"Failed to execute callback input/output streaming: {e}") - - return _parse_streaming_result(meta) - - def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bool = False) -> ExecutionResult: - if not self._initialized: - raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") - - if inputs is None: - inputs = {} - - normalized_inputs = {key: _normalize_input_value(val) for key, val in inputs.items()} - inputs_json = json.dumps(normalized_inputs) - - try: - result_ptr = self._lib.run_script( - self._thread, - script.encode("utf-8"), - inputs_json.encode("utf-8"), - ) - raw = self._decode_and_free(result_ptr) - result = _parse_native_encoded_response(raw) - except Exception as e: - raise DataWeaveError(f"Failed to execute script: {e}") - - if raise_on_error and not result.success: - raise DataWeaveScriptError(result) - return result - - def __enter__(self): - self.initialize() - return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.cleanup() - return False +from typing import Any, Dict, Iterable, Optional + +from .encoding import normalize_input_value as _normalize_input_value +from .encoding import parse_native_encoded_response as _parse_native_encoded_response +from .encoding import parse_streaming_result as _parse_streaming_result +from .models import ( + READ_CALLBACK, + WRITE_CALLBACK, + DataWeaveError, + DataWeaveLibraryNotFoundError, + DataWeaveScriptError, + ExecutionResult, + InputValue, + ReadCallback, + Stream, + StreamingResult, + WriteCallback, +) +from .native import candidate_library_paths as _candidate_library_paths +from .native import find_library as _find_library +from .runtime import DataWeave _global_instance: Optional[DataWeave] = None @@ -967,48 +42,23 @@ def run(script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bo return _get_global_instance().run(script, inputs, raise_on_error=raise_on_error) -def run_streaming( - script: str, inputs: Optional[Dict[str, Any]] = None, -) -> Stream: - """Execute a script and yield output chunks. See :meth:`DataWeave.run_streaming`.""" +def run_streaming(script: str, inputs: Optional[Dict[str, Any]] = None) -> Stream: return _get_global_instance().run_streaming(script, inputs) def run_callback(script: str, write_callback: WriteCallback, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: - """Execute a script and stream output via a write callback. See :meth:`DataWeave.run_callback`.""" return _get_global_instance().run_callback(script, write_callback, inputs) -def run_transform( - script: str, - input_stream: Iterable[bytes], - input_name: str = "payload", - input_mime_type: str = "application/json", - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, -) -> Stream: - """Execute a script with streaming input and output. See :meth:`DataWeave.run_transform`.""" - return _get_global_instance().run_transform( - script, input_stream, input_name, input_mime_type, input_charset, inputs, - ) +def run_transform(script: str, input_stream: Iterable[bytes], input_name: str = "payload", input_mime_type: str = "application/json", input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> Stream: + return _get_global_instance().run_transform(script, input_stream, input_name, input_mime_type, input_charset, inputs) -def run_input_output_callback( - script: str, - input_name: str, - input_mime_type: str, - read_callback: ReadCallback, - write_callback: WriteCallback, - input_charset: Optional[str] = None, - inputs: Optional[Dict[str, Any]] = None, -) -> StreamingResult: - """Execute a script with callback-driven input and output. See :meth:`DataWeave.run_input_output_callback`.""" - return _get_global_instance().run_input_output_callback( - script, input_name, input_mime_type, read_callback, write_callback, input_charset, inputs, - ) +def run_input_output_callback(script: str, input_name: str, input_mime_type: str, read_callback: ReadCallback, write_callback: WriteCallback, input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: + return _get_global_instance().run_input_output_callback(script, input_name, input_mime_type, read_callback, write_callback, input_charset, inputs) -def cleanup(): +def cleanup() -> None: global _global_instance if _global_instance is not None: _global_instance.cleanup() @@ -1016,22 +66,8 @@ def cleanup(): __all__ = [ - "DataWeave", - "DataWeaveError", - "DataWeaveLibraryNotFoundError", - "DataWeaveScriptError", - "ExecutionResult", - "InputValue", - "ReadCallback", - "Stream", - "StreamingResult", - "WriteCallback", - "READ_CALLBACK", - "WRITE_CALLBACK", - "run", - "run_callback", - "run_input_output_callback", - "run_streaming", - "run_transform", - "cleanup", + "DataWeave", "DataWeaveError", "DataWeaveLibraryNotFoundError", "DataWeaveScriptError", + "ExecutionResult", "InputValue", "ReadCallback", "Stream", "StreamingResult", "WriteCallback", + "READ_CALLBACK", "WRITE_CALLBACK", "run", "run_callback", "run_input_output_callback", + "run_streaming", "run_transform", "cleanup", ] diff --git a/native-lib/python/src/dataweave/encoding.py b/native-lib/python/src/dataweave/encoding.py new file mode 100644 index 00000000..0f3c0881 --- /dev/null +++ b/native-lib/python/src/dataweave/encoding.py @@ -0,0 +1,123 @@ +import base64 +import json +from typing import Any, Dict, Optional + +from .models import DataWeaveError, ExecutionResult, InputValue, StreamingResult + + +def parse_native_encoded_response(raw: str) -> ExecutionResult: + if raw is None: + return ExecutionResult(False, None, "Native returned null", False, None, None) + + if raw == "": + return ExecutionResult(False, None, "Native returned empty response", False, None, None) + + try: + parsed = json.loads(raw) + except Exception as e: + return ExecutionResult(False, None, f"Failed to parse native JSON response: {e}", False, None, None) + + if not isinstance(parsed, dict): + return ExecutionResult(False, None, "Native response JSON is not an object", False, None, None) + + success = bool(parsed.get("success", False)) + if not success: + return ExecutionResult(False, None, parsed.get("error"), False, None, None) + + return ExecutionResult( + success=True, + result=parsed.get("result"), + error=None, + binary=bool(parsed.get("binary", False)), + mime_type=parsed.get("mimeType"), + charset=parsed.get("charset"), + ) + + +def parse_streaming_result(meta: dict) -> StreamingResult: + success = meta.get("success", False) + if not success: + return StreamingResult( + success=False, + error=meta.get("error"), + mime_type=None, + charset=None, + binary=False, + ) + return StreamingResult( + success=True, + error=None, + mime_type=meta.get("mimeType"), + charset=meta.get("charset"), + binary=meta.get("binary", False), + ) + + +def normalize_input_value(value: Any, mime_type: Optional[str] = None) -> Dict[str, Any]: + if isinstance(value, dict): + allowed_keys = {"content", "mimeType", "charset", "properties"} + extra_keys = set(value.keys()) - allowed_keys + if extra_keys: + raise DataWeaveError( + "Explicit input dict contains unsupported keys: " + ", ".join(sorted(extra_keys)) + ) + + if "content" in value or "mimeType" in value: + if "content" not in value or "mimeType" not in value: + raise DataWeaveError( + "Explicit input dict must include both 'content' and 'mimeType'" + ) + + raw_content = value.get("content") + charset = value.get("charset") or "utf-8" + if isinstance(raw_content, bytes): + encoded_content = base64.b64encode(raw_content).decode("ascii") + else: + encoded_content = base64.b64encode(str(raw_content).encode(charset)).decode("ascii") + + normalized: Dict[str, Any] = { + "content": encoded_content, + "mimeType": value.get("mimeType"), + } + if "charset" in value: + normalized["charset"] = value.get("charset") + if "properties" in value: + normalized["properties"] = value.get("properties") + return normalized + + if isinstance(value, InputValue): + out: Dict[str, Any] = { + "content": value.encode_content(), + "mimeType": value.mime_type or mime_type, + } + if value.charset is not None: + out["charset"] = value.charset + if value.properties is not None: + out["properties"] = value.properties + return out + + if isinstance(value, str): + content = value + default_mime = "text/plain" + elif isinstance(value, (int, float, bool)): + content = json.dumps(value) + default_mime = "application/json" + elif value is None: + content = "null" + default_mime = "application/json" + else: + try: + content = json.dumps(value) + default_mime = "application/json" + except (TypeError, ValueError): + content = str(value) + default_mime = "text/plain" + + charset = "utf-8" + encoded_content = base64.b64encode(content.encode(charset)).decode("ascii") + + return { + "content": encoded_content, + "mimeType": mime_type or default_mime, + "charset": charset, + } diff --git a/native-lib/python/src/dataweave/models.py b/native-lib/python/src/dataweave/models.py new file mode 100644 index 00000000..ae33ceac --- /dev/null +++ b/native-lib/python/src/dataweave/models.py @@ -0,0 +1,139 @@ +import base64 +import ctypes +from dataclasses import dataclass +from typing import Callable, Dict, Generator, Optional, Union + + +class DataWeaveError(Exception): + pass + + +class DataWeaveScriptError(DataWeaveError): + """Raised when a DataWeave script fails (compile or runtime error). + + Carries the full result object so callers can inspect details. + """ + + def __init__(self, result): + self.result = result + super().__init__(result.error or "Script execution failed") + + +class DataWeaveLibraryNotFoundError(Exception): + pass + + +# ctypes callback signatures matching NativeCallbacks.WriteCallback / ReadCallback. +# Buffer parameters use c_void_p (not c_char_p) because ctypes gives c_char_p +# special treatment that prevents writing into the buffer. +# int (*WriteCallback)(void *ctx, const char *buffer, int length) +WRITE_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) +# int (*ReadCallback)(void *ctx, char *buffer, int bufferSize) +READ_CALLBACK = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int) + + +WriteCallback = Callable[[bytes], int] +ReadCallback = Callable[[int], bytes] + + +@dataclass +class InputValue: + content: Union[str, bytes] + mime_type: Optional[str] = None + charset: Optional[str] = None + properties: Optional[Dict[str, Union[str, int, bool]]] = None + + def encode_content(self) -> str: + if isinstance(self.content, bytes): + raw = self.content + else: + raw = self.content.encode(self.charset or "utf-8") + return base64.b64encode(raw).decode("ascii") + + +@dataclass(repr=False) +class ExecutionResult: + success: bool + result: Optional[str] + error: Optional[str] + binary: bool + mime_type: Optional[str] + charset: Optional[str] + + def __repr__(self): + if not self.success: + return f"ExecutionResult(success=False, error={self.error!r})" + preview = (self.result[:50] + "...") if self.result and len(self.result) > 50 else self.result + return f"ExecutionResult(success=True, mime_type={self.mime_type!r}, charset={self.charset!r}, result={preview!r})" + + def get_bytes(self) -> Optional[bytes]: + if not self.success or self.result is None: + return None + return base64.b64decode(self.result) + + def get_string(self) -> Optional[str]: + if not self.success or self.result is None: + return None + if self.binary: + return self.result + return self.get_bytes().decode(self.charset or "utf-8") + + +@dataclass +class StreamingResult: + """Metadata returned after a streaming execution completes.""" + success: bool + error: Optional[str] + mime_type: Optional[str] + charset: Optional[str] + binary: bool + + +class Stream: + """Wrapper around a streaming generator that captures metadata. + + Iterate to consume output chunks. After iteration completes, + access ``.metadata`` for the :class:`StreamingResult`. + """ + + def __init__(self, gen: Generator[bytes, None, StreamingResult]): + self._gen = gen + self._metadata: Optional[StreamingResult] = None + + def __iter__(self): + return self + + def __next__(self) -> bytes: + try: + return next(self._gen) + except StopIteration as e: + self._metadata = e.value + raise + + @property + def metadata(self) -> Optional[StreamingResult]: + return self._metadata + + def close(self) -> None: + """Stop consuming output and request bounded worker cleanup.""" + on_close = getattr(self, "_on_close", None) + if on_close is not None: + on_close() + try: + self._gen.close() + except Exception: + # Stream cancellation is best-effort; native calls cannot be forcibly + # interrupted from Python and cleanup must not escape finalization. + pass + + def __enter__(self): + return self + + def __exit__(self, _exc_type, _exc_val, _exc_tb): + self.close() + return False + + def __del__(self): + self.close() + + _close = close diff --git a/native-lib/python/src/dataweave/native.py b/native-lib/python/src/dataweave/native.py new file mode 100644 index 00000000..98f771a1 --- /dev/null +++ b/native-lib/python/src/dataweave/native.py @@ -0,0 +1,210 @@ +import ctypes +import os +from pathlib import Path +from typing import Optional + +from .models import DataWeaveError, DataWeaveLibraryNotFoundError, READ_CALLBACK, WRITE_CALLBACK + + +_ENV_NATIVE_LIB = "DATAWEAVE_NATIVE_LIB" + + +class graal_isolate_t(ctypes.Structure): + pass + + +class graal_isolatethread_t(ctypes.Structure): + pass + + +GraalIsolatePointer = ctypes.POINTER(graal_isolate_t) +GraalIsolateThreadPointer = ctypes.POINTER(graal_isolatethread_t) + + +def candidate_library_paths() -> list[Path]: + paths: list[Path] = [] + env_value = (os.environ.get(_ENV_NATIVE_LIB) or "").strip() + if env_value: + paths.append(Path(env_value)) + + pkg_dir = Path(__file__).resolve().parent + native_dir = pkg_dir / "native" + paths.extend(native_dir / name for name in ("dwlib.dylib", "dwlib.so", "dwlib.dll")) + + for parent in pkg_dir.parents: + build_dir = parent / "build" / "native" / "nativeCompile" + if build_dir.exists(): + paths.extend(build_dir / name for name in ("dwlib.dylib", "dwlib.so", "dwlib.dll")) + break + + paths.extend(Path(name) for name in ("dwlib.dylib", "dwlib.so", "dwlib.dll")) + return paths + + +def find_library() -> str: + for path in candidate_library_paths(): + if path.exists() and path.is_file(): + return str(path) + raise DataWeaveLibraryNotFoundError( + "Could not find DataWeave native library (dwlib). " + f"Set {_ENV_NATIVE_LIB} to an absolute path or install a wheel that bundles the native library." + ) + + +class NativeRuntime: + """Owns the native library handle, isolate lifecycle, and ctypes ABI.""" + + def __init__(self, lib_path: Optional[str] = None): + self.lib_path = lib_path or find_library() + self.lib = None + self.isolate = None + self.thread = None + self.initialized = False + self.has_callback_streaming = False + self.has_callback_input_output = False + + def initialize(self) -> None: + if self.initialized: + return + try: + self.lib = ctypes.CDLL(self.lib_path) + except OSError as error: + raise DataWeaveError(f"Failed to load library from {self.lib_path}: {error}") + isolate_created = False + try: + self._create_isolate() + isolate_created = True + self._setup_functions() + self.initialized = True + except Exception: + if isolate_created: + self._tear_down_isolate(suppress_errors=True) + self._reset() + raise + + def _create_isolate(self) -> None: + self._require_export("graal_create_isolate") + self.lib.graal_create_isolate.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(GraalIsolatePointer), + ctypes.POINTER(GraalIsolateThreadPointer), + ] + self.lib.graal_create_isolate.restype = ctypes.c_int + self.isolate = GraalIsolatePointer() + self.thread = GraalIsolateThreadPointer() + try: + result = self.lib.graal_create_isolate(None, ctypes.byref(self.isolate), ctypes.byref(self.thread)) + except Exception as error: + raise DataWeaveError(f"Failed to create GraalVM isolate: {error}") from error + if result != 0: + raise DataWeaveError(f"Failed to create GraalVM isolate. Error code: {result}") + + def _setup_functions(self) -> None: + self._require_export("run_script") + self._require_export("free_cstring") + self._require_export("graal_tear_down_isolate") + self.lib.run_script.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p] + self.lib.run_script.restype = ctypes.c_void_p + self.lib.free_cstring.argtypes = [GraalIsolateThreadPointer, ctypes.c_void_p] + self.lib.free_cstring.restype = None + self.lib.graal_tear_down_isolate.argtypes = [GraalIsolateThreadPointer] + self.lib.graal_tear_down_isolate.restype = ctypes.c_int + if hasattr(self.lib, "run_script_callback"): + self._require_streaming_lifecycle_exports("run_script_callback") + self.lib.run_script_callback.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p, WRITE_CALLBACK, ctypes.c_void_p] + self.lib.run_script_callback.restype = ctypes.c_void_p + self.has_callback_streaming = True + if hasattr(self.lib, "run_script_input_output_callback"): + self._require_streaming_lifecycle_exports("run_script_input_output_callback") + self.lib.run_script_input_output_callback.argtypes = [GraalIsolateThreadPointer, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, READ_CALLBACK, WRITE_CALLBACK, ctypes.c_void_p] + self.lib.run_script_input_output_callback.restype = ctypes.c_void_p + self.has_callback_input_output = True + + def _require_export(self, name: str) -> None: + if not hasattr(self.lib, name): + raise DataWeaveError(f"Native library does not export {name}") + + def _require_streaming_lifecycle_exports(self, callback_name: str) -> None: + for name in ("free_cstring", "graal_attach_thread", "graal_detach_thread"): + if not hasattr(self.lib, name): + raise DataWeaveError(f"{callback_name} requires native export {name}") + self.lib.graal_attach_thread.argtypes = [GraalIsolatePointer, ctypes.POINTER(GraalIsolateThreadPointer)] + self.lib.graal_attach_thread.restype = ctypes.c_int + self.lib.graal_detach_thread.argtypes = [GraalIsolateThreadPointer] + self.lib.graal_detach_thread.restype = ctypes.c_int + + def attach_thread(self): + worker_thread = GraalIsolateThreadPointer() + try: + result = self.lib.graal_attach_thread(self.isolate, ctypes.byref(worker_thread)) + except Exception as error: + raise DataWeaveError(f"Failed to attach worker thread to isolate: {error}") from error + if result != 0: + raise DataWeaveError(f"Failed to attach worker thread to isolate (code {result})") + return worker_thread + + def detach_thread(self, thread) -> None: + try: + result = self.lib.graal_detach_thread(thread) + except Exception as error: + raise DataWeaveError(f"Failed to detach worker thread from isolate: {error}") from error + if result != 0: + raise DataWeaveError(f"Failed to detach worker thread from isolate. Error code: {result}") + + def decode_and_free(self, ptr, thread=None) -> str: + if not ptr: + return "" + primary_error = None + try: + return ctypes.string_at(ptr).decode("utf-8") + except Exception as error: + primary_error = error + raise + finally: + try: + if self.lib is not None: + self.lib.free_cstring(thread or self.thread, ptr) + except Exception: + if primary_error is None: + raise + + def run_script(self, thread, script: bytes, inputs: bytes): + return self.lib.run_script(thread, script, inputs) + + def run_script_callback(self, thread, script: bytes, inputs: bytes, write_callback): + return self.lib.run_script_callback(thread, script, inputs, write_callback, None) + + def run_script_input_output_callback(self, thread, script: bytes, inputs: bytes, input_name: bytes, input_mime_type: bytes, input_charset: Optional[bytes], read_callback, write_callback): + return self.lib.run_script_input_output_callback( + thread, script, inputs, input_name, input_mime_type, input_charset, read_callback, write_callback, None, + ) + + def cleanup(self) -> None: + if not self.initialized: + return + try: + self._tear_down_isolate() + finally: + self._reset() + + def _tear_down_isolate(self, suppress_errors: bool = False) -> None: + if self.thread is None: + return + try: + result = self.lib.graal_tear_down_isolate(self.thread) + if result != 0: + raise DataWeaveError(f"Failed to tear down GraalVM isolate. Error code: {result}") + except DataWeaveError: + if not suppress_errors: + raise + except Exception as error: + if not suppress_errors: + raise DataWeaveError(f"Failed to tear down GraalVM isolate: {error}") from error + + def _reset(self) -> None: + self.initialized = False + self.thread = None + self.isolate = None + self.lib = None + self.has_callback_streaming = False + self.has_callback_input_output = False diff --git a/native-lib/python/src/dataweave/runtime.py b/native-lib/python/src/dataweave/runtime.py new file mode 100644 index 00000000..8d57d918 --- /dev/null +++ b/native-lib/python/src/dataweave/runtime.py @@ -0,0 +1,269 @@ +import ctypes +import json +from queue import Empty, Full, Queue +from threading import Event, Lock, Thread, current_thread +from typing import Any, Dict, Generator, Iterable, Optional + +from .encoding import normalize_input_value, parse_native_encoded_response, parse_streaming_result +from .models import ( + READ_CALLBACK, + WRITE_CALLBACK, + DataWeaveError, + DataWeaveScriptError, + ExecutionResult, + ReadCallback, + Stream, + StreamingResult, + WriteCallback, +) +from .native import NativeRuntime + + +_OUTPUT_QUEUE_MAXSIZE = 512 +_WORKER_TIMEOUT_SECONDS = 30 +_WORKER_JOIN_TIMEOUT_SECONDS = 0.1 + + +class DataWeave: + """High-level execution API backed by a :class:`NativeRuntime`.""" + + def __init__(self, lib_path: Optional[str] = None): + self._native = NativeRuntime(lib_path) + self._stream_workers = set() + self._stream_workers_lock = Lock() + self._cleaning_up = False + + def initialize(self): + self._native.initialize() + + def cleanup(self): + workers, lock = self._worker_registry() + with lock: + if workers: + raise DataWeaveError("Cannot clean up DataWeave runtime while an active streaming worker is attached.") + self._cleaning_up = True + try: + self._native.cleanup() + finally: + with lock: + self._cleaning_up = False + + def _worker_registry(self): + if not hasattr(self, "_stream_workers"): + self._stream_workers = set() + self._stream_workers_lock = Lock() + return self._stream_workers, self._stream_workers_lock + + def _register_stream_worker(self, worker: Thread) -> None: + workers, lock = self._worker_registry() + with lock: + if getattr(self, "_cleaning_up", False): + raise DataWeaveError("Cannot start a streaming worker while the DataWeave runtime is being cleaned up.") + workers.add(worker) + + def _unregister_stream_worker(self, worker: Optional[Thread] = None) -> None: + workers, lock = self._worker_registry() + with lock: + workers.discard(worker or current_thread()) + + def _require_initialized(self, supported: bool, api_name: str) -> None: + if not self._native.initialized: + raise DataWeaveError("DataWeave runtime not initialized. Call initialize() first.") + if not supported: + raise DataWeaveError(f"Native library does not support {api_name}.") + + @staticmethod + def _inputs_json(inputs: Optional[Dict[str, Any]]) -> bytes: + return json.dumps({key: normalize_input_value(value) for key, value in (inputs or {}).items()}).encode("utf-8") + + def run(self, script: str, inputs: Optional[Dict[str, Any]] = None, raise_on_error: bool = False) -> ExecutionResult: + self._require_initialized(True, "script execution") + try: + raw = self._native.decode_and_free(self._native.run_script(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs))) + result = parse_native_encoded_response(raw) + except Exception as error: + raise DataWeaveError(f"Failed to execute script: {error}") + if raise_on_error and not result.success: + raise DataWeaveScriptError(result) + return result + + def run_callback(self, script: str, write_callback: WriteCallback, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: + self._require_initialized(self._native.has_callback_streaming, "callback streaming API (run_script_callback not found)") + @WRITE_CALLBACK + def write_cb(_context, buffer, length): + try: + return write_callback(ctypes.string_at(buffer, length)) + except Exception: + return -1 + try: + ptr = self._native.run_script_callback(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs), write_cb) + raw = self._native.decode_and_free(ptr) + return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) + except Exception as error: + raise DataWeaveError(f"Failed to execute callback streaming: {error}") + + def _stream_worker(self, invoke, cancelled: Event) -> Generator[bytes, None, StreamingResult]: + sentinel = object() + queue: Queue = Queue(maxsize=_OUTPUT_QUEUE_MAXSIZE) + + class CleanupFailure: + def __init__(self, error: Exception): + self.error = error + + def publish(item) -> None: + while not cancelled.is_set(): + try: + queue.put(item, timeout=0.1) + return + except Full: + pass + + @WRITE_CALLBACK + def write_cb(_context, buffer, length): + try: + if cancelled.is_set(): + return -1 + queue.put(ctypes.string_at(buffer, length), timeout=_WORKER_TIMEOUT_SECONDS) + return 0 + except Exception: + return -1 + + def worker_main(): + worker_thread = None + primary_error = None + primary_outcome = False + try: + worker_thread = self._native.attach_thread() + raw = self._native.decode_and_free(invoke(worker_thread, write_cb), worker_thread) + metadata = json.loads(raw) if raw else {"success": False, "error": "Empty response"} + primary_outcome = not metadata.get("success", False) + publish(metadata) + except Exception as error: + primary_error = error + publish({"success": False, "error": str(error)}) + finally: + if worker_thread is not None: + try: + self._native.detach_thread(worker_thread) + except Exception as error: + if primary_error is None and not primary_outcome: + publish(CleanupFailure(error)) + publish(sentinel) + self._unregister_stream_worker() + + # Python cannot cancel a native call. Daemon workers keep an abandoned + # call from extending interpreter lifetime after bounded cancellation. + worker = Thread(target=worker_main, name="dw-streaming-worker", daemon=True) + self._register_stream_worker(worker) + try: + worker.start() + except Exception: + self._unregister_stream_worker(worker) + raise + metadata = None + try: + while True: + try: + item = queue.get(timeout=_WORKER_TIMEOUT_SECONDS) + except Empty: + cancelled.set() + worker.join(timeout=_WORKER_JOIN_TIMEOUT_SECONDS) + raise DataWeaveError(f"Worker thread timeout after {_WORKER_TIMEOUT_SECONDS} seconds") + if item is sentinel: + break + if isinstance(item, CleanupFailure): + raise item.error + if isinstance(item, dict): + metadata = item + else: + yield item + finally: + cancelled.set() + worker.join(timeout=_WORKER_JOIN_TIMEOUT_SECONDS) + return parse_streaming_result(metadata or {"success": False, "error": "No metadata received from native call"}) + + def run_streaming(self, script: str, inputs: Optional[Dict[str, Any]] = None) -> Stream: + self._require_initialized(self._native.has_callback_streaming, "callback streaming API (run_script_callback not found)") + cancelled = Event() + encoded_inputs = self._inputs_json(inputs) + stream = Stream(self._stream_worker(lambda thread, write_cb: self._native.run_script_callback(thread, script.encode("utf-8"), encoded_inputs, write_cb), cancelled)) + stream._on_close = cancelled.set + stream._cancelled = cancelled + return stream + + @staticmethod + def _chunk_reader(input_stream: Iterable[bytes]): + iterator = iter(input_stream) + state = {"chunk": b"", "offset": 0, "done": False} + @READ_CALLBACK + def read_cb(_context, buffer, buffer_size): + try: + while True: + chunk = state["chunk"] + if state["offset"] < len(chunk): + size = min(len(chunk) - state["offset"], buffer_size) + ctypes.memmove(buffer, chunk[state["offset"]:state["offset"] + size], size) + state["offset"] += size + return size + if state["done"]: + return 0 + chunk = next(iterator, None) + if not chunk: + state["done"] = True + return 0 + state["chunk"] = chunk + state["offset"] = 0 + except Exception: + return -1 + return read_cb + + def run_transform(self, script: str, input_stream: Iterable[bytes], input_name: str = "payload", input_mime_type: str = "application/json", input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> Stream: + self._require_initialized(self._native.has_callback_input_output, "callback input/output API (run_script_input_output_callback not found)") + cancelled = Event() + read_cb = self._chunk_reader(input_stream) + encoded_inputs = self._inputs_json(inputs) + def invoke(thread, write_cb): + return self._native.run_script_input_output_callback(thread, script.encode("utf-8"), encoded_inputs, input_name.encode("utf-8"), input_mime_type.encode("utf-8"), input_charset.encode("utf-8") if input_charset else None, read_cb, write_cb) + stream = Stream(self._stream_worker(invoke, cancelled)) + stream._on_close = cancelled.set + stream._cancelled = cancelled + return stream + + def run_input_output_callback(self, script: str, input_name: str, input_mime_type: str, read_callback: ReadCallback, write_callback: WriteCallback, input_charset: Optional[str] = None, inputs: Optional[Dict[str, Any]] = None) -> StreamingResult: + self._require_initialized(self._native.has_callback_input_output, "callback input/output API (run_script_input_output_callback not found)") + @READ_CALLBACK + def read_cb(_context, buffer, buffer_size): + try: + data = read_callback(buffer_size) + if not data: + return 0 + if len(data) > buffer_size: + return -1 + ctypes.memmove(buffer, data, len(data)) + return len(data) + except Exception: + return -1 + @WRITE_CALLBACK + def write_cb(_context, buffer, length): + try: + return write_callback(ctypes.string_at(buffer, length)) + except Exception: + return -1 + try: + ptr = self._native.run_script_input_output_callback(self._native.thread, script.encode("utf-8"), self._inputs_json(inputs), input_name.encode("utf-8"), input_mime_type.encode("utf-8"), input_charset.encode("utf-8") if input_charset else None, read_cb, write_cb) + raw = self._native.decode_and_free(ptr) + return parse_streaming_result(json.loads(raw) if raw else {"success": False, "error": "Empty response"}) + except Exception as error: + raise DataWeaveError(f"Failed to execute callback input/output streaming: {error}") + + def __enter__(self): + self.initialize() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + self.cleanup() + except Exception: + if exc_type is None: + raise + return False diff --git a/native-lib/python/tests/__init__.py b/native-lib/python/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/native-lib/python/tests/conftest.py b/native-lib/python/tests/conftest.py new file mode 100644 index 00000000..91acf043 --- /dev/null +++ b/native-lib/python/tests/conftest.py @@ -0,0 +1,119 @@ +import sys +from pathlib import Path + +import pytest + + +PYTHON_SRC_DIR = Path(__file__).resolve().parents[1] / "src" +sys.path.insert(0, str(PYTHON_SRC_DIR)) +sys.path.insert(0, str(Path(__file__).parent)) + +import dataweave + + +def _tck_discovery(): + from tck.case_loader import discover_cases + from tck.ignore_list import EXCLUDED_CASES, STRUCTURAL_MODULE_CASES, validate_exclusions + + suites_dir = Path(__file__).resolve().parents[2] / "node" / "tests" / "tck" / "suites" + discovery = discover_cases(suites_dir) + scenarios = [ + scenario + for discovered_case in discovery.cases + for scenario in discovered_case.scenarios + ] + errors = validate_exclusions(EXCLUDED_CASES, scenarios) + if errors: + raise pytest.UsageError("Invalid active TCK exclusions: " + "; ".join(errors)) + exclusions = [ + scenario for scenario in scenarios + if scenario.identifier.rsplit(":", 1)[0] in EXCLUDED_CASES + ] + structural_modules = set(discovery.structural_case_identifiers) & STRUCTURAL_MODULE_CASES + return discovery, scenarios, exclusions, structural_modules + + +def pytest_configure(config): + if config.option.markexpr == "tck": + config._tck_discovery = _tck_discovery() + + +def pytest_report_header(config): + if not hasattr(config, "_tck_discovery"): + return None + discovery, scenarios, exclusions, structural_modules = config._tck_discovery + categories = {} + from tck.ignore_list import exclusion_for + + for scenario in exclusions: + exclusion = exclusion_for(scenario.identifier.rsplit(":", 1)[0]) + categories[exclusion.category] = categories.get(exclusion.category, 0) + 1 + category_totals = ", ".join( + f"{category}={count}" for category, count in sorted(categories.items()) + ) or "none" + return ( + f"TCK: discovered={len(scenarios)}, structural-skips={discovery.structural_skips}, " + f"structural-module-cases={len(structural_modules)}, " + f"active-exclusions={len(exclusions)} ({category_totals})" + ) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + if not hasattr(config, "_tck_discovery"): + return + discovery, scenarios, exclusions, structural_modules = config._tck_discovery + reports = [ + report + for reports in terminalreporter.stats.values() + for report in reports + if getattr(report, "when", None) == "call" + and "::test_tck_scenario[" in report.nodeid + ] + totals = { + outcome: sum( + report.outcome == outcome + and (outcome != "skipped" or not getattr(report, "wasxfail", None)) + for report in reports + ) + for outcome in ("passed", "failed", "skipped") + } + xfailed = sum(bool(getattr(report, "wasxfail", None)) for report in reports) + executed = totals["passed"] + totals["failed"] + terminalreporter.write_line( + "TCK totals: " + f"selected={len(scenarios)}, structural-skips={discovery.structural_skips}, " + f"structural-module-cases={len(structural_modules)}, " + f"executed={executed}, active-exclusions={totals['skipped']}, passed={totals['passed']}, " + f"failed={totals['failed']}, xfail={xfailed}" + ) + + +@pytest.fixture(autouse=True) +def clean_dataweave_runtime(request): + """Keep module-level isolate state from leaking between integration tests.""" + if request.node.get_closest_marker("tck"): + yield + return + dataweave.cleanup() + yield + dataweave.cleanup() + + +@pytest.fixture(scope="session") +def tck_runtime(): + """Own one isolate for the TCK session and release it after the lane.""" + runtime = dataweave.DataWeave() + runtime.initialize() + try: + yield runtime + finally: + runtime.cleanup() + + +@pytest.fixture +def collect_stream(): + def collect(stream): + chunks = list(stream) + return b"".join(chunks), stream.metadata + + return collect diff --git a/native-lib/python/tests/integration/test_callbacks.py b/native-lib/python/tests/integration/test_callbacks.py new file mode 100644 index 00000000..b7ee1c53 --- /dev/null +++ b/native-lib/python/tests/integration/test_callbacks.py @@ -0,0 +1,135 @@ +import io + +import pytest + +import dataweave + + +@pytest.mark.integration +def test_callback_streams_basic_output(): + chunks = [] + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_callback("2 + 2", on_write) + + assert result.success is True + assert b"".join(chunks).decode(result.charset or "utf-8") == "4" + + +@pytest.mark.integration +def test_callback_streams_output_with_inputs(): + chunks = [] + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_callback("num1 + num2", on_write, inputs={"num1": 25, "num2": 17}) + + assert result.success is True + assert b"".join(chunks).decode(result.charset or "utf-8") == "42" + + +@pytest.mark.integration +def test_callback_translates_write_callback_exception_to_unsuccessful_result(): + def on_write(_data: bytes) -> int: + raise RuntimeError("write callback failed") + + result = dataweave.run_callback("2 + 2", on_write) + + assert result.success is False + assert result.error is not None + + +@pytest.mark.integration +def test_callback_transforms_streamed_input_and_output(): + source = io.BytesIO(b"[10, 20, 30, 40, 50]") + chunks = [] + + def on_read(buffer_size: int) -> bytes: + return source.read(buffer_size) + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_input_output_callback( + "output application/json\n---\npayload map ($ * 2)", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + output = b"".join(chunks).decode(result.charset or "utf-8") + assert result.success is True + assert "20" in output + assert "100" in output + + +@pytest.mark.integration +def test_callback_accepts_large_streamed_input(): + records = b"[" + b",".join(f'{{"id":{index}}}'.encode() for index in range(1, 1001)) + b"]" + source = io.BytesIO(records) + chunks = [] + + def on_read(buffer_size: int) -> bytes: + return source.read(buffer_size) + + def on_write(data: bytes) -> int: + chunks.append(data) + return 0 + + result = dataweave.run_input_output_callback( + "output application/json\n---\nsizeOf(payload)", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + assert result.success is True + assert b"".join(chunks).decode(result.charset or "utf-8") == "1000" + + +@pytest.mark.integration +def test_input_output_callback_translates_read_callback_exception_to_unsuccessful_result(): + def on_read(_buffer_size: int) -> bytes: + raise RuntimeError("read callback failed") + + def on_write(_data: bytes) -> int: + return 0 + + result = dataweave.run_input_output_callback( + "output application/json\n---\npayload", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + assert result.success is False + assert result.error is not None + + +@pytest.mark.integration +def test_input_output_callback_translates_write_callback_exception_to_unsuccessful_result(): + def on_read(_buffer_size: int) -> bytes: + return b"[1]" + + def on_write(_data: bytes) -> int: + raise RuntimeError("write callback failed") + + result = dataweave.run_input_output_callback( + "output application/json\n---\npayload", + input_name="payload", + input_mime_type="application/json", + read_callback=on_read, + write_callback=on_write, + ) + + assert result.success is False + assert result.error is not None diff --git a/native-lib/python/tests/integration/test_execution.py b/native-lib/python/tests/integration/test_execution.py new file mode 100644 index 00000000..3768163d --- /dev/null +++ b/native-lib/python/tests/integration/test_execution.py @@ -0,0 +1,63 @@ +from pathlib import Path + +import pytest + +import dataweave + + +@pytest.mark.integration +def test_input_value_accepts_public_mime_type_constructor_keyword(): + value = dataweave.InputValue( + content="1234567", + mime_type="application/csv", + properties={"header": False, "separator": "4"}, + ) + + assert value.mime_type == "application/csv" + + +@pytest.mark.integration +def test_runs_basic_script(): + result = dataweave.run("2 + 2", {}) + + assert result.get_string() == "4" + + +@pytest.mark.integration +def test_runs_script_with_inputs(): + result = dataweave.run("num1 + num2", {"num1": 25, "num2": 17}) + + assert result.get_string() == "42" + + +@pytest.mark.integration +def test_converts_utf16_xml_input_to_csv(): + xml_path = Path(__file__).resolve().parents[1] / "person.xml" + script = """output application/csv header=true +--- +[payload.person] +""" + + result = dataweave.run( + script, + { + "payload": { + "content": xml_path.read_bytes(), + "mimeType": "application/xml", + "charset": "UTF-16", + } + }, + ) + + output = result.get_string() or "" + assert result.success is True + assert "name" in output and "age" in output + assert "Billy" in output + assert "31" in output + + +@pytest.mark.integration +def test_converts_python_list_input_automatically(): + result = dataweave.run("numbers[0]", {"numbers": [1, 2, 3]}) + + assert result.get_string() == "1" diff --git a/native-lib/python/tests/integration/test_lifecycle.py b/native-lib/python/tests/integration/test_lifecycle.py new file mode 100644 index 00000000..cfabda1a --- /dev/null +++ b/native-lib/python/tests/integration/test_lifecycle.py @@ -0,0 +1,30 @@ +import pytest + +import dataweave + + +@pytest.mark.integration +def test_context_manager_runs_multiple_scripts(): + with dataweave.DataWeave() as dw: + assert dw.run("sqrt(144)").get_string() == "12" + assert dw.run("sqrt(10000)").get_string() == "100" + + +@pytest.mark.unit +def test_context_exit_preserves_body_exception_when_cleanup_fails(monkeypatch): + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + monkeypatch.setattr(runtime, "initialize", lambda: None) + monkeypatch.setattr(runtime, "cleanup", lambda: (_ for _ in ()).throw(dataweave.DataWeaveError("cleanup failed"))) + + with pytest.raises(ValueError, match="body failed"): + with runtime: + raise ValueError("body failed") + + +@pytest.mark.unit +def test_context_exit_surfaces_cleanup_failure_without_body_exception(monkeypatch): + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + monkeypatch.setattr(runtime, "cleanup", lambda: (_ for _ in ()).throw(dataweave.DataWeaveError("cleanup failed"))) + + with pytest.raises(dataweave.DataWeaveError, match="cleanup failed"): + runtime.__exit__(None, None, None) diff --git a/native-lib/python/tests/integration/test_streaming.py b/native-lib/python/tests/integration/test_streaming.py new file mode 100644 index 00000000..dc92c272 --- /dev/null +++ b/native-lib/python/tests/integration/test_streaming.py @@ -0,0 +1,115 @@ +import json +from pathlib import Path + +import pytest + +import dataweave + + +@pytest.mark.integration +def test_run_streaming_returns_chunks_and_metadata(collect_stream): + output, metadata = collect_stream(dataweave.run_streaming("output application/json --- {a: 1, b: 2}")) + + text = output.decode(metadata.charset or "utf-8") + assert output + assert '"a": 1' in text or '"a":1' in text + assert metadata.success is True + assert metadata.mime_type == "application/json" + + +@pytest.mark.integration +def test_run_streaming_splits_large_output_into_multiple_chunks(collect_stream): + stream = dataweave.run_streaming('output application/json --- (1 to 5000) map {id: $, name: "item_" ++ $}') + chunks = list(stream) + output = b"".join(chunks) + metadata = stream.metadata + + text = output.decode(metadata.charset or "utf-8") + assert metadata.success is True + assert len(chunks) > 1 + assert '"id": 5000' in text or '"id":5000' in text + + +@pytest.mark.integration +def test_run_streaming_returns_error_metadata(collect_stream): + output, metadata = collect_stream(dataweave.run_streaming("output application/json --- invalid_var")) + + assert metadata.success is False + assert metadata.error is not None + assert output == b"" + + +@pytest.mark.integration +def test_run_streaming_accepts_input_bindings(collect_stream): + output, metadata = collect_stream(dataweave.run_streaming("num1 + num2", {"num1": 25, "num2": 17})) + + assert metadata.success is True + assert output.decode(metadata.charset or "utf-8").strip() == "42" + + +@pytest.mark.integration +def test_run_transform_streams_iterable_input(collect_stream): + stream = dataweave.run_transform( + "output application/json\n---\npayload map ($ * 2)", + input_stream=[b"[10, 20, 30, 40, 50]"], + input_mime_type="application/json", + ) + output, metadata = collect_stream(stream) + + text = output.decode(metadata.charset or "utf-8") + assert metadata.success is True + assert "20" in text + assert "100" in text + + +@pytest.mark.integration +def test_run_transform_reads_chunked_input(collect_stream): + input_data = b"[" + b",".join(f'{{"id":{index}}}'.encode() for index in range(1, 1001)) + b"]" + + def chunked(): + for index in range(0, len(input_data), 4096): + yield input_data[index:index + 4096] + + stream = dataweave.run_transform( + "output application/json\n---\nsizeOf(payload)", + input_stream=chunked(), + input_mime_type="application/json", + ) + output, metadata = collect_stream(stream) + + assert metadata.success is True + assert output.decode(metadata.charset or "utf-8") == "1000" + + +@pytest.mark.integration +def test_run_transform_preserves_large_single_input_chunk(collect_stream): + payload = json.dumps([{"id": index, "name": f"item_{index}", "value": index * 3} for index in range(1, 2001)]).encode() + assert len(payload) > 8192 + + stream = dataweave.run_transform( + "output application/json\n---\nsizeOf(payload)", + input_stream=iter([payload]), + input_mime_type="application/json", + ) + output, metadata = collect_stream(stream) + + assert metadata.success is True + assert output.decode(metadata.charset or "utf-8") == "2000" + + +@pytest.mark.integration +def test_run_transform_reads_file_input(collect_stream): + xml_path = Path(__file__).resolve().parents[1] / "person.xml" + with xml_path.open("rb") as source: + stream = dataweave.run_transform( + "output application/csv header=true\n---\n[payload.person]", + input_stream=iter(lambda: source.read(4096), b""), + input_mime_type="application/xml", + input_charset="UTF-16", + ) + output, metadata = collect_stream(stream) + + text = output.decode(metadata.charset or "utf-8") + assert metadata.success is True + assert "Billy" in text + assert "31" in text diff --git a/native-lib/python/tests/tck/__init__.py b/native-lib/python/tests/tck/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/native-lib/python/tests/tck/case_loader.py b/native-lib/python/tests/tck/case_loader.py new file mode 100644 index 00000000..d042e7f6 --- /dev/null +++ b/native-lib/python/tests/tck/case_loader.py @@ -0,0 +1,122 @@ +"""Filesystem loader for the Gradle-staged DataWeave TCK corpus.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional + +from dataweave import InputValue + + +EXTENSION_TO_MIME = { + "bin": "application/octet-stream", + "csv": "application/csv", + "dwl": "application/dw", + "json": "application/json", + "multipart": "multipart/form-data", + "properties": "text/x-java-properties", + "txt": "text/plain", + "urlencoded": "application/x-www-form-urlencoded", + "xml": "application/xml", +} + + +@dataclass(frozen=True) +class TckScenario: + identifier: str + transform: str + inputs: Dict[str, InputValue] + expected: bytes + output_extension: str + charset: Optional[str] + + +@dataclass(frozen=True) +class DiscoveredCase: + identifier: str + scenarios: List[TckScenario] + + +@dataclass(frozen=True) +class Discovery: + cases: List[DiscoveredCase] + structural_skips: int + structural_case_identifiers: List[str] + + +def extension_of(name: str) -> str: + return Path(name).suffix.removeprefix(".").lower() + + +def discover_cases(suites_dir: Path) -> Discovery: + cases: List[DiscoveredCase] = [] + structural_skips = 0 + structural_case_identifiers: List[str] = [] + if not suites_dir.exists(): + return Discovery(cases, structural_skips, structural_case_identifiers) + + for suite_dir in sorted(path for path in suites_dir.iterdir() if path.is_dir()): + for case_dir in sorted(path for path in suite_dir.iterdir() if path.is_dir()): + if not case_dir.exists(): + structural_skips += 1 + structural_case_identifiers.append(f"{suite_dir.name}/{case_dir.name}") + continue + scenarios = _load_case(suite_dir.name, case_dir) + if scenarios is None: + structural_skips += 1 + structural_case_identifiers.append(f"{suite_dir.name}/{case_dir.name}") + else: + cases.append(DiscoveredCase(f"{suite_dir.name}/{case_dir.name}", scenarios)) + return Discovery(cases, structural_skips, structural_case_identifiers) + + +def _load_case(suite_name: str, case_dir: Path) -> Optional[List[TckScenario]]: + files = {path.name: path for path in case_dir.iterdir() if path.is_file()} + case_name = case_dir.name + if case_name.endswith("_wip") or case_name.endswith("wip"): + return None + if "config.properties" in files or any( + name.startswith(("in", "out")) and name.endswith("-config.properties") for name in files + ): + return None + if any(extension_of(name) == "groovy" for name in files): + return None + + transforms = [ + path for name, path in files.items() + if extension_of(name) == "dwl" and not name.startswith("in") and not name.startswith("out") + ] + if len(transforms) != 1 or "transform.dwl" not in files: + return None + + input_paths = sorted( + (path for name, path in files.items() if name.startswith("in") and name[2:].split(".", 1)[0].isdigit()), + key=lambda path: path.name, + ) + if any(extension_of(path.name) not in EXTENSION_TO_MIME for path in input_paths): + return None + output_paths = sorted( + (path for name, path in files.items() if name.startswith("out.") and extension_of(name) in EXTENSION_TO_MIME), + key=lambda path: path.name, + ) + if not output_paths: + return None + + inputs = { + path.stem: InputValue(path.read_bytes(), mime_type=EXTENSION_TO_MIME[extension_of(path.name)]) + for path in input_paths + } + transform = files["transform.dwl"].read_text(encoding="utf-8") + encoding = files.get("encoding") + charset = encoding.read_text(encoding="utf-8").strip() if encoding else None + case_id = f"{suite_name}/{case_name}" + return [ + TckScenario( + identifier=f"{case_id}:{path.name}", + transform=transform, + inputs=inputs, + expected=path.read_bytes(), + output_extension=extension_of(path.name), + charset=charset, + ) + for path in output_paths + ] diff --git a/native-lib/python/tests/tck/compare.py b/native-lib/python/tests/tck/compare.py new file mode 100644 index 00000000..16adc3fe --- /dev/null +++ b/native-lib/python/tests/tck/compare.py @@ -0,0 +1,128 @@ +"""Output comparators for the formats emitted by the staged TCK corpus.""" + +import json +from xml.dom import Node, minidom +from dataclasses import dataclass +from typing import Any, Optional + + +@dataclass(frozen=True) +class CompareResult: + match: bool + detail: str = "" + + +def compare_output( + extension: str, actual: bytes, expected: bytes, charset: Optional[str] = None +) -> CompareResult: + extension = extension.removeprefix(".").lower() + if extension == "bin": + return _result(actual == expected, "binary mismatch") + if extension not in { + "json", "xml", "csv", "txt", "dwl", "properties", "urlencoded", "multipart" + }: + return CompareResult(False, f"unknown output extension: {extension}") + + encoding = _python_encoding(charset) + try: + actual_text = actual.decode(encoding) + expected_text = expected.decode(encoding) + except UnicodeDecodeError as error: + return CompareResult(False, f"cannot decode output as {encoding}: {error}") + if extension == "json": + return _compare_json(actual_text, expected_text) + if extension == "xml": + return _compare_xml(actual_text, expected_text) + if extension == "dwl": + return _result(_strip_whitespace(actual_text) == _strip_whitespace(expected_text), "DWL mismatch") + return _result(_normalize_text(actual_text) == _normalize_text(expected_text), "text mismatch") + + +def _compare_json(actual: str, expected: str) -> CompareResult: + try: + actual_value = json.loads(actual) + except json.JSONDecodeError as error: + return CompareResult(False, f"actual is not valid JSON: {error}") + try: + expected_value = json.loads(expected) + except json.JSONDecodeError as error: + return CompareResult(False, f"expected is not valid JSON: {error}") + return _result(_json_equal(actual_value, expected_value), "JSON mismatch") + + +def _compare_xml(actual: str, expected: str) -> CompareResult: + try: + actual_value = _xml_value(minidom.parseString(actual).documentElement) + except Exception as error: + return CompareResult(False, f"actual is not valid XML: {error}") + try: + expected_value = _xml_value(minidom.parseString(expected).documentElement) + except Exception as error: + return CompareResult(False, f"expected is not valid XML: {error}") + return _result(actual_value == expected_value, "XML mismatch") + + +def _xml_value(element) -> Any: + return ( + element.tagName, + tuple(sorted( + (attribute.name, attribute.value) + for attribute in element.attributes.values() + if not attribute.name.startswith("xmlns") + )), + tuple( + _xml_child_value(child) + for child in element.childNodes + if child.nodeType == Node.ELEMENT_NODE or child.data.strip() + ), + ) + + +def _xml_child_value(node) -> Any: + if node.nodeType == Node.ELEMENT_NODE: + return _xml_value(node) + return ("#text", node.data.strip()) + + +def _json_equal(actual: Any, expected: Any) -> bool: + if isinstance(actual, bool) or isinstance(expected, bool): + return type(actual) is type(expected) and actual == expected + if _is_number(actual) and _is_number(expected): + return actual == expected + if type(actual) is not type(expected): + return False + if isinstance(actual, str) and isinstance(expected, str): + return _normalize_eol(actual) == _normalize_eol(expected) + if isinstance(actual, list) and isinstance(expected, list): + return len(actual) == len(expected) and all( + _json_equal(left, right) for left, right in zip(actual, expected) + ) + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all( + _json_equal(actual[key], expected[key]) for key in actual + ) + return actual == expected + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _python_encoding(charset: Optional[str]) -> str: + return (charset or "utf-8").replace("UTF-16", "utf-16") + + +def _normalize_eol(value: str) -> str: + return value.replace("\r\n", "\n") + + +def _normalize_text(value: str) -> str: + return _normalize_eol(value).strip() + + +def _strip_whitespace(value: str) -> str: + return "".join(value.split()) + + +def _result(match: bool, detail: str) -> CompareResult: + return CompareResult(match, "" if match else detail) diff --git a/native-lib/python/tests/tck/ignore_list.py b/native-lib/python/tests/tck/ignore_list.py new file mode 100644 index 00000000..9edb7cca --- /dev/null +++ b/native-lib/python/tests/tck/ignore_list.py @@ -0,0 +1,290 @@ +"""Auditable exclusions for TCK cases Python cannot execute deterministically.""" + +from dataclasses import dataclass +from typing import Dict, Iterable, List, Mapping, Optional + + +UNSUPPORTED_DW_MODULE_RESOLUTION = "unsupported-dw-module-resolution" +UNAVAILABLE_JAVA_MODULE = "unavailable-java-module" +UNAVAILABLE_CLASSPATH_TEST_RESOURCE = "unavailable-classpath-test-resource" + +SUPPORTED_CATEGORIES = frozenset( + ( + UNSUPPORTED_DW_MODULE_RESOLUTION, + UNAVAILABLE_JAVA_MODULE, + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + ) +) + + +# These cases are transform-shape structural skips because each bundles its +# imported module beside transform.dwl. They are reported separately and are +# deliberately not active exclusions, so they cannot hide runnable failures. +STRUCTURAL_MODULE_CASES = frozenset( + ( + "runtime/implicit_type_parameters-out.json", + "runtime/import_mapping-out.json", + "runtime/import_mapping_with_functions-out.json", + "runtime/import_mapping_with_implicit_input-out.json", + "runtime/import_namespace-out.xml", + "runtime/infinit_list-out.json", + "runtime/interceptor_functions-out.json", + "runtime/lazy_metadata_definition-out.json", + "runtime/location-out.json", + "runtime/locationString-out.json", + "runtime/logwith_function-out.json", + "runtime/type_selector_materialize-out.json", + "runtime/weave_multiple_namespace-out.dwl", + ) +) + + +@dataclass(frozen=True) +class Exclusion: + case_identifier: str + category: str + reason: str + + +def _exclusion(case_identifier: str, category: str, reason: str) -> Exclusion: + return Exclusion(case_identifier, category, reason) + + +# Each entry has a full case identifier and direct runtime evidence. Categories +# describe only an observed, unsupported limitation; they never match patterns. +EXCLUDED_CASES: Dict[str, Exclusion] = { + "runtime/import-component-alias-lib-out.json": _exclusion( + "runtime/import-component-alias-lib-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-lib-out.json": _exclusion( + "runtime/import-lib-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-lib-with-alias-out.json": _exclusion( + "runtime/import-lib-with-alias-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-named-lib-out.json": _exclusion( + "runtime/import-named-lib-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/import-star-out.json": _exclusion( + "runtime/import-star-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/module-singleton-out.json": _exclusion( + "runtime/module-singleton-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports a test-only DW module; the Python binding has no module resolver", + ), + "runtime/is-empty-using-empty-stream-out.json": _exclusion( + "runtime/is-empty-using-empty-stream-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Client, which is not resolved by the Python binding", + ), + "runtime/streaming_binary_inside_value-out.json": _exclusion( + "runtime/streaming_binary_inside_value-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-array-value-with-failures-out.json": _exclusion( + "runtime/try-handle-array-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-attribute-delegate-with-failures-out.json": _exclusion( + "runtime/try-handle-attribute-delegate-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-attributes-value-with-failures-out.json": _exclusion( + "runtime/try-handle-attributes-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-binary-value-with-failures-out.json": _exclusion( + "runtime/try-handle-binary-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-delegate-value-with-failures-out.json": _exclusion( + "runtime/try-handle-delegate-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-key-value-pair-value-with-failures-out.json": _exclusion( + "runtime/try-handle-key-value-pair-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-materialized-object-with-failures-out.json": _exclusion( + "runtime/try-handle-materialized-object-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-name-value-pair-value-with-failures-out.json": _exclusion( + "runtime/try-handle-name-value-pair-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-schema-property-value-with-failures-out.json": _exclusion( + "runtime/try-handle-schema-property-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "runtime/try-handle-schema-value-with-failures-out.json": _exclusion( + "runtime/try-handle-schema-value-with-failures-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports dw::Natives, which is not resolved by the Python binding", + ), + "core-modules/multipart-write-binary-out.json": _exclusion( + "core-modules/multipart-write-binary-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions before exercising multipart binary output", + ), + "core-modules/read-binary-files-out.bin": _exclusion( + "core-modules/read-binary-files-out.bin", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions before reading the binary fixture", + ), + "runtime/full-qualified-name-ref-out.json": _exclusion( + "runtime/full-qualified-name-ref-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve org::mule::weave::v2::libs::lib test modules", + ), + "runtime/private_scope_directives-out.xml": _exclusion( + "runtime/private_scope_directives-out.xml", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::Module", + ), + "runtime/try-out.json": _exclusion( + "runtime/try-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions", + ), + "runtime/urlEncodeDecode-out.json": _exclusion( + "runtime/urlEncodeDecode-out.json", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "cannot resolve dw::core::Assertions", + ), + "runtime/java-big-decimal-out.xml": _exclusion( + "runtime/java-big-decimal-out.xml", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve java::lang::String::valueOf", + ), + "runtime/java-field-ref-out.json": _exclusion( + "runtime/java-field-ref-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve test POJO Constants or java::lang::String", + ), + "runtime/java-interop-enum-out.json": _exclusion( + "runtime/java-interop-enum-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve test POJO GenderEnum or java::lang::String", + ), + "runtime/java-interop-function-call-out.json": _exclusion( + "runtime/java-interop-function-call-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve test POJO MyCompanyUtils or java::lang::String", + ), + "runtime/java_epoch_bridge-out.json": _exclusion( + "runtime/java_epoch_bridge-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve java::time::Instant members", + ), + "runtime/runtime_run_coercionException-out.json": _exclusion( + "runtime/runtime_run_coercionException-out.json", + UNAVAILABLE_JAVA_MODULE, + "dw::Runtime.run cannot resolve application/java and returns UnknownContentTypeException", + ), + "runtime/runtime_run_fibo-out.json": _exclusion( + "runtime/runtime_run_fibo-out.json", + UNAVAILABLE_JAVA_MODULE, + "dw::Runtime.run cannot resolve application/java and returns UnknownContentTypeException", + ), + "runtime/runtime_run_null_java-out.json": _exclusion( + "runtime/runtime_run_null_java-out.json", + UNAVAILABLE_JAVA_MODULE, + "application/java returns UnknownContentTypeException", + ), + "runtime/sql_date_mapping-out.json": _exclusion( + "runtime/sql_date_mapping-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve Java test class org::mule::weave::v2::pojo::SqlDateTest", + ), + "runtime/underflow-out.json": _exclusion( + "runtime/underflow-out.json", + UNAVAILABLE_JAVA_MODULE, + "cannot resolve java::lang::Long::{MIN_VALUE,MAX_VALUE}", + ), + "runtime/write-function-with-null-out.xml": _exclusion( + "runtime/write-function-with-null-out.xml", + UNAVAILABLE_JAVA_MODULE, + "write(null, application/java) reports unknown content type", + ), + "runtime/dw-binary-out.dwl": _exclusion( + "runtime/dw-binary-out.dwl", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://dw-binary/in0.bin", + ), + "runtime/read-function-by-id-out.json": _exclusion( + "runtime/read-function-by-id-out.json", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://read-function-by-id/include.dwl", + ), + "runtime/read-function-out.json": _exclusion( + "runtime/read-function-out.json", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://read-function/include.dwl", + ), + "runtime/read_lines-out.json": _exclusion( + "runtime/read_lines-out.json", + UNAVAILABLE_CLASSPATH_TEST_RESOURCE, + "readUrl cannot find classpath://read_lines/test.txt", + ), +} + + +def exclusion_for(case_identifier: str) -> Optional[Exclusion]: + return EXCLUDED_CASES.get(case_identifier) + + +def validate_exclusions( + entries: Mapping[str, object], scenarios: Optional[Iterable[object]] = None +) -> List[str]: + errors = [] + for identifier, entry in entries.items(): + case_identifier = _entry_field(entry, "case_identifier") + category = _entry_field(entry, "category") + reason = _entry_field(entry, "reason") + if not case_identifier: + errors.append(f"{identifier}: missing case identity") + elif case_identifier != identifier: + errors.append(f"{identifier}: case identity must match registry key") + if not category: + errors.append(f"{identifier}: missing category") + elif category not in SUPPORTED_CATEGORIES: + errors.append(f"{identifier}: unsupported category {category}") + if not reason or not reason.strip(): + errors.append(f"{identifier}: missing reason") + if scenarios is not None: + discovered = { + scenario.identifier.rsplit(":", 1)[0] + for scenario in scenarios + } + for identifier in entries: + if identifier not in discovered: + errors.append(f"{identifier}: not a discovered runnable case") + return errors + + +def _entry_field(entry: object, field: str) -> Optional[str]: + if isinstance(entry, Exclusion): + return getattr(entry, field) + return entry.get(field) diff --git a/native-lib/python/tests/tck/test_conformance.py b/native-lib/python/tests/tck/test_conformance.py new file mode 100644 index 00000000..e1aaaed9 --- /dev/null +++ b/native-lib/python/tests/tck/test_conformance.py @@ -0,0 +1,444 @@ +import base64 +import json +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, Union + +import pytest + +import dataweave + +sys.path.insert(0, str(Path(__file__).parent)) + +from case_loader import TckScenario, discover_cases +from compare import compare_output +from ignore_list import ( + EXCLUDED_CASES, + Exclusion, + UNSUPPORTED_DW_MODULE_RESOLUTION, + exclusion_for, + validate_exclusions, +) +from conftest import pytest_terminal_summary + + +pytestmark = pytest.mark.tck + + +SUITES_DIR = Path(__file__).resolve().parents[3] / "node" / "tests" / "tck" / "suites" +DISCOVERY = discover_cases(SUITES_DIR) +SCENARIOS = [ + scenario + for discovered_case in DISCOVERY.cases + for scenario in discovered_case.scenarios +] + + +ACCEPTED_BASELINE_MISMATCHES = { + "core-modules/csv-invalid-utf8-out.csv:out.csv": ( + "runtime emits a replacement character where the fixture expects an empty CSV value" + ), + "core-modules/number-addition-out.json:out.json": ( + "runtime emits a numeric result that differs from the accepted baseline fixture" + ), + "core-modules/number-subtraction-out.json:out.json": ( + "runtime emits a numeric result that differs from the accepted baseline fixture" + ), + "core-modules/multipart-binary-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-class-cast-issue-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-empty-part-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-mixed-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-write-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-write-subtype-override-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/properties-passthrough-out.properties:out.properties": "properties writer output differs from the baseline fixture", + "runtime/access_raw_value-out.json:out.json": "runtime coercion output differs from the baseline fixture", + "runtime/coerciones_toString-out.json:out.json": "locale-sensitive runtime output differs from the baseline fixture", + "runtime/properties-writer-out.properties:out.properties": "properties writer output differs from the baseline fixture", + "runtime/read-concat-out.json:out.json": "runtime coercion output differs from the baseline fixture", + "runtime/runtime_dataFormatsDescriptors-out.json:out.json": "dw::Runtime output differs from the baseline fixture", + "runtime/runtime_orElseTry-out.json:out.json": "source-location runtime output differs from the baseline fixture", + "runtime/runtime_run-out.json:out.json": "dw::Runtime output differs from the baseline fixture", + "runtime/try-recursive-call-out.json:out.json": "source-location runtime output differs from the baseline fixture", + "runtime/update-op-out.dwl:out.dwl": "runtime coercion output differs from the baseline fixture", +} + +DEFERRED_WRITER_CASE = "core-modules/deferred-write-should-terminate-out.json:out.json" + + +def tck_params(): + return [ + pytest.param( + scenario, + marks=pytest.mark.xfail(strict=True, reason=reason), + id=scenario.identifier, + ) + if (reason := ACCEPTED_BASELINE_MISMATCHES.get(scenario.identifier)) + else pytest.param(scenario, id=scenario.identifier) + for scenario in SCENARIOS + ] + + +def test_accepted_baseline_mismatches_are_strict_xfails_with_reasons(): + params = tck_params() + xfails = { + parameter.id: next(mark for mark in parameter.marks if mark.name == "xfail") + for parameter in params + if any(mark.name == "xfail" for mark in parameter.marks) + } + + assert set(xfails) == set(ACCEPTED_BASELINE_MISMATCHES) + for identifier, mark in xfails.items(): + assert xfails[identifier].kwargs["strict"] is True + assert xfails[identifier].kwargs["reason"] == ACCEPTED_BASELINE_MISMATCHES[identifier] + + +@pytest.mark.unit +def test_tck_summary_counts_xfails_as_visible_expected_mismatches(): + output = [] + terminalreporter = SimpleNamespace( + stats={ + "xfailed": [ + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[core-modules/csv-invalid-utf8-out.csv:out.csv]", + outcome="skipped", + wasxfail="accepted mismatch", + ) + ] + }, + write_line=output.append, + ) + config = SimpleNamespace(_tck_discovery=(DISCOVERY, SCENARIOS, [], set())) + + pytest_terminal_summary(terminalreporter, 0, config) + + assert "active-exclusions=0" in output[-1] + assert "xfail=1" in output[-1] + + +@pytest.mark.parametrize("scenario", tck_params()) +def test_tck_scenario(scenario, tck_runtime): + """Runs each non-excluded staged corpus scenario against the Python binding.""" + exclusion = exclusion_for(scenario.identifier.rsplit(":", 1)[0]) + if exclusion: + pytest.skip(f"{exclusion.category}: {exclusion.reason}") + + if scenario.identifier == DEFERRED_WRITER_CASE: + success, error, output = _run_deferred_writer_in_subprocess(scenario) + assert success, error + else: + result = tck_runtime.run(scenario.transform, scenario.inputs) + assert result.success, result.error + output = result.get_bytes() + comparison = compare_output( + scenario.output_extension, + output, + scenario.expected, + scenario.charset, + ) + assert comparison.match, comparison.detail + + +def test_tck_session_runtime_runs_after_subprocess_deferred_write(tck_runtime): + """Deferred writers must not prevent the managed TCK runtime from continuing.""" + deferred = next( + scenario + for scenario in SCENARIOS + if scenario.identifier + == "core-modules/deferred-write-should-terminate-out.json:out.json" + ) + + success, error, _output = _run_deferred_writer_in_subprocess(deferred) + following_result = tck_runtime.run("%dw 2.0\noutput application/json\n--- 1") + + assert success, error + assert following_result.success, following_result.error + assert following_result.get_bytes() == b"1" + + +def _run_deferred_writer_in_subprocess(scenario): + """Contain the native deferred writer whose isolate teardown can block.""" + source_dir = Path(__file__).resolve().parents[2] / "src" + inputs = { + name: { + "content": base64.b64encode(value.content if isinstance(value.content, bytes) else value.content.encode()).decode(), + "mime_type": value.mime_type, + "charset": value.charset, + "properties": value.properties, + } + for name, value in scenario.inputs.items() + } + code = """ +import base64 +import json +import sys +from dataweave import DataWeave, InputValue + +script, inputs_json = sys.argv[1:] +inputs = { + name: InputValue(base64.b64decode(value['content']), value['mime_type'], value['charset'], value['properties']) + for name, value in json.loads(inputs_json).items() +} +runtime = DataWeave() +runtime.initialize() +result = runtime.run(script, inputs) +print(json.dumps({'success': result.success, 'error': result.error, 'result': result.result})) +""" + environment = os.environ.copy() + environment["PYTHONPATH"] = str(source_dir) + os.pathsep + environment.get("PYTHONPATH", "") + completed = subprocess.run( + [sys.executable, "-c", code, scenario.transform, json.dumps(inputs)], + capture_output=True, + check=False, + env=environment, + text=True, + timeout=30, + ) + assert completed.returncode == 0, completed.stderr + response = json.loads(completed.stdout) + return response["success"], response["error"], base64.b64decode(response["result"] or "") + + +def write_case(root: Path, name: str, files: Dict[str, Union[bytes, str]]) -> Path: + case = root / "runtime" / name + case.mkdir(parents=True) + for file_name, content in files.items(): + path = case / file_name + if isinstance(content, bytes): + path.write_bytes(content) + else: + path.write_text(content, encoding="utf-8") + return case + + +def test_discover_cases_loads_transform_input_output_and_scenarios(tmp_path: Path): + """Catches a loader that ignores TCK inputs or expected-output scenarios.""" + write_case( + tmp_path, + "maps-out.json", + { + "transform.dwl": "%dw 2.0\noutput application/json\n---\nin0", + "in0.json": '{"answer": 42}', + "out.json": '{"answer": 42}', + "out.xml": "42", + }, + ) + + discovery = discover_cases(tmp_path) + + assert discovery.structural_skips == 0 + assert discovery.structural_case_identifiers == [] + assert len(discovery.cases) == 1 + scenarios = discovery.cases[0].scenarios + assert [scenario.identifier for scenario in scenarios] == [ + "runtime/maps-out.json:out.json", + "runtime/maps-out.json:out.xml", + ] + assert scenarios[0].transform == "%dw 2.0\noutput application/json\n---\nin0" + assert scenarios[0].inputs["in0"].mime_type == "application/json" + assert scenarios[0].inputs["in0"].content == b'{"answer": 42}' + assert scenarios[0].expected == b'{"answer": 42}' + + +@pytest.mark.parametrize( + ("extension", "actual", "expected"), + [ + ("json", b'{"a": 1, "b": 2}', b'{"b":2,"a":1}'), + ("xml", b"\n 1\n", b"1"), + ("xml", b"1tail", b"1tail"), + ("csv", b"a,b\r\n1,2\r\n", b"a,b\n1,2"), + ("txt", b"value\r\n", b"value\n"), + ("dwl", b"fun f(x) = x + 1", b"fun f(x)=x+1"), + ("properties", b"a=b\r\n", b"a=b\n"), + ("urlencoded", b"a=1&b=2\r\n", b"a=1&b=2"), + ("multipart", b"--boundary\r\nbody", b"--boundary\nbody"), + ("bin", b"\x00\x01", b"\x00\x01"), + ], +) +def test_compare_output_matches_required_corpus_extensions( + extension: str, actual: bytes, expected: bytes +): + """Catches extension dispatch that compares TCK writer output too strictly.""" + assert compare_output(extension, actual, expected).match + + +def test_compare_output_rejects_unknown_extension(): + """Catches silent fallbacks that hide unsupported corpus output formats.""" + result = compare_output("unknown", b"actual", b"expected") + + assert not result.match + assert "unknown output extension" in result.detail + + +@pytest.mark.parametrize(("actual", "expected"), [(b"true", b"1"), (b"false", b"0")]) +def test_compare_output_rejects_json_booleans_as_numbers(actual: bytes, expected: bytes): + assert not compare_output("json", actual, expected).match + + +def test_compare_output_rejects_different_xml_tail_text(): + """Catches structural XML comparison that discards text after child elements.""" + result = compare_output( + "xml", + b"1actual tail", + b"1expected tail", + ) + + assert not result.match + + +def test_compare_output_rejects_different_namespace_prefixes(): + """Matches Node's policy: declaration placement is ignored, prefixes are not.""" + result = compare_output( + "xml", + b'1', + b'1', + ) + + assert not result.match + + +def test_exclusion_registry_requires_category_and_reason(): + """Catches exclusions that cannot be audited by category and rationale.""" + errors = validate_exclusions( + { + "missing-category": { + "case_identifier": "missing-category", + "reason": "needs a module", + }, + "missing-reason": { + "case_identifier": "missing-reason", + "category": "unsupported-dw-module-resolution", + }, + } + ) + + assert errors == [ + "missing-category: missing category", + "missing-reason: missing reason", + ] + + +def test_exclusion_registry_requires_case_identity_supported_category_and_reason(): + """Catches exclusions that cannot be traced to one approved runtime limitation.""" + errors = validate_exclusions( + { + "runtime/missing-identity": { + "category": "unsupported-dw-module-resolution", + "reason": "Cannot resolve dw::core::Assertions", + }, + "runtime/mismatched-identity": { + "case_identifier": "runtime/another-case", + "category": "unsupported-dw-module-resolution", + "reason": "Cannot resolve dw::core::Assertions", + }, + "runtime/unsupported-category": { + "case_identifier": "runtime/unsupported-category", + "category": "broad-runtime-exception", + "reason": "runtime failure", + }, + "runtime/blank-reason": { + "case_identifier": "runtime/blank-reason", + "category": "unsupported-dw-module-resolution", + "reason": " ", + }, + } + ) + + assert errors == [ + "runtime/missing-identity: missing case identity", + "runtime/mismatched-identity: case identity must match registry key", + "runtime/unsupported-category: unsupported category broad-runtime-exception", + "runtime/blank-reason: missing reason", + ] + + +def test_only_declared_case_identifiers_are_excluded(): + """Catches broad exclusion matching that can skip unrelated failures.""" + assert validate_exclusions(EXCLUDED_CASES, SCENARIOS) == [] + assert exclusion_for("unknown-case") is None + exclusion = exclusion_for("runtime/import-lib-out.json") + assert exclusion.case_identifier == "runtime/import-lib-out.json" + assert exclusion.category == "unsupported-dw-module-resolution" + assert len(EXCLUDED_CASES) == 39 + + +def test_exclusion_registry_uses_the_inventory_categories(): + """Catches category collapse that would conceal the unsupported boundary.""" + categories = {} + for exclusion in EXCLUDED_CASES.values(): + categories[exclusion.category] = categories.get(exclusion.category, 0) + 1 + + assert categories == { + "unavailable-classpath-test-resource": 4, + "unavailable-java-module": 11, + "unsupported-dw-module-resolution": 24, + } + + +def test_exclusion_registry_rejects_unreachable_active_entries(): + """Catches active exclusions that cannot affect any discovered runnable scenario.""" + scenario = TckScenario( + "runtime/imports:out.json", + "%dw 2.0\nimport sample from test::module\n--- sample", + {}, + b"null", + "json", + None, + ) + + errors = validate_exclusions( + { + "runtime/imports": Exclusion( + "runtime/imports", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "imports test module", + ), + "runtime/not-discovered": Exclusion( + "runtime/not-discovered", + UNSUPPORTED_DW_MODULE_RESOLUTION, + "stale entry", + ), + }, + [scenario], + ) + + assert errors == ["runtime/not-discovered: not a discovered runnable case"] + + +def test_tck_summary_ignores_collection_nodes(): + """Catches pytest collection nodes being counted as test reports.""" + output = [] + terminalreporter = SimpleNamespace( + stats={ + "passed": [ + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/plain:out.json]", + outcome="passed", + ), + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/excluded:out.json]", + outcome="skipped", + ), + SimpleNamespace( + when="call", + nodeid="tests/tck/test_conformance.py::test_tck_scenario[runtime/failing:out.json]", + outcome="failed", + ), + ], + "": [SimpleNamespace(nodeid="tests/tck/test_conformance.py")], + }, + write_line=output.append, + ) + config = SimpleNamespace(_tck_discovery=(DISCOVERY, SCENARIOS, [SCENARIOS[0]], set())) + + pytest_terminal_summary(terminalreporter, 0, config) + + assert output[-1] == ( + "TCK totals: selected=731, structural-skips=191, structural-module-cases=0, executed=2, " + "active-exclusions=1, passed=1, failed=1, xfail=0" + ) diff --git a/native-lib/python/tests/test_dataweave_module.py b/native-lib/python/tests/test_dataweave_module.py deleted file mode 100755 index 959834b3..00000000 --- a/native-lib/python/tests/test_dataweave_module.py +++ /dev/null @@ -1,539 +0,0 @@ -#!/usr/bin/env python3 -""" -Quick test script for the DataWeave Python module. -""" - -import sys -from pathlib import Path - -_PYTHON_SRC_DIR = Path(__file__).resolve().parents[1] / "src" -sys.path.insert(0, str(_PYTHON_SRC_DIR)) - -import dataweave - -def test_input_value_mime_type_constructor(): - """Test InputValue accepts the public mime_type constructor keyword.""" - print("Testing InputValue mime_type constructor...") - try: - value = dataweave.InputValue( - content="1234567", - mime_type="application/csv", - properties={"header": False, "separator": "4"}, - ) - assert value.mime_type == "application/csv" - print("[OK] InputValue mime_type constructor works") - return True - except Exception as e: - print(f"[FAIL] InputValue mime_type constructor failed: {e}") - return False - -def test_basic(): - """Test basic functionality""" - print("Testing basic script execution...") - try: - result = dataweave.run("2 + 2", {}) - assert result.get_string() == "4", f"Expected '4', got '{result.get_string()}'" - print("[OK] Basic script execution works") - return True - except Exception as e: - print(f"[FAIL] Basic script execution failed: {e}") - return False - -def test_with_inputs(): - """Test script with inputs""" - print("\nTesting script with inputs...") - try: - result = dataweave.run("num1 + num2", {"num1": 25, "num2": 17}) - assert result.get_string() == "42", f"Expected '42', got '{result.get_string()}'" - print("[OK] Script with inputs works") - return True - except Exception as e: - print(f"[FAIL] Script with inputs failed: {e}") - return False - -def test_context_manager(): - """Test context manager""" - print("\nTesting with context manager...") - try: - with dataweave.DataWeave() as dw: - - result = dw.run("sqrt(144)") - assert result.get_string() == "12", f"Expected '12', got '{result.get_string()}'" - result = dw.run("sqrt(10000)") - assert result.get_string() == "100", f"Expected '100', got '{result.get_string()}'" - print("[OK] Script execution witch context manager works") - return True - except Exception as e: - print(f"[FAIL] Script execution witch context manager failed: {e}") - return False - -def test_encoding(): - """Test reading UTF-16 XML input and producing CSV output""" - print("\nTesting encoding (UTF-16 XML -> CSV)...") - try: - xml_path = ( - Path(__file__).resolve().parent / "person.xml" - ) - xml_bytes = xml_path.read_bytes() - - script = """output application/csv header=true ---- -[payload.person] -""" - - result = dataweave.run( - script, - { - "payload": { - "content": xml_bytes, - "mimeType": "application/xml", - "charset": "UTF-16", - } - }, - ) - - out = result.get_string() or "" - print(f"out: \n{out}") - assert result.success is True, f"Expected success=true, got: {result}" - assert "name" in out and "age" in out, f"CSV header missing, got: {out!r}" - assert "Billy" in out, f"Expected name 'Billy' in CSV, got: {out!r}" - assert "31" in out, f"Expected age '31' in CSV, got: {out!r}" - - print("[OK] Encoding conversion works") - return True - except Exception as e: - print(f"[FAIL] Encoding conversion failed: {e}") - return False - -def test_auto_conversion(): - """Test auto-conversion of different types""" - print("\nTesting auto-conversion...") - try: - - # Test array - result = dataweave.run( - "numbers[0]", - {"numbers": [1, 2, 3]} - ) - assert result.get_string() == "1", f"Expected '1', got '{result.get_string()}'" - - print("[OK] Auto-conversion works") - return True - except Exception as e: - print(f"[FAIL] Auto-conversion failed: {e}") - return False - -def test_callback_output_basic(): - """Test callback-based output streaming""" - print("\nTesting callback output basic...") - try: - chunks = [] - - def on_write(data: bytes) -> int: - chunks.append(data) - return 0 - - result = dataweave.run_callback("2 + 2", on_write) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(chunks) - text = full.decode(result.charset or "utf-8") - assert text == "4", f"Expected '4', got '{text}'" - print(f"[OK] Callback output basic works (chunks={len(chunks)}, result='{text}')") - return True - except Exception as e: - print(f"[FAIL] Callback output basic failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_callback_output_with_inputs(): - """Test callback-based output streaming with inputs""" - print("\nTesting callback output with inputs...") - try: - chunks = [] - - def on_write(data: bytes) -> int: - chunks.append(data) - return 0 - - result = dataweave.run_callback("num1 + num2", on_write, inputs={"num1": 25, "num2": 17}) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(chunks) - text = full.decode(result.charset or "utf-8") - assert text == "42", f"Expected '42', got '{text}'" - print(f"[OK] Callback output with inputs works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] Callback output with inputs failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_callback_input_output(): - """Test callback-based input and output streaming""" - print("\nTesting callback input+output...") - try: - import io as _io - - source = _io.BytesIO(b'[10, 20, 30, 40, 50]') - output_chunks = [] - - def on_read(buf_size: int) -> bytes: - return source.read(buf_size) - - def on_write(data: bytes) -> int: - output_chunks.append(data) - return 0 - - script = "output application/json\n---\npayload map ($ * 2)" - result = dataweave.run_input_output_callback( - script, - input_name="payload", - input_mime_type="application/json", - read_callback=on_read, - write_callback=on_write, - ) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(output_chunks) - text = full.decode(result.charset or "utf-8") - assert "20" in text, f"Expected 20 in result (10*2), got: {text}" - assert "100" in text, f"Expected 100 in result (50*2), got: {text}" - print(f"[OK] Callback input+output works (chunks={len(output_chunks)}, result={text.strip()[:80]}...)") - return True - except Exception as e: - print(f"[FAIL] Callback input+output failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_basic(): - """Test run_streaming yields chunks and returns metadata""" - print("\nTesting run_streaming basic...") - try: - stream = dataweave.run_streaming("output application/json --- {a: 1, b: 2}") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - assert len(full) > 0, "Expected non-empty output" - text = full.decode(metadata.charset or "utf-8") - assert '"a": 1' in text or '"a":1' in text, f"Expected key 'a' in JSON, got: {text}" - assert metadata.success is True, f"Expected success, got: {metadata}" - assert metadata.mime_type == "application/json", f"Expected json mime, got: {metadata.mime_type}" - print(f"[OK] run_streaming basic works (chunks={len(chunks)}, result={text.strip()[:60]})") - return True - except Exception as e: - print(f"[FAIL] run_streaming basic failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_large(): - """Test run_streaming with large output to verify true streaming (multiple chunks)""" - print("\nTesting run_streaming large...") - try: - script = 'output application/json --- (1 to 5000) map {id: $, name: "item_" ++ $}' - stream = dataweave.run_streaming(script) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert len(chunks) > 1, f"Expected multiple chunks for large output, got {len(chunks)}" - assert '"id": 5000' in text or '"id":5000' in text, f"Expected last item in output" - print(f"[OK] run_streaming large works (chunks={len(chunks)}, bytes={len(full)})") - return True - except Exception as e: - print(f"[FAIL] run_streaming large failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_error(): - """Test run_streaming with an invalid script returns error metadata""" - print("\nTesting run_streaming error...") - try: - stream = dataweave.run_streaming("output application/json --- invalid_var") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - assert metadata.success is False, f"Expected failure, got: {metadata}" - assert metadata.error is not None, "Expected error message" - assert len(chunks) == 0, f"Expected no chunks on error, got {len(chunks)}" - print(f"[OK] run_streaming error works (error={metadata.error[:60]}...)") - return True - except Exception as e: - print(f"[FAIL] run_streaming error failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_streaming_with_inputs(): - """Test run_streaming with input bindings""" - print("\nTesting run_streaming with inputs...") - try: - stream = dataweave.run_streaming("num1 + num2", {"num1": 25, "num2": 17}) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert text.strip() == "42", f"Expected '42', got '{text.strip()}'" - print(f"[OK] run_streaming with inputs works (result='{text.strip()}')") - return True - except Exception as e: - print(f"[FAIL] run_streaming with inputs failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_callback_input_output_large(): - """Test callback-based input+output streaming with large data""" - print("\nTesting callback input+output large...") - try: - import io as _io - - # Build a large JSON array - parts = [b"["] - for i in range(1, 1001): - if i > 1: - parts.append(b",") - parts.append(f'{{"id":{i}}}'.encode()) - parts.append(b"]") - source = _io.BytesIO(b"".join(parts)) - output_chunks = [] - - def on_read(buf_size: int) -> bytes: - return source.read(buf_size) - - def on_write(data: bytes) -> int: - output_chunks.append(data) - return 0 - - result = dataweave.run_input_output_callback( - "output application/json\n---\nsizeOf(payload)", - input_name="payload", - input_mime_type="application/json", - read_callback=on_read, - write_callback=on_write, - ) - assert result.success is True, f"Expected success, got: {result}" - full = b"".join(output_chunks) - text = full.decode(result.charset or "utf-8") - assert text == "1000", f"Expected '1000', got '{text}'" - print(f"[OK] Callback input+output large works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] Callback input+output large failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_basic(): - """Test run_transform with an iterable input and streaming output""" - print("\nTesting run_transform basic...") - try: - input_data = [b'[10, 20, 30, 40, 50]'] - script = "output application/json\n---\npayload map ($ * 2)" - stream = dataweave.run_transform(script, input_stream=input_data, input_mime_type="application/json") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert "20" in text, f"Expected 20 in result (10*2), got: {text}" - assert "100" in text, f"Expected 100 in result (50*2), got: {text}" - print(f"[OK] run_transform basic works (chunks={len(chunks)}, result={text.strip()[:60]})") - return True - except Exception as e: - print(f"[FAIL] run_transform basic failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_large(): - """Test run_transform with large chunked input to verify streaming both directions""" - print("\nTesting run_transform large...") - try: - import io as _io - - # Build a large JSON array as chunked input - parts = [b"["] - for i in range(1, 1001): - if i > 1: - parts.append(b",") - parts.append(f'{{"id":{i}}}'.encode()) - parts.append(b"]") - full_input = b"".join(parts) - - # Feed in 4KB chunks (simulating a file/network read) - def chunked(data, size=4096): - for i in range(0, len(data), size): - yield data[i:i+size] - - script = "output application/json\n---\nsizeOf(payload)" - stream = dataweave.run_transform( - script, - input_stream=chunked(full_input), - input_mime_type="application/json", - ) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert text == "1000", f"Expected '1000', got '{text}'" - print(f"[OK] run_transform large works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] run_transform large failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_large_single_chunk(): - """Regression: a single input chunk > 8KB native buffer must not be truncated.""" - print("\nTesting run_transform large single chunk (>8KB, regression)...") - try: - import json as _json - records = [{"id": i, "name": f"item_{i}", "value": i*3} for i in range(1, 2001)] - payload = _json.dumps(records).encode() # ~97KB, one chunk - assert len(payload) > 8192, f"Expected payload > 8192 bytes, got {len(payload)}" - script = "output application/json\n---\nsizeOf(payload)" - stream = dataweave.run_transform(script, input_stream=iter([payload]), input_mime_type="application/json") - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - text = b"".join(chunks).decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert text == "2000", f"Expected '2000', got '{text}'" - print(f"[OK] run_transform large single chunk works (result='{text}')") - return True - except Exception as e: - print(f"[FAIL] run_transform large single chunk failed: {e}") - import traceback - traceback.print_exc() - return False - -def test_run_transform_with_file(): - """Test run_transform reading from a file-like object""" - print("\nTesting run_transform with file...") - try: - from pathlib import Path - - xml_path = Path(__file__).resolve().parent / "person.xml" - - with open(xml_path, "rb") as f: - stream = dataweave.run_transform( - "output application/csv header=true\n---\n[payload.person]", - input_stream=iter(lambda: f.read(4096), b""), - input_mime_type="application/xml", - input_charset="UTF-16", - ) - chunks = [] - try: - while True: - chunks.append(next(stream)) - except StopIteration as e: - metadata = e.value - - full = b"".join(chunks) - text = full.decode(metadata.charset or "utf-8") - assert metadata.success is True, f"Expected success, got: {metadata}" - assert "Billy" in text, f"Expected 'Billy' in CSV, got: {text}" - assert "31" in text, f"Expected '31' in CSV, got: {text}" - print(f"[OK] run_transform with file works (result={text.strip()[:60]})") - return True - except Exception as e: - print(f"[FAIL] run_transform with file failed: {e}") - import traceback - traceback.print_exc() - return False - -def main(): - """Run all tests""" - print("="*70) - print("DataWeave Python Module - Test Suite") - print("="*70) - - try: - results = [] - results.append(test_input_value_mime_type_constructor()) - results.append(test_basic()) - results.append(test_with_inputs()) - results.append(test_context_manager()) - results.append(test_encoding()) - results.append(test_auto_conversion()) - results.append(test_callback_output_basic()) - results.append(test_callback_output_with_inputs()) - results.append(test_callback_input_output()) - results.append(test_callback_input_output_large()) - results.append(test_run_streaming_basic()) - results.append(test_run_streaming_large()) - results.append(test_run_streaming_error()) - results.append(test_run_streaming_with_inputs()) - results.append(test_run_transform_basic()) - results.append(test_run_transform_large()) - results.append(test_run_transform_large_single_chunk()) - results.append(test_run_transform_with_file()) - - # Cleanup - dataweave.cleanup() - - print("\n" + "="*70) - passed = sum(results) - total = len(results) - print(f"Results: {passed}/{total} tests passed") - print("="*70) - - if passed == total: - print("\n[OK] All tests passed!") - sys.exit(0) - else: - print(f"\n[FAIL] {total - passed} test(s) failed") - sys.exit(1) - - except dataweave.DataWeaveLibraryNotFoundError as e: - print(f"\n[ERROR] {e}") - print("\nPlease build the native library first:") - print(" ./gradlew nativeCompile") - sys.exit(2) - except Exception as e: - print(f"\n[ERROR] Unexpected error: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py new file mode 100644 index 00000000..560f8687 --- /dev/null +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -0,0 +1,114 @@ +from pathlib import Path +import re + +import pytest + + +def named_step_if(document: str, name: str) -> str: + step = re.search( + rf"^(?P[ \t]*)- name: {re.escape(name)}\n" + rf"(?P(?:^(?P=indent)[ \t]+.*\n?)*)", + document, + re.MULTILINE, + ) + assert step, f"missing step {name!r}" + + guard = re.search(r"^[ \t]+if: (?P.+)$", step.group("body"), re.MULTILINE) + assert guard, f"missing if guard for step {name!r}" + return guard.group("guard") + + +@pytest.mark.unit +def test_python_artifact_runs_python_test_before_building_wheel(): + action = (Path(__file__).resolve().parents[4] / ".github/actions/python/action.yml").read_text() + + assert "native-lib:pythonTest" in action + assert action.index("native-lib:pythonTest") < action.index("native-lib:buildPythonWheel") + + +@pytest.mark.unit +def test_python_tck_is_gated_by_the_master_only_workflow_input(): + root = Path(__file__).resolve().parents[4] + action = (root / ".github/actions/python/action.yml").read_text() + workflow = (root / ".github/workflows/main.yml").read_text() + + assert named_step_if(action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" + assert "native-lib:pythonTck" in action + assert "run-tck: ${{ github.ref == 'refs/heads/master' }}" in workflow + + +@pytest.mark.unit +def test_python_artifact_owns_test_dependencies_and_tck_junit_upload(): + root = Path(__file__).resolve().parents[4] + foundation = (root / ".github/actions/build-foundation/action.yml").read_text() + action = (root / ".github/actions/python/action.yml").read_text() + + assert "Install Python test dependencies" not in foundation + assert "native-lib/python[test]" in action + assert "platform:" in action + assert "python-tck-junit-${{ inputs.platform }}" in action + assert action.index("Create Native Lib Python Wheel") < action.index("Run Python TCK Conformance") + assert action.index("Upload Python wheel (artifact)") < action.index("Run Python TCK Conformance") + assert action.index("Upload Python wheel to release") < action.index("Run Python TCK Conformance") + assert action.index("Run Python TCK Conformance") < action.index("Upload Python TCK JUnit") + assert named_step_if(action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" + assert "native-lib/build/test-results/pythonTck.xml" in action + + +@pytest.mark.unit +def test_master_tck_stages_the_shared_corpus_once_before_python_and_node(): + root = Path(__file__).resolve().parents[4] + gradle = (root / "native-lib/build.gradle").read_text() + workflow = (root / ".github/workflows/main.yml").read_text() + node_action = (root / ".github/actions/node/action.yml").read_text() + + python_tck = gradle[gradle.index("tasks.register('pythonTck'"):gradle.index("tasks.register('buildNodePackage'")] + assert "dependsOn tasks.named('stageTckSuites')" not in python_tck + assert "native-lib:stageTckSuites" not in node_action + assert workflow.index("Stage TCK corpus") < workflow.index("- name: Python") + assert workflow.index("Stage TCK corpus") < workflow.index("- name: Node") + + +@pytest.mark.unit +def test_tck_metadata_validation_is_not_selected_by_the_pr_python_test_lane(): + root = Path(__file__).resolve().parents[4] + conformance = (root / "native-lib/python/tests/tck/test_conformance.py").read_text() + + assert "@pytest.mark.unit\ndef test_accepted_baseline_mismatches" not in conformance + + +@pytest.mark.unit +def test_foundation_skips_python_tests_and_master_aggregates_binding_failures(): + root = Path(__file__).resolve().parents[4] + foundation = (root / ".github/actions/build-foundation/action.yml").read_text() + workflow = (root / ".github/workflows/main.yml").read_text() + python_action = (root / ".github/actions/python/action.yml").read_text() + node_action = (root / ".github/actions/node/action.yml").read_text() + + assert "-PskipPythonTests=true" in foundation + assert "id: python" in workflow + assert "id: node" in workflow + assert workflow.count("continue-on-error: true") >= 2 + assert "Fail if binding artifacts failed" in workflow + assert "steps.python.outcome == 'failure'" in workflow + assert "steps.node.outcome == 'failure'" in workflow + assert "platform: ${{ matrix.script_name }}" in workflow + assert named_step_if(python_action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" + assert named_step_if(node_action, "Run Node.js TCK Conformance") == "always() && inputs.run-tck == 'true'" + assert named_step_if(workflow, "Fail if binding artifacts failed") == ( + "always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure')" + ) + assert workflow.index("- name: Native library") < workflow.index("- name: Fail if binding artifacts failed") + + +@pytest.mark.unit +def test_named_step_if_does_not_read_a_later_step_guard(): + mutated_workflow = """\ + - name: Fail if binding artifacts failed + run: exit 1 + - name: Later step + if: always() && (steps.python.outcome == 'failure' || steps.node.outcome == 'failure') +""" + + with pytest.raises(AssertionError, match="missing if guard"): + named_step_if(mutated_workflow, "Fail if binding artifacts failed") diff --git a/native-lib/python/tests/unit/test_encoding.py b/native-lib/python/tests/unit/test_encoding.py new file mode 100644 index 00000000..d16bd357 --- /dev/null +++ b/native-lib/python/tests/unit/test_encoding.py @@ -0,0 +1,61 @@ +import base64 + +import pytest + +import dataweave +from dataweave import encoding + + +@pytest.mark.unit +def test_public_encoding_functions_are_used_by_compatibility_aliases(): + assert dataweave._normalize_input_value is encoding.normalize_input_value + assert dataweave._parse_native_encoded_response is encoding.parse_native_encoded_response + assert dataweave._parse_streaming_result is encoding.parse_streaming_result + + +@pytest.mark.unit +def test_normalize_plain_text_uses_text_mime_type(): + normalized = dataweave._normalize_input_value("hello") + + assert normalized == { + "content": base64.b64encode(b"hello").decode("ascii"), + "mimeType": "text/plain", + "charset": "utf-8", + } + + +@pytest.mark.unit +def test_normalize_explicit_bytes_preserves_mime_charset_and_properties(): + normalized = dataweave._normalize_input_value({ + "content": b"caf\xe9", + "mimeType": "text/plain", + "charset": "latin-1", + "properties": {"header": False}, + }) + + assert normalized == { + "content": base64.b64encode(b"caf\xe9").decode("ascii"), + "mimeType": "text/plain", + "charset": "latin-1", + "properties": {"header": False}, + } + + +@pytest.mark.unit +@pytest.mark.parametrize("value", [ + {"content": "body"}, + {"mimeType": "text/plain"}, +]) +def test_normalize_explicit_input_requires_content_and_mime_type(value): + with pytest.raises(dataweave.DataWeaveError, match="must include both 'content' and 'mimeType'"): + dataweave._normalize_input_value(value) + + +@pytest.mark.unit +def test_normalize_explicit_input_rejects_unsupported_keys(): + with pytest.raises(dataweave.DataWeaveError, match="unsupported keys: unexpected"): + dataweave._normalize_input_value({ + "content": "body", + "mimeType": "text/plain", + "unexpected": True, + }) diff --git a/native-lib/python/tests/unit/test_facade.py b/native-lib/python/tests/unit/test_facade.py new file mode 100644 index 00000000..c281495e --- /dev/null +++ b/native-lib/python/tests/unit/test_facade.py @@ -0,0 +1,93 @@ +import pytest + +import dataweave + + +@pytest.mark.unit +def test_facade_preserves_fixed_legacy_public_exports(): + legacy_exports = [ + "DataWeave", + "DataWeaveError", + "DataWeaveLibraryNotFoundError", + "DataWeaveScriptError", + "ExecutionResult", + "InputValue", + "ReadCallback", + "Stream", + "StreamingResult", + "WriteCallback", + "READ_CALLBACK", + "WRITE_CALLBACK", + "run", + "run_callback", + "run_input_output_callback", + "run_streaming", + "run_transform", + "cleanup", + ] + + for name in legacy_exports: + assert name in dataweave.__all__ + getattr(dataweave, name) + + +@pytest.mark.unit +def test_global_facade_initializes_once_and_cleanup_allows_recreation(monkeypatch): + created = [] + registered = [] + + class FakeRuntime: + def __init__(self): + self.cleaned = False + created.append(self) + + def initialize(self): + pass + + def cleanup(self): + self.cleaned = True + + monkeypatch.setattr(dataweave, "DataWeave", FakeRuntime) + monkeypatch.setattr("atexit.register", registered.append) + + first = dataweave._get_global_instance() + second = dataweave._get_global_instance() + dataweave.cleanup() + third = dataweave._get_global_instance() + + assert first is second + assert first.cleaned is True + assert third is not first + assert registered == [dataweave.cleanup, dataweave.cleanup] + + +@pytest.mark.unit +def test_cleanup_is_noop_without_global_runtime(): + dataweave.cleanup() + + assert dataweave._global_instance is None + + +@pytest.mark.unit +def test_global_cleanup_retains_failed_runtime_for_retry(monkeypatch): + created = [] + + class FakeRuntime: + def __init__(self): + created.append(self) + + def initialize(self): + pass + + def cleanup(self): + raise dataweave.DataWeaveError("teardown failed") + + monkeypatch.setattr(dataweave, "DataWeave", FakeRuntime) + first = dataweave._get_global_instance() + + with pytest.raises(dataweave.DataWeaveError, match="teardown failed"): + dataweave.cleanup() + + second = dataweave._get_global_instance() + assert second is first + dataweave._global_instance = None diff --git a/native-lib/python/tests/unit/test_models.py b/native-lib/python/tests/unit/test_models.py new file mode 100644 index 00000000..b337896d --- /dev/null +++ b/native-lib/python/tests/unit/test_models.py @@ -0,0 +1,84 @@ +import base64 + +import pytest + +import dataweave +from dataweave import models + + +@pytest.mark.unit +def test_public_models_are_exported_from_models_module(): + assert models.ExecutionResult is dataweave.ExecutionResult + assert models.InputValue is dataweave.InputValue + assert models.StreamingResult is dataweave.StreamingResult + assert models.DataWeaveError is dataweave.DataWeaveError + assert models.DataWeaveScriptError is dataweave.DataWeaveScriptError + assert models.DataWeaveLibraryNotFoundError is dataweave.DataWeaveLibraryNotFoundError + assert models.READ_CALLBACK is dataweave.READ_CALLBACK + assert models.WRITE_CALLBACK is dataweave.WRITE_CALLBACK + + +@pytest.mark.unit +def test_input_value_encodes_text_with_its_charset(): + value = dataweave.InputValue("caf\u00e9", charset="latin-1") + + assert value.encode_content() == base64.b64encode(b"caf\xe9").decode("ascii") + + +@pytest.mark.unit +def test_execution_result_decodes_text_payload(): + result = dataweave.ExecutionResult( + success=True, + result=base64.b64encode(b"hello").decode("ascii"), + error=None, + binary=False, + mime_type="text/plain", + charset="utf-8", + ) + + assert result.get_bytes() == b"hello" + assert result.get_string() == "hello" + + +@pytest.mark.unit +def test_execution_result_keeps_binary_payload_as_base64_text(): + payload = base64.b64encode(b"\x00\xff").decode("ascii") + result = dataweave.ExecutionResult(True, payload, None, True, "application/octet-stream", None) + + assert result.get_bytes() == b"\x00\xff" + assert result.get_string() == payload + + +@pytest.mark.unit +def test_execution_result_returns_none_for_unsuccessful_execution(): + result = dataweave.ExecutionResult(False, None, "failed", False, None, None) + + assert result.get_bytes() is None + assert result.get_string() is None + + +@pytest.mark.unit +def test_parse_native_response_preserves_error_result(): + result = dataweave._parse_native_encoded_response('{"success": false, "error": "script failed"}') + + assert result == dataweave.ExecutionResult(False, None, "script failed", False, None, None) + + +@pytest.mark.unit +def test_stream_public_close_and_context_manager_close_the_underlying_generator(): + closed = [] + + def generate(): + try: + yield b"chunk" + finally: + closed.append(True) + + stream = dataweave.Stream(generate()) + + with stream as managed: + assert next(managed) == b"chunk" + + stream.close() + + assert closed == [True] diff --git a/native-lib/python/tests/unit/test_native.py b/native-lib/python/tests/unit/test_native.py new file mode 100644 index 00000000..edaeb5f0 --- /dev/null +++ b/native-lib/python/tests/unit/test_native.py @@ -0,0 +1,280 @@ +from pathlib import Path + +import pytest + +import dataweave +from dataweave import native + + +@pytest.mark.unit +def test_parse_native_response_rejects_malformed_json(): + result = dataweave._parse_native_encoded_response("not json") + + assert result.success is False + assert result.error.startswith("Failed to parse native JSON response:") + + +@pytest.mark.unit +@pytest.mark.parametrize("raw, expected_error", [ + (None, "Native returned null"), + ("", "Native returned empty response"), + ("[]", "Native response JSON is not an object"), +]) +def test_parse_native_response_rejects_invalid_native_values(raw, expected_error): + result = dataweave._parse_native_encoded_response(raw) + + assert result == dataweave.ExecutionResult(False, None, expected_error, False, None, None) + + +@pytest.mark.unit +def test_candidate_paths_prioritize_environment_override(monkeypatch, tmp_path): + override = tmp_path / "custom-dwlib" + monkeypatch.setenv("DATAWEAVE_NATIVE_LIB", str(override)) + + paths = dataweave._candidate_library_paths() + + assert paths[0] == override + assert paths[1:4] == [ + Path(dataweave.__file__).resolve().parent / "native" / "dwlib.dylib", + Path(dataweave.__file__).resolve().parent / "native" / "dwlib.so", + Path(dataweave.__file__).resolve().parent / "native" / "dwlib.dll", + ] + + +@pytest.mark.unit +def test_decode_and_free_releases_native_string_when_decoding_fails(monkeypatch): + freed = [] + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.thread = "thread" + runtime.lib = type("Native", (), {"free_cstring": lambda _self, thread, ptr: freed.append((thread, ptr))})() + monkeypatch.setattr(native.ctypes, "string_at", lambda _ptr: b"\xff") + + with pytest.raises(UnicodeDecodeError): + runtime.decode_and_free(123) + + assert freed == [("thread", 123)] + + +@pytest.mark.unit +def test_decode_and_free_preserves_decode_failure_when_free_also_fails(monkeypatch): + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.thread = "thread" + runtime.lib = type("Native", (), {"free_cstring": lambda _self, _thread, _ptr: (_ for _ in ()).throw(RuntimeError("free failed"))})() + monkeypatch.setattr(native.ctypes, "string_at", lambda _ptr: b"\xff") + + with pytest.raises(UnicodeDecodeError): + runtime.decode_and_free(123) + + +@pytest.mark.unit +def test_native_runtime_registers_abi_and_cleans_up_idempotently(monkeypatch): + class Function: + pass + + class FakeLibrary: + run_script = Function() + free_cstring = Function() + graal_attach_thread = Function() + graal_detach_thread = Function() + + def __init__(self): + self.tear_down_threads = [] + self.graal_create_isolate = Function() + self.graal_create_isolate.__call__ = lambda _params, _isolate, _thread: 0 + self.graal_tear_down_isolate = Function() + self.graal_tear_down_isolate.__call__ = lambda thread: self.tear_down_threads.append(thread) or 0 + + class CallableFunction(Function): + def __init__(self, callback): + self.callback = callback + + def __call__(self, *args): + return self.callback(*args) + + library = FakeLibrary() + library.graal_create_isolate = CallableFunction(lambda _params, _isolate, _thread: 0) + library.graal_tear_down_isolate = CallableFunction(lambda thread: library.tear_down_threads.append(thread) or 0) + + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + runtime = native.NativeRuntime("/tmp/dwlib") + runtime.initialize() + runtime.cleanup() + runtime.cleanup() + + assert library.run_script.argtypes[1:] == [native.ctypes.c_char_p, native.ctypes.c_char_p] + assert library.free_cstring.argtypes[1] is native.ctypes.c_void_p + assert len(library.tear_down_threads) == 1 + assert runtime.initialized is False + + +@pytest.mark.unit +def test_native_runtime_wraps_library_load_errors(monkeypatch): + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: (_ for _ in ()).throw(OSError("bad image"))) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to load library from /tmp/dwlib: bad image"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_initialize_resets_state_when_isolate_creation_fails(monkeypatch): + class Function: + def __call__(self, *_args): + return 9 + + library = type("Native", (), {"graal_create_isolate": Function()})() + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + runtime = native.NativeRuntime("/tmp/dwlib") + + with pytest.raises(dataweave.DataWeaveError, match="Failed to create GraalVM isolate. Error code: 9"): + runtime.initialize() + + assert runtime.lib is None + assert runtime.isolate is None + assert runtime.thread is None + assert runtime.initialized is False + + +@pytest.mark.unit +def test_initialize_requires_create_isolate_export(monkeypatch): + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: object()) + + with pytest.raises(dataweave.DataWeaveError, match="Native library does not export graal_create_isolate"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_initialize_wraps_create_isolate_exception(monkeypatch): + class Function: + def __call__(self, *_args): + raise RuntimeError("native create failure") + + library = type("Native", (), {"graal_create_isolate": Function()})() + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to create GraalVM isolate: native create failure"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_cleanup_wraps_teardown_exception(): + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.initialized = True + runtime.thread = object() + runtime.isolate = object() + runtime.lib = type("Native", (), {"graal_tear_down_isolate": lambda _self, _thread: (_ for _ in ()).throw(RuntimeError("native teardown failure"))})() + + with pytest.raises(dataweave.DataWeaveError, match="Failed to tear down GraalVM isolate: native teardown failure"): + runtime.cleanup() + + +@pytest.mark.unit +def test_initialize_tears_down_isolate_when_required_export_is_missing(monkeypatch): + class Function: + def __init__(self, callback): + self.callback = callback + + def __call__(self, *args): + return self.callback(*args) + + torn_down = [] + library = type("Native", (), {})() + library.graal_create_isolate = Function(lambda _params, _isolate, _thread: 0) + library.graal_tear_down_isolate = Function(lambda thread: torn_down.append(thread) or 0) + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + runtime = native.NativeRuntime("/tmp/dwlib") + + with pytest.raises(dataweave.DataWeaveError, match="Native library does not export run_script"): + runtime.initialize() + + assert len(torn_down) == 1 + assert runtime.lib is None + assert runtime.isolate is None + assert runtime.thread is None + + +@pytest.mark.unit +def test_initialize_rejects_streaming_export_without_required_lifecycle_symbols(monkeypatch): + class Function: + def __init__(self, callback=lambda *_args: 0): + self.callback = callback + + def __call__(self, *args): + return self.callback(*args) + + torn_down = [] + library = type("Native", (), {})() + library.graal_create_isolate = Function() + library.graal_tear_down_isolate = Function(lambda thread: torn_down.append(thread) or 0) + library.run_script = Function() + library.run_script_callback = Function() + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + + with pytest.raises(dataweave.DataWeaveError, match="Native library does not export free_cstring"): + native.NativeRuntime("/tmp/dwlib").initialize() + + assert len(torn_down) == 1 + + +@pytest.mark.unit +@pytest.mark.parametrize("missing_symbol", ["graal_attach_thread", "graal_detach_thread"]) +def test_initialize_rejects_streaming_export_without_thread_lifecycle_symbols(monkeypatch, missing_symbol): + class Function: + def __call__(self, *_args): + return 0 + + library = type("Native", (), {})() + library.graal_create_isolate = Function() + library.graal_tear_down_isolate = Function() + library.run_script = Function() + library.free_cstring = Function() + library.run_script_callback = Function() + for symbol in ("graal_attach_thread", "graal_detach_thread"): + if symbol != missing_symbol: + setattr(library, symbol, Function()) + monkeypatch.setattr(native.ctypes, "CDLL", lambda _path: library) + + with pytest.raises(dataweave.DataWeaveError, match=f"run_script_callback requires native export {missing_symbol}"): + native.NativeRuntime("/tmp/dwlib").initialize() + + +@pytest.mark.unit +def test_cleanup_surfaces_native_teardown_error_code(monkeypatch): + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.initialized = True + runtime.thread = object() + runtime.isolate = object() + runtime.lib = type("Native", (), {"graal_tear_down_isolate": lambda _self, _thread: 7})() + + with pytest.raises(dataweave.DataWeaveError, match="Failed to tear down GraalVM isolate. Error code: 7"): + runtime.cleanup() + + assert runtime.lib is None + assert runtime.thread is None + assert runtime.isolate is None + + +@pytest.mark.unit +@pytest.mark.parametrize( + ("method_name", "error_message"), + [ + ("attach_thread", "Failed to attach worker thread to isolate: native attach failure"), + ("detach_thread", "Failed to detach worker thread from isolate: native detach failure"), + ], +) +def test_thread_lifecycle_wraps_native_invocation_errors(method_name, error_message): + class Native: + def graal_attach_thread(self, _isolate, _thread): + raise RuntimeError("native attach failure") + + def graal_detach_thread(self, _thread): + raise RuntimeError("native detach failure") + + runtime = native.NativeRuntime.__new__(native.NativeRuntime) + runtime.lib = Native() + runtime.isolate = object() + + with pytest.raises(dataweave.DataWeaveError, match=error_message): + if method_name == "attach_thread": + runtime.attach_thread() + else: + runtime.detach_thread(object()) diff --git a/native-lib/python/tests/unit/test_streaming.py b/native-lib/python/tests/unit/test_streaming.py new file mode 100644 index 00000000..b489da0e --- /dev/null +++ b/native-lib/python/tests/unit/test_streaming.py @@ -0,0 +1,406 @@ +import ctypes +from queue import Full, Queue +from threading import Event, Thread +from time import sleep + +import pytest + +import dataweave +from dataweave import runtime as runtime_module + + +class FakeNative: + def __init__(self, metadata=None, attach_code=0, emit=b"", consume_input=False): + self.metadata = metadata + self.attach_code = attach_code + self.emit = emit + self.consume_input = consume_input + self.detached = [] + self.freed = [] + self._buffers = [] + self.detached_event = Event() + + def graal_attach_thread(self, _isolate, _thread): + return self.attach_code + + def graal_detach_thread(self, thread): + self.detached.append(thread) + self.detached_event.set() + return 0 + + def free_cstring(self, thread, ptr): + self.freed.append((thread, ptr)) + + def _response_pointer(self): + if self.metadata is None: + return None + buffer = ctypes.create_string_buffer(self.metadata.encode("utf-8")) + self._buffers.append(buffer) + return ctypes.addressof(buffer) + + def run_script_callback(self, _thread, _script, _inputs, write_callback, _context): + if self.emit: + buffer = ctypes.create_string_buffer(self.emit) + self.write_status = write_callback(None, ctypes.addressof(buffer), len(self.emit)) + return self._response_pointer() + + def run_script_input_output_callback( + self, _thread, _script, _inputs, _input_name, _mime_type, _charset, read_callback, write_callback, _context, + ): + if self.consume_input: + buffer = ctypes.create_string_buffer(3) + read = [] + while True: + size = read_callback(None, ctypes.addressof(buffer), len(buffer)) + self.read_status = size + if size == 0: + break + if size < 0: + return self._response_pointer() + read.append(bytes(buffer.raw[:size])) + self.read_input = b"".join(read) + if self.emit: + buffer = ctypes.create_string_buffer(self.emit) + assert write_callback(None, ctypes.addressof(buffer), len(self.emit)) == 0 + return self._response_pointer() + + +def configured_runtime(native): + runtime = dataweave.DataWeave.__new__(dataweave.DataWeave) + native_runtime = runtime_module.NativeRuntime.__new__(runtime_module.NativeRuntime) + native_runtime.initialized = True + native_runtime.has_callback_streaming = True + native_runtime.has_callback_input_output = True + native_runtime.lib = native + native_runtime.isolate = object() + native_runtime.thread = object() + runtime._native = native_runtime + return runtime + + +@pytest.mark.unit +def test_run_callback_converts_write_callback_exception_to_abort_result(): + native = FakeNative('{"success": false, "error": "callback aborted"}', emit=b"chunk") + runtime = configured_runtime(native) + + result = runtime.run_callback("script", lambda _chunk: (_ for _ in ()).throw(RuntimeError("stop"))) + + assert result == dataweave.StreamingResult(False, "callback aborted", None, None, False) + assert native.write_status == -1 + + +@pytest.mark.unit +def test_run_input_output_callback_converts_read_exception_to_abort_result(): + native = FakeNative('{"success": false, "error": "read aborted"}', consume_input=True) + runtime = configured_runtime(native) + + result = runtime.run_input_output_callback( + "script", "payload", "application/json", lambda _size: (_ for _ in ()).throw(RuntimeError("stop")), lambda _data: 0, + ) + + assert result == dataweave.StreamingResult(False, "read aborted", None, None, False) + assert native.read_status == -1 + + +@pytest.mark.unit +@pytest.mark.parametrize( + "invoke", + [ + lambda runtime: runtime.run_callback("script", lambda _data: 0), + lambda runtime: runtime.run_input_output_callback( + "script", "payload", "application/json", lambda _size: b"", lambda _data: 0, + ), + ], +) +def test_callback_apis_translate_malformed_native_metadata(invoke): + runtime = configured_runtime(FakeNative("not-json")) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to execute callback"): + invoke(runtime) + + +@pytest.mark.unit +def test_run_transform_preserves_remainder_of_large_input_chunk(): + native = FakeNative('{"success": true, "mimeType": "application/json", "charset": "utf-8"}', consume_input=True) + runtime = configured_runtime(native) + stream = runtime.run_transform("script", [b"abcdefgh"], input_mime_type="application/json") + + assert list(stream) == [] + assert native.read_input == b"abcdefgh" + assert stream.metadata == dataweave.StreamingResult(True, None, "application/json", "utf-8", False) + + +@pytest.mark.unit +def test_run_streaming_returns_failure_metadata_when_worker_produces_no_metadata(monkeypatch): + class MetadataDroppingQueue(Queue): + def put(self, item, *args, **kwargs): + if isinstance(item, dict): + return None + return super().put(item, *args, **kwargs) + + monkeypatch.setattr(runtime_module, "Queue", MetadataDroppingQueue) + runtime = configured_runtime(FakeNative('{"success": true}')) + stream = runtime.run_streaming("script") + + assert list(stream) == [] + assert stream.metadata == dataweave.StreamingResult(False, "No metadata received from native call", None, None, False) + + +@pytest.mark.unit +def test_run_streaming_returns_attach_failure_without_detaching_unattached_thread(): + native = FakeNative(attach_code=9) + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + + assert list(stream) == [] + assert stream.metadata == dataweave.StreamingResult(False, "Failed to attach worker thread to isolate (code 9)", None, None, False) + assert native.detached == [] + + +@pytest.mark.unit +def test_stream_public_close_aborts_worker_and_detaches_after_consumer_abandons_output(): + class BlockingFakeNative(FakeNative): + def __init__(self): + super().__init__('{"success": false, "error": "aborted"}') + self.first_chunk_written = Event() + self.cancelled = None + + def run_script_callback(self, _thread, _script, _inputs, write_callback, _context): + first = ctypes.create_string_buffer(b"first") + assert write_callback(None, ctypes.addressof(first), 5) == 0 + self.first_chunk_written.set() + assert self.cancelled.wait(1) + second = ctypes.create_string_buffer(b"second") + self.write_status = write_callback(None, ctypes.addressof(second), 6) + return self._response_pointer() + + native = BlockingFakeNative() + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + native.cancelled = stream._cancelled + + assert next(stream) == b"first" + assert native.first_chunk_written.wait(1) + stream.close() + + assert native.detached_event.wait(1) + assert native.write_status == -1 + + +@pytest.mark.unit +def test_run_input_output_callback_rejects_oversized_read_chunk_without_truncating(): + class OversizedInputNative(FakeNative): + def run_script_input_output_callback( + self, _thread, _script, _inputs, _input_name, _mime_type, _charset, read_callback, _write_callback, _context, + ): + buffer = ctypes.create_string_buffer(3) + self.read_status = read_callback(None, ctypes.addressof(buffer), len(buffer)) + self.read_input = bytes(buffer.raw[:max(self.read_status, 0)]) + return self._response_pointer() + + native = OversizedInputNative('{"success": false, "error": "read aborted"}') + runtime = configured_runtime(native) + + result = runtime.run_input_output_callback( + "script", "payload", "application/json", lambda _size: b"oversized", lambda _data: 0, + ) + + assert result == dataweave.StreamingResult(False, "read aborted", None, None, False) + assert native.read_status == -1 + assert native.read_input == b"" + + +@pytest.mark.unit +def test_stream_early_close_does_not_block_terminal_publication_on_full_queue(monkeypatch): + cancelled = [] + terminal_blocked = Event() + + class CancellationAwareQueue(Queue): + def put(self, item, block=True, timeout=None): + if not isinstance(item, bytes) and cancelled[0].is_set(): + if timeout is None: + terminal_blocked.set() + raise Full + return super().put(item, block, timeout) + + class FullQueueFakeNative(FakeNative): + def __init__(self): + super().__init__('{"success": false, "error": "aborted"}') + self.queue_full = Event() + self.cancelled = None + + def run_script_callback(self, _thread, _script, _inputs, write_callback, _context): + first = ctypes.create_string_buffer(b"first") + assert write_callback(None, ctypes.addressof(first), 5) == 0 + second = ctypes.create_string_buffer(b"second") + assert write_callback(None, ctypes.addressof(second), 6) == 0 + self.queue_full.set() + assert self.cancelled.wait(1) + third = ctypes.create_string_buffer(b"third") + self.write_status = write_callback(None, ctypes.addressof(third), 5) + return self._response_pointer() + + monkeypatch.setattr(runtime_module, "Queue", CancellationAwareQueue) + monkeypatch.setattr(runtime_module, "_OUTPUT_QUEUE_MAXSIZE", 1) + native = FullQueueFakeNative() + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + cancelled.append(stream._cancelled) + native.cancelled = stream._cancelled + + assert next(stream) == b"first" + assert native.queue_full.wait(1) + stream._close() + + assert native.write_status == -1 + assert native.detached_event.wait(1) + assert terminal_blocked.is_set() is False + + +@pytest.mark.unit +def test_runtime_module_owns_dataweave_orchestration(): + assert dataweave.DataWeave is runtime_module.DataWeave + + +@pytest.mark.unit +def test_run_streaming_reports_worker_timeout_when_native_call_produces_no_output(monkeypatch): + class BlockingFakeNative(FakeNative): + def run_script_callback(self, _thread, _script, _inputs, _write_callback, _context): + sleep(0.05) + return self._response_pointer() + + monkeypatch.setattr(runtime_module, "_WORKER_TIMEOUT_SECONDS", 0.01) + runtime = configured_runtime(BlockingFakeNative('{"success": true}')) + + with pytest.raises(dataweave.DataWeaveError, match="Worker thread timeout after 0.01 seconds"): + list(runtime.run_streaming("script")) + + +@pytest.mark.unit +def test_stream_worker_start_failure_does_not_block_cleanup(monkeypatch): + native = FakeNative('{"success": true}') + native.graal_tear_down_isolate = lambda _thread: 0 + runtime = configured_runtime(native) + + def fail_start(_worker): + raise RuntimeError("cannot start") + + monkeypatch.setattr(runtime_module.Thread, "start", fail_start) + + with pytest.raises(RuntimeError, match="cannot start"): + list(runtime.run_streaming("script")) + runtime.cleanup() + + +@pytest.mark.unit +def test_stream_finalization_does_not_raise_when_a_native_worker_cannot_cancel(monkeypatch): + class UncancellableNative(FakeNative): + def run_script_callback(self, _thread, _script, _inputs, _write_callback, _context): + sleep(0.1) + return self._response_pointer() + + monkeypatch.setattr(runtime_module, "_WORKER_JOIN_TIMEOUT_SECONDS", 0.001) + runtime = configured_runtime(UncancellableNative('{"success": true}')) + stream = runtime.run_streaming("script") + + stream.close() + + +@pytest.mark.unit +def test_cleanup_refuses_to_tear_down_isolate_while_stream_worker_is_active(monkeypatch): + class BlockingNative(FakeNative): + def __init__(self): + super().__init__('{"success": true}') + self.started = Event() + self.release = Event() + self.torn_down = False + + def run_script_callback(self, _thread, _script, _inputs, _write_callback, _context): + self.started.set() + self.release.wait() + return self._response_pointer() + + def graal_tear_down_isolate(self, _thread): + self.torn_down = True + return 0 + + monkeypatch.setattr(runtime_module, "_WORKER_JOIN_TIMEOUT_SECONDS", 0.001) + native = BlockingNative() + runtime = configured_runtime(native) + stream = runtime.run_streaming("script") + consumer_error = [] + + def consume(): + try: + next(stream, None) + except dataweave.DataWeaveError as error: + consumer_error.append(error) + + consumer = Thread(target=consume) + consumer.start() + assert native.started.wait(timeout=1) + stream.close() + + with pytest.raises(dataweave.DataWeaveError, match="active streaming worker"): + runtime.cleanup() + assert native.torn_down is False + + native.release.set() + consumer.join(timeout=1) + assert consumer_error == [] + assert native.detached_event.wait(timeout=1) + runtime.cleanup() + assert native.torn_down is True + + +@pytest.mark.unit +def test_stream_worker_cannot_register_after_isolate_teardown_starts(): + class BlockingCleanupNative(FakeNative): + def __init__(self): + super().__init__('{"success": true}') + self.cleanup_started = Event() + self.release_cleanup = Event() + + def graal_tear_down_isolate(self, _thread): + self.cleanup_started.set() + self.release_cleanup.wait() + return 0 + + native = BlockingCleanupNative() + runtime = configured_runtime(native) + cleanup = Thread(target=runtime.cleanup) + cleanup.start() + assert native.cleanup_started.wait(timeout=1) + + with pytest.raises(dataweave.DataWeaveError, match="being cleaned up"): + runtime._register_stream_worker(Thread()) + + native.release_cleanup.set() + cleanup.join(timeout=1) + + +@pytest.mark.unit +def test_run_streaming_surfaces_detach_failure_without_primary_execution_failure(): + class DetachFailingNative(FakeNative): + def graal_detach_thread(self, thread): + super().graal_detach_thread(thread) + return 7 + + runtime = configured_runtime(DetachFailingNative('{"success": true}')) + + with pytest.raises(dataweave.DataWeaveError, match="Failed to detach worker thread from isolate. Error code: 7"): + list(runtime.run_streaming("script")) + + +@pytest.mark.unit +def test_run_streaming_preserves_unsuccessful_metadata_when_detach_fails(): + class DetachFailingNative(FakeNative): + def graal_detach_thread(self, thread): + super().graal_detach_thread(thread) + return 7 + + runtime = configured_runtime(DetachFailingNative('{"success": false, "error": "script failed"}')) + stream = runtime.run_streaming("script") + + assert list(stream) == [] + assert stream.metadata == dataweave.StreamingResult(False, "script failed", None, None, False)