From cbb1f34f2b2d09cdf9c81aaa0f01ee9732b28a8b Mon Sep 17 00:00:00 2001 From: A Date: Wed, 12 Aug 2026 07:49:14 -0700 Subject: [PATCH] Ship linkable libraries in the macOS wheel The macOS wheel shipped one fused Python extension and no linkable libraries, so a C++ application got nothing from it: the headers and the CMake package were there, but every component a consumer asked for resolved to nothing. On Linux the same wheel ships the runtime, the kernels, the delegates, the thread pool and the profiler as separate libraries, and a C++ application links them directly. The three mechanisms that made this Linux only now have Mach-O equivalents: runtime search path $ORIGIN loader_path keeping a registration only library linked --no-as-needed -force_load, already present library identity ELF soname install name relative to rpath Windows is still refused, because there the runtime carries no export annotations for a DLL, which is a missing capability rather than a different spelling of one. The packaging entries and the package config asked for a .so by name, so they would have looked for a file the build never emits. Both now derive the suffix from the platform, and the config also accounts for Mach-O putting the version before the suffix, libfoo.1.dylib, where ELF puts it after, libfoo.so.1. The two wheel test suites now run on macOS as well, which is what makes the split verified rather than claimed: the Python extension links these libraries itself, so it passes whether or not the package config names them or the shipped headers are complete. Porting them needed a suffix helper, the Mach-O spelling of the symbol queries, since nm -D asks for a dynamic symbol table that Mach-O does not have, and otool in place of readelf. One check keeps a documented skip on macOS: ldd resolves dependencies transitively and reports undefined symbols, and otool -L only lists recorded names, so claiming equivalence there would weaken the check while appearing to strengthen coverage. Test Plan: exercised both platform branches of every changed helper. The CMake helpers, driven with real CMake and the Apple branch forced, so the macOS answers are checked rather than assumed: shipped runtime path Linux $ORIGIN macOS loader_path two entry path Linux $ORIGIN/../../lib:... macOS loader_path/../../lib;... library identity Linux version only macOS install name rpath The Python helpers, loaded under each platform: suffix .so .dylib defined symbols nm -DC nm -gU -C undefined symbols nm -DC --undefined-only nm -gu -C library file name libexecutorch.so libexecutorch.dylib The symbol queries and the load command reader were then run against a real Mach-O library on macOS: 137 defined and 133 undefined symbols listed, and the load commands read, including the dependency and search path entries and the torch dependency the libtorch check looks for. Linux is unaffected, verified by building a wheel from this change and confirming it still ships the same six libraries. ghstack-source-id: ac0cac5f775f0ac1ac5e470964b7c45a366af6df ghstack-comment-id: 5263018041 Pull-Request: https://github.com/pytorch/executorch/pull/21771 --- .ci/scripts/wheel/test_cpp_sdk.py | 183 +++++++++++++++++----- .ci/scripts/wheel/test_macos.py | 18 +++ CMakeLists.txt | 44 ++++-- extension/training/CMakeLists.txt | 9 +- kernels/quantized/CMakeLists.txt | 13 +- setup.py | 142 +++++++++++++---- tools/cmake/Utils.cmake | 113 +++++++++---- tools/cmake/executorch-wheel-config.cmake | 43 ++++- tools/cmake/preset/pybind.cmake | 10 +- 9 files changed, 450 insertions(+), 125 deletions(-) diff --git a/.ci/scripts/wheel/test_cpp_sdk.py b/.ci/scripts/wheel/test_cpp_sdk.py index 68597de8c03..678b6da070c 100644 --- a/.ci/scripts/wheel/test_cpp_sdk.py +++ b/.ci/scripts/wheel/test_cpp_sdk.py @@ -33,6 +33,7 @@ _EXPORT_SCRIPT = """ import json import sys +from pathlib import Path import torch from executorch.exir import to_edge_transform_and_lower @@ -64,7 +65,15 @@ def forward(self, x, image): # variants of the quantized operators with torch. Without it the export fails with # "Missing out variants: quantized_decomposed::quantize_per_tensor", because the # lowering step has no out variant to select. - import executorch.kernels.quantized # noqa: F401 + # Loaded directly rather than through executorch.kernels.quantized, whose __init__ + # swallows every exception, so a load failure would otherwise appear much later as + # "Missing out variants" with no indication of why. + import executorch as _executorch + + _root = Path(list(_executorch.__path__)[0]) / "kernels" / "quantized" + _libs = sorted(_root.glob("*quantized_ops_aot_lib.*")) + assert len(_libs) == 1, f"expected one ahead-of-time library, found {_libs}" + torch.ops.load_library(str(_libs[0])) from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import ( get_symmetric_quantization_config, XNNPACKQuantizer, @@ -261,6 +270,62 @@ def _consumer_cmake(components) -> str: """ +def _mach_o_runtime_paths(binary) -> list: + """The runtime search path entries a Mach-O binary records. + + Mach-O keeps one entry per LC_RPATH load command, where ELF keeps a single colon joined + string, so they are read rather than split. + """ + otool = _tool("otool") + assert otool is not None, "otool is required to read a Mach-O runtime search path" + listing = subprocess.run( + [otool, "-l", str(binary)], capture_output=True, text=True, check=False + ).stdout + entries = [] + lines = listing.splitlines() + for index, line in enumerate(lines): + if "LC_RPATH" not in line: + continue + for following in lines[index + 1 : index + 4]: + stripped = following.strip() + if stripped.startswith("path "): + entries.append(stripped.split(" (offset", 1)[0][len("path ") :]) + break + return entries + + +def _dynamic_lib_suffix() -> str: + """The loadable library suffix on this platform, including the dot.""" + return ".dylib" if sys.platform == "darwin" else ".so" + + +def _library_file_name(base_name: str) -> str: + """The file name a library has on this platform.""" + return f"{base_name}{_dynamic_lib_suffix()}" + + +def _recorded_dependencies(binary) -> str: + """What a built binary records about its dependencies and search paths. + + readelf prints the ELF dynamic section, otool -l the Mach-O load commands. Both + carry the same facts: a dependency entry and a runtime search path entry, named + NEEDED and RUNPATH on ELF, LC_LOAD_DYLIB and LC_RPATH on Mach-O. + """ + if sys.platform == "darwin": + tool, args = _tool("otool"), ["-l"] + needed = "otool" + else: + tool, args = _tool("readelf"), ["-d"] + needed = "readelf" + assert tool is not None, f"{needed} is needed to read the runtime search path" + return subprocess.run( + [tool, *args, str(binary)], + capture_output=True, + text=True, + check=True, + ).stdout + + def _tool(name: str) -> str: """Locate a build tool, including one pip installed beside this interpreter. @@ -560,19 +625,14 @@ def test_consumer_is_relocatable(work_dir: Path) -> None: model, reference = _export(work_dir, "plain") consumer = _build_consumer(work_dir, "relocate", ["runtime", "kernels_optimized"]) - assert shutil.which("readelf") is not None, "readelf is needed to read the RUNPATH" - dynamic = subprocess.run( - [_tool("readelf"), "-d", str(consumer)], - capture_output=True, - text=True, - check=True, - ).stdout - assert "libexecutorch.so" in dynamic, ( + dynamic = _recorded_dependencies(consumer) + assert _library_file_name("libexecutorch") in dynamic, ( "the application records no dependency on the shipped runtime, so it is not " f"linking what the wheel ships:\n{dynamic}" ) - assert "$ORIGIN" in dynamic, ( - "the application has no $ORIGIN-relative runtime search path, so it cannot " + token = "@loader_path" if sys.platform == "darwin" else "$ORIGIN" + assert token in dynamic, ( + f"the application has no {token} relative runtime search path, so it cannot " f"work anywhere but where it was built:\n{dynamic}" ) # The newer tag specifically, not just any search path. DT_RPATH is searched @@ -580,12 +640,15 @@ def test_consumer_is_relocatable(work_dir: Path) -> None: # a consumer given DT_RPATH cannot point an instrumented or locally built # runtime at their application. Both tags satisfy the check above, so without # this the package could silently go back to the older one. - assert "(RUNPATH)" in dynamic, ( - "the application's runtime search path is recorded as DT_RPATH rather than " - "DT_RUNPATH. DT_RPATH outranks LD_LIBRARY_PATH and is inherited by " - "dependencies, so a consumer could not override a packaged library with " - f"their own build:\n{dynamic}" - ) + # ELF only. Mach-O records one LC_RPATH with no weaker older variant, so there is no + # equivalent preference to check there. + if sys.platform != "darwin": + assert "(RUNPATH)" in dynamic, ( + "the application's runtime search path is recorded as DT_RPATH rather than " + "DT_RUNPATH. DT_RPATH outranks LD_LIBRARY_PATH and is inherited by " + "dependencies, so a consumer could not override a packaged library with " + f"their own build:\n{dynamic}" + ) package_dir = _installed_package_dir() deployed = work_dir / "deployed" @@ -598,7 +661,7 @@ def test_consumer_is_relocatable(work_dir: Path) -> None: directory = package_dir / source if not directory.is_dir(): continue - for library in sorted(directory.glob("lib*.so*")): + for library in sorted(directory.glob(_library_file_name("lib*") + "*")): if library.is_file() and not library.is_symlink(): shutil.copy2(library, deployed / library.name) @@ -625,20 +688,50 @@ def test_consumer_is_relocatable(work_dir: Path) -> None: "be installed. Without it the relocated application resolves the original " "package and the check would pass without testing anything." ) - current = subprocess.run( - [patchelf, "--print-rpath", str(moved)], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - kept = [ - entry - for entry in current.split(":") - if entry and not entry.startswith(str(package_dir)) - ] - subprocess.run( - [patchelf, "--set-rpath", ":".join(kept) or "$ORIGIN", str(moved)], check=True - ) + if sys.platform == "darwin": + entries = _mach_o_runtime_paths(moved) + else: + entries = [ + entry + for entry in subprocess.run( + [patchelf, "--print-rpath", str(moved)], + capture_output=True, + text=True, + check=True, + ) + .stdout.strip() + .split(":") + if entry + ] + kept = [entry for entry in entries if not entry.startswith(str(package_dir))] + + if sys.platform == "darwin": + # One entry per load command, so each unwanted one is deleted individually and the + # fallback is added only when stripping emptied the list. + install_name_tool = _tool("install_name_tool") + assert install_name_tool is not None, ( + "install_name_tool is required to strip the wheel's absolute directory from the " + "relocated application, and without it this check passes for the wrong reason" + ) + for entry in entries: + if entry in kept: + continue + subprocess.run( + [install_name_tool, "-delete_rpath", entry, str(moved)], + capture_output=True, + check=False, + ) + if not kept: + subprocess.run( + [install_name_tool, "-add_rpath", "@loader_path", str(moved)], + capture_output=True, + check=False, + ) + else: + subprocess.run( + [patchelf, "--set-rpath", ":".join(kept) or "$ORIGIN", str(moved)], + check=True, + ) output = _run_consumer(moved, model, reference, work_dir) print(f"✓ the application still runs deployed away from the wheel ({output})") @@ -790,7 +883,9 @@ def test_profiler_component_is_usable(work_dir: Path) -> None: # Globbed, not an exact name: the library carries a version suffix outside a wheel build, and an exact # match would silently skip this check there. The profiler is required elsewhere in this suite, so its # absence is a fault rather than a reason to skip. - shipped = sorted((package_dir / "lib").glob("libexecutorch_etdump.so*")) + shipped = sorted( + (package_dir / "lib").glob(_library_file_name("libexecutorch_etdump") + "*") + ) assert shipped, ( f"the wheel ships no profiler library under {package_dir / 'lib'}, so the etdump component it " "advertises cannot be linked" @@ -1026,7 +1121,7 @@ def test_shipped_headers_have_implementations(work_dir: Path) -> None: "executorch_kernels_optimized", "executorch_threadpool", ) - if (library_dir / f"lib{name}.so").is_file() + if (library_dir / (f"lib{name}" + _dynamic_lib_suffix())).is_file() ], f"-Wl,-rpath,{library_dir}", ], @@ -1236,10 +1331,8 @@ def test_pre_3_28_route_builds_a_consumer_through_variables(work_dir: Path) -> N # the only thing that carries them on this route. Reading the dynamic section rather # than running because running needs a model, which the modern-CMake tests above # cover once and this one only owns the variables path. - dependencies = subprocess.run( - ["readelf", "-d", str(consumer)], capture_output=True, text=True, check=True - ).stdout - assert "libexecutorch.so" in dependencies, ( + dependencies = _recorded_dependencies(consumer) + assert _library_file_name("libexecutorch") in dependencies, ( "a consumer built through EXECUTORCH_LIBRARIES on pre-3.28 CMake does not " f"depend on the runtime:\n{dependencies}" ) @@ -1269,7 +1362,11 @@ def test_quantized_kernels_component_runs_a_model(work_dir: Path) -> None: package_dir = _installed_package_dir() # Globbed for the same reason the profiler check is: the library carries a version suffix outside a # wheel build, and an exact name would skip this silently there rather than running it. - shipped = sorted((package_dir / "lib").glob("libexecutorch_kernels_quantized.so*")) + shipped = sorted( + (package_dir / "lib").glob( + _library_file_name("libexecutorch_kernels_quantized") + "*" + ) + ) assert shipped, ( "the wheel ships no quantized kernels library. The preset that builds it enables " "them unconditionally, so this is a packaging or build regression rather than an " @@ -1318,7 +1415,9 @@ def test_aggregate_variable_excludes_the_quantized_kernels(work_dir: Path) -> No # always enables these kernels, so their absence is a regression rather than a # configuration to tolerate, and skipping would report this as coverage. assert sorted( - (package_dir / "lib").glob("libexecutorch_kernels_quantized.so*") + (package_dir / "lib").glob( + _library_file_name("libexecutorch_kernels_quantized") + "*" + ) ), "the wheel ships no quantized kernels library, so this check cannot run" source_dir = work_dir / "aggregate-only" @@ -1353,9 +1452,7 @@ def test_aggregate_variable_excludes_the_quantized_kernels(work_dir: Path) -> No ) consumer = build_dir / "consumer" - dependencies = subprocess.run( - ["readelf", "-d", str(consumer)], capture_output=True, text=True, check=True - ).stdout + dependencies = _recorded_dependencies(consumer) assert "libexecutorch_kernels_quantized" not in dependencies, ( "an application that linked only ${EXECUTORCH_LIBRARIES} depends on the " "quantized kernels. That library collides with the export-time plugin, so it " diff --git a/.ci/scripts/wheel/test_macos.py b/.ci/scripts/wheel/test_macos.py index c139f828d63..a3c61f51417 100644 --- a/.ci/scripts/wheel/test_macos.py +++ b/.ci/scripts/wheel/test_macos.py @@ -6,12 +6,30 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +import tempfile +from pathlib import Path + import test_base +import test_cpp_sdk +import test_shared_libraries from examples.models import Backend, Model if __name__ == "__main__": test_base.test_cmsis_nn_install() + # The wheel ships the runtime, the kernels, the delegate, the thread pool and + # the profiler as separate libraries here too, so check that each has exactly + # one owner and that all of them are loadable. + with tempfile.TemporaryDirectory() as work_dir: + test_shared_libraries.run_tests(Path(work_dir)) + + # And that a C++ application outside the wheel can actually use them. Nothing + # else covers this: the Python extension links those libraries itself, so it + # passes whether or not the package config names them or the shipped headers + # are complete. + with tempfile.TemporaryDirectory() as work_dir: + test_cpp_sdk.run_tests(Path(work_dir)) + test_base.run_tests( model_tests=[ test_base.ModelTest( diff --git a/CMakeLists.txt b/CMakeLists.txt index d51fcbc25fa..efa61a5fc06 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,17 +193,16 @@ if(DEFINED EXECUTORCH_BAREMETAL_SKIP_INSTALL endif() if(EXECUTORCH_BUILD_SHARED) - # Linux only, and said here rather than left to fail somewhere downstream. The - # shared build names libraries with an ELF soname, records $ORIGIN runtime - # paths, and uses GNU linker options to keep a registration-only library on a - # link line. None of that applies on Apple, which is served by the Swift - # package distribution, or on Windows, where the runtime carries no export - # annotations for a DLL. Enabling it elsewhere failed much later and less - # clearly, when packaging looked for a .so the build never emitted. - if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Said here rather than left to fail somewhere downstream, where packaging + # looked for a library the build never emitted. Windows is still refused: + # there the runtime carries no export annotations, so a DLL would link against + # nothing, which is a missing capability rather than a different spelling of + # one. + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT APPLE) message( - FATAL_ERROR "EXECUTORCH_BUILD_SHARED is supported on Linux only, not " - "${CMAKE_SYSTEM_NAME}." + FATAL_ERROR + "EXECUTORCH_BUILD_SHARED is supported on Linux and macOS only, not " + "${CMAKE_SYSTEM_NAME}." ) endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -1192,8 +1191,17 @@ if(EXECUTORCH_BUILD_PYBIND) # RPATH for _portable_lib.so. It sits in # /executorch/extension/pybindings, so torch is three levels up - # and the wheel's own lib/ directory is two. - set(_portable_lib_rpath "$ORIGIN/../../../torch/lib") + # and the wheel's own lib/ directory is two. Mach-O spells the loader relative + # token differently and takes a list rather than a colon joined string, so + # both differ here while the layout reasoning does not. + if(APPLE) + set(_portable_lib_origin "@loader_path") + set(_portable_lib_rpath_separator ";") + else() + set(_portable_lib_origin "$ORIGIN") + set(_portable_lib_rpath_separator ":") + endif() + set(_portable_lib_rpath "${_portable_lib_origin}/../../../torch/lib") if(EXECUTORCH_BUILD_EXTENSION_MODULE) # extension_module_static is already bundled into libexecutorch.so; linking @@ -1243,12 +1251,20 @@ if(EXECUTORCH_BUILD_PYBIND) endif() if(EXECUTORCH_BUILD_CUDA) - string(APPEND _portable_lib_rpath ":$ORIGIN/../../backends/cuda") + string( + APPEND + _portable_lib_rpath + "${_portable_lib_rpath_separator}${_portable_lib_origin}/../../backends/cuda" + ) endif() if(EXECUTORCH_BUILD_QNN) list(APPEND _dep_libs qnn_executorch_backend) - string(APPEND _portable_lib_rpath ":$ORIGIN/../../backends/qualcomm") + string( + APPEND + _portable_lib_rpath + "${_portable_lib_rpath_separator}${_portable_lib_origin}/../../backends/qualcomm" + ) endif() if(EXECUTORCH_BUILD_ENN) diff --git a/extension/training/CMakeLists.txt b/extension/training/CMakeLists.txt index 04a880a4043..7b76723fb08 100644 --- a/extension/training/CMakeLists.txt +++ b/extension/training/CMakeLists.txt @@ -108,12 +108,17 @@ if(EXECUTORCH_BUILD_PYBIND) endif() executorch_target_link_shared_runtime(_training_lib) - if(EXECUTORCH_BUILD_SHARED AND NOT APPLE) + if(EXECUTORCH_BUILD_SHARED) # This module links Torch directly, and the only other entry reaching it is # the absolute build directory CMake adds, which does not exist anywhere # else, so the Torch path is recorded here rather than left implicit. + if(APPLE) + set(_training_torch_path "@loader_path/../../../../torch/lib") + else() + set(_training_torch_path "$ORIGIN/../../../../torch/lib") + endif() set_target_properties( - _training_lib PROPERTIES INSTALL_RPATH "$ORIGIN/../../../../torch/lib" + _training_lib PROPERTIES INSTALL_RPATH "${_training_torch_path}" ) executorch_target_shared_runtime_path( _training_lib "extension/training/pybindings" diff --git a/kernels/quantized/CMakeLists.txt b/kernels/quantized/CMakeLists.txt index 3722ec7528d..578352d9c76 100644 --- a/kernels/quantized/CMakeLists.txt +++ b/kernels/quantized/CMakeLists.txt @@ -153,8 +153,19 @@ if(NOT CMAKE_GENERATOR STREQUAL "Xcode" # route to the runtime, and overwriting the property here would drop it. get_target_property(_existing quantized_ops_aot_lib INSTALL_RPATH) if(_existing) - set(RPATH "${_existing}:${RPATH}") + # Mach-O keeps one entry per load command, so the two are a CMake list + # there. Joining them with a colon produced a single unusable path + # containing both, and the library then found neither the runtime nor + # the extension. + if(APPLE) + set(RPATH "${_existing};${RPATH}") + else() + set(RPATH "${_existing}:${RPATH}") + endif() endif() + # Quoted, because on Apple this is a list: unquoted it expands into + # separate arguments and the trailing entries are read as further property + # keywords, leaving the search path empty. set_target_properties( quantized_ops_aot_lib PROPERTIES BUILD_RPATH "${RPATH}" INSTALL_RPATH "${RPATH}" diff --git a/setup.py b/setup.py index d7ff2a37ed3..0abfe2c9007 100644 --- a/setup.py +++ b/setup.py @@ -439,7 +439,18 @@ def _sibling_library_search_paths(depth: int = 1) -> List[str]: matching SONAME could satisfy the dependency first. """ up = "/".join([".."] * depth) - return [f"$ORIGIN/{up}/{directory}" for directory in _SIBLING_LIBRARY_DIRECTORIES] + token = _loader_relative_token() + return [f"{token}/{up}/{directory}" for directory in _SIBLING_LIBRARY_DIRECTORIES] + + +def _loader_relative_token() -> str: + """The token a runtime search path uses to mean "the directory this file is in". + + ELF spells it $ORIGIN and Mach-O spells it @loader_path. Both are literal text in the + recorded path, so the wrong one becomes a directory of that name and resolves to + nothing. + """ + return "@loader_path" if sys.platform == "darwin" else "$ORIGIN" def _cuda_runtime_search_paths(depth: int = 1) -> List[str]: @@ -453,7 +464,7 @@ def _cuda_runtime_search_paths(depth: int = 1) -> List[str]: train = _cuda_train() out = "/".join([".."] * (depth + 1)) return [ - f"$ORIGIN/{out}/{directory}" + f"{_loader_relative_token()}/{out}/{directory}" for directory in _CUDA_LIBRARY_DIRECTORIES.get(train, ()) ] @@ -1059,6 +1070,69 @@ def _append_relative_search_paths(entries: List[str], depth: int = 1) -> None: entries.append(search_path) +def _write_runtime_paths( + library: Path, tool: str, original: str, found: List[str], entries: List[str] +) -> None: + """Record the filtered runtime search path back onto the library. + + ELF holds one value that is replaced outright. Mach-O holds one load command per entry + with nothing to overwrite, so the difference is applied as deletes and adds. + """ + if library.suffix != ".dylib": + rewritten = ":".join(entries) + if rewritten == original: + return + subprocess.run( + [tool, "--set-rpath", rewritten, os.fspath(library)], + check=True, + ) + return + install_name_tool = shutil.which("install_name_tool") + if install_name_tool is None: + # The caller checks for this tool before deciding to clean a Mach-O library, so reaching + # here means the environment changed underneath. Leaving the paths in place would ship a + # library naming the build machine, so say so rather than continue quietly. + raise RuntimeError( + "install_name_tool disappeared while cleaning " + os.fspath(library) + ) + for entry in found: + if entry not in entries: + subprocess.run( + [install_name_tool, "-delete_rpath", entry, os.fspath(library)], + capture_output=True, + check=False, + ) + for entry in entries: + if entry not in found: + subprocess.run( + [install_name_tool, "-add_rpath", entry, os.fspath(library)], + capture_output=True, + check=False, + ) + + +def _parse_runtime_paths(original: str, is_mach_o: bool) -> List[str]: + """The runtime search path entries a tool reported, as a list. + + The two formats differ in shape, not just spelling: ELF keeps one colon separated + string, while Mach-O keeps a separate load command per entry, so one cannot be + split the way the other is. + """ + if not is_mach_o: + return original.split(":") + found = [] + lines = original.splitlines() + for index, line in enumerate(lines): + if "LC_RPATH" not in line: + continue + for following in lines[index + 1 : index + 4]: + stripped = following.strip() + if stripped.startswith("path "): + found.append(stripped.split(" (offset", 1)[0][len("path ") :]) + break + return found + + def _is_usable_runtime_path( entry: str, safe_to_drop_toolkit_paths: bool, @@ -1143,13 +1217,22 @@ def _strip_absolute_runtime_paths(library: Path, ships_cuda: bool) -> None: the release tests treats a missing patchelf as a failure rather than a skip: the wheel-build environment has it, and that is where the guarantee belongs. """ - if library.suffix != ".so" and ".so." not in library.name: + is_mach_o = library.suffix == ".dylib" + if not is_mach_o and library.suffix != ".so" and ".so." not in library.name: + return + # Same best effort contract on either platform: a build without the tools still + # produces a working wheel, and the release check is where the guarantee is enforced. + tool = shutil.which("otool") if is_mach_o else shutil.which("patchelf") + if tool is None: return - patchelf = shutil.which("patchelf") - if patchelf is None: + if is_mach_o and shutil.which("install_name_tool") is None: return result = subprocess.run( - [patchelf, "--print-rpath", os.fspath(library)], + ( + [tool, "-l", os.fspath(library)] + if is_mach_o + else [tool, "--print-rpath", os.fspath(library)] + ), capture_output=True, text=True, check=False, @@ -1172,13 +1255,15 @@ def _strip_absolute_runtime_paths(library: Path, ships_cuda: bool) -> None: _cuda_runtime_search_paths(_package_relative_depth(library)) ) + found = _parse_runtime_paths(original, is_mach_o) + # Whether the library can still reach torch without the absolute entry. has_relative_torch_route = any( not entry.startswith("/") and entry.rstrip("/").endswith("/torch/lib") - for entry in original.split(":") + for entry in found ) entries = [ entry - for entry in original.split(":") + for entry in found if _is_usable_runtime_path( entry, safe_to_drop_toolkit_paths, has_relative_torch_route ) @@ -1189,13 +1274,7 @@ def _strip_absolute_runtime_paths(library: Path, ships_cuda: bool) -> None: # which names the build machine and will not exist for a user who installed from an # index. Appended, so a path already present keeps its position. _append_relative_search_paths(entries, _package_relative_depth(library)) - rewritten = ":".join(entries) - if rewritten == original: - return - subprocess.run( - [patchelf, "--set-rpath", rewritten, os.fspath(library)], - check=True, - ) + _write_runtime_paths(library, tool, original, found, entries) class CustomBuildPy(build_py): @@ -1800,8 +1879,8 @@ def run(self): # noqa C901 # what links it, which never happens inside a wheel. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/", - src_name="libexecutorch.so", - dst="executorch/lib/libexecutorch.so", + src_name=get_dynamic_lib_name("executorch"), + dst="executorch/lib/" + get_dynamic_lib_name("executorch"), dependent_cmake_flags=["EXECUTORCH_BUILD_SHARED"], ), # Install the profiler next to it, as its own library rather than @@ -1809,8 +1888,8 @@ def run(self): # noqa C901 # it however many consumers load. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/devtools/etdump/", - src_name="libexecutorch_etdump.so", - dst="executorch/lib/libexecutorch_etdump.so", + src_name=get_dynamic_lib_name("executorch_etdump"), + dst="executorch/lib/" + get_dynamic_lib_name("executorch_etdump"), # Not gated on EXECUTORCH_BUILD_DEVTOOLS. The shared build adds # the devtools subdirectory itself, so the library exists # whenever the shared build does. The Python extension carries a @@ -1823,8 +1902,9 @@ def run(self): # noqa C901 # component that uses it. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/extension/threadpool/", - src_name="libexecutorch_threadpool.so", - dst="executorch/lib/libexecutorch_threadpool.so", + src_name=get_dynamic_lib_name("executorch_threadpool"), + dst="executorch/lib/" + + get_dynamic_lib_name("executorch_threadpool"), # The target only exists when both of its dependencies are # enabled, so packaging has to require them too or a shared # build with either turned off looks for a file that was @@ -1839,8 +1919,9 @@ def run(self): # noqa C901 # registered once per process rather than once per component. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/configurations/", - src_name="libexecutorch_kernels_optimized.so", - dst="executorch/lib/libexecutorch_kernels_optimized.so", + src_name=get_dynamic_lib_name("executorch_kernels_optimized"), + dst="executorch/lib/" + + get_dynamic_lib_name("executorch_kernels_optimized"), # The target is only created when the optimized kernels are # enabled, so packaging has to require that too rather than # looking for a file a shared build may never have produced. @@ -1855,8 +1936,9 @@ def run(self): # noqa C901 # CPU-only build never produced. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/backends/cuda/", - src_name="libexecutorch_backend_cuda.so", - dst="executorch/lib/libexecutorch_backend_cuda.so", + src_name=get_dynamic_lib_name("executorch_backend_cuda"), + dst="executorch/lib/" + + get_dynamic_lib_name("executorch_backend_cuda"), dependent_cmake_flags=[ "EXECUTORCH_BUILD_SHARED", "EXECUTORCH_BUILD_CUDA", @@ -1888,8 +1970,9 @@ def run(self): # noqa C901 # them before. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/kernels/quantized/", - src_name="libexecutorch_kernels_quantized.so", - dst="executorch/lib/libexecutorch_kernels_quantized.so", + src_name=get_dynamic_lib_name("executorch_kernels_quantized"), + dst="executorch/lib/" + + get_dynamic_lib_name("executorch_kernels_quantized"), dependent_cmake_flags=[ "EXECUTORCH_BUILD_SHARED", "EXECUTORCH_BUILD_KERNELS_QUANTIZED", @@ -1911,8 +1994,9 @@ def run(self): # noqa C901 # copy of it instead of one per component that uses it. BuiltFile( src_dir="%CMAKE_CACHE_DIR%/backends/xnnpack/", - src_name="libexecutorch_backend_xnnpack.so", - dst="executorch/lib/libexecutorch_backend_xnnpack.so", + src_name=get_dynamic_lib_name("executorch_backend_xnnpack"), + dst="executorch/lib/" + + get_dynamic_lib_name("executorch_backend_xnnpack"), dependent_cmake_flags=[ "EXECUTORCH_BUILD_SHARED", "EXECUTORCH_BUILD_XNNPACK", diff --git a/tools/cmake/Utils.cmake b/tools/cmake/Utils.cmake index e5c5af38c71..b62faf5c6bd 100644 --- a/tools/cmake/Utils.cmake +++ b/tools/cmake/Utils.cmake @@ -68,12 +68,22 @@ function(executorch_target_whole_archive target_name archive_target) # de-duplication problem above, and because this file's pre-existing # SHELL:LINKER: helpers already break on a path containing a space, which is # the more common case. Both fail loudly at link time rather than producing a - # binary whose registrations are quietly missing. - target_link_options( - ${target_name} - PRIVATE - "LINKER:--push-state,--whole-archive,$,--pop-state" - ) + # binary whose registrations are quietly missing. Mach-O has no bracketing + # pair: -force_load takes the archive directly, so the push and pop that scope + # --whole-archive have no counterpart and must not be emitted. ld rejects them + # outright rather than ignoring them. + if(APPLE) + target_link_options( + ${target_name} PRIVATE + "SHELL:LINKER:-force_load,$" + ) + else() + target_link_options( + ${target_name} + PRIVATE + "LINKER:--push-state,--whole-archive,$,--pop-state" + ) + endif() # Also link it the ordinary way. A link option naming a file is not a build # prerequisite, so on its own it lets the archive be rebuilt while the library # bundling it keeps the previous contents, which is a stale registration @@ -90,16 +100,26 @@ function(executorch_target_link_options_shared_lib target_name) # constructor. Export scoped --no-as-needed retention instead, which is what # actually keeps a registration-only shared library on the link line. get_target_property(_target_type ${target_name} TYPE) - if(_target_type STREQUAL "SHARED_LIBRARY" AND NOT (APPLE OR MSVC)) - target_link_options( - ${target_name} - INTERFACE - # One option with the library inside it, for two reasons. A SHELL: string - # would split on spaces and break a path containing one, and separate - # options repeat identical text that CMake de-duplicates, which silently - # leaves every library after the first outside any --no-as-needed scope. - "LINKER:--push-state,--no-as-needed,$,--pop-state" - ) + # A shared library is never an archive, so the archive handling below does not + # apply to one on any platform. On Apple it actively harms: -force_load on a + # shared library makes every consumer absorb a copy of its contents, which put + # a second operator registry inside the runtime library. + if(_target_type STREQUAL "SHARED_LIBRARY" AND NOT MSVC) + # Mach-O keeps a library named on the link line whether or not anything + # references it, so there is nothing to counter and ld rejects the GNU + # flags. + if(NOT APPLE) + target_link_options( + ${target_name} + INTERFACE + # One option with the library inside it, for two reasons. A SHELL: + # string would split on spaces and break a path containing one, and + # separate options repeat identical text that CMake de-duplicates, which + # silently leaves every library after the first outside any + # --no-as-needed scope. + "LINKER:--push-state,--no-as-needed,$,--pop-state" + ) + endif() # Retention is fully handled above, and applying whole-archive to a shared # library below would do nothing: that flag governs archive member # extraction, and this target is not an archive. @@ -335,12 +355,27 @@ function(executorch_target_retain_shared_library target_name library_target) # Scoped per library for the same reason as whole-archive above: unique option # text, so nothing is de-duplicated out of the retention scope. Without this a # registration-only library is dropped under the default --as-needed and its - # static initializer never runs. - target_link_options( - ${target_name} - PRIVATE - "LINKER:--push-state,--no-as-needed,$,--pop-state" - ) + # static initializer never runs. Mach-O records a library named on the link + # line whether or not anything references it, so there is nothing to counter + # and ld rejects the GNU flags. Named as a link option on Apple too, because + # CMake emits options before every ordered link library. That ordering is what + # makes the shared runtime resolve the registry symbols ahead of a static + # archive that also defines them, so no archive member is extracted and the + # process keeps one registry. Measured on the GNU side: the option sits at + # link slot 9 and the archive at 17, and the archive member is never + # extracted. Mach-O keeps a named library regardless, so the path alone is + # enough there and the --as-needed dance does not apply. + if(APPLE) + target_link_options( + ${target_name} PRIVATE "SHELL:$" + ) + else() + target_link_options( + ${target_name} + PRIVATE + "LINKER:--push-state,--no-as-needed,$,--pop-state" + ) + endif() target_link_libraries(${target_name} PRIVATE ${library_target}) endfunction() @@ -355,8 +390,15 @@ endfunction() # recorded directories are what resolves them there, and packaging strips them # so nothing absolute ships. function(executorch_target_shipped_runtime_path target_name) + # Mach-O spells the same idea @loader_path. + if(APPLE) + set(_origin "@loader_path") + else() + set(_origin "$ORIGIN") + endif() set_target_properties( - ${target_name} PROPERTIES BUILD_RPATH "$ORIGIN" INSTALL_RPATH "$ORIGIN" + ${target_name} PROPERTIES BUILD_RPATH "${_origin}" INSTALL_RPATH + "${_origin}" ) endfunction() @@ -378,12 +420,21 @@ endfunction() function(executorch_target_shared_runtime_path target_name wheel_subdir install_destination ) - if(NOT EXECUTORCH_BUILD_SHARED OR APPLE) + if(NOT EXECUTORCH_BUILD_SHARED) return() endif() + # Mach-O spells the token differently and takes a list rather than a colon + # joined string, so both differ here while the mechanism does not. + if(APPLE) + set(_origin "@loader_path") + set(_separator ";") + else() + set(_origin "$ORIGIN") + set(_separator ":") + endif() # Up out of the subdirectory, then into the package's lib/. string(REGEX REPLACE "[^/]+" ".." _up "${wheel_subdir}") - set(_paths "$ORIGIN/${_up}/lib") + set(_paths "${_origin}/${_up}/lib") # Made absolute lexically, so a destination that is already absolute, as a # ${CMAKE_INSTALL_LIBDIR} based one becomes, is handled the same as a prefix # relative one. @@ -408,10 +459,10 @@ function(executorch_target_shared_runtime_path target_name wheel_subdir file(RELATIVE_PATH _to_libdir "${_installed_dir}" "${CMAKE_INSTALL_FULL_LIBDIR}" ) - string(APPEND _paths ":$ORIGIN/${_to_libdir}") + string(APPEND _paths "${_separator}${_origin}/${_to_libdir}") get_target_property(_existing ${target_name} INSTALL_RPATH) if(_existing) - set(_paths "${_existing}:${_paths}") + set(_paths "${_existing}${_separator}${_paths}") endif() set_target_properties( ${target_name} PROPERTIES BUILD_RPATH "${_paths}" INSTALL_RPATH "${_paths}" @@ -434,6 +485,14 @@ endfunction() # instead is not equivalent, because a wheel is a zip and the format has no # portable symlink support. function(executorch_target_soname_policy target_name) + # Mach-O records the path a library expects to live at, and a consumer copies + # that path verbatim, so a library shipped somewhere other than where it was + # built is unfindable unless the recorded name is relative to whoever loads + # it. Set that before the wheel check, because the wheel is exactly the case + # that relocates. + if(APPLE) + set_target_properties(${target_name} PROPERTIES INSTALL_NAME_DIR "@rpath") + endif() if(EXECUTORCH_BUILD_WHEEL_DO_NOT_USE) return() endif() diff --git a/tools/cmake/executorch-wheel-config.cmake b/tools/cmake/executorch-wheel-config.cmake index e11c58ffac0..998f7578002 100644 --- a/tools/cmake/executorch-wheel-config.cmake +++ b/tools/cmake/executorch-wheel-config.cmake @@ -237,9 +237,18 @@ function(_executorch_find_library _output _base_name) "" PARENT_SCOPE ) - file(GLOB _matches "${_executorch_package_root}/lib/${_base_name}.so" - "${_executorch_package_root}/lib/${_base_name}.so.*" - ) + # Mach-O puts the version before the suffix, libfoo.1.dylib, where ELF puts it + # after, libfoo.so.1, so the versioned pattern differs and not just the + # suffix. + if(APPLE) + file(GLOB _matches "${_executorch_package_root}/lib/${_base_name}.dylib" + "${_executorch_package_root}/lib/${_base_name}.*.dylib" + ) + else() + file(GLOB _matches "${_executorch_package_root}/lib/${_base_name}.so" + "${_executorch_package_root}/lib/${_base_name}.so.*" + ) + endif() list(LENGTH _matches _count) if(_count EQUAL 0) return() @@ -432,6 +441,16 @@ elseif(_executorch_runtime_library) "LINKER:-rpath,$ORIGIN" "LINKER:-rpath,$ORIGIN/../lib" "LINKER:-rpath,${_executorch_package_root}/lib" ) + elseif(APPLE) + # Same purpose, in Mach-O spelling. The token differs, and there is no + # weaker older variant of the load command, so the tag selection flag has + # no counterpart here. + set_property( + TARGET executorch::runtime + APPEND + PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-rpath,@loader_path" + "LINKER:-rpath,@loader_path/../lib" + ) endif() endif() endif() @@ -499,9 +518,11 @@ function(_executorch_define_component _suffix _library_name) PROPERTY INTERFACE_LINK_LIBRARIES executorch::runtime ) endif() - # Guarded on Linux because these are GNU linker options. A wheel only ships - # these components on Linux, so a consumer configured for another system is - # either cross-compiling from the wrong package or has nothing to retain. + # Spelled per platform because the options and the loader relative token + # differ. ELF takes $ORIGIN and needs a flag to choose the newer tag; Mach-O + # takes @loader_path and has no weaker older variant to choose against. A + # consumer configured for any other system is either cross-compiling from the + # wrong package or has nothing to retain. if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set_property( TARGET ${_target} @@ -524,6 +545,16 @@ function(_executorch_define_component _suffix _library_name) # state, so the pop restores whatever the consumer had. "LINKER:--push-state,--no-as-needed,${_library},--pop-state" ) + elseif(APPLE) + # Mach-O spelling of the same two search paths. There is no --no-as-needed + # equivalent to bracket: the linker records a dependency on a dylib it was + # given, so nothing needs to be forced to stay. + set_property( + TARGET ${_target} + APPEND + PROPERTY INTERFACE_LINK_OPTIONS "LINKER:-rpath,@loader_path" + "LINKER:-rpath,@loader_path/../lib" + ) endif() if(NOT _component_OPT_IN) set(EXECUTORCH_LIBRARIES diff --git a/tools/cmake/preset/pybind.cmake b/tools/cmake/preset/pybind.cmake index 068f80d1e2b..33e33f4a916 100644 --- a/tools/cmake/preset/pybind.cmake +++ b/tools/cmake/preset/pybind.cmake @@ -42,6 +42,11 @@ endif() # TODO(larryliu0820): Temporarily disable building llm_runner for Windows wheel # due to the issue of tokenizer file path length limitation. if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # Same reason as on Linux: one shared runtime so a process has a single + # backend registry, and a C++ consumer can link the wheel instead of building + # from source. The Swift package remains the better fit for an application + # bundle. + set_overridable_option(EXECUTORCH_BUILD_SHARED ON) set_overridable_option(EXECUTORCH_BUILD_VGF ${_executorch_pybind_enable_vgf}) set_overridable_option(EXECUTORCH_BUILD_COREML ON) set_overridable_option(EXECUTORCH_BUILD_EXTENSION_TRAINING ON) @@ -105,9 +110,8 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") endif() set_overridable_option(EXECUTORCH_BUILD_OPENVINO OFF) # Ship one shared runtime that both the pybind extension and standalone C++ - # consumers link, so a process has a single backend registry. Linux only: - # macOS C++ consumers are served by the Swift package distribution, and the - # runtime has no export annotations for a Windows DLL. + # consumers link, so a process has a single backend registry. Not set on + # Windows, where the runtime has no export annotations for a DLL. set_overridable_option(EXECUTORCH_BUILD_SHARED ON) elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL "WIN32"