Ship a C++ SDK in the wheel

.ci/scripts/wheel/test_linux.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import test_base
+import test_cpp_sdk
 import test_shared_libraries
 from examples.models import Backend, Model
 
@@ -51,6 +52,13 @@
         with tempfile.TemporaryDirectory() as work_dir:
             test_shared_libraries.run_tests(Path(work_dir))
 
+        # And that a C++ application outside the wheel can actually use them.
+        # Nothing above covers this: the Python extension links those libraries
+        # itself, so it passes whether or not the package config names them or the
+        # shipped headers are complete.
+        with tempfile.TemporaryDirectory() as work_dir:
+            test_cpp_sdk.run_tests(Path(work_dir))
+
     model_tests = [
         test_base.ModelTest(
             model=Model.Mv3,

.ci/scripts/wheel/test_linux_aarch64.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 
 import test_base
+import test_cpp_sdk
 import test_shared_libraries
 from examples.models import Backend, Model
 
@@ -36,6 +37,11 @@
     with tempfile.TemporaryDirectory() as work_dir:
         test_shared_libraries.run_tests(Path(work_dir))
 
+    # And that a C++ application outside the wheel can actually use those
+    # libraries, which nothing above covers.
+    with tempfile.TemporaryDirectory() as work_dir:
+        test_cpp_sdk.run_tests(Path(work_dir))
+
     test_base.run_tests(
         model_tests=[
             test_base.ModelTest(

README-wheel.md

@@ -22,8 +22,8 @@ The prebuilt `executorch.runtime` module included in this package provides a way
 to run ExecuTorch `.pte` files, with some restrictions:
 * Only [core ATen operators](docs/source/ir-ops-set-definition.md) are linked into the prebuilt module
 * Only the [XNNPACK backend delegate](docs/source/backends/xnnpack/xnnpack-overview.md) is linked into the prebuilt module.
-* \[macOS only] [Core ML](docs/source/backends/coreml/coreml-overview.md) and [MPS](docs/source/backends/mps/mps-overview.md) backend
-  are also linked into the prebuilt module.
+* \[macOS only] [Core ML](docs/source/backends/coreml/coreml-overview.md) and MLX backends are
+  also linked into the prebuilt module.
 * \[Linux x86_64] [QNN](docs/source/backends-qualcomm.md) backend is linked into the prebuilt module.
 * \[Linux] [OpenVINO](docs/source/build-run-openvino.md) backend is also linked into the
   prebuilt module. OpenVINO requires the runtime to be installed separately:

docs/source/using-executorch-cpp.md

@@ -40,6 +40,226 @@ Running a model using the low-level runtime APIs allows for a high-degree of con
 
 ## Building with CMake
 
+There are two ways to get the C++ runtime. Linking the prebuilt libraries from the pip
+package needs no source checkout and is the quicker option. Building from source gives
+you every option the project has, and is what you need for a platform the wheel does not
+cover.
+
+### Using the prebuilt libraries from the pip package
+
+On Linux, `pip install executorch` ships the runtime as prebuilt shared libraries together with the
+headers and a CMake package. So a C++ program can use ExecuTorch without building it from source,
+and without knowing much CMake.
+
+#### Run your first model in four steps
+
+Copy these three files into an empty folder and follow along. No prior CMake knowledge needed.
+
+**1. Install, and make a model file.**
+
+```
+pip install executorch torch --extra-index-url https://download.pytorch.org/whl/cpu
+```
+
+`torch` is named explicitly because the wheel does not depend on it, so you bring your own torch and
+keep the version under your control. You need it only to create a model file in step 1, not to run
+the C++ program.
+
+A C++ program loads a `.pte` file, which is a model that has already been exported. C++ cannot
+create one, so make it in Python first:
+
+```python
+# export.py
+import torch
+from executorch.exir import to_edge_transform_and_lower
+
+class Add(torch.nn.Module):
+    def forward(self, x, y):
+        return x + y
+
+example = (torch.ones(2, 2), torch.ones(2, 2))
+program = to_edge_transform_and_lower(
+    torch.export.export(Add(), example)
+).to_executorch()
+open("model.pte", "wb").write(program.buffer)
+```
+
+```
+python export.py
+```
+
+**2. Write the program.**
+
+```cpp
+// main.cpp
+#include <executorch/extension/module/module.h>
+#include <executorch/extension/tensor/tensor.h>
+#include <cstdio>
+
+using namespace executorch::extension;
+
+int main() {
+  Module module("model.pte");
+
+  std::array<float, 4> a{1, 2, 3, 4};
+  std::array<float, 4> b{10, 20, 30, 40};
+
+  const auto result = module.forward({make_tensor_ptr({2, 2}, a.data()),
+                                      make_tensor_ptr({2, 2}, b.data())});
+  if (!result.ok()) {
+    std::printf("forward failed: 0x%x\n", (unsigned)result.error());
+    return 1;
+  }
+
+  const auto out = result->at(0).toTensor();
+  for (int i = 0; i < out.numel(); ++i) {
+    std::printf("%g ", out.const_data_ptr<float>()[i]);
+  }
+  std::printf("\n");
+  return 0;
+}
+```
+
+**3. Write six lines of CMake.**
+
+```cmake
+# CMakeLists.txt
+cmake_minimum_required(VERSION 3.28)
+project(app CXX)
+
+find_package(executorch REQUIRED COMPONENTS kernels_optimized)
+
+add_executable(app main.cpp)
+target_link_libraries(app PRIVATE executorch::runtime
+                                  executorch::kernels_optimized)
+```
+
+Two lines matter. `find_package` finds the installed ExecuTorch, and `target_link_libraries` says
+which parts you want. Every model needs at least these two: `runtime` is the engine that executes a
+program, and a kernel component such as `kernels_optimized` provides the maths the model computes
+with. With only the engine, a model loads and then fails with a missing operator.
+
+**4. Build and run.**
+
+```
+cmake -S . -B build \
+  -DCMAKE_PREFIX_PATH="$(python -c 'import executorch, pathlib; print(pathlib.Path(executorch.__path__[0]) / "share" / "cmake")')"
+cmake --build build
+./build/app
+```
+
+```
+11 22 33 44
+```
+
+That is the two input arrays added together. The long `python -c` part just prints where pip put the
+CMake package, so CMake can find it. Run `./build/app` from the folder holding `model.pte`, because
+the path in `main.cpp` is relative.
+
+#### Adding kernels and backends
+
+Add a component to both lines to get more. Nothing else in the program changes.
+
+```cmake
+find_package(executorch REQUIRED COMPONENTS kernels_optimized backend_xnnpack)
+
+target_link_libraries(app PRIVATE executorch::runtime
+                                  executorch::kernels_optimized
+                                  executorch::backend_xnnpack)
+```
+
+These are the components the Linux package provides:
+
+| Component | What it gives you | Where |
+| --- | --- | --- |
+| `runtime` | The engine. Always needed. | Linux, macOS |
+| `kernels_optimized` | Fast CPU operators. The usual choice. | Linux, macOS |
+| `kernels_quantized` | Operators for quantized models. | Linux, macOS |
+| `backend_xnnpack` | The XNNPACK backend, for models exported with it. | Linux, macOS |
+| `threadpool` | Multi-threaded execution. | Linux, macOS |
+| `etdump` | Profiling, to record what ran and how long it took. | Linux, macOS |
+
+To see what your own install offers, ask CMake:
+
+```cmake
+find_package(executorch REQUIRED)
+foreach(_component runtime kernels_optimized kernels_quantized backend_xnnpack
+                   threadpool etdump)
+  if(TARGET executorch::${_component})
+    message(STATUS "have ${_component}")
+  endif()
+endforeach()
+```
+
+On macOS the Core ML and MLX
+delegates are registered inside the Python extension rather than shipped as separate C++ libraries,
+so a C++ application there cannot link them as components; use them from Python, or build from
+source if you need them in C++.
+
+A backend is only needed if the model was exported for it. Linking XNNPACK does not make a plain
+model faster, and a model exported for XNNPACK will fail to load without it. If you are not sure
+what a model needs, start with `runtime` and `kernels_optimized` and add what the error asks for.
+
+If you would rather not choose, one variable links the common set:
+
+```cmake
+find_package(executorch REQUIRED)
+target_link_libraries(app PRIVATE ${EXECUTORCH_LIBRARIES})
+```
+
+The quantized kernels are deliberately left out of that variable, because loading
+`executorch.kernels.quantized` in Python registers the same operators and a duplicate registration
+stops the runtime. Name `executorch::kernels_quantized` when you want them.
+
+#### When something does not work
+
+- `find_package` could not find executorch: the `-DCMAKE_PREFIX_PATH=...` argument is missing or
+  points somewhere else. Run the `python -c` line on its own and check the folder exists.
+- The program builds but fails to load the model: the path is relative, so run it from the folder
+  containing the `.pte` file.
+- A missing operator at run time: add a kernel component, usually
+  `executorch::kernels_optimized`.
+- The model fails to load complaining about a backend: link the backend it was exported for.
+- `executorch::runtime` is not a target: imported targets need CMake 3.28 or newer. On an older
+  CMake the package still works, but you name variables instead of targets, and you have to pass on
+  the definitions and the C++20 requirement yourself:
+
+  ```cmake
+  cmake_minimum_required(VERSION 3.19)
+  project(app CXX)
+
+  find_package(executorch REQUIRED)
+
+  add_executable(app main.cpp)
+  target_include_directories(app PRIVATE ${EXECUTORCH_INCLUDE_DIRS})
+  target_compile_definitions(app PRIVATE ${EXECUTORCH_COMPILE_DEFINITIONS})
+  target_compile_features(app PRIVATE cxx_std_${EXECUTORCH_CXX_STANDARD})
+  target_link_libraries(app PRIVATE ${EXECUTORCH_LIBRARIES})
+  set_target_properties(
+    app PROPERTIES INSTALL_RPATH "${EXECUTORCH_RUNTIME_LIBRARY_DIR}"
+  )
+  ```
+
+  Leaving out `EXECUTORCH_COMPILE_DEFINITIONS` fails with a missing
+  `torch/headeronly/macros/cmake_macros.h`, because the vendored headers look for a file that only
+  exists inside a PyTorch build.
+
+  `INSTALL_RPATH` matters once you run `cmake --install`. CMake gives your program a search path while
+  it sits in the build directory and removes that path when installing, so an installed program cannot
+  find the libraries unless you record where they live.
+
+  Quantized kernels are not part of `EXECUTORCH_LIBRARIES`, so add them when your model needs them:
+
+  ```cmake
+  target_link_libraries(app PRIVATE ${EXECUTORCH_QUANTIZED_KERNELS_LIBRARY})
+  ```
+
+You should not need `LD_LIBRARY_PATH`. The shipped libraries record where their neighbours live, so
+they find each other once the program links against the installed package.
+
+### Building from source
+
+
 ExecuTorch uses CMake as the primary build system. Inclusion of the module and tensor APIs are controlled by the `EXECUTORCH_BUILD_EXTENSION_MODULE` and `EXECUTORCH_BUILD_EXTENSION_TENSOR` CMake options. As these APIs may not be supported on embedded systems, they are disabled by default when building from source. The low-level API surface is always included. To link, add the `executorch` target as a CMake dependency, along with `executorch_backends`, `executorch_extensions`, and `extension_kernels`, to link all configured backends, extensions, and kernels.
 
 ```

extension/memory_allocator/memory_allocator_utils.h

@@ -16,12 +16,10 @@
 #include <executorch/runtime/core/result.h>
 #include <executorch/runtime/platform/compiler.h>
 
-using executorch::runtime::Error;
-using executorch::runtime::Result;
 namespace executorch::extension::utils {
 
 // Util to get alighment adjusted allocation size
-inline Result<size_t> get_aligned_size(size_t size, size_t alignment) {
+inline runtime::Result<size_t> get_aligned_size(size_t size, size_t alignment) {
   // The minimum alignment that malloc() is guaranteed to provide.
   static constexpr size_t kMallocAlignment = alignof(std::max_align_t);
   if (alignment > kMallocAlignment) {
@@ -31,7 +29,7 @@ inline Result<size_t> get_aligned_size(size_t size, size_t alignment) {
     const size_t extra = alignment - 1;
     if ET_UNLIKELY (extra >= SIZE_MAX - size) {
       ET_LOG(Error, "Malloc size overflow: size=%zu + extra=%zu", size, extra);
-      return Result<size_t>(Error::InvalidArgument);
+      return runtime::Result<size_t>(runtime::Error::InvalidArgument);
     }
     size += extra;
   }

runtime/executor/platform_memory_allocator.h

@@ -13,6 +13,7 @@
 #include <cstdint>
 
 #include <c10/util/safe_numerics.h>
+#include <executorch/runtime/core/exec_aten/exec_aten.h>
 #include <executorch/runtime/core/memory_allocator.h>
 #include <executorch/runtime/platform/log.h>
 #include <executorch/runtime/platform/platform.h>