diff --git a/.github/workflows/platforms.yml b/.github/workflows/platforms.yml new file mode 100644 index 0000000..cabc778 --- /dev/null +++ b/.github/workflows/platforms.yml @@ -0,0 +1,178 @@ +name: platforms + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + CMAKE_BUILD_PARALLEL_LEVEL: "2" + CMAKE_GENERATOR: Ninja + NATIVE_SHA: 34aedfbe8ed9779697905541a62e2160ccfd9c05 + +jobs: + msrv: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Check out exact sources + uses: actions/checkout@v4.4.0 + with: + submodules: recursive + - name: Verify checkout and native pin + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(git rev-parse HEAD:vllm-cpp-sys/vllm.cpp)" = "$NATIVE_SHA" + test "$(git -C vllm-cpp-sys/vllm.cpp rev-parse HEAD)" = "$NATIVE_SHA" + test -z "$(git -C vllm-cpp-sys/vllm.cpp status --porcelain)" + - name: Install exact Rust toolchain + uses: dtolnay/rust-toolchain@1.85.0 + - name: Install native dependencies + run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends build-essential cmake ninja-build + - name: Install Just + uses: extractions/setup-just@v3.1.0 + with: + just-version: "1.40.0" + - name: Check exact MSRV + env: + RUSTUP_TOOLCHAIN: 1.85.0 + run: just msrv + + linux-arm64-cpu: + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + steps: + - name: Check out exact sources + uses: actions/checkout@v4.4.0 + with: + submodules: recursive + - name: Verify checkout, native pin, and host + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = aarch64 + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(git rev-parse HEAD:vllm-cpp-sys/vllm.cpp)" = "$NATIVE_SHA" + test "$(git -C vllm-cpp-sys/vllm.cpp rev-parse HEAD)" = "$NATIVE_SHA" + test -z "$(git -C vllm-cpp-sys/vllm.cpp status --porcelain)" + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Install native dependencies + run: sudo apt-get update && sudo apt-get install --yes --no-install-recommends build-essential cmake ninja-build + - name: Test model-free ARM64 CPU targets + run: env -u VLLM_CPP_TEST_MODEL cargo test --locked --workspace --all-targets --features vllm-cpp/serde + + macos-arm64-cpu: + runs-on: macos-15 + timeout-minutes: 60 + steps: + - name: Check out exact sources + uses: actions/checkout@v4.4.0 + with: + submodules: recursive + - name: Verify checkout, native pin, and host + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = arm64 + test "$(sw_vers -productVersion | cut -d. -f1)" -ge 15 + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(git rev-parse HEAD:vllm-cpp-sys/vllm.cpp)" = "$NATIVE_SHA" + test "$(git -C vllm-cpp-sys/vllm.cpp rev-parse HEAD)" = "$NATIVE_SHA" + test -z "$(git -C vllm-cpp-sys/vllm.cpp status --porcelain)" + command -v cmake + command -v ninja + xcrun --find clang++ + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Test model-free Apple ARM64 CPU targets + run: env -u VLLM_CPP_TEST_MODEL cargo test --locked --workspace --all-targets --features vllm-cpp/serde + + macos-arm64-metal-build: + runs-on: macos-15 + timeout-minutes: 60 + steps: + - name: Check out exact sources + uses: actions/checkout@v4.4.0 + with: + submodules: recursive + - name: Verify checkout, native pin, and Metal toolchain + shell: bash + run: | + set -euo pipefail + test "$(uname -m)" = arm64 + test "$(sw_vers -productVersion | cut -d. -f1)" -ge 15 + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(git rev-parse HEAD:vllm-cpp-sys/vllm.cpp)" = "$NATIVE_SHA" + test "$(git -C vllm-cpp-sys/vllm.cpp rev-parse HEAD)" = "$NATIVE_SHA" + test -z "$(git -C vllm-cpp-sys/vllm.cpp status --porcelain)" + command -v cmake + command -v ninja + xcrun --find clang++ + sdk=$(xcrun --sdk macosx --show-sdk-path) + test -d "$sdk/System/Library/Frameworks/Metal.framework" + test -d "$sdk/System/Library/Frameworks/Foundation.framework" + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + - name: Compile and link Metal Rust targets + run: cargo test --locked -p vllm-cpp-sys --release --tests --features metal --no-run + + vulkan-llvmpipe: + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - name: Check out exact sources + uses: actions/checkout@v4.4.0 + with: + submodules: recursive + - name: Verify checkout and native pin + shell: bash + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(git rev-parse HEAD:vllm-cpp-sys/vllm.cpp)" = "$NATIVE_SHA" + test "$(git -C vllm-cpp-sys/vllm.cpp rev-parse HEAD)" = "$NATIVE_SHA" + test -z "$(git -C vllm-cpp-sys/vllm.cpp status --porcelain)" + - name: Install Rust and Vulkan dependencies + uses: dtolnay/rust-toolchain@stable + - name: Install Mesa llvmpipe + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install --yes --no-install-recommends build-essential cmake mesa-vulkan-drivers ninja-build python3 vulkan-tools + mapfile -t icds < <(dpkg -L mesa-vulkan-drivers | grep -E '/lvp_icd[^/]*\.json$') + test "${#icds[@]}" -eq 1 + echo "VK_DRIVER_FILES=${icds[0]}" >> "$GITHUB_ENV" + echo "VK_ICD_FILENAMES=${icds[0]}" >> "$GITHUB_ENV" + - name: Require llvmpipe Vulkan 1.1 and 16-bit storage + shell: bash + run: | + set -euo pipefail + vulkaninfo --summary | tee vulkan-summary.txt + grep -Eiq 'deviceName.*llvmpipe' vulkan-summary.txt + grep -Eq 'apiVersion.*1\.[1-9]' vulkan-summary.txt + vulkaninfo | tee vulkan-info.txt + grep -Eq 'storageBuffer16BitAccess[[:space:]]*=[[:space:]]*true' vulkan-info.txt + - name: Compile and link the Rust Vulkan surface + run: cargo test --locked -p vllm-cpp-sys --release --tests --features vulkan --no-run + - name: Build and run native Vulkan backend and op gates + shell: bash + run: | + set -euo pipefail + cmake -S vllm-cpp-sys/vllm.cpp -B target/vulkan-native \ + -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DVLLM_CPP_BUILD_TESTS=ON -DVLLM_CPP_BUILD_EXAMPLES=OFF \ + -DVLLM_CPP_SERVER=OFF -DVLLM_CPP_CUDA=OFF \ + -DVLLM_CPP_METAL=OFF -DVLLM_CPP_MLX=OFF \ + -DVLLM_CPP_VULKAN=ON -DVLLM_CPP_TRITON=OFF + cmake --build target/vulkan-native --target test_vulkan_backend test_backend_cross_device + ctest --test-dir target/vulkan-native --output-on-failure \ + --tests-regex '^(test_vulkan_backend|test_backend_cross_device)$' diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac77f2..40651c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ All notable changes to this project will be documented in this file. - RAII ownership for native engines, requests, completions, and strings, including callback panic containment and callback-thread-safe deferred request cleanup. - Linux x86_64 CPU builds for bundled and system libraries with static or dynamic linking. - Experimental bundled Linux x86_64/aarch64 build integration for CUDA, external CUTLASS, Triton AOT, and Vulkan. +- Bundled Apple ARM64 CPU and Metal build/link integration, plus optional external MLX integration with deterministic target/root/file validation and no packaged MLX payload or rpath. +- Manual hosted exact Rust 1.85.0, Linux ARM64 CPU, Apple ARM64 CPU/Metal compile-link, and Mesa llvmpipe Vulkan lanes. ### Compatibility @@ -20,5 +22,6 @@ All notable changes to this project will be documented in this file. ### Known limitations - The supported runtime tier is native Linux x86_64 CPU. Accelerator features are experimental build/configuration surfaces, not runtime-support claims. -- Known native blockers include a CUDA teardown SIGSEGV after otherwise successful tests, a CUDA bf16 numerical tolerance failure, CUTLASS concurrent-output differences, and incomplete Vulkan runtime coverage. -- Dynamic builds require callers to deploy `libvllm.so` and its runtime dependencies through a loader-visible path. System static builds must also provide the matching private BLAKE3 archive. +- Known native blockers include a CUDA teardown SIGSEGV after otherwise successful tests, a CUDA bf16 numerical tolerance failure, CUTLASS concurrent-output differences, incomplete Vulkan attention/model runtime, and external MLX deployment plus unvalidated release-lane model/runtime behavior. +- The hosted Metal lane checks compile/link only, the software Vulkan lane checks backend/ops only, and accelerator builds do not establish runtime correctness. +- Dynamic builds require callers to deploy `libvllm.so` or `libvllm.dylib` and its runtime dependencies through a loader-visible path. System static builds must also provide the matching private BLAKE3 archive. diff --git a/README.md b/README.md index 747673e..1592fe7 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,11 @@ Rust bindings for [vllm.cpp](https://github.com/mudler/vllm.cpp), organized as: The safe crate provides a cloneable engine API for model loading, blocking completion and streaming, non-blocking concurrent requests, structured output, and raw-JSON chat. An optional `serde` feature adds `serde_json::Value` chat helpers. The sys crate provides checked-in generated FFI declarations with C/Rust layout checks and coverage for all 19 exported C symbols. -Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. Experimental bundled builds also expose Linux x86_64/aarch64 build configuration for CUDA, external CUTLASS, Triton AOT, and Vulkan. These accelerator features are build-only integration surfaces, not runtime-support claims. vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. +Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. Bundled CPU builds also target Linux aarch64 and Apple ARM64. Experimental bundled builds expose Linux x86_64/aarch64 build configuration for CUDA, external CUTLASS, Triton AOT, and Vulkan, plus Apple ARM64 Metal and external MLX configuration. Accelerator features are build integration surfaces, not runtime-support claims. vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. ## Prerequisites -Initial development and testing support Linux CPU builds. They require: +Native builds require: - Rust and Cargo. - CMake 3.24 or newer. @@ -62,18 +62,20 @@ cargo test --locked -p vllm-cpp --release --features serde just ci ``` -Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The default bundled build remains deterministic and CPU-only: native tests, examples, the HTTP server, CUDA, Metal, MLX, Vulkan, Triton, and CUTLASS fetching are disabled explicitly. Use `nix develop .#msrv -c just msrv` for the exact local Rust 1.85.0 policy check; hosted exact-MSRV validation is deferred to a later CI slice. +Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The default bundled build remains deterministic and CPU-only: native tests, examples, the HTTP server, CUDA, Metal, MLX, Vulkan, Triton, and CUTLASS fetching are disabled explicitly. Use `nix develop .#msrv -c just msrv` for the exact local Rust 1.85.0 policy check; the manual `platforms` workflow runs the same exact toolchain policy. `build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. ## Experimental Backend Builds -Backend features apply to bundled Linux x86_64/aarch64 builds only and are mutually exclusive with `system`; CUDA and Vulkan are also mutually exclusive. Backend features do not enable `bundled`: normal default-feature commands may use `--features cuda`, while `--no-default-features` callers must include it explicitly, for example `--features bundled,cuda`. Use a fresh `CARGO_TARGET_DIR` for every backend and link mode. +Backend features are bundled-only and mutually exclusive with `system`; CUDA and Vulkan are also mutually exclusive. CUDA/CUTLASS/Triton/Vulkan target Linux x86_64/aarch64, while Metal/MLX require exact `aarch64-apple-darwin`. Backend features do not enable `bundled`: normal default-feature commands may use `--features cuda`, while `--no-default-features` callers must include it explicitly, for example `--features bundled,cuda`. Use a fresh `CARGO_TARGET_DIR` for every backend and link mode. - `cuda` requires `VLLM_CPP_CUDA_ARCHITECTURES` equal to `80`, `86`, `87`, `89`, `90a`, `100a`, `103a`, `110`, `120a`, `121a`, or `120a;121a`. Leave this variable unset when `cuda` is disabled, including CPU and system builds. - `cuda-cutlass` implies `cuda`, requires an explicit canonical `VLLM_CPP_CUTLASS_DIR` containing CUTLASS >=4.5.0, disables fetching, and rejects `103a` and `110`. Plain CUDA uses a nonexistent sentinel CUTLASS root so an ambient checkout cannot alter the build. - `triton-aot` implies `cuda`, enables only checked-in AOT artifacts for one of `80`, `86`, `89`, `90a`, `100a`, or `121a`, and forces regeneration off. - `vulkan` uses packaged Khronos headers and checked-in SPIR-V. It does not link a Vulkan SDK library; the native library opens the runtime loader dynamically. +- `metal` enables the native Metal backend on Apple ARM64 and links Apple's `Metal` and `Foundation` frameworks. Its MSL is compiled at runtime. +- `mlx` implies `metal` and requires canonical `MLX_ROOT` containing `include/mlx/array.h`, `lib/libmlx.dylib`, and `lib/mlx.metallib`. MLX remains an external dependency: Cargo neither fetches nor packages it and emits no machine-local rpath. For example: @@ -88,11 +90,16 @@ VLLM_CPP_CUDA_ARCHITECTURES=120a \ nix develop .#vulkan CARGO_TARGET_DIR=target/vulkan-static cargo build --locked --release --features vulkan + +# Apple ARM64 only +CARGO_TARGET_DIR=target/metal-static cargo build --locked --release --features metal +MLX_ROOT=/absolute/path/to/mlx CARGO_TARGET_DIR=target/mlx-static \ + cargo build --locked --release --features mlx ``` -Static CUDA links the exact `cudart`, `cublasLt`, and, for Triton, CUDA driver locations selected by CMake. Dynamic builds rely on `libvllm.so` `DT_NEEDED` entries instead of repeating those transitive Cargo links; deploy the shared library and toolkit libraries through normal loader paths. +Static CUDA links the exact `cudart`, `cublasLt`, and, for Triton, CUDA driver locations selected by CMake. Static Apple builds link `libc++`; Metal adds the `Metal` and `Foundation` frameworks, while MLX adds its canonical `lib` search path before `dylib=mlx`. Dynamic builds rely on the shared native library's transitive dependencies instead of repeating them through Cargo. Deploy `libvllm.so`/`libvllm.dylib` and optional toolkit/MLX libraries through normal loader paths. -Compilation does not establish runtime correctness. Known native evidence blockers remain: CUDA teardown can SIGSEGV after otherwise successful tests; CUDA bf16 testing has a numerical tolerance failure; CUTLASS concurrent output differs from the non-concurrent path; Vulkan runtime coverage is incomplete. No runtime support is claimed here. +Compilation does not establish runtime correctness. Known native evidence blockers remain: CUDA teardown can SIGSEGV after otherwise successful tests; CUDA bf16 testing has a numerical tolerance failure; CUTLASS concurrent output differs from the non-concurrent path; Vulkan attention/model runtime is incomplete; and MLX is an external, numerically distinct provider without release-lane model evidence. No accelerator runtime support is claimed here. ## Test Model and Sanitizers @@ -138,9 +145,13 @@ The package gate validates deterministic inventories for both crates, package me `just publish-dry-run` performs a sys-then-safe workspace packaging dry-run without uploading; it uses `--no-verify` to avoid the pre-publication registry cycle. As required by [RELEASING.md](RELEASING.md), after `vllm-cpp-sys` is available from crates.io, run the full `cargo publish -p vllm-cpp --locked --dry-run` verification before publishing the safe crate. +## Platform and Backend Validation + +The manual `platforms` workflow provides exact Rust 1.85.0, Linux ARM64 CPU, Apple ARM64 CPU, Apple ARM64 Metal compile/link, and Mesa llvmpipe Vulkan jobs without duplicating ordinary Linux x86_64 CPU CI. The Vulkan job requires a real llvmpipe device and `storageBuffer16BitAccess`, then runs native backend/op gates; its scope is backend/op checking, not attention or model-inference support. The hosted Metal job checks compile/link only, not runtime correctness. + ## Support -The supported runtime target is native Linux x86_64 CPU. Maintainer tests cover the four bundled/system static/dynamic CPU link modes plus bundled blocking and concurrent request inference with the pinned Qwen fixture. Sanitizer evidence covers native ASan/UBSan/leak detection and selected native-only GCC TSan lifecycle paths as described above. Linux CUDA/CUTLASS/Triton/Vulkan features remain experimental build-only surfaces with the limitations listed above; Apple and other accelerator targets are out of scope. +The supported runtime target is native Linux x86_64 CPU. Maintainer tests cover the four bundled/system static/dynamic CPU link modes plus bundled blocking and concurrent request inference with the pinned Qwen fixture. Sanitizer evidence covers native ASan/UBSan/leak detection and selected native-only GCC TSan lifecycle paths as described above. The manual Linux ARM64 and Apple ARM64 CPU jobs are configured for model-free build/test coverage. CUDA/CUTLASS/Triton/Vulkan/Metal/MLX remain experimental surfaces with the evidence boundaries and limitations listed above; CPU is the only supported runtime family. ## Licensing and Affiliation diff --git a/vllm-cpp-sys/Cargo.toml b/vllm-cpp-sys/Cargo.toml index f1bc77c..6b639d2 100644 --- a/vllm-cpp-sys/Cargo.toml +++ b/vllm-cpp-sys/Cargo.toml @@ -48,6 +48,8 @@ cuda = [] cuda-cutlass = ["cuda"] triton-aot = ["cuda"] vulkan = [] +metal = [] +mlx = ["metal"] [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/vllm-cpp-sys/README.md b/vllm-cpp-sys/README.md index a6bfe57..cbc3649 100644 --- a/vllm-cpp-sys/README.md +++ b/vllm-cpp-sys/README.md @@ -12,13 +12,13 @@ The package contains Rust declarations and conformance tests, native build/link - `dynamic-link` makes either source mode link `libvllm` dynamically. - `system` disables the bundled build and links a caller-provided installation. Disable default features when selecting it. -`bundled` and `system` are mutually exclusive. CPU runtime support is limited to native Linux x86_64; Linux x86_64/aarch64 accelerator features below are experimental build-only configuration. +`bundled` and `system` are mutually exclusive. CPU runtime support is limited to native Linux x86_64; Linux aarch64 and Apple ARM64 have model-free build/test surfaces. Accelerator features below are experimental build configuration. System mode requires `VLLM_CPP_ROOT`, whose prefix must contain `include/vllm.h` plus a `lib` or `lib64` directory. `VLLM_CPP_LIB_DIR` can override the vllm library directory. Consumer builds validate that the selected system header exists but do not compare its layout. The maintainer integration test compiles its C probe at test runtime against that header, compares its C layouts with the generated Rust declarations, and the runtime test requires ABI version 10. Upstream's normal CMake install provides `libvllm` and `vllm.h` but does not install the private `libblake3_vendored.a` target. A stock install therefore works directly with `system,dynamic-link`. System static mode requires callers to provision the matching `libblake3_vendored.a` separately and set `VLLM_CPP_BLAKE3_LIB_DIR`; when the variable is unset, the build script checks the selected vllm library directory for backward compatibility. It does not search arbitrary build trees. -`dynamic-link` does not copy or package the shared library. Tests and applications must install `libvllm` in a loader-visible location or configure `LD_LIBRARY_PATH`, rpath, or another loader search path. +`dynamic-link` does not copy or package the shared library. Tests and applications must install `libvllm` in a loader-visible location or configure `LD_LIBRARY_PATH`, `DYLD_LIBRARY_PATH`, an application-owned rpath, or another loader search path. This crate emits no machine-local rpath. ## Backend Features @@ -30,10 +30,14 @@ Feature selection is build configuration, not runtime or hardware evidence. Acce | `cuda-cutlass` | Implies CUDA; requires canonical `VLLM_CPP_CUTLASS_DIR` with CUTLASS >=4.5.0; fetch is OFF; `103a`/`110` rejected | Header-only external input | | `triton-aot` | Implies CUDA; checked-in single-architecture artifacts only; regeneration is OFF | Static mode also links CMake's exact `CUDA_cuda_driver_LIBRARY`; dynamic mode relies on `libvllm.so` | | `vulkan` | Linux x86_64/aarch64; packaged Khronos headers and checked-in SPIR-V | No Vulkan SDK link; runtime loader uses `dlopen` | +| `metal` | Exact `aarch64-apple-darwin`; native MSL backend | Static mode links `c++`, then `Metal` and `Foundation` frameworks | +| `mlx` | Implies Metal; requires canonical external `MLX_ROOT` with `include/mlx/array.h`, `lib/libmlx.dylib`, and `lib/mlx.metallib` | Static mode adds `$MLX_ROOT/lib` before `dylib=mlx`; no rpath or packaged MLX payload | CUDA architectures are exactly `80`, `86`, `87`, `89`, `90a`, `100a`, `103a`, `110`, `120a`, `121a`, or `120a;121a`. Triton accepts only `80`, `86`, `89`, `90a`, `100a`, or `121a`. Plain CUDA passes a nonexistent CUTLASS root to CMake so ambient source cannot silently change the build. Use separate `CARGO_TARGET_DIR` values for each backend/link combination. -These features do not claim runtime support. Known native blockers remain: a CUDA teardown SIGSEGV after otherwise successful tests, a CUDA bf16 numerical tolerance failure, CUTLASS concurrent output differences, and incomplete Vulkan runtime coverage. Metal and MLX are not exposed by this crate. +Metal and MLX remain bundled-only and do not imply `bundled`; `--no-default-features` callers must explicitly select `bundled,metal` or `bundled,mlx`. Setting `MLX_ROOT` without `mlx` is rejected. MLX is never downloaded, copied, or packaged, and applications are responsible for making `libmlx.dylib` and its metallib deployment visible at runtime. + +These features do not claim runtime support. Known native blockers remain: a CUDA teardown SIGSEGV after otherwise successful tests, a CUDA bf16 numerical tolerance failure, CUTLASS concurrent output differences, incomplete Vulkan attention/model runtime, and external MLX provider behavior without release-lane model/runtime validation. The hosted Metal lane checks compile/link only; the software Vulkan lane checks backend/ops only. CPU is the only supported runtime family. ## Generated Bindings @@ -45,7 +49,7 @@ just sys just link-modes ``` -The conformance gate verifies the generated output, C and C++ header compatibility, C/Rust layout, the exact 19-symbol export set, pure backend plans/cache parsing, pinned CUDA architecture mappings, Triton AOT drift, and compile-time/runtime ABI 10. CI separately runs bundled static/dynamic and fixture-backed system static/dynamic CPU tests; dynamic tests set the required loader path. `build.rs` only performs consumer native build/link integration and does not compile or execute the layout probe. `tests/layout.rs` compiles and executes `tests/layout.c` with the bundled header or `VLLM_CPP_ROOT/include/vllm.h` at test runtime using the Rust standard library. Native Linux CPU is the supported runtime target; cross-compiling that integration test is unsupported. +The conformance gate verifies the generated output, C and C++ header compatibility, C/Rust layout, the exact 19-symbol export set, pure backend plans/cache parsing, Apple/MLX validation and link ordering, pinned CUDA architecture mappings, Triton AOT drift, and compile-time/runtime ABI 10. CI separately runs bundled static/dynamic and fixture-backed system static/dynamic CPU tests; dynamic tests set the required loader path. `build.rs` only performs consumer native build/link integration and does not compile or execute the layout probe. `tests/layout.rs` compiles and executes `tests/layout.c` with the bundled header or `VLLM_CPP_ROOT/include/vllm.h` at test runtime using the Rust standard library. Native Linux CPU is the supported runtime target; cross-compiling that integration test is unsupported. The Rust crate is dual-licensed under MIT or Apache-2.0. The bundled vllm.cpp source retains its upstream Apache-2.0 license and notices. diff --git a/vllm-cpp-sys/build.rs b/vllm-cpp-sys/build.rs index fe08bcb..fa2cf13 100644 --- a/vllm-cpp-sys/build.rs +++ b/vllm-cpp-sys/build.rs @@ -25,6 +25,7 @@ const RERUN_ENV: &[&str] = &[ "VLLM_CPP_BLAKE3_LIB_DIR", "VLLM_CPP_CUDA_ARCHITECTURES", "VLLM_CPP_CUTLASS_DIR", + "MLX_ROOT", "VLLM_CPP_SANITIZE", "CUDA_PATH", "CUDA_HOME", @@ -85,6 +86,8 @@ fn build_inputs() -> Inputs { cuda_cutlass: cfg!(feature = "cuda-cutlass"), triton_aot: cfg!(feature = "triton-aot"), vulkan: cfg!(feature = "vulkan"), + metal: cfg!(feature = "metal"), + mlx: cfg!(feature = "mlx"), }, target: Target { triple: required_env("TARGET"), @@ -94,6 +97,7 @@ fn build_inputs() -> Inputs { environment: Environment { cuda_architectures: env::var("VLLM_CPP_CUDA_ARCHITECTURES").ok(), cutlass_dir: env::var_os("VLLM_CPP_CUTLASS_DIR").map(PathBuf::from), + mlx_root: env::var_os("MLX_ROOT").map(PathBuf::from), sanitizer: env::var("VLLM_CPP_SANITIZE").ok(), }, } @@ -122,7 +126,8 @@ fn build_bundled(plan: &BuildPlan) -> PathBuf { let dynamic_link = cfg!(feature = "dynamic-link"); let vllm_artifact = if dynamic_link { - shared_library_name("vllm") + shared_library_name("vllm", &required_env("CARGO_CFG_TARGET_OS")) + .unwrap_or_else(|error| config_error(error)) } else { static_library_name("vllm") }; @@ -198,7 +203,8 @@ fn validate_system_root() -> PathBuf { fn link_system(root: &Path) { let dynamic_link = cfg!(feature = "dynamic-link"); let vllm_artifact = if dynamic_link { - shared_library_name("vllm") + shared_library_name("vllm", &required_env("CARGO_CFG_TARGET_OS")) + .unwrap_or_else(|error| config_error(error)) } else { static_library_name("vllm") }; @@ -275,6 +281,12 @@ fn emit_link_requirements(plan: &BuildPlan, cmake_cache: Option<&Path>) { LinkRequirement::Library(name) => { println!("cargo:rustc-link-lib=dylib={name}"); } + LinkRequirement::Framework(name) => { + println!("cargo:rustc-link-lib=framework={name}"); + } + LinkRequirement::NativeSearch(path) => { + println!("cargo:rustc-link-search=native={}", path.display()); + } LinkRequirement::CudaToolkit(component) => { let cache = cmake_cache.unwrap_or_else(|| { config_error( diff --git a/vllm-cpp-sys/src/build_config.rs b/vllm-cpp-sys/src/build_config.rs index 75cc19a..0ee338f 100644 --- a/vllm-cpp-sys/src/build_config.rs +++ b/vllm-cpp-sys/src/build_config.rs @@ -28,11 +28,13 @@ pub struct Features { pub cuda_cutlass: bool, pub triton_aot: bool, pub vulkan: bool, + pub metal: bool, + pub mlx: bool, } impl Features { fn has_backend(&self) -> bool { - self.cuda || self.cuda_cutlass || self.triton_aot || self.vulkan + self.cuda || self.cuda_cutlass || self.triton_aot || self.vulkan || self.metal || self.mlx } } @@ -47,6 +49,7 @@ pub struct Target { pub struct Environment { pub cuda_architectures: Option, pub cutlass_dir: Option, + pub mlx_root: Option, pub sanitizer: Option, } @@ -60,6 +63,8 @@ pub struct Inputs { #[derive(Clone, Debug, Eq, PartialEq)] pub enum LinkRequirement { Library(&'static str), + Framework(&'static str), + NativeSearch(PathBuf), CudaToolkit(CudaComponent), } @@ -153,6 +158,7 @@ pub fn plan(inputs: &Inputs, probe: &impl PathProbe) -> Result Result Result Result Result<(), ConfigError> if features.triton_aot && !features.cuda { return Err(ConfigError::new("`triton-aot` requires the `cuda` feature")); } + if features.mlx && !features.metal { + return Err(ConfigError::new("`mlx` requires the `metal` feature")); + } if features.cuda && features.vulkan { return Err(ConfigError::new( "`cuda` and `vulkan` cannot be combined in release 0.1; build separate backend artifacts", @@ -232,15 +250,19 @@ fn validate_feature_implications(features: &Features) -> Result<(), ConfigError> } fn validate_targets(inputs: &Inputs) -> Result<(), ConfigError> { - if inputs.target.os != "linux" { + let linux = + inputs.target.os == "linux" && matches!(inputs.target.arch.as_str(), "x86_64" | "aarch64"); + let apple_arm64 = inputs.target.triple == "aarch64-apple-darwin" + && inputs.target.os == "macos" + && inputs.target.arch == "aarch64"; + + if !linux && !apple_arm64 { return Err(ConfigError::new(format!( - "linking is implemented only for Linux, not target {}", + "supported targets are Linux x86_64/aarch64 and aarch64-apple-darwin; target {} is unsupported", inputs.target.triple ))); } - if (inputs.features.cuda || inputs.features.vulkan) - && !matches!(inputs.target.arch.as_str(), "x86_64" | "aarch64") - { + if (inputs.features.cuda || inputs.features.vulkan) && !linux { let feature = if inputs.features.cuda { "cuda" } else { @@ -251,6 +273,13 @@ fn validate_targets(inputs: &Inputs) -> Result<(), ConfigError> { inputs.target.triple ))); } + if (inputs.features.metal || inputs.features.mlx) && !apple_arm64 { + let feature = if inputs.features.mlx { "mlx" } else { "metal" }; + return Err(ConfigError::new(format!( + "`{feature}` supports only aarch64-apple-darwin; target {} is unsupported", + inputs.target.triple + ))); + } Ok(()) } @@ -366,6 +395,57 @@ fn validate_cutlass( Ok(canonical) } +fn validate_mlx(inputs: &Inputs, probe: &impl PathProbe) -> Result, ConfigError> { + if !inputs.features.mlx { + if inputs.environment.mlx_root.is_some() { + return Err(ConfigError::new( + "MLX_ROOT is set but the `mlx` feature is disabled; remove it or enable `mlx`", + )); + } + return Ok(None); + } + + let root = inputs.environment.mlx_root.as_deref().ok_or_else(|| { + ConfigError::new( + "the `mlx` feature requires MLX_ROOT pointing to an existing MLX install; MLX is external and is never fetched or packaged", + ) + })?; + if !probe.is_dir(root) { + return Err(ConfigError::new(format!( + "MLX_ROOT={} is not an existing directory", + root.display() + ))); + } + let canonical = probe.canonicalize(root).map_err(|error| { + ConfigError::new(format!( + "failed to canonicalize MLX_ROOT={}: {error}", + root.display() + )) + })?; + if !canonical.is_absolute() { + return Err(ConfigError::new(format!( + "MLX_ROOT must resolve to an absolute path; got {}", + canonical.display() + ))); + } + + for relative in [ + "include/mlx/array.h", + "lib/libmlx.dylib", + "lib/mlx.metallib", + ] { + let path = canonical.join(relative); + if !probe.is_file(&path) { + return Err(ConfigError::new(format!( + "MLX_ROOT must contain {relative}; missing {}", + path.display() + ))); + } + } + + Ok(Some(canonical)) +} + fn parse_cutlass_version(contents: &str) -> Option<(u32, u32, u32)> { fn value(contents: &str, name: &str) -> Option { contents.lines().find_map(|line| { @@ -386,15 +466,25 @@ fn platform_link_requirements(inputs: &Inputs) -> Result, C if inputs.features.dynamic_link { return Ok(Vec::new()); } - if inputs.target.os != "linux" { - return Err(ConfigError::new(format!( - "static linking is implemented only for Linux, not {}", - inputs.target.os - ))); + + match inputs.target.os.as_str() { + "linux" => Ok(vec![ + LinkRequirement::Library("stdc++"), + LinkRequirement::Library("pthread"), + LinkRequirement::Library("dl"), + ]), + "macos" => { + let mut requirements = vec![LinkRequirement::Library("c++")]; + if inputs.features.metal { + requirements.extend([ + LinkRequirement::Framework("Metal"), + LinkRequirement::Framework("Foundation"), + ]); + } + Ok(requirements) + } + os => Err(ConfigError::new(format!( + "static linking is not implemented for target OS {os}" + ))), } - Ok(vec![ - LinkRequirement::Library("stdc++"), - LinkRequirement::Library("pthread"), - LinkRequirement::Library("dl"), - ]) } diff --git a/vllm-cpp-sys/src/build_support.rs b/vllm-cpp-sys/src/build_support.rs index cd83911..d6b5263 100644 --- a/vllm-cpp-sys/src/build_support.rs +++ b/vllm-cpp-sys/src/build_support.rs @@ -3,8 +3,17 @@ use std::path::{Path, PathBuf}; const LIB_DIR_CANDIDATES: [&str; 2] = ["lib64", "lib"]; -pub(crate) fn shared_library_name(stem: &str) -> String { - format!("lib{stem}.so") +pub(crate) fn shared_library_name(stem: &str, target_os: &str) -> Result { + let suffix = match target_os { + "linux" => ".so", + "macos" => ".dylib", + other => { + return Err(format!( + "shared libraries are unsupported for target OS {other}" + )) + } + }; + Ok(format!("lib{stem}{suffix}")) } pub(crate) fn require_library_file( diff --git a/vllm-cpp-sys/src/lib.rs b/vllm-cpp-sys/src/lib.rs index 34ad6f4..9da0808 100644 --- a/vllm-cpp-sys/src/lib.rs +++ b/vllm-cpp-sys/src/lib.rs @@ -22,10 +22,11 @@ //! //! The default `bundled` feature compiles and statically links the packaged //! pinned source. `system` links a caller-provided compatible installation, and -//! `dynamic-link` selects `libvllm.so` in either source mode. Dynamic consumers -//! must deploy the shared library and dependencies through the platform loader. -//! CUDA, external CUTLASS, Triton AOT, and Vulkan features are experimental -//! bundled build configuration and require their documented native inputs. +//! `dynamic-link` selects the platform `libvllm` shared library in either source +//! mode. Dynamic consumers must deploy that library and its dependencies through +//! the platform loader. CUDA, external CUTLASS, Triton AOT, Vulkan, Metal, and +//! external MLX features are experimental bundled build configuration and require +//! their documented targets and native inputs. //! //! Native Linux x86_64 CPU is the supported runtime tier. Building the bundled //! source requires CMake 3.24 or newer, a build tool, C11 and C++20 compilers, diff --git a/vllm-cpp-sys/tests/build_config.rs b/vllm-cpp-sys/tests/build_config.rs index 7c0245a..25b7478 100644 --- a/vllm-cpp-sys/tests/build_config.rs +++ b/vllm-cpp-sys/tests/build_config.rs @@ -19,6 +19,25 @@ struct MockProbe { } impl MockProbe { + fn mlx() -> Self { + let mut probe = Self::default(); + probe + .canonical + .insert(PathBuf::from("mlx"), PathBuf::from("/mlx")); + probe.dirs.insert(PathBuf::from("mlx")); + probe.dirs.insert(PathBuf::from("/mlx")); + for relative in [ + "include/mlx/array.h", + "lib/libmlx.dylib", + "lib/mlx.metallib", + ] { + probe + .files + .insert(PathBuf::from("/mlx").join(relative), String::new()); + } + probe + } + fn cutlass(version: (u32, u32, u32)) -> Self { let mut probe = Self::default(); probe @@ -84,6 +103,31 @@ fn linux() -> Inputs { } } +fn linux_arm64() -> Inputs { + let mut inputs = linux(); + inputs.target = Target { + triple: "aarch64-unknown-linux-gnu".to_owned(), + os: "linux".to_owned(), + arch: "aarch64".to_owned(), + }; + inputs +} + +fn apple() -> Inputs { + Inputs { + features: Features { + bundled: true, + ..Features::default() + }, + target: Target { + triple: "aarch64-apple-darwin".to_owned(), + os: "macos".to_owned(), + arch: "aarch64".to_owned(), + }, + environment: Environment::default(), + } +} + fn error(inputs: &Inputs, probe: &MockProbe) -> String { plan(inputs, probe).unwrap_err().to_string() } @@ -91,12 +135,20 @@ fn error(inputs: &Inputs, probe: &MockProbe) -> String { #[test] fn cpu_and_vulkan_plans_are_deterministic() { let cpu = plan(&linux(), &MockProbe::default()).unwrap(); + let arm_cpu = plan(&linux_arm64(), &MockProbe::default()).unwrap(); + assert_eq!(arm_cpu, cpu); assert!(cpu .cmake_defines .contains(&("VLLM_CPP_CUDA", "OFF".to_owned()))); assert!(cpu .cmake_defines .contains(&("VLLM_CPP_VULKAN", "OFF".to_owned()))); + assert!(cpu + .cmake_defines + .contains(&("VLLM_CPP_METAL", "OFF".to_owned()))); + assert!(cpu + .cmake_defines + .contains(&("VLLM_CPP_MLX", "OFF".to_owned()))); assert!(!cpu .cmake_defines .iter() @@ -120,6 +172,139 @@ fn cpu_and_vulkan_plans_are_deterministic() { .contains(&("VLLM_CPP_VULKAN", "ON".to_owned()))); } +#[test] +fn apple_cpu_metal_and_mlx_plans_are_deterministic() { + let cpu = plan(&apple(), &MockProbe::default()).unwrap(); + assert_eq!(cpu.link_requirements, vec![LinkRequirement::Library("c++")]); + assert!(cpu + .cmake_defines + .contains(&("VLLM_CPP_METAL", "OFF".to_owned()))); + assert!(cpu + .cmake_defines + .contains(&("VLLM_CPP_MLX", "OFF".to_owned()))); + + let mut metal = apple(); + metal.features.metal = true; + let metal_plan = plan(&metal, &MockProbe::default()).unwrap(); + assert!(metal_plan + .cmake_defines + .contains(&("VLLM_CPP_METAL", "ON".to_owned()))); + assert!(metal_plan + .cmake_defines + .contains(&("VLLM_CPP_MLX", "OFF".to_owned()))); + assert_eq!( + metal_plan.link_requirements, + vec![ + LinkRequirement::Library("c++"), + LinkRequirement::Framework("Metal"), + LinkRequirement::Framework("Foundation"), + ] + ); + + let mut mlx = metal; + mlx.features.mlx = true; + mlx.environment.mlx_root = Some(PathBuf::from("mlx")); + let mlx_plan = plan(&mlx, &MockProbe::mlx()).unwrap(); + assert!(mlx_plan + .cmake_defines + .contains(&("VLLM_CPP_MLX", "ON".to_owned()))); + assert!(mlx_plan + .cmake_defines + .contains(&("MLX_ROOT", "/mlx".to_owned()))); + assert_eq!( + mlx_plan.link_requirements, + vec![ + LinkRequirement::Library("c++"), + LinkRequirement::Framework("Metal"), + LinkRequirement::Framework("Foundation"), + LinkRequirement::NativeSearch(PathBuf::from("/mlx/lib")), + LinkRequirement::Library("mlx"), + ] + ); + + mlx.features.dynamic_link = true; + let dynamic_mlx = plan(&mlx, &MockProbe::mlx()).unwrap(); + assert!(dynamic_mlx.link_requirements.is_empty()); + assert!(dynamic_mlx + .cmake_defines + .contains(&("MLX_ROOT", "/mlx".to_owned()))); +} + +#[test] +fn mlx_validation_reports_each_configuration_error() { + let mut inputs = apple(); + inputs.features.metal = true; + inputs.features.mlx = true; + assert_eq!( + error(&inputs, &MockProbe::default()), + format!( + "{ERROR_PREFIX} the `mlx` feature requires MLX_ROOT pointing to an existing MLX install; MLX is external and is never fetched or packaged" + ) + ); + + inputs.environment.mlx_root = Some(PathBuf::from("mlx")); + assert_eq!( + error(&inputs, &MockProbe::default()), + format!("{ERROR_PREFIX} MLX_ROOT=mlx is not an existing directory") + ); + + let mut relative_probe = MockProbe::mlx(); + relative_probe + .canonical + .insert(PathBuf::from("mlx"), PathBuf::from("canonical-mlx")); + assert!(error(&inputs, &relative_probe).contains("must resolve to an absolute path")); + + for missing in [ + "include/mlx/array.h", + "lib/libmlx.dylib", + "lib/mlx.metallib", + ] { + let mut probe = MockProbe::mlx(); + probe.files.remove(&PathBuf::from("/mlx").join(missing)); + assert_eq!( + error(&inputs, &probe), + format!("{ERROR_PREFIX} MLX_ROOT must contain {missing}; missing /mlx/{missing}") + ); + } + + let mut no_feature = apple(); + no_feature.environment.mlx_root = Some(PathBuf::from("mlx")); + assert!(error(&no_feature, &MockProbe::mlx()).contains("feature is disabled")); +} + +#[test] +fn apple_backends_reject_other_targets_and_system_mode() { + for feature in ["metal", "mlx"] { + let mut inputs = linux(); + inputs.features.metal = true; + inputs.features.mlx = feature == "mlx"; + if inputs.features.mlx { + inputs.environment.mlx_root = Some(PathBuf::from("mlx")); + } + assert!(error(&inputs, &MockProbe::mlx()).contains("aarch64-apple-darwin")); + } + + let mut apple_cuda = apple(); + apple_cuda.features.cuda = true; + apple_cuda.environment.cuda_architectures = Some("80".to_owned()); + assert!(error(&apple_cuda, &MockProbe::default()) + .contains("`cuda` supports only Linux x86_64/aarch64 targets")); + + let mut x86_macos = apple(); + x86_macos.target = Target { + triple: "x86_64-apple-darwin".to_owned(), + os: "macos".to_owned(), + arch: "x86_64".to_owned(), + }; + assert!(error(&x86_macos, &MockProbe::default()).contains("target x86_64-apple-darwin")); + + let mut system_metal = apple(); + system_metal.features.bundled = false; + system_metal.features.system = true; + system_metal.features.metal = true; + assert!(error(&system_metal, &MockProbe::default()).contains("bundled-only")); +} + #[test] fn accepts_exact_cuda_architecture_spellings() { for architecture in [ @@ -164,7 +349,7 @@ fn cuda_requires_arch_and_supported_target() { os: "windows".to_owned(), arch: "x86_64".to_owned(), }; - assert!(error(&inputs, &MockProbe::default()).contains("only for Linux")); + assert!(error(&inputs, &MockProbe::default()).contains("target x86_64-pc-windows-msvc")); } #[test] @@ -198,6 +383,13 @@ fn rejects_broken_feature_implications() { let mut triton = linux(); triton.features.triton_aot = true; assert!(error(&triton, &MockProbe::default()).contains("requires the `cuda`")); + + let mut mlx = apple(); + mlx.features.mlx = true; + assert_eq!( + error(&mlx, &MockProbe::default()), + format!("{ERROR_PREFIX} `mlx` requires the `metal` feature") + ); } #[test] diff --git a/vllm-cpp-sys/tests/build_support.rs b/vllm-cpp-sys/tests/build_support.rs index c56682e..1ec5b56 100644 --- a/vllm-cpp-sys/tests/build_support.rs +++ b/vllm-cpp-sys/tests/build_support.rs @@ -51,8 +51,16 @@ fn write_file(path: &Path) { } #[test] -fn shared_library_name_uses_linux_soname() { - assert_eq!(shared_library_name("vllm"), "libvllm.so"); +fn shared_library_name_is_target_aware() { + assert_eq!(shared_library_name("vllm", "linux").unwrap(), "libvllm.so"); + assert_eq!( + shared_library_name("vllm", "macos").unwrap(), + "libvllm.dylib" + ); + assert_eq!( + shared_library_name("vllm", "windows").unwrap_err(), + "shared libraries are unsupported for target OS windows" + ); } #[test] diff --git a/vllm-cpp/Cargo.toml b/vllm-cpp/Cargo.toml index f4e3bf6..6e9e481 100644 --- a/vllm-cpp/Cargo.toml +++ b/vllm-cpp/Cargo.toml @@ -20,6 +20,8 @@ cuda = ["vllm-cpp-sys/cuda"] cuda-cutlass = ["cuda", "vllm-cpp-sys/cuda-cutlass"] triton-aot = ["cuda", "vllm-cpp-sys/triton-aot"] vulkan = ["vllm-cpp-sys/vulkan"] +metal = ["vllm-cpp-sys/metal"] +mlx = ["metal", "vllm-cpp-sys/mlx"] serde = ["dep:serde_json"] [package.metadata.docs.rs] diff --git a/vllm-cpp/README.md b/vllm-cpp/README.md index 2c9d1a6..ffcf185 100644 --- a/vllm-cpp/README.md +++ b/vllm-cpp/README.md @@ -37,18 +37,20 @@ The model argument is a directory understood by the pinned native engine, not a | `cuda-cutlass` | Experimental CUDA build with a caller-provided CUTLASS >=4.5.0 tree | | `triton-aot` | Experimental CUDA build using checked-in Triton AOT artifacts | | `vulkan` | Experimental bundled Vulkan build configuration | +| `metal` | Experimental native Metal build on Apple ARM64 | +| `mlx` | Experimental external MLX provider on Apple ARM64; implies `metal` | -`bundled` and `system` conflict. CUDA and Vulkan conflict, and accelerator features are bundled-only but do not implicitly enable `bundled` for `--no-default-features` builds. The workspace [backend documentation](https://github.com/querymt/vllm-cpp-rs#experimental-backend-builds) records exact environment variables, supported build architectures, and current blockers. +`bundled` and `system` conflict. CUDA and Vulkan conflict, and accelerator features are bundled-only but do not implicitly enable `bundled` for `--no-default-features` builds. Metal/MLX require exact `aarch64-apple-darwin`; MLX additionally requires an external `MLX_ROOT` with its headers, dylib, and metallib. The workspace [backend documentation](https://github.com/querymt/vllm-cpp-rs#experimental-backend-builds) records exact environment variables, supported build architectures, and current blockers. ## ABI and deployment This crate is tied to the exact same `vllm-cpp-sys` crate version and the pinned vllm.cpp commit `34aedfbe8ed9779697905541a62e2160ccfd9c05`. Model loading requires exact C ABI version 10 before any versioned struct crosses FFI. A system library must implement the same ABI; the consumer build checks for its header, while maintainer conformance tests check layout and symbols. -Static bundled builds include the native archive in the application link. Dynamic bundled or system builds do not deploy `libvllm.so`: install it and its backend/toolkit dependencies in a loader-visible location using `LD_LIBRARY_PATH`, rpath, or the system loader configuration. System mode uses `VLLM_CPP_ROOT`; `VLLM_CPP_LIB_DIR` can choose a nonstandard library directory. System static linking also requires the matching `libblake3_vendored.a` through `VLLM_CPP_BLAKE3_LIB_DIR` or the selected vllm library directory. +Static bundled builds include the native archive in the application link. Dynamic bundled or system builds do not deploy `libvllm.so`/`libvllm.dylib`: install it and its backend/toolkit dependencies in a loader-visible location using `LD_LIBRARY_PATH`, `DYLD_LIBRARY_PATH`, rpath supplied by the application, or the system loader configuration. System mode uses `VLLM_CPP_ROOT`; `VLLM_CPP_LIB_DIR` can choose a nonstandard library directory. System static linking also requires the matching `libblake3_vendored.a` through `VLLM_CPP_BLAKE3_LIB_DIR` or the selected vllm library directory. ## Support boundary -The supported runtime tier is native Linux x86_64 CPU, covering bundled/system and static/dynamic link modes. Linux x86_64/aarch64 CUDA, external CUTLASS, Triton AOT, and Vulkan features are experimental build/configuration surfaces only. Known native blockers include CUDA teardown failure, CUDA bf16 numerical tolerance failure, CUTLASS concurrent-output differences, and incomplete Vulkan runtime coverage. Successful compilation is not a runtime-support claim. +The supported runtime tier is native Linux x86_64 CPU, covering bundled/system and static/dynamic link modes. Linux ARM64 and Apple ARM64 CPU have manual hosted jobs configured for model-free build/test coverage. CUDA, external CUTLASS, Triton AOT, Vulkan, Metal, and MLX are experimental build/configuration surfaces. The hosted Metal job checks compilation/linking only; Vulkan software-device gates check backend/ops, not attention or model inference; MLX remains external and has no release-lane model/runtime evidence. Known native blockers include CUDA teardown failure, CUDA bf16 numerical tolerance failure, CUTLASS concurrent-output differences, incomplete Vulkan attention/model runtime, and MLX's numerically distinct provider behavior. CPU remains the only supported runtime family. See the repository [changelog](https://github.com/querymt/vllm-cpp-rs/blob/main/CHANGELOG.md), [release process](https://github.com/querymt/vllm-cpp-rs/blob/main/RELEASING.md), and [root support details](https://github.com/querymt/vllm-cpp-rs#support) for the current release boundary. diff --git a/vllm-cpp/src/lib.rs b/vllm-cpp/src/lib.rs index 8a98169..43c4919 100644 --- a/vllm-cpp/src/lib.rs +++ b/vllm-cpp/src/lib.rs @@ -30,12 +30,13 @@ //! [`expected_abi_version`] before versioned structs cross FFI. The default //! `bundled` feature builds the pinned native source. `system` selects a //! caller-provided installation, `dynamic-link` selects shared linking, and -//! `serde` adds typed JSON helpers. CUDA, CUTLASS, Triton AOT, and Vulkan features -//! are experimental bundled build configuration. +//! `serde` adds typed JSON helpers. CUDA, CUTLASS, Triton AOT, Vulkan, Metal, and +//! external MLX features are experimental bundled build configuration. //! -//! Dynamic linking does not deploy `libvllm.so`; applications must make it and -//! its runtime dependencies visible through `LD_LIBRARY_PATH`, rpath, or system -//! loader configuration. The supported runtime tier is native Linux x86_64 CPU. +//! Dynamic linking does not deploy `libvllm.so` or `libvllm.dylib`; applications +//! must make it and its runtime dependencies visible through the platform loader, +//! such as `LD_LIBRARY_PATH`, `DYLD_LIBRARY_PATH`, or an application-owned rpath. +//! The supported runtime tier is native Linux x86_64 CPU. //! Accelerator features are build/configuration surfaces with known runtime //! blockers, not complete accelerator runtime support.