From 7274bdee75320962b9e93f87876c66bf3926b23e Mon Sep 17 00:00:00 2001 From: Ivan Zatevakhin Date: Mon, 10 Aug 2026 23:48:25 +0100 Subject: [PATCH] feat: add safe blocking inference API --- Cargo.lock | 96 +++++++ Justfile | 162 ++++++++++- README.md | 67 ++++- vllm-cpp-sys/build.rs | 41 ++- vllm-cpp/Cargo.toml | 7 +- vllm-cpp/examples/chat.rs | 17 ++ vllm-cpp/examples/complete.rs | 14 + vllm-cpp/examples/stream.rs | 21 ++ vllm-cpp/examples/structured.rs | 19 ++ vllm-cpp/src/callback.rs | 121 ++++++++ vllm-cpp/src/engine.rs | 477 ++++++++++++++++++++++++++++++++ vllm-cpp/src/error.rs | 95 +++++++ vllm-cpp/src/lib.rs | 28 +- vllm-cpp/src/params.rs | 330 ++++++++++++++++++++++ vllm-cpp/tests/qwen3.rs | 205 ++++++++++++++ vllm-cpp/tests/safe_api.rs | 58 ++++ 16 files changed, 1740 insertions(+), 18 deletions(-) create mode 100644 vllm-cpp/examples/chat.rs create mode 100644 vllm-cpp/examples/complete.rs create mode 100644 vllm-cpp/examples/stream.rs create mode 100644 vllm-cpp/examples/structured.rs create mode 100644 vllm-cpp/src/callback.rs create mode 100644 vllm-cpp/src/engine.rs create mode 100644 vllm-cpp/src/error.rs create mode 100644 vllm-cpp/src/params.rs create mode 100644 vllm-cpp/tests/qwen3.rs create mode 100644 vllm-cpp/tests/safe_api.rs diff --git a/Cargo.lock b/Cargo.lock index 26ed156..2ea39b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,16 +27,106 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + [[package]] name = "vllm-cpp" version = "0.1.0" dependencies = [ + "serde_json", "vllm-cpp-sys", ] @@ -46,3 +136,9 @@ version = "0.1.0" dependencies = [ "cmake", ] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Justfile b/Justfile index bf281fd..487f7a1 100644 --- a/Justfile +++ b/Justfile @@ -2,6 +2,7 @@ set shell := ["bash", "-euo", "pipefail", "-c"] root := justfile_directory() bindings_file := root + "/vllm-cpp-sys/src/bindings.rs" +model_revision := "c1899de289a04d12100db370d81485cdf75e47ca" # Maintainer workflows require Just 1.40 or newer. @@ -255,6 +256,23 @@ link-modes: "expected VLLM_CPP_LIB_DIR override artifact libvllm.a at $prefix/lib64/libvllm.a" \ "$system_override_static_log" + system_sanitize_target="$work/system-sanitize" + system_sanitize_log="$work/system-sanitize.log" + echo '==> system sanitizer rejection' + if VLLM_CPP_ROOT="$prefix" \ + VLLM_CPP_BLAKE3_LIB_DIR="$prefix/blake3-lib" \ + VLLM_CPP_SANITIZE=address \ + CARGO_TARGET_DIR="$system_sanitize_target" \ + cargo test --locked -vv -p vllm-cpp-sys --release --tests \ + --no-default-features --features system 2>&1 \ + | tee "$system_sanitize_log"; then + echo 'expected system mode to reject VLLM_CPP_SANITIZE' >&2 + exit 1 + fi + grep -Fq \ + 'VLLM_CPP_SANITIZE is supported only for bundled builds' \ + "$system_sanitize_log" + system_dynamic_target="$work/system-dynamic" system_dynamic_log="$work/system-dynamic.log" echo '==> system dynamic' @@ -475,6 +493,144 @@ package-test: cargo run --locked --release --offline ) + safe_version=$(cargo metadata --locked --offline --no-deps --format-version 1 \ + | jq -er '[.packages[] | select(.name == "vllm-cpp") | .version] | if length == 1 then .[0] else error("expected exactly one vllm-cpp package") end') + safe_package_args=() + if [[ -n $(git status --porcelain=v1 --untracked-files=all -- vllm-cpp vllm-cpp-sys) ]]; then + safe_package_args+=(--allow-dirty) + fi + cargo package --workspace --locked --offline --no-verify "${safe_package_args[@]}" + safe_package_file="$package_target/package/vllm-cpp-$safe_version.crate" + tar -xzf "$safe_package_file" -C "$temp" + safe_root="$temp/vllm-cpp-$safe_version" + safe_manifest="$safe_root/Cargo.toml" + sed -i \ + "/\[dependencies.vllm-cpp-sys\]/a path = \"$package_root\"" \ + "$safe_manifest" + ( + cd "$safe_root" + CARGO_NET_OFFLINE=true cargo generate-lockfile --offline + ) + + package_listing="$temp/safe-package.list" + tar -tzf "$safe_package_file" > "$package_listing" + if grep -Eq '(^|/)(target|model\.safetensors)(/|$)' "$package_listing"; then + echo 'local build output or model fixture leaked into the safe crate package' >&2 + exit 1 + fi + if grep -RIlF "$repo_root" "$safe_root" --exclude=Cargo.toml >/dev/null; then + echo 'local repository path leaked into the safe crate package' >&2 + exit 1 + fi + [[ ! -e $safe_root/target ]] || { + echo 'local build output leaked into the safe crate package' >&2 + exit 1 + } + [[ ! -e $safe_root/model.safetensors ]] || { + echo 'model fixture leaked into the safe crate package' >&2 + exit 1 + } + ( + cd "$safe_root" + env -u VLLM_CPP_TEST_MODEL \ + CARGO_NET_OFFLINE=true CARGO_TARGET_DIR="$temp_target" \ + cargo test --locked --release --offline --features bundled,serde + ) + +# Download and verify the pinned Qwen3-0.6B model fixture. +setup-test-model destination=env_var_or_default("VLLM_CPP_TEST_MODEL", env_var_or_default("XDG_CACHE_HOME", env_var("HOME") + "/.cache") + "/vllm-cpp-rs/Qwen3-0.6B-" + model_revision): + #!/usr/bin/env bash + set -euo pipefail + revision={{ quote(model_revision) }} + base="https://huggingface.co/Qwen/Qwen3-0.6B/resolve/$revision" + destination={{ quote(destination) }} + mkdir -p "$destination" + + files=( + LICENSE + config.json + generation_config.json + merges.txt + model.safetensors + tokenizer.json + tokenizer_config.json + vocab.json + ) + for file in "${files[@]}"; do + if [[ ! -f $destination/$file ]]; then + echo "downloading $file" >&2 + curl --fail --location --retry 3 --continue-at - \ + "$base/$file" --output "$destination/$file" + fi + done + + cat > "$destination/SHA256SUMS.expected" <<'EOF' + 832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e LICENSE + 660db3b73d788119c04535e48cf9be5f55bc3100841a718637ae695b442f27dd config.json + 2325da0f15bb848e018c5ae071b7943332e9f871d6b60e2ed22ca97d4cb993d2 generation_config.json + 8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 merges.txt + f47f71177f32bcd101b7573ec9171e6a57f4f4d31148d38e382306f42996874b model.safetensors + aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4 tokenizer.json + d5d09f07b48c3086c508b30d1c9114bd1189145b74e982a265350c923acd8101 tokenizer_config.json + ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 vocab.json + EOF + (cd "$destination" && sha256sum --check SHA256SUMS.expected) >&2 + printf '%s\n' "$destination" + +# Run the blocking safe API and Qwen model suites with ASan, UBSan, and leak detection. +sanitizers model=env_var_or_default("VLLM_CPP_TEST_MODEL", ""): + #!/usr/bin/env bash + set -euo pipefail + model={{ quote(model) }} + if [[ -z $model ]]; then + echo 'set VLLM_CPP_TEST_MODEL or pass model=' >&2 + exit 1 + fi + required_model_files=( + model.safetensors + config.json + tokenizer.json + tokenizer_config.json + ) + missing=() + for file in "${required_model_files[@]}"; do + [[ -f $model/$file ]] || missing+=("$file") + done + if ((${#missing[@]})); then + printf 'model fixture is incomplete at %s; missing:' "$model" >&2 + printf ' %s' "${missing[@]}" >&2 + printf '\n' >&2 + exit 1 + fi + cd {{ quote(root) }} + export VLLM_CPP_TEST_MODEL="$model" + export VLLM_CPP_SANITIZE=address,undefined + export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-{{ quote(root + "/target/sanitize") }}} + + cargo test --locked -p vllm-cpp --test safe_api --test qwen3 --no-run + + asan=$(gcc -print-file-name=libasan.so) + ubsan=$(gcc -print-file-name=libubsan.so) + if [[ ! -f $asan || ! -f $ubsan ]]; then + echo 'GCC sanitizer runtimes are unavailable' >&2 + exit 1 + fi + export LD_PRELOAD="$asan:$ubsan${LD_PRELOAD:+:$LD_PRELOAD}" + export ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=1:halt_on_error=1} + export UBSAN_OPTIONS=${UBSAN_OPTIONS:-halt_on_error=1:print_stacktrace=1} + export VT_POOL_BYPASS=1 + + for pattern in safe_api qwen3; do + binary=$(find "$CARGO_TARGET_DIR/debug/deps" -maxdepth 1 -type f \ + -name "$pattern-*" -executable -printf '%T@ %p\n' \ + | sort -n | tail -1 | cut -d' ' -f2-) || true + if [[ -z $binary ]]; then + echo "could not find $pattern test binary" >&2 + exit 1 + fi + "$binary" --test-threads=1 + done + # Check Just and Rust formatting. fmt-check: just --unstable --justfile {{ quote(root + "/Justfile") }} --fmt --check @@ -484,5 +640,9 @@ fmt-check: lint: cargo clippy --locked --workspace --all-targets -- -D warnings +# Build API documentation with warnings denied. +docs: + RUSTDOCFLAGS="-D warnings" cargo doc --locked --workspace --no-deps --features vllm-cpp/serde + # Run the complete maintainer gate serially; do not pass --jobs. -ci: fmt-check lint sys link-modes package-test +ci: fmt-check lint docs sys link-modes package-test diff --git a/README.md b/README.md index 92b359a..a6d5d4a 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ Rust bindings for [vllm.cpp](https://github.com/mudler/vllm.cpp), organized as: - `vllm-cpp-sys`: raw C API bindings and the pinned native source build. -- `vllm-cpp`: the application-facing Rust bindings. +- `vllm-cpp`: the application-facing safe blocking API. ## Status -The sys crate provides checked-in generated raw FFI declarations. Conformance checks cover C/Rust layout, all 19 exported C symbols, and C ABI version 10. Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. A high-level API follows in a later PR. +The safe crate provides an owned blocking engine API for model loading, completion, streaming, structured output, and raw-JSON chat. An optional `serde` feature adds `serde_json::Value` chat helpers. The sys crate provides checked-in generated FFI declarations with C/Rust layout checks and coverage for all 19 exported C symbols. -vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. +Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. ## Prerequisites @@ -20,7 +20,7 @@ Initial development and testing support Linux CPU builds. They require: - Ninja or another CMake build tool. - A C11 and C++20 compiler. - A system linker and C++ standard library. -- Just 1.40 or newer for maintainer workflows, plus Git, `jq`, and GNU tar. +- Just 1.40 or newer for maintainer workflows, plus Git, `jq`, GNU tar, and `curl` for the model fixture recipe. This repository provides a Nix development shell with the pinned development tools: @@ -36,6 +36,33 @@ vllm.cpp is a Git submodule. Clone recursively, or initialize it after cloning: git submodule update --init --recursive ``` +## Safe API + +```rust +use vllm_cpp::{Engine, SamplingParams}; + +let engine = Engine::load("/models/Qwen3-0.6B")?; +let completion = engine.complete( + "The capital of France is", + &SamplingParams::greedy().max_tokens(16), +)?; +println!("{}", completion.text); +# Ok::<(), vllm_cpp::Error>(()) +``` + +`EngineBuilder` owns model settings and converts them to temporary C strings only for the load call. `SamplingParams` owns stop strings and structured constraints. Completion and chat strings are copied into Rust values before the matching native free function runs. + +Blocking streaming callbacks receive copied UTF-8 deltas. Callback panics are caught before the C boundary and resumed only after the native call has stopped and returned. Chat methods accept raw OpenAI-compatible request JSON; enable `serde` for `serde_json::Value` request and response helpers. + +Run the practical examples with a model directory: + +```console +cargo run -p vllm-cpp --example complete -- +cargo run -p vllm-cpp --example stream -- +cargo run -p vllm-cpp --example chat -- +cargo run -p vllm-cpp --example structured -- +``` + ## Build and Test Inside the Nix shell, use the root `Justfile` (requires Just 1.40 or newer) for maintainer workflows: @@ -43,12 +70,33 @@ Inside the Nix shell, use the root `Justfile` (requires Just 1.40 or newer) for ```console CMAKE_GENERATOR=Ninja cargo build --locked --release CMAKE_GENERATOR=Ninja cargo test --locked --workspace --release +cargo test --locked -p vllm-cpp --release --features serde just ci ``` Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The bundled build is deterministic and CPU-only: native tests, examples, the HTTP server, CUDA, Metal, MLX, Vulkan, Triton, and CUTLASS fetching are disabled explicitly. -`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. `just package-test` requires `jq` and GNU tar, fetches locked Cargo dependencies when `CARGO_NET_OFFLINE` is not `true`, and then validates the package offline. +`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. + +## Test Model and Sanitizers + +Model-backed tests use Apache-2.0 `Qwen/Qwen3-0.6B` at pinned revision `c1899de289a04d12100db370d81485cdf75e47ca`. Download or reuse the cache and verify every file, then run exactly the six blocking model tests serially: + +```console +model=$(just setup-test-model) +VLLM_CPP_TEST_MODEL="$model" \ + cargo test --locked -p vllm-cpp --release --test qwen3 -- --test-threads=1 +``` + +The approximately 1.5 GB model stays in the user cache and is not included in repository or crate packages. Model-backed tests skip with an explanatory message when `VLLM_CPP_TEST_MODEL` is unset. When it is set, the test helper and sanitizer gate require `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json` and report every missing file. + +Address, undefined-behavior, and leak checks run the blocking safe API and six model tests with native instrumentation: + +```console +just sanitizers "$model" +``` + +`VLLM_CPP_SANITIZE` is a bundled-build test input. System mode rejects it because Cargo cannot infer whether an externally built native library carries matching instrumentation. ## Link Modes @@ -57,22 +105,23 @@ Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The bundled buil - `system` requires `--no-default-features` and links a prefix selected by `VLLM_CPP_ROOT`. - `system,dynamic-link` dynamically links the selected system library. -System-mode consumer builds validate that `VLLM_CPP_ROOT/include/vllm.h` exists; they do not compare its layout. The maintainer `layout-test` recipe compiles `tests/layout.c` at test runtime against the bundled header and compares its C layouts with the generated Rust declarations. The `link-modes` recipe repeats layout conformance against bundled and system fixture headers and also runs the runtime ABI test. `VLLM_CPP_LIB_DIR` selects a nonstandard vllm library directory. Upstream's normal CMake install does not install `libblake3_vendored.a`, so a stock install works directly with `system,dynamic-link`; system static users must provision that archive separately and set `VLLM_CPP_BLAKE3_LIB_DIR` (or place it in the vllm library directory). Dynamic tests and applications must make `libvllm.so` loader-visible with `LD_LIBRARY_PATH`, rpath, or an installed loader path; Cargo neither deploys the library nor configures rpath. Native Linux CPU is the supported target; cross-compiling the layout integration test is unsupported because it executes the compiled target probe. +System-mode consumer builds validate that `VLLM_CPP_ROOT/include/vllm.h` exists; they do not compare its layout. The maintainer `layout-test` recipe compiles `tests/layout.c` at test runtime against the bundled header and compares its C layouts with the generated Rust declarations. The `link-modes` recipe repeats layout conformance against bundled and system fixture headers and also runs the runtime ABI test. `VLLM_CPP_LIB_DIR` selects a nonstandard vllm library directory. Upstream's normal CMake install does not install `libblake3_vendored.a`, so a stock install works directly with `system,dynamic-link`; system static users must provision that archive separately and set `VLLM_CPP_BLAKE3_LIB_DIR` (or place it in the vllm library directory). Dynamic tests and applications must make `libvllm.so` loader-visible with `LD_LIBRARY_PATH`, rpath, or an installed loader path; Cargo neither deploys the library nor configures rpath. ## Packaging -Inspect and build the sys package with: +Inspect and test both packaged crates with: ```console cargo package -p vllm-cpp-sys --locked --list +cargo package -p vllm-cpp --locked --list just package-test ``` -The package carries only the native build inputs and required licenses/notices; upstream tests, large fixtures, media, benchmarks, and agent records are excluded. The package measures approximately 30 MiB unpacked and 4.2 MiB compressed. +The package gate preserves the sys crate inventory, tests the extracted sys crate and downstream fixture offline, then points the extracted safe crate at the extracted sys crate and tests it offline with `bundled,serde`. It also rejects local paths, build output, and model files in the safe package. The sys package carries only native build inputs and required licenses/notices; upstream tests, large fixtures, media, benchmarks, and agent records are excluded. ## Support -CI runs Linux x86_64 CPU tests in all four link modes: bundled static/dynamic and system static/dynamic. The system prefix is a test fixture assembled from bundled build outputs; its separately copied BLAKE3 archive demonstrates the explicit system-static contract rather than claiming that a stock upstream install provides one. Other operating systems, architectures, and accelerator backends are not yet supported by the Rust build. +The supported target is native Linux x86_64 CPU. Maintainer tests cover the four bundled/system static/dynamic link modes and bundled blocking inference with the pinned Qwen fixture. Other operating systems, architectures, and accelerator builds are not supported by this Rust build. ## Licensing and Affiliation diff --git a/vllm-cpp-sys/build.rs b/vllm-cpp-sys/build.rs index 737cdcf..3dc6ff0 100644 --- a/vllm-cpp-sys/build.rs +++ b/vllm-cpp-sys/build.rs @@ -16,6 +16,7 @@ const RERUN_ENV: &[&str] = &[ "VLLM_CPP_ROOT", "VLLM_CPP_LIB_DIR", "VLLM_CPP_BLAKE3_LIB_DIR", + "VLLM_CPP_SANITIZE", "CC", "CFLAGS", "CXX", @@ -55,15 +56,17 @@ fn main() { panic!("linking is implemented only for Linux, not {target_os}"); } + let sanitizer = sanitizer_config(bundled); let system_root = system.then(validate_system_root); if bundled { - build_bundled(); + build_bundled(sanitizer.as_deref()); } else { link_system(system_root.as_deref().expect("system root was validated")); } + link_sanitizer_runtimes(sanitizer.as_deref()); } -fn build_bundled() { +fn build_bundled(sanitizer: Option<&str>) { let source = Path::new("vllm.cpp"); if !source.join("CMakeLists.txt").is_file() { panic!( @@ -84,7 +87,7 @@ fn build_bundled() { .define("VLLM_CPP_TRITON", "OFF") .define("VLLM_CPP_TRITON_REGEN", "OFF") .define("VLLM_CPP_CUTLASS_FETCH", "OFF") - .define("VLLM_CPP_SANITIZE", "OFF"); + .define("VLLM_CPP_SANITIZE", sanitizer.unwrap_or("OFF")); let dynamic_link = cfg!(feature = "dynamic-link"); let vllm_artifact = if dynamic_link { @@ -218,6 +221,38 @@ fn system_library_dir(root: &Path, expected_artifact: &str) -> PathBuf { }) } +fn sanitizer_config(bundled: bool) -> Option { + let value = env::var_os("VLLM_CPP_SANITIZE")?; + let value = value + .into_string() + .unwrap_or_else(|_| panic!("VLLM_CPP_SANITIZE must be valid UTF-8")); + if value == "OFF" { + return None; + } + if !bundled { + panic!("VLLM_CPP_SANITIZE is supported only for bundled builds"); + } + match value.as_str() { + "address" | "undefined" | "address,undefined" => Some(value), + "thread" => panic!("thread sanitization is not part of the blocking API test lane"), + _ => panic!( + "unsupported VLLM_CPP_SANITIZE value `{value}`; expected OFF, address, undefined, or address,undefined" + ), + } +} + +fn link_sanitizer_runtimes(sanitizer: Option<&str>) { + let Some(sanitizer) = sanitizer else { + return; + }; + if sanitizer.split(',').any(|name| name == "address") { + println!("cargo:rustc-link-lib=dylib=asan"); + } + if sanitizer.split(',').any(|name| name == "undefined") { + println!("cargo:rustc-link-lib=dylib=ubsan"); + } +} + fn link_platform_dependencies() { println!("cargo:rustc-link-lib=dylib=stdc++"); println!("cargo:rustc-link-lib=dylib=pthread"); diff --git a/vllm-cpp/Cargo.toml b/vllm-cpp/Cargo.toml index b6570b9..6ebcf79 100644 --- a/vllm-cpp/Cargo.toml +++ b/vllm-cpp/Cargo.toml @@ -5,13 +5,18 @@ edition.workspace = true license.workspace = true repository.workspace = true rust-version.workspace = true -description = "Rust bindings for vllm.cpp" +description = "Safe Rust bindings for vllm.cpp" [features] default = ["bundled"] bundled = ["vllm-cpp-sys/bundled"] system = ["vllm-cpp-sys/system"] dynamic-link = ["vllm-cpp-sys/dynamic-link"] +serde = ["dep:serde_json"] [dependencies] +serde_json = { version = "1.0.149", optional = true } vllm-cpp-sys = { workspace = true, default-features = false } + +[dev-dependencies] +serde_json = "1.0.149" diff --git a/vllm-cpp/examples/chat.rs b/vllm-cpp/examples/chat.rs new file mode 100644 index 0000000..77341d7 --- /dev/null +++ b/vllm-cpp/examples/chat.rs @@ -0,0 +1,17 @@ +use vllm_cpp::Engine; + +fn main() -> Result<(), Box> { + let model = std::env::args_os() + .nth(1) + .ok_or("usage: chat ")?; + let engine = Engine::load(model)?; + let response = engine.chat_json( + r#"{ + "messages": [{"role": "user", "content": "Reply with hello."}], + "temperature": 0, + "max_tokens": 16 + }"#, + )?; + println!("{response}"); + Ok(()) +} diff --git a/vllm-cpp/examples/complete.rs b/vllm-cpp/examples/complete.rs new file mode 100644 index 0000000..71cb53d --- /dev/null +++ b/vllm-cpp/examples/complete.rs @@ -0,0 +1,14 @@ +use vllm_cpp::{Engine, SamplingParams}; + +fn main() -> Result<(), Box> { + let model = std::env::args_os() + .nth(1) + .ok_or("usage: complete ")?; + let engine = Engine::load(model)?; + let completion = engine.complete( + "The capital of France is", + &SamplingParams::greedy().max_tokens(16), + )?; + println!("{}", completion.text); + Ok(()) +} diff --git a/vllm-cpp/examples/stream.rs b/vllm-cpp/examples/stream.rs new file mode 100644 index 0000000..b852a6c --- /dev/null +++ b/vllm-cpp/examples/stream.rs @@ -0,0 +1,21 @@ +use std::io::{self, Write}; + +use vllm_cpp::{Engine, SamplingParams, StreamControl}; + +fn main() -> Result<(), Box> { + let model = std::env::args_os() + .nth(1) + .ok_or("usage: stream ")?; + let engine = Engine::load(model)?; + engine.complete_stream( + "Write one short sentence about Rust:", + &SamplingParams::greedy().max_tokens(32), + |event| { + print!("{}", event.delta); + io::stdout().flush().expect("flush stdout"); + StreamControl::Continue + }, + )?; + println!(); + Ok(()) +} diff --git a/vllm-cpp/examples/structured.rs b/vllm-cpp/examples/structured.rs new file mode 100644 index 0000000..ec48207 --- /dev/null +++ b/vllm-cpp/examples/structured.rs @@ -0,0 +1,19 @@ +use vllm_cpp::{Engine, SamplingParams, StructuredOutput}; + +fn main() -> Result<(), Box> { + let model = std::env::args_os() + .nth(1) + .ok_or("usage: structured ")?; + let engine = Engine::load(model)?; + let completion = engine.complete( + "Choose exactly one color: red or blue. Answer:", + &SamplingParams::greedy() + .max_tokens(8) + .structured_output(StructuredOutput::Choice(vec![ + "red".to_owned(), + "blue".to_owned(), + ])), + )?; + println!("{}", completion.text); + Ok(()) +} diff --git a/vllm-cpp/src/callback.rs b/vllm-cpp/src/callback.rs new file mode 100644 index 0000000..64ce67c --- /dev/null +++ b/vllm-cpp/src/callback.rs @@ -0,0 +1,121 @@ +use std::any::Any; +use std::ffi::CStr; +use std::os::raw::{c_char, c_void}; +use std::panic::{catch_unwind, AssertUnwindSafe}; + +use crate::error::Error; + +/// Controls whether native streaming continues after a callback. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum StreamControl { + Continue, + Stop, +} + +/// One copied streaming delta. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct StreamEvent { + pub delta: String, + pub finished: bool, +} + +/// How a successful blocking stream ended. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct StreamOutcome { + pub stopped_by_callback: bool, +} + +pub(crate) struct CallbackState<'callback, F> { + callback: &'callback mut F, + stopped: bool, + error: Option, + panic: Option>, +} + +impl<'callback, F> CallbackState<'callback, F> { + pub(crate) fn new(callback: &'callback mut F) -> Self { + Self { + callback, + stopped: false, + error: None, + panic: None, + } + } + + pub(crate) const fn stopped(&self) -> bool { + self.stopped + } + + pub(crate) fn take_error(&mut self) -> Option { + self.error.take() + } + + pub(crate) fn take_panic(&mut self) -> Option> { + self.panic.take() + } + + fn apply_control(&mut self, control: StreamControl, finished: bool) -> bool { + match control { + StreamControl::Continue => true, + StreamControl::Stop => { + if !finished { + self.stopped = true; + } + false + } + } + } +} + +pub(crate) unsafe extern "C" fn callback_trampoline( + delta_text: *const c_char, + finished: bool, + user_data: *mut c_void, +) -> bool +where + F: FnMut(StreamEvent) -> StreamControl, +{ + // SAFETY: callers pass a stable pointer to CallbackState and the native + // blocking function cannot retain it after returning. + let state = unsafe { &mut *user_data.cast::>() }; + if delta_text.is_null() { + state.error = Some(Error::InvalidUtf8 { + field: "stream delta", + }); + return false; + } + // SAFETY: vllm.cpp promises a borrowed NUL-terminated string for the callback. + let delta = match unsafe { CStr::from_ptr(delta_text) }.to_str() { + Ok(delta) => delta.to_owned(), + Err(_) => { + state.error = Some(Error::InvalidUtf8 { + field: "stream delta", + }); + return false; + } + }; + let event = StreamEvent { delta, finished }; + match catch_unwind(AssertUnwindSafe(|| (state.callback)(event))) { + Ok(control) => state.apply_control(control, finished), + Err(payload) => { + state.panic = Some(payload); + false + } + } +} + +#[cfg(test)] +mod tests { + use super::{CallbackState, StreamControl, StreamEvent}; + + #[test] + fn only_nonterminal_stop_marks_callback_stop() { + let mut callback = |_: StreamEvent| StreamControl::Continue; + let mut state = CallbackState::new(&mut callback); + + assert!(!state.apply_control(StreamControl::Stop, true)); + assert!(!state.stopped()); + assert!(!state.apply_control(StreamControl::Stop, false)); + assert!(state.stopped()); + } +} diff --git a/vllm-cpp/src/engine.rs b/vllm-cpp/src/engine.rs new file mode 100644 index 0000000..cc5f084 --- /dev/null +++ b/vllm-cpp/src/engine.rs @@ -0,0 +1,477 @@ +use std::ffi::{CStr, CString}; +use std::mem::MaybeUninit; +use std::os::raw::c_char; +use std::path::{Path, PathBuf}; +use std::ptr::{self, NonNull}; + +use vllm_cpp_sys as ffi; + +use crate::callback::{ + callback_trampoline, CallbackState, StreamControl, StreamEvent, StreamOutcome, +}; +use crate::error::{invalid_configuration, status_result, Error}; +use crate::params::{SamplingParams, SchedulerPolicy, Toggle}; + +/// An owned vllm.cpp serving engine. +pub struct Engine { + raw: NonNull, +} + +impl std::fmt::Debug for Engine { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("Engine") + .field("raw", &self.raw) + .finish_non_exhaustive() + } +} + +/// Builder for one complete serving engine. +#[derive(Clone, Debug)] +pub struct EngineBuilder { + model_path: PathBuf, + tokenizer_config_path: Option, + block_size: Option, + num_blocks: Option, + max_model_len: Option, + max_num_seqs: Option, + tool_parser: Option, + reasoning_parser: Option, + speculative_config: Option, + prefix_caching: Toggle, + max_num_batched_tokens: Option, + scheduler: SchedulerPolicy, + kv_transfer_config: Option, + jump_forward: Toggle, +} + +/// Why native generation finished. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum FinishReason { + Stop, + Length, + Abort, + Error, + Repetition, + Unknown, + Other(String), +} + +/// A Rust-owned blocking completion result. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Completion { + pub text: String, + pub finish_reason: Option, + pub prompt_tokens: u32, + pub completion_tokens: u32, +} + +impl Engine { + /// Starts configuring an engine for a model directory or GGUF file. + pub fn builder(model_path: impl Into) -> EngineBuilder { + EngineBuilder::new(model_path) + } + + /// Loads an engine using native defaults. + pub fn load(model_path: impl Into) -> Result { + Self::builder(model_path).load() + } + + /// Runs one blocking text completion. + pub fn complete(&self, prompt: &str, params: &SamplingParams) -> Result { + let prompt = to_cstring(prompt, "prompt")?; + let params = params.marshal()?; + let mut raw = MaybeUninit::::uninit(); + // SAFETY: the engine is owned and live, all pointers remain valid for the + // call, and out storage is initialized by native code on success. + let status = unsafe { + ffi::vllm_complete( + self.raw.as_ptr(), + prompt.as_ptr(), + params.raw(), + raw.as_mut_ptr(), + ) + }; + status_result(status)?; + // SAFETY: VLLM_OK initializes every completion field. + let raw = unsafe { raw.assume_init() }; + let guard = CompletionGuard(raw); + completion_from_raw(&guard.0) + } + + /// Runs one blocking streaming text completion. + /// + /// A callback panic is resumed only after native code has aborted the request + /// and returned across the FFI boundary. + pub fn complete_stream( + &self, + prompt: &str, + params: &SamplingParams, + mut callback: F, + ) -> Result + where + F: FnMut(StreamEvent) -> StreamControl, + { + let prompt = to_cstring(prompt, "prompt")?; + let params = params.marshal()?; + let mut state = CallbackState::new(&mut callback); + // SAFETY: state has a stable stack address for this blocking call; the C + // API does not retain user_data after returning. + let status = unsafe { + ffi::vllm_complete_stream( + self.raw.as_ptr(), + prompt.as_ptr(), + params.raw(), + Some(callback_trampoline::), + ptr::from_mut(&mut state).cast(), + ) + }; + if let Some(payload) = state.take_panic() { + std::panic::resume_unwind(payload); + } + if let Some(error) = state.take_error() { + return Err(error); + } + status_result(status)?; + Ok(StreamOutcome { + stopped_by_callback: state.stopped(), + }) + } + + /// Runs one blocking OpenAI-style chat request and returns response JSON. + pub fn chat_json(&self, request_json: &str) -> Result { + let request = to_cstring(request_json, "chat request JSON")?; + let mut output: *mut c_char = ptr::null_mut(); + // SAFETY: the engine and request pointers are valid for the call and the + // returned string is released by NativeStringGuard. + let status = unsafe { ffi::vllm_chat(self.raw.as_ptr(), request.as_ptr(), &mut output) }; + status_result(status)?; + let output = NonNull::new(output).ok_or_else(|| Error::Runtime { + message: "vllm_chat succeeded without a response".to_owned(), + })?; + let guard = NativeStringGuard(output); + c_string_to_owned(guard.0.as_ptr(), "chat response") + } + + /// Runs one blocking OpenAI-style streaming chat request. + pub fn chat_stream_json( + &self, + request_json: &str, + mut callback: F, + ) -> Result + where + F: FnMut(StreamEvent) -> StreamControl, + { + let request = to_cstring(request_json, "chat request JSON")?; + let mut state = CallbackState::new(&mut callback); + // SAFETY: state remains valid for this blocking call and native code does + // not retain it after returning. + let status = unsafe { + ffi::vllm_chat_stream( + self.raw.as_ptr(), + request.as_ptr(), + Some(callback_trampoline::), + ptr::from_mut(&mut state).cast(), + ) + }; + if let Some(payload) = state.take_panic() { + std::panic::resume_unwind(payload); + } + if let Some(error) = state.take_error() { + return Err(error); + } + status_result(status)?; + Ok(StreamOutcome { + stopped_by_callback: state.stopped(), + }) + } + + #[cfg(feature = "serde")] + pub fn chat(&self, request: &serde_json::Value) -> Result { + let request_json = serde_json::to_string(request).map_err(|error| Error::Json { + context: "failed to serialize chat request", + message: error.to_string(), + })?; + let response = self.chat_json(&request_json)?; + serde_json::from_str(&response).map_err(|error| Error::Json { + context: "failed to parse chat response", + message: error.to_string(), + }) + } +} + +impl Drop for Engine { + fn drop(&mut self) { + // SAFETY: Engine exclusively owns this live handle and drops it once. + unsafe { ffi::vllm_engine_free(self.raw.as_ptr()) }; + } +} + +// Moving the sole owner is safe because the native handle has no thread affinity; +// ownership still keeps the handle live until all Rust access has ended. +unsafe impl Send for Engine {} + +impl EngineBuilder { + #[must_use] + pub fn new(model_path: impl Into) -> Self { + Self { + model_path: model_path.into(), + tokenizer_config_path: None, + block_size: None, + num_blocks: None, + max_model_len: None, + max_num_seqs: None, + tool_parser: None, + reasoning_parser: None, + speculative_config: None, + prefix_caching: Toggle::Default, + max_num_batched_tokens: None, + scheduler: SchedulerPolicy::Fcfs, + kv_transfer_config: None, + jump_forward: Toggle::Default, + } + } + + #[must_use] + pub fn tokenizer_config_path(mut self, value: impl Into) -> Self { + self.tokenizer_config_path = Some(value.into()); + self + } + + #[must_use] + pub fn block_size(mut self, value: u32) -> Self { + self.block_size = Some(value); + self + } + + #[must_use] + pub fn num_blocks(mut self, value: u32) -> Self { + self.num_blocks = Some(value); + self + } + + #[must_use] + pub fn max_model_len(mut self, value: u32) -> Self { + self.max_model_len = Some(value); + self + } + + #[must_use] + pub fn max_num_seqs(mut self, value: u32) -> Self { + self.max_num_seqs = Some(value); + self + } + + #[must_use] + pub fn tool_parser(mut self, value: impl Into) -> Self { + self.tool_parser = Some(value.into()); + self + } + + #[must_use] + pub fn reasoning_parser(mut self, value: impl Into) -> Self { + self.reasoning_parser = Some(value.into()); + self + } + + #[must_use] + pub fn speculative_config(mut self, value: impl Into) -> Self { + self.speculative_config = Some(value.into()); + self + } + + #[must_use] + pub fn prefix_caching(mut self, value: Toggle) -> Self { + self.prefix_caching = value; + self + } + + #[must_use] + pub fn max_num_batched_tokens(mut self, value: u32) -> Self { + self.max_num_batched_tokens = Some(value); + self + } + + #[must_use] + pub fn scheduler(mut self, value: SchedulerPolicy) -> Self { + self.scheduler = value; + self + } + + #[must_use] + pub fn kv_transfer_config(mut self, value: impl Into) -> Self { + self.kv_transfer_config = Some(value.into()); + self + } + + #[must_use] + pub fn jump_forward(mut self, value: Toggle) -> Self { + self.jump_forward = value; + self + } + + pub fn load(self) -> Result { + ensure_abi()?; + let model_path = path_to_cstring(&self.model_path, "model path")?; + let tokenizer_config_path = self + .tokenizer_config_path + .as_deref() + .map(|path| path_to_cstring(path, "tokenizer config path")) + .transpose()?; + let tool_parser = optional_cstring(self.tool_parser.as_deref(), "tool parser")?; + let reasoning_parser = + optional_cstring(self.reasoning_parser.as_deref(), "reasoning parser")?; + let speculative_config = optional_cstring( + self.speculative_config.as_deref(), + "speculative configuration", + )?; + let scheduling_policy = to_cstring(self.scheduler.as_str(), "scheduler policy")?; + let kv_transfer_config = optional_cstring( + self.kv_transfer_config.as_deref(), + "KV transfer configuration", + )?; + + // ABI equality is checked before this struct-returning native call. + let mut raw = unsafe { ffi::vllm_model_params_default() }; + raw.model_path = model_path.as_ptr(); + raw.tokenizer_config_path = optional_pointer(tokenizer_config_path.as_ref()); + raw.block_size = optional_u32_to_i32(self.block_size, "block_size")?; + raw.num_blocks = optional_u32_to_i32(self.num_blocks, "num_blocks")?; + raw.max_model_len = optional_u32_to_i32(self.max_model_len, "max_model_len")?; + raw.max_num_seqs = optional_u32_to_i32(self.max_num_seqs, "max_num_seqs")?; + raw.tool_parser = optional_pointer(tool_parser.as_ref()); + raw.reasoning_parser = optional_pointer(reasoning_parser.as_ref()); + raw.speculative_config = optional_pointer(speculative_config.as_ref()); + raw.enable_prefix_caching = self.prefix_caching.as_native(); + raw.max_num_batched_tokens = + optional_u32_to_i32(self.max_num_batched_tokens, "max_num_batched_tokens")?; + raw.scheduling_policy = scheduling_policy.as_ptr(); + raw.kv_transfer_config = optional_pointer(kv_transfer_config.as_ref()); + raw.enable_jump_forward = self.jump_forward.as_native(); + + let mut output = ptr::null_mut(); + // SAFETY: all string storage remains live for the call and output points + // to writable handle storage. + let status = unsafe { ffi::vllm_engine_load(&raw, &mut output) }; + status_result(status)?; + let raw = NonNull::new(output).ok_or_else(|| Error::ModelLoad { + message: "vllm_engine_load succeeded without a handle".to_owned(), + })?; + Ok(Engine { raw }) + } +} + +fn ensure_abi() -> Result<(), Error> { + // SAFETY: this base ABI function takes no pointers or versioned structs. + let actual = unsafe { ffi::vllm_abi_version() }; + let expected = ffi::VLLM_ABI_VERSION as i32; + if actual == expected { + Ok(()) + } else { + Err(Error::AbiMismatch { expected, actual }) + } +} + +fn completion_from_raw(raw: &ffi::vllm_completion) -> Result { + if raw.text.is_null() { + return Err(Error::Runtime { + message: "vllm_complete succeeded without text".to_owned(), + }); + } + let text = c_string_to_owned(raw.text, "completion text")?; + let finish_reason = if raw.finish_reason.is_null() { + None + } else { + Some(parse_finish_reason(c_string_to_owned( + raw.finish_reason, + "finish reason", + )?)) + }; + Ok(Completion { + text, + finish_reason, + prompt_tokens: count_to_u32(raw.prompt_tokens, "prompt token count")?, + completion_tokens: count_to_u32(raw.completion_tokens, "completion token count")?, + }) +} + +fn parse_finish_reason(value: String) -> FinishReason { + match value.as_str() { + "stop" => FinishReason::Stop, + "length" => FinishReason::Length, + "abort" => FinishReason::Abort, + "error" => FinishReason::Error, + "repetition" => FinishReason::Repetition, + "unknown" => FinishReason::Unknown, + _ => FinishReason::Other(value), + } +} + +fn count_to_u32(value: i32, field: &'static str) -> Result { + u32::try_from(value).map_err(|_| invalid_configuration(format!("native {field} was negative"))) +} + +fn optional_u32_to_i32(value: Option, field: &'static str) -> Result { + match value { + Some(0) | None => Ok(0), + Some(value) => i32::try_from(value) + .map_err(|_| invalid_configuration(format!("{field} exceeds native i32 range"))), + } +} + +fn optional_cstring(value: Option<&str>, field: &'static str) -> Result, Error> { + value.map(|value| to_cstring(value, field)).transpose() +} + +fn optional_pointer(value: Option<&CString>) -> *const c_char { + value.map_or(ptr::null(), |value| value.as_ptr()) +} + +fn to_cstring(value: &str, field: &'static str) -> Result { + CString::new(value).map_err(|_| Error::InteriorNul { field }) +} + +#[cfg(unix)] +fn path_to_cstring(path: &Path, field: &'static str) -> Result { + use std::os::unix::ffi::OsStrExt; + + CString::new(path.as_os_str().as_bytes()).map_err(|_| Error::InteriorNul { field }) +} + +#[cfg(not(unix))] +fn path_to_cstring(path: &Path, field: &'static str) -> Result { + path.to_str() + .ok_or(Error::PathEncoding) + .and_then(|value| to_cstring(value, field)) +} + +fn c_string_to_owned(pointer: *const c_char, field: &'static str) -> Result { + if pointer.is_null() { + return Err(Error::InvalidUtf8 { field }); + } + // SAFETY: callers pass a live native NUL-terminated string. + unsafe { CStr::from_ptr(pointer) } + .to_str() + .map(str::to_owned) + .map_err(|_| Error::InvalidUtf8 { field }) +} + +struct CompletionGuard(ffi::vllm_completion); + +impl Drop for CompletionGuard { + fn drop(&mut self) { + // SAFETY: the native function initialized this completion and the guard + // releases its owned members exactly once. + unsafe { ffi::vllm_completion_free(&mut self.0) }; + } +} + +struct NativeStringGuard(NonNull); + +impl Drop for NativeStringGuard { + fn drop(&mut self) { + // SAFETY: vllm_chat allocated this string and this guard owns it once. + unsafe { ffi::vllm_string_free(self.0.as_ptr()) }; + } +} diff --git a/vllm-cpp/src/error.rs b/vllm-cpp/src/error.rs new file mode 100644 index 0000000..c2694cf --- /dev/null +++ b/vllm-cpp/src/error.rs @@ -0,0 +1,95 @@ +use std::ffi::CStr; +use std::fmt; + +use vllm_cpp_sys as ffi; + +/// An error returned by the safe vllm.cpp wrapper. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum Error { + /// The loaded native library does not match the generated C ABI. + AbiMismatch { expected: i32, actual: i32 }, + /// Native code rejected caller input. + InvalidArgument { message: String }, + /// The model, tokenizer, configuration, or weights could not be loaded. + ModelLoad { message: String }, + /// Native generation failed at runtime. + Runtime { message: String }, + /// Native code reported an unclassified failure. + NativeUnknown { message: String }, + /// Native code returned a status unknown to these bindings. + UnknownStatus { status: u32, message: String }, + /// A value cannot cross the C boundary because it contains a NUL byte. + InteriorNul { field: &'static str }, + /// A platform path cannot be represented by the native UTF-8 API. + PathEncoding, + /// Native code returned bytes that are not valid UTF-8. + InvalidUtf8 { field: &'static str }, + /// A Rust-side parameter cannot be represented by the native API. + InvalidConfiguration { message: String }, + /// JSON serialization or parsing failed. + Json { + context: &'static str, + message: String, + }, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AbiMismatch { expected, actual } => { + write!( + f, + "vllm.cpp ABI mismatch: expected {expected}, found {actual}" + ) + } + Self::InvalidArgument { message } => write!(f, "invalid argument: {message}"), + Self::ModelLoad { message } => write!(f, "model load failed: {message}"), + Self::Runtime { message } => write!(f, "vllm.cpp runtime failure: {message}"), + Self::NativeUnknown { message } => write!(f, "unknown native failure: {message}"), + Self::UnknownStatus { status, message } => { + write!(f, "unknown native status {status}: {message}") + } + Self::InteriorNul { field } => write!(f, "{field} contains an interior NUL byte"), + Self::PathEncoding => write!(f, "path cannot be represented by the native API"), + Self::InvalidUtf8 { field } => write!(f, "native {field} is not valid UTF-8"), + Self::InvalidConfiguration { message } => { + write!(f, "invalid configuration: {message}") + } + Self::Json { context, message } => write!(f, "{context}: {message}"), + } + } +} + +impl std::error::Error for Error {} + +pub(crate) fn status_result(status: ffi::vllm_status) -> Result<(), Error> { + if status == ffi::vllm_status_VLLM_OK { + return Ok(()); + } + + // The native diagnostic is thread-local and valid only until the next C API + // call on this thread, so copy it before doing any other FFI work. + let message = unsafe { + let pointer = ffi::vllm_last_error(); + if pointer.is_null() { + String::new() + } else { + CStr::from_ptr(pointer).to_string_lossy().into_owned() + } + }; + let error = match status { + ffi::vllm_status_VLLM_ERR_INVALID_ARGUMENT => Error::InvalidArgument { message }, + ffi::vllm_status_VLLM_ERR_MODEL_LOAD => Error::ModelLoad { message }, + ffi::vllm_status_VLLM_ERR_RUNTIME => Error::Runtime { message }, + ffi::vllm_status_VLLM_ERR_UNKNOWN => Error::NativeUnknown { message }, + status => Error::UnknownStatus { status, message }, + }; + Err(error) +} + +pub(crate) fn invalid_configuration(message: impl Into) -> Error { + Error::InvalidConfiguration { + message: message.into(), + } +} diff --git a/vllm-cpp/src/lib.rs b/vllm-cpp/src/lib.rs index 8ccd04d..755c20a 100644 --- a/vllm-cpp/src/lib.rs +++ b/vllm-cpp/src/lib.rs @@ -1,9 +1,29 @@ -//! Rust bindings for vllm.cpp. +//! Safe Rust bindings for the stable vllm.cpp C API. //! -//! The high-level API will be added in follow-up work. This bootstrap crate establishes -//! the final workspace shape and verifies the linked C ABI. +//! The central [`Engine`] owns a complete native serving stack. Construct +//! request parameters in Rust, then use blocking completion or chat methods +//! without handling native pointers or free functions. -/// Returns the C ABI version reported by the linked vllm.cpp library. +mod callback; +mod engine; +mod error; +mod params; + +pub use callback::{StreamControl, StreamEvent, StreamOutcome}; +pub use engine::{Completion, Engine, EngineBuilder, FinishReason}; +pub use error::Error; +pub use params::{SamplingParams, SchedulerPolicy, StructuredOutput, Toggle}; + +/// Returns the compile-time C ABI expected by this crate. +#[must_use] +pub const fn expected_abi_version() -> i32 { + vllm_cpp_sys::VLLM_ABI_VERSION as i32 +} + +/// Returns the C ABI reported by the linked vllm.cpp library. +/// +/// Engine loading compares this value for exact equality before passing any +/// versioned native struct. #[must_use] pub fn abi_version() -> i32 { // SAFETY: this base ABI function takes no pointers and returns a plain i32. diff --git a/vllm-cpp/src/params.rs b/vllm-cpp/src/params.rs new file mode 100644 index 0000000..6154f1d --- /dev/null +++ b/vllm-cpp/src/params.rs @@ -0,0 +1,330 @@ +use std::ffi::CString; +use std::os::raw::c_char; +use std::ptr; + +use vllm_cpp_sys as ffi; + +use crate::error::{invalid_configuration, Error}; + +const NATIVE_DEFAULT_MAX_TOKENS: u32 = 16; + +/// Native scheduler admission order. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum SchedulerPolicy { + /// Process requests in arrival order. + #[default] + Fcfs, + /// Order requests by priority and then arrival time. + Priority, + /// Prefer requests sharing the longest cached prefix. + LongestPrefixMatch, +} + +impl SchedulerPolicy { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Fcfs => "fcfs", + Self::Priority => "priority", + Self::LongestPrefixMatch => "lpm", + } + } +} + +/// A native tri-state setting whose default is resolved by vllm.cpp. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum Toggle { + /// Let vllm.cpp resolve the model or environment default. + #[default] + Default, + /// Force the feature on. + On, + /// Force the feature off. + Off, +} + +impl Toggle { + pub(crate) const fn as_native(self) -> i32 { + match self { + Self::Default => 0, + Self::On => 1, + Self::Off => 2, + } + } +} + +/// One engine-side structured decoding constraint. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum StructuredOutput { + JsonSchema(String), + Regex(String), + Choice(Vec), + Grammar(String), + JsonObject, +} + +/// Owned sampling configuration for one generation request. +#[derive(Clone, Debug, PartialEq)] +pub struct SamplingParams { + temperature: f32, + top_p: f32, + top_k: i32, + min_p: f32, + max_tokens: Option, + seed: Option, + presence_penalty: f32, + frequency_penalty: f32, + repetition_penalty: f32, + min_tokens: u32, + ignore_eos: bool, + stop: Vec, + structured_output: Option, +} + +impl Default for SamplingParams { + fn default() -> Self { + Self { + temperature: 1.0, + top_p: 1.0, + top_k: 0, + min_p: 0.0, + max_tokens: Some(NATIVE_DEFAULT_MAX_TOKENS), + seed: None, + presence_penalty: 0.0, + frequency_penalty: 0.0, + repetition_penalty: 1.0, + min_tokens: 0, + ignore_eos: false, + stop: Vec::new(), + structured_output: None, + } + } +} + +impl SamplingParams { + /// Returns deterministic argmax sampling with native defaults otherwise. + #[must_use] + pub fn greedy() -> Self { + Self::default().temperature(0.0) + } + + #[must_use] + pub fn temperature(mut self, value: f32) -> Self { + self.temperature = value; + self + } + + #[must_use] + pub fn top_p(mut self, value: f32) -> Self { + self.top_p = value; + self + } + + #[must_use] + pub fn top_k(mut self, value: i32) -> Self { + self.top_k = value; + self + } + + #[must_use] + pub fn min_p(mut self, value: f32) -> Self { + self.min_p = value; + self + } + + #[must_use] + pub fn max_tokens(mut self, value: u32) -> Self { + self.max_tokens = Some(value); + self + } + + #[must_use] + pub fn unbounded(mut self) -> Self { + self.max_tokens = None; + self + } + + #[must_use] + pub fn seed(mut self, value: u64) -> Self { + self.seed = Some(value); + self + } + + #[must_use] + pub fn clear_seed(mut self) -> Self { + self.seed = None; + self + } + + #[must_use] + pub fn presence_penalty(mut self, value: f32) -> Self { + self.presence_penalty = value; + self + } + + #[must_use] + pub fn frequency_penalty(mut self, value: f32) -> Self { + self.frequency_penalty = value; + self + } + + #[must_use] + pub fn repetition_penalty(mut self, value: f32) -> Self { + self.repetition_penalty = value; + self + } + + #[must_use] + pub fn min_tokens(mut self, value: u32) -> Self { + self.min_tokens = value; + self + } + + #[must_use] + pub fn ignore_eos(mut self, value: bool) -> Self { + self.ignore_eos = value; + self + } + + #[must_use] + pub fn stop(mut self, value: impl Into) -> Self { + self.stop.push(value.into()); + self + } + + #[must_use] + pub fn stop_all(mut self, values: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.stop.extend(values.into_iter().map(Into::into)); + self + } + + #[must_use] + pub fn structured_output(mut self, value: StructuredOutput) -> Self { + self.structured_output = Some(value); + self + } + + pub(crate) fn marshal(&self) -> Result { + MarshaledSamplingParams::new(self) + } +} + +pub(crate) struct MarshaledSamplingParams { + raw: ffi::vllm_sampling_params, + _stop: Vec, + _stop_pointers: Vec<*const c_char>, + _structured_string: Option, + _choices: Vec, + _choice_pointers: Vec<*const c_char>, +} + +impl MarshaledSamplingParams { + fn new(params: &SamplingParams) -> Result { + // ABI equality is checked before this struct-returning call. + let mut raw = unsafe { ffi::vllm_sampling_params_default() }; + raw.temperature = params.temperature; + raw.top_p = params.top_p; + raw.top_k = params.top_k; + raw.min_p = params.min_p; + raw.max_tokens = optional_u32_to_i32(params.max_tokens, "max_tokens")?; + raw.seed = params.seed.unwrap_or(0); + raw.has_seed = i32::from(params.seed.is_some()); + raw.presence_penalty = params.presence_penalty; + raw.frequency_penalty = params.frequency_penalty; + raw.repetition_penalty = params.repetition_penalty; + raw.min_tokens = u32_to_i32(params.min_tokens, "min_tokens")?; + raw.ignore_eos = i32::from(params.ignore_eos); + + let stop = strings_to_cstrings(¶ms.stop, "stop string")?; + let stop_pointers = stop.iter().map(|value| value.as_ptr()).collect::>(); + raw.stop = pointer_or_null(&stop_pointers); + raw.n_stop = length_to_i32(stop_pointers.len(), "stop strings")?; + + let mut structured_string = None; + let mut choices = Vec::new(); + let mut choice_pointers = Vec::new(); + if let Some(structured) = ¶ms.structured_output { + match structured { + StructuredOutput::JsonSchema(value) => { + structured_string = Some(to_cstring(value, "JSON schema")?); + raw.structured_json = structured_string.as_ref().unwrap().as_ptr(); + } + StructuredOutput::Regex(value) => { + structured_string = Some(to_cstring(value, "structured regex")?); + raw.structured_regex = structured_string.as_ref().unwrap().as_ptr(); + } + StructuredOutput::Choice(values) => { + if values.is_empty() { + return Err(invalid_configuration( + "structured choices must contain at least one value", + )); + } + choices = strings_to_cstrings(values, "structured choice")?; + choice_pointers = choices + .iter() + .map(|value| value.as_ptr()) + .collect::>(); + raw.structured_choice = pointer_or_null(&choice_pointers); + raw.n_structured_choice = + length_to_i32(choice_pointers.len(), "structured choices")?; + } + StructuredOutput::Grammar(value) => { + structured_string = Some(to_cstring(value, "structured grammar")?); + raw.structured_grammar = structured_string.as_ref().unwrap().as_ptr(); + } + StructuredOutput::JsonObject => raw.structured_json_object = 1, + } + } + + Ok(Self { + raw, + _stop: stop, + _stop_pointers: stop_pointers, + _structured_string: structured_string, + _choices: choices, + _choice_pointers: choice_pointers, + }) + } + + pub(crate) const fn raw(&self) -> &ffi::vllm_sampling_params { + &self.raw + } +} + +pub(crate) fn to_cstring(value: &str, field: &'static str) -> Result { + CString::new(value).map_err(|_| Error::InteriorNul { field }) +} + +fn strings_to_cstrings(values: &[String], field: &'static str) -> Result, Error> { + values + .iter() + .map(|value| to_cstring(value, field)) + .collect() +} + +fn pointer_or_null(values: &[*const c_char]) -> *const *const c_char { + if values.is_empty() { + ptr::null() + } else { + values.as_ptr() + } +} + +fn optional_u32_to_i32(value: Option, field: &'static str) -> Result { + match value { + Some(0) | None => Ok(0), + Some(value) => u32_to_i32(value, field), + } +} + +fn u32_to_i32(value: u32, field: &'static str) -> Result { + i32::try_from(value) + .map_err(|_| invalid_configuration(format!("{field} exceeds native i32 range"))) +} + +fn length_to_i32(value: usize, field: &'static str) -> Result { + i32::try_from(value).map_err(|_| invalid_configuration(format!("too many {field}"))) +} diff --git a/vllm-cpp/tests/qwen3.rs b/vllm-cpp/tests/qwen3.rs new file mode 100644 index 0000000..5e6bee0 --- /dev/null +++ b/vllm-cpp/tests/qwen3.rs @@ -0,0 +1,205 @@ +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock}; + +use vllm_cpp::{Engine, FinishReason, SamplingParams, StreamControl, StructuredOutput}; + +const REQUIRED_MODEL_FILES: [&str; 4] = [ + "model.safetensors", + "config.json", + "tokenizer.json", + "tokenizer_config.json", +]; + +fn model_path() -> Option { + let path = std::env::var_os("VLLM_CPP_TEST_MODEL").map(PathBuf::from)?; + let missing = REQUIRED_MODEL_FILES + .iter() + .filter(|file| !path.join(file).is_file()) + .copied() + .collect::>(); + assert!( + missing.is_empty(), + "VLLM_CPP_TEST_MODEL fixture is incomplete at {}: missing {}", + path.display(), + missing.join(", ") + ); + Some(path) +} + +fn with_engine(test: impl FnOnce(&Engine, &Path)) { + let Some(path) = model_path() else { + eprintln!("skipping model test; set VLLM_CPP_TEST_MODEL with `just setup-test-model`"); + return; + }; + static ENGINE: OnceLock> = OnceLock::new(); + let engine = ENGINE.get_or_init(|| { + Mutex::new( + Engine::builder(&path) + .num_blocks(64) + .max_model_len(256) + .max_num_seqs(2) + .max_num_batched_tokens(256) + .load() + .expect("load Qwen3-0.6B"), + ) + }); + let engine = engine.lock().expect("model test engine lock"); + test(&engine, &path); +} + +#[test] +fn greedy_completion_and_streaming_match() { + with_engine(|engine, _| { + let params = SamplingParams::greedy().max_tokens(8); + let completion = engine + .complete("The capital of France is", ¶ms) + .expect("blocking completion"); + assert!(!completion.text.is_empty()); + assert_eq!(completion.completion_tokens, 8); + assert_eq!(completion.finish_reason, Some(FinishReason::Length)); + + let mut streamed = String::new(); + let outcome = engine + .complete_stream("The capital of France is", ¶ms, |event| { + streamed.push_str(&event.delta); + StreamControl::Continue + }) + .expect("streaming completion"); + assert!(!outcome.stopped_by_callback); + assert_eq!(streamed, completion.text); + }); +} + +#[test] +fn seeded_sampling_is_repeatable() { + with_engine(|engine, _| { + let params = SamplingParams::default() + .temperature(0.8) + .seed(42) + .max_tokens(8); + let first = engine + .complete("A surprising fact about rust is", ¶ms) + .expect("first seeded completion"); + let second = engine + .complete("A surprising fact about rust is", ¶ms) + .expect("second seeded completion"); + assert_eq!(first, second); + }); +} + +#[test] +fn early_stop_and_callback_panic_leave_engine_reusable() { + with_engine(|engine, _| { + let params = SamplingParams::greedy().max_tokens(8); + let outcome = engine + .complete_stream("Count from one to ten:", ¶ms, |_| StreamControl::Stop) + .expect("early stop"); + assert!(outcome.stopped_by_callback); + + let panic = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = engine.complete_stream("Say hello", ¶ms, |_| { + panic!("intentional callback panic") + }); + })); + assert!(panic.is_err()); + + let completion = engine + .complete("Say hello", &SamplingParams::greedy().max_tokens(2)) + .expect("engine remains reusable"); + assert!(!completion.text.is_empty()); + }); +} + +#[test] +fn structured_choice_is_enforced() { + with_engine(|engine, _| { + let params = + SamplingParams::greedy() + .max_tokens(8) + .structured_output(StructuredOutput::Choice(vec![ + "red".to_owned(), + "blue".to_owned(), + ])); + let completion = engine + .complete("Choose exactly one color: red or blue. Answer:", ¶ms) + .expect("structured completion"); + assert!( + completion.text.trim() == "red" || completion.text.trim() == "blue", + "unexpected choice: {:?}", + completion.text + ); + }); +} + +#[test] +fn terminal_stop_is_natural_finish_for_completion_and_chat() { + with_engine(|engine, _| { + let mut completion_finished = false; + let completion_outcome = engine + .complete_stream( + "Reply with hello.", + &SamplingParams::greedy().max_tokens(4), + |event| { + completion_finished |= event.finished; + if event.finished { + StreamControl::Stop + } else { + StreamControl::Continue + } + }, + ) + .expect("terminal completion stop"); + assert!(completion_finished); + assert!(!completion_outcome.stopped_by_callback); + + let request = r#"{ + "messages":[{"role":"user","content":"Reply with hello."}], + "temperature":0, + "max_tokens":4 + }"#; + let mut chat_finished = false; + let chat_outcome = engine + .chat_stream_json(request, |event| { + chat_finished |= event.finished; + if event.finished { + StreamControl::Stop + } else { + StreamControl::Continue + } + }) + .expect("terminal chat stop"); + assert!(chat_finished); + assert!(!chat_outcome.stopped_by_callback); + }); +} + +#[test] +fn blocking_and_streaming_chat_return_json() { + with_engine(|engine, _| { + let request = r#"{ + "messages":[{"role":"user","content":"Reply with hello."}], + "temperature":0, + "max_tokens":4 + }"#; + let response = engine.chat_json(request).expect("blocking chat"); + let json: serde_json::Value = serde_json::from_str(&response).expect("valid response JSON"); + assert_eq!(json["object"], "chat.completion"); + assert!(json["choices"] + .as_array() + .is_some_and(|choices| !choices.is_empty())); + + let mut chunks = Vec::new(); + let outcome = engine + .chat_stream_json(request, |event| { + if !event.finished { + let chunk: serde_json::Value = + serde_json::from_str(&event.delta).expect("valid chunk JSON"); + chunks.push(chunk); + } + StreamControl::Continue + }) + .expect("streaming chat"); + assert!(!outcome.stopped_by_callback); + assert!(!chunks.is_empty()); + }); +} diff --git a/vllm-cpp/tests/safe_api.rs b/vllm-cpp/tests/safe_api.rs new file mode 100644 index 0000000..8ce090f --- /dev/null +++ b/vllm-cpp/tests/safe_api.rs @@ -0,0 +1,58 @@ +use vllm_cpp::{Engine, Error, SchedulerPolicy, Toggle}; + +fn missing_model() -> &'static str { + "/nonexistent/vllm-cpp-rs-safe-api-model" +} + +#[test] +fn reports_expected_abi() { + assert_eq!(vllm_cpp::expected_abi_version(), 10); + assert_eq!(vllm_cpp::abi_version(), 10); +} + +#[test] +fn missing_model_is_typed() { + let error = Engine::load(missing_model()).unwrap_err(); + assert!(matches!(error, Error::ModelLoad { .. }), "{error:?}"); + assert!(!error.to_string().is_empty()); +} + +#[test] +fn malformed_engine_json_is_invalid_argument_before_loading() { + let error = Engine::builder(missing_model()) + .speculative_config("{") + .load() + .unwrap_err(); + assert!(matches!(error, Error::InvalidArgument { .. }), "{error:?}"); +} + +#[test] +fn interior_nul_fails_before_ffi() { + let error = Engine::builder("bad\0model").load().unwrap_err(); + assert_eq!( + error, + Error::InteriorNul { + field: "model path" + } + ); +} + +#[test] +fn engine_builder_accepts_all_safe_options() { + let error = Engine::builder(missing_model()) + .tokenizer_config_path("/nonexistent/tokenizer_config.json") + .block_size(16) + .num_blocks(32) + .max_model_len(128) + .max_num_seqs(2) + .tool_parser("hermes") + .reasoning_parser("none") + .prefix_caching(Toggle::Off) + .max_num_batched_tokens(128) + .scheduler(SchedulerPolicy::LongestPrefixMatch) + .kv_transfer_config("") + .jump_forward(Toggle::Off) + .load() + .unwrap_err(); + assert!(matches!(error, Error::ModelLoad { .. }), "{error:?}"); +}