Skip to content

feat: add safe blocking inference API #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zatevakhin merged 1 commit into from
Aug 21, 2026
Merged

feat: add safe blocking inference API #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

162 changes: 161 additions & 1 deletion Justfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ set shell := ["bash", "-euo", "pipefail", "-c"]

root := justfile_directory()
bindings_file := root + "/vllm-cpp-sys/src/bindings.rs"
model_revision := "c1899de289a04d12100db370d81485cdf75e47ca"

# Maintainer workflows require Just 1.40 or newer.

Expand Down Expand Up @@ -255,6 +256,23 @@ link-modes:
"expected VLLM_CPP_LIB_DIR override artifact libvllm.a at $prefix/lib64/libvllm.a" \
"$system_override_static_log"

system_sanitize_target="$work/system-sanitize"
system_sanitize_log="$work/system-sanitize.log"
echo '==> system sanitizer rejection'
if VLLM_CPP_ROOT="$prefix" \
VLLM_CPP_BLAKE3_LIB_DIR="$prefix/blake3-lib" \
VLLM_CPP_SANITIZE=address \
CARGO_TARGET_DIR="$system_sanitize_target" \
cargo test --locked -vv -p vllm-cpp-sys --release --tests \
--no-default-features --features system 2>&1 \
| tee "$system_sanitize_log"; then
echo 'expected system mode to reject VLLM_CPP_SANITIZE' >&2
exit 1
fi
grep -Fq \
'VLLM_CPP_SANITIZE is supported only for bundled builds' \
"$system_sanitize_log"

system_dynamic_target="$work/system-dynamic"
system_dynamic_log="$work/system-dynamic.log"
echo '==> system dynamic'
Expand Down Expand Up @@ -475,6 +493,144 @@ package-test:
cargo run --locked --release --offline
)

safe_version=$(cargo metadata --locked --offline --no-deps --format-version 1 \
| jq -er '[.packages[] | select(.name == "vllm-cpp") | .version] | if length == 1 then .[0] else error("expected exactly one vllm-cpp package") end')
safe_package_args=()
if [[ -n $(git status --porcelain=v1 --untracked-files=all -- vllm-cpp vllm-cpp-sys) ]]; then
safe_package_args+=(--allow-dirty)
fi
cargo package --workspace --locked --offline --no-verify "${safe_package_args[@]}"
safe_package_file="$package_target/package/vllm-cpp-$safe_version.crate"
tar -xzf "$safe_package_file" -C "$temp"
safe_root="$temp/vllm-cpp-$safe_version"
safe_manifest="$safe_root/Cargo.toml"
sed -i \
"/\[dependencies.vllm-cpp-sys\]/a path = \"$package_root\"" \
"$safe_manifest"
(
cd "$safe_root"
CARGO_NET_OFFLINE=true cargo generate-lockfile --offline
)

package_listing="$temp/safe-package.list"
tar -tzf "$safe_package_file" > "$package_listing"
if grep -Eq '(^|/)(target|model\.safetensors)(/|$)' "$package_listing"; then
echo 'local build output or model fixture leaked into the safe crate package' >&2
exit 1
fi
if grep -RIlF "$repo_root" "$safe_root" --exclude=Cargo.toml >/dev/null; then
echo 'local repository path leaked into the safe crate package' >&2
exit 1
fi
[[ ! -e $safe_root/target ]] || {
echo 'local build output leaked into the safe crate package' >&2
exit 1
}
[[ ! -e $safe_root/model.safetensors ]] || {
echo 'model fixture leaked into the safe crate package' >&2
exit 1
}
(
cd "$safe_root"
env -u VLLM_CPP_TEST_MODEL \
CARGO_NET_OFFLINE=true CARGO_TARGET_DIR="$temp_target" \
cargo test --locked --release --offline --features bundled,serde
)

# Download and verify the pinned Qwen3-0.6B model fixture.
setup-test-model destination=env_var_or_default("VLLM_CPP_TEST_MODEL", env_var_or_default("XDG_CACHE_HOME", env_var("HOME") + "/.cache") + "/vllm-cpp-rs/Qwen3-0.6B-" + model_revision):
#!/usr/bin/env bash
set -euo pipefail
revision={{ quote(model_revision) }}
base="https://huggingface.co/Qwen/Qwen3-0.6B/resolve/$revision"
destination={{ quote(destination) }}
mkdir -p "$destination"

files=(
LICENSE
config.json
generation_config.json
merges.txt
model.safetensors
tokenizer.json
tokenizer_config.json
vocab.json
)
for file in "${files[@]}"; do
if [[ ! -f $destination/$file ]]; then
echo "downloading $file" >&2
curl --fail --location --retry 3 --continue-at - \
"$base/$file" --output "$destination/$file"
fi
done

cat > "$destination/SHA256SUMS.expected" <<'EOF'
832dd9e00a68dd83b3c3fb9f5588dad7dcf337a0db50f7d9483f310cd292e92e LICENSE
660db3b73d788119c04535e48cf9be5f55bc3100841a718637ae695b442f27dd config.json
2325da0f15bb848e018c5ae071b7943332e9f871d6b60e2ed22ca97d4cb993d2 generation_config.json
8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 merges.txt
f47f71177f32bcd101b7573ec9171e6a57f4f4d31148d38e382306f42996874b model.safetensors
aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4 tokenizer.json
d5d09f07b48c3086c508b30d1c9114bd1189145b74e982a265350c923acd8101 tokenizer_config.json
ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 vocab.json
EOF
(cd "$destination" && sha256sum --check SHA256SUMS.expected) >&2
printf '%s\n' "$destination"

# Run the blocking safe API and Qwen model suites with ASan, UBSan, and leak detection.
sanitizers model=env_var_or_default("VLLM_CPP_TEST_MODEL", ""):
#!/usr/bin/env bash
set -euo pipefail
model={{ quote(model) }}
if [[ -z $model ]]; then
echo 'set VLLM_CPP_TEST_MODEL or pass model=<verified-model-directory>' >&2
exit 1
fi
required_model_files=(
model.safetensors
config.json
tokenizer.json
tokenizer_config.json
)
missing=()
for file in "${required_model_files[@]}"; do
[[ -f $model/$file ]] || missing+=("$file")
done
if ((${#missing[@]})); then
printf 'model fixture is incomplete at %s; missing:' "$model" >&2
printf ' %s' "${missing[@]}" >&2
printf '\n' >&2
exit 1
fi
cd {{ quote(root) }}
export VLLM_CPP_TEST_MODEL="$model"
export VLLM_CPP_SANITIZE=address,undefined
export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-{{ quote(root + "/target/sanitize") }}}

cargo test --locked -p vllm-cpp --test safe_api --test qwen3 --no-run

asan=$(gcc -print-file-name=libasan.so)
ubsan=$(gcc -print-file-name=libubsan.so)
if [[ ! -f $asan || ! -f $ubsan ]]; then
echo 'GCC sanitizer runtimes are unavailable' >&2
exit 1
fi
export LD_PRELOAD="$asan:$ubsan${LD_PRELOAD:+:$LD_PRELOAD}"
export ASAN_OPTIONS=${ASAN_OPTIONS:-detect_leaks=1:halt_on_error=1}
export UBSAN_OPTIONS=${UBSAN_OPTIONS:-halt_on_error=1:print_stacktrace=1}
export VT_POOL_BYPASS=1

for pattern in safe_api qwen3; do
binary=$(find "$CARGO_TARGET_DIR/debug/deps" -maxdepth 1 -type f \
-name "$pattern-*" -executable -printf '%T@ %p\n' \
| sort -n | tail -1 | cut -d' ' -f2-) || true
if [[ -z $binary ]]; then
echo "could not find $pattern test binary" >&2
exit 1
fi
"$binary" --test-threads=1
done

# Check Just and Rust formatting.
fmt-check:
just --unstable --justfile {{ quote(root + "/Justfile") }} --fmt --check
Expand All @@ -484,5 +640,9 @@ fmt-check:
lint:
cargo clippy --locked --workspace --all-targets -- -D warnings

# Build API documentation with warnings denied.
docs:
RUSTDOCFLAGS="-D warnings" cargo doc --locked --workspace --no-deps --features vllm-cpp/serde

# Run the complete maintainer gate serially; do not pass --jobs.
ci: fmt-check lint sys link-modes package-test
ci: fmt-check lint docs sys link-modes package-test
Loading
Loading