From 4a43d3a732901b9445327b336414f6e8da6e999a Mon Sep 17 00:00:00 2001 From: Ivan Zatevakhin Date: Mon, 17 Aug 2026 23:46:35 +0100 Subject: [PATCH] feat: add hugging face loading and interactive chat add cached gguf and safetensors resolution with optional revisions, and share local or hub model loading across the examples. expand the chat example with streaming, history, sampling controls, and recoverable per-turn errors. --- CHANGELOG.md | 3 + Cargo.lock | 1349 +++++++++++++++++++++++++++++-- Justfile | 19 +- README.md | 12 +- vllm-cpp/Cargo.toml | 7 +- vllm-cpp/README.md | 27 +- vllm-cpp/examples/README.md | 80 +- vllm-cpp/examples/chat.rs | 1059 +++++++++++++++++++++++- vllm-cpp/examples/common/mod.rs | 254 ++++++ vllm-cpp/examples/complete.rs | 6 +- vllm-cpp/examples/concurrent.rs | 6 +- vllm-cpp/examples/stream.rs | 6 +- vllm-cpp/examples/structured.rs | 25 +- vllm-cpp/src/error.rs | 35 + vllm-cpp/src/hf.rs | 1022 +++++++++++++++++++++++ vllm-cpp/src/lib.rs | 15 +- vllm-cpp/tests/qwen3.rs | 38 + vllm-cpp/tests/safe_api.rs | 14 +- 18 files changed, 3848 insertions(+), 129 deletions(-) create mode 100644 vllm-cpp/examples/common/mod.rs create mode 100644 vllm-cpp/src/hf.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 40651c9..31be1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ All notable changes to this project will be documented in this file. - Checked-in raw Rust declarations for the 19-symbol stable vllm.cpp C API at ABI version 10, with header, symbol, layout, and runtime conformance checks. - A safe API for model loading, blocking completion and streaming, raw-JSON and optional serde chat, structured output, owned sampling parameters, and concurrent request submission, cancellation, waiting, and diagnostics. +- An always-available synchronous `hf-hub` resolver for standalone GGUF files and runtime-complete sparse Safetensors snapshots, defaulting to the Hub's mutable `main` revision with an explicit branch/tag/commit override, cache/token/progress/offline controls, and no async runtime. +- Consistent local, Hugging Face GGUF, and Hugging Face Safetensors model-source arguments across every runnable example, with cache reuse and optional revisions; plus a weather extraction example and model-backed test using JSON-Schema structured output. +- A Clap-based interactive `chat` example with prompt/file startup input, retained system/user/assistant history, supported sampling controls, default streaming or blocking output, and shared local/Hugging Face resolution. - RAII ownership for native engines, requests, completions, and strings, including callback panic containment and callback-thread-safe deferred request cleanup. - Linux x86_64 CPU builds for bundled and system libraries with static or dynamic linking. - Experimental bundled Linux x86_64/aarch64 build integration for CUDA, external CUTLASS, Triton AOT, and Vulkan. diff --git a/Cargo.lock b/Cargo.lock index d7e4e88..06171a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,86 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.4.0" @@ -12,6 +92,52 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -22,126 +148,1233 @@ dependencies = [ ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "colorchoice" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "itoa" -version = "1.0.18" +name = "console" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] [[package]] -name = "memchr" -version = "2.8.3" +name = "cookie" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] [[package]] -name = "proc-macro2" -version = "1.0.107" +name = "cookie_store" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" dependencies = [ - "unicode-ident", + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", ] [[package]] -name = "quote" -version = "1.0.47" +name = "crc32fast" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "proc-macro2", + "cfg-if", ] [[package]] -name = "serde" -version = "1.0.229" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "serde_core", + "powerfmt", ] [[package]] -name = "serde_core" -version = "1.0.229" +name = "dirs" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "serde_derive", + "dirs-sys", ] [[package]] -name = "serde_derive" -version = "1.0.229" +name = "dirs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] -name = "serde_json" -version = "1.0.151" +name = "document-features" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ - "itoa", - "memchr", + "litrs", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hf-hub" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +dependencies = [ + "dirs", + "http", + "indicatif", + "libc", + "log", + "rand", "serde", - "serde_core", - "zmij", + "serde_json", + "thiserror", + "ureq", + "windows-sys 0.61.2", ] [[package]] -name = "shlex" -version = "2.0.1" +name = "http" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] [[package]] -name = "static_assertions" +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] [[package]] -name = "syn" -version = "3.0.3" +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "icu_normalizer", + "icu_properties", ] [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "indexmap" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] [[package]] -name = "vllm-cpp" -version = "0.1.0" +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ - "serde_json", - "static_assertions", - "vllm-cpp-sys", + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", ] [[package]] -name = "vllm-cpp-sys" -version = "0.1.0" +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ - "cmake", + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vllm-cpp" +version = "0.1.0" +dependencies = [ + "clap", + "hf-hub", + "serde_json", + "static_assertions", + "vllm-cpp-sys", +] + +[[package]] +name = "vllm-cpp-sys" +version = "0.1.0" +dependencies = [ + "cmake", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] diff --git a/Justfile b/Justfile index de2666c..08af319 100644 --- a/Justfile +++ b/Justfile @@ -384,6 +384,9 @@ package-test: and (map(select(.name == "vllm-cpp" and .documentation == "https://docs.rs/vllm-cpp")) | length == 1) and (map(select(.name == "vllm-cpp-sys" and .documentation == "https://docs.rs/vllm-cpp-sys")) | length == 1) and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "vllm-cpp-sys" and .req == ("=" + $version) and .uses_default_features == false)) | length == 1) + and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "hf-hub" and .req == "^0.5.0" and .optional == false and .uses_default_features == false and .features == ["ureq"])) | length == 1) + and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "serde_json" and .optional == false and .kind == null)) | length == 1) + and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "clap" and .req == "=4.6.1" and .kind == "dev" and .optional == false and .features == ["derive"])) | length == 1) ' "$metadata" >/dev/null sys_list="$temp/vllm-cpp-sys.list" @@ -485,6 +488,7 @@ package-test: README.md \ examples/README.md \ examples/chat.rs \ + examples/common/mod.rs \ examples/complete.rs \ examples/concurrent.rs \ examples/stream.rs \ @@ -492,6 +496,7 @@ package-test: src/callback.rs \ src/engine.rs \ src/error.rs \ + src/hf.rs \ src/lib.rs \ src/params.rs \ src/request.rs \ @@ -727,8 +732,11 @@ package-test: and .[0].license == "MIT OR Apache-2.0" and .[0].rust_version == "1.85" and .[0].features.default == ["bundled"] - and .[0].features.serde == ["dep:serde_json"] + and .[0].features.serde == [] and (.[0].dependencies | map(select(.name == "vllm-cpp-sys" and .req == ("=" + $version) and .uses_default_features == false)) | length == 1) + and (.[0].dependencies | map(select(.name == "hf-hub" and .req == "^0.5.0" and .optional == false and .uses_default_features == false and .features == ["ureq"])) | length == 1) + and (.[0].dependencies | map(select(.name == "serde_json" and .optional == false and .kind == null)) | length == 1) + and (.[0].dependencies | map(select(.name == "clap" and .req == "=4.6.1" and .kind == "dev" and .optional == false and .features == ["derive"])) | length == 1) ' <(cargo metadata --manifest-path "$safe_root/Cargo.toml" \ --locked --offline --no-deps --format-version 1) >/dev/null @@ -748,12 +756,19 @@ package-test: vllm-cpp-sys = { path = "$sys_root" } EOF cat > "$safe_consumer/src/main.rs" <<'EOF' - use vllm_cpp::{abi_version, expected_abi_version, Engine, Error, SamplingParams}; + use vllm_cpp::{ + abi_version, expected_abi_version, Engine, Error, HuggingFaceModel, SamplingParams, + }; fn main() { assert_eq!(expected_abi_version(), 10); assert_eq!(abi_version(), 10); let _params = SamplingParams::greedy().max_tokens(1); + let resolver = HuggingFaceModel::gguf("owner/model", "model.gguf") + .revision("revision") + .cache_dir("/nonexistent/vllm-cpp-rs-safe-package-hf-cache") + .offline(true); + assert!(resolver.resolve().is_err()); assert!(matches!( Engine::load("/nonexistent/vllm-cpp-rs-safe-package-smoke"), Err(Error::ModelLoad { .. }) diff --git a/README.md b/README.md index 1592fe7..995e1d9 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Rust bindings for [vllm.cpp](https://github.com/mudler/vllm.cpp), organized as: ## Status -The safe crate provides a cloneable engine API for model loading, blocking completion and streaming, non-blocking concurrent requests, structured output, and raw-JSON chat. An optional `serde` feature adds `serde_json::Value` chat helpers. The sys crate provides checked-in generated FFI declarations with C/Rust layout checks and coverage for all 19 exported C symbols. +The safe crate provides a cloneable engine API for local model loading, blocking completion and streaming, non-blocking concurrent requests, structured output, and raw-JSON chat. It also provides an always-available synchronous Hugging Face resolver for standalone GGUF files and runtime-complete sparse Safetensors snapshots, plus a Clap-based interactive chat example using those APIs. An optional `serde` feature adds `serde_json::Value` chat helpers. The sys crate provides checked-in generated FFI declarations with C/Rust layout checks and coverage for all 19 exported C symbols. Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. Bundled CPU builds also target Linux aarch64 and Apple ARM64. Experimental bundled builds expose Linux x86_64/aarch64 build configuration for CUDA, external CUTLASS, Triton AOT, and Vulkan, plus Apple ARM64 Metal and external MLX configuration. Accelerator features are build integration surfaces, not runtime-support claims. vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. @@ -41,7 +41,9 @@ git submodule update --init --recursive ## Safe API -The packaged [`vllm-cpp` guide](vllm-cpp/README.md) covers model requirements, safe ownership, callbacks, concurrency, features, link modes, and deployment. The [`vllm-cpp-sys` guide](vllm-cpp-sys/README.md) documents the raw ABI and native build boundary. +The packaged [`vllm-cpp` guide](vllm-cpp/README.md) covers local and Hugging Face model resolution, safe ownership, callbacks, concurrency, features, link modes, and deployment. The [`vllm-cpp-sys` guide](vllm-cpp-sys/README.md) documents the raw ABI and native build boundary. + +`Engine::load` accepts a native-compatible model directory or standalone GGUF. `HuggingFaceModel` synchronously resolves into the normal Hugging Face cache before engine construction, defaulting to the Hub's mutable `main` revision; `.revision(...)` can pin a branch, tag, or commit. GGUF mode selects one safe root file. Safetensors mode pins downloads to repository metadata's commit SHA and retrieves only native runtime requirements: root configuration/tokenizer files and either unsharded weights or an index plus all root shards. Every runnable example accepts a bare or explicit local path and both Hub artifact forms with optional `--revision`. Cached downloads are reused. Retrieval does not prove model/backend compatibility. `EngineBuilder` owns model settings and converts them to temporary C strings only for the load call. `SamplingParams` owns stop strings and structured constraints. Completion and chat strings are copied into Rust values before the matching native free function runs. @@ -49,7 +51,7 @@ The packaged [`vllm-cpp` guide](vllm-cpp/README.md) covers model requirements, s All streaming callbacks receive copied UTF-8 deltas. Blocking callbacks may borrow stack data; their panics are caught before the C boundary and resumed only after the native call returns. Asynchronous callbacks must be `Send + 'static`, run on a native delivery thread, and report panic as `Error::CallbackPanicked` from `wait`. Waiting for or freeing a request from its own callback thread is prohibited by ABI v10: `wait` returns `Error::RequestCallbackThread`, while drop transfers cleanup to a prestarted reaper that owns the request, callback, and engine until native free/cancel/join completes. Chat methods accept raw OpenAI-compatible request JSON; enable `serde` for `serde_json::Value` request and response helpers. -See [the examples guide](vllm-cpp/examples/README.md) for ordinary Linux and optional Nix setup and commands for every example. Release-facing changes are recorded in the [changelog](CHANGELOG.md), and maintainers use the manual [release process](RELEASING.md). +See [the examples guide](vllm-cpp/examples/README.md) for ordinary Linux and optional Nix setup, commands for every example, and the interactive chat CLI's local/Hub model forms and generation options. Release-facing changes are recorded in the [changelog](CHANGELOG.md), and maintainers use the manual [release process](RELEASING.md). ## Build and Test @@ -64,7 +66,7 @@ just ci Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The default bundled build remains deterministic and CPU-only: native tests, examples, the HTTP server, CUDA, Metal, MLX, Vulkan, Triton, and CUTLASS fetching are disabled explicitly. Use `nix develop .#msrv -c just msrv` for the exact local Rust 1.85.0 policy check; the manual `platforms` workflow runs the same exact toolchain policy. -`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. +`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. The high-level crate's required `hf-hub` 0.5 dependency uses only its synchronous `ureq` feature, without Tokio or another async runtime. Library download progress is disabled by default. ## Experimental Backend Builds @@ -103,7 +105,7 @@ Compilation does not establish runtime correctness. Known native evidence blocke ## Test Model and Sanitizers -Model-backed tests use Apache-2.0 `Qwen/Qwen3-0.6B` at pinned revision `c1899de289a04d12100db370d81485cdf75e47ca`. Download or reuse the cache and verify every file, then run exactly 14 blocking and request-lifecycle model tests serially: +Model-backed tests use Apache-2.0 `Qwen/Qwen3-0.6B` at pinned revision `c1899de289a04d12100db370d81485cdf75e47ca`. Download or reuse the cache and verify every file, then run exactly 15 blocking and request-lifecycle model tests serially, including choice and JSON-Schema structured-output enforcement: ```console model=$(just setup-test-model) diff --git a/vllm-cpp/Cargo.toml b/vllm-cpp/Cargo.toml index 6e9e481..e9d0754 100644 --- a/vllm-cpp/Cargo.toml +++ b/vllm-cpp/Cargo.toml @@ -22,16 +22,17 @@ triton-aot = ["cuda", "vllm-cpp-sys/triton-aot"] vulkan = ["vllm-cpp-sys/vulkan"] metal = ["vllm-cpp-sys/metal"] mlx = ["metal", "vllm-cpp-sys/mlx"] -serde = ["dep:serde_json"] +serde = [] [package.metadata.docs.rs] features = ["serde"] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde_json = { version = "1.0.149", optional = true } +hf-hub = { version = "0.5.0", default-features = false, features = ["ureq"] } +serde_json = "1.0.149" vllm-cpp-sys = { workspace = true, default-features = false } [dev-dependencies] -serde_json = "1.0.149" +clap = { version = "=4.6.1", features = ["derive"] } static_assertions = "1.1.0" diff --git a/vllm-cpp/README.md b/vllm-cpp/README.md index ffcf185..4c0f5ae 100644 --- a/vllm-cpp/README.md +++ b/vllm-cpp/README.md @@ -14,7 +14,28 @@ println!("{}", completion.text); # Ok::<(), vllm_cpp::Error>(()) ``` -The model argument is a directory understood by the pinned native engine, not a single weights file. The known-good test layout contains `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; model-family compatibility remains a native vllm.cpp concern. See the packaged [examples guide](examples/README.md) for blocking completion, streaming, chat, structured output, and concurrent-request commands. +`Engine::load` accepts either a model directory or a standalone GGUF file understood by the pinned native engine. The known-good Safetensors test layout contains `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; model-family compatibility remains a native vllm.cpp concern. See the packaged [examples guide](examples/README.md) for local and Hugging Face loading, blocking completion, streaming, JSON-Schema output, concurrent-request commands, and the Clap-based interactive chat CLI with retained history and supported sampling controls. + +## Hugging Face models + +`HuggingFaceModel` is an always-available synchronous resolver backed by the required `hf-hub` 0.5 dependency. It returns a local `PathBuf` that can be passed unchanged to `Engine::load`: + +```rust,no_run +use vllm_cpp::{Engine, HuggingFaceModel}; + +let path = HuggingFaceModel::gguf("owner/repository", "model.gguf") + // Omit this builder to follow the Hub's mutable `main` revision. + .revision("0123456789abcdef0123456789abcdef01234567") + .resolve()?; +let engine = Engine::load(path)?; +# Ok::<(), Box>(()) +``` + +`HuggingFaceModel::gguf(repo, filename)` and `HuggingFaceModel::safetensors(repo)` default to the Hub's mutable `main` revision. The `.revision(...)` builder accepts a branch, tag, or commit; immutable commit SHAs are recommended for reproducibility. The default cache honors `HF_HOME` through the normal Hugging Face layout and uses the cached login token when available. Builders can select a Hub cache directory, override the token, enable progress, or require cache-only offline resolution. Explicit tokens are redacted from resolver `Debug` output. The official endpoint is fixed; `HF_ENDPOINT` is not used. + +GGUF mode retrieves one root-level lowercase `.gguf` file and rejects split sets. Safetensors mode first reads repository metadata for `main` or the explicit revision, pins downloads to its commit SHA, and retrieves only the root files required by this native loader: `config.json`, `tokenizer.json`, optional `tokenizer_config.json`, and either `model.safetensors` or the root index plus every indexed root shard. It returns the shared `snapshots/` directory and creates that revision's cache ref only after a complete successful retrieval. It does not download unrelated repository assets. Offline mode constructs no network API, reads the cached `main` ref by default, and distinguishes a missing cache revision from an incomplete cached snapshot. + +Retrieval validates cache and snapshot completeness; it does not prove that the pinned native engine supports the repository's model architecture, tokenizer, quantization, or backend. ## API and ownership @@ -32,7 +53,7 @@ The model argument is a directory understood by the pinned native engine, not a | `bundled` (default) | Build and statically link the pinned CPU native source | | `system` | Link a caller-provided installation; use with `--no-default-features` | | `dynamic-link` | Link `libvllm` dynamically in bundled or system mode | -| `serde` | Add `serde_json::Value` chat helpers | +| `serde` | Add `serde_json::Value` chat helpers; JSON parsing for Hub resolution is always present | | `cuda` | Experimental bundled CUDA build configuration | | `cuda-cutlass` | Experimental CUDA build with a caller-provided CUTLASS >=4.5.0 tree | | `triton-aot` | Experimental CUDA build using checked-in Triton AOT artifacts | @@ -40,6 +61,8 @@ The model argument is a directory understood by the pinned native engine, not a | `metal` | Experimental native Metal build on Apple ARM64 | | `mlx` | Experimental external MLX provider on Apple ARM64; implies `metal` | +Hugging Face resolution is not a Cargo feature: synchronous `hf-hub` support is a normal dependency in every build and does not add Tokio or another async runtime. + `bundled` and `system` conflict. CUDA and Vulkan conflict, and accelerator features are bundled-only but do not implicitly enable `bundled` for `--no-default-features` builds. Metal/MLX require exact `aarch64-apple-darwin`; MLX additionally requires an external `MLX_ROOT` with its headers, dylib, and metallib. The workspace [backend documentation](https://github.com/querymt/vllm-cpp-rs#experimental-backend-builds) records exact environment variables, supported build architectures, and current blockers. ## ABI and deployment diff --git a/vllm-cpp/examples/README.md b/vllm-cpp/examples/README.md index 81e0e19..3eac25b 100644 --- a/vllm-cpp/examples/README.md +++ b/vllm-cpp/examples/README.md @@ -1,16 +1,33 @@ # examples -these examples exercise the safe `vllm-cpp` api with fixed prompts and settings: +these examples exercise the safe `vllm-cpp` api. four use fixed prompts and settings; `chat` is an interactive command-line application: | example | behavior | |---|---| | [`complete`](complete.rs) | runs one blocking text completion | | [`stream`](stream.rs) | prints one completion as token deltas arrive | | [`concurrent`](concurrent.rs) | submits two asynchronous streaming requests and waits for both | -| [`chat`](chat.rs) | sends a fixed raw-json chat request; the optional `serde` feature is not required | -| [`structured`](structured.rs) | constrains one completion to the choice `red` or `blue` | +| [`chat`](chat.rs) | runs a Clap-based interactive chat with conversation history and streaming output | +| [`structured`](structured.rs) | extracts a fixed weather report under a JSON Schema | -each example reads the first positional argument as a model directory and implements no additional options. a usable directory must contain the runtime files `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; the pinned [test model fixture and layout](https://github.com/querymt/vllm-cpp-rs#test-model-and-sanitizers) is the known-good reference. model compatibility depends on the native engine, so an arbitrary model directory is not guaranteed to work. +The four fixed examples (`complete`, `stream`, `concurrent`, and `structured`) accept the same manual model-source forms: + +```console +EXAMPLE +EXAMPLE local +EXAMPLE hf-gguf [--revision ] +EXAMPLE hf-safetensors [--revision ] +``` + +The bare path remains an alias for `local`. Hub forms enable download progress, use `main` when `--revision` is omitted, and reuse files already present in Hugging Face's normal cache. `Engine::load` receives the local or resolved `PathBuf` unchanged. It accepts either a model directory or a standalone GGUF file understood by the pinned native engine. A known-good Safetensors directory contains `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; indexed models instead use `model.safetensors.index.json` and all referenced root shards. Model compatibility depends on the native engine, so an arbitrary directory, GGUF, or Hub repository is not guaranteed to work. + +## hugging face loading + +Omitting `--revision` follows the Hub's mutable default `main` branch. Use `.revision(...)` in the library or `--revision` in an example to select a branch, tag, or commit; an immutable commit SHA is recommended for reproducibility. The resolver is synchronous and always available because `hf-hub` is a normal dependency, not a Cargo feature. + +By default the resolver uses Hugging Face's normal cache, honoring `HF_HOME`, and selects the cached token created by Hugging Face login. Library callers can use `.revision(...)`, `.cache_dir(...)`, `.token(...)`, `.progress(...)`, and `.offline(true)`. An explicit token overrides the selected cached token and is redacted from resolver `Debug`. Offline default resolution reads only the cached `main` ref; an explicit revision reads only that cached ref. Offline mode constructs no API. The official Hugging Face endpoint remains fixed; `HF_ENDPOINT` is not used. + +GGUF mode retrieves one safe root-level filename ending in lowercase `.gguf`; split GGUF sets are unsupported. Safetensors mode queries metadata for `main` or the explicit revision, pins file retrieval to the returned commit SHA, and creates that revision's cache ref after complete success. It retrieves only `config.json`, `tokenizer.json`, optional `tokenizer_config.json`, and either root `model.safetensors` or root `model.safetensors.index.json` plus every unique root shard in `weight_map`. Unrelated repository assets are not downloaded. Successful retrieval establishes a complete sparse snapshot for the pinned loader, not model-family or backend compatibility. ## ordinary linux @@ -30,28 +47,51 @@ git submodule update --init --recursive the commands below select ninja explicitly with `CMAKE_GENERATOR=Ninja`; merely installing ninja does not configure cmake to use it. if you choose another cmake generator and build tool, install them and set or otherwise configure `CMAKE_GENERATOR` accordingly. -the common command shape is: +The common command shape is: ```console CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example EXAMPLE -- /path/to/model + cargo run --locked --release -p vllm-cpp --features bundled --example EXAMPLE -- MODEL_SOURCE ``` -`bundled` is the default feature; it is shown explicitly here to identify the cpu backend. run any of the five examples with: +`bundled` is the default feature; it is shown explicitly here to identify the CPU backend. These commands demonstrate all five examples and the shared source syntax: ```console -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example complete -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example stream -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example concurrent -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example chat -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example structured -- /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example complete -- /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example stream -- local /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example concurrent -- hf-gguf owner/repository model.gguf +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example chat -- --system "Answer concisely." hf-safetensors owner/repository +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example structured -- hf-safetensors owner/repository --revision release ``` +Replace the fixed examples' arguments after `--` with another model-source form from the syntax block. The chat command uses the same source names through Clap; its exact syntax and interactive options follow. No example requires a revision. + +## interactive chat CLI + +`chat` accepts global chat options before or after the model subcommand and uses one of these model forms: + +```console +chat [OPTIONS] +chat [OPTIONS] local [OPTIONS] +chat [OPTIONS] hf-gguf [--revision ] [OPTIONS] +chat [OPTIONS] hf-safetensors [--revision ] [OPTIONS] +``` + +The bare path remains a local alias. The Hub subcommands reuse the shared resolver: progress is enabled, cached files are reused, omitted revisions follow mutable `main`, and `--revision` can select a branch, tag, or commit. These complete commands show local and Hub ordering: + +```console +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --example chat -- \ + --system "Answer concisely." --prompt "Hello" local /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --example chat -- \ + hf-gguf owner/repository model.gguf --revision release --temperature 0.5 +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --example chat -- \ + --file prompt.txt hf-safetensors owner/repository --no-stream +``` + +`--prompt/-p ` and `--file/-f ` are mutually exclusive optional first user messages; prompt files must be UTF-8. `--system ` adds a retained system message. Generation options are `--max-tokens` (default `256`, maximum `2147483647`), `--temperature` (default `0.7`), `--top-p` (default `1`), `--top-k` (default `0`), `--min-p` (default `0`), and optional `--seed`. Responses stream by default; `--no-stream` selects blocking `Engine::chat_json` output. The CLI maintains the complete user/assistant history, submits it on each turn, and prints only assistant content rather than raw response JSON. Native role, reasoning, tool-call, and finish metadata is ignored; a valid response with no content is stored as an empty assistant message. + +At `user>` enter `/clear` to retain the system message while clearing other history, or `/quit`/`/exit` to stop. EOF also exits cleanly. A per-turn request or response error is reported as `chat: `, the attempted turn is removed from history, and the prompt continues; terminal input/output errors and model startup failures still exit. This high-level example intentionally exposes only controls supported by the existing chat request and engine APIs; it does not add low-level model, device, context, batch, token, or timing controls from other runtimes. + ## optional nix shell nix is optional and works on supported linux installations with nix; it does not require nixos. the default development shell supplies the cpu build dependencies. from the workspace root, run: @@ -61,7 +101,7 @@ CMAKE_GENERATOR=Ninja \ nix develop -c cargo run --locked --release -p vllm-cpp --features bundled --example complete -- /path/to/model ``` -replace `complete` with any other example name from the table. +replace `complete` with another example and its arguments from the table. ## experimental cuda @@ -97,6 +137,8 @@ other accelerator features have stricter limits: - if the native source or cmake inputs are missing, rerun `git submodule update --init --recursive`. - the first bundled build compiles the native c++ library and can take substantially longer than later runs. -- if model loading fails, verify the directory argument and its model, configuration, and tokenizer files, then confirm that vllm.cpp supports the model. +- if local model loading fails, verify the directory or standalone GGUF argument and required files, then confirm that vllm.cpp supports the model. +- if Hub resolution fails offline, verify that the exact requested revision has a cache ref and a complete single snapshot; offline mode never contacts the network. +- successful Hub resolution does not establish architecture, tokenizer, quantization, or backend support in the pinned native engine. - for `dynamic-link` or `system` builds, follow the root [link mode and loader-path requirements](https://github.com/querymt/vllm-cpp-rs#link-modes); cargo does not deploy `libvllm.so` or configure its runtime search path. - for accelerator configuration errors, use a fresh target directory and consult the root [experimental backend build details](https://github.com/querymt/vllm-cpp-rs#experimental-backend-builds). diff --git a/vllm-cpp/examples/chat.rs b/vllm-cpp/examples/chat.rs index 77341d7..6c41069 100644 --- a/vllm-cpp/examples/chat.rs +++ b/vllm-cpp/examples/chat.rs @@ -1,17 +1,1044 @@ -use vllm_cpp::Engine; - -fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: chat ")?; - let engine = Engine::load(model)?; - let response = engine.chat_json( - r#"{ - "messages": [{"role": "user", "content": "Reply with hello."}], - "temperature": 0, - "max_tokens": 16 - }"#, - )?; - println!("{response}"); - Ok(()) +mod common; + +use std::error::Error; +use std::fmt; +use std::fs; +use std::io::{self, BufRead, Write}; +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::{Parser, Subcommand}; +use serde_json::{json, Value}; +use vllm_cpp::{Engine, StreamControl}; + +use common::ModelSource; + +#[derive(Debug, Parser)] +#[command( + name = "chat", + about = "Chat interactively with a local or Hugging Face model", + override_usage = "chat [OPTIONS] \n chat [OPTIONS] ", + subcommand_negates_reqs = true +)] +struct Args { + /// Bare local model directory or GGUF path (an alias for `local `). + #[arg(value_name = "MODEL", required = true)] + model_path: Option, + + #[command(subcommand)] + model: Option, + + /// Initial user message. + #[arg( + short = 'p', + long, + global = true, + conflicts_with = "file", + value_name = "TEXT" + )] + prompt: Option, + + /// Read the initial user message from a UTF-8 file. + #[arg( + short = 'f', + long, + global = true, + conflicts_with = "prompt", + value_name = "PATH" + )] + file: Option, + + /// System message added at the start of the conversation. + #[arg(long, global = true, value_name = "TEXT")] + system: Option, + + /// Maximum tokens generated per response, from 1 through 2147483647. + #[arg(long, global = true, default_value_t = 256, value_parser = parse_max_tokens)] + max_tokens: u32, + + /// Sampling temperature in [0, 2]. + #[arg(long, global = true, default_value_t = 0.7, value_parser = parse_temperature)] + temperature: f64, + + /// Nucleus-sampling probability in (0, 1]. + #[arg(long, global = true, default_value_t = 1.0, value_parser = parse_top_p)] + top_p: f64, + + /// Number of top tokens to consider; use 0 or -1 for all tokens. + #[arg(long, global = true, default_value_t = 0, allow_hyphen_values = true, value_parser = parse_top_k)] + top_k: i32, + + /// Minimum token probability relative to the most likely token, in [0, 1]. + #[arg(long, global = true, default_value_t = 0.0, value_parser = parse_min_p)] + min_p: f64, + + /// Random seed for generation. + #[arg(long, global = true, allow_hyphen_values = true)] + seed: Option, + + /// Wait for each complete response instead of printing streamed deltas. + #[arg(long, global = true)] + no_stream: bool, +} + +#[derive(Debug, Subcommand)] +enum Model { + /// Use a local model directory or standalone GGUF file. + Local { + #[arg(value_name = "PATH")] + path: PathBuf, + }, + + /// Download one GGUF file from Hugging Face or reuse its cached copy. + #[command(name = "hf-gguf")] + HuggingFaceGguf { + /// Hugging Face repository, for example `owner/model`. + #[arg(value_name = "REPO")] + repo: String, + /// Root-level GGUF filename in the repository. + #[arg(value_name = "FILENAME")] + filename: String, + /// Branch, tag, or commit; defaults to the mutable `main` revision. + #[arg(long, value_name = "REVISION")] + revision: Option, + }, + + /// Download a runtime-complete Safetensors snapshot from Hugging Face. + #[command(name = "hf-safetensors")] + HuggingFaceSafetensors { + /// Hugging Face repository, for example `owner/model`. + #[arg(value_name = "REPO")] + repo: String, + /// Branch, tag, or commit; defaults to the mutable `main` revision. + #[arg(long, value_name = "REVISION")] + revision: Option, + }, +} + +impl Model { + fn into_source(self) -> ModelSource { + match self { + Self::Local { path } => ModelSource::Local(path), + Self::HuggingFaceGguf { + repo, + filename, + revision, + } => ModelSource::Gguf { + repo, + filename, + revision, + }, + Self::HuggingFaceSafetensors { repo, revision } => { + ModelSource::Safetensors { repo, revision } + } + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Role { + System, + User, + Assistant, +} + +impl Role { + const fn as_str(self) -> &'static str { + match self { + Self::System => "system", + Self::User => "user", + Self::Assistant => "assistant", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct Message { + role: Role, + content: String, +} + +impl Message { + fn new(role: Role, content: impl Into) -> Self { + Self { + role, + content: content.into(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +struct GenerationConfig { + max_tokens: u32, + temperature: f64, + top_p: f64, + top_k: i32, + min_p: f64, + seed: Option, +} + +#[derive(Debug)] +enum ChatError { + Turn(String), + Fatal(String), +} + +impl ChatError { + fn turn(message: impl Into) -> Self { + Self::Turn(message.into()) + } + + fn fatal(message: impl Into) -> Self { + Self::Fatal(message.into()) + } + + const fn is_fatal(&self) -> bool { + matches!(self, Self::Fatal(_)) + } +} + +impl fmt::Display for ChatError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Turn(message) | Self::Fatal(message) => formatter.write_str(message), + } + } +} + +impl Error for ChatError {} + +type ChatResult = Result; + +fn main() -> ExitCode { + match run(Args::parse()) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("chat: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(args: Args) -> ChatResult<()> { + let Args { + model_path, + model, + prompt, + file, + system, + max_tokens, + temperature, + top_p, + top_k, + min_p, + seed, + no_stream, + } = args; + + let initial_prompt = read_initial_prompt(prompt, file)?; + let source = match (model_path, model) { + (Some(path), None) => ModelSource::Local(path), + (None, Some(model)) => model.into_source(), + _ => return Err(ChatError::fatal("exactly one model source is required")), + }; + let model = source + .resolve() + .map_err(|error| ChatError::fatal(format!("failed to resolve model source: {error}")))?; + let engine = Engine::load(model) + .map_err(|error| ChatError::fatal(format!("failed to load model: {error}")))?; + let config = GenerationConfig { + max_tokens, + temperature, + top_p, + top_k, + min_p, + seed, + }; + + let mut history = Vec::new(); + if let Some(system) = system { + history.push(Message::new(Role::System, system)); + } + + write_stdout_line("Interactive chat: use /clear to reset the conversation or /quit to exit.")?; + if let Some(prompt) = initial_prompt { + handle_turn( + &mut history, + prompt, + |history, prompt| run_turn(&engine, history, &config, !no_stream, prompt), + report_turn_error, + )?; + } + interactive_loop(&engine, &mut history, &config, !no_stream) +} + +fn read_initial_prompt( + prompt: Option, + file: Option, +) -> ChatResult> { + match (prompt, file) { + (Some(prompt), None) => Ok(Some(prompt)), + (None, Some(path)) => fs::read_to_string(&path).map(Some).map_err(|error| { + ChatError::fatal(format!( + "failed to read UTF-8 prompt file `{}`: {error}", + path.display() + )) + }), + (None, None) => Ok(None), + (Some(_), Some(_)) => Err(ChatError::fatal( + "--prompt and --file cannot be used together", + )), + } +} + +fn interactive_loop( + engine: &Engine, + history: &mut Vec, + config: &GenerationConfig, + stream: bool, +) -> ChatResult<()> { + let stdin = io::stdin(); + let mut stdin = stdin.lock(); + let mut line = String::new(); + + loop { + { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + write!(stdout, "user> ") + .and_then(|()| stdout.flush()) + .map_err(|error| { + ChatError::fatal(format!("failed to write input prompt: {error}")) + })?; + } + + line.clear(); + let bytes = stdin + .read_line(&mut line) + .map_err(|error| ChatError::fatal(format!("failed to read standard input: {error}")))?; + if bytes == 0 { + write_stdout_line("")?; + return Ok(()); + } + + let input = line.trim_end_matches(['\r', '\n']); + match classify_input(input) { + Input::Quit => return Ok(()), + Input::Clear => { + history.retain(|message| message.role == Role::System); + write_stdout_line("Conversation cleared.")?; + } + Input::Empty => {} + Input::Message(message) => { + handle_turn( + history, + message.to_owned(), + |history, message| run_turn(engine, history, config, stream, message), + report_turn_error, + )?; + } + } + } +} + +fn run_turn( + engine: &Engine, + history: &mut Vec, + config: &GenerationConfig, + stream: bool, + user_message: String, +) -> ChatResult<()> { + run_history_turn(history, user_message, |history| { + let request = build_request(history, config, stream)?; + if stream { + write_assistant_prefix()?; + let result = stream_assistant(engine, &request); + let content = result.as_ref().map_or("", |assistant| assistant.as_str()); + finish_assistant_output(content)?; + result + } else { + engine + .chat_json(&request) + .map_err(|error| ChatError::turn(format!("blocking chat request failed: {error}"))) + .and_then(|response| extract_blocking_content(&response)) + .and_then(|assistant| { + write_assistant_prefix()?; + { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + stdout.write_all(assistant.as_bytes()).map_err(|error| { + ChatError::fatal(format!("failed to write assistant response: {error}")) + })?; + } + finish_assistant_output(&assistant)?; + Ok(assistant) + }) + } + }) +} + +fn run_history_turn( + history: &mut Vec, + user_message: String, + run_assistant: F, +) -> ChatResult<()> +where + F: FnOnce(&mut Vec) -> ChatResult, +{ + let checkpoint = history.len(); + history.push(Message::new(Role::User, user_message)); + match run_assistant(history) { + Ok(assistant) => { + history.push(Message::new(Role::Assistant, assistant)); + Ok(()) + } + Err(error) => { + rollback_turn(history, checkpoint); + Err(error) + } + } +} + +fn handle_turn( + history: &mut Vec, + user_message: String, + run: F, + mut report_error: R, +) -> ChatResult<()> +where + F: FnOnce(&mut Vec, String) -> ChatResult<()>, + R: FnMut(&ChatError), +{ + match run(history, user_message) { + Ok(()) => Ok(()), + Err(error) if error.is_fatal() => Err(error), + Err(error) => { + report_error(&error); + Ok(()) + } + } +} + +fn report_turn_error(error: &ChatError) { + eprintln!("chat: {error}"); +} + +fn rollback_turn(history: &mut Vec, checkpoint: usize) { + history.truncate(checkpoint); +} + +fn stream_assistant(engine: &Engine, request: &str) -> ChatResult { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + let mut assistant = String::new(); + let mut callback_error = None; + + let outcome = engine.chat_stream_json(request, |event| { + if callback_error.is_some() { + return StreamControl::Stop; + } + if event.finished { + return StreamControl::Continue; + } + + match accumulate_stream_content(&mut assistant, &event.delta) { + Ok(delta) => { + if let Err(error) = stdout + .write_all(delta.as_bytes()) + .and_then(|()| stdout.flush()) + { + callback_error = Some(ChatError::fatal(format!( + "failed to write streaming assistant response: {error}" + ))); + StreamControl::Stop + } else { + StreamControl::Continue + } + } + Err(error) => { + callback_error = Some(error); + StreamControl::Stop + } + } + }); + drop(stdout); + + if let Some(error) = callback_error { + return Err(error); + } + let outcome = outcome + .map_err(|error| ChatError::turn(format!("streaming chat request failed: {error}")))?; + complete_stream(assistant, outcome.stopped_by_callback) +} + +fn accumulate_stream_content(assistant: &mut String, chunk: &str) -> ChatResult { + let content = extract_stream_content(chunk)?; + assistant.push_str(&content); + Ok(content) +} + +fn complete_stream(assistant: String, stopped_by_callback: bool) -> ChatResult { + if stopped_by_callback { + Err(ChatError::turn( + "streaming chat response stopped unexpectedly", + )) + } else { + Ok(assistant) + } +} + +fn write_stdout_line(message: &str) -> ChatResult<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + writeln!(stdout, "{message}") + .and_then(|()| stdout.flush()) + .map_err(|error| ChatError::fatal(format!("failed to write standard output: {error}"))) +} + +fn write_assistant_prefix() -> ChatResult<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + write!(stdout, "assistant> ") + .and_then(|()| stdout.flush()) + .map_err(|error| ChatError::fatal(format!("failed to write assistant prompt: {error}"))) +} + +fn finish_assistant_output(content: &str) -> ChatResult<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + if !content.ends_with('\n') { + writeln!(stdout).map_err(|error| { + ChatError::fatal(format!("failed to finish assistant response: {error}")) + })?; + } + stdout + .flush() + .map_err(|error| ChatError::fatal(format!("failed to flush assistant response: {error}"))) +} + +fn build_request( + messages: &[Message], + config: &GenerationConfig, + stream: bool, +) -> ChatResult { + let messages: Vec<_> = messages + .iter() + .map(|message| { + json!({ + "role": message.role.as_str(), + "content": message.content, + }) + }) + .collect(); + let mut request = json!({ + "messages": messages, + "max_tokens": config.max_tokens, + "temperature": config.temperature, + "top_p": config.top_p, + "top_k": config.top_k, + "min_p": config.min_p, + "stream": stream, + }); + if let Some(seed) = config.seed { + request["seed"] = json!(seed); + } + serde_json::to_string(&request) + .map_err(|error| ChatError::turn(format!("failed to serialize chat request: {error}"))) +} + +fn extract_blocking_content(response: &str) -> ChatResult { + let response: Value = serde_json::from_str(response).map_err(|error| { + ChatError::turn(format!("invalid blocking chat response JSON: {error}")) + })?; + let choices = response + .get("choices") + .and_then(Value::as_array) + .ok_or_else(|| ChatError::turn("blocking chat response is missing array `choices`"))?; + let choice = choices + .first() + .and_then(Value::as_object) + .ok_or_else(|| ChatError::turn("blocking chat response is missing object `choices[0]`"))?; + let message = choice + .get("message") + .and_then(Value::as_object) + .ok_or_else(|| { + ChatError::turn("blocking chat response is missing object `choices[0].message`") + })?; + + match message.get("content") { + Some(Value::String(content)) => Ok(content.clone()), + None | Some(Value::Null) => Ok(String::new()), + Some(_) => Err(ChatError::turn( + "blocking chat response has non-string `choices[0].message.content`", + )), + } +} + +fn extract_stream_content(chunk: &str) -> ChatResult { + let chunk: Value = serde_json::from_str(chunk) + .map_err(|error| ChatError::turn(format!("invalid streaming chat chunk JSON: {error}")))?; + let choices = chunk + .get("choices") + .and_then(Value::as_array) + .ok_or_else(|| ChatError::turn("streaming chat chunk is missing array `choices`"))?; + let mut content = String::new(); + for (index, choice) in choices.iter().enumerate() { + let delta = choice + .get("delta") + .and_then(Value::as_object) + .ok_or_else(|| { + ChatError::turn(format!( + "streaming chat chunk is missing object `choices[{index}].delta`" + )) + })?; + match delta.get("content") { + Some(Value::String(part)) => content.push_str(part), + None | Some(Value::Null) => {} + Some(_) => { + return Err(ChatError::turn(format!( + "streaming chat chunk has non-string `choices[{index}].delta.content`" + ))) + } + } + } + Ok(content) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Input<'a> { + Quit, + Clear, + Empty, + Message(&'a str), +} + +fn classify_input(input: &str) -> Input<'_> { + match input.trim() { + "/quit" | "/exit" => Input::Quit, + "/clear" => Input::Clear, + "" => Input::Empty, + _ => Input::Message(input), + } +} + +fn parse_max_tokens(value: &str) -> Result { + let value = value + .parse::() + .map_err(|error| format!("invalid token count: {error}"))?; + if value == 0 { + return Err("max tokens must be greater than zero".to_owned()); + } + if value > i32::MAX as u32 { + return Err(format!("max tokens must not exceed {}", i32::MAX)); + } + Ok(value) +} + +fn parse_temperature(value: &str) -> Result { + parse_float_range(value, "temperature", 0.0, 2.0, false) +} + +fn parse_top_p(value: &str) -> Result { + parse_float_range(value, "top-p", 0.0, 1.0, true) +} + +fn parse_top_k(value: &str) -> Result { + let value = value + .parse::() + .map_err(|error| format!("invalid top-k value: {error}"))?; + if value < -1 { + return Err("top-k must be -1, 0, or a positive integer".to_owned()); + } + Ok(value) +} + +fn parse_min_p(value: &str) -> Result { + parse_float_range(value, "min-p", 0.0, 1.0, false) +} + +fn parse_float_range( + value: &str, + name: &str, + minimum: f64, + maximum: f64, + minimum_is_exclusive: bool, +) -> Result { + let value = value + .parse::() + .map_err(|error| format!("invalid {name} value: {error}"))?; + let below_minimum = if minimum_is_exclusive { + value <= minimum + } else { + value < minimum + }; + if !value.is_finite() || below_minimum || value > maximum { + let opening = if minimum_is_exclusive { "(" } else { "[" }; + return Err(format!( + "{name} must be finite and in {opening}{minimum}, {maximum}]" + )); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use clap::error::ErrorKind; + + use super::*; + + fn source(args: Args) -> ModelSource { + match (args.model_path, args.model) { + (Some(path), None) => ModelSource::Local(path), + (None, Some(model)) => model.into_source(), + _ => panic!("invalid parsed model source"), + } + } + + #[test] + fn parses_bare_and_local_model_forms_with_global_options() { + let args = Args::try_parse_from(["chat", "--prompt", "hello", "model.gguf", "--no-stream"]) + .unwrap(); + assert_eq!(source(args), ModelSource::Local("model.gguf".into())); + + let args = Args::try_parse_from([ + "chat", + "--system", + "be concise", + "local", + "model-directory", + "--temperature", + "0.25", + ]) + .unwrap(); + assert_eq!(args.temperature, 0.25); + assert_eq!(source(args), ModelSource::Local("model-directory".into())); + } + + #[test] + fn rejects_missing_model_source() { + assert!(Args::try_parse_from(["chat"]).is_err()); + } + + #[test] + fn parses_hugging_face_forms_and_revisions() { + let gguf = Args::try_parse_from([ + "chat", + "hf-gguf", + "owner/model", + "model.gguf", + "--max-tokens", + "64", + ]) + .unwrap(); + assert_eq!(gguf.max_tokens, 64); + assert_eq!( + source(gguf), + ModelSource::Gguf { + repo: "owner/model".to_owned(), + filename: "model.gguf".to_owned(), + revision: None, + } + ); + + let safetensors = Args::try_parse_from([ + "chat", + "--top-p", + "0.9", + "hf-safetensors", + "owner/model", + "--revision", + "release", + "--min-p", + "0.05", + ]) + .unwrap(); + assert_eq!(safetensors.min_p, 0.05); + assert_eq!( + source(safetensors), + ModelSource::Safetensors { + repo: "owner/model".to_owned(), + revision: Some("release".to_owned()), + } + ); + } + + #[test] + fn parses_supported_sampling_options() { + let args = Args::try_parse_from([ + "chat", + "local", + "model", + "--max-tokens", + "512", + "--temperature", + "1.25", + "--top-p", + "0.8", + "--top-k", + "-1", + "--min-p", + "0.1", + "--seed", + "-2", + ]) + .unwrap(); + assert_eq!(args.max_tokens, 512); + assert_eq!(args.temperature, 1.25); + assert_eq!(args.top_p, 0.8); + assert_eq!(args.top_k, -1); + assert_eq!(args.min_p, 0.1); + assert_eq!(args.seed, Some(-2)); + } + + #[test] + fn uses_documented_generation_defaults() { + let args = Args::try_parse_from(["chat", "local", "model"]).unwrap(); + assert_eq!(args.max_tokens, 256); + assert_eq!(args.temperature, 0.7); + assert_eq!(args.top_p, 1.0); + assert_eq!(args.top_k, 0); + assert_eq!(args.min_p, 0.0); + assert_eq!(args.seed, None); + assert!(!args.no_stream); + } + + #[test] + fn rejects_prompt_file_conflict_and_invalid_sampling_values() { + let error = Args::try_parse_from([ + "chat", + "local", + "model", + "--prompt", + "hello", + "--file", + "prompt.txt", + ]) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ArgumentConflict); + + for arguments in [ + vec!["chat", "local", "model", "--max-tokens", "0"], + vec!["chat", "local", "model", "--max-tokens", "2147483648"], + vec!["chat", "local", "model", "--temperature", "nan"], + vec!["chat", "local", "model", "--temperature", "2.1"], + vec!["chat", "local", "model", "--top-p", "0"], + vec!["chat", "local", "model", "--top-k", "-2"], + vec!["chat", "local", "model", "--min-p", "1.1"], + ] { + assert!(Args::try_parse_from(arguments).is_err()); + } + assert!( + Args::try_parse_from(["chat", "local", "model", "--max-tokens", "2147483647",]).is_ok() + ); + } + + #[test] + fn rolls_back_failed_turn_and_continues_with_the_next_input() { + let mut history = vec![Message::new(Role::System, "system")]; + let mut errors = Vec::new(); + let mut attempted_messages = Vec::new(); + + handle_turn( + &mut history, + "failed question".to_owned(), + |history, message| { + attempted_messages.push(message.clone()); + run_history_turn(history, message, |_history| { + Err(ChatError::turn("request failed")) + }) + }, + |error| errors.push(error.to_string()), + ) + .unwrap(); + assert_eq!(history, vec![Message::new(Role::System, "system")]); + assert_eq!(attempted_messages, ["failed question"]); + assert_eq!(errors, ["request failed"]); + + handle_turn( + &mut history, + "next question".to_owned(), + |history, message| { + run_history_turn(history, message, |_history| Ok("answer".to_owned())) + }, + |_| panic!("successful turn must not report an error"), + ) + .unwrap(); + assert_eq!( + history, + vec![ + Message::new(Role::System, "system"), + Message::new(Role::User, "next question"), + Message::new(Role::Assistant, "answer"), + ] + ); + } + + #[test] + fn propagates_fatal_errors_instead_of_recovering() { + let mut history = Vec::new(); + let mut reported = false; + let error = handle_turn( + &mut history, + "question".to_owned(), + |_history, _message| Err(ChatError::fatal("standard output failed")), + |_| reported = true, + ) + .unwrap_err(); + assert!(error.is_fatal()); + assert!(!reported); + } + + #[test] + fn builds_request_with_history_and_sampling_configuration() { + let messages = vec![ + Message::new(Role::System, "Be concise."), + Message::new(Role::User, "Hello"), + Message::new(Role::Assistant, "Hi"), + ]; + let request = build_request( + &messages, + &GenerationConfig { + max_tokens: 42, + temperature: 0.5, + top_p: 0.9, + top_k: 10, + min_p: 0.1, + seed: Some(7), + }, + true, + ) + .unwrap(); + let request: Value = serde_json::from_str(&request).unwrap(); + assert_eq!(request["messages"][0]["role"], "system"); + assert_eq!(request["messages"][1]["content"], "Hello"); + assert_eq!(request["messages"][2]["content"], "Hi"); + assert_eq!(request["max_tokens"], 42); + assert_eq!(request["temperature"], 0.5); + assert_eq!(request["top_p"], 0.9); + assert_eq!(request["top_k"], 10); + assert_eq!(request["min_p"], 0.1); + assert_eq!(request["seed"], 7); + assert_eq!(request["stream"], true); + } + + #[test] + fn extracts_native_blocking_response_content() { + let response = r#"{ + "id":"chatcmpl-test", + "object":"chat.completion", + "choices":[{ + "index":0, + "message":{"role":"assistant","content":"hello"}, + "finish_reason":"stop" + }] + }"#; + assert_eq!(extract_blocking_content(response).unwrap(), "hello"); + + let reasoning_only = r#"{ + "id":"chatcmpl-test", + "object":"chat.completion", + "choices":[{ + "index":0, + "message":{ + "role":"assistant", + "content":null, + "reasoning":"private reasoning" + }, + "finish_reason":"stop" + }] + }"#; + assert_eq!(extract_blocking_content(reasoning_only).unwrap(), ""); + + let missing_content = r#"{ + "choices":[{ + "message":{ + "role":"assistant", + "tool_calls":[{ + "id":"chatcmpl-tool-0", + "type":"function", + "function":{"name":"weather","arguments":"{}"} + }] + }, + "finish_reason":"tool_calls" + }] + }"#; + assert_eq!(extract_blocking_content(missing_content).unwrap(), ""); + } + + #[test] + fn rejects_malformed_blocking_response_shapes() { + for response in [ + r#"{}"#, + r#"{"choices":"not an array"}"#, + r#"{"choices":[]}"#, + r#"{"choices":[{"message":"not an object"}]}"#, + r#"{"choices":[{"message":{"content":42}}]}"#, + "not json", + ] { + assert!(extract_blocking_content(response).is_err()); + } + } + + #[test] + fn extracts_native_stream_chunks_and_accepts_empty_content() { + let chunks = [ + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"private"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null},{"index":1,"delta":{"content":"!"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + ]; + let mut content = String::new(); + for chunk in chunks { + assert_eq!( + accumulate_stream_content(&mut content, chunk).unwrap(), + extract_stream_content(chunk).unwrap() + ); + } + assert_eq!(content, "Hello!"); + + let empty_chunks = [ + r#"{"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{"reasoning":"private"},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"name":"weather"}}]},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{"content":null},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}"#, + ]; + let mut empty = String::new(); + for chunk in empty_chunks { + assert_eq!(accumulate_stream_content(&mut empty, chunk).unwrap(), ""); + } + assert_eq!(complete_stream(empty, false).unwrap(), ""); + + let mut history = vec![Message::new(Role::User, "question")]; + run_history_turn(&mut history, "next".to_owned(), |_history| { + complete_stream(String::new(), false) + }) + .unwrap(); + assert_eq!(history.last(), Some(&Message::new(Role::Assistant, ""))); + } + + #[test] + fn rejects_malformed_stream_content_types_and_shapes() { + for chunk in [ + r#"{}"#, + r#"{"choices":"not an array"}"#, + r#"{"choices":[{}]}"#, + r#"{"choices":[{"delta":"not an object"}]}"#, + r#"{"choices":[{"delta":{"content":42}}]}"#, + "not json", + ] { + assert!(extract_stream_content(chunk).is_err()); + } + assert!(complete_stream(String::new(), true).is_err()); + } + + #[test] + fn classifies_interactive_commands_without_trimming_messages() { + assert_eq!(classify_input(" /quit "), Input::Quit); + assert_eq!(classify_input("/exit"), Input::Quit); + assert_eq!(classify_input("/clear"), Input::Clear); + assert_eq!(classify_input(" "), Input::Empty); + assert_eq!(classify_input(" hello "), Input::Message(" hello ")); + } } diff --git a/vllm-cpp/examples/common/mod.rs b/vllm-cpp/examples/common/mod.rs new file mode 100644 index 0000000..2fd79d3 --- /dev/null +++ b/vllm-cpp/examples/common/mod.rs @@ -0,0 +1,254 @@ +use std::env; +use std::error::Error; +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::path::PathBuf; + +use vllm_cpp::HuggingFaceModel; + +#[derive(Debug)] +struct UsageError(String); + +impl fmt::Display for UsageError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Error for UsageError {} + +#[derive(Debug, Eq, PartialEq)] +pub enum ModelSource { + Local(PathBuf), + Gguf { + repo: String, + filename: String, + revision: Option, + }, + Safetensors { + repo: String, + revision: Option, + }, +} + +impl ModelSource { + pub fn resolve(self) -> Result> { + match self { + Self::Local(path) => Ok(path), + Self::Gguf { + repo, + filename, + revision, + } => { + let resolver = HuggingFaceModel::gguf(repo, filename); + let resolver = if let Some(revision) = revision { + resolver.revision(revision) + } else { + resolver + }; + Ok(resolver.progress(true).resolve()?) + } + Self::Safetensors { repo, revision } => { + let resolver = HuggingFaceModel::safetensors(repo); + let resolver = if let Some(revision) = revision { + resolver.revision(revision) + } else { + resolver + }; + Ok(resolver.progress(true).resolve()?) + } + } + } +} + +#[allow(dead_code)] +pub fn resolve_model(example: &str) -> Result> { + parse_model_source(example, env::args_os().skip(1))?.resolve() +} + +fn parse_model_source( + example: &str, + arguments: impl IntoIterator, +) -> Result { + let mut args = arguments.into_iter(); + let first = args.next().ok_or_else(|| usage(example))?; + match first.to_str() { + Some("local") => { + let path = args.next().ok_or_else(|| usage(example))?; + if path.is_empty() { + return Err(usage(example)); + } + require_end(example, &mut args)?; + Ok(ModelSource::Local(path.into())) + } + Some("hf-gguf") => { + let repo = required_utf8(example, &mut args)?; + let filename = required_utf8(example, &mut args)?; + let revision = optional_revision(example, &mut args)?; + Ok(ModelSource::Gguf { + repo, + filename, + revision, + }) + } + Some("hf-safetensors") => { + let repo = required_utf8(example, &mut args)?; + let revision = optional_revision(example, &mut args)?; + Ok(ModelSource::Safetensors { repo, revision }) + } + Some(value) if value.starts_with('-') => Err(usage(example)), + _ => { + if first.is_empty() { + return Err(usage(example)); + } + require_end(example, &mut args)?; + Ok(ModelSource::Local(first.into())) + } + } +} + +fn required_utf8( + example: &str, + args: &mut impl Iterator, +) -> Result { + let value = args + .next() + .ok_or_else(|| usage(example))? + .into_string() + .map_err(|_| usage(example))?; + if value.is_empty() { + return Err(usage(example)); + } + Ok(value) +} + +fn optional_revision( + example: &str, + args: &mut impl Iterator, +) -> Result, UsageError> { + let Some(flag) = args.next() else { + return Ok(None); + }; + if flag != OsStr::new("--revision") { + return Err(usage(example)); + } + let revision = required_utf8(example, args)?; + require_end(example, args)?; + Ok(Some(revision)) +} + +fn require_end(example: &str, args: &mut impl Iterator) -> Result<(), UsageError> { + if args.next().is_some() { + return Err(usage(example)); + } + Ok(()) +} + +fn usage(example: &str) -> UsageError { + UsageError(format!( + "usage:\n {example} \n {example} local \n {example} hf-gguf [--revision ]\n {example} hf-safetensors [--revision ]" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn os(values: &[&str]) -> Vec { + values.iter().map(OsString::from).collect() + } + + #[test] + fn parses_local_aliases() { + assert_eq!( + parse_model_source("example", os(&["model.gguf"])).unwrap(), + ModelSource::Local("model.gguf".into()) + ); + assert_eq!( + parse_model_source("example", os(&["local", "model"])).unwrap(), + ModelSource::Local("model".into()) + ); + } + + #[test] + fn parses_hugging_face_sources_with_optional_revisions() { + assert_eq!( + parse_model_source("example", os(&["hf-gguf", "owner/model", "model.gguf"])).unwrap(), + ModelSource::Gguf { + repo: "owner/model".to_owned(), + filename: "model.gguf".to_owned(), + revision: None, + } + ); + assert_eq!( + parse_model_source("example", os(&["hf-safetensors", "owner/model"])).unwrap(), + ModelSource::Safetensors { + repo: "owner/model".to_owned(), + revision: None, + } + ); + assert_eq!( + parse_model_source( + "example", + os(&[ + "hf-gguf", + "owner/model", + "model.gguf", + "--revision", + "release", + ]), + ) + .unwrap(), + ModelSource::Gguf { + repo: "owner/model".to_owned(), + filename: "model.gguf".to_owned(), + revision: Some("release".to_owned()), + } + ); + assert_eq!( + parse_model_source( + "example", + os(&["hf-safetensors", "owner/model", "--revision", "release",]), + ) + .unwrap(), + ModelSource::Safetensors { + repo: "owner/model".to_owned(), + revision: Some("release".to_owned()), + } + ); + } + + #[test] + fn rejects_missing_extra_duplicate_and_misordered_arguments() { + for args in [ + os(&[]), + os(&[""]), + os(&["local"]), + os(&["local", ""]), + os(&["model", "extra"]), + os(&["hf-gguf", "owner/model"]), + os(&["hf-gguf", "owner/model", ""]), + os(&["hf-safetensors", ""]), + os(&["hf-safetensors", "owner/model", "release"]), + os(&["hf-safetensors", "owner/model", "--revision"]), + os(&["hf-safetensors", "owner/model", "--revision", ""]), + os(&[ + "hf-safetensors", + "owner/model", + "--revision", + "one", + "--revision", + "two", + ]), + os(&[ + "hf-gguf", + "--revision", + "release", + "owner/model", + "model.gguf", + ]), + ] { + assert!(parse_model_source("example", args).is_err()); + } + } +} diff --git a/vllm-cpp/examples/complete.rs b/vllm-cpp/examples/complete.rs index 71cb53d..bc26b2e 100644 --- a/vllm-cpp/examples/complete.rs +++ b/vllm-cpp/examples/complete.rs @@ -1,9 +1,9 @@ +mod common; + use vllm_cpp::{Engine, SamplingParams}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: complete ")?; + let model = common::resolve_model("complete")?; let engine = Engine::load(model)?; let completion = engine.complete( "The capital of France is", diff --git a/vllm-cpp/examples/concurrent.rs b/vllm-cpp/examples/concurrent.rs index 51432f7..87804cf 100644 --- a/vllm-cpp/examples/concurrent.rs +++ b/vllm-cpp/examples/concurrent.rs @@ -1,11 +1,11 @@ +mod common; + use std::io::{self, Write}; use vllm_cpp::{Engine, SamplingParams, StreamControl}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: concurrent ")?; + let model = common::resolve_model("concurrent")?; let engine = Engine::load(model)?; let params = SamplingParams::greedy().max_tokens(16); diff --git a/vllm-cpp/examples/stream.rs b/vllm-cpp/examples/stream.rs index b852a6c..f4c1980 100644 --- a/vllm-cpp/examples/stream.rs +++ b/vllm-cpp/examples/stream.rs @@ -1,11 +1,11 @@ +mod common; + use std::io::{self, Write}; use vllm_cpp::{Engine, SamplingParams, StreamControl}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: stream ")?; + let model = common::resolve_model("stream")?; let engine = Engine::load(model)?; engine.complete_stream( "Write one short sentence about Rust:", diff --git a/vllm-cpp/examples/structured.rs b/vllm-cpp/examples/structured.rs index ec48207..4107025 100644 --- a/vllm-cpp/examples/structured.rs +++ b/vllm-cpp/examples/structured.rs @@ -1,18 +1,25 @@ +mod common; + use vllm_cpp::{Engine, SamplingParams, StructuredOutput}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: structured ")?; + let model = common::resolve_model("structured")?; let engine = Engine::load(model)?; + let schema = r#"{ + "type": "object", + "properties": { + "location": { "type": "string" }, + "temperature_celsius": { "type": "number" }, + "condition": { "type": "string" } + }, + "required": ["location", "temperature_celsius", "condition"], + "additionalProperties": false + }"#; let completion = engine.complete( - "Choose exactly one color: red or blue. Answer:", + "Extract the weather report as JSON: Paris is sunny and 22 degrees Celsius.", &SamplingParams::greedy() - .max_tokens(8) - .structured_output(StructuredOutput::Choice(vec![ - "red".to_owned(), - "blue".to_owned(), - ])), + .max_tokens(64) + .structured_output(StructuredOutput::JsonSchema(schema.to_owned())), )?; println!("{}", completion.text); Ok(()) diff --git a/vllm-cpp/src/error.rs b/vllm-cpp/src/error.rs index 9090f93..d9a15ae 100644 --- a/vllm-cpp/src/error.rs +++ b/vllm-cpp/src/error.rs @@ -3,6 +3,41 @@ use std::fmt; use vllm_cpp_sys as ffi; +/// An error returned while resolving a model from the Hugging Face Hub. +/// +/// External transport errors are converted to contextual strings so this type +/// remains stable, cloneable, and comparable. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum HuggingFaceError { + /// A repository, revision, or filename is invalid. + InvalidInput { message: String }, + /// The requested revision is not present in the selected local cache. + CacheMiss { message: String }, + /// A repository snapshot lacks required runtime files or metadata. + Incomplete { message: String }, + /// A Hugging Face API or download operation failed. + Hub { message: String }, + /// A local cache operation failed. + Io { message: String }, +} + +impl fmt::Display for HuggingFaceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidInput { message } => write!(f, "invalid Hugging Face model: {message}"), + Self::CacheMiss { message } => write!(f, "Hugging Face cache miss: {message}"), + Self::Incomplete { message } => { + write!(f, "incomplete Hugging Face snapshot: {message}") + } + Self::Hub { message } => write!(f, "Hugging Face Hub failure: {message}"), + Self::Io { message } => write!(f, "Hugging Face cache I/O failure: {message}"), + } + } +} + +impl std::error::Error for HuggingFaceError {} + /// An error returned by the safe vllm.cpp wrapper. #[derive(Clone, Debug, Eq, PartialEq)] #[non_exhaustive] diff --git a/vllm-cpp/src/hf.rs b/vllm-cpp/src/hf.rs new file mode 100644 index 0000000..84f9676 --- /dev/null +++ b/vllm-cpp/src/hf.rs @@ -0,0 +1,1022 @@ +use std::collections::{BTreeSet, HashMap}; +use std::fmt; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +use hf_hub::api::sync::ApiBuilder; +use hf_hub::api::RepoInfo; +use hf_hub::{Cache, Repo, RepoType}; +use serde_json::Value; + +use crate::HuggingFaceError; + +const CONFIG: &str = "config.json"; +const TOKENIZER: &str = "tokenizer.json"; +const TOKENIZER_CONFIG: &str = "tokenizer_config.json"; +const SAFETENSORS: &str = "model.safetensors"; +const SAFETENSORS_INDEX: &str = "model.safetensors.index.json"; +const DEFAULT_REVISION: &str = "main"; + +/// A synchronous Hugging Face model resolver. +/// +/// Resolution is separate from [`crate::Engine::load`]. The resolver returns a +/// standalone GGUF path or a sparse, runtime-complete Safetensors snapshot +/// directory in the normal Hugging Face cache layout. +#[derive(Clone)] +pub struct HuggingFaceModel { + repo_id: String, + revision: String, + kind: ModelKind, + cache_dir: Option, + token: Option, + progress: bool, + offline: bool, +} + +#[derive(Clone, Debug)] +enum ModelKind { + Gguf { filename: String }, + Safetensors, +} + +impl fmt::Debug for HuggingFaceModel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HuggingFaceModel") + .field("repo_id", &self.repo_id) + .field("revision", &self.revision) + .field("kind", &self.kind) + .field("cache_dir", &self.cache_dir) + .field("token", &self.token.as_ref().map(|_| "[REDACTED]")) + .field("progress", &self.progress) + .field("offline", &self.offline) + .finish() + } +} + +impl HuggingFaceModel { + /// Selects one standalone GGUF file from the repository's `main` revision. + #[must_use] + pub fn gguf(repo_id: impl Into, filename: impl Into) -> Self { + Self { + repo_id: repo_id.into(), + revision: DEFAULT_REVISION.to_owned(), + kind: ModelKind::Gguf { + filename: filename.into(), + }, + cache_dir: None, + token: None, + progress: false, + offline: false, + } + } + + /// Selects a runtime-complete Safetensors directory from the repository's `main` revision. + #[must_use] + pub fn safetensors(repo_id: impl Into) -> Self { + Self { + repo_id: repo_id.into(), + revision: DEFAULT_REVISION.to_owned(), + kind: ModelKind::Safetensors, + cache_dir: None, + token: None, + progress: false, + offline: false, + } + } + + /// Overrides the repository revision with a branch, tag, or commit. + #[must_use] + pub fn revision(mut self, revision: impl Into) -> Self { + self.revision = revision.into(); + self + } + + /// Overrides the Hugging Face Hub cache directory. + /// + /// The path is the Hub cache itself (for example, `~/.cache/huggingface/hub`), + /// not its parent. + #[must_use] + pub fn cache_dir(mut self, cache_dir: impl Into) -> Self { + self.cache_dir = Some(cache_dir.into()); + self + } + + /// Overrides the cached Hugging Face token for this resolver. + #[must_use] + pub fn token(mut self, token: impl Into) -> Self { + self.token = Some(token.into()); + self + } + + /// Enables or disables download progress bars. Progress is disabled by default. + #[must_use] + pub fn progress(mut self, progress: bool) -> Self { + self.progress = progress; + self + } + + /// Enables or disables cache-only resolution. Offline mode never builds an API. + #[must_use] + pub fn offline(mut self, offline: bool) -> Self { + self.offline = offline; + self + } + + /// Resolves the selected model into the normal Hugging Face cache. + pub fn resolve(&self) -> Result { + self.validate()?; + let cache = self.cache(); + if self.offline { + return self.resolve_offline(&cache); + } + + match &self.kind { + ModelKind::Gguf { filename } => self.resolve_gguf_online(cache, filename), + ModelKind::Safetensors => self.resolve_safetensors_online(cache), + } + } + + fn validate(&self) -> Result<(), HuggingFaceError> { + validate_nonempty("repository ID", &self.repo_id)?; + validate_nonempty("revision", &self.revision)?; + validate_repo_id(&self.repo_id)?; + validate_revision(&self.revision)?; + + if let Some(token) = &self.token { + validate_nonempty("token", token)?; + } + if let ModelKind::Gguf { filename } = &self.kind { + validate_root_filename(filename, "GGUF filename")?; + if !filename.ends_with(".gguf") { + return Err(invalid("GGUF filename must end with lowercase `.gguf`")); + } + if is_split_gguf(filename) { + return Err(invalid("split GGUF sets are not supported")); + } + } + Ok(()) + } + + fn cache(&self) -> Cache { + self.cache_dir + .clone() + .map(Cache::new) + .unwrap_or_else(Cache::from_env) + } + + fn api_builder(&self, cache: Cache) -> ApiBuilder { + let builder = ApiBuilder::from_cache(cache).with_progress(self.progress); + match &self.token { + Some(token) => builder.with_token(Some(token.clone())), + None => builder, + } + } + + fn requested_repo(&self) -> Repo { + Repo::with_revision(self.repo_id.clone(), RepoType::Model, self.revision.clone()) + } + + fn resolve_gguf_online( + &self, + cache: Cache, + filename: &str, + ) -> Result { + let api = self + .api_builder(cache) + .build() + .map_err(|error| hub(format!("could not create API: {error}")))?; + let path = api + .repo(self.requested_repo()) + .get(filename) + .map_err(|error| { + hub(format!( + "could not resolve `{filename}` from `{}` at `{}`: {error}", + self.repo_id, self.revision + )) + })?; + verify_gguf_path(&path, filename)?; + Ok(path) + } + + fn resolve_safetensors_online(&self, cache: Cache) -> Result { + let api = self + .api_builder(cache.clone()) + .build() + .map_err(|error| hub(format!("could not create API: {error}")))?; + let info = api.repo(self.requested_repo()).info().map_err(|error| { + hub(format!( + "could not read metadata for `{}` at `{}`: {error}", + self.repo_id, self.revision + )) + })?; + let sha = info.sha.trim(); + if sha.is_empty() { + return Err(incomplete("repository metadata has an empty commit SHA")); + } + validate_root_filename(sha, "repository metadata SHA") + .map_err(|_| incomplete("repository metadata has an unsafe SHA"))?; + + let available = sibling_names(&info); + let bootstrap = plan_safetensors(&available)?; + let pinned_repo = + Repo::with_revision(self.repo_id.clone(), RepoType::Model, sha.to_owned()); + let pinned = api.repo(pinned_repo.clone()); + let index = if bootstrap.indexed { + let path = pinned.get(SAFETENSORS_INDEX).map_err(|error| { + hub(format!( + "could not resolve `{SAFETENSORS_INDEX}` for `{}` at `{sha}`: {error}", + self.repo_id + )) + })?; + let bytes = fs::read(&path).map_err(|error| { + io_error(format!("could not read `{}`: {error}", path.display())) + })?; + Some((path, bytes)) + } else { + None + }; + let (plan, index_path) = match index { + Some((path, bytes)) => (plan_indexed_safetensors(&available, &bytes)?, Some(path)), + None => (bootstrap, None), + }; + + let mut paths = HashMap::new(); + if let Some(path) = index_path { + paths.insert(SAFETENSORS_INDEX.to_owned(), path); + } + for filename in &plan.files { + if paths.contains_key(filename) { + continue; + } + let path = pinned.get(filename).map_err(|error| { + hub(format!( + "could not resolve `{filename}` for `{}` at `{sha}`: {error}", + self.repo_id + )) + })?; + paths.insert(filename.clone(), path); + } + + let snapshot = verify_snapshot_paths(&paths, sha)?; + cache + .repo(self.requested_repo()) + .create_ref(sha) + .map_err(|error| { + io_error(format!( + "could not update cache ref `{}` for `{}`: {error}", + self.revision, self.repo_id + )) + })?; + Ok(snapshot) + } + + fn resolve_offline(&self, cache: &Cache) -> Result { + let repo = cache.repo(self.requested_repo()); + match &self.kind { + ModelKind::Gguf { filename } => { + let path = repo.get(filename).ok_or_else(|| { + cache_miss(format!( + "`{filename}` for `{}` at `{}` is not cached", + self.repo_id, self.revision + )) + })?; + let snapshot = snapshot_for_cached_revision(cache, &self.requested_repo())?; + verify_gguf_path(&path, filename)?; + if path.parent() != Some(snapshot.as_path()) { + return Err(incomplete(format!( + "resolved `{filename}` does not belong to the requested revision snapshot" + ))); + } + Ok(path) + } + ModelKind::Safetensors => { + let snapshot = snapshot_for_cached_revision(cache, &self.requested_repo())?; + let available = cached_root_files(&snapshot)?; + let bootstrap = plan_safetensors(&available)?; + let plan = if bootstrap.indexed { + let index_path = snapshot.join(SAFETENSORS_INDEX); + let bytes = fs::read(&index_path).map_err(|error| { + io_error(format!( + "could not read `{}`: {error}", + index_path.display() + )) + })?; + plan_indexed_safetensors(&available, &bytes)? + } else { + bootstrap + }; + + let paths = plan + .files + .iter() + .map(|filename| (filename.clone(), snapshot.join(filename))) + .collect::>(); + verify_snapshot_paths(&paths, snapshot_sha(&snapshot)?) + } + } + } +} + +#[derive(Debug, Eq, PartialEq)] +struct SafetensorsPlan { + files: Vec, + indexed: bool, +} + +fn plan_safetensors(available: &BTreeSet) -> Result { + for required in [CONFIG, TOKENIZER] { + if !available.contains(required) { + return Err(incomplete(format!("required `{required}` is missing"))); + } + } + + let mut files = vec![CONFIG.to_owned(), TOKENIZER.to_owned()]; + if available.contains(TOKENIZER_CONFIG) { + files.push(TOKENIZER_CONFIG.to_owned()); + } + if available.contains(SAFETENSORS) { + files.push(SAFETENSORS.to_owned()); + Ok(SafetensorsPlan { + files, + indexed: false, + }) + } else if available.contains(SAFETENSORS_INDEX) { + files.push(SAFETENSORS_INDEX.to_owned()); + Ok(SafetensorsPlan { + files, + indexed: true, + }) + } else { + Err(incomplete(format!( + "neither `{SAFETENSORS}` nor `{SAFETENSORS_INDEX}` is present" + ))) + } +} + +fn plan_indexed_safetensors( + available: &BTreeSet, + bytes: &[u8], +) -> Result { + let mut plan = plan_safetensors(available)?; + if !plan.indexed { + return Ok(plan); + } + + let value: Value = serde_json::from_slice(bytes) + .map_err(|error| incomplete(format!("`{SAFETENSORS_INDEX}` is malformed JSON: {error}")))?; + let weight_map = value + .get("weight_map") + .and_then(Value::as_object) + .ok_or_else(|| incomplete(format!("`{SAFETENSORS_INDEX}` has no object `weight_map`")))?; + if weight_map.is_empty() { + return Err(incomplete(format!( + "`{SAFETENSORS_INDEX}` has an empty `weight_map`" + ))); + } + + let mut shards = BTreeSet::new(); + for value in weight_map.values() { + let shard = value.as_str().ok_or_else(|| { + incomplete(format!( + "`{SAFETENSORS_INDEX}` contains a non-string shard path" + )) + })?; + validate_root_filename(shard, "Safetensors shard") + .map_err(|error| incomplete(error.to_string()))?; + if !shard.ends_with(".safetensors") { + return Err(incomplete(format!( + "indexed shard `{shard}` must end with `.safetensors`" + ))); + } + if !available.contains(shard) { + return Err(incomplete(format!( + "indexed shard `{shard}` is missing from repository metadata or cache" + ))); + } + shards.insert(shard.to_owned()); + } + plan.files.extend(shards); + Ok(plan) +} + +fn sibling_names(info: &RepoInfo) -> BTreeSet { + info.siblings + .iter() + .filter(|sibling| { + validate_root_filename(&sibling.rfilename, "repository metadata filename").is_ok() + }) + .map(|sibling| sibling.rfilename.clone()) + .collect() +} + +fn cached_root_files(snapshot: &Path) -> Result, HuggingFaceError> { + let entries = fs::read_dir(snapshot).map_err(|error| { + io_error(format!( + "could not read cached snapshot `{}`: {error}", + snapshot.display() + )) + })?; + let mut files = BTreeSet::new(); + for entry in entries { + let entry = entry.map_err(|error| { + io_error(format!( + "could not inspect cached snapshot `{}`: {error}", + snapshot.display() + )) + })?; + if entry.path().is_file() { + if let Some(filename) = entry.file_name().to_str() { + files.insert(filename.to_owned()); + } + } + } + Ok(files) +} + +fn snapshot_for_cached_revision(cache: &Cache, repo: &Repo) -> Result { + let cache_repo = cache.repo(repo.clone()); + let ref_path = cache + .path() + .join(repo.folder_name()) + .join("refs") + .join(repo.revision()); + let sha = fs::read_to_string(&ref_path).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + cache_miss(format!( + "revision `{}` for `{}` has no cache ref", + repo.revision(), + repo.folder_name() + )) + } else { + io_error(format!( + "could not read cache ref `{}`: {error}", + ref_path.display() + )) + } + })?; + let sha = sha.trim(); + if sha.is_empty() { + return Err(incomplete(format!( + "cache ref `{}` has an empty SHA", + ref_path.display() + ))); + } + validate_root_filename(sha, "cached revision SHA") + .map_err(|_| incomplete("cached revision has an unsafe SHA"))?; + let snapshot = cache_repo.pointer_path(sha); + if !snapshot.is_dir() { + return Err(cache_miss(format!( + "snapshot `{}` is not cached", + snapshot.display() + ))); + } + Ok(snapshot) +} + +fn verify_gguf_path(path: &Path, filename: &str) -> Result<(), HuggingFaceError> { + if !path.is_file() { + return Err(incomplete(format!( + "resolved `{filename}` is not a file at `{}`", + path.display() + ))); + } + if path.file_name().and_then(|name| name.to_str()) != Some(filename) { + return Err(incomplete(format!( + "resolved GGUF path does not match requested filename `{filename}`" + ))); + } + let snapshot = path + .parent() + .ok_or_else(|| incomplete(format!("resolved `{filename}` has no snapshot directory")))?; + if snapshot + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + != Some("snapshots") + { + return Err(incomplete(format!( + "resolved `{filename}` is not directly under a `snapshots` directory" + ))); + } + Ok(()) +} + +fn snapshot_sha(snapshot: &Path) -> Result<&str, HuggingFaceError> { + snapshot + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| incomplete("cached snapshot has a non-UTF-8 SHA")) +} + +fn verify_snapshot_paths( + paths: &HashMap, + sha: &str, +) -> Result { + if paths.is_empty() { + return Err(incomplete("no snapshot files were resolved")); + } + let mut common = None; + for (filename, path) in paths { + if !path.is_file() { + return Err(incomplete(format!( + "resolved `{filename}` is not a file at `{}`", + path.display() + ))); + } + let parent = path.parent().ok_or_else(|| { + incomplete(format!("resolved `{filename}` has no snapshot directory")) + })?; + let valid_layout = parent.file_name().and_then(|name| name.to_str()) == Some(sha) + && parent + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + == Some("snapshots"); + if !valid_layout { + return Err(incomplete(format!( + "resolved `{filename}` is outside `snapshots/{sha}`" + ))); + } + match &common { + Some(expected) if expected != parent => { + return Err(incomplete("resolved files belong to different snapshots")); + } + None => common = Some(parent.to_owned()), + _ => {} + } + } + common.ok_or_else(|| incomplete("no snapshot directory was resolved")) +} + +fn validate_nonempty(field: &str, value: &str) -> Result<(), HuggingFaceError> { + if value.trim().is_empty() { + Err(invalid(format!("{field} must not be empty"))) + } else { + Ok(()) + } +} + +fn validate_repo_id(repo_id: &str) -> Result<(), HuggingFaceError> { + validate_repo_relative_path(repo_id, "repository ID")?; + if repo_id.split('/').count() > 2 { + return Err(invalid("repository ID must be `name` or `namespace/name`")); + } + Ok(()) +} + +fn validate_revision(revision: &str) -> Result<(), HuggingFaceError> { + validate_repo_relative_path(revision, "revision") +} + +fn validate_root_filename(filename: &str, field: &str) -> Result<(), HuggingFaceError> { + validate_repo_relative_path(filename, field)?; + if Path::new(filename).components().count() != 1 { + return Err(invalid(format!("{field} must be a root-level filename"))); + } + Ok(()) +} + +fn validate_repo_relative_path(path: &str, field: &str) -> Result<(), HuggingFaceError> { + validate_nonempty(field, path)?; + if path.contains('\\') + || path.contains('\0') + || path.contains(':') + || path.starts_with('~') + || path.split('/').any(str::is_empty) + { + return Err(invalid(format!( + "{field} must use portable repository-relative path syntax" + ))); + } + let path = Path::new(path); + if path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(invalid(format!( + "{field} must not be absolute or contain `.` or `..` components" + ))); + } + for component in path.components() { + let Component::Normal(component) = component else { + unreachable!("non-normal components were rejected above"); + }; + let component = component.to_string_lossy(); + if component.ends_with([' ', '.']) + || component.chars().any(|value| { + value.is_control() || matches!(value, '<' | '>' | '"' | '|' | '?' | '*') + }) + { + return Err(invalid(format!( + "{field} contains characters that are not portable path syntax" + ))); + } + } + Ok(()) +} + +fn is_split_gguf(filename: &str) -> bool { + let stem = filename.strip_suffix(".gguf").unwrap_or(filename); + let Some((prefix, total)) = stem.rsplit_once("-of-") else { + return false; + }; + let Some((_, part)) = prefix.rsplit_once('-') else { + return false; + }; + !part.is_empty() + && !total.is_empty() + && part.bytes().all(|value| value.is_ascii_digit()) + && total.bytes().all(|value| value.is_ascii_digit()) +} + +fn invalid(message: impl Into) -> HuggingFaceError { + HuggingFaceError::InvalidInput { + message: message.into(), + } +} + +fn cache_miss(message: impl Into) -> HuggingFaceError { + HuggingFaceError::CacheMiss { + message: message.into(), + } +} + +fn incomplete(message: impl Into) -> HuggingFaceError { + HuggingFaceError::Incomplete { + message: message.into(), + } +} + +fn hub(message: impl Into) -> HuggingFaceError { + HuggingFaceError::Hub { + message: message.into(), + } +} + +fn io_error(message: impl Into) -> HuggingFaceError { + HuggingFaceError::Io { + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use hf_hub::api::Siblings; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); + const REPO: &str = "owner/model"; + const REVISION: &str = "release"; + const SHA: &str = "0123456789abcdef"; + + struct TempDir(PathBuf); + + impl TempDir { + fn new() -> Self { + let id = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("vllm-cpp-hf-{}-{id}", std::process::id())); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() + } + + fn assert_incomplete_contains( + result: Result, + expected: &str, + ) { + match result { + Err(HuggingFaceError::Incomplete { message }) => assert!( + message.contains(expected), + "expected `{message}` to contain `{expected}`" + ), + other => panic!("expected incomplete error containing `{expected}`, got {other:?}"), + } + } + + fn cache_fixture_at(revision: &str, files: &[(&str, &[u8])]) -> TempDir { + let temp = TempDir::new(); + let cache = Cache::new(temp.0.clone()); + let repo = Repo::with_revision(REPO.to_owned(), RepoType::Model, revision.to_owned()); + let cache_repo = cache.repo(repo); + cache_repo.create_ref(SHA).unwrap(); + let snapshot = cache_repo.pointer_path(SHA); + fs::create_dir_all(&snapshot).unwrap(); + for (filename, contents) in files { + fs::write(snapshot.join(filename), contents).unwrap(); + } + temp + } + + fn cache_fixture(files: &[(&str, &[u8])]) -> TempDir { + cache_fixture_at(DEFAULT_REVISION, files) + } + + #[test] + fn validates_inputs_and_rejects_split_gguf() { + let cases = [ + HuggingFaceModel::gguf("", "model.gguf"), + HuggingFaceModel::gguf(REPO, "model.gguf").revision(""), + HuggingFaceModel::gguf(REPO, "model.gguf").revision("../main"), + HuggingFaceModel::gguf("owner/model/extra", "model.gguf"), + HuggingFaceModel::gguf(REPO, "/model.gguf"), + HuggingFaceModel::gguf(REPO, "nested/model.gguf"), + HuggingFaceModel::gguf(REPO, "model.GGUF"), + HuggingFaceModel::gguf(REPO, "model-00001-of-00002.gguf"), + HuggingFaceModel::gguf(REPO, "model-1-of-2.gguf"), + HuggingFaceModel::gguf(REPO, "model?.gguf"), + ]; + for model in cases { + assert!(matches!( + model.validate(), + Err(HuggingFaceError::InvalidInput { .. }) + )); + } + assert!(HuggingFaceModel::gguf(REPO, "model.gguf") + .revision("refs/pr/1") + .validate() + .is_ok()); + } + + #[test] + fn defaults_to_main_and_accepts_revision_override() { + let default_gguf = HuggingFaceModel::gguf(REPO, "model.gguf"); + let default_safetensors = HuggingFaceModel::safetensors(REPO); + assert_eq!(default_gguf.revision, DEFAULT_REVISION); + assert_eq!(default_safetensors.revision, DEFAULT_REVISION); + + let pinned = default_safetensors.revision(REVISION); + assert_eq!(pinned.revision, REVISION); + assert_eq!(pinned.requested_repo().revision(), REVISION); + } + + #[test] + fn debug_redacts_explicit_token() { + let model = HuggingFaceModel::safetensors(REPO) + .revision(REVISION) + .token("hf_secret_value"); + let debug = format!("{model:?}"); + assert!(debug.contains("[REDACTED]")); + assert!(!debug.contains("hf_secret_value")); + } + + #[test] + fn plans_unsharded_and_indexed_snapshots() { + let unsharded = names(&[CONFIG, TOKENIZER, TOKENIZER_CONFIG, SAFETENSORS]); + assert_eq!( + plan_safetensors(&unsharded).unwrap(), + SafetensorsPlan { + files: vec![CONFIG, TOKENIZER, TOKENIZER_CONFIG, SAFETENSORS] + .into_iter() + .map(str::to_owned) + .collect(), + indexed: false, + } + ); + + let indexed = names(&[ + CONFIG, + TOKENIZER, + SAFETENSORS_INDEX, + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + ]); + let bytes = br#"{"weight_map":{"a":"model-00002-of-00002.safetensors","b":"model-00001-of-00002.safetensors","c":"model-00002-of-00002.safetensors"}}"#; + let plan = plan_indexed_safetensors(&indexed, bytes).unwrap(); + assert_eq!( + plan.files, + vec![ + CONFIG, + TOKENIZER, + SAFETENSORS_INDEX, + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + ] + ); + } + + #[test] + fn rejects_incomplete_or_malformed_safetensors_metadata() { + let missing_core = names(&[CONFIG, SAFETENSORS]); + assert!(matches!( + plan_safetensors(&missing_core), + Err(HuggingFaceError::Incomplete { .. }) + )); + + let indexed = names(&[CONFIG, TOKENIZER, SAFETENSORS_INDEX, "part.safetensors"]); + for (bytes, expected) in [ + (br#"not json"#.as_slice(), "malformed JSON"), + (br#"{}"#.as_slice(), "has no object `weight_map`"), + (br#"{"weight_map":{}}"#.as_slice(), "empty `weight_map`"), + ( + br#"{"weight_map":{"a":3}}"#.as_slice(), + "non-string shard path", + ), + ( + br#"{"weight_map":{"a":"../part.safetensors"}}"#.as_slice(), + "must not be absolute or contain `.` or `..` components", + ), + ( + br#"{"weight_map":{"a":"/part.safetensors"}}"#.as_slice(), + "must use portable repository-relative path syntax", + ), + ( + br#"{"weight_map":{"a":"nested/part.safetensors"}}"#.as_slice(), + "must be a root-level filename", + ), + ( + br#"{"weight_map":{"a":"missing.safetensors"}}"#.as_slice(), + "indexed shard `missing.safetensors` is missing", + ), + ( + br#"{"weight_map":{"a":"part.bin"}}"#.as_slice(), + "must end with `.safetensors`", + ), + ] { + assert_incomplete_contains(plan_indexed_safetensors(&indexed, bytes), expected); + } + } + + #[test] + fn ignores_unrelated_unsafe_safetensors_siblings() { + let info = RepoInfo { + sha: SHA.to_owned(), + siblings: [ + CONFIG, + TOKENIZER, + SAFETENSORS, + "../junk.json", + "/junk.json", + "nested/junk.json", + ] + .into_iter() + .map(|rfilename| Siblings { + rfilename: rfilename.to_owned(), + }) + .collect(), + }; + + let available = sibling_names(&info); + assert_eq!(available, names(&[CONFIG, TOKENIZER, SAFETENSORS])); + assert!(plan_safetensors(&available).is_ok()); + + let unsafe_required = RepoInfo { + sha: SHA.to_owned(), + siblings: ["../config.json", TOKENIZER, SAFETENSORS] + .into_iter() + .map(|rfilename| Siblings { + rfilename: rfilename.to_owned(), + }) + .collect(), + }; + assert_incomplete_contains( + plan_safetensors(&sibling_names(&unsafe_required)), + "required `config.json` is missing", + ); + } + + #[test] + fn resolves_complete_offline_unsharded_cache() { + let fixture = cache_fixture(&[ + (CONFIG, b"{}"), + (TOKENIZER, b"{}"), + (TOKENIZER_CONFIG, b"{}"), + (SAFETENSORS, b"weights"), + ]); + let resolved = HuggingFaceModel::safetensors(REPO) + .cache_dir(&fixture.0) + .offline(true) + .resolve() + .unwrap(); + assert_eq!(resolved.file_name().unwrap(), SHA); + assert_eq!(resolved.parent().unwrap().file_name().unwrap(), "snapshots"); + } + + #[test] + fn resolves_complete_offline_indexed_cache() { + let index = br#"{"weight_map":{"a":"model-00001-of-00002.safetensors","b":"model-00002-of-00002.safetensors"}}"#; + let fixture = cache_fixture_at( + REVISION, + &[ + (CONFIG, b"{}"), + (TOKENIZER, b"{}"), + (SAFETENSORS_INDEX, index), + ("model-00001-of-00002.safetensors", b"one"), + ("model-00002-of-00002.safetensors", b"two"), + ], + ); + assert!(HuggingFaceModel::safetensors(REPO) + .revision(REVISION) + .cache_dir(&fixture.0) + .offline(true) + .resolve() + .is_ok()); + } + + #[test] + fn distinguishes_offline_cache_miss_and_incomplete_snapshot() { + let empty = TempDir::new(); + let miss = HuggingFaceModel::safetensors(REPO) + .cache_dir(&empty.0) + .offline(true) + .resolve(); + assert!(matches!(miss, Err(HuggingFaceError::CacheMiss { .. }))); + + let partial = cache_fixture(&[(CONFIG, b"{}"), (TOKENIZER, b"{}")]); + let incomplete = HuggingFaceModel::safetensors(REPO) + .cache_dir(&partial.0) + .offline(true) + .resolve(); + assert!(matches!( + incomplete, + Err(HuggingFaceError::Incomplete { .. }) + )); + + let index = br#"{"weight_map":{"a":"missing.safetensors"}}"#; + let missing_shard = cache_fixture(&[ + (CONFIG, b"{}"), + (TOKENIZER, b"{}"), + (SAFETENSORS_INDEX, index), + ]); + assert_incomplete_contains( + HuggingFaceModel::safetensors(REPO) + .cache_dir(&missing_shard.0) + .offline(true) + .resolve(), + "indexed shard `missing.safetensors` is missing", + ); + } + + #[test] + fn resolves_offline_gguf_and_reports_missing_file() { + let fixture = cache_fixture(&[("model.gguf", b"gguf")]); + let resolved = HuggingFaceModel::gguf(REPO, "model.gguf") + .cache_dir(&fixture.0) + .offline(true) + .resolve() + .unwrap(); + assert_eq!(resolved.file_name().unwrap(), "model.gguf"); + + let missing = HuggingFaceModel::gguf(REPO, "missing.gguf") + .cache_dir(&fixture.0) + .offline(true) + .resolve(); + assert!(matches!(missing, Err(HuggingFaceError::CacheMiss { .. }))); + } + + #[test] + fn verifies_gguf_filename_and_snapshot_layout() { + let fixture = cache_fixture(&[("model.gguf", b"gguf")]); + let path = Cache::new(fixture.0.clone()) + .repo(Repo::with_revision( + REPO.to_owned(), + RepoType::Model, + DEFAULT_REVISION.to_owned(), + )) + .get("model.gguf") + .unwrap(); + assert!(verify_gguf_path(&path, "model.gguf").is_ok()); + assert_incomplete_contains( + verify_gguf_path(&path, "other.gguf"), + "does not match requested filename", + ); + + let outside = fixture.0.join("outside.gguf"); + fs::write(&outside, b"gguf").unwrap(); + assert_incomplete_contains( + verify_gguf_path(&outside, "outside.gguf"), + "not directly under a `snapshots` directory", + ); + } + + #[test] + fn rejects_mixed_snapshot_paths() { + let temp = TempDir::new(); + let first = temp.0.join("snapshots").join(SHA); + let second = temp.0.join("snapshots").join("other"); + fs::create_dir_all(&first).unwrap(); + fs::create_dir_all(&second).unwrap(); + fs::write(first.join(CONFIG), b"{}").unwrap(); + fs::write(second.join(TOKENIZER), b"{}").unwrap(); + let paths = HashMap::from([ + (CONFIG.to_owned(), first.join(CONFIG)), + (TOKENIZER.to_owned(), second.join(TOKENIZER)), + ]); + assert!(matches!( + verify_snapshot_paths(&paths, SHA), + Err(HuggingFaceError::Incomplete { .. }) + )); + } +} diff --git a/vllm-cpp/src/lib.rs b/vllm-cpp/src/lib.rs index 43c4919..e4a422a 100644 --- a/vllm-cpp/src/lib.rs +++ b/vllm-cpp/src/lib.rs @@ -2,8 +2,9 @@ //! //! # Entry points //! -//! Create an [`Engine`] with [`Engine::load`] or configure native model settings -//! through [`EngineBuilder`]. [`SamplingParams`] owns sampling, stop-string, and +//! Resolve a Hub model with [`HuggingFaceModel`] (default `main`, or an explicit +//! revision), then create an [`Engine`] with [`Engine::load`] or configure native +//! model settings through [`EngineBuilder`]. [`SamplingParams`] owns sampling, stop-string, and //! [`StructuredOutput`] settings for completion calls. The engine provides //! blocking completion, streaming, raw-JSON chat, and [`Engine::submit`] for a //! concurrent [`Request`]. Enable `serde` for `serde_json::Value` chat helpers. @@ -30,8 +31,10 @@ //! [`expected_abi_version`] before versioned structs cross FFI. The default //! `bundled` feature builds the pinned native source. `system` selects a //! caller-provided installation, `dynamic-link` selects shared linking, and -//! `serde` adds typed JSON helpers. CUDA, CUTLASS, Triton AOT, Vulkan, Metal, and -//! external MLX features are experimental bundled build configuration. +//! `serde` adds typed JSON helpers. The non-optional `hf-hub` dependency provides +//! synchronous, cache-aware model retrieval without an async runtime. CUDA, +//! CUTLASS, Triton AOT, Vulkan, Metal, and external MLX features are experimental +//! bundled build configuration. //! //! Dynamic linking does not deploy `libvllm.so` or `libvllm.dylib`; applications //! must make it and its runtime dependencies visible through the platform loader, @@ -43,12 +46,14 @@ mod callback; mod engine; mod error; +mod hf; mod params; mod request; pub use callback::{StreamControl, StreamEvent, StreamOutcome}; pub use engine::{Completion, Engine, EngineBuilder, FinishReason}; -pub use error::Error; +pub use error::{Error, HuggingFaceError}; +pub use hf::HuggingFaceModel; pub use params::{SamplingParams, SchedulerPolicy, StructuredOutput, Toggle}; pub use request::{Request, RequestOutcome}; diff --git a/vllm-cpp/tests/qwen3.rs b/vllm-cpp/tests/qwen3.rs index 967f49e..3ae1aaf 100644 --- a/vllm-cpp/tests/qwen3.rs +++ b/vllm-cpp/tests/qwen3.rs @@ -550,6 +550,44 @@ fn structured_choice_is_enforced() { }); } +#[test] +fn structured_json_schema_is_enforced() { + with_engine(|engine, _| { + let schema = r#"{ + "type": "object", + "properties": { + "location": { "type": "string" }, + "temperature_celsius": { "type": "number" }, + "condition": { "type": "string" } + }, + "required": ["location", "temperature_celsius", "condition"], + "additionalProperties": false + }"#; + let params = SamplingParams::greedy() + .max_tokens(64) + .structured_output(StructuredOutput::JsonSchema(schema.to_owned())); + let completion = engine + .complete( + "Extract the weather report as JSON: Paris is sunny and 22 degrees Celsius.", + ¶ms, + ) + .expect("JSON Schema completion"); + let value: serde_json::Value = + serde_json::from_str(completion.text.trim()).expect("valid structured JSON"); + let object = value.as_object().expect("JSON object"); + assert_eq!(object.len(), 3, "unexpected properties: {object:?}"); + assert!(object + .get("location") + .is_some_and(|value| value.is_string())); + assert!(object + .get("temperature_celsius") + .is_some_and(|value| value.is_number())); + assert!(object + .get("condition") + .is_some_and(|value| value.is_string())); + }); +} + #[test] fn terminal_stop_is_natural_finish_for_completion_and_chat() { with_engine(|engine, _| { diff --git a/vllm-cpp/tests/safe_api.rs b/vllm-cpp/tests/safe_api.rs index 8e25025..f5f69f5 100644 --- a/vllm-cpp/tests/safe_api.rs +++ b/vllm-cpp/tests/safe_api.rs @@ -1,7 +1,11 @@ use static_assertions::{assert_impl_all, assert_not_impl_any}; -use vllm_cpp::{Engine, Error, Request, SchedulerPolicy, Toggle}; +use vllm_cpp::{ + Engine, Error, HuggingFaceError, HuggingFaceModel, Request, SchedulerPolicy, Toggle, +}; assert_impl_all!(Engine: Send, Sync, Clone); +assert_impl_all!(HuggingFaceError: Clone, std::fmt::Debug, Eq, PartialEq); +assert_impl_all!(HuggingFaceModel: Clone, std::fmt::Debug); assert_impl_all!(Request: Send); assert_not_impl_any!(Request: Sync); @@ -9,6 +13,14 @@ fn missing_model() -> &'static str { "/nonexistent/vllm-cpp-rs-safe-api-model" } +#[test] +fn hugging_face_constructors_accept_default_and_explicit_revisions() { + let gguf = HuggingFaceModel::gguf("owner/model", "model.gguf"); + let safetensors = HuggingFaceModel::safetensors("owner/model").revision("release"); + assert!(format!("{gguf:?}").contains("revision: \"main\"")); + assert!(format!("{safetensors:?}").contains("revision: \"release\"")); +} + #[test] fn reports_expected_abi() { assert_eq!(vllm_cpp::expected_abi_version(), 10);