diff --git a/CHANGELOG.md b/CHANGELOG.md index 40651c9..31be1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ All notable changes to this project will be documented in this file. - Checked-in raw Rust declarations for the 19-symbol stable vllm.cpp C API at ABI version 10, with header, symbol, layout, and runtime conformance checks. - A safe API for model loading, blocking completion and streaming, raw-JSON and optional serde chat, structured output, owned sampling parameters, and concurrent request submission, cancellation, waiting, and diagnostics. +- An always-available synchronous `hf-hub` resolver for standalone GGUF files and runtime-complete sparse Safetensors snapshots, defaulting to the Hub's mutable `main` revision with an explicit branch/tag/commit override, cache/token/progress/offline controls, and no async runtime. +- Consistent local, Hugging Face GGUF, and Hugging Face Safetensors model-source arguments across every runnable example, with cache reuse and optional revisions; plus a weather extraction example and model-backed test using JSON-Schema structured output. +- A Clap-based interactive `chat` example with prompt/file startup input, retained system/user/assistant history, supported sampling controls, default streaming or blocking output, and shared local/Hugging Face resolution. - RAII ownership for native engines, requests, completions, and strings, including callback panic containment and callback-thread-safe deferred request cleanup. - Linux x86_64 CPU builds for bundled and system libraries with static or dynamic linking. - Experimental bundled Linux x86_64/aarch64 build integration for CUDA, external CUTLASS, Triton AOT, and Vulkan. diff --git a/Cargo.lock b/Cargo.lock index d7e4e88..06171a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,86 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.4.0" @@ -12,6 +92,52 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -22,126 +148,1233 @@ dependencies = [ ] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "colorchoice" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] -name = "itoa" -version = "1.0.18" +name = "console" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys 0.61.2", +] [[package]] -name = "memchr" -version = "2.8.3" +name = "cookie" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] [[package]] -name = "proc-macro2" -version = "1.0.107" +name = "cookie_store" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" dependencies = [ - "unicode-ident", + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", ] [[package]] -name = "quote" -version = "1.0.47" +name = "crc32fast" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "proc-macro2", + "cfg-if", ] [[package]] -name = "serde" -version = "1.0.229" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "serde_core", + "powerfmt", ] [[package]] -name = "serde_core" -version = "1.0.229" +name = "dirs" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "serde_derive", + "dirs-sys", ] [[package]] -name = "serde_derive" -version = "1.0.229" +name = "dirs-sys" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] -name = "serde_json" -version = "1.0.151" +name = "document-features" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" dependencies = [ - "itoa", - "memchr", + "litrs", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hf-hub" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +dependencies = [ + "dirs", + "http", + "indicatif", + "libc", + "log", + "rand", "serde", - "serde_core", - "zmij", + "serde_json", + "thiserror", + "ureq", + "windows-sys 0.61.2", ] [[package]] -name = "shlex" -version = "2.0.1" +name = "http" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] [[package]] -name = "static_assertions" +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] [[package]] -name = "syn" -version = "3.0.3" +name = "idna_adapter" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "icu_normalizer", + "icu_properties", ] [[package]] -name = "unicode-ident" -version = "1.0.24" +name = "indexmap" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] [[package]] -name = "vllm-cpp" -version = "0.1.0" +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ - "serde_json", - "static_assertions", - "vllm-cpp-sys", + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", ] [[package]] -name = "vllm-cpp-sys" -version = "0.1.0" +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ - "cmake", + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vllm-cpp" +version = "0.1.0" +dependencies = [ + "clap", + "hf-hub", + "serde_json", + "static_assertions", + "vllm-cpp-sys", +] + +[[package]] +name = "vllm-cpp-sys" +version = "0.1.0" +dependencies = [ + "cmake", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47402523226a02bfe5230160dc3ccc089aa6f6f19e7fcbb4e6f824bbb1b4aa62" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", ] [[package]] diff --git a/Justfile b/Justfile index de2666c..08af319 100644 --- a/Justfile +++ b/Justfile @@ -384,6 +384,9 @@ package-test: and (map(select(.name == "vllm-cpp" and .documentation == "https://docs.rs/vllm-cpp")) | length == 1) and (map(select(.name == "vllm-cpp-sys" and .documentation == "https://docs.rs/vllm-cpp-sys")) | length == 1) and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "vllm-cpp-sys" and .req == ("=" + $version) and .uses_default_features == false)) | length == 1) + and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "hf-hub" and .req == "^0.5.0" and .optional == false and .uses_default_features == false and .features == ["ureq"])) | length == 1) + and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "serde_json" and .optional == false and .kind == null)) | length == 1) + and (map(select(.name == "vllm-cpp") | .dependencies[] | select(.name == "clap" and .req == "=4.6.1" and .kind == "dev" and .optional == false and .features == ["derive"])) | length == 1) ' "$metadata" >/dev/null sys_list="$temp/vllm-cpp-sys.list" @@ -485,6 +488,7 @@ package-test: README.md \ examples/README.md \ examples/chat.rs \ + examples/common/mod.rs \ examples/complete.rs \ examples/concurrent.rs \ examples/stream.rs \ @@ -492,6 +496,7 @@ package-test: src/callback.rs \ src/engine.rs \ src/error.rs \ + src/hf.rs \ src/lib.rs \ src/params.rs \ src/request.rs \ @@ -727,8 +732,11 @@ package-test: and .[0].license == "MIT OR Apache-2.0" and .[0].rust_version == "1.85" and .[0].features.default == ["bundled"] - and .[0].features.serde == ["dep:serde_json"] + and .[0].features.serde == [] and (.[0].dependencies | map(select(.name == "vllm-cpp-sys" and .req == ("=" + $version) and .uses_default_features == false)) | length == 1) + and (.[0].dependencies | map(select(.name == "hf-hub" and .req == "^0.5.0" and .optional == false and .uses_default_features == false and .features == ["ureq"])) | length == 1) + and (.[0].dependencies | map(select(.name == "serde_json" and .optional == false and .kind == null)) | length == 1) + and (.[0].dependencies | map(select(.name == "clap" and .req == "=4.6.1" and .kind == "dev" and .optional == false and .features == ["derive"])) | length == 1) ' <(cargo metadata --manifest-path "$safe_root/Cargo.toml" \ --locked --offline --no-deps --format-version 1) >/dev/null @@ -748,12 +756,19 @@ package-test: vllm-cpp-sys = { path = "$sys_root" } EOF cat > "$safe_consumer/src/main.rs" <<'EOF' - use vllm_cpp::{abi_version, expected_abi_version, Engine, Error, SamplingParams}; + use vllm_cpp::{ + abi_version, expected_abi_version, Engine, Error, HuggingFaceModel, SamplingParams, + }; fn main() { assert_eq!(expected_abi_version(), 10); assert_eq!(abi_version(), 10); let _params = SamplingParams::greedy().max_tokens(1); + let resolver = HuggingFaceModel::gguf("owner/model", "model.gguf") + .revision("revision") + .cache_dir("/nonexistent/vllm-cpp-rs-safe-package-hf-cache") + .offline(true); + assert!(resolver.resolve().is_err()); assert!(matches!( Engine::load("/nonexistent/vllm-cpp-rs-safe-package-smoke"), Err(Error::ModelLoad { .. }) diff --git a/README.md b/README.md index 1592fe7..995e1d9 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Rust bindings for [vllm.cpp](https://github.com/mudler/vllm.cpp), organized as: ## Status -The safe crate provides a cloneable engine API for model loading, blocking completion and streaming, non-blocking concurrent requests, structured output, and raw-JSON chat. An optional `serde` feature adds `serde_json::Value` chat helpers. The sys crate provides checked-in generated FFI declarations with C/Rust layout checks and coverage for all 19 exported C symbols. +The safe crate provides a cloneable engine API for local model loading, blocking completion and streaming, non-blocking concurrent requests, structured output, and raw-JSON chat. It also provides an always-available synchronous Hugging Face resolver for standalone GGUF files and runtime-complete sparse Safetensors snapshots, plus a Clap-based interactive chat example using those APIs. An optional `serde` feature adds `serde_json::Value` chat helpers. The sys crate provides checked-in generated FFI declarations with C/Rust layout checks and coverage for all 19 exported C symbols. Linux x86_64 CPU builds support bundled static, bundled dynamic, system static, and system dynamic linking. Bundled CPU builds also target Linux aarch64 and Apple ARM64. Experimental bundled builds expose Linux x86_64/aarch64 build configuration for CUDA, external CUTLASS, Triton AOT, and Vulkan, plus Apple ARM64 Metal and external MLX configuration. Accelerator features are build integration surfaces, not runtime-support claims. vllm.cpp is pinned at `34aedfbe8ed9779697905541a62e2160ccfd9c05`, which exposes C ABI version 10. @@ -41,7 +41,9 @@ git submodule update --init --recursive ## Safe API -The packaged [`vllm-cpp` guide](vllm-cpp/README.md) covers model requirements, safe ownership, callbacks, concurrency, features, link modes, and deployment. The [`vllm-cpp-sys` guide](vllm-cpp-sys/README.md) documents the raw ABI and native build boundary. +The packaged [`vllm-cpp` guide](vllm-cpp/README.md) covers local and Hugging Face model resolution, safe ownership, callbacks, concurrency, features, link modes, and deployment. The [`vllm-cpp-sys` guide](vllm-cpp-sys/README.md) documents the raw ABI and native build boundary. + +`Engine::load` accepts a native-compatible model directory or standalone GGUF. `HuggingFaceModel` synchronously resolves into the normal Hugging Face cache before engine construction, defaulting to the Hub's mutable `main` revision; `.revision(...)` can pin a branch, tag, or commit. GGUF mode selects one safe root file. Safetensors mode pins downloads to repository metadata's commit SHA and retrieves only native runtime requirements: root configuration/tokenizer files and either unsharded weights or an index plus all root shards. Every runnable example accepts a bare or explicit local path and both Hub artifact forms with optional `--revision`. Cached downloads are reused. Retrieval does not prove model/backend compatibility. `EngineBuilder` owns model settings and converts them to temporary C strings only for the load call. `SamplingParams` owns stop strings and structured constraints. Completion and chat strings are copied into Rust values before the matching native free function runs. @@ -49,7 +51,7 @@ The packaged [`vllm-cpp` guide](vllm-cpp/README.md) covers model requirements, s All streaming callbacks receive copied UTF-8 deltas. Blocking callbacks may borrow stack data; their panics are caught before the C boundary and resumed only after the native call returns. Asynchronous callbacks must be `Send + 'static`, run on a native delivery thread, and report panic as `Error::CallbackPanicked` from `wait`. Waiting for or freeing a request from its own callback thread is prohibited by ABI v10: `wait` returns `Error::RequestCallbackThread`, while drop transfers cleanup to a prestarted reaper that owns the request, callback, and engine until native free/cancel/join completes. Chat methods accept raw OpenAI-compatible request JSON; enable `serde` for `serde_json::Value` request and response helpers. -See [the examples guide](vllm-cpp/examples/README.md) for ordinary Linux and optional Nix setup and commands for every example. Release-facing changes are recorded in the [changelog](CHANGELOG.md), and maintainers use the manual [release process](RELEASING.md). +See [the examples guide](vllm-cpp/examples/README.md) for ordinary Linux and optional Nix setup, commands for every example, and the interactive chat CLI's local/Hub model forms and generation options. Release-facing changes are recorded in the [changelog](CHANGELOG.md), and maintainers use the manual [release process](RELEASING.md). ## Build and Test @@ -64,7 +66,7 @@ just ci Set `CMAKE_BUILD_PARALLEL_LEVEL` to control native parallelism. The default bundled build remains deterministic and CPU-only: native tests, examples, the HTTP server, CUDA, Metal, MLX, Vulkan, Triton, and CUTLASS fetching are disabled explicitly. Use `nix develop .#msrv -c just msrv` for the exact local Rust 1.85.0 policy check; the manual `platforms` workflow runs the same exact toolchain policy. -`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. +`build.rs` is consumer-only native build/link integration; it does not download dependencies or compile/execute the maintainer layout probe. Ordinary consumers do not need Just, bindgen, or libclang. Normal first-time Cargo dependency resolution may access crates.io; use Cargo's standard `--offline` mode after dependencies are cached. The high-level crate's required `hf-hub` 0.5 dependency uses only its synchronous `ureq` feature, without Tokio or another async runtime. Library download progress is disabled by default. ## Experimental Backend Builds @@ -103,7 +105,7 @@ Compilation does not establish runtime correctness. Known native evidence blocke ## Test Model and Sanitizers -Model-backed tests use Apache-2.0 `Qwen/Qwen3-0.6B` at pinned revision `c1899de289a04d12100db370d81485cdf75e47ca`. Download or reuse the cache and verify every file, then run exactly 14 blocking and request-lifecycle model tests serially: +Model-backed tests use Apache-2.0 `Qwen/Qwen3-0.6B` at pinned revision `c1899de289a04d12100db370d81485cdf75e47ca`. Download or reuse the cache and verify every file, then run exactly 15 blocking and request-lifecycle model tests serially, including choice and JSON-Schema structured-output enforcement: ```console model=$(just setup-test-model) diff --git a/vllm-cpp/Cargo.toml b/vllm-cpp/Cargo.toml index 6e9e481..e9d0754 100644 --- a/vllm-cpp/Cargo.toml +++ b/vllm-cpp/Cargo.toml @@ -22,16 +22,17 @@ triton-aot = ["cuda", "vllm-cpp-sys/triton-aot"] vulkan = ["vllm-cpp-sys/vulkan"] metal = ["vllm-cpp-sys/metal"] mlx = ["metal", "vllm-cpp-sys/mlx"] -serde = ["dep:serde_json"] +serde = [] [package.metadata.docs.rs] features = ["serde"] targets = ["x86_64-unknown-linux-gnu"] [dependencies] -serde_json = { version = "1.0.149", optional = true } +hf-hub = { version = "0.5.0", default-features = false, features = ["ureq"] } +serde_json = "1.0.149" vllm-cpp-sys = { workspace = true, default-features = false } [dev-dependencies] -serde_json = "1.0.149" +clap = { version = "=4.6.1", features = ["derive"] } static_assertions = "1.1.0" diff --git a/vllm-cpp/README.md b/vllm-cpp/README.md index ffcf185..4c0f5ae 100644 --- a/vllm-cpp/README.md +++ b/vllm-cpp/README.md @@ -14,7 +14,28 @@ println!("{}", completion.text); # Ok::<(), vllm_cpp::Error>(()) ``` -The model argument is a directory understood by the pinned native engine, not a single weights file. The known-good test layout contains `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; model-family compatibility remains a native vllm.cpp concern. See the packaged [examples guide](examples/README.md) for blocking completion, streaming, chat, structured output, and concurrent-request commands. +`Engine::load` accepts either a model directory or a standalone GGUF file understood by the pinned native engine. The known-good Safetensors test layout contains `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; model-family compatibility remains a native vllm.cpp concern. See the packaged [examples guide](examples/README.md) for local and Hugging Face loading, blocking completion, streaming, JSON-Schema output, concurrent-request commands, and the Clap-based interactive chat CLI with retained history and supported sampling controls. + +## Hugging Face models + +`HuggingFaceModel` is an always-available synchronous resolver backed by the required `hf-hub` 0.5 dependency. It returns a local `PathBuf` that can be passed unchanged to `Engine::load`: + +```rust,no_run +use vllm_cpp::{Engine, HuggingFaceModel}; + +let path = HuggingFaceModel::gguf("owner/repository", "model.gguf") + // Omit this builder to follow the Hub's mutable `main` revision. + .revision("0123456789abcdef0123456789abcdef01234567") + .resolve()?; +let engine = Engine::load(path)?; +# Ok::<(), Box>(()) +``` + +`HuggingFaceModel::gguf(repo, filename)` and `HuggingFaceModel::safetensors(repo)` default to the Hub's mutable `main` revision. The `.revision(...)` builder accepts a branch, tag, or commit; immutable commit SHAs are recommended for reproducibility. The default cache honors `HF_HOME` through the normal Hugging Face layout and uses the cached login token when available. Builders can select a Hub cache directory, override the token, enable progress, or require cache-only offline resolution. Explicit tokens are redacted from resolver `Debug` output. The official endpoint is fixed; `HF_ENDPOINT` is not used. + +GGUF mode retrieves one root-level lowercase `.gguf` file and rejects split sets. Safetensors mode first reads repository metadata for `main` or the explicit revision, pins downloads to its commit SHA, and retrieves only the root files required by this native loader: `config.json`, `tokenizer.json`, optional `tokenizer_config.json`, and either `model.safetensors` or the root index plus every indexed root shard. It returns the shared `snapshots/` directory and creates that revision's cache ref only after a complete successful retrieval. It does not download unrelated repository assets. Offline mode constructs no network API, reads the cached `main` ref by default, and distinguishes a missing cache revision from an incomplete cached snapshot. + +Retrieval validates cache and snapshot completeness; it does not prove that the pinned native engine supports the repository's model architecture, tokenizer, quantization, or backend. ## API and ownership @@ -32,7 +53,7 @@ The model argument is a directory understood by the pinned native engine, not a | `bundled` (default) | Build and statically link the pinned CPU native source | | `system` | Link a caller-provided installation; use with `--no-default-features` | | `dynamic-link` | Link `libvllm` dynamically in bundled or system mode | -| `serde` | Add `serde_json::Value` chat helpers | +| `serde` | Add `serde_json::Value` chat helpers; JSON parsing for Hub resolution is always present | | `cuda` | Experimental bundled CUDA build configuration | | `cuda-cutlass` | Experimental CUDA build with a caller-provided CUTLASS >=4.5.0 tree | | `triton-aot` | Experimental CUDA build using checked-in Triton AOT artifacts | @@ -40,6 +61,8 @@ The model argument is a directory understood by the pinned native engine, not a | `metal` | Experimental native Metal build on Apple ARM64 | | `mlx` | Experimental external MLX provider on Apple ARM64; implies `metal` | +Hugging Face resolution is not a Cargo feature: synchronous `hf-hub` support is a normal dependency in every build and does not add Tokio or another async runtime. + `bundled` and `system` conflict. CUDA and Vulkan conflict, and accelerator features are bundled-only but do not implicitly enable `bundled` for `--no-default-features` builds. Metal/MLX require exact `aarch64-apple-darwin`; MLX additionally requires an external `MLX_ROOT` with its headers, dylib, and metallib. The workspace [backend documentation](https://github.com/querymt/vllm-cpp-rs#experimental-backend-builds) records exact environment variables, supported build architectures, and current blockers. ## ABI and deployment diff --git a/vllm-cpp/examples/README.md b/vllm-cpp/examples/README.md index 81e0e19..3eac25b 100644 --- a/vllm-cpp/examples/README.md +++ b/vllm-cpp/examples/README.md @@ -1,16 +1,33 @@ # examples -these examples exercise the safe `vllm-cpp` api with fixed prompts and settings: +these examples exercise the safe `vllm-cpp` api. four use fixed prompts and settings; `chat` is an interactive command-line application: | example | behavior | |---|---| | [`complete`](complete.rs) | runs one blocking text completion | | [`stream`](stream.rs) | prints one completion as token deltas arrive | | [`concurrent`](concurrent.rs) | submits two asynchronous streaming requests and waits for both | -| [`chat`](chat.rs) | sends a fixed raw-json chat request; the optional `serde` feature is not required | -| [`structured`](structured.rs) | constrains one completion to the choice `red` or `blue` | +| [`chat`](chat.rs) | runs a Clap-based interactive chat with conversation history and streaming output | +| [`structured`](structured.rs) | extracts a fixed weather report under a JSON Schema | -each example reads the first positional argument as a model directory and implements no additional options. a usable directory must contain the runtime files `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; the pinned [test model fixture and layout](https://github.com/querymt/vllm-cpp-rs#test-model-and-sanitizers) is the known-good reference. model compatibility depends on the native engine, so an arbitrary model directory is not guaranteed to work. +The four fixed examples (`complete`, `stream`, `concurrent`, and `structured`) accept the same manual model-source forms: + +```console +EXAMPLE +EXAMPLE local +EXAMPLE hf-gguf [--revision ] +EXAMPLE hf-safetensors [--revision ] +``` + +The bare path remains an alias for `local`. Hub forms enable download progress, use `main` when `--revision` is omitted, and reuse files already present in Hugging Face's normal cache. `Engine::load` receives the local or resolved `PathBuf` unchanged. It accepts either a model directory or a standalone GGUF file understood by the pinned native engine. A known-good Safetensors directory contains `model.safetensors`, `config.json`, `tokenizer.json`, and `tokenizer_config.json`; indexed models instead use `model.safetensors.index.json` and all referenced root shards. Model compatibility depends on the native engine, so an arbitrary directory, GGUF, or Hub repository is not guaranteed to work. + +## hugging face loading + +Omitting `--revision` follows the Hub's mutable default `main` branch. Use `.revision(...)` in the library or `--revision` in an example to select a branch, tag, or commit; an immutable commit SHA is recommended for reproducibility. The resolver is synchronous and always available because `hf-hub` is a normal dependency, not a Cargo feature. + +By default the resolver uses Hugging Face's normal cache, honoring `HF_HOME`, and selects the cached token created by Hugging Face login. Library callers can use `.revision(...)`, `.cache_dir(...)`, `.token(...)`, `.progress(...)`, and `.offline(true)`. An explicit token overrides the selected cached token and is redacted from resolver `Debug`. Offline default resolution reads only the cached `main` ref; an explicit revision reads only that cached ref. Offline mode constructs no API. The official Hugging Face endpoint remains fixed; `HF_ENDPOINT` is not used. + +GGUF mode retrieves one safe root-level filename ending in lowercase `.gguf`; split GGUF sets are unsupported. Safetensors mode queries metadata for `main` or the explicit revision, pins file retrieval to the returned commit SHA, and creates that revision's cache ref after complete success. It retrieves only `config.json`, `tokenizer.json`, optional `tokenizer_config.json`, and either root `model.safetensors` or root `model.safetensors.index.json` plus every unique root shard in `weight_map`. Unrelated repository assets are not downloaded. Successful retrieval establishes a complete sparse snapshot for the pinned loader, not model-family or backend compatibility. ## ordinary linux @@ -30,28 +47,51 @@ git submodule update --init --recursive the commands below select ninja explicitly with `CMAKE_GENERATOR=Ninja`; merely installing ninja does not configure cmake to use it. if you choose another cmake generator and build tool, install them and set or otherwise configure `CMAKE_GENERATOR` accordingly. -the common command shape is: +The common command shape is: ```console CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example EXAMPLE -- /path/to/model + cargo run --locked --release -p vllm-cpp --features bundled --example EXAMPLE -- MODEL_SOURCE ``` -`bundled` is the default feature; it is shown explicitly here to identify the cpu backend. run any of the five examples with: +`bundled` is the default feature; it is shown explicitly here to identify the CPU backend. These commands demonstrate all five examples and the shared source syntax: ```console -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example complete -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example stream -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example concurrent -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example chat -- /path/to/model -CMAKE_GENERATOR=Ninja \ - cargo run --locked --release -p vllm-cpp --features bundled --example structured -- /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example complete -- /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example stream -- local /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example concurrent -- hf-gguf owner/repository model.gguf +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example chat -- --system "Answer concisely." hf-safetensors owner/repository +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --features bundled --example structured -- hf-safetensors owner/repository --revision release ``` +Replace the fixed examples' arguments after `--` with another model-source form from the syntax block. The chat command uses the same source names through Clap; its exact syntax and interactive options follow. No example requires a revision. + +## interactive chat CLI + +`chat` accepts global chat options before or after the model subcommand and uses one of these model forms: + +```console +chat [OPTIONS] +chat [OPTIONS] local [OPTIONS] +chat [OPTIONS] hf-gguf [--revision ] [OPTIONS] +chat [OPTIONS] hf-safetensors [--revision ] [OPTIONS] +``` + +The bare path remains a local alias. The Hub subcommands reuse the shared resolver: progress is enabled, cached files are reused, omitted revisions follow mutable `main`, and `--revision` can select a branch, tag, or commit. These complete commands show local and Hub ordering: + +```console +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --example chat -- \ + --system "Answer concisely." --prompt "Hello" local /path/to/model +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --example chat -- \ + hf-gguf owner/repository model.gguf --revision release --temperature 0.5 +CMAKE_GENERATOR=Ninja cargo run --locked --release -p vllm-cpp --example chat -- \ + --file prompt.txt hf-safetensors owner/repository --no-stream +``` + +`--prompt/-p ` and `--file/-f ` are mutually exclusive optional first user messages; prompt files must be UTF-8. `--system ` adds a retained system message. Generation options are `--max-tokens` (default `256`, maximum `2147483647`), `--temperature` (default `0.7`), `--top-p` (default `1`), `--top-k` (default `0`), `--min-p` (default `0`), and optional `--seed`. Responses stream by default; `--no-stream` selects blocking `Engine::chat_json` output. The CLI maintains the complete user/assistant history, submits it on each turn, and prints only assistant content rather than raw response JSON. Native role, reasoning, tool-call, and finish metadata is ignored; a valid response with no content is stored as an empty assistant message. + +At `user>` enter `/clear` to retain the system message while clearing other history, or `/quit`/`/exit` to stop. EOF also exits cleanly. A per-turn request or response error is reported as `chat: `, the attempted turn is removed from history, and the prompt continues; terminal input/output errors and model startup failures still exit. This high-level example intentionally exposes only controls supported by the existing chat request and engine APIs; it does not add low-level model, device, context, batch, token, or timing controls from other runtimes. + ## optional nix shell nix is optional and works on supported linux installations with nix; it does not require nixos. the default development shell supplies the cpu build dependencies. from the workspace root, run: @@ -61,7 +101,7 @@ CMAKE_GENERATOR=Ninja \ nix develop -c cargo run --locked --release -p vllm-cpp --features bundled --example complete -- /path/to/model ``` -replace `complete` with any other example name from the table. +replace `complete` with another example and its arguments from the table. ## experimental cuda @@ -97,6 +137,8 @@ other accelerator features have stricter limits: - if the native source or cmake inputs are missing, rerun `git submodule update --init --recursive`. - the first bundled build compiles the native c++ library and can take substantially longer than later runs. -- if model loading fails, verify the directory argument and its model, configuration, and tokenizer files, then confirm that vllm.cpp supports the model. +- if local model loading fails, verify the directory or standalone GGUF argument and required files, then confirm that vllm.cpp supports the model. +- if Hub resolution fails offline, verify that the exact requested revision has a cache ref and a complete single snapshot; offline mode never contacts the network. +- successful Hub resolution does not establish architecture, tokenizer, quantization, or backend support in the pinned native engine. - for `dynamic-link` or `system` builds, follow the root [link mode and loader-path requirements](https://github.com/querymt/vllm-cpp-rs#link-modes); cargo does not deploy `libvllm.so` or configure its runtime search path. - for accelerator configuration errors, use a fresh target directory and consult the root [experimental backend build details](https://github.com/querymt/vllm-cpp-rs#experimental-backend-builds). diff --git a/vllm-cpp/examples/chat.rs b/vllm-cpp/examples/chat.rs index 77341d7..6c41069 100644 --- a/vllm-cpp/examples/chat.rs +++ b/vllm-cpp/examples/chat.rs @@ -1,17 +1,1044 @@ -use vllm_cpp::Engine; - -fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: chat ")?; - let engine = Engine::load(model)?; - let response = engine.chat_json( - r#"{ - "messages": [{"role": "user", "content": "Reply with hello."}], - "temperature": 0, - "max_tokens": 16 - }"#, - )?; - println!("{response}"); - Ok(()) +mod common; + +use std::error::Error; +use std::fmt; +use std::fs; +use std::io::{self, BufRead, Write}; +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::{Parser, Subcommand}; +use serde_json::{json, Value}; +use vllm_cpp::{Engine, StreamControl}; + +use common::ModelSource; + +#[derive(Debug, Parser)] +#[command( + name = "chat", + about = "Chat interactively with a local or Hugging Face model", + override_usage = "chat [OPTIONS] \n chat [OPTIONS] ", + subcommand_negates_reqs = true +)] +struct Args { + /// Bare local model directory or GGUF path (an alias for `local `). + #[arg(value_name = "MODEL", required = true)] + model_path: Option, + + #[command(subcommand)] + model: Option, + + /// Initial user message. + #[arg( + short = 'p', + long, + global = true, + conflicts_with = "file", + value_name = "TEXT" + )] + prompt: Option, + + /// Read the initial user message from a UTF-8 file. + #[arg( + short = 'f', + long, + global = true, + conflicts_with = "prompt", + value_name = "PATH" + )] + file: Option, + + /// System message added at the start of the conversation. + #[arg(long, global = true, value_name = "TEXT")] + system: Option, + + /// Maximum tokens generated per response, from 1 through 2147483647. + #[arg(long, global = true, default_value_t = 256, value_parser = parse_max_tokens)] + max_tokens: u32, + + /// Sampling temperature in [0, 2]. + #[arg(long, global = true, default_value_t = 0.7, value_parser = parse_temperature)] + temperature: f64, + + /// Nucleus-sampling probability in (0, 1]. + #[arg(long, global = true, default_value_t = 1.0, value_parser = parse_top_p)] + top_p: f64, + + /// Number of top tokens to consider; use 0 or -1 for all tokens. + #[arg(long, global = true, default_value_t = 0, allow_hyphen_values = true, value_parser = parse_top_k)] + top_k: i32, + + /// Minimum token probability relative to the most likely token, in [0, 1]. + #[arg(long, global = true, default_value_t = 0.0, value_parser = parse_min_p)] + min_p: f64, + + /// Random seed for generation. + #[arg(long, global = true, allow_hyphen_values = true)] + seed: Option, + + /// Wait for each complete response instead of printing streamed deltas. + #[arg(long, global = true)] + no_stream: bool, +} + +#[derive(Debug, Subcommand)] +enum Model { + /// Use a local model directory or standalone GGUF file. + Local { + #[arg(value_name = "PATH")] + path: PathBuf, + }, + + /// Download one GGUF file from Hugging Face or reuse its cached copy. + #[command(name = "hf-gguf")] + HuggingFaceGguf { + /// Hugging Face repository, for example `owner/model`. + #[arg(value_name = "REPO")] + repo: String, + /// Root-level GGUF filename in the repository. + #[arg(value_name = "FILENAME")] + filename: String, + /// Branch, tag, or commit; defaults to the mutable `main` revision. + #[arg(long, value_name = "REVISION")] + revision: Option, + }, + + /// Download a runtime-complete Safetensors snapshot from Hugging Face. + #[command(name = "hf-safetensors")] + HuggingFaceSafetensors { + /// Hugging Face repository, for example `owner/model`. + #[arg(value_name = "REPO")] + repo: String, + /// Branch, tag, or commit; defaults to the mutable `main` revision. + #[arg(long, value_name = "REVISION")] + revision: Option, + }, +} + +impl Model { + fn into_source(self) -> ModelSource { + match self { + Self::Local { path } => ModelSource::Local(path), + Self::HuggingFaceGguf { + repo, + filename, + revision, + } => ModelSource::Gguf { + repo, + filename, + revision, + }, + Self::HuggingFaceSafetensors { repo, revision } => { + ModelSource::Safetensors { repo, revision } + } + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Role { + System, + User, + Assistant, +} + +impl Role { + const fn as_str(self) -> &'static str { + match self { + Self::System => "system", + Self::User => "user", + Self::Assistant => "assistant", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct Message { + role: Role, + content: String, +} + +impl Message { + fn new(role: Role, content: impl Into) -> Self { + Self { + role, + content: content.into(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +struct GenerationConfig { + max_tokens: u32, + temperature: f64, + top_p: f64, + top_k: i32, + min_p: f64, + seed: Option, +} + +#[derive(Debug)] +enum ChatError { + Turn(String), + Fatal(String), +} + +impl ChatError { + fn turn(message: impl Into) -> Self { + Self::Turn(message.into()) + } + + fn fatal(message: impl Into) -> Self { + Self::Fatal(message.into()) + } + + const fn is_fatal(&self) -> bool { + matches!(self, Self::Fatal(_)) + } +} + +impl fmt::Display for ChatError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Turn(message) | Self::Fatal(message) => formatter.write_str(message), + } + } +} + +impl Error for ChatError {} + +type ChatResult = Result; + +fn main() -> ExitCode { + match run(Args::parse()) { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("chat: {error}"); + ExitCode::FAILURE + } + } +} + +fn run(args: Args) -> ChatResult<()> { + let Args { + model_path, + model, + prompt, + file, + system, + max_tokens, + temperature, + top_p, + top_k, + min_p, + seed, + no_stream, + } = args; + + let initial_prompt = read_initial_prompt(prompt, file)?; + let source = match (model_path, model) { + (Some(path), None) => ModelSource::Local(path), + (None, Some(model)) => model.into_source(), + _ => return Err(ChatError::fatal("exactly one model source is required")), + }; + let model = source + .resolve() + .map_err(|error| ChatError::fatal(format!("failed to resolve model source: {error}")))?; + let engine = Engine::load(model) + .map_err(|error| ChatError::fatal(format!("failed to load model: {error}")))?; + let config = GenerationConfig { + max_tokens, + temperature, + top_p, + top_k, + min_p, + seed, + }; + + let mut history = Vec::new(); + if let Some(system) = system { + history.push(Message::new(Role::System, system)); + } + + write_stdout_line("Interactive chat: use /clear to reset the conversation or /quit to exit.")?; + if let Some(prompt) = initial_prompt { + handle_turn( + &mut history, + prompt, + |history, prompt| run_turn(&engine, history, &config, !no_stream, prompt), + report_turn_error, + )?; + } + interactive_loop(&engine, &mut history, &config, !no_stream) +} + +fn read_initial_prompt( + prompt: Option, + file: Option, +) -> ChatResult> { + match (prompt, file) { + (Some(prompt), None) => Ok(Some(prompt)), + (None, Some(path)) => fs::read_to_string(&path).map(Some).map_err(|error| { + ChatError::fatal(format!( + "failed to read UTF-8 prompt file `{}`: {error}", + path.display() + )) + }), + (None, None) => Ok(None), + (Some(_), Some(_)) => Err(ChatError::fatal( + "--prompt and --file cannot be used together", + )), + } +} + +fn interactive_loop( + engine: &Engine, + history: &mut Vec, + config: &GenerationConfig, + stream: bool, +) -> ChatResult<()> { + let stdin = io::stdin(); + let mut stdin = stdin.lock(); + let mut line = String::new(); + + loop { + { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + write!(stdout, "user> ") + .and_then(|()| stdout.flush()) + .map_err(|error| { + ChatError::fatal(format!("failed to write input prompt: {error}")) + })?; + } + + line.clear(); + let bytes = stdin + .read_line(&mut line) + .map_err(|error| ChatError::fatal(format!("failed to read standard input: {error}")))?; + if bytes == 0 { + write_stdout_line("")?; + return Ok(()); + } + + let input = line.trim_end_matches(['\r', '\n']); + match classify_input(input) { + Input::Quit => return Ok(()), + Input::Clear => { + history.retain(|message| message.role == Role::System); + write_stdout_line("Conversation cleared.")?; + } + Input::Empty => {} + Input::Message(message) => { + handle_turn( + history, + message.to_owned(), + |history, message| run_turn(engine, history, config, stream, message), + report_turn_error, + )?; + } + } + } +} + +fn run_turn( + engine: &Engine, + history: &mut Vec, + config: &GenerationConfig, + stream: bool, + user_message: String, +) -> ChatResult<()> { + run_history_turn(history, user_message, |history| { + let request = build_request(history, config, stream)?; + if stream { + write_assistant_prefix()?; + let result = stream_assistant(engine, &request); + let content = result.as_ref().map_or("", |assistant| assistant.as_str()); + finish_assistant_output(content)?; + result + } else { + engine + .chat_json(&request) + .map_err(|error| ChatError::turn(format!("blocking chat request failed: {error}"))) + .and_then(|response| extract_blocking_content(&response)) + .and_then(|assistant| { + write_assistant_prefix()?; + { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + stdout.write_all(assistant.as_bytes()).map_err(|error| { + ChatError::fatal(format!("failed to write assistant response: {error}")) + })?; + } + finish_assistant_output(&assistant)?; + Ok(assistant) + }) + } + }) +} + +fn run_history_turn( + history: &mut Vec, + user_message: String, + run_assistant: F, +) -> ChatResult<()> +where + F: FnOnce(&mut Vec) -> ChatResult, +{ + let checkpoint = history.len(); + history.push(Message::new(Role::User, user_message)); + match run_assistant(history) { + Ok(assistant) => { + history.push(Message::new(Role::Assistant, assistant)); + Ok(()) + } + Err(error) => { + rollback_turn(history, checkpoint); + Err(error) + } + } +} + +fn handle_turn( + history: &mut Vec, + user_message: String, + run: F, + mut report_error: R, +) -> ChatResult<()> +where + F: FnOnce(&mut Vec, String) -> ChatResult<()>, + R: FnMut(&ChatError), +{ + match run(history, user_message) { + Ok(()) => Ok(()), + Err(error) if error.is_fatal() => Err(error), + Err(error) => { + report_error(&error); + Ok(()) + } + } +} + +fn report_turn_error(error: &ChatError) { + eprintln!("chat: {error}"); +} + +fn rollback_turn(history: &mut Vec, checkpoint: usize) { + history.truncate(checkpoint); +} + +fn stream_assistant(engine: &Engine, request: &str) -> ChatResult { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + let mut assistant = String::new(); + let mut callback_error = None; + + let outcome = engine.chat_stream_json(request, |event| { + if callback_error.is_some() { + return StreamControl::Stop; + } + if event.finished { + return StreamControl::Continue; + } + + match accumulate_stream_content(&mut assistant, &event.delta) { + Ok(delta) => { + if let Err(error) = stdout + .write_all(delta.as_bytes()) + .and_then(|()| stdout.flush()) + { + callback_error = Some(ChatError::fatal(format!( + "failed to write streaming assistant response: {error}" + ))); + StreamControl::Stop + } else { + StreamControl::Continue + } + } + Err(error) => { + callback_error = Some(error); + StreamControl::Stop + } + } + }); + drop(stdout); + + if let Some(error) = callback_error { + return Err(error); + } + let outcome = outcome + .map_err(|error| ChatError::turn(format!("streaming chat request failed: {error}")))?; + complete_stream(assistant, outcome.stopped_by_callback) +} + +fn accumulate_stream_content(assistant: &mut String, chunk: &str) -> ChatResult { + let content = extract_stream_content(chunk)?; + assistant.push_str(&content); + Ok(content) +} + +fn complete_stream(assistant: String, stopped_by_callback: bool) -> ChatResult { + if stopped_by_callback { + Err(ChatError::turn( + "streaming chat response stopped unexpectedly", + )) + } else { + Ok(assistant) + } +} + +fn write_stdout_line(message: &str) -> ChatResult<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + writeln!(stdout, "{message}") + .and_then(|()| stdout.flush()) + .map_err(|error| ChatError::fatal(format!("failed to write standard output: {error}"))) +} + +fn write_assistant_prefix() -> ChatResult<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + write!(stdout, "assistant> ") + .and_then(|()| stdout.flush()) + .map_err(|error| ChatError::fatal(format!("failed to write assistant prompt: {error}"))) +} + +fn finish_assistant_output(content: &str) -> ChatResult<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + if !content.ends_with('\n') { + writeln!(stdout).map_err(|error| { + ChatError::fatal(format!("failed to finish assistant response: {error}")) + })?; + } + stdout + .flush() + .map_err(|error| ChatError::fatal(format!("failed to flush assistant response: {error}"))) +} + +fn build_request( + messages: &[Message], + config: &GenerationConfig, + stream: bool, +) -> ChatResult { + let messages: Vec<_> = messages + .iter() + .map(|message| { + json!({ + "role": message.role.as_str(), + "content": message.content, + }) + }) + .collect(); + let mut request = json!({ + "messages": messages, + "max_tokens": config.max_tokens, + "temperature": config.temperature, + "top_p": config.top_p, + "top_k": config.top_k, + "min_p": config.min_p, + "stream": stream, + }); + if let Some(seed) = config.seed { + request["seed"] = json!(seed); + } + serde_json::to_string(&request) + .map_err(|error| ChatError::turn(format!("failed to serialize chat request: {error}"))) +} + +fn extract_blocking_content(response: &str) -> ChatResult { + let response: Value = serde_json::from_str(response).map_err(|error| { + ChatError::turn(format!("invalid blocking chat response JSON: {error}")) + })?; + let choices = response + .get("choices") + .and_then(Value::as_array) + .ok_or_else(|| ChatError::turn("blocking chat response is missing array `choices`"))?; + let choice = choices + .first() + .and_then(Value::as_object) + .ok_or_else(|| ChatError::turn("blocking chat response is missing object `choices[0]`"))?; + let message = choice + .get("message") + .and_then(Value::as_object) + .ok_or_else(|| { + ChatError::turn("blocking chat response is missing object `choices[0].message`") + })?; + + match message.get("content") { + Some(Value::String(content)) => Ok(content.clone()), + None | Some(Value::Null) => Ok(String::new()), + Some(_) => Err(ChatError::turn( + "blocking chat response has non-string `choices[0].message.content`", + )), + } +} + +fn extract_stream_content(chunk: &str) -> ChatResult { + let chunk: Value = serde_json::from_str(chunk) + .map_err(|error| ChatError::turn(format!("invalid streaming chat chunk JSON: {error}")))?; + let choices = chunk + .get("choices") + .and_then(Value::as_array) + .ok_or_else(|| ChatError::turn("streaming chat chunk is missing array `choices`"))?; + let mut content = String::new(); + for (index, choice) in choices.iter().enumerate() { + let delta = choice + .get("delta") + .and_then(Value::as_object) + .ok_or_else(|| { + ChatError::turn(format!( + "streaming chat chunk is missing object `choices[{index}].delta`" + )) + })?; + match delta.get("content") { + Some(Value::String(part)) => content.push_str(part), + None | Some(Value::Null) => {} + Some(_) => { + return Err(ChatError::turn(format!( + "streaming chat chunk has non-string `choices[{index}].delta.content`" + ))) + } + } + } + Ok(content) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Input<'a> { + Quit, + Clear, + Empty, + Message(&'a str), +} + +fn classify_input(input: &str) -> Input<'_> { + match input.trim() { + "/quit" | "/exit" => Input::Quit, + "/clear" => Input::Clear, + "" => Input::Empty, + _ => Input::Message(input), + } +} + +fn parse_max_tokens(value: &str) -> Result { + let value = value + .parse::() + .map_err(|error| format!("invalid token count: {error}"))?; + if value == 0 { + return Err("max tokens must be greater than zero".to_owned()); + } + if value > i32::MAX as u32 { + return Err(format!("max tokens must not exceed {}", i32::MAX)); + } + Ok(value) +} + +fn parse_temperature(value: &str) -> Result { + parse_float_range(value, "temperature", 0.0, 2.0, false) +} + +fn parse_top_p(value: &str) -> Result { + parse_float_range(value, "top-p", 0.0, 1.0, true) +} + +fn parse_top_k(value: &str) -> Result { + let value = value + .parse::() + .map_err(|error| format!("invalid top-k value: {error}"))?; + if value < -1 { + return Err("top-k must be -1, 0, or a positive integer".to_owned()); + } + Ok(value) +} + +fn parse_min_p(value: &str) -> Result { + parse_float_range(value, "min-p", 0.0, 1.0, false) +} + +fn parse_float_range( + value: &str, + name: &str, + minimum: f64, + maximum: f64, + minimum_is_exclusive: bool, +) -> Result { + let value = value + .parse::() + .map_err(|error| format!("invalid {name} value: {error}"))?; + let below_minimum = if minimum_is_exclusive { + value <= minimum + } else { + value < minimum + }; + if !value.is_finite() || below_minimum || value > maximum { + let opening = if minimum_is_exclusive { "(" } else { "[" }; + return Err(format!( + "{name} must be finite and in {opening}{minimum}, {maximum}]" + )); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use clap::error::ErrorKind; + + use super::*; + + fn source(args: Args) -> ModelSource { + match (args.model_path, args.model) { + (Some(path), None) => ModelSource::Local(path), + (None, Some(model)) => model.into_source(), + _ => panic!("invalid parsed model source"), + } + } + + #[test] + fn parses_bare_and_local_model_forms_with_global_options() { + let args = Args::try_parse_from(["chat", "--prompt", "hello", "model.gguf", "--no-stream"]) + .unwrap(); + assert_eq!(source(args), ModelSource::Local("model.gguf".into())); + + let args = Args::try_parse_from([ + "chat", + "--system", + "be concise", + "local", + "model-directory", + "--temperature", + "0.25", + ]) + .unwrap(); + assert_eq!(args.temperature, 0.25); + assert_eq!(source(args), ModelSource::Local("model-directory".into())); + } + + #[test] + fn rejects_missing_model_source() { + assert!(Args::try_parse_from(["chat"]).is_err()); + } + + #[test] + fn parses_hugging_face_forms_and_revisions() { + let gguf = Args::try_parse_from([ + "chat", + "hf-gguf", + "owner/model", + "model.gguf", + "--max-tokens", + "64", + ]) + .unwrap(); + assert_eq!(gguf.max_tokens, 64); + assert_eq!( + source(gguf), + ModelSource::Gguf { + repo: "owner/model".to_owned(), + filename: "model.gguf".to_owned(), + revision: None, + } + ); + + let safetensors = Args::try_parse_from([ + "chat", + "--top-p", + "0.9", + "hf-safetensors", + "owner/model", + "--revision", + "release", + "--min-p", + "0.05", + ]) + .unwrap(); + assert_eq!(safetensors.min_p, 0.05); + assert_eq!( + source(safetensors), + ModelSource::Safetensors { + repo: "owner/model".to_owned(), + revision: Some("release".to_owned()), + } + ); + } + + #[test] + fn parses_supported_sampling_options() { + let args = Args::try_parse_from([ + "chat", + "local", + "model", + "--max-tokens", + "512", + "--temperature", + "1.25", + "--top-p", + "0.8", + "--top-k", + "-1", + "--min-p", + "0.1", + "--seed", + "-2", + ]) + .unwrap(); + assert_eq!(args.max_tokens, 512); + assert_eq!(args.temperature, 1.25); + assert_eq!(args.top_p, 0.8); + assert_eq!(args.top_k, -1); + assert_eq!(args.min_p, 0.1); + assert_eq!(args.seed, Some(-2)); + } + + #[test] + fn uses_documented_generation_defaults() { + let args = Args::try_parse_from(["chat", "local", "model"]).unwrap(); + assert_eq!(args.max_tokens, 256); + assert_eq!(args.temperature, 0.7); + assert_eq!(args.top_p, 1.0); + assert_eq!(args.top_k, 0); + assert_eq!(args.min_p, 0.0); + assert_eq!(args.seed, None); + assert!(!args.no_stream); + } + + #[test] + fn rejects_prompt_file_conflict_and_invalid_sampling_values() { + let error = Args::try_parse_from([ + "chat", + "local", + "model", + "--prompt", + "hello", + "--file", + "prompt.txt", + ]) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::ArgumentConflict); + + for arguments in [ + vec!["chat", "local", "model", "--max-tokens", "0"], + vec!["chat", "local", "model", "--max-tokens", "2147483648"], + vec!["chat", "local", "model", "--temperature", "nan"], + vec!["chat", "local", "model", "--temperature", "2.1"], + vec!["chat", "local", "model", "--top-p", "0"], + vec!["chat", "local", "model", "--top-k", "-2"], + vec!["chat", "local", "model", "--min-p", "1.1"], + ] { + assert!(Args::try_parse_from(arguments).is_err()); + } + assert!( + Args::try_parse_from(["chat", "local", "model", "--max-tokens", "2147483647",]).is_ok() + ); + } + + #[test] + fn rolls_back_failed_turn_and_continues_with_the_next_input() { + let mut history = vec![Message::new(Role::System, "system")]; + let mut errors = Vec::new(); + let mut attempted_messages = Vec::new(); + + handle_turn( + &mut history, + "failed question".to_owned(), + |history, message| { + attempted_messages.push(message.clone()); + run_history_turn(history, message, |_history| { + Err(ChatError::turn("request failed")) + }) + }, + |error| errors.push(error.to_string()), + ) + .unwrap(); + assert_eq!(history, vec![Message::new(Role::System, "system")]); + assert_eq!(attempted_messages, ["failed question"]); + assert_eq!(errors, ["request failed"]); + + handle_turn( + &mut history, + "next question".to_owned(), + |history, message| { + run_history_turn(history, message, |_history| Ok("answer".to_owned())) + }, + |_| panic!("successful turn must not report an error"), + ) + .unwrap(); + assert_eq!( + history, + vec![ + Message::new(Role::System, "system"), + Message::new(Role::User, "next question"), + Message::new(Role::Assistant, "answer"), + ] + ); + } + + #[test] + fn propagates_fatal_errors_instead_of_recovering() { + let mut history = Vec::new(); + let mut reported = false; + let error = handle_turn( + &mut history, + "question".to_owned(), + |_history, _message| Err(ChatError::fatal("standard output failed")), + |_| reported = true, + ) + .unwrap_err(); + assert!(error.is_fatal()); + assert!(!reported); + } + + #[test] + fn builds_request_with_history_and_sampling_configuration() { + let messages = vec![ + Message::new(Role::System, "Be concise."), + Message::new(Role::User, "Hello"), + Message::new(Role::Assistant, "Hi"), + ]; + let request = build_request( + &messages, + &GenerationConfig { + max_tokens: 42, + temperature: 0.5, + top_p: 0.9, + top_k: 10, + min_p: 0.1, + seed: Some(7), + }, + true, + ) + .unwrap(); + let request: Value = serde_json::from_str(&request).unwrap(); + assert_eq!(request["messages"][0]["role"], "system"); + assert_eq!(request["messages"][1]["content"], "Hello"); + assert_eq!(request["messages"][2]["content"], "Hi"); + assert_eq!(request["max_tokens"], 42); + assert_eq!(request["temperature"], 0.5); + assert_eq!(request["top_p"], 0.9); + assert_eq!(request["top_k"], 10); + assert_eq!(request["min_p"], 0.1); + assert_eq!(request["seed"], 7); + assert_eq!(request["stream"], true); + } + + #[test] + fn extracts_native_blocking_response_content() { + let response = r#"{ + "id":"chatcmpl-test", + "object":"chat.completion", + "choices":[{ + "index":0, + "message":{"role":"assistant","content":"hello"}, + "finish_reason":"stop" + }] + }"#; + assert_eq!(extract_blocking_content(response).unwrap(), "hello"); + + let reasoning_only = r#"{ + "id":"chatcmpl-test", + "object":"chat.completion", + "choices":[{ + "index":0, + "message":{ + "role":"assistant", + "content":null, + "reasoning":"private reasoning" + }, + "finish_reason":"stop" + }] + }"#; + assert_eq!(extract_blocking_content(reasoning_only).unwrap(), ""); + + let missing_content = r#"{ + "choices":[{ + "message":{ + "role":"assistant", + "tool_calls":[{ + "id":"chatcmpl-tool-0", + "type":"function", + "function":{"name":"weather","arguments":"{}"} + }] + }, + "finish_reason":"tool_calls" + }] + }"#; + assert_eq!(extract_blocking_content(missing_content).unwrap(), ""); + } + + #[test] + fn rejects_malformed_blocking_response_shapes() { + for response in [ + r#"{}"#, + r#"{"choices":"not an array"}"#, + r#"{"choices":[]}"#, + r#"{"choices":[{"message":"not an object"}]}"#, + r#"{"choices":[{"message":{"content":42}}]}"#, + "not json", + ] { + assert!(extract_blocking_content(response).is_err()); + } + } + + #[test] + fn extracts_native_stream_chunks_and_accepts_empty_content() { + let chunks = [ + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"reasoning":"private"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Hel"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"lo"},"finish_reason":null},{"index":1,"delta":{"content":"!"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl-test","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + ]; + let mut content = String::new(); + for chunk in chunks { + assert_eq!( + accumulate_stream_content(&mut content, chunk).unwrap(), + extract_stream_content(chunk).unwrap() + ); + } + assert_eq!(content, "Hello!"); + + let empty_chunks = [ + r#"{"choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{"reasoning":"private"},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"name":"weather"}}]},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{"content":null},"finish_reason":null}]}"#, + r#"{"choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}"#, + ]; + let mut empty = String::new(); + for chunk in empty_chunks { + assert_eq!(accumulate_stream_content(&mut empty, chunk).unwrap(), ""); + } + assert_eq!(complete_stream(empty, false).unwrap(), ""); + + let mut history = vec![Message::new(Role::User, "question")]; + run_history_turn(&mut history, "next".to_owned(), |_history| { + complete_stream(String::new(), false) + }) + .unwrap(); + assert_eq!(history.last(), Some(&Message::new(Role::Assistant, ""))); + } + + #[test] + fn rejects_malformed_stream_content_types_and_shapes() { + for chunk in [ + r#"{}"#, + r#"{"choices":"not an array"}"#, + r#"{"choices":[{}]}"#, + r#"{"choices":[{"delta":"not an object"}]}"#, + r#"{"choices":[{"delta":{"content":42}}]}"#, + "not json", + ] { + assert!(extract_stream_content(chunk).is_err()); + } + assert!(complete_stream(String::new(), true).is_err()); + } + + #[test] + fn classifies_interactive_commands_without_trimming_messages() { + assert_eq!(classify_input(" /quit "), Input::Quit); + assert_eq!(classify_input("/exit"), Input::Quit); + assert_eq!(classify_input("/clear"), Input::Clear); + assert_eq!(classify_input(" "), Input::Empty); + assert_eq!(classify_input(" hello "), Input::Message(" hello ")); + } } diff --git a/vllm-cpp/examples/common/mod.rs b/vllm-cpp/examples/common/mod.rs new file mode 100644 index 0000000..2fd79d3 --- /dev/null +++ b/vllm-cpp/examples/common/mod.rs @@ -0,0 +1,254 @@ +use std::env; +use std::error::Error; +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::path::PathBuf; + +use vllm_cpp::HuggingFaceModel; + +#[derive(Debug)] +struct UsageError(String); + +impl fmt::Display for UsageError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Error for UsageError {} + +#[derive(Debug, Eq, PartialEq)] +pub enum ModelSource { + Local(PathBuf), + Gguf { + repo: String, + filename: String, + revision: Option, + }, + Safetensors { + repo: String, + revision: Option, + }, +} + +impl ModelSource { + pub fn resolve(self) -> Result> { + match self { + Self::Local(path) => Ok(path), + Self::Gguf { + repo, + filename, + revision, + } => { + let resolver = HuggingFaceModel::gguf(repo, filename); + let resolver = if let Some(revision) = revision { + resolver.revision(revision) + } else { + resolver + }; + Ok(resolver.progress(true).resolve()?) + } + Self::Safetensors { repo, revision } => { + let resolver = HuggingFaceModel::safetensors(repo); + let resolver = if let Some(revision) = revision { + resolver.revision(revision) + } else { + resolver + }; + Ok(resolver.progress(true).resolve()?) + } + } + } +} + +#[allow(dead_code)] +pub fn resolve_model(example: &str) -> Result> { + parse_model_source(example, env::args_os().skip(1))?.resolve() +} + +fn parse_model_source( + example: &str, + arguments: impl IntoIterator, +) -> Result { + let mut args = arguments.into_iter(); + let first = args.next().ok_or_else(|| usage(example))?; + match first.to_str() { + Some("local") => { + let path = args.next().ok_or_else(|| usage(example))?; + if path.is_empty() { + return Err(usage(example)); + } + require_end(example, &mut args)?; + Ok(ModelSource::Local(path.into())) + } + Some("hf-gguf") => { + let repo = required_utf8(example, &mut args)?; + let filename = required_utf8(example, &mut args)?; + let revision = optional_revision(example, &mut args)?; + Ok(ModelSource::Gguf { + repo, + filename, + revision, + }) + } + Some("hf-safetensors") => { + let repo = required_utf8(example, &mut args)?; + let revision = optional_revision(example, &mut args)?; + Ok(ModelSource::Safetensors { repo, revision }) + } + Some(value) if value.starts_with('-') => Err(usage(example)), + _ => { + if first.is_empty() { + return Err(usage(example)); + } + require_end(example, &mut args)?; + Ok(ModelSource::Local(first.into())) + } + } +} + +fn required_utf8( + example: &str, + args: &mut impl Iterator, +) -> Result { + let value = args + .next() + .ok_or_else(|| usage(example))? + .into_string() + .map_err(|_| usage(example))?; + if value.is_empty() { + return Err(usage(example)); + } + Ok(value) +} + +fn optional_revision( + example: &str, + args: &mut impl Iterator, +) -> Result, UsageError> { + let Some(flag) = args.next() else { + return Ok(None); + }; + if flag != OsStr::new("--revision") { + return Err(usage(example)); + } + let revision = required_utf8(example, args)?; + require_end(example, args)?; + Ok(Some(revision)) +} + +fn require_end(example: &str, args: &mut impl Iterator) -> Result<(), UsageError> { + if args.next().is_some() { + return Err(usage(example)); + } + Ok(()) +} + +fn usage(example: &str) -> UsageError { + UsageError(format!( + "usage:\n {example} \n {example} local \n {example} hf-gguf [--revision ]\n {example} hf-safetensors [--revision ]" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn os(values: &[&str]) -> Vec { + values.iter().map(OsString::from).collect() + } + + #[test] + fn parses_local_aliases() { + assert_eq!( + parse_model_source("example", os(&["model.gguf"])).unwrap(), + ModelSource::Local("model.gguf".into()) + ); + assert_eq!( + parse_model_source("example", os(&["local", "model"])).unwrap(), + ModelSource::Local("model".into()) + ); + } + + #[test] + fn parses_hugging_face_sources_with_optional_revisions() { + assert_eq!( + parse_model_source("example", os(&["hf-gguf", "owner/model", "model.gguf"])).unwrap(), + ModelSource::Gguf { + repo: "owner/model".to_owned(), + filename: "model.gguf".to_owned(), + revision: None, + } + ); + assert_eq!( + parse_model_source("example", os(&["hf-safetensors", "owner/model"])).unwrap(), + ModelSource::Safetensors { + repo: "owner/model".to_owned(), + revision: None, + } + ); + assert_eq!( + parse_model_source( + "example", + os(&[ + "hf-gguf", + "owner/model", + "model.gguf", + "--revision", + "release", + ]), + ) + .unwrap(), + ModelSource::Gguf { + repo: "owner/model".to_owned(), + filename: "model.gguf".to_owned(), + revision: Some("release".to_owned()), + } + ); + assert_eq!( + parse_model_source( + "example", + os(&["hf-safetensors", "owner/model", "--revision", "release",]), + ) + .unwrap(), + ModelSource::Safetensors { + repo: "owner/model".to_owned(), + revision: Some("release".to_owned()), + } + ); + } + + #[test] + fn rejects_missing_extra_duplicate_and_misordered_arguments() { + for args in [ + os(&[]), + os(&[""]), + os(&["local"]), + os(&["local", ""]), + os(&["model", "extra"]), + os(&["hf-gguf", "owner/model"]), + os(&["hf-gguf", "owner/model", ""]), + os(&["hf-safetensors", ""]), + os(&["hf-safetensors", "owner/model", "release"]), + os(&["hf-safetensors", "owner/model", "--revision"]), + os(&["hf-safetensors", "owner/model", "--revision", ""]), + os(&[ + "hf-safetensors", + "owner/model", + "--revision", + "one", + "--revision", + "two", + ]), + os(&[ + "hf-gguf", + "--revision", + "release", + "owner/model", + "model.gguf", + ]), + ] { + assert!(parse_model_source("example", args).is_err()); + } + } +} diff --git a/vllm-cpp/examples/complete.rs b/vllm-cpp/examples/complete.rs index 71cb53d..bc26b2e 100644 --- a/vllm-cpp/examples/complete.rs +++ b/vllm-cpp/examples/complete.rs @@ -1,9 +1,9 @@ +mod common; + use vllm_cpp::{Engine, SamplingParams}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: complete ")?; + let model = common::resolve_model("complete")?; let engine = Engine::load(model)?; let completion = engine.complete( "The capital of France is", diff --git a/vllm-cpp/examples/concurrent.rs b/vllm-cpp/examples/concurrent.rs index 51432f7..87804cf 100644 --- a/vllm-cpp/examples/concurrent.rs +++ b/vllm-cpp/examples/concurrent.rs @@ -1,11 +1,11 @@ +mod common; + use std::io::{self, Write}; use vllm_cpp::{Engine, SamplingParams, StreamControl}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: concurrent ")?; + let model = common::resolve_model("concurrent")?; let engine = Engine::load(model)?; let params = SamplingParams::greedy().max_tokens(16); diff --git a/vllm-cpp/examples/stream.rs b/vllm-cpp/examples/stream.rs index b852a6c..f4c1980 100644 --- a/vllm-cpp/examples/stream.rs +++ b/vllm-cpp/examples/stream.rs @@ -1,11 +1,11 @@ +mod common; + use std::io::{self, Write}; use vllm_cpp::{Engine, SamplingParams, StreamControl}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: stream ")?; + let model = common::resolve_model("stream")?; let engine = Engine::load(model)?; engine.complete_stream( "Write one short sentence about Rust:", diff --git a/vllm-cpp/examples/structured.rs b/vllm-cpp/examples/structured.rs index ec48207..4107025 100644 --- a/vllm-cpp/examples/structured.rs +++ b/vllm-cpp/examples/structured.rs @@ -1,18 +1,25 @@ +mod common; + use vllm_cpp::{Engine, SamplingParams, StructuredOutput}; fn main() -> Result<(), Box> { - let model = std::env::args_os() - .nth(1) - .ok_or("usage: structured ")?; + let model = common::resolve_model("structured")?; let engine = Engine::load(model)?; + let schema = r#"{ + "type": "object", + "properties": { + "location": { "type": "string" }, + "temperature_celsius": { "type": "number" }, + "condition": { "type": "string" } + }, + "required": ["location", "temperature_celsius", "condition"], + "additionalProperties": false + }"#; let completion = engine.complete( - "Choose exactly one color: red or blue. Answer:", + "Extract the weather report as JSON: Paris is sunny and 22 degrees Celsius.", &SamplingParams::greedy() - .max_tokens(8) - .structured_output(StructuredOutput::Choice(vec![ - "red".to_owned(), - "blue".to_owned(), - ])), + .max_tokens(64) + .structured_output(StructuredOutput::JsonSchema(schema.to_owned())), )?; println!("{}", completion.text); Ok(()) diff --git a/vllm-cpp/src/error.rs b/vllm-cpp/src/error.rs index 9090f93..d9a15ae 100644 --- a/vllm-cpp/src/error.rs +++ b/vllm-cpp/src/error.rs @@ -3,6 +3,41 @@ use std::fmt; use vllm_cpp_sys as ffi; +/// An error returned while resolving a model from the Hugging Face Hub. +/// +/// External transport errors are converted to contextual strings so this type +/// remains stable, cloneable, and comparable. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub enum HuggingFaceError { + /// A repository, revision, or filename is invalid. + InvalidInput { message: String }, + /// The requested revision is not present in the selected local cache. + CacheMiss { message: String }, + /// A repository snapshot lacks required runtime files or metadata. + Incomplete { message: String }, + /// A Hugging Face API or download operation failed. + Hub { message: String }, + /// A local cache operation failed. + Io { message: String }, +} + +impl fmt::Display for HuggingFaceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidInput { message } => write!(f, "invalid Hugging Face model: {message}"), + Self::CacheMiss { message } => write!(f, "Hugging Face cache miss: {message}"), + Self::Incomplete { message } => { + write!(f, "incomplete Hugging Face snapshot: {message}") + } + Self::Hub { message } => write!(f, "Hugging Face Hub failure: {message}"), + Self::Io { message } => write!(f, "Hugging Face cache I/O failure: {message}"), + } + } +} + +impl std::error::Error for HuggingFaceError {} + /// An error returned by the safe vllm.cpp wrapper. #[derive(Clone, Debug, Eq, PartialEq)] #[non_exhaustive] diff --git a/vllm-cpp/src/hf.rs b/vllm-cpp/src/hf.rs new file mode 100644 index 0000000..84f9676 --- /dev/null +++ b/vllm-cpp/src/hf.rs @@ -0,0 +1,1022 @@ +use std::collections::{BTreeSet, HashMap}; +use std::fmt; +use std::fs; +use std::path::{Component, Path, PathBuf}; + +use hf_hub::api::sync::ApiBuilder; +use hf_hub::api::RepoInfo; +use hf_hub::{Cache, Repo, RepoType}; +use serde_json::Value; + +use crate::HuggingFaceError; + +const CONFIG: &str = "config.json"; +const TOKENIZER: &str = "tokenizer.json"; +const TOKENIZER_CONFIG: &str = "tokenizer_config.json"; +const SAFETENSORS: &str = "model.safetensors"; +const SAFETENSORS_INDEX: &str = "model.safetensors.index.json"; +const DEFAULT_REVISION: &str = "main"; + +/// A synchronous Hugging Face model resolver. +/// +/// Resolution is separate from [`crate::Engine::load`]. The resolver returns a +/// standalone GGUF path or a sparse, runtime-complete Safetensors snapshot +/// directory in the normal Hugging Face cache layout. +#[derive(Clone)] +pub struct HuggingFaceModel { + repo_id: String, + revision: String, + kind: ModelKind, + cache_dir: Option, + token: Option, + progress: bool, + offline: bool, +} + +#[derive(Clone, Debug)] +enum ModelKind { + Gguf { filename: String }, + Safetensors, +} + +impl fmt::Debug for HuggingFaceModel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HuggingFaceModel") + .field("repo_id", &self.repo_id) + .field("revision", &self.revision) + .field("kind", &self.kind) + .field("cache_dir", &self.cache_dir) + .field("token", &self.token.as_ref().map(|_| "[REDACTED]")) + .field("progress", &self.progress) + .field("offline", &self.offline) + .finish() + } +} + +impl HuggingFaceModel { + /// Selects one standalone GGUF file from the repository's `main` revision. + #[must_use] + pub fn gguf(repo_id: impl Into, filename: impl Into) -> Self { + Self { + repo_id: repo_id.into(), + revision: DEFAULT_REVISION.to_owned(), + kind: ModelKind::Gguf { + filename: filename.into(), + }, + cache_dir: None, + token: None, + progress: false, + offline: false, + } + } + + /// Selects a runtime-complete Safetensors directory from the repository's `main` revision. + #[must_use] + pub fn safetensors(repo_id: impl Into) -> Self { + Self { + repo_id: repo_id.into(), + revision: DEFAULT_REVISION.to_owned(), + kind: ModelKind::Safetensors, + cache_dir: None, + token: None, + progress: false, + offline: false, + } + } + + /// Overrides the repository revision with a branch, tag, or commit. + #[must_use] + pub fn revision(mut self, revision: impl Into) -> Self { + self.revision = revision.into(); + self + } + + /// Overrides the Hugging Face Hub cache directory. + /// + /// The path is the Hub cache itself (for example, `~/.cache/huggingface/hub`), + /// not its parent. + #[must_use] + pub fn cache_dir(mut self, cache_dir: impl Into) -> Self { + self.cache_dir = Some(cache_dir.into()); + self + } + + /// Overrides the cached Hugging Face token for this resolver. + #[must_use] + pub fn token(mut self, token: impl Into) -> Self { + self.token = Some(token.into()); + self + } + + /// Enables or disables download progress bars. Progress is disabled by default. + #[must_use] + pub fn progress(mut self, progress: bool) -> Self { + self.progress = progress; + self + } + + /// Enables or disables cache-only resolution. Offline mode never builds an API. + #[must_use] + pub fn offline(mut self, offline: bool) -> Self { + self.offline = offline; + self + } + + /// Resolves the selected model into the normal Hugging Face cache. + pub fn resolve(&self) -> Result { + self.validate()?; + let cache = self.cache(); + if self.offline { + return self.resolve_offline(&cache); + } + + match &self.kind { + ModelKind::Gguf { filename } => self.resolve_gguf_online(cache, filename), + ModelKind::Safetensors => self.resolve_safetensors_online(cache), + } + } + + fn validate(&self) -> Result<(), HuggingFaceError> { + validate_nonempty("repository ID", &self.repo_id)?; + validate_nonempty("revision", &self.revision)?; + validate_repo_id(&self.repo_id)?; + validate_revision(&self.revision)?; + + if let Some(token) = &self.token { + validate_nonempty("token", token)?; + } + if let ModelKind::Gguf { filename } = &self.kind { + validate_root_filename(filename, "GGUF filename")?; + if !filename.ends_with(".gguf") { + return Err(invalid("GGUF filename must end with lowercase `.gguf`")); + } + if is_split_gguf(filename) { + return Err(invalid("split GGUF sets are not supported")); + } + } + Ok(()) + } + + fn cache(&self) -> Cache { + self.cache_dir + .clone() + .map(Cache::new) + .unwrap_or_else(Cache::from_env) + } + + fn api_builder(&self, cache: Cache) -> ApiBuilder { + let builder = ApiBuilder::from_cache(cache).with_progress(self.progress); + match &self.token { + Some(token) => builder.with_token(Some(token.clone())), + None => builder, + } + } + + fn requested_repo(&self) -> Repo { + Repo::with_revision(self.repo_id.clone(), RepoType::Model, self.revision.clone()) + } + + fn resolve_gguf_online( + &self, + cache: Cache, + filename: &str, + ) -> Result { + let api = self + .api_builder(cache) + .build() + .map_err(|error| hub(format!("could not create API: {error}")))?; + let path = api + .repo(self.requested_repo()) + .get(filename) + .map_err(|error| { + hub(format!( + "could not resolve `{filename}` from `{}` at `{}`: {error}", + self.repo_id, self.revision + )) + })?; + verify_gguf_path(&path, filename)?; + Ok(path) + } + + fn resolve_safetensors_online(&self, cache: Cache) -> Result { + let api = self + .api_builder(cache.clone()) + .build() + .map_err(|error| hub(format!("could not create API: {error}")))?; + let info = api.repo(self.requested_repo()).info().map_err(|error| { + hub(format!( + "could not read metadata for `{}` at `{}`: {error}", + self.repo_id, self.revision + )) + })?; + let sha = info.sha.trim(); + if sha.is_empty() { + return Err(incomplete("repository metadata has an empty commit SHA")); + } + validate_root_filename(sha, "repository metadata SHA") + .map_err(|_| incomplete("repository metadata has an unsafe SHA"))?; + + let available = sibling_names(&info); + let bootstrap = plan_safetensors(&available)?; + let pinned_repo = + Repo::with_revision(self.repo_id.clone(), RepoType::Model, sha.to_owned()); + let pinned = api.repo(pinned_repo.clone()); + let index = if bootstrap.indexed { + let path = pinned.get(SAFETENSORS_INDEX).map_err(|error| { + hub(format!( + "could not resolve `{SAFETENSORS_INDEX}` for `{}` at `{sha}`: {error}", + self.repo_id + )) + })?; + let bytes = fs::read(&path).map_err(|error| { + io_error(format!("could not read `{}`: {error}", path.display())) + })?; + Some((path, bytes)) + } else { + None + }; + let (plan, index_path) = match index { + Some((path, bytes)) => (plan_indexed_safetensors(&available, &bytes)?, Some(path)), + None => (bootstrap, None), + }; + + let mut paths = HashMap::new(); + if let Some(path) = index_path { + paths.insert(SAFETENSORS_INDEX.to_owned(), path); + } + for filename in &plan.files { + if paths.contains_key(filename) { + continue; + } + let path = pinned.get(filename).map_err(|error| { + hub(format!( + "could not resolve `{filename}` for `{}` at `{sha}`: {error}", + self.repo_id + )) + })?; + paths.insert(filename.clone(), path); + } + + let snapshot = verify_snapshot_paths(&paths, sha)?; + cache + .repo(self.requested_repo()) + .create_ref(sha) + .map_err(|error| { + io_error(format!( + "could not update cache ref `{}` for `{}`: {error}", + self.revision, self.repo_id + )) + })?; + Ok(snapshot) + } + + fn resolve_offline(&self, cache: &Cache) -> Result { + let repo = cache.repo(self.requested_repo()); + match &self.kind { + ModelKind::Gguf { filename } => { + let path = repo.get(filename).ok_or_else(|| { + cache_miss(format!( + "`{filename}` for `{}` at `{}` is not cached", + self.repo_id, self.revision + )) + })?; + let snapshot = snapshot_for_cached_revision(cache, &self.requested_repo())?; + verify_gguf_path(&path, filename)?; + if path.parent() != Some(snapshot.as_path()) { + return Err(incomplete(format!( + "resolved `{filename}` does not belong to the requested revision snapshot" + ))); + } + Ok(path) + } + ModelKind::Safetensors => { + let snapshot = snapshot_for_cached_revision(cache, &self.requested_repo())?; + let available = cached_root_files(&snapshot)?; + let bootstrap = plan_safetensors(&available)?; + let plan = if bootstrap.indexed { + let index_path = snapshot.join(SAFETENSORS_INDEX); + let bytes = fs::read(&index_path).map_err(|error| { + io_error(format!( + "could not read `{}`: {error}", + index_path.display() + )) + })?; + plan_indexed_safetensors(&available, &bytes)? + } else { + bootstrap + }; + + let paths = plan + .files + .iter() + .map(|filename| (filename.clone(), snapshot.join(filename))) + .collect::>(); + verify_snapshot_paths(&paths, snapshot_sha(&snapshot)?) + } + } + } +} + +#[derive(Debug, Eq, PartialEq)] +struct SafetensorsPlan { + files: Vec, + indexed: bool, +} + +fn plan_safetensors(available: &BTreeSet) -> Result { + for required in [CONFIG, TOKENIZER] { + if !available.contains(required) { + return Err(incomplete(format!("required `{required}` is missing"))); + } + } + + let mut files = vec![CONFIG.to_owned(), TOKENIZER.to_owned()]; + if available.contains(TOKENIZER_CONFIG) { + files.push(TOKENIZER_CONFIG.to_owned()); + } + if available.contains(SAFETENSORS) { + files.push(SAFETENSORS.to_owned()); + Ok(SafetensorsPlan { + files, + indexed: false, + }) + } else if available.contains(SAFETENSORS_INDEX) { + files.push(SAFETENSORS_INDEX.to_owned()); + Ok(SafetensorsPlan { + files, + indexed: true, + }) + } else { + Err(incomplete(format!( + "neither `{SAFETENSORS}` nor `{SAFETENSORS_INDEX}` is present" + ))) + } +} + +fn plan_indexed_safetensors( + available: &BTreeSet, + bytes: &[u8], +) -> Result { + let mut plan = plan_safetensors(available)?; + if !plan.indexed { + return Ok(plan); + } + + let value: Value = serde_json::from_slice(bytes) + .map_err(|error| incomplete(format!("`{SAFETENSORS_INDEX}` is malformed JSON: {error}")))?; + let weight_map = value + .get("weight_map") + .and_then(Value::as_object) + .ok_or_else(|| incomplete(format!("`{SAFETENSORS_INDEX}` has no object `weight_map`")))?; + if weight_map.is_empty() { + return Err(incomplete(format!( + "`{SAFETENSORS_INDEX}` has an empty `weight_map`" + ))); + } + + let mut shards = BTreeSet::new(); + for value in weight_map.values() { + let shard = value.as_str().ok_or_else(|| { + incomplete(format!( + "`{SAFETENSORS_INDEX}` contains a non-string shard path" + )) + })?; + validate_root_filename(shard, "Safetensors shard") + .map_err(|error| incomplete(error.to_string()))?; + if !shard.ends_with(".safetensors") { + return Err(incomplete(format!( + "indexed shard `{shard}` must end with `.safetensors`" + ))); + } + if !available.contains(shard) { + return Err(incomplete(format!( + "indexed shard `{shard}` is missing from repository metadata or cache" + ))); + } + shards.insert(shard.to_owned()); + } + plan.files.extend(shards); + Ok(plan) +} + +fn sibling_names(info: &RepoInfo) -> BTreeSet { + info.siblings + .iter() + .filter(|sibling| { + validate_root_filename(&sibling.rfilename, "repository metadata filename").is_ok() + }) + .map(|sibling| sibling.rfilename.clone()) + .collect() +} + +fn cached_root_files(snapshot: &Path) -> Result, HuggingFaceError> { + let entries = fs::read_dir(snapshot).map_err(|error| { + io_error(format!( + "could not read cached snapshot `{}`: {error}", + snapshot.display() + )) + })?; + let mut files = BTreeSet::new(); + for entry in entries { + let entry = entry.map_err(|error| { + io_error(format!( + "could not inspect cached snapshot `{}`: {error}", + snapshot.display() + )) + })?; + if entry.path().is_file() { + if let Some(filename) = entry.file_name().to_str() { + files.insert(filename.to_owned()); + } + } + } + Ok(files) +} + +fn snapshot_for_cached_revision(cache: &Cache, repo: &Repo) -> Result { + let cache_repo = cache.repo(repo.clone()); + let ref_path = cache + .path() + .join(repo.folder_name()) + .join("refs") + .join(repo.revision()); + let sha = fs::read_to_string(&ref_path).map_err(|error| { + if error.kind() == std::io::ErrorKind::NotFound { + cache_miss(format!( + "revision `{}` for `{}` has no cache ref", + repo.revision(), + repo.folder_name() + )) + } else { + io_error(format!( + "could not read cache ref `{}`: {error}", + ref_path.display() + )) + } + })?; + let sha = sha.trim(); + if sha.is_empty() { + return Err(incomplete(format!( + "cache ref `{}` has an empty SHA", + ref_path.display() + ))); + } + validate_root_filename(sha, "cached revision SHA") + .map_err(|_| incomplete("cached revision has an unsafe SHA"))?; + let snapshot = cache_repo.pointer_path(sha); + if !snapshot.is_dir() { + return Err(cache_miss(format!( + "snapshot `{}` is not cached", + snapshot.display() + ))); + } + Ok(snapshot) +} + +fn verify_gguf_path(path: &Path, filename: &str) -> Result<(), HuggingFaceError> { + if !path.is_file() { + return Err(incomplete(format!( + "resolved `{filename}` is not a file at `{}`", + path.display() + ))); + } + if path.file_name().and_then(|name| name.to_str()) != Some(filename) { + return Err(incomplete(format!( + "resolved GGUF path does not match requested filename `{filename}`" + ))); + } + let snapshot = path + .parent() + .ok_or_else(|| incomplete(format!("resolved `{filename}` has no snapshot directory")))?; + if snapshot + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + != Some("snapshots") + { + return Err(incomplete(format!( + "resolved `{filename}` is not directly under a `snapshots` directory" + ))); + } + Ok(()) +} + +fn snapshot_sha(snapshot: &Path) -> Result<&str, HuggingFaceError> { + snapshot + .file_name() + .and_then(|value| value.to_str()) + .ok_or_else(|| incomplete("cached snapshot has a non-UTF-8 SHA")) +} + +fn verify_snapshot_paths( + paths: &HashMap, + sha: &str, +) -> Result { + if paths.is_empty() { + return Err(incomplete("no snapshot files were resolved")); + } + let mut common = None; + for (filename, path) in paths { + if !path.is_file() { + return Err(incomplete(format!( + "resolved `{filename}` is not a file at `{}`", + path.display() + ))); + } + let parent = path.parent().ok_or_else(|| { + incomplete(format!("resolved `{filename}` has no snapshot directory")) + })?; + let valid_layout = parent.file_name().and_then(|name| name.to_str()) == Some(sha) + && parent + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + == Some("snapshots"); + if !valid_layout { + return Err(incomplete(format!( + "resolved `{filename}` is outside `snapshots/{sha}`" + ))); + } + match &common { + Some(expected) if expected != parent => { + return Err(incomplete("resolved files belong to different snapshots")); + } + None => common = Some(parent.to_owned()), + _ => {} + } + } + common.ok_or_else(|| incomplete("no snapshot directory was resolved")) +} + +fn validate_nonempty(field: &str, value: &str) -> Result<(), HuggingFaceError> { + if value.trim().is_empty() { + Err(invalid(format!("{field} must not be empty"))) + } else { + Ok(()) + } +} + +fn validate_repo_id(repo_id: &str) -> Result<(), HuggingFaceError> { + validate_repo_relative_path(repo_id, "repository ID")?; + if repo_id.split('/').count() > 2 { + return Err(invalid("repository ID must be `name` or `namespace/name`")); + } + Ok(()) +} + +fn validate_revision(revision: &str) -> Result<(), HuggingFaceError> { + validate_repo_relative_path(revision, "revision") +} + +fn validate_root_filename(filename: &str, field: &str) -> Result<(), HuggingFaceError> { + validate_repo_relative_path(filename, field)?; + if Path::new(filename).components().count() != 1 { + return Err(invalid(format!("{field} must be a root-level filename"))); + } + Ok(()) +} + +fn validate_repo_relative_path(path: &str, field: &str) -> Result<(), HuggingFaceError> { + validate_nonempty(field, path)?; + if path.contains('\\') + || path.contains('\0') + || path.contains(':') + || path.starts_with('~') + || path.split('/').any(str::is_empty) + { + return Err(invalid(format!( + "{field} must use portable repository-relative path syntax" + ))); + } + let path = Path::new(path); + if path.is_absolute() + || path + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(invalid(format!( + "{field} must not be absolute or contain `.` or `..` components" + ))); + } + for component in path.components() { + let Component::Normal(component) = component else { + unreachable!("non-normal components were rejected above"); + }; + let component = component.to_string_lossy(); + if component.ends_with([' ', '.']) + || component.chars().any(|value| { + value.is_control() || matches!(value, '<' | '>' | '"' | '|' | '?' | '*') + }) + { + return Err(invalid(format!( + "{field} contains characters that are not portable path syntax" + ))); + } + } + Ok(()) +} + +fn is_split_gguf(filename: &str) -> bool { + let stem = filename.strip_suffix(".gguf").unwrap_or(filename); + let Some((prefix, total)) = stem.rsplit_once("-of-") else { + return false; + }; + let Some((_, part)) = prefix.rsplit_once('-') else { + return false; + }; + !part.is_empty() + && !total.is_empty() + && part.bytes().all(|value| value.is_ascii_digit()) + && total.bytes().all(|value| value.is_ascii_digit()) +} + +fn invalid(message: impl Into) -> HuggingFaceError { + HuggingFaceError::InvalidInput { + message: message.into(), + } +} + +fn cache_miss(message: impl Into) -> HuggingFaceError { + HuggingFaceError::CacheMiss { + message: message.into(), + } +} + +fn incomplete(message: impl Into) -> HuggingFaceError { + HuggingFaceError::Incomplete { + message: message.into(), + } +} + +fn hub(message: impl Into) -> HuggingFaceError { + HuggingFaceError::Hub { + message: message.into(), + } +} + +fn io_error(message: impl Into) -> HuggingFaceError { + HuggingFaceError::Io { + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use hf_hub::api::Siblings; + use std::sync::atomic::{AtomicU64, Ordering}; + + static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); + const REPO: &str = "owner/model"; + const REVISION: &str = "release"; + const SHA: &str = "0123456789abcdef"; + + struct TempDir(PathBuf); + + impl TempDir { + fn new() -> Self { + let id = NEXT_TEMP.fetch_add(1, Ordering::Relaxed); + let path = + std::env::temp_dir().join(format!("vllm-cpp-hf-{}-{id}", std::process::id())); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + fn names(values: &[&str]) -> BTreeSet { + values.iter().map(|value| (*value).to_owned()).collect() + } + + fn assert_incomplete_contains( + result: Result, + expected: &str, + ) { + match result { + Err(HuggingFaceError::Incomplete { message }) => assert!( + message.contains(expected), + "expected `{message}` to contain `{expected}`" + ), + other => panic!("expected incomplete error containing `{expected}`, got {other:?}"), + } + } + + fn cache_fixture_at(revision: &str, files: &[(&str, &[u8])]) -> TempDir { + let temp = TempDir::new(); + let cache = Cache::new(temp.0.clone()); + let repo = Repo::with_revision(REPO.to_owned(), RepoType::Model, revision.to_owned()); + let cache_repo = cache.repo(repo); + cache_repo.create_ref(SHA).unwrap(); + let snapshot = cache_repo.pointer_path(SHA); + fs::create_dir_all(&snapshot).unwrap(); + for (filename, contents) in files { + fs::write(snapshot.join(filename), contents).unwrap(); + } + temp + } + + fn cache_fixture(files: &[(&str, &[u8])]) -> TempDir { + cache_fixture_at(DEFAULT_REVISION, files) + } + + #[test] + fn validates_inputs_and_rejects_split_gguf() { + let cases = [ + HuggingFaceModel::gguf("", "model.gguf"), + HuggingFaceModel::gguf(REPO, "model.gguf").revision(""), + HuggingFaceModel::gguf(REPO, "model.gguf").revision("../main"), + HuggingFaceModel::gguf("owner/model/extra", "model.gguf"), + HuggingFaceModel::gguf(REPO, "/model.gguf"), + HuggingFaceModel::gguf(REPO, "nested/model.gguf"), + HuggingFaceModel::gguf(REPO, "model.GGUF"), + HuggingFaceModel::gguf(REPO, "model-00001-of-00002.gguf"), + HuggingFaceModel::gguf(REPO, "model-1-of-2.gguf"), + HuggingFaceModel::gguf(REPO, "model?.gguf"), + ]; + for model in cases { + assert!(matches!( + model.validate(), + Err(HuggingFaceError::InvalidInput { .. }) + )); + } + assert!(HuggingFaceModel::gguf(REPO, "model.gguf") + .revision("refs/pr/1") + .validate() + .is_ok()); + } + + #[test] + fn defaults_to_main_and_accepts_revision_override() { + let default_gguf = HuggingFaceModel::gguf(REPO, "model.gguf"); + let default_safetensors = HuggingFaceModel::safetensors(REPO); + assert_eq!(default_gguf.revision, DEFAULT_REVISION); + assert_eq!(default_safetensors.revision, DEFAULT_REVISION); + + let pinned = default_safetensors.revision(REVISION); + assert_eq!(pinned.revision, REVISION); + assert_eq!(pinned.requested_repo().revision(), REVISION); + } + + #[test] + fn debug_redacts_explicit_token() { + let model = HuggingFaceModel::safetensors(REPO) + .revision(REVISION) + .token("hf_secret_value"); + let debug = format!("{model:?}"); + assert!(debug.contains("[REDACTED]")); + assert!(!debug.contains("hf_secret_value")); + } + + #[test] + fn plans_unsharded_and_indexed_snapshots() { + let unsharded = names(&[CONFIG, TOKENIZER, TOKENIZER_CONFIG, SAFETENSORS]); + assert_eq!( + plan_safetensors(&unsharded).unwrap(), + SafetensorsPlan { + files: vec![CONFIG, TOKENIZER, TOKENIZER_CONFIG, SAFETENSORS] + .into_iter() + .map(str::to_owned) + .collect(), + indexed: false, + } + ); + + let indexed = names(&[ + CONFIG, + TOKENIZER, + SAFETENSORS_INDEX, + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + ]); + let bytes = br#"{"weight_map":{"a":"model-00002-of-00002.safetensors","b":"model-00001-of-00002.safetensors","c":"model-00002-of-00002.safetensors"}}"#; + let plan = plan_indexed_safetensors(&indexed, bytes).unwrap(); + assert_eq!( + plan.files, + vec![ + CONFIG, + TOKENIZER, + SAFETENSORS_INDEX, + "model-00001-of-00002.safetensors", + "model-00002-of-00002.safetensors", + ] + ); + } + + #[test] + fn rejects_incomplete_or_malformed_safetensors_metadata() { + let missing_core = names(&[CONFIG, SAFETENSORS]); + assert!(matches!( + plan_safetensors(&missing_core), + Err(HuggingFaceError::Incomplete { .. }) + )); + + let indexed = names(&[CONFIG, TOKENIZER, SAFETENSORS_INDEX, "part.safetensors"]); + for (bytes, expected) in [ + (br#"not json"#.as_slice(), "malformed JSON"), + (br#"{}"#.as_slice(), "has no object `weight_map`"), + (br#"{"weight_map":{}}"#.as_slice(), "empty `weight_map`"), + ( + br#"{"weight_map":{"a":3}}"#.as_slice(), + "non-string shard path", + ), + ( + br#"{"weight_map":{"a":"../part.safetensors"}}"#.as_slice(), + "must not be absolute or contain `.` or `..` components", + ), + ( + br#"{"weight_map":{"a":"/part.safetensors"}}"#.as_slice(), + "must use portable repository-relative path syntax", + ), + ( + br#"{"weight_map":{"a":"nested/part.safetensors"}}"#.as_slice(), + "must be a root-level filename", + ), + ( + br#"{"weight_map":{"a":"missing.safetensors"}}"#.as_slice(), + "indexed shard `missing.safetensors` is missing", + ), + ( + br#"{"weight_map":{"a":"part.bin"}}"#.as_slice(), + "must end with `.safetensors`", + ), + ] { + assert_incomplete_contains(plan_indexed_safetensors(&indexed, bytes), expected); + } + } + + #[test] + fn ignores_unrelated_unsafe_safetensors_siblings() { + let info = RepoInfo { + sha: SHA.to_owned(), + siblings: [ + CONFIG, + TOKENIZER, + SAFETENSORS, + "../junk.json", + "/junk.json", + "nested/junk.json", + ] + .into_iter() + .map(|rfilename| Siblings { + rfilename: rfilename.to_owned(), + }) + .collect(), + }; + + let available = sibling_names(&info); + assert_eq!(available, names(&[CONFIG, TOKENIZER, SAFETENSORS])); + assert!(plan_safetensors(&available).is_ok()); + + let unsafe_required = RepoInfo { + sha: SHA.to_owned(), + siblings: ["../config.json", TOKENIZER, SAFETENSORS] + .into_iter() + .map(|rfilename| Siblings { + rfilename: rfilename.to_owned(), + }) + .collect(), + }; + assert_incomplete_contains( + plan_safetensors(&sibling_names(&unsafe_required)), + "required `config.json` is missing", + ); + } + + #[test] + fn resolves_complete_offline_unsharded_cache() { + let fixture = cache_fixture(&[ + (CONFIG, b"{}"), + (TOKENIZER, b"{}"), + (TOKENIZER_CONFIG, b"{}"), + (SAFETENSORS, b"weights"), + ]); + let resolved = HuggingFaceModel::safetensors(REPO) + .cache_dir(&fixture.0) + .offline(true) + .resolve() + .unwrap(); + assert_eq!(resolved.file_name().unwrap(), SHA); + assert_eq!(resolved.parent().unwrap().file_name().unwrap(), "snapshots"); + } + + #[test] + fn resolves_complete_offline_indexed_cache() { + let index = br#"{"weight_map":{"a":"model-00001-of-00002.safetensors","b":"model-00002-of-00002.safetensors"}}"#; + let fixture = cache_fixture_at( + REVISION, + &[ + (CONFIG, b"{}"), + (TOKENIZER, b"{}"), + (SAFETENSORS_INDEX, index), + ("model-00001-of-00002.safetensors", b"one"), + ("model-00002-of-00002.safetensors", b"two"), + ], + ); + assert!(HuggingFaceModel::safetensors(REPO) + .revision(REVISION) + .cache_dir(&fixture.0) + .offline(true) + .resolve() + .is_ok()); + } + + #[test] + fn distinguishes_offline_cache_miss_and_incomplete_snapshot() { + let empty = TempDir::new(); + let miss = HuggingFaceModel::safetensors(REPO) + .cache_dir(&empty.0) + .offline(true) + .resolve(); + assert!(matches!(miss, Err(HuggingFaceError::CacheMiss { .. }))); + + let partial = cache_fixture(&[(CONFIG, b"{}"), (TOKENIZER, b"{}")]); + let incomplete = HuggingFaceModel::safetensors(REPO) + .cache_dir(&partial.0) + .offline(true) + .resolve(); + assert!(matches!( + incomplete, + Err(HuggingFaceError::Incomplete { .. }) + )); + + let index = br#"{"weight_map":{"a":"missing.safetensors"}}"#; + let missing_shard = cache_fixture(&[ + (CONFIG, b"{}"), + (TOKENIZER, b"{}"), + (SAFETENSORS_INDEX, index), + ]); + assert_incomplete_contains( + HuggingFaceModel::safetensors(REPO) + .cache_dir(&missing_shard.0) + .offline(true) + .resolve(), + "indexed shard `missing.safetensors` is missing", + ); + } + + #[test] + fn resolves_offline_gguf_and_reports_missing_file() { + let fixture = cache_fixture(&[("model.gguf", b"gguf")]); + let resolved = HuggingFaceModel::gguf(REPO, "model.gguf") + .cache_dir(&fixture.0) + .offline(true) + .resolve() + .unwrap(); + assert_eq!(resolved.file_name().unwrap(), "model.gguf"); + + let missing = HuggingFaceModel::gguf(REPO, "missing.gguf") + .cache_dir(&fixture.0) + .offline(true) + .resolve(); + assert!(matches!(missing, Err(HuggingFaceError::CacheMiss { .. }))); + } + + #[test] + fn verifies_gguf_filename_and_snapshot_layout() { + let fixture = cache_fixture(&[("model.gguf", b"gguf")]); + let path = Cache::new(fixture.0.clone()) + .repo(Repo::with_revision( + REPO.to_owned(), + RepoType::Model, + DEFAULT_REVISION.to_owned(), + )) + .get("model.gguf") + .unwrap(); + assert!(verify_gguf_path(&path, "model.gguf").is_ok()); + assert_incomplete_contains( + verify_gguf_path(&path, "other.gguf"), + "does not match requested filename", + ); + + let outside = fixture.0.join("outside.gguf"); + fs::write(&outside, b"gguf").unwrap(); + assert_incomplete_contains( + verify_gguf_path(&outside, "outside.gguf"), + "not directly under a `snapshots` directory", + ); + } + + #[test] + fn rejects_mixed_snapshot_paths() { + let temp = TempDir::new(); + let first = temp.0.join("snapshots").join(SHA); + let second = temp.0.join("snapshots").join("other"); + fs::create_dir_all(&first).unwrap(); + fs::create_dir_all(&second).unwrap(); + fs::write(first.join(CONFIG), b"{}").unwrap(); + fs::write(second.join(TOKENIZER), b"{}").unwrap(); + let paths = HashMap::from([ + (CONFIG.to_owned(), first.join(CONFIG)), + (TOKENIZER.to_owned(), second.join(TOKENIZER)), + ]); + assert!(matches!( + verify_snapshot_paths(&paths, SHA), + Err(HuggingFaceError::Incomplete { .. }) + )); + } +} diff --git a/vllm-cpp/src/lib.rs b/vllm-cpp/src/lib.rs index 43c4919..e4a422a 100644 --- a/vllm-cpp/src/lib.rs +++ b/vllm-cpp/src/lib.rs @@ -2,8 +2,9 @@ //! //! # Entry points //! -//! Create an [`Engine`] with [`Engine::load`] or configure native model settings -//! through [`EngineBuilder`]. [`SamplingParams`] owns sampling, stop-string, and +//! Resolve a Hub model with [`HuggingFaceModel`] (default `main`, or an explicit +//! revision), then create an [`Engine`] with [`Engine::load`] or configure native +//! model settings through [`EngineBuilder`]. [`SamplingParams`] owns sampling, stop-string, and //! [`StructuredOutput`] settings for completion calls. The engine provides //! blocking completion, streaming, raw-JSON chat, and [`Engine::submit`] for a //! concurrent [`Request`]. Enable `serde` for `serde_json::Value` chat helpers. @@ -30,8 +31,10 @@ //! [`expected_abi_version`] before versioned structs cross FFI. The default //! `bundled` feature builds the pinned native source. `system` selects a //! caller-provided installation, `dynamic-link` selects shared linking, and -//! `serde` adds typed JSON helpers. CUDA, CUTLASS, Triton AOT, Vulkan, Metal, and -//! external MLX features are experimental bundled build configuration. +//! `serde` adds typed JSON helpers. The non-optional `hf-hub` dependency provides +//! synchronous, cache-aware model retrieval without an async runtime. CUDA, +//! CUTLASS, Triton AOT, Vulkan, Metal, and external MLX features are experimental +//! bundled build configuration. //! //! Dynamic linking does not deploy `libvllm.so` or `libvllm.dylib`; applications //! must make it and its runtime dependencies visible through the platform loader, @@ -43,12 +46,14 @@ mod callback; mod engine; mod error; +mod hf; mod params; mod request; pub use callback::{StreamControl, StreamEvent, StreamOutcome}; pub use engine::{Completion, Engine, EngineBuilder, FinishReason}; -pub use error::Error; +pub use error::{Error, HuggingFaceError}; +pub use hf::HuggingFaceModel; pub use params::{SamplingParams, SchedulerPolicy, StructuredOutput, Toggle}; pub use request::{Request, RequestOutcome}; diff --git a/vllm-cpp/tests/qwen3.rs b/vllm-cpp/tests/qwen3.rs index 967f49e..3ae1aaf 100644 --- a/vllm-cpp/tests/qwen3.rs +++ b/vllm-cpp/tests/qwen3.rs @@ -550,6 +550,44 @@ fn structured_choice_is_enforced() { }); } +#[test] +fn structured_json_schema_is_enforced() { + with_engine(|engine, _| { + let schema = r#"{ + "type": "object", + "properties": { + "location": { "type": "string" }, + "temperature_celsius": { "type": "number" }, + "condition": { "type": "string" } + }, + "required": ["location", "temperature_celsius", "condition"], + "additionalProperties": false + }"#; + let params = SamplingParams::greedy() + .max_tokens(64) + .structured_output(StructuredOutput::JsonSchema(schema.to_owned())); + let completion = engine + .complete( + "Extract the weather report as JSON: Paris is sunny and 22 degrees Celsius.", + ¶ms, + ) + .expect("JSON Schema completion"); + let value: serde_json::Value = + serde_json::from_str(completion.text.trim()).expect("valid structured JSON"); + let object = value.as_object().expect("JSON object"); + assert_eq!(object.len(), 3, "unexpected properties: {object:?}"); + assert!(object + .get("location") + .is_some_and(|value| value.is_string())); + assert!(object + .get("temperature_celsius") + .is_some_and(|value| value.is_number())); + assert!(object + .get("condition") + .is_some_and(|value| value.is_string())); + }); +} + #[test] fn terminal_stop_is_natural_finish_for_completion_and_chat() { with_engine(|engine, _| { diff --git a/vllm-cpp/tests/safe_api.rs b/vllm-cpp/tests/safe_api.rs index 8e25025..f5f69f5 100644 --- a/vllm-cpp/tests/safe_api.rs +++ b/vllm-cpp/tests/safe_api.rs @@ -1,7 +1,11 @@ use static_assertions::{assert_impl_all, assert_not_impl_any}; -use vllm_cpp::{Engine, Error, Request, SchedulerPolicy, Toggle}; +use vllm_cpp::{ + Engine, Error, HuggingFaceError, HuggingFaceModel, Request, SchedulerPolicy, Toggle, +}; assert_impl_all!(Engine: Send, Sync, Clone); +assert_impl_all!(HuggingFaceError: Clone, std::fmt::Debug, Eq, PartialEq); +assert_impl_all!(HuggingFaceModel: Clone, std::fmt::Debug); assert_impl_all!(Request: Send); assert_not_impl_any!(Request: Sync); @@ -9,6 +13,14 @@ fn missing_model() -> &'static str { "/nonexistent/vllm-cpp-rs-safe-api-model" } +#[test] +fn hugging_face_constructors_accept_default_and_explicit_revisions() { + let gguf = HuggingFaceModel::gguf("owner/model", "model.gguf"); + let safetensors = HuggingFaceModel::safetensors("owner/model").revision("release"); + assert!(format!("{gguf:?}").contains("revision: \"main\"")); + assert!(format!("{safetensors:?}").contains("revision: \"release\"")); +} + #[test] fn reports_expected_abi() { assert_eq!(vllm_cpp::expected_abi_version(), 10);