From 813c30673b571a96dad66f337766a9d337e21da1 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 4 Sep 2026 22:06:16 -0600 Subject: [PATCH 1/9] feat: scaffold native fulltext package --- .github/workflows/ci.yml | 64 ++ .gitignore | 5 + .prettierignore | 4 + .prettierrc.json | 6 + CONTRIBUTING.md | 37 + Cargo.lock | 1348 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 33 + README.md | 62 +- build.rs | 3 + deny.toml | 21 + dependencies.md | 28 + docs/scaffold-design.md | 341 +++++++++ package-lock.json | 86 +++ package.json | 69 ++ rust-toolchain.toml | 4 + rustfmt.toml | 3 + src/boundary.rs | 39 ++ src/directory_harness.rs | 334 +++++++++ src/lib.rs | 39 ++ test/dependencies.test.mjs | 21 + test/native.test.mjs | 25 + test/package.test.mjs | 64 ++ ts/addon.d.ts | 12 + ts/errors.ts | 27 + ts/load-addon.ts | 57 ++ ts/native.ts | 33 + tsconfig.json | 17 + 27 files changed, 2780 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .prettierignore create mode 100644 .prettierrc.json create mode 100644 CONTRIBUTING.md create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 build.rs create mode 100644 deny.toml create mode 100644 dependencies.md create mode 100644 docs/scaffold-design.md create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 rust-toolchain.toml create mode 100644 rustfmt.toml create mode 100644 src/boundary.rs create mode 100644 src/directory_harness.rs create mode 100644 src/lib.rs create mode 100644 test/dependencies.test.mjs create mode 100644 test/native.test.mjs create mode 100644 test/package.test.mjs create mode 100644 ts/addon.d.ts create mode 100644 ts/errors.ts create mode 100644 ts/load-addon.ts create mode 100644 ts/native.ts create mode 100644 tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..46350ad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.os }} / Node ${{ matrix.node }} + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + node: '22.18.0' + - os: ubuntu-latest + node: '24' + - os: macos-14 + node: '24' + - os: windows-latest + node: '24' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + - uses: dtolnay/rust-toolchain@1.90.0 + with: + components: clippy,rustfmt + - uses: Swatinem/rust-cache@v2 + - run: npm ci --ignore-scripts + - run: npm run format:check + - run: npm run lint + - run: npm test + + supply-chain: + name: Supply chain and native linkage + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '24' + cache: npm + - uses: dtolnay/rust-toolchain@1.90.0 + - uses: taiki-e/install-action@cargo-deny + - run: npm ci --ignore-scripts + - run: cargo deny check + - run: npm audit --audit-level=high + - run: npm run build:native + - name: Reject linked RocksDB symbols + run: | + if cargo tree --locked | grep -i rocksdb; then exit 1; fi + if nm -D fulltext.linux-x64-gnu.node | grep -i rocksdb; then exit 1; fi + - name: Verify unwind profile + run: cargo rustc --locked --release --features node-api -- --print cfg | grep 'panic="unwind"' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb2269a --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +dist/ +target/ +*.node +*.tgz diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..c2e398e --- /dev/null +++ b/.prettierignore @@ -0,0 +1,4 @@ +Cargo.lock +dist/ +target/ +ts/addon.d.ts diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..eddeebc --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,6 @@ +{ + "printWidth": 120, + "singleQuote": true, + "useTabs": true, + "trailingComma": "all" +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5aafbfb --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Contributing + +## Setup + +Install Node.js 22.18 or newer and Rust 1.90, then install JavaScript dependencies without running +package lifecycle scripts: + +```bash +npm ci --ignore-scripts +``` + +## Checks + +Run the same primary checks used in CI: + +```bash +npm run format:check +npm run lint +npm test +``` + +Rust tests exercise the engine and directory contract without Node.js. Node tests build the addon, +load it through the public `./native` entry point, verify panic containment, and install the output +of `npm pack` into a temporary consumer project. + +## Design constraints + +- Keep search, indexing, scheduling, and lifecycle behavior shared across storage backends. +- Keep backend choice explicit; do not introduce automatic fallback. +- Do not link RocksDB into this addon. The future Rocks backend must use the versioned lease owned + by rocksdb-js. +- Do not expose generated Node-API declarations as the public TypeScript API. +- Keep CPU and I/O work off the Node.js event loop. +- Add a direct test for each source module. + +Open an issue before changing a public package entry point, native ABI, persistence contract, or +supported-platform matrix. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..88d34f4 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1348 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "bon" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3fac94a66da67200398458a25412bcc3f9b6443b5119a6cad9cf3ccfcd8cc6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4654961ad0494e4774c5c60b4cb4cd0ae9b9d92d039d901638b1dba97ebebf5" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "ctor" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.5", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "harper-fulltext" +version = "0.0.0" +dependencies = [ + "napi", + "napi-build", + "napi-derive", + "tantivy", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" + +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + +[[package]] +name = "napi" +version = "2.16.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55740c4ae1d8696773c78fdafd5d0e5fe9bc9f1b071c7ba493ba5c413a9184f3" +dependencies = [ + "bitflags", + "ctor", + "napi-derive", + "napi-sys", + "once_cell", +] + +[[package]] +name = "napi-build" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab" + +[[package]] +name = "napi-derive" +version = "2.16.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbe2585d8ac223f7d34f13701434b9d5f4eb9c332cccce8dee57ea18ab8ab0c" +dependencies = [ + "cfg-if", + "convert_case", + "napi-derive-backend", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "napi-derive-backend" +version = "1.0.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1639aaa9eeb76e91c6ae66da8ce3e89e921cd3885e99ec85f4abacae72fc91bf" +dependencies = [ + "convert_case", + "once_cell", + "proc-macro2", + "quote", + "regex", + "semver", + "syn 2.0.119", +] + +[[package]] +name = "napi-sys" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427802e8ec3a734331fec1035594a210ce1ff4dc5bc1950530920ab717964ea3" +dependencies = [ + "libloading", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "ordered-float" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c9e0d9b23589f26070720bac724174bfec1083e82f7854cdd0267518343c0" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.5", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "sketches-ddsketch" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513c3f5f732bfd6fbb187619c2dfe9d2f25f1a2976f01d575f0fd329d565df56" +dependencies = [ + "serde", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tantivy" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edde6a10743fff00a4e1a8c9ef020bf5f3cbad301b7d2d39f2b07f123c4eac07" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time", + "typetag", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fed3d674429bcd2de5d0a6d1aa5495fed8afd9c5ecce993019caf7615f53fa4" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf10915aa75da3c3b0d58b58853d2e889efbaf32d4982a4c3715dde6bba23e5" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" +dependencies = [ + "fnv", + "nom", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" +dependencies = [ + "futures-util", + "itertools", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbb051742da9d53ca9e8fff43a9b10e319338b24e2c0e15d0372df19ffeb951" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.5", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.5", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..b42ebf9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "harper-fulltext" +version = "0.0.0" +edition = "2021" +rust-version = "1.90" +description = "Tantivy full-text indexing for Node.js and Harper" +license = "Apache-2.0" +repository = "https://github.com/HarperFast/fulltext" + +[lib] +crate-type = ["cdylib", "rlib"] + +[features] +default = [] +node-api = ["dep:napi", "dep:napi-derive"] +test-panic = ["node-api"] + +[dependencies] +napi = { version = "=2.16.17", default-features = false, features = ["napi8"], optional = true } +napi-derive = { version = "=2.16.13", optional = true } +tantivy = "=0.26.1" + +[build-dependencies] +napi-build = "=2.4.1" + +[profile.release] +lto = "thin" +codegen-units = 1 +panic = "unwind" + +[profile.bench] +lto = "thin" +codegen-units = 1 diff --git a/README.md b/README.md index 34c3891..9b9d24b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,60 @@ -# fulltext -Native Tantivy full-text search for Node.js with native and caller-owned RocksDB storage +# @harperfast/fulltext + +Native Tantivy full-text indexing for Node.js, with a native filesystem backend and a planned +caller-owned rocksdb-js backend for Harper. + +This repository is under active development. The initial scaffold exposes runtime capability +information through the native entry point; indexing and search APIs are tracked separately and +are not yet available. + +## Requirements + +- Node.js 22.18 or newer, or Node.js 24 or newer +- Rust 1.90 when building from source + +The package never compiles or downloads native code during installation. A supported prebuilt +artifact must be present for the executing platform. + +## Native usage + +```js +import { runtimeInfo } from '@harperfast/fulltext/native'; + +const info = await runtimeInfo(); +console.log(info.tantivyVersion); +``` + +`runtimeInfo()` is asynchronous so later search and indexing operations can remain off the Node.js +event loop without changing the public calling convention. + +## Storage boundaries + +The package is designed around two explicit entry points: + +- `@harperfast/fulltext/native` uses Tantivy's native directory implementation and has no + rocksdb-js dependency. +- `@harperfast/fulltext/rocks` will use a caller-owned rocksdb-js database through a versioned + native capability lease. It is not exported until that contract is implemented and tested. + +There is no generic storage selector and no fallback between backends. Harper will consume only the +Rocks entry point. The fulltext addon will not link its own copy of RocksDB. + +## Development + +```bash +npm ci --ignore-scripts +npm run build:debug +npm test +npm run lint +npm run format:check +``` + +Generated Node-API declarations in `ts/addon.d.ts` are private implementation types. Consumers use +only the types exported from a package entry point. + +See [CONTRIBUTING.md](CONTRIBUTING.md) for the development workflow and +[docs/scaffold-design.md](docs/scaffold-design.md) for the architecture behind the initial package. + +## License + +Apache-2.0 diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..bed4fb7 --- /dev/null +++ b/build.rs @@ -0,0 +1,3 @@ +fn main() { + napi_build::setup(); +} diff --git a/deny.toml b/deny.toml new file mode 100644 index 0000000..b5b3d35 --- /dev/null +++ b/deny.toml @@ -0,0 +1,21 @@ +[advisories] +yanked = "deny" + +[licenses] +allow = [ + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-3-Clause", + "MIT", + "Unicode-3.0", + "Zlib", +] + +[bans] +multiple-versions = "deny" +wildcards = "deny" +skip = [{ name = "windows-sys", version = "0.59.0" }] + +[sources] +unknown-registry = "deny" +unknown-git = "deny" diff --git a/dependencies.md b/dependencies.md new file mode 100644 index 0000000..ffe67e5 --- /dev/null +++ b/dependencies.md @@ -0,0 +1,28 @@ +# Dependencies + +Direct dependency versions are exact so the native artifact is reproducible and upgrades are +reviewed deliberately. + +## Rust runtime and build graph + +| Dependency | Scope | Purpose | +| --------------------- | ------------------------------- | ------------------------------------------------------------------------- | +| `tantivy` 0.26.1 | runtime | Full-text indexing and search engine, including the `Directory` contract. | +| `napi` 2.16.17 | optional runtime | Node-API values and error conversion for the addon build. | +| `napi-derive` 2.16.13 | optional build/runtime boundary | Generates Node-API exports. | +| `napi-build` 2.4.1 | build | Configures platform-specific addon linking. | + +The Rust dependency graph must not include RocksDB. The future Rocks backend calls a C-ABI +capability table owned by rocksdb-js rather than linking a second RocksDB runtime. + +## JavaScript development graph + +| Dependency | Scope | Purpose | +| --------------------- | ----------- | --------------------------------------------------------------------- | +| `@napi-rs/cli` 2.18.4 | development | Builds and names native artifacts and generates private declarations. | +| `@types/node` 24.10.0 | development | Type information for supported Node.js APIs. | +| `prettier` 3.6.2 | development | Repository formatting checks. | +| `typescript` 5.9.3 | development | Compiles the public façade and declarations. | + +The native entry point has no production npm dependencies. rocksdb-js will be an optional peer +dependency only when the Rocks entry point is implemented. diff --git a/docs/scaffold-design.md b/docs/scaffold-design.md new file mode 100644 index 0000000..59aca77 --- /dev/null +++ b/docs/scaffold-design.md @@ -0,0 +1,341 @@ +# Fulltext package scaffold + +- **Issue:** [HarperFast/fulltext#9](https://github.com/HarperFast/fulltext/issues/9) +- **Status:** implementation plan +- **Base:** `origin/main` +- **Initial engine:** Tantivy 0.26.1 + +## Objective + +Establish the repository, build, package, and test architecture for `@harperfast/fulltext` without +prematurely implementing the search engine or committing to unresolved rocksdb-js bridge details. +The target architecture supports one shared Rust addon behind two explicit package entry points: + +- `@harperfast/fulltext/native`, backed by Tantivy `MmapDirectory`; +- `@harperfast/fulltext/rocks`, backed by a future `RocksDbDirectory` over a caller-owned + rocksdb-js lease. + +Harper will consume only the Rocks entry point. The native entry point exists for standalone use, +directory conformance, and comparative performance measurement. Issue #9 exports only the tested +native entry point. The Rocks entry point is added after Phase 0 defines and tests its lease. + +## Design assessment + +This work establishes a public package API, a JavaScript/Rust boundary, and a future +fulltext/rocksdb-js native boundary. Those are durable interfaces, so the scaffold requires a +reviewed design and must keep unresolved storage details behind explicit seams. + +The invariant is: **backend selection is explicit and never falls back, all indexing and search +behavior is implemented once, the addon links no RocksDB symbols, no native panic crosses Node-API, +and no opaque native lease is used before its identity, layout, ABI, capabilities, and lifetime are +validated.** + +## Source-grounded baseline + +The plan was checked against: + +- HarperFast/hnsw at `42c71850536afce0b72d7511158fe600cd0879c6`: Rust `cdylib` plus + `rlib`, a narrow Node binding, Rust tests, Node smoke coverage, and platform artifacts; +- HarperFast/symphony at `eb2c1b6760605f908475345672f9cab3ae5548cc`: napi-rs artifact + packaging, generated low-level declarations, a TypeScript façade, multi-platform CI, and + integration testing; +- Tantivy 0.26.1 at `d8f4c0b703120ed98f06297724dc1522df6019b9`: the pinned engine and + `Directory` contract; +- rocksdb-js 2.8.0 at `7ab102ca3e9600343bcefe6f19204b111836ec52`: the current caller-owned + database and column-family baseline. + +The scaffold reuses those patterns selectively. HNSW's hand-written loader is not copied when +napi-rs artifact packaging supplies the same function. Symphony's generated native declarations +remain private rather than becoming the user API. Neither reference determines the unresolved +Rocks lease or durability contract. + +## Repository shape + +```text +. +├── Cargo.toml +├── Cargo.lock +├── build.rs +├── package.json +├── tsconfig.json +├── rust-toolchain.toml +├── rustfmt.toml +├── src/ +│ ├── lib.rs +│ ├── boundary.rs +│ └── directory_harness.rs +├── ts/ +│ ├── addon.d.ts +│ ├── shared.ts +│ └── native.ts +├── test/ +├── docs/ +└── .github/workflows/ +``` + +The first scaffold commit creates only enough source to build and smoke-test the package. Empty +architectural directories are not added. Modules appear with the behavior and unit tests that +justify them. + +## Package and API boundaries + +`package.json` initially exports only the backend exercised end to end: + +```json +{ + "exports": { + "./native": { + "types": "./dist/native.d.ts", + "import": "./dist/native.js" + } + } +} +``` + +The TypeScript façade is the public API. Generated napi-rs declarations describe the low-level +addon only. Public types must not expose Tantivy, napi-rs, or RocksDB implementation types. + +The native entry point loads the shared artifact and calls only the native factory. A later Rocks +entry point loads the same artifact and calls only the Rocks factory. There is no generic public +factory, storage option bag, or fallback between them. Issue #9 neither exports the unimplemented +Rocks subpath nor invents the lease shape before Phase 0 establishes it. + +The future lease has non-negotiable safety constraints now: it is a type-tagged N-API `External` +carrying a fixed magic value, integer ABI version, structure size, capability set, and a token +generated for each loaded rocksdb-js addon instance. The wrapper validates every field before +calling through the table; a +mismatch returns a stable coded JavaScript error. The lease exposes an opaque handle and C-ABI +function table owned by rocksdb-js, never a `rocksdb::DB*`, numeric pointer, untagged external, or +Rust/C++ implementation type. It retains rocksdb-js lifetime ownership for every admitted operation; +database close rejects new work and joins or safely revokes all holders before releasing handles. + +## Rust boundary + +The crate builds as: + +- `rlib` so engine and storage behavior can be tested directly in Rust; +- `cdylib` for the Node-API addon. The initial artifact contains only implemented native code; the + same artifact gains RocksDbDirectory only when that backend is tested and exported. + +The initial N-API surface is deliberately small: typed package/version/ABI capability reporting +proves loading and generated declarations without JSON serialization. The public façade exposes it +as a Promise so no future work API inherits a synchronous public signature. Constant-time native +introspection may be implemented synchronously underneath that façade; blocking, `block_on`, and +CPU- or I/O-bearing synchronous exports are forbidden. Issue #17 introduces the bounded +package-owned executor before any search, indexing, commit, or storage operation is exposed. +Packed operations use borrowed request buffers and transfer owned `Vec` responses rather than +JSON strings. Batch operations are the default shape; the façade never invokes native code once per +document. Work is admitted to a bounded package-owned pool and never creates a runtime task or +thread per call. + +The release and benchmark profiles use thin LTO, one codegen unit, and `panic = "unwind"`. +The addon does not install a global allocator because doing so could change allocation behavior for +the host Node process. Every package-owned N-API boundary invokes a shared +`catch_unwind` adapter that converts a panic into a JavaScript error with a stable `code`; a panic +reaching that boundary may not unwind through FFI. A test-only Cargo feature exposes a panic probe +to the Node smoke suite, but the probe and feature are absent from published artifacts. All public +errors carry stable codes from the first release. + +Before implementing the adapter, the pinned napi-rs behavior is verified so the wrapper neither +double-wraps panics nor replaces useful native error mapping. Every package-owned worker entry +catches and records its panic before completing its promise with a coded failure. Query depth and +complexity are bounded before native execution. Allocation failure, stack overflow, double panic +during unwinding, and a panic wholly inside an upstream-owned thread can still abort the process; +those paths are explicitly out of scope for `catch_unwind` and remain targets for input bounds, +upstream qualification, and process supervision rather than false error-containment claims. + +Any caught panic poisons its owning object. That object and all work derived from it fail fast with +`E_POISONED`; callers cannot retry through potentially corrupted Tantivy or adapter state. The +panic smoke route verifies both the original coded failure and terminal poison behavior. CI also +asserts that the effective release profile retains `panic = "unwind"`. + +Tantivy is pinned exactly to 0.26.1. napi-rs is a build and binding dependency, not part of the +public API. The Cargo feature layout must allow Rust unit tests to exercise engine code without +requiring a Node environment. + +## Build and packaging + +The scaffold uses npm and committed `package-lock.json`, matching current Symphony practice and +Harper CI. Scripts cover: + +- TypeScript build and type checking; +- debug and release native builds; +- Rust unit tests and Clippy; +- Node smoke tests; +- formatting checks; +- napi-rs artifact assembly. + +The initial platform matrix follows the qualified intersection of HNSW and Symphony: + +- Linux x64 and arm64, glibc and musl; +- macOS arm64 and x64 build artifacts; +- Windows x64. + +One addon artifact is built per platform rather than one per backend. Every matrix artifact is +loaded and smoke-tested on its target architecture; producing a file is +not sufficient. A missing or mismatched artifact fails with the resolved platform triple and never +falls back to an install-time source build or download. Platform packages and publishing automation +are structured now but completed under the dedicated release issue. No install script compiles or +downloads code silently in this scaffold. + +## Testing and end-to-end route + +Every introduced source module has a direct test. The scaffold gates: + +1. `cargo test`; +2. `cargo clippy --all-targets -- -D warnings`; +3. TypeScript type checking; +4. native addon build; +5. a Node smoke test that imports the public native entry point, awaits typed capability reporting, + and proves a test-only native panic becomes a stable coded JavaScript error; +6. a backend-parameterized Rust `Directory` conformance harness, initially run against Tantivy + `MmapDirectory`, covering atomic metadata writes, exclusive writer/meta locks, open-file + deletion, boundary range reads, `meta.json` watch notification, and `sync_directory`; the + harness includes two-writer lock races, watch/delete/read concurrency, and counters for read + calls and requested bytes; the Rocks adapter adds copied-byte accounting so later implementations + expose read amplification; +7. negative-control Directory implementations with non-atomic metadata and always-successful locks + that the harness must reject, plus a realistic KV-shaped control using process-local locks, + polling watches, and non-atomic chunk publication; +8. a smoke test installed from `npm pack` output rather than the repository tree, proving the + exports map, packaged files, addon resolution, stable errors, and platform artifact together; +9. package-content and loaded-addon inspection proving private generated bindings, unintended + source artifacts, unlisted subpaths, test-only probes, and package install scripts are absent; +10. `cargo deny` gates for advisories, licenses, sources, bans, and duplicate native libraries, + plus a policy-scoped npm audit. + +All Cargo operations use `--locked`; npm CI operations use `npm ci`. The scaffold gates loading +on the executing architecture, not merely cross-compilation. It does not claim to exercise +indexing, search, Rocks durability, concurrency, or catalog performance. + +The end-to-end route for this issue is the Node smoke test against the built addon through the +published `./native` entry point. Rocks behavior is intentionally not observable end to end until +the Phase 0 bridge is selected. + +## Dependency decisions + +Production dependencies are limited to Tantivy and the napi-rs binding crates needed to build the +addon. TypeScript, Node types, napi-rs CLI, and formatting/lint tooling are development-only. +rocksdb-js is not installed by the native entry point and is not bundled into the addon. The future +Rocks entry point uses an optional peer relationship so the caller owns the qualified version. + +The Cargo dependency graph must contain no RocksDB crate or linked RocksDB native library. CI checks +`cargo tree` and built-artifact linkage. All Rocks operations eventually call the versioned +function table supplied and owned by rocksdb-js. + +Every dependency added by the scaffold is recorded in `dependencies.md` with its purpose and +whether it appears in the runtime, build, or development graph. CI checks the documented direct +dependency names against Cargo and npm manifests so the ledger cannot silently drift. + +Platform package names are reserved before the first npm release. They are published with +provenance and referenced at the exact root-package version, preventing an unrelated package or +version from satisfying artifact resolution. + +## Approaches considered + +### Different layer: implement the wrapper inside Harper + +Rejected because it prevents standalone rocksdb-js use, couples native artifact releases to Harper, +and makes the native reference backend a Harper concern. The Node wrapper is an independently +versioned deliverable with Harper as one consumer. + +### Deeper cause: add fulltext behavior to the rocksdb-js addon + +Rejected because it would make rocksdb-js own Tantivy, query behavior, and the fulltext release +cadence. That removes the cross-addon boundary but collapses two independently useful libraries +into one. The narrower fix is to keep all RocksDB calls and handle ownership in rocksdb-js while +exposing only the minimum versioned C-ABI capability table to fulltext. + +### Do less: ship only the native entry point in issue #9 + +Chosen for this issue. The scaffold proves package loading, generated bindings, error containment, +and the Tantivy `Directory` conformance harness without exporting placeholder Rocks APIs or lease +types. The Rocks entry point is added only after Phase 0 selects and tests the bridge contract. + +### Chosen: one shared addon with tested subpath façades + +The long-term package uses one platform artifact and explicit `./native` and `./rocks` façades. +This keeps indexing, search, scheduling, errors, and lifecycle single-sourced; permits native +conformance and performance comparison; leaves Rocks ownership with rocksdb-js; and avoids a doubled +artifact matrix. The artifact links no RocksDB symbols. Harper imports and exposes only the Rocks +façade. The presence of compiled `MmapDirectory` code is not a storage fallback: no Harper code +calls its factory, and no generic factory can select it. + +### Rejected bridge: link RocksDB into the fulltext addon + +A second statically linked RocksDB runtime cannot safely operate on handles created by rocksdb-js +and can introduce incompatible vtables, allocators, static state, and teardown. The build therefore +enforces a zero-RocksDB dependency graph; all future Rocks operations call the versioned function +table supplied and owned by rocksdb-js. + +### Alternative: compile one native artifact per backend + +Rejected because it doubles every platform build and release artifact and creates a mode where one +backend compiles while the other silently drifts. Once the addon links zero RocksDB symbols and +backend choice is available only through explicit factories, binary separation does not prevent an +additional unsafe operation. + +### Alternative: place fulltext behind a process boundary + +Rejected because a sidecar cannot use the caller-owned RocksDB handle: the database is already open +for writing inside Harper's process. IPC would isolate panics but cannot satisfy the single-owner +storage requirement without adding a second store or moving Harper's primary database ownership. + +## Storage seam considered + +### Chosen: implement Tantivy Directory directly over RocksDB + +Tantivy keeps its segment and metadata formats while the adapter maps logical files to RocksDB +objects. This avoids a second persistent store and permits bounded recovery from the caller-owned +database. + +### Alternative: materialize Rocks-backed segments into a local MmapDirectory + +Rejected because every activation and recovery would require a second physical copy, introduce +local-filesystem capacity and cleanup semantics into Harper, and add a checkpoint/materialization +protocol whose correctness is separate from Tantivy publication. + +### Alternative: persist immutable segments in RocksDB and use local mmap files as a read cache + +Rejected for the Harper release because each node needs local capacity proportional to the index, +startup requires cache hydration, and correctness acquires an additional cache lifecycle. Phase 0 +may retain this as a comparative benchmark because it avoids a RocksDB lookup and copy on every +Tantivy range read, but it cannot become an implicit fallback. + +### Alternative: checkpoint RamDirectory into RocksDB + +Rejected because index residency and rebuild memory would scale with index size. Product catalogs +with hundreds of millions of records cannot use whole-index RAM as their persistence staging +contract. + +The direct Directory remains a Phase 0 proof obligation rather than an assumed success. If it +cannot satisfy Tantivy semantics and the performance gates, Harper does not release the feature. +Phase 0 must also assign durability ownership: whether objects use WAL, which operation implements +`sync_directory`, what makes object bytes durable before metadata publication, and which +capability reports that guarantee. A commit cannot be reported durable until that sequence is +proven by crash tests. The reserved durability capability must express the invariant that all +segment bytes become durable before atomic `meta.json` publication; successful commit reporting +comes only after that publication is durable. + +## Sequencing + +1. Land the minimal buildable Rust/Node scaffold and native smoke route. +2. Define the public façade and packed operation ABI in + [#14](https://github.com/HarperFast/fulltext/issues/14). +3. Implement bounded runtime and writer coordination in + [#17](https://github.com/HarperFast/fulltext/issues/17). +4. Implement the native reference backend in + [#16](https://github.com/HarperFast/fulltext/issues/16). +5. Resolve the minimum Rocks bridge through + [#7](https://github.com/HarperFast/fulltext/issues/7) and + [HarperFast/rocksdb-js#834](https://github.com/HarperFast/rocksdb-js/issues/834) before fixing + its public lease surface. + +## Out of scope + +- Search/index implementation. +- A concrete Rocks lease ABI. +- Harper schema or query integration. +- Performance claims beyond build and smoke overhead. +- npm publication. +- Treating native storage as a Harper fallback. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..4e0e022 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,86 @@ +{ + "name": "@harperfast/fulltext", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@harperfast/fulltext", + "version": "0.0.0", + "license": "Apache-2.0", + "devDependencies": { + "@napi-rs/cli": "2.18.4", + "@types/node": "24.10.0", + "prettier": "3.6.2", + "typescript": "5.9.3" + }, + "engines": { + "node": "^22.18.0 || >=24.0.0" + } + }, + "node_modules/@napi-rs/cli": { + "version": "2.18.4", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.4.tgz", + "integrity": "sha512-SgJeA4df9DE2iAEpr3M2H0OKl/yjtg1BnRI5/JyowS71tUWhrfSu2LT0V3vlHET+g1hBVlrO60PmEXwUEKp8Mg==", + "dev": true, + "license": "MIT", + "bin": { + "napi": "scripts/index.js" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + } + }, + "node_modules/@types/node": { + "version": "24.10.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.0.tgz", + "integrity": "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..edcde0a --- /dev/null +++ b/package.json @@ -0,0 +1,69 @@ +{ + "name": "@harperfast/fulltext", + "version": "0.0.0", + "description": "Tantivy full-text indexing for Node.js and Harper", + "type": "module", + "exports": { + "./native": { + "types": "./dist/native.d.ts", + "import": "./dist/native.js" + } + }, + "files": [ + "dist/", + "fulltext.*.node", + "README.md", + "LICENSE", + "dependencies.md" + ], + "napi": { + "name": "fulltext", + "triples": { + "defaults": false, + "additional": [ + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] + } + }, + "scripts": { + "build": "npm run build:typescript && npm run build:native", + "build:debug": "npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features node-api", + "build:native": "napi build --platform --js false --dts ts/addon.d.ts --release --features node-api", + "build:test-native": "napi build --platform --js false --dts ts/addon.d.ts --features test-panic", + "build:typescript": "tsc -p tsconfig.json", + "check": "npm run format:check && npm run lint && npm run test", + "format": "prettier --write . && cargo fmt", + "format:check": "prettier --check . && cargo fmt --check", + "lint": "tsc -p tsconfig.json --noEmit && cargo clippy --locked --all-targets --all-features -- -D warnings", + "test": "cargo test --locked --all-features && npm run build:test-native && npm run build:typescript && node --test --test-concurrency=1 test/*.test.mjs", + "test:rust": "cargo test --locked --all-features", + "test:node": "node --test --test-concurrency=1 test/*.test.mjs" + }, + "engines": { + "node": "^22.18.0 || >=24.0.0" + }, + "keywords": [ + "tantivy", + "full-text-search", + "node-api", + "rocksdb", + "harper" + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/HarperFast/fulltext.git" + }, + "devDependencies": { + "@napi-rs/cli": "2.18.4", + "@types/node": "24.10.0", + "prettier": "3.6.2", + "typescript": "5.9.3" + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..66f329f --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.90.0" +profile = "minimal" +components = ["clippy", "rustfmt"] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..22b75b0 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,3 @@ +edition = "2021" +hard_tabs = true +max_width = 120 diff --git a/src/boundary.rs b/src/boundary.rs new file mode 100644 index 0000000..70d6055 --- /dev/null +++ b/src/boundary.rs @@ -0,0 +1,39 @@ +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use napi::{Error, Result, Status}; + +static POISONED: AtomicBool = AtomicBool::new(false); + +pub fn run(operation: impl FnOnce() -> T) -> Result { + if POISONED.load(Ordering::Acquire) { + return Err(coded_error("E_POISONED", "the native addon is in a terminal state")); + } + match catch_unwind(AssertUnwindSafe(operation)) { + Ok(value) => Ok(value), + Err(_) => { + POISONED.store(true, Ordering::Release); + Err(coded_error("E_NATIVE_PANIC", "native operation panicked")) + } + } +} + +fn coded_error(code: &str, message: impl AsRef) -> Error { + Error::new(Status::GenericFailure, format!("[{code}] {}", message.as_ref())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn panic_poisoning_is_terminal() { + POISONED.store(false, Ordering::Release); + let panic_error = run(|| panic!("boom")).unwrap_err(); + assert!(panic_error.reason.contains("[E_NATIVE_PANIC]")); + + let poisoned_error = run(|| 1).unwrap_err(); + assert!(poisoned_error.reason.contains("[E_POISONED]")); + POISONED.store(false, Ordering::Release); + } +} diff --git a/src/directory_harness.rs b/src/directory_harness.rs new file mode 100644 index 0000000..0786f49 --- /dev/null +++ b/src/directory_harness.rs @@ -0,0 +1,334 @@ +use std::fmt; +use std::io::{self, Write}; +use std::ops::Range; +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError}; +use tantivy::directory::{ + Directory, DirectoryLock, FileHandle, OwnedBytes, WatchCallback, WatchHandle, WritePtr, INDEX_WRITER_LOCK, +}; +use tantivy::HasLen; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ReadMetrics { + pub calls: u64, + pub bytes_requested: u64, +} + +#[derive(Debug, Default)] +struct Counters { + calls: AtomicU64, + bytes_requested: AtomicU64, +} + +#[derive(Clone)] +pub struct InstrumentedDirectory { + inner: D, + counters: Arc, +} + +impl InstrumentedDirectory { + pub fn new(inner: D) -> Self { + Self { + inner, + counters: Arc::new(Counters::default()), + } + } + + pub fn read_metrics(&self) -> ReadMetrics { + ReadMetrics { + calls: self.counters.calls.load(Ordering::Relaxed), + bytes_requested: self.counters.bytes_requested.load(Ordering::Relaxed), + } + } +} + +impl fmt::Debug for InstrumentedDirectory { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_tuple("InstrumentedDirectory") + .field(&self.inner) + .finish() + } +} + +#[derive(Debug)] +struct InstrumentedFileHandle { + inner: Arc, + counters: Arc, +} + +impl HasLen for InstrumentedFileHandle { + fn len(&self) -> usize { + self.inner.len() + } +} + +impl FileHandle for InstrumentedFileHandle { + fn read_bytes(&self, range: Range) -> io::Result { + self.counters.calls.fetch_add(1, Ordering::Relaxed); + self.counters + .bytes_requested + .fetch_add((range.end - range.start) as u64, Ordering::Relaxed); + self.inner.read_bytes(range) + } +} + +impl Directory for InstrumentedDirectory +where + D: Directory + Clone, +{ + fn get_file_handle(&self, path: &Path) -> Result, OpenReadError> { + Ok(Arc::new(InstrumentedFileHandle { + inner: self.inner.get_file_handle(path)?, + counters: self.counters.clone(), + })) + } + + fn delete(&self, path: &Path) -> Result<(), DeleteError> { + self.inner.delete(path) + } + + fn exists(&self, path: &Path) -> Result { + self.inner.exists(path) + } + + fn open_write(&self, path: &Path) -> Result { + self.inner.open_write(path) + } + + fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { + self.inner.atomic_read(path) + } + + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { + self.inner.atomic_write(path, data) + } + + fn sync_directory(&self) -> io::Result<()> { + self.inner.sync_directory() + } + + fn acquire_lock(&self, lock: &tantivy::directory::Lock) -> Result { + self.inner.acquire_lock(lock) + } + + fn watch(&self, callback: WatchCallback) -> tantivy::Result { + self.inner.watch(callback) + } +} + +pub fn verify_directory(directory: D) -> Result +where + D: Directory + Clone, +{ + let directory = InstrumentedDirectory::new(directory); + verify_write_read_delete(&directory)?; + verify_atomic_metadata(&directory)?; + verify_writer_exclusion(&directory)?; + verify_watch(&directory)?; + directory.sync_directory().map_err(|error| error.to_string())?; + Ok(directory.read_metrics()) +} + +fn verify_write_read_delete(directory: &D) -> Result<(), String> +where + D: Directory + Clone, +{ + let path = Path::new("segment"); + let mut writer = directory.open_write(path).map_err(|error| error.to_string())?; + writer.write_all(b"0123456789").map_err(|error| error.to_string())?; + writer.flush().map_err(|error| error.to_string())?; + + let file = directory.open_read(path).map_err(|error| error.to_string())?; + let middle = file.slice(2..7).read_bytes().map_err(|error| error.to_string())?; + if middle.as_slice() != b"23456" { + return Err("range read returned unexpected bytes".to_owned()); + } + let deleting_directory = directory.clone(); + std::thread::spawn(move || deleting_directory.delete(Path::new("segment"))) + .join() + .map_err(|_| "segment deletion panicked".to_owned())? + .map_err(|error| error.to_string())?; + let retained = file.read_bytes().map_err(|error| error.to_string())?; + if retained.as_slice() != b"0123456789" { + return Err("an open file changed after deletion".to_owned()); + } + Ok(()) +} + +pub fn verify_failed_atomic_replacement( + directory: &dyn Directory, + replacement: impl FnOnce() -> io::Result<()>, +) -> Result<(), String> { + let path = Path::new("meta.json"); + let before = directory.atomic_read(path).map_err(|error| error.to_string())?; + if replacement().is_ok() { + return Err("fault injection did not fail the metadata replacement".to_owned()); + } + let after = directory.atomic_read(path).map_err(|error| error.to_string())?; + if before != after { + return Err("failed metadata replacement became observable".to_owned()); + } + Ok(()) +} + +fn verify_atomic_metadata(directory: &dyn Directory) -> Result<(), String> { + let path = Path::new("meta.json"); + directory + .atomic_write(path, b"first") + .map_err(|error| error.to_string())?; + if directory.atomic_read(path).map_err(|error| error.to_string())? != b"first" { + return Err("atomic metadata read returned unexpected bytes".to_owned()); + } + directory + .atomic_write(path, b"second") + .map_err(|error| error.to_string())?; + if directory.atomic_read(path).map_err(|error| error.to_string())? != b"second" { + return Err("atomic metadata replacement returned unexpected bytes".to_owned()); + } + Ok(()) +} + +fn verify_writer_exclusion(directory: &D) -> Result<(), String> +where + D: Directory + Clone, +{ + let held = directory + .acquire_lock(&INDEX_WRITER_LOCK) + .map_err(|error| error.to_string())?; + let contender = directory.clone(); + let result = std::thread::spawn(move || contender.acquire_lock(&INDEX_WRITER_LOCK).is_err()) + .join() + .map_err(|_| "writer lock contender panicked".to_owned())?; + if !result { + return Err("two writers acquired the index lock".to_owned()); + } + drop(held); + directory + .acquire_lock(&INDEX_WRITER_LOCK) + .map_err(|error| error.to_string())?; + Ok(()) +} + +fn verify_watch(directory: &dyn Directory) -> Result<(), String> { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let _handle = directory + .watch(WatchCallback::new(move || { + let _ = sender.try_send(()); + })) + .map_err(|error| error.to_string())?; + directory + .atomic_write(Path::new("meta.json"), b"watched") + .map_err(|error| error.to_string())?; + receiver + .recv_timeout(Duration::from_secs(2)) + .map_err(|_| "meta.json watch did not fire".to_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + + use tantivy::directory::{Lock, MmapDirectory}; + + #[derive(Clone, Debug)] + struct BrokenDirectory { + inner: MmapDirectory, + ignore_locks: bool, + fail_next_atomic_write: Arc, + } + + impl BrokenDirectory { + fn new(ignore_locks: bool) -> Self { + Self { + inner: MmapDirectory::create_from_tempdir().unwrap(), + ignore_locks, + fail_next_atomic_write: Arc::new(AtomicBool::new(false)), + } + } + + fn fail_next_atomic_write(&self) { + self.fail_next_atomic_write.store(true, Ordering::Release); + } + } + + impl Directory for BrokenDirectory { + fn get_file_handle(&self, path: &Path) -> Result, OpenReadError> { + self.inner.get_file_handle(path) + } + + fn delete(&self, path: &Path) -> Result<(), DeleteError> { + self.inner.delete(path) + } + + fn exists(&self, path: &Path) -> Result { + self.inner.exists(path) + } + + fn open_write(&self, path: &Path) -> Result { + self.inner.open_write(path) + } + + fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { + self.inner.atomic_read(path) + } + + fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { + if !self.fail_next_atomic_write.swap(false, Ordering::AcqRel) { + return self.inner.atomic_write(path, data); + } + if self.inner.exists(path).unwrap_or(false) { + self.inner.delete(path).map_err(io::Error::other)?; + } + let mut writer = self.inner.open_write(path).map_err(io::Error::other)?; + writer.write_all(&data[..data.len() / 2])?; + writer.flush()?; + Err(io::Error::other("injected partial write")) + } + + fn sync_directory(&self) -> io::Result<()> { + self.inner.sync_directory() + } + + fn acquire_lock(&self, lock: &Lock) -> Result { + if self.ignore_locks { + return Ok(DirectoryLock::from(Box::new(()))); + } + self.inner.acquire_lock(lock) + } + + fn watch(&self, callback: WatchCallback) -> tantivy::Result { + self.inner.watch(callback) + } + } + + #[test] + fn mmap_directory_satisfies_the_contract() { + let directory = MmapDirectory::create_from_tempdir().unwrap(); + let metrics = verify_directory(directory).unwrap(); + assert_eq!(metrics.calls, 2); + assert_eq!(metrics.bytes_requested, 15); + } + + #[test] + fn harness_rejects_non_exclusive_locks() { + let directory = BrokenDirectory::new(true); + assert!(verify_writer_exclusion(&directory).is_err()); + } + + #[test] + fn harness_rejects_visible_partial_metadata() { + let directory = BrokenDirectory::new(false); + directory.atomic_write(Path::new("meta.json"), b"stable").unwrap(); + directory.fail_next_atomic_write(); + assert!(verify_failed_atomic_replacement(&directory, || { + directory.atomic_write(Path::new("meta.json"), b"replacement") + }) + .is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..98584b7 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,39 @@ +#![deny(clippy::all)] +#![deny(unsafe_op_in_unsafe_fn)] + +pub mod directory_harness; + +#[cfg(feature = "node-api")] +mod boundary; + +#[cfg(feature = "node-api")] +use napi_derive::napi; + +pub const NATIVE_ABI_VERSION: u32 = 1; +pub const TANTIVY_VERSION: &str = "0.26.1"; + +#[cfg(feature = "node-api")] +#[napi(object)] +pub struct RuntimeInfo { + pub package_version: String, + pub tantivy_version: String, + pub native_abi_version: u32, + pub storage_backends: Vec, +} + +#[cfg(feature = "node-api")] +#[napi(js_name = "runtimeInfo")] +pub fn runtime_info() -> napi::Result { + boundary::run(|| RuntimeInfo { + package_version: env!("CARGO_PKG_VERSION").to_owned(), + tantivy_version: TANTIVY_VERSION.to_owned(), + native_abi_version: NATIVE_ABI_VERSION, + storage_backends: vec!["native".to_owned()], + }) +} + +#[cfg(feature = "test-panic")] +#[napi(js_name = "__testPanic")] +pub fn test_panic() -> napi::Result<()> { + boundary::run(|| panic!("test panic")) +} diff --git a/test/dependencies.test.mjs b/test/dependencies.test.mjs new file mode 100644 index 0000000..c8545af --- /dev/null +++ b/test/dependencies.test.mjs @@ -0,0 +1,21 @@ +import assert from 'node:assert'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; + +const cargoManifest = readFileSync(new URL('../Cargo.toml', import.meta.url), 'utf8'); +const packageManifest = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const dependencyLedger = readFileSync(new URL('../dependencies.md', import.meta.url), 'utf8'); + +test('every direct dependency is documented', () => { + for (const dependency of ['tantivy', 'napi', 'napi-derive', 'napi-build']) { + assert.match(cargoManifest, new RegExp(`^${dependency.replace('-', '\\-')}\\s*=`, 'm')); + assert(dependencyLedger.includes(`\`${dependency}\``), `${dependency} is absent from dependencies.md`); + } + for (const dependency of Object.keys(packageManifest.devDependencies)) { + assert(dependencyLedger.includes(`\`${dependency}\``), `${dependency} is absent from dependencies.md`); + } +}); + +test('the native dependency graph does not declare RocksDB', () => { + assert(!/rocksdb/i.test(cargoManifest)); +}); diff --git a/test/native.test.mjs b/test/native.test.mjs new file mode 100644 index 0000000..5505c13 --- /dev/null +++ b/test/native.test.mjs @@ -0,0 +1,25 @@ +import assert from 'node:assert'; +import test from 'node:test'; + +import { runtimeInfo } from '@harperfast/fulltext/native'; +import { normalizeNativeError } from '../dist/errors.js'; +import { loadAddon, platformTriple } from '../dist/load-addon.js'; + +test('loads the artifact for the executing platform', async () => { + const info = await runtimeInfo(); + assert.deepStrictEqual(info, { + packageVersion: '0.0.0', + tantivyVersion: '0.26.1', + nativeAbiVersion: 1, + storageBackends: ['native'], + }); + assert.match(platformTriple(), /^(darwin|linux|win32)-(arm64|x64)(-(gnu|musl|msvc))?$/); +}); + +test('turns a panic into a coded terminal error', async () => { + assert.throws( + () => loadAddon().__testPanic(), + (error) => normalizeNativeError(error).code === 'E_NATIVE_PANIC', + ); + await assert.rejects(runtimeInfo(), { code: 'E_POISONED' }); +}); diff --git a/test/package.test.mjs b/test/package.test.mjs new file mode 100644 index 0000000..991af56 --- /dev/null +++ b/test/package.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; + +test('the packed package loads without lifecycle scripts', (context) => { + const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'harper-fulltext-')); + context.after(() => rmSync(temporaryDirectory, { force: true, recursive: true })); + execFileSync('npm', ['run', 'build:native'], { stdio: 'pipe' }); + const packOutput = execFileSync( + 'npm', + ['pack', '--json', '--ignore-scripts', '--pack-destination', temporaryDirectory], + { + encoding: 'utf8', + }, + ); + const parsedOutput = JSON.parse(packOutput); + const pack = Array.isArray(parsedOutput) ? parsedOutput[0] : parsedOutput['@harperfast/fulltext']; + const { filename, files } = pack; + const includedPaths = files.map((file) => file.path); + assert(includedPaths.includes('dist/native.js')); + assert(includedPaths.some((file) => /^fulltext\..+\.node$/.test(file))); + assert(!includedPaths.some((file) => file.startsWith('src/') || file === 'ts/addon.d.ts')); + + const projectDirectory = path.join(temporaryDirectory, 'consumer'); + mkdirSync(projectDirectory); + writeFileSync( + path.join(projectDirectory, 'package.json'), + JSON.stringify({ + type: 'module', + private: true, + dependencies: { '@harperfast/fulltext': `file:${path.join(temporaryDirectory, filename)}` }, + }), + ); + execFileSync('npm', ['install', '--package-lock-only', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], { + cwd: projectDirectory, + stdio: 'pipe', + }); + execFileSync('npm', ['ci', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], { + cwd: projectDirectory, + stdio: 'pipe', + }); + const installedManifest = JSON.parse( + readFileSync(path.join(projectDirectory, 'node_modules/@harperfast/fulltext/package.json'), 'utf8'), + ); + assert(!installedManifest.scripts?.install); + const artifact = includedPaths.find((file) => /^fulltext\..+\.node$/.test(file)); + const require = createRequire(import.meta.url); + const installedAddon = require(path.join(projectDirectory, 'node_modules/@harperfast/fulltext', artifact)); + assert(!('__testPanic' in installedAddon)); + const output = execFileSync( + process.execPath, + [ + '--input-type=module', + '--eval', + "import('@harperfast/fulltext/native').then(x => x.runtimeInfo()).then(console.log)", + ], + { cwd: projectDirectory, encoding: 'utf8' }, + ); + assert.match(output, /tantivyVersion: '0\.26\.1'/); +}); diff --git a/ts/addon.d.ts b/ts/addon.d.ts new file mode 100644 index 0000000..d8af278 --- /dev/null +++ b/ts/addon.d.ts @@ -0,0 +1,12 @@ +/* tslint:disable */ +/* eslint-disable */ + +/* auto-generated by NAPI-RS */ + +export interface RuntimeInfo { + packageVersion: string + tantivyVersion: string + nativeAbiVersion: number + storageBackends: Array +} +export declare function runtimeInfo(): RuntimeInfo diff --git a/ts/errors.ts b/ts/errors.ts new file mode 100644 index 0000000..f6cfa38 --- /dev/null +++ b/ts/errors.ts @@ -0,0 +1,27 @@ +export type FulltextErrorCode = 'E_NATIVE_ADDON_NOT_FOUND' | 'E_NATIVE_PANIC' | 'E_POISONED' | 'E_NATIVE_FAILURE'; + +export class FulltextError extends Error { + readonly code: FulltextErrorCode; + + constructor(code: FulltextErrorCode, message: string, cause?: unknown) { + super(message, { cause }); + this.name = 'FulltextError'; + this.code = code; + } +} + +export function normalizeNativeError(error: unknown): FulltextError { + if (error instanceof FulltextError) { + return error; + } + const message = error instanceof Error ? error.message : String(error); + const match = /^\[(E_[A-Z_]+)]\s*(.*)$/.exec(message); + if (match && isErrorCode(match[1])) { + return new FulltextError(match[1], match[2] || match[1], error); + } + return new FulltextError('E_NATIVE_FAILURE', message, error); +} + +function isErrorCode(value: string): value is FulltextErrorCode { + return value === 'E_NATIVE_ADDON_NOT_FOUND' || value === 'E_NATIVE_PANIC' || value === 'E_POISONED'; +} diff --git a/ts/load-addon.ts b/ts/load-addon.ts new file mode 100644 index 0000000..94fc2eb --- /dev/null +++ b/ts/load-addon.ts @@ -0,0 +1,57 @@ +import { existsSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +import type * as NativeAddon from './addon.js'; +import { FulltextError } from './errors.js'; + +const require = createRequire(import.meta.url); +let loadedAddon: typeof NativeAddon | undefined; + +export function loadAddon(): typeof NativeAddon { + if (loadedAddon) { + return loadedAddon; + } + const triple = platformTriple(); + const artifact = `fulltext.${triple}.node`; + const localPath = fileURLToPath(new URL(`../${artifact}`, import.meta.url)); + if (existsSync(localPath)) { + loadedAddon = require(localPath) as typeof NativeAddon; + return loadedAddon; + } + + const platformPackage = `@harperfast/fulltext-${triple}`; + let packagePath: string; + try { + packagePath = require.resolve(platformPackage); + } catch (error) { + throw new FulltextError( + 'E_NATIVE_ADDON_NOT_FOUND', + `No fulltext native artifact is installed for ${triple}`, + error, + ); + } + loadedAddon = require(packagePath) as typeof NativeAddon; + return loadedAddon; +} + +export function platformTriple(): string { + if (process.platform === 'linux') { + return `linux-${process.arch}-${usesGlibc() ? 'gnu' : 'musl'}`; + } + if (process.platform === 'darwin') { + return `darwin-${process.arch}`; + } + if (process.platform === 'win32') { + return `win32-${process.arch}-msvc`; + } + throw new FulltextError( + 'E_NATIVE_ADDON_NOT_FOUND', + `Fulltext does not provide a native artifact for ${process.platform}-${process.arch}`, + ); +} + +function usesGlibc(): boolean { + const report = process.report?.getReport() as { header?: { glibcVersionRuntime?: string } } | undefined; + return Boolean(report?.header?.glibcVersionRuntime); +} diff --git a/ts/native.ts b/ts/native.ts new file mode 100644 index 0000000..e167ce0 --- /dev/null +++ b/ts/native.ts @@ -0,0 +1,33 @@ +import type { RuntimeInfo as NativeRuntimeInfo } from './addon.js'; +import { normalizeNativeError } from './errors.js'; +import { loadAddon } from './load-addon.js'; + +export { FulltextError } from './errors.js'; +export type { FulltextErrorCode } from './errors.js'; + +export interface RuntimeInfo { + packageVersion: string; + tantivyVersion: string; + nativeAbiVersion: number; + storageBackends: ReadonlyArray<'native'>; +} + +export async function runtimeInfo(): Promise { + try { + return toRuntimeInfo(loadAddon().runtimeInfo()); + } catch (error) { + throw normalizeNativeError(error); + } +} + +function toRuntimeInfo(info: NativeRuntimeInfo): RuntimeInfo { + if (info.storageBackends.length !== 1 || info.storageBackends[0] !== 'native') { + throw new Error(`Unexpected storage capabilities: ${info.storageBackends.join(', ')}`); + } + return { + packageVersion: info.packageVersion, + tantivyVersion: info.tantivyVersion, + nativeAbiVersion: info.nativeAbiVersion, + storageBackends: ['native'], + }; +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..02fe5ef --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "ts", + "strict": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": ["ts/**/*.ts"], + "exclude": ["node_modules", "dist"] +} From e014f41fd9526219fbb78822379e4d3712f029aa Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 4 Sep 2026 22:57:58 -0600 Subject: [PATCH 2/9] fix: address scaffold review findings --- .github/workflows/ci.yml | 8 +++-- README.md | 3 ++ deny.toml | 9 +++++- dependencies.md | 4 +++ docs/scaffold-design.md | 64 ++++++++++++++++++++------------------ package.json | 9 ++---- src/boundary.rs | 54 +++++++++++++++++++++----------- src/directory_harness.rs | 55 +++++++++++++++++++++----------- src/lib.rs | 32 +++++++++++++++---- test/dependencies.test.mjs | 31 ++++++++++++++++-- test/native.test.mjs | 11 +++++-- test/package.test.mjs | 31 +++++++++++------- ts/errors.ts | 27 ++++++++++++---- ts/load-addon.ts | 58 +++++++++++++++++++++++----------- ts/native.ts | 21 +++++-------- 15 files changed, 282 insertions(+), 135 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46350ad..e70a09f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,11 @@ jobs: - run: npm run build:native - name: Reject linked RocksDB symbols run: | - if cargo tree --locked | grep -i rocksdb; then exit 1; fi - if nm -D fulltext.linux-x64-gnu.node | grep -i rocksdb; then exit 1; fi + set -euo pipefail + artifact=fulltext.linux-x64-gnu.node + test -f "$artifact" + if cargo tree --locked --all-features | grep -qi rocksdb; then exit 1; fi + if nm --defined-only "$artifact" | grep -qi rocksdb; then exit 1; fi + if ldd "$artifact" | grep -qi rocksdb; then exit 1; fi - name: Verify unwind profile run: cargo rustc --locked --release --features node-api -- --print cfg | grep 'panic="unwind"' diff --git a/README.md b/README.md index 9b9d24b..8b01351 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ are not yet available. The package never compiles or downloads native code during installation. A supported prebuilt artifact must be present for the executing platform. +The initial CI-qualified targets are Linux x64 glibc, macOS arm64, and Windows x64. Additional +targets are added only after their artifacts are loaded and tested on the target runtime. + ## Native usage ```js diff --git a/deny.toml b/deny.toml index b5b3d35..50da30f 100644 --- a/deny.toml +++ b/deny.toml @@ -1,3 +1,6 @@ +[graph] +all-features = true + [advisories] yanked = "deny" @@ -6,6 +9,7 @@ allow = [ "Apache-2.0", "Apache-2.0 WITH LLVM-exception", "BSD-3-Clause", + "ISC", "MIT", "Unicode-3.0", "Zlib", @@ -14,7 +18,10 @@ allow = [ [bans] multiple-versions = "deny" wildcards = "deny" -skip = [{ name = "windows-sys", version = "0.59.0" }] +skip = [ + { name = "syn", version = "2.0.119" }, + { name = "windows-sys", version = "0.59.0" }, +] [sources] unknown-registry = "deny" diff --git a/dependencies.md b/dependencies.md index ffe67e5..c76f340 100644 --- a/dependencies.md +++ b/dependencies.md @@ -15,6 +15,10 @@ reviewed deliberately. The Rust dependency graph must not include RocksDB. The future Rocks backend calls a C-ABI capability table owned by rocksdb-js rather than linking a second RocksDB runtime. +napi-rs 2.16 generates an outer unwind boundary only for exports marked `catch_unwind`. Every +fulltext function and method uses that option to cover argument and result conversion; the inner +boundary adds stable error codes and per-handle poison state for package-owned operations. + ## JavaScript development graph | Dependency | Scope | Purpose | diff --git a/docs/scaffold-design.md b/docs/scaffold-design.md index 59aca77..0e7f531 100644 --- a/docs/scaffold-design.md +++ b/docs/scaffold-design.md @@ -130,11 +130,12 @@ thread per call. The release and benchmark profiles use thin LTO, one codegen unit, and `panic = "unwind"`. The addon does not install a global allocator because doing so could change allocation behavior for -the host Node process. Every package-owned N-API boundary invokes a shared -`catch_unwind` adapter that converts a panic into a JavaScript error with a stable `code`; a panic -reaching that boundary may not unwind through FFI. A test-only Cargo feature exposes a panic probe -to the Node smoke suite, but the probe and feature are absent from published artifacts. All public -errors carry stable codes from the first release. +the host Node process. Every package-owned N-API boundary invokes a shared `catch_unwind` adapter +that converts a panic into a JavaScript error with a stable native `code`; every exported function +and method also enables napi-rs 2.16's generated `catch_unwind` wrapper so result conversion remains +inside an unwind boundary. A test-only Cargo feature exposes a handle-level panic probe to the Node +smoke suite, but the probe and feature are absent from published artifacts. All public errors carry +stable codes from the first release. Before implementing the adapter, the pinned napi-rs behavior is verified so the wrapper neither double-wraps panics nor replaces useful native error mapping. Every package-owned worker entry @@ -144,10 +145,11 @@ during unwinding, and a panic wholly inside an upstream-owned thread can still a those paths are explicitly out of scope for `catch_unwind` and remain targets for input bounds, upstream qualification, and process supervision rather than false error-containment claims. -Any caught panic poisons its owning object. That object and all work derived from it fail fast with -`E_POISONED`; callers cannot retry through potentially corrupted Tantivy or adapter state. The -panic smoke route verifies both the original coded failure and terminal poison behavior. CI also -asserts that the effective release profile retains `panic = "unwind"`. +Any caught panic on a stateful handle poisons that handle. It and all work derived from it fail fast +with `E_POISONED`; callers cannot retry through potentially corrupted Tantivy or adapter state. +Stateless capability inspection remains available, and an unrelated handle remains healthy. The +panic smoke route verifies the original coded failure, terminal handle poisoning, and isolation +from another handle. CI also asserts that the effective release profile retains `panic = "unwind"`. Tantivy is pinned exactly to 0.26.1. napi-rs is a build and binding dependency, not part of the public API. The Cargo feature layout must allow Rust unit tests to exercise engine code without @@ -165,18 +167,20 @@ Harper CI. Scripts cover: - formatting checks; - napi-rs artifact assembly. -The initial platform matrix follows the qualified intersection of HNSW and Symphony: +The initial scaffold qualifies only the targets exercised end to end in CI: -- Linux x64 and arm64, glibc and musl; -- macOS arm64 and x64 build artifacts; +- Linux x64 glibc; +- macOS arm64; - Windows x64. One addon artifact is built per platform rather than one per backend. Every matrix artifact is loaded and smoke-tested on its target architecture; producing a file is not sufficient. A missing or mismatched artifact fails with the resolved platform triple and never -falls back to an install-time source build or download. Platform packages and publishing automation -are structured now but completed under the dedicated release issue. No install script compiles or -downloads code silently in this scaffold. +falls back to an install-time source build or download. Linux arm64, Linux musl, and macOS x64 are +added to the manifest only with target-native loading coverage under the dedicated release issue. +Platform packages and publishing automation are completed there as well. No install script compiles +or downloads code silently in this scaffold; `prepack` creates a release artifact so test-only +exports cannot enter a package assembled from developer state. ## Testing and end-to-end route @@ -188,15 +192,13 @@ Every introduced source module has a direct test. The scaffold gates: 4. native addon build; 5. a Node smoke test that imports the public native entry point, awaits typed capability reporting, and proves a test-only native panic becomes a stable coded JavaScript error; -6. a backend-parameterized Rust `Directory` conformance harness, initially run against Tantivy - `MmapDirectory`, covering atomic metadata writes, exclusive writer/meta locks, open-file - deletion, boundary range reads, `meta.json` watch notification, and `sync_directory`; the - harness includes two-writer lock races, watch/delete/read concurrency, and counters for read - calls and requested bytes; the Rocks adapter adds copied-byte accounting so later implementations - expose read amplification; -7. negative-control Directory implementations with non-atomic metadata and always-successful locks - that the harness must reject, plus a realistic KV-shaped control using process-local locks, - polling watches, and non-atomic chunk publication; +6. a backend-parameterized Rust `Directory` baseline harness, initially run against Tantivy + `MmapDirectory`, covering successful atomic metadata replacement, in-process writer exclusion, + platform-correct open-file deletion, boundary range reads, `meta.json` watch notification, and + `sync_directory`; logical read-call and requested-byte counters establish the Tantivy access + pattern, while the Rocks adapter adds physical fetched/copied-byte accounting; +7. negative controls proving the harness rejects always-successful locks and visible partial + metadata after a fault-injected replacement; 8. a smoke test installed from `npm pack` output rather than the repository tree, proving the exports map, packaged files, addon resolution, stable errors, and platform artifact together; 9. package-content and loaded-addon inspection proving private generated bindings, unintended @@ -204,9 +206,11 @@ Every introduced source module has a direct test. The scaffold gates: 10. `cargo deny` gates for advisories, licenses, sources, bans, and duplicate native libraries, plus a policy-scoped npm audit. -All Cargo operations use `--locked`; npm CI operations use `npm ci`. The scaffold gates loading -on the executing architecture, not merely cross-compilation. It does not claim to exercise -indexing, search, Rocks durability, concurrency, or catalog performance. +All Cargo operations use `--locked`; npm CI operations use `npm ci`. The scaffold gates loading on +the executing architecture, not merely cross-compilation. Cross-process Rocks writer exclusion, +crash durability, physical read amplification, chunk publication, and bounded watch latency require +the concrete Rocks lease and remain mandatory Phase 0 tests. The scaffold does not claim to +exercise indexing, search, Rocks durability, cross-process concurrency, or catalog performance. The end-to-end route for this issue is the Node smoke test against the built addon through the published `./native` entry point. Rocks behavior is intentionally not observable end to end until @@ -227,9 +231,9 @@ Every dependency added by the scaffold is recorded in `dependencies.md` with its whether it appears in the runtime, build, or development graph. CI checks the documented direct dependency names against Cargo and npm manifests so the ledger cannot silently drift. -Platform package names are reserved before the first npm release. They are published with -provenance and referenced at the exact root-package version, preventing an unrelated package or -version from satisfying artifact resolution. +Before the first npm release, the release work reserves the qualified platform package names, +publishes them with provenance, and references each at the exact root-package version. The scaffold +does not resolve undeclared platform packages. ## Approaches considered diff --git a/package.json b/package.json index edcde0a..7808052 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,6 @@ "defaults": false, "additional": [ "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "x86_64-apple-darwin", "aarch64-apple-darwin", "x86_64-pc-windows-msvc" ] @@ -41,9 +37,10 @@ "format": "prettier --write . && cargo fmt", "format:check": "prettier --check . && cargo fmt --check", "lint": "tsc -p tsconfig.json --noEmit && cargo clippy --locked --all-targets --all-features -- -D warnings", - "test": "cargo test --locked --all-features && npm run build:test-native && npm run build:typescript && node --test --test-concurrency=1 test/*.test.mjs", + "prepack": "npm run build:native && npm run build:typescript", + "test": "cargo test --locked --all-features && npm run test:node", "test:rust": "cargo test --locked --all-features", - "test:node": "node --test --test-concurrency=1 test/*.test.mjs" + "test:node": "npm run build:test-native && npm run build:typescript && node --test --test-concurrency=1 test/*.test.mjs" }, "engines": { "node": "^22.18.0 || >=24.0.0" diff --git a/src/boundary.rs b/src/boundary.rs index 70d6055..ecdae7b 100644 --- a/src/boundary.rs +++ b/src/boundary.rs @@ -1,25 +1,39 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::sync::atomic::{AtomicBool, Ordering}; -use napi::{Error, Result, Status}; +use napi::Error; -static POISONED: AtomicBool = AtomicBool::new(false); +pub type Result = napi::Result; -pub fn run(operation: impl FnOnce() -> T) -> Result { - if POISONED.load(Ordering::Acquire) { - return Err(coded_error("E_POISONED", "the native addon is in a terminal state")); +#[derive(Default)] +pub struct PoisonState { + poisoned: AtomicBool, +} + +impl PoisonState { + pub fn run(&self, operation: impl FnOnce() -> T) -> Result { + if self.poisoned.load(Ordering::Acquire) { + return Err(coded_error("E_POISONED", "the native handle is in a terminal state")); + } + match catch_unwind(AssertUnwindSafe(operation)) { + Ok(value) => Ok(value), + Err(_) => { + self.poisoned.store(true, Ordering::Release); + Err(coded_error("E_NATIVE_PANIC", "native operation panicked")) + } + } } +} + +pub fn run_stateless(operation: impl FnOnce() -> T) -> Result { match catch_unwind(AssertUnwindSafe(operation)) { Ok(value) => Ok(value), - Err(_) => { - POISONED.store(true, Ordering::Release); - Err(coded_error("E_NATIVE_PANIC", "native operation panicked")) - } + Err(_) => Err(coded_error("E_NATIVE_PANIC", "native operation panicked")), } } -fn coded_error(code: &str, message: impl AsRef) -> Error { - Error::new(Status::GenericFailure, format!("[{code}] {}", message.as_ref())) +fn coded_error(code: &'static str, message: impl AsRef) -> Error<&'static str> { + Error::new(code, message.as_ref()) } #[cfg(test)] @@ -27,13 +41,15 @@ mod tests { use super::*; #[test] - fn panic_poisoning_is_terminal() { - POISONED.store(false, Ordering::Release); - let panic_error = run(|| panic!("boom")).unwrap_err(); - assert!(panic_error.reason.contains("[E_NATIVE_PANIC]")); - - let poisoned_error = run(|| 1).unwrap_err(); - assert!(poisoned_error.reason.contains("[E_POISONED]")); - POISONED.store(false, Ordering::Release); + fn panic_poisoning_is_scoped_to_the_handle() { + let poisoned = PoisonState::default(); + let healthy = PoisonState::default(); + let panic_error = poisoned.run(|| panic!("boom")).unwrap_err(); + assert_eq!(panic_error.status, "E_NATIVE_PANIC"); + + let poisoned_error = poisoned.run(|| 1).unwrap_err(); + assert_eq!(poisoned_error.status, "E_POISONED"); + assert_eq!(healthy.run(|| 1).unwrap(), 1); + assert_eq!(run_stateless(|| 1).unwrap(), 1); } } diff --git a/src/directory_harness.rs b/src/directory_harness.rs index 0786f49..63ee2e2 100644 --- a/src/directory_harness.rs +++ b/src/directory_harness.rs @@ -13,7 +13,7 @@ use tantivy::directory::{ use tantivy::HasLen; #[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ReadMetrics { +pub struct LogicalReadMetrics { pub calls: u64, pub bytes_requested: u64, } @@ -38,8 +38,8 @@ impl InstrumentedDirectory { } } - pub fn read_metrics(&self) -> ReadMetrics { - ReadMetrics { + pub fn logical_read_metrics(&self) -> LogicalReadMetrics { + LogicalReadMetrics { calls: self.counters.calls.load(Ordering::Relaxed), bytes_requested: self.counters.bytes_requested.load(Ordering::Relaxed), } @@ -101,7 +101,12 @@ where } fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { - self.inner.atomic_read(path) + let bytes = self.inner.atomic_read(path)?; + self.counters.calls.fetch_add(1, Ordering::Relaxed); + self.counters + .bytes_requested + .fetch_add(bytes.len() as u64, Ordering::Relaxed); + Ok(bytes) } fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { @@ -121,17 +126,17 @@ where } } -pub fn verify_directory(directory: D) -> Result +pub fn verify_directory_baseline(directory: D) -> Result where D: Directory + Clone, { let directory = InstrumentedDirectory::new(directory); verify_write_read_delete(&directory)?; verify_atomic_metadata(&directory)?; - verify_writer_exclusion(&directory)?; + verify_in_process_writer_exclusion(&directory)?; verify_watch(&directory)?; directory.sync_directory().map_err(|error| error.to_string())?; - Ok(directory.read_metrics()) + Ok(directory.logical_read_metrics()) } fn verify_write_read_delete(directory: &D) -> Result<(), String> @@ -149,13 +154,27 @@ where return Err("range read returned unexpected bytes".to_owned()); } let deleting_directory = directory.clone(); - std::thread::spawn(move || deleting_directory.delete(Path::new("segment"))) + let delete_result = std::thread::spawn(move || deleting_directory.delete(Path::new("segment"))) .join() - .map_err(|_| "segment deletion panicked".to_owned())? - .map_err(|error| error.to_string())?; - let retained = file.read_bytes().map_err(|error| error.to_string())?; - if retained.as_slice() != b"0123456789" { - return Err("an open file changed after deletion".to_owned()); + .map_err(|_| "segment deletion panicked".to_owned())?; + if cfg!(windows) { + if delete_result.is_ok() { + return Err("Windows deleted a mapped file unexpectedly".to_owned()); + } + let retained = file.read_bytes().map_err(|error| error.to_string())?; + if retained.as_slice() != b"0123456789" { + return Err("an open file changed after a rejected deletion".to_owned()); + } + drop(retained); + drop(middle); + drop(file); + directory.delete(path).map_err(|error| error.to_string())?; + } else { + delete_result.map_err(|error| error.to_string())?; + let retained = file.read_bytes().map_err(|error| error.to_string())?; + if retained.as_slice() != b"0123456789" { + return Err("an open file changed after deletion".to_owned()); + } } Ok(()) } @@ -193,7 +212,7 @@ fn verify_atomic_metadata(directory: &dyn Directory) -> Result<(), String> { Ok(()) } -fn verify_writer_exclusion(directory: &D) -> Result<(), String> +fn verify_in_process_writer_exclusion(directory: &D) -> Result<(), String> where D: Directory + Clone, { @@ -310,15 +329,15 @@ mod tests { #[test] fn mmap_directory_satisfies_the_contract() { let directory = MmapDirectory::create_from_tempdir().unwrap(); - let metrics = verify_directory(directory).unwrap(); - assert_eq!(metrics.calls, 2); - assert_eq!(metrics.bytes_requested, 15); + let metrics = verify_directory_baseline(directory).unwrap(); + assert_eq!(metrics.calls, 4); + assert_eq!(metrics.bytes_requested, 26); } #[test] fn harness_rejects_non_exclusive_locks() { let directory = BrokenDirectory::new(true); - assert!(verify_writer_exclusion(&directory).is_err()); + assert!(verify_in_process_writer_exclusion(&directory).is_err()); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 98584b7..9a24efc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,9 +22,9 @@ pub struct RuntimeInfo { } #[cfg(feature = "node-api")] -#[napi(js_name = "runtimeInfo")] -pub fn runtime_info() -> napi::Result { - boundary::run(|| RuntimeInfo { +#[napi(catch_unwind, js_name = "runtimeInfo")] +pub fn runtime_info() -> boundary::Result { + boundary::run_stateless(|| RuntimeInfo { package_version: env!("CARGO_PKG_VERSION").to_owned(), tantivy_version: TANTIVY_VERSION.to_owned(), native_abi_version: NATIVE_ABI_VERSION, @@ -33,7 +33,27 @@ pub fn runtime_info() -> napi::Result { } #[cfg(feature = "test-panic")] -#[napi(js_name = "__testPanic")] -pub fn test_panic() -> napi::Result<()> { - boundary::run(|| panic!("test panic")) +#[napi] +#[derive(Default)] +pub struct TestHandle { + poison: boundary::PoisonState, +} + +#[cfg(feature = "test-panic")] +#[napi] +impl TestHandle { + #[napi(constructor)] + pub fn new() -> Self { + Self::default() + } + + #[napi(catch_unwind)] + pub fn panic(&self) -> boundary::Result<()> { + self.poison.run(|| panic!("test panic")) + } + + #[napi(catch_unwind)] + pub fn check(&self) -> boundary::Result { + self.poison.run(|| true) + } } diff --git a/test/dependencies.test.mjs b/test/dependencies.test.mjs index c8545af..ae59aee 100644 --- a/test/dependencies.test.mjs +++ b/test/dependencies.test.mjs @@ -3,12 +3,12 @@ import { readFileSync } from 'node:fs'; import test from 'node:test'; const cargoManifest = readFileSync(new URL('../Cargo.toml', import.meta.url), 'utf8'); +const cargoLock = readFileSync(new URL('../Cargo.lock', import.meta.url), 'utf8'); const packageManifest = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); const dependencyLedger = readFileSync(new URL('../dependencies.md', import.meta.url), 'utf8'); test('every direct dependency is documented', () => { - for (const dependency of ['tantivy', 'napi', 'napi-derive', 'napi-build']) { - assert.match(cargoManifest, new RegExp(`^${dependency.replace('-', '\\-')}\\s*=`, 'm')); + for (const dependency of cargoDependencies(cargoManifest)) { assert(dependencyLedger.includes(`\`${dependency}\``), `${dependency} is absent from dependencies.md`); } for (const dependency of Object.keys(packageManifest.devDependencies)) { @@ -18,4 +18,31 @@ test('every direct dependency is documented', () => { test('the native dependency graph does not declare RocksDB', () => { assert(!/rocksdb/i.test(cargoManifest)); + assert(!/^name\s*=\s*"[^"]*rocksdb[^"]*"/im.test(cargoLock)); }); + +function cargoDependencies(manifest) { + const dependencySections = new Set(['dependencies', 'build-dependencies', 'dev-dependencies']); + let currentSection; + const dependencies = new Set(); + for (const line of manifest.split('\n')) { + const section = /^\[([^\]]+)]$/.exec(line.trim()); + if (section) { + currentSection = section[1]; + continue; + } + const isDependencySection = + dependencySections.has(currentSection) || + currentSection?.endsWith('.dependencies') || + currentSection?.endsWith('.build-dependencies') || + currentSection?.endsWith('.dev-dependencies'); + if (!isDependencySection) { + continue; + } + const dependency = /^([A-Za-z0-9_-]+)\s*=/.exec(line); + if (dependency) { + dependencies.add(dependency[1]); + } + } + return dependencies; +} diff --git a/test/native.test.mjs b/test/native.test.mjs index 5505c13..2e81473 100644 --- a/test/native.test.mjs +++ b/test/native.test.mjs @@ -17,9 +17,16 @@ test('loads the artifact for the executing platform', async () => { }); test('turns a panic into a coded terminal error', async () => { + const firstHandle = new (loadAddon().TestHandle)(); + const secondHandle = new (loadAddon().TestHandle)(); assert.throws( - () => loadAddon().__testPanic(), + () => firstHandle.panic(), (error) => normalizeNativeError(error).code === 'E_NATIVE_PANIC', ); - await assert.rejects(runtimeInfo(), { code: 'E_POISONED' }); + assert.throws( + () => firstHandle.check(), + (error) => normalizeNativeError(error).code === 'E_POISONED', + ); + assert.strictEqual(secondHandle.check(), true); + await assert.doesNotReject(runtimeInfo()); }); diff --git a/test/package.test.mjs b/test/package.test.mjs index 991af56..c010b3d 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -6,17 +6,26 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; +const npmInvocation = process.env.npm_execpath + ? { executable: process.execPath, prefix: [process.env.npm_execpath] } + : { executable: process.platform === 'win32' ? process.env.ComSpec || 'cmd.exe' : 'npm', prefix: [] }; + +function runNpm(arguments_, options) { + const argumentsWithPrefix = [...npmInvocation.prefix]; + if (process.platform === 'win32' && npmInvocation.prefix.length === 0) { + argumentsWithPrefix.push('/d', '/s', '/c', 'npm.cmd'); + } + argumentsWithPrefix.push(...arguments_); + return execFileSync(npmInvocation.executable, argumentsWithPrefix, options); +} + test('the packed package loads without lifecycle scripts', (context) => { const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'harper-fulltext-')); context.after(() => rmSync(temporaryDirectory, { force: true, recursive: true })); - execFileSync('npm', ['run', 'build:native'], { stdio: 'pipe' }); - const packOutput = execFileSync( - 'npm', - ['pack', '--json', '--ignore-scripts', '--pack-destination', temporaryDirectory], - { - encoding: 'utf8', - }, - ); + runNpm(['run', 'build:native'], { stdio: 'pipe' }); + const packOutput = runNpm(['pack', '--json', '--ignore-scripts', '--pack-destination', temporaryDirectory], { + encoding: 'utf8', + }); const parsedOutput = JSON.parse(packOutput); const pack = Array.isArray(parsedOutput) ? parsedOutput[0] : parsedOutput['@harperfast/fulltext']; const { filename, files } = pack; @@ -35,11 +44,11 @@ test('the packed package loads without lifecycle scripts', (context) => { dependencies: { '@harperfast/fulltext': `file:${path.join(temporaryDirectory, filename)}` }, }), ); - execFileSync('npm', ['install', '--package-lock-only', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], { + runNpm(['install', '--package-lock-only', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: projectDirectory, stdio: 'pipe', }); - execFileSync('npm', ['ci', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], { + runNpm(['ci', '--offline', '--ignore-scripts', '--no-audit', '--no-fund'], { cwd: projectDirectory, stdio: 'pipe', }); @@ -50,7 +59,7 @@ test('the packed package loads without lifecycle scripts', (context) => { const artifact = includedPaths.find((file) => /^fulltext\..+\.node$/.test(file)); const require = createRequire(import.meta.url); const installedAddon = require(path.join(projectDirectory, 'node_modules/@harperfast/fulltext', artifact)); - assert(!('__testPanic' in installedAddon)); + assert(!('TestHandle' in installedAddon)); const output = execFileSync( process.execPath, [ diff --git a/ts/errors.ts b/ts/errors.ts index f6cfa38..25f4223 100644 --- a/ts/errors.ts +++ b/ts/errors.ts @@ -1,4 +1,13 @@ -export type FulltextErrorCode = 'E_NATIVE_ADDON_NOT_FOUND' | 'E_NATIVE_PANIC' | 'E_POISONED' | 'E_NATIVE_FAILURE'; +const errorCodes = [ + 'E_NATIVE_ADDON_NOT_FOUND', + 'E_NATIVE_ABI_MISMATCH', + 'E_NATIVE_CAPABILITY_MISMATCH', + 'E_NATIVE_PANIC', + 'E_POISONED', + 'E_NATIVE_FAILURE', +] as const; + +export type FulltextErrorCode = (typeof errorCodes)[number]; export class FulltextError extends Error { readonly code: FulltextErrorCode; @@ -15,13 +24,19 @@ export function normalizeNativeError(error: unknown): FulltextError { return error; } const message = error instanceof Error ? error.message : String(error); - const match = /^\[(E_[A-Z_]+)]\s*(.*)$/.exec(message); - if (match && isErrorCode(match[1])) { - return new FulltextError(match[1], match[2] || match[1], error); + const code = readErrorCode(error); + if (code) { + return new FulltextError(code, message || code, error); } return new FulltextError('E_NATIVE_FAILURE', message, error); } -function isErrorCode(value: string): value is FulltextErrorCode { - return value === 'E_NATIVE_ADDON_NOT_FOUND' || value === 'E_NATIVE_PANIC' || value === 'E_POISONED'; +function readErrorCode(error: unknown): FulltextErrorCode | undefined { + if (typeof error !== 'object' || error === null || !('code' in error)) { + return undefined; + } + const code = error.code; + return typeof code === 'string' && errorCodes.includes(code as FulltextErrorCode) + ? (code as FulltextErrorCode) + : undefined; } diff --git a/ts/load-addon.ts b/ts/load-addon.ts index 94fc2eb..5940a02 100644 --- a/ts/load-addon.ts +++ b/ts/load-addon.ts @@ -2,13 +2,30 @@ import { existsSync } from 'node:fs'; import { createRequire } from 'node:module'; import { fileURLToPath } from 'node:url'; -import type * as NativeAddon from './addon.js'; import { FulltextError } from './errors.js'; +interface NativeRuntimeInfo { + packageVersion: string; + tantivyVersion: string; + nativeAbiVersion: number; + storageBackends: Array; +} + +interface NativeTestHandle { + panic(): void; + check(): boolean; +} + +interface NativeAddonApi { + runtimeInfo(): NativeRuntimeInfo; + TestHandle?: new () => NativeTestHandle; +} + const require = createRequire(import.meta.url); -let loadedAddon: typeof NativeAddon | undefined; +const expectedNativeAbiVersion = 1; +let loadedAddon: NativeAddonApi | undefined; -export function loadAddon(): typeof NativeAddon { +export function loadAddon(): NativeAddonApi { if (loadedAddon) { return loadedAddon; } @@ -16,23 +33,12 @@ export function loadAddon(): typeof NativeAddon { const artifact = `fulltext.${triple}.node`; const localPath = fileURLToPath(new URL(`../${artifact}`, import.meta.url)); if (existsSync(localPath)) { - loadedAddon = require(localPath) as typeof NativeAddon; + const addon = require(localPath) as NativeAddonApi; + validateAddon(addon, localPath); + loadedAddon = addon; return loadedAddon; } - - const platformPackage = `@harperfast/fulltext-${triple}`; - let packagePath: string; - try { - packagePath = require.resolve(platformPackage); - } catch (error) { - throw new FulltextError( - 'E_NATIVE_ADDON_NOT_FOUND', - `No fulltext native artifact is installed for ${triple}`, - error, - ); - } - loadedAddon = require(packagePath) as typeof NativeAddon; - return loadedAddon; + throw new FulltextError('E_NATIVE_ADDON_NOT_FOUND', `No fulltext native artifact is installed for ${triple}`); } export function platformTriple(): string { @@ -55,3 +61,19 @@ function usesGlibc(): boolean { const report = process.report?.getReport() as { header?: { glibcVersionRuntime?: string } } | undefined; return Boolean(report?.header?.glibcVersionRuntime); } + +function validateAddon(addon: NativeAddonApi, artifactPath: string): void { + const info = addon.runtimeInfo(); + if (info.nativeAbiVersion !== expectedNativeAbiVersion) { + throw new FulltextError( + 'E_NATIVE_ABI_MISMATCH', + `Fulltext native ABI ${info.nativeAbiVersion} from ${artifactPath} does not match ${expectedNativeAbiVersion}`, + ); + } + if (info.storageBackends.length !== 1 || info.storageBackends[0] !== 'native') { + throw new FulltextError( + 'E_NATIVE_CAPABILITY_MISMATCH', + `Unexpected storage capabilities from ${artifactPath}: ${info.storageBackends.join(', ')}`, + ); + } +} diff --git a/ts/native.ts b/ts/native.ts index e167ce0..f1e8664 100644 --- a/ts/native.ts +++ b/ts/native.ts @@ -1,4 +1,3 @@ -import type { RuntimeInfo as NativeRuntimeInfo } from './addon.js'; import { normalizeNativeError } from './errors.js'; import { loadAddon } from './load-addon.js'; @@ -14,20 +13,14 @@ export interface RuntimeInfo { export async function runtimeInfo(): Promise { try { - return toRuntimeInfo(loadAddon().runtimeInfo()); + const info = loadAddon().runtimeInfo(); + return { + packageVersion: info.packageVersion, + tantivyVersion: info.tantivyVersion, + nativeAbiVersion: info.nativeAbiVersion, + storageBackends: ['native'], + }; } catch (error) { throw normalizeNativeError(error); } } - -function toRuntimeInfo(info: NativeRuntimeInfo): RuntimeInfo { - if (info.storageBackends.length !== 1 || info.storageBackends[0] !== 'native') { - throw new Error(`Unexpected storage capabilities: ${info.storageBackends.join(', ')}`); - } - return { - packageVersion: info.packageVersion, - tantivyVersion: info.tantivyVersion, - nativeAbiVersion: info.nativeAbiVersion, - storageBackends: ['native'], - }; -} From 926a8d2c4e3d4d4107a6ea31e53470927f431a09 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 4 Sep 2026 23:33:25 -0600 Subject: [PATCH 3/9] fix: harden native scaffold contracts --- .github/workflows/ci.yml | 10 +- .prettierignore | 1 + CONTRIBUTING.md | 4 +- README.md | 3 +- dependencies.md | 5 +- docs/scaffold-design.md | 46 +++--- package.json | 5 +- scripts/verify-package-artifacts.mjs | 14 ++ src/boundary.rs | 34 ++++- src/directory_harness.rs | 217 ++++++++++++++++++++++++--- src/lib.rs | 2 +- test/dependencies.test.mjs | 6 +- test/napi-boundaries.test.mjs | 18 +++ test/package-artifacts.test.mjs | 14 ++ test/package.test.mjs | 9 +- 15 files changed, 333 insertions(+), 55 deletions(-) create mode 100644 scripts/verify-package-artifacts.mjs create mode 100644 test/napi-boundaries.test.mjs create mode 100644 test/package-artifacts.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e70a09f..e6a6d48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - run: npm run format:check - run: npm run lint - run: npm test + - run: git diff --exit-code -- ts/addon.d.ts supply-chain: name: Supply chain and native linkage @@ -62,7 +63,12 @@ jobs: artifact=fulltext.linux-x64-gnu.node test -f "$artifact" if cargo tree --locked --all-features | grep -qi rocksdb; then exit 1; fi - if nm --defined-only "$artifact" | grep -qi rocksdb; then exit 1; fi - if ldd "$artifact" | grep -qi rocksdb; then exit 1; fi + nm -D --defined-only "$artifact" > "$RUNNER_TEMP/fulltext-symbols.txt" + test -s "$RUNNER_TEMP/fulltext-symbols.txt" + grep -q napi_register_module_v1 "$RUNNER_TEMP/fulltext-symbols.txt" + if grep -qi rocksdb "$RUNNER_TEMP/fulltext-symbols.txt"; then exit 1; fi + ldd "$artifact" > "$RUNNER_TEMP/fulltext-linkage.txt" + test -s "$RUNNER_TEMP/fulltext-linkage.txt" + if grep -qi rocksdb "$RUNNER_TEMP/fulltext-linkage.txt"; then exit 1; fi - name: Verify unwind profile run: cargo rustc --locked --release --features node-api -- --print cfg | grep 'panic="unwind"' diff --git a/.prettierignore b/.prettierignore index c2e398e..996144f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ Cargo.lock +package-lock.json dist/ target/ ts/addon.d.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5aafbfb..fd22a20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,9 @@ of `npm pack` into a temporary consumer project. - Do not link RocksDB into this addon. The future Rocks backend must use the versioned lease owned by rocksdb-js. - Do not expose generated Node-API declarations as the public TypeScript API. -- Keep CPU and I/O work off the Node.js event loop. +- Keep CPU and sustained I/O work off the Node.js event loop. The promise-shaped capability call may + synchronously load the addon once; no search, indexing, commit, or storage operation gets that + exception. - Add a direct test for each source module. Open an issue before changing a public package entry point, native ABI, persistence contract, or diff --git a/README.md b/README.md index 8b01351..7b2f2c3 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,8 @@ console.log(info.tantivyVersion); ``` `runtimeInfo()` is asynchronous so later search and indexing operations can remain off the Node.js -event loop without changing the public calling convention. +event loop without changing the public calling convention. Its first call may synchronously load +the native artifact; search, indexing, commit, and storage work will use the package executor. ## Storage boundaries diff --git a/dependencies.md b/dependencies.md index c76f340..ed2768d 100644 --- a/dependencies.md +++ b/dependencies.md @@ -16,8 +16,9 @@ The Rust dependency graph must not include RocksDB. The future Rocks backend cal capability table owned by rocksdb-js rather than linking a second RocksDB runtime. napi-rs 2.16 generates an outer unwind boundary only for exports marked `catch_unwind`. Every -fulltext function and method uses that option to cover argument and result conversion; the inner -boundary adds stable error codes and per-handle poison state for package-owned operations. +fulltext function, method, and constructor uses that option to contain argument and result +conversion panics. That outer boundary uses napi-rs error mapping; the inner boundary adds stable +error codes and per-handle poison state for package-owned operations. ## JavaScript development graph diff --git a/docs/scaffold-design.md b/docs/scaffold-design.md index 0e7f531..0ad7796 100644 --- a/docs/scaffold-design.md +++ b/docs/scaffold-design.md @@ -119,9 +119,9 @@ The crate builds as: The initial N-API surface is deliberately small: typed package/version/ABI capability reporting proves loading and generated declarations without JSON serialization. The public façade exposes it -as a Promise so no future work API inherits a synchronous public signature. Constant-time native -introspection may be implemented synchronously underneath that façade; blocking, `block_on`, and -CPU- or I/O-bearing synchronous exports are forbidden. Issue #17 introduces the bounded +as a Promise so no future work API inherits a synchronous public signature. The first call may +synchronously load the native addon before performing constant-time introspection; sustained +blocking, `block_on`, and CPU- or I/O-bearing synchronous exports are forbidden. Issue #17 introduces the bounded package-owned executor before any search, indexing, commit, or storage operation is exposed. Packed operations use borrowed request buffers and transfer owned `Vec` responses rather than JSON strings. Batch operations are the default shape; the façade never invokes native code once per @@ -132,8 +132,11 @@ The release and benchmark profiles use thin LTO, one codegen unit, and `panic = The addon does not install a global allocator because doing so could change allocation behavior for the host Node process. Every package-owned N-API boundary invokes a shared `catch_unwind` adapter that converts a panic into a JavaScript error with a stable native `code`; every exported function -and method also enables napi-rs 2.16's generated `catch_unwind` wrapper so result conversion remains -inside an unwind boundary. A test-only Cargo feature exposes a handle-level panic probe to the Node +and method, including constructors, also enables napi-rs 2.16's generated `catch_unwind` wrapper so +argument and result conversion remain inside an unwind boundary. A source-level CI gate rejects new +Node-API functions without that wrapper. The outer wrapper is a process-safety net: conversion-time +panics use napi-rs error mapping and do not participate in handle poisoning. A test-only Cargo +feature exposes a handle-level panic probe to the Node smoke suite, but the probe and feature are absent from published artifacts. All public errors carry stable codes from the first release. @@ -145,11 +148,15 @@ during unwinding, and a panic wholly inside an upstream-owned thread can still a those paths are explicitly out of scope for `catch_unwind` and remain targets for input bounds, upstream qualification, and process supervision rather than false error-containment claims. -Any caught panic on a stateful handle poisons that handle. It and all work derived from it fail fast +Any panic caught by a stateful handle's inner boundary poisons that handle. It and all work derived from it fail fast with `E_POISONED`; callers cannot retry through potentially corrupted Tantivy or adapter state. Stateless capability inspection remains available, and an unrelated handle remains healthy. The panic smoke route verifies the original coded failure, terminal handle poisoning, and isolation -from another handle. CI also asserts that the effective release profile retains `panic = "unwind"`. +from another handle. An operation completing after a concurrent panic also returns `E_POISONED`. +Caught panics still invoke Rust's process-wide panic hook before returning the coded error; Harper +logging must classify the subsequent error as contained rather than treating the hook output alone +as evidence of process failure. CI also asserts that the effective release profile retains +`panic = "unwind"`. Tantivy is pinned exactly to 0.26.1. napi-rs is a build and binding dependency, not part of the public API. The Cargo feature layout must allow Rust unit tests to exercise engine code without @@ -178,9 +185,10 @@ loaded and smoke-tested on its target architecture; producing a file is not sufficient. A missing or mismatched artifact fails with the resolved platform triple and never falls back to an install-time source build or download. Linux arm64, Linux musl, and macOS x64 are added to the manifest only with target-native loading coverage under the dedicated release issue. -Platform packages and publishing automation are completed there as well. No install script compiles -or downloads code silently in this scaffold; `prepack` creates a release artifact so test-only -exports cannot enter a package assembled from developer state. +Platform packages and publishing automation are completed there as well. No consumer install script +compiles or downloads code silently in this scaffold. `prepack` creates the host release artifact +and rejects any additional native artifact in the package root, preventing a stale or test-feature +cross-build from entering the tarball. ## Testing and end-to-end route @@ -193,16 +201,20 @@ Every introduced source module has a direct test. The scaffold gates: 5. a Node smoke test that imports the public native entry point, awaits typed capability reporting, and proves a test-only native panic becomes a stable coded JavaScript error; 6. a backend-parameterized Rust `Directory` baseline harness, initially run against Tantivy - `MmapDirectory`, covering successful atomic metadata replacement, in-process writer exclusion, - platform-correct open-file deletion, boundary range reads, `meta.json` watch notification, and - `sync_directory`; logical read-call and requested-byte counters establish the Tantivy access - pattern, while the Rocks adapter adds physical fetched/copied-byte accounting; -7. negative controls proving the harness rejects always-successful locks and visible partial - metadata after a fault-injected replacement; + `MmapDirectory`, covering concurrent atomic metadata visibility, missing-file error variants, + in-process writer exclusion, backend-neutral open-handle deletion semantics, synchronous and + asynchronous boundary reads, write termination, `meta.json` watch notification, and + `sync_directory`; logical read-call and requested-byte counters prove the instrumentation and a + fixed baseline for the harness's own operations, while the Rocks adapter adds physical + fetched/copied-byte accounting around real indexing and search; +7. negative controls proving the harness rejects always-successful locks, failed partial metadata, + and partial metadata exposed during a successful replacement; 8. a smoke test installed from `npm pack` output rather than the repository tree, proving the exports map, packaged files, addon resolution, stable errors, and platform artifact together; 9. package-content and loaded-addon inspection proving private generated bindings, unintended - source artifacts, unlisted subpaths, test-only probes, and package install scripts are absent; + source artifacts, unlisted subpaths, test-only probes, consumer lifecycle scripts, and stale + native artifacts are absent; CI also verifies committed generated declarations against a release + build; 10. `cargo deny` gates for advisories, licenses, sources, bans, and duplicate native libraries, plus a policy-scoped npm audit. diff --git a/package.json b/package.json index 7808052..645eab9 100644 --- a/package.json +++ b/package.json @@ -37,10 +37,11 @@ "format": "prettier --write . && cargo fmt", "format:check": "prettier --check . && cargo fmt --check", "lint": "tsc -p tsconfig.json --noEmit && cargo clippy --locked --all-targets --all-features -- -D warnings", - "prepack": "npm run build:native && npm run build:typescript", + "prepack": "npm run build:native && npm run build:typescript && npm run verify:package-artifacts", "test": "cargo test --locked --all-features && npm run test:node", "test:rust": "cargo test --locked --all-features", - "test:node": "npm run build:test-native && npm run build:typescript && node --test --test-concurrency=1 test/*.test.mjs" + "test:node": "npm run build:test-native && npm run build:typescript && node --test test/dependencies.test.mjs test/napi-boundaries.test.mjs test/native.test.mjs test/package-artifacts.test.mjs && npm run build:native && npm run build:typescript && node --test test/package.test.mjs", + "verify:package-artifacts": "node scripts/verify-package-artifacts.mjs" }, "engines": { "node": "^22.18.0 || >=24.0.0" diff --git a/scripts/verify-package-artifacts.mjs b/scripts/verify-package-artifacts.mjs new file mode 100644 index 0000000..89c8083 --- /dev/null +++ b/scripts/verify-package-artifacts.mjs @@ -0,0 +1,14 @@ +import { readdirSync } from 'node:fs'; + +import { platformTriple } from '../dist/load-addon.js'; + +export function verifyPackageArtifacts(entries, expectedArtifact) { + const artifacts = entries.filter((entry) => /^fulltext\..+\.node$/.test(entry)); + if (artifacts.length !== 1 || artifacts[0] !== expectedArtifact) { + throw new Error( + `Expected only ${expectedArtifact} before packing; found ${artifacts.length === 0 ? 'none' : artifacts.join(', ')}`, + ); + } +} + +verifyPackageArtifacts(readdirSync(new URL('../', import.meta.url)), `fulltext.${platformTriple()}.node`); diff --git a/src/boundary.rs b/src/boundary.rs index ecdae7b..a85b2d7 100644 --- a/src/boundary.rs +++ b/src/boundary.rs @@ -6,17 +6,25 @@ use napi::Error; pub type Result = napi::Result; #[derive(Default)] +#[cfg_attr(not(feature = "test-panic"), allow(dead_code))] pub struct PoisonState { poisoned: AtomicBool, } impl PoisonState { + #[cfg_attr(not(feature = "test-panic"), allow(dead_code))] pub fn run(&self, operation: impl FnOnce() -> T) -> Result { if self.poisoned.load(Ordering::Acquire) { return Err(coded_error("E_POISONED", "the native handle is in a terminal state")); } match catch_unwind(AssertUnwindSafe(operation)) { - Ok(value) => Ok(value), + Ok(value) => { + if self.poisoned.load(Ordering::Acquire) { + Err(coded_error("E_POISONED", "the native handle is in a terminal state")) + } else { + Ok(value) + } + } Err(_) => { self.poisoned.store(true, Ordering::Release); Err(coded_error("E_NATIVE_PANIC", "native operation panicked")) @@ -39,6 +47,8 @@ fn coded_error(code: &'static str, message: impl AsRef) -> Error<&'static s #[cfg(test)] mod tests { use super::*; + use std::sync::{Arc, Barrier}; + use std::thread; #[test] fn panic_poisoning_is_scoped_to_the_handle() { @@ -52,4 +62,26 @@ mod tests { assert_eq!(healthy.run(|| 1).unwrap(), 1); assert_eq!(run_stateless(|| 1).unwrap(), 1); } + + #[test] + fn operation_finishing_after_a_concurrent_panic_is_rejected() { + let state = Arc::new(PoisonState::default()); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + let concurrent_state = state.clone(); + let concurrent_entered = entered.clone(); + let concurrent_release = release.clone(); + let operation = thread::spawn(move || { + concurrent_state.run(|| { + concurrent_entered.wait(); + concurrent_release.wait(); + 1 + }) + }); + + entered.wait(); + assert_eq!(state.run(|| panic!("boom")).unwrap_err().status, "E_NATIVE_PANIC"); + release.wait(); + assert_eq!(operation.join().unwrap().unwrap_err().status, "E_POISONED"); + } } diff --git a/src/directory_harness.rs b/src/directory_harness.rs index 63ee2e2..a5ce715 100644 --- a/src/directory_harness.rs +++ b/src/directory_harness.rs @@ -1,14 +1,19 @@ use std::fmt; +use std::future::Future; use std::io::{self, Write}; use std::ops::Range; use std::path::Path; +use std::pin::Pin; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::{mpsc, Arc}; +use std::task::{Context, Poll, Wake, Waker}; +use std::thread; use std::time::Duration; use tantivy::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteError}; use tantivy::directory::{ - Directory, DirectoryLock, FileHandle, OwnedBytes, WatchCallback, WatchHandle, WritePtr, INDEX_WRITER_LOCK, + Directory, DirectoryLock, FileHandle, OwnedBytes, TerminatingWrite, WatchCallback, WatchHandle, WritePtr, + INDEX_WRITER_LOCK, }; use tantivy::HasLen; @@ -75,6 +80,21 @@ impl FileHandle for InstrumentedFileHandle { .fetch_add((range.end - range.start) as u64, Ordering::Relaxed); self.inner.read_bytes(range) } + + fn read_bytes_async<'life0, 'async_trait>( + &'life0 self, + range: Range, + ) -> Pin> + Send + 'async_trait>> + where + 'life0: 'async_trait, + Self: 'async_trait, + { + self.counters.calls.fetch_add(1, Ordering::Relaxed); + self.counters + .bytes_requested + .fetch_add((range.end - range.start) as u64, Ordering::Relaxed); + self.inner.read_bytes_async(range) + } } impl Directory for InstrumentedDirectory @@ -130,7 +150,9 @@ pub fn verify_directory_baseline(directory: D) -> Result Result<(), String> { + let path = Path::new("missing"); + if !matches!(directory.open_read(path), Err(OpenReadError::FileDoesNotExist(_))) { + return Err("opening a missing file returned the wrong result".to_owned()); + } + if !matches!(directory.delete(path), Err(DeleteError::FileDoesNotExist(_))) { + return Err("deleting a missing file returned the wrong result".to_owned()); + } + Ok(()) +} + fn verify_write_read_delete(directory: &D) -> Result<(), String> where D: Directory + Clone, @@ -146,39 +179,129 @@ where let path = Path::new("segment"); let mut writer = directory.open_write(path).map_err(|error| error.to_string())?; writer.write_all(b"0123456789").map_err(|error| error.to_string())?; - writer.flush().map_err(|error| error.to_string())?; + writer.terminate().map_err(|error| error.to_string())?; + if !matches!(directory.open_write(path), Err(OpenWriteError::FileAlreadyExists(_))) { + return Err("opening an existing file for write returned the wrong result".to_owned()); + } let file = directory.open_read(path).map_err(|error| error.to_string())?; let middle = file.slice(2..7).read_bytes().map_err(|error| error.to_string())?; if middle.as_slice() != b"23456" { return Err("range read returned unexpected bytes".to_owned()); } + let asynchronous = block_on(file.slice(7..10).read_bytes_async()).map_err(|error| error.to_string())?; + if asynchronous.as_slice() != b"789" { + return Err("asynchronous range read returned unexpected bytes".to_owned()); + } + let empty = file.slice(4..4).read_bytes().map_err(|error| error.to_string())?; + if !empty.is_empty() { + return Err("zero-length range read returned bytes".to_owned()); + } let deleting_directory = directory.clone(); let delete_result = std::thread::spawn(move || deleting_directory.delete(Path::new("segment"))) .join() .map_err(|_| "segment deletion panicked".to_owned())?; - if cfg!(windows) { - if delete_result.is_ok() { - return Err("Windows deleted a mapped file unexpectedly".to_owned()); - } - let retained = file.read_bytes().map_err(|error| error.to_string())?; - if retained.as_slice() != b"0123456789" { - return Err("an open file changed after a rejected deletion".to_owned()); - } - drop(retained); - drop(middle); - drop(file); + let retained = file.read_bytes().map_err(|error| error.to_string())?; + if retained.as_slice() != b"0123456789" { + return Err("an open file changed while deletion was attempted".to_owned()); + } + let deletion_succeeded = delete_result.is_ok(); + drop(retained); + drop(asynchronous); + drop(empty); + drop(middle); + drop(file); + if !deletion_succeeded { directory.delete(path).map_err(|error| error.to_string())?; - } else { - delete_result.map_err(|error| error.to_string())?; - let retained = file.read_bytes().map_err(|error| error.to_string())?; - if retained.as_slice() != b"0123456789" { - return Err("an open file changed after deletion".to_owned()); + } + if !matches!(directory.open_read(path), Err(OpenReadError::FileDoesNotExist(_))) { + return Err("deleted file remained visible".to_owned()); + } + Ok(()) +} + +fn verify_concurrent_atomic_visibility(directory: &D) -> Result<(), String> +where + D: Directory + Clone, +{ + let path = Path::new("meta.json"); + let before = vec![b'a'; 4096]; + let after = vec![b'b'; 4096]; + directory + .atomic_write(path, &before) + .map_err(|error| error.to_string())?; + verify_concurrent_atomic_replacement(directory, &before, &after) +} + +fn verify_concurrent_atomic_replacement(directory: &D, before: &[u8], after: &[u8]) -> Result<(), String> +where + D: Directory + Clone, +{ + let path = Path::new("meta.json"); + let observer_directory = directory.clone(); + let observer_before = before.to_vec(); + let observer_after = after.to_vec(); + let (ready_sender, ready_receiver) = mpsc::sync_channel(0); + let (stop_sender, stop_receiver) = mpsc::channel(); + let observer = thread::spawn(move || -> Result<(), String> { + let initial = observer_directory + .atomic_read(path) + .map_err(|error| error.to_string())?; + if initial != observer_before { + return Err("atomic metadata observer did not read the initial value".to_owned()); + } + ready_sender.send(()).map_err(|error| error.to_string())?; + loop { + match stop_receiver.try_recv() { + Ok(()) | Err(mpsc::TryRecvError::Disconnected) => break, + Err(mpsc::TryRecvError::Empty) => {} + } + let observed = observer_directory + .atomic_read(path) + .map_err(|error| error.to_string())?; + if observed != observer_before && observed != observer_after { + return Err("atomic metadata observer saw a partial value".to_owned()); + } + thread::yield_now(); } + Ok(()) + }); + ready_receiver + .recv_timeout(Duration::from_secs(2)) + .map_err(|error| error.to_string())?; + let write_result = directory.atomic_write(path, after).map_err(|error| error.to_string()); + let _ = stop_sender.send(()); + let observer_result = observer + .join() + .map_err(|_| "atomic metadata observer panicked".to_owned())?; + write_result?; + observer_result?; + if directory.atomic_read(path).map_err(|error| error.to_string())? != after { + return Err("atomic metadata replacement returned unexpected bytes".to_owned()); } Ok(()) } +fn block_on(future: F) -> F::Output { + struct ThreadWake(thread::Thread); + + impl Wake for ThreadWake { + fn wake(self: Arc) { + self.0.unpark(); + } + } + + let waker = Waker::from(Arc::new(ThreadWake(thread::current()))); + let mut context = Context::from_waker(&waker); + let mut future = std::pin::pin!(future); + loop { + match future.as_mut().poll(&mut context) { + Poll::Ready(output) => return output, + Poll::Pending => thread::park(), + } + } +} + pub fn verify_failed_atomic_replacement( directory: &dyn Directory, replacement: impl FnOnce() -> io::Result<()>, @@ -260,6 +383,9 @@ mod tests { inner: MmapDirectory, ignore_locks: bool, fail_next_atomic_write: Arc, + non_atomic_next_write: Arc, + non_atomic_write_active: Arc, + partial_write_observed: Arc, } impl BrokenDirectory { @@ -268,12 +394,20 @@ mod tests { inner: MmapDirectory::create_from_tempdir().unwrap(), ignore_locks, fail_next_atomic_write: Arc::new(AtomicBool::new(false)), + non_atomic_next_write: Arc::new(AtomicBool::new(false)), + non_atomic_write_active: Arc::new(AtomicBool::new(false)), + partial_write_observed: Arc::new(AtomicBool::new(false)), } } fn fail_next_atomic_write(&self) { self.fail_next_atomic_write.store(true, Ordering::Release); } + + fn make_next_write_non_atomic(&self) { + self.non_atomic_next_write.store(true, Ordering::Release); + self.partial_write_observed.store(false, Ordering::Release); + } } impl Directory for BrokenDirectory { @@ -294,10 +428,39 @@ mod tests { } fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { - self.inner.atomic_read(path) + let result = self.inner.atomic_read(path); + if self.non_atomic_write_active.load(Ordering::Acquire) + && match &result { + Ok(bytes) => bytes.len() != 4096, + Err(_) => true, + } { + self.partial_write_observed.store(true, Ordering::Release); + } + result } fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { + if self.non_atomic_next_write.swap(false, Ordering::AcqRel) { + self.non_atomic_write_active.store(true, Ordering::Release); + if self.inner.exists(path).unwrap_or(false) { + self.inner.delete(path).map_err(io::Error::other)?; + } + let mut writer = self.inner.open_write(path).map_err(io::Error::other)?; + writer.write_all(&data[..data.len() / 2])?; + writer.flush()?; + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !self.partial_write_observed.load(Ordering::Acquire) { + if std::time::Instant::now() >= deadline { + self.non_atomic_write_active.store(false, Ordering::Release); + return Err(io::Error::other("observer did not inspect partial metadata")); + } + thread::yield_now(); + } + writer.write_all(&data[data.len() / 2..])?; + let result = writer.terminate(); + self.non_atomic_write_active.store(false, Ordering::Release); + return result; + } if !self.fail_next_atomic_write.swap(false, Ordering::AcqRel) { return self.inner.atomic_write(path, data); } @@ -330,8 +493,8 @@ mod tests { fn mmap_directory_satisfies_the_contract() { let directory = MmapDirectory::create_from_tempdir().unwrap(); let metrics = verify_directory_baseline(directory).unwrap(); - assert_eq!(metrics.calls, 4); - assert_eq!(metrics.bytes_requested, 26); + assert_eq!(metrics.calls, 6); + assert_eq!(metrics.bytes_requested, 29); } #[test] @@ -350,4 +513,14 @@ mod tests { }) .is_err()); } + + #[test] + fn harness_rejects_partial_metadata_during_successful_replacement() { + let directory = BrokenDirectory::new(false); + let before = vec![b'a'; 4096]; + let after = vec![b'b'; 4096]; + directory.atomic_write(Path::new("meta.json"), &before).unwrap(); + directory.make_next_write_non_atomic(); + assert!(verify_concurrent_atomic_replacement(&directory, &before, &after).is_err()); + } } diff --git a/src/lib.rs b/src/lib.rs index 9a24efc..f875cb7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,7 +42,7 @@ pub struct TestHandle { #[cfg(feature = "test-panic")] #[napi] impl TestHandle { - #[napi(constructor)] + #[napi(catch_unwind, constructor)] pub fn new() -> Self { Self::default() } diff --git a/test/dependencies.test.mjs b/test/dependencies.test.mjs index ae59aee..05567cc 100644 --- a/test/dependencies.test.mjs +++ b/test/dependencies.test.mjs @@ -11,8 +11,10 @@ test('every direct dependency is documented', () => { for (const dependency of cargoDependencies(cargoManifest)) { assert(dependencyLedger.includes(`\`${dependency}\``), `${dependency} is absent from dependencies.md`); } - for (const dependency of Object.keys(packageManifest.devDependencies)) { - assert(dependencyLedger.includes(`\`${dependency}\``), `${dependency} is absent from dependencies.md`); + for (const section of ['dependencies', 'optionalDependencies', 'peerDependencies', 'devDependencies']) { + for (const dependency of Object.keys(packageManifest[section] ?? {})) { + assert(dependencyLedger.includes(`\`${dependency}\``), `${dependency} is absent from dependencies.md`); + } } }); diff --git a/test/napi-boundaries.test.mjs b/test/napi-boundaries.test.mjs new file mode 100644 index 0000000..a499806 --- /dev/null +++ b/test/napi-boundaries.test.mjs @@ -0,0 +1,18 @@ +import assert from 'node:assert'; +import { readdirSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import test from 'node:test'; + +test('every Node-API function has an outer unwind boundary', () => { + const sourceDirectory = fileURLToPath(new URL('../src/', import.meta.url)); + for (const entry of readdirSync(sourceDirectory, { recursive: true })) { + if (typeof entry !== 'string' || !entry.endsWith('.rs')) { + continue; + } + const source = readFileSync(path.join(sourceDirectory, entry), 'utf8'); + for (const match of source.matchAll(/#\[napi(?:\(([^)]*)\))?]\s+pub\s+(?:async\s+)?fn\s+(\w+)/g)) { + assert.match(match[1] ?? '', /(?:^|,)\s*catch_unwind\s*(?:,|$)/, `${entry}:${match[2]} lacks catch_unwind`); + } + } +}); diff --git a/test/package-artifacts.test.mjs b/test/package-artifacts.test.mjs new file mode 100644 index 0000000..0c58433 --- /dev/null +++ b/test/package-artifacts.test.mjs @@ -0,0 +1,14 @@ +import assert from 'node:assert'; +import test from 'node:test'; + +import { verifyPackageArtifacts } from '../scripts/verify-package-artifacts.mjs'; + +test('package verification rejects missing and additional native artifacts', () => { + const expected = 'fulltext.linux-x64-gnu.node'; + assert.doesNotThrow(() => verifyPackageArtifacts([expected], expected)); + assert.throws(() => verifyPackageArtifacts([], expected), /found none/); + assert.throws( + () => verifyPackageArtifacts([expected, 'fulltext.win32-x64-msvc.node'], expected), + /fulltext\.win32-x64-msvc\.node/, + ); +}); diff --git a/test/package.test.mjs b/test/package.test.mjs index c010b3d..42cb144 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -19,11 +19,10 @@ function runNpm(arguments_, options) { return execFileSync(npmInvocation.executable, argumentsWithPrefix, options); } -test('the packed package loads without lifecycle scripts', (context) => { +test('the packed package loads without consumer lifecycle scripts', (context) => { const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'harper-fulltext-')); context.after(() => rmSync(temporaryDirectory, { force: true, recursive: true })); - runNpm(['run', 'build:native'], { stdio: 'pipe' }); - const packOutput = runNpm(['pack', '--json', '--ignore-scripts', '--pack-destination', temporaryDirectory], { + const packOutput = runNpm(['pack', '--json', '--pack-destination', temporaryDirectory], { encoding: 'utf8', }); const parsedOutput = JSON.parse(packOutput); @@ -55,7 +54,9 @@ test('the packed package loads without lifecycle scripts', (context) => { const installedManifest = JSON.parse( readFileSync(path.join(projectDirectory, 'node_modules/@harperfast/fulltext/package.json'), 'utf8'), ); - assert(!installedManifest.scripts?.install); + for (const lifecycle of ['preinstall', 'install', 'postinstall', 'prepare']) { + assert(!installedManifest.scripts?.[lifecycle], `${lifecycle} must not run in a consumer installation`); + } const artifact = includedPaths.find((file) => /^fulltext\..+\.node$/.test(file)); const require = createRequire(import.meta.url); const installedAddon = require(path.join(projectDirectory, 'node_modules/@harperfast/fulltext', artifact)); From bca9434c0bd836e86edd67bc47521792bb3ea05d Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Fri, 4 Sep 2026 23:55:38 -0600 Subject: [PATCH 4/9] test: close final scaffold review gaps --- dependencies.md | 5 ++ package.json | 9 +-- scripts/clean-native-build-state.mjs | 22 +++++++ scripts/run-node-tests.mjs | 25 ++++++++ scripts/verify-package-artifacts.mjs | 5 +- src/directory_harness.rs | 52 ++++++++++------ test/napi-boundaries.test.mjs | 60 +++++++++++++++++-- ...kage.test.mjs => package.release.test.mjs} | 18 +++--- 8 files changed, 162 insertions(+), 34 deletions(-) create mode 100644 scripts/clean-native-build-state.mjs create mode 100644 scripts/run-node-tests.mjs rename test/{package.test.mjs => package.release.test.mjs} (85%) diff --git a/dependencies.md b/dependencies.md index ed2768d..b7e03ad 100644 --- a/dependencies.md +++ b/dependencies.md @@ -20,6 +20,11 @@ fulltext function, method, and constructor uses that option to contain argument conversion panics. That outer boundary uses napi-rs error mapping; the inner boundary adds stable error codes and per-handle poison state for package-owned operations. +napi-rs 2.16 accumulates generated type records in a checkout-specific temporary file. Native build +scripts remove those exact intermediate files and refresh the crate entry point's modification time +before rebuilding, so switching between test and release features cannot retain test-only +declarations through Cargo's incremental compilation. + ## JavaScript development graph | Dependency | Scope | Purpose | diff --git a/package.json b/package.json index 645eab9..5dbd906 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,11 @@ }, "scripts": { "build": "npm run build:typescript && npm run build:native", - "build:debug": "npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features node-api", - "build:native": "napi build --platform --js false --dts ts/addon.d.ts --release --features node-api", - "build:test-native": "napi build --platform --js false --dts ts/addon.d.ts --features test-panic", + "build:debug": "npm run clean:generated-native && npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features node-api", + "build:native": "npm run clean:generated-native && napi build --platform --js false --dts ts/addon.d.ts --release --features node-api", + "build:test-native": "npm run clean:generated-native && napi build --platform --js false --dts ts/addon.d.ts --features test-panic", "build:typescript": "tsc -p tsconfig.json", + "clean:generated-native": "node scripts/clean-native-build-state.mjs", "check": "npm run format:check && npm run lint && npm run test", "format": "prettier --write . && cargo fmt", "format:check": "prettier --check . && cargo fmt --check", @@ -40,7 +41,7 @@ "prepack": "npm run build:native && npm run build:typescript && npm run verify:package-artifacts", "test": "cargo test --locked --all-features && npm run test:node", "test:rust": "cargo test --locked --all-features", - "test:node": "npm run build:test-native && npm run build:typescript && node --test test/dependencies.test.mjs test/napi-boundaries.test.mjs test/native.test.mjs test/package-artifacts.test.mjs && npm run build:native && npm run build:typescript && node --test test/package.test.mjs", + "test:node": "npm run build:test-native && npm run build:typescript && node scripts/run-node-tests.mjs test && npm run build:native && npm run build:typescript && node scripts/run-node-tests.mjs release", "verify:package-artifacts": "node scripts/verify-package-artifacts.mjs" }, "engines": { diff --git a/scripts/clean-native-build-state.mjs b/scripts/clean-native-build-state.mjs new file mode 100644 index 0000000..916ce39 --- /dev/null +++ b/scripts/clean-native-build-state.mjs @@ -0,0 +1,22 @@ +import { createHash } from 'node:crypto'; +import { readFileSync, rmSync, utimesSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const require = createRequire(import.meta.url); +const cargoManifest = readFileSync(new URL('../Cargo.toml', import.meta.url), 'utf8'); +const cargoPackageName = /^name\s*=\s*"([^"]+)"/m.exec(cargoManifest)?.[1]; +if (!cargoPackageName) { + throw new Error('Cargo package name is missing'); +} +const cargoArtifactName = cargoPackageName.replaceAll('-', '_'); +const napiCliVersion = require('@napi-rs/cli/package.json').version; +const cwdHash = createHash('sha256').update(process.cwd()).update(napiCliVersion).digest('hex').slice(0, 8); + +rmSync(new URL('../ts/addon.d.ts', import.meta.url), { force: true }); +for (const suffix of ['napi_type_def.tmp', 'napi_wasi_register.tmp']) { + rmSync(path.join(tmpdir(), `${cargoArtifactName}-${cwdHash}.${suffix}`), { force: true }); +} +const now = new Date(); +utimesSync(new URL('../src/lib.rs', import.meta.url), now, now); diff --git a/scripts/run-node-tests.mjs b/scripts/run-node-tests.mjs new file mode 100644 index 0000000..b42da19 --- /dev/null +++ b/scripts/run-node-tests.mjs @@ -0,0 +1,25 @@ +import { spawnSync } from 'node:child_process'; +import { readdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +const phase = process.argv[2]; +if (phase !== 'test' && phase !== 'release') { + throw new Error('Expected the test phase to be "test" or "release"'); +} + +const testDirectory = fileURLToPath(new URL('../test/', import.meta.url)); +const files = readdirSync(testDirectory, { recursive: true }) + .filter((entry) => typeof entry === 'string' && entry.endsWith('.test.mjs')) + .filter((entry) => (phase === 'release') === entry.endsWith('.release.test.mjs')) + .map((entry) => path.join(testDirectory, entry)); + +if (files.length === 0) { + throw new Error(`No ${phase} test files were discovered`); +} + +const result = spawnSync(process.execPath, ['--test', ...files], { stdio: 'inherit' }); +if (result.error) { + throw result.error; +} +process.exitCode = result.status ?? 1; diff --git a/scripts/verify-package-artifacts.mjs b/scripts/verify-package-artifacts.mjs index 89c8083..110396d 100644 --- a/scripts/verify-package-artifacts.mjs +++ b/scripts/verify-package-artifacts.mjs @@ -1,4 +1,5 @@ import { readdirSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; import { platformTriple } from '../dist/load-addon.js'; @@ -11,4 +12,6 @@ export function verifyPackageArtifacts(entries, expectedArtifact) { } } -verifyPackageArtifacts(readdirSync(new URL('../', import.meta.url)), `fulltext.${platformTriple()}.node`); +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + verifyPackageArtifacts(readdirSync(new URL('../', import.meta.url)), `fulltext.${platformTriple()}.node`); +} diff --git a/src/directory_harness.rs b/src/directory_harness.rs index a5ce715..a875d3a 100644 --- a/src/directory_harness.rs +++ b/src/directory_harness.rs @@ -374,13 +374,17 @@ fn verify_watch(directory: &dyn Directory) -> Result<(), String> { #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; + use std::path::PathBuf; use std::sync::atomic::AtomicBool; + use std::sync::RwLock; use tantivy::directory::{Lock, MmapDirectory}; #[derive(Clone, Debug)] struct BrokenDirectory { inner: MmapDirectory, + atomic_files: Arc>>>, ignore_locks: bool, fail_next_atomic_write: Arc, non_atomic_next_write: Arc, @@ -392,6 +396,7 @@ mod tests { fn new(ignore_locks: bool) -> Self { Self { inner: MmapDirectory::create_from_tempdir().unwrap(), + atomic_files: Arc::new(RwLock::new(HashMap::new())), ignore_locks, fail_next_atomic_write: Arc::new(AtomicBool::new(false)), non_atomic_next_write: Arc::new(AtomicBool::new(false)), @@ -428,7 +433,13 @@ mod tests { } fn atomic_read(&self, path: &Path) -> Result, OpenReadError> { - let result = self.inner.atomic_read(path); + let result = self + .atomic_files + .read() + .unwrap() + .get(path) + .cloned() + .ok_or_else(|| OpenReadError::FileDoesNotExist(path.to_path_buf())); if self.non_atomic_write_active.load(Ordering::Acquire) && match &result { Ok(bytes) => bytes.len() != 4096, @@ -442,12 +453,10 @@ mod tests { fn atomic_write(&self, path: &Path, data: &[u8]) -> io::Result<()> { if self.non_atomic_next_write.swap(false, Ordering::AcqRel) { self.non_atomic_write_active.store(true, Ordering::Release); - if self.inner.exists(path).unwrap_or(false) { - self.inner.delete(path).map_err(io::Error::other)?; - } - let mut writer = self.inner.open_write(path).map_err(io::Error::other)?; - writer.write_all(&data[..data.len() / 2])?; - writer.flush()?; + self.atomic_files + .write() + .unwrap() + .insert(path.to_path_buf(), data[..data.len() / 2].to_vec()); let deadline = std::time::Instant::now() + Duration::from_secs(2); while !self.partial_write_observed.load(Ordering::Acquire) { if std::time::Instant::now() >= deadline { @@ -456,20 +465,24 @@ mod tests { } thread::yield_now(); } - writer.write_all(&data[data.len() / 2..])?; - let result = writer.terminate(); + self.atomic_files + .write() + .unwrap() + .insert(path.to_path_buf(), data.to_vec()); self.non_atomic_write_active.store(false, Ordering::Release); - return result; + return Ok(()); } if !self.fail_next_atomic_write.swap(false, Ordering::AcqRel) { - return self.inner.atomic_write(path, data); - } - if self.inner.exists(path).unwrap_or(false) { - self.inner.delete(path).map_err(io::Error::other)?; + self.atomic_files + .write() + .unwrap() + .insert(path.to_path_buf(), data.to_vec()); + return Ok(()); } - let mut writer = self.inner.open_write(path).map_err(io::Error::other)?; - writer.write_all(&data[..data.len() / 2])?; - writer.flush()?; + self.atomic_files + .write() + .unwrap() + .insert(path.to_path_buf(), data[..data.len() / 2].to_vec()); Err(io::Error::other("injected partial write")) } @@ -521,6 +534,9 @@ mod tests { let after = vec![b'b'; 4096]; directory.atomic_write(Path::new("meta.json"), &before).unwrap(); directory.make_next_write_non_atomic(); - assert!(verify_concurrent_atomic_replacement(&directory, &before, &after).is_err()); + assert_eq!( + verify_concurrent_atomic_replacement(&directory, &before, &after).unwrap_err(), + "atomic metadata observer saw a partial value" + ); } } diff --git a/test/napi-boundaries.test.mjs b/test/napi-boundaries.test.mjs index a499806..3d0b2b2 100644 --- a/test/napi-boundaries.test.mjs +++ b/test/napi-boundaries.test.mjs @@ -10,9 +10,61 @@ test('every Node-API function has an outer unwind boundary', () => { if (typeof entry !== 'string' || !entry.endsWith('.rs')) { continue; } - const source = readFileSync(path.join(sourceDirectory, entry), 'utf8'); - for (const match of source.matchAll(/#\[napi(?:\(([^)]*)\))?]\s+pub\s+(?:async\s+)?fn\s+(\w+)/g)) { - assert.match(match[1] ?? '', /(?:^|,)\s*catch_unwind\s*(?:,|$)/, `${entry}:${match[2]} lacks catch_unwind`); - } + verifyNapiBoundaries(readFileSync(path.join(sourceDirectory, entry), 'utf8'), entry); } }); + +test('the Node-API boundary check rejects syntax that previously failed open', () => { + assert.throws( + () => verifyNapiBoundaries('#[napi]\n#[allow(dead_code)]\npub fn missing() {}', 'extra-attribute.rs'), + /lacks catch_unwind/, + ); + assert.throws( + () => verifyNapiBoundaries('#[napi(ts_args_type = "Array<(string)>")]\npub fn nested() {}', 'nested.rs'), + /lacks catch_unwind/, + ); + assert.throws( + () => verifyNapiBoundaries('#[napi(\ncatch_unwind\n)]\npub fn multiline() {}', 'multiline.rs'), + /must be written on one line/, + ); + assert.doesNotThrow(() => + verifyNapiBoundaries( + '#[napi(ts_args_type = "Array<(string)>", catch_unwind)]\n#[allow(dead_code)]\npub fn guarded() {}', + 'guarded.rs', + ), + ); +}); + +function verifyNapiBoundaries(source, entry) { + const lines = source.split('\n'); + for (let index = 0; index < lines.length; index++) { + const attribute = lines[index].trim(); + if (!attribute.startsWith('#[napi')) { + continue; + } + assert.match( + attribute, + /^#\[napi(?:\(.*\))?]$/, + `${entry}:${index + 1} napi attributes must be written on one line`, + ); + + let itemIndex = index + 1; + while (itemIndex < lines.length) { + const line = lines[itemIndex].trim(); + if (line === '' || line.startsWith('//')) { + itemIndex++; + continue; + } + if (line.startsWith('#[')) { + assert.match(line, /^#\[.*]$/, `${entry}:${itemIndex + 1} item attributes must be written on one line`); + itemIndex++; + continue; + } + const functionName = /(?:^|\s)fn\s+(\w+)/.exec(line)?.[1]; + if (functionName) { + assert.match(attribute, /(?:\(|,)\s*catch_unwind\s*(?:,|\))/, `${entry}:${functionName} lacks catch_unwind`); + } + break; + } + } +} diff --git a/test/package.test.mjs b/test/package.release.test.mjs similarity index 85% rename from test/package.test.mjs rename to test/package.release.test.mjs index 42cb144..9ff05c1 100644 --- a/test/package.test.mjs +++ b/test/package.release.test.mjs @@ -6,25 +6,28 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; -const npmInvocation = process.env.npm_execpath +const npmCommand = process.env.npm_execpath ? { executable: process.execPath, prefix: [process.env.npm_execpath] } : { executable: process.platform === 'win32' ? process.env.ComSpec || 'cmd.exe' : 'npm', prefix: [] }; function runNpm(arguments_, options) { - const argumentsWithPrefix = [...npmInvocation.prefix]; - if (process.platform === 'win32' && npmInvocation.prefix.length === 0) { + const argumentsWithPrefix = [...npmCommand.prefix]; + if (process.platform === 'win32' && npmCommand.prefix.length === 0) { argumentsWithPrefix.push('/d', '/s', '/c', 'npm.cmd'); } argumentsWithPrefix.push(...arguments_); - return execFileSync(npmInvocation.executable, argumentsWithPrefix, options); + return execFileSync(npmCommand.executable, argumentsWithPrefix, options); } test('the packed package loads without consumer lifecycle scripts', (context) => { const temporaryDirectory = mkdtempSync(path.join(tmpdir(), 'harper-fulltext-')); context.after(() => rmSync(temporaryDirectory, { force: true, recursive: true })); - const packOutput = runNpm(['pack', '--json', '--pack-destination', temporaryDirectory], { - encoding: 'utf8', - }); + const packOutput = runNpm( + ['pack', '--json', '--foreground-scripts=false', '--pack-destination', temporaryDirectory], + { + encoding: 'utf8', + }, + ); const parsedOutput = JSON.parse(packOutput); const pack = Array.isArray(parsedOutput) ? parsedOutput[0] : parsedOutput['@harperfast/fulltext']; const { filename, files } = pack; @@ -32,6 +35,7 @@ test('the packed package loads without consumer lifecycle scripts', (context) => assert(includedPaths.includes('dist/native.js')); assert(includedPaths.some((file) => /^fulltext\..+\.node$/.test(file))); assert(!includedPaths.some((file) => file.startsWith('src/') || file === 'ts/addon.d.ts')); + assert.doesNotMatch(readFileSync(new URL('../ts/addon.d.ts', import.meta.url), 'utf8'), /TestHandle/); const projectDirectory = path.join(temporaryDirectory, 'consumer'); mkdirSync(projectDirectory); From a013b01a16a43d50aed1cee8595d8888694cb315 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sat, 5 Sep 2026 00:15:50 -0600 Subject: [PATCH 5/9] test: isolate test-only native probes --- dependencies.md | 5 --- docs/scaffold-design.md | 4 ++ package.json | 7 ++-- scripts/clean-native-build-state.mjs | 22 ---------- src/lib.rs | 61 ++++++++++++++++++++-------- test/napi-boundaries.test.mjs | 15 +++++++ test/native.test.mjs | 12 +++--- test/package.release.test.mjs | 4 +- ts/load-addon.ts | 9 ++-- 9 files changed, 77 insertions(+), 62 deletions(-) delete mode 100644 scripts/clean-native-build-state.mjs diff --git a/dependencies.md b/dependencies.md index b7e03ad..ed2768d 100644 --- a/dependencies.md +++ b/dependencies.md @@ -20,11 +20,6 @@ fulltext function, method, and constructor uses that option to contain argument conversion panics. That outer boundary uses napi-rs error mapping; the inner boundary adds stable error codes and per-handle poison state for package-owned operations. -napi-rs 2.16 accumulates generated type records in a checkout-specific temporary file. Native build -scripts remove those exact intermediate files and refresh the crate entry point's modification time -before rebuilding, so switching between test and release features cannot retain test-only -declarations through Cargo's incremental compilation. - ## JavaScript development graph | Dependency | Scope | Purpose | diff --git a/docs/scaffold-design.md b/docs/scaffold-design.md index 0ad7796..e42b6ac 100644 --- a/docs/scaffold-design.md +++ b/docs/scaffold-design.md @@ -153,6 +153,10 @@ with `E_POISONED`; callers cannot retry through potentially corrupted Tantivy or Stateless capability inspection remains available, and an unrelated handle remains healthy. The panic smoke route verifies the original coded failure, terminal handle poisoning, and isolation from another handle. An operation completing after a concurrent panic also returns `E_POISONED`. +The poison flag is not a substitute for synchronization: an owning writer or other mutable handle +must serialize its state-changing operations before entering this boundary. Concurrent operations +are permitted only over immutable or independently synchronized state, so no successful operation +can observe a sibling mutation while that sibling is unwinding. Caught panics still invoke Rust's process-wide panic hook before returning the coded error; Harper logging must classify the subsequent error as contained rather than treating the hook output alone as evidence of process failure. CI also asserts that the effective release profile retains diff --git a/package.json b/package.json index 5dbd906..8730779 100644 --- a/package.json +++ b/package.json @@ -29,11 +29,10 @@ }, "scripts": { "build": "npm run build:typescript && npm run build:native", - "build:debug": "npm run clean:generated-native && npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features node-api", - "build:native": "npm run clean:generated-native && napi build --platform --js false --dts ts/addon.d.ts --release --features node-api", - "build:test-native": "npm run clean:generated-native && napi build --platform --js false --dts ts/addon.d.ts --features test-panic", + "build:debug": "npm run build:typescript && napi build --platform --js false --dts ts/addon.d.ts --features node-api", + "build:native": "napi build --platform --js false --dts ts/addon.d.ts --release --features node-api", + "build:test-native": "napi build --platform --js false --dts ts/addon.d.ts --features test-panic", "build:typescript": "tsc -p tsconfig.json", - "clean:generated-native": "node scripts/clean-native-build-state.mjs", "check": "npm run format:check && npm run lint && npm run test", "format": "prettier --write . && cargo fmt", "format:check": "prettier --check . && cargo fmt --check", diff --git a/scripts/clean-native-build-state.mjs b/scripts/clean-native-build-state.mjs deleted file mode 100644 index 916ce39..0000000 --- a/scripts/clean-native-build-state.mjs +++ /dev/null @@ -1,22 +0,0 @@ -import { createHash } from 'node:crypto'; -import { readFileSync, rmSync, utimesSync } from 'node:fs'; -import { createRequire } from 'node:module'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; - -const require = createRequire(import.meta.url); -const cargoManifest = readFileSync(new URL('../Cargo.toml', import.meta.url), 'utf8'); -const cargoPackageName = /^name\s*=\s*"([^"]+)"/m.exec(cargoManifest)?.[1]; -if (!cargoPackageName) { - throw new Error('Cargo package name is missing'); -} -const cargoArtifactName = cargoPackageName.replaceAll('-', '_'); -const napiCliVersion = require('@napi-rs/cli/package.json').version; -const cwdHash = createHash('sha256').update(process.cwd()).update(napiCliVersion).digest('hex').slice(0, 8); - -rmSync(new URL('../ts/addon.d.ts', import.meta.url), { force: true }); -for (const suffix of ['napi_type_def.tmp', 'napi_wasi_register.tmp']) { - rmSync(path.join(tmpdir(), `${cargoArtifactName}-${cwdHash}.${suffix}`), { force: true }); -} -const now = new Date(); -utimesSync(new URL('../src/lib.rs', import.meta.url), now, now); diff --git a/src/lib.rs b/src/lib.rs index f875cb7..127f130 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,6 +9,13 @@ mod boundary; #[cfg(feature = "node-api")] use napi_derive::napi; +#[cfg(feature = "test-panic")] +use std::collections::HashMap; +#[cfg(feature = "test-panic")] +use std::sync::atomic::{AtomicU32, Ordering}; +#[cfg(feature = "test-panic")] +use std::sync::{Arc, Mutex, OnceLock}; + pub const NATIVE_ABI_VERSION: u32 = 1; pub const TANTIVY_VERSION: &str = "0.26.1"; @@ -33,27 +40,45 @@ pub fn runtime_info() -> boundary::Result { } #[cfg(feature = "test-panic")] -#[napi] -#[derive(Default)] -pub struct TestHandle { - poison: boundary::PoisonState, +static NEXT_TEST_HANDLE: AtomicU32 = AtomicU32::new(1); +#[cfg(feature = "test-panic")] +static TEST_HANDLES: OnceLock>>> = OnceLock::new(); + +#[cfg(feature = "test-panic")] +#[napi(catch_unwind, skip_typescript, js_name = "__testCreateHandle")] +pub fn test_create_handle() -> boundary::Result { + boundary::run_stateless(|| { + let id = NEXT_TEST_HANDLE.fetch_add(1, Ordering::Relaxed); + TEST_HANDLES + .get_or_init(Default::default) + .lock() + .unwrap() + .insert(id, Arc::new(boundary::PoisonState::default())); + id + }) } #[cfg(feature = "test-panic")] -#[napi] -impl TestHandle { - #[napi(catch_unwind, constructor)] - pub fn new() -> Self { - Self::default() - } +#[napi(catch_unwind, skip_typescript, js_name = "__testPanic")] +pub fn test_panic(id: u32) -> boundary::Result<()> { + test_handle(id)?.run(|| panic!("test panic")) +} - #[napi(catch_unwind)] - pub fn panic(&self) -> boundary::Result<()> { - self.poison.run(|| panic!("test panic")) - } +#[cfg(feature = "test-panic")] +#[napi(catch_unwind, skip_typescript, js_name = "__testCheck")] +pub fn test_check(id: u32) -> boundary::Result { + test_handle(id)?.run(|| true) +} - #[napi(catch_unwind)] - pub fn check(&self) -> boundary::Result { - self.poison.run(|| true) - } +#[cfg(feature = "test-panic")] +fn test_handle(id: u32) -> boundary::Result> { + boundary::run_stateless(|| { + TEST_HANDLES + .get_or_init(Default::default) + .lock() + .unwrap() + .get(&id) + .cloned() + .expect("unknown test handle") + }) } diff --git a/test/napi-boundaries.test.mjs b/test/napi-boundaries.test.mjs index 3d0b2b2..5ef39e2 100644 --- a/test/napi-boundaries.test.mjs +++ b/test/napi-boundaries.test.mjs @@ -27,6 +27,10 @@ test('the Node-API boundary check rejects syntax that previously failed open', ( () => verifyNapiBoundaries('#[napi(\ncatch_unwind\n)]\npub fn multiline() {}', 'multiline.rs'), /must be written on one line/, ); + assert.throws( + () => verifyNapiBoundaries('#[napi]\n/* boundary note */\npub fn commented() {}', 'commented.rs'), + /lacks catch_unwind/, + ); assert.doesNotThrow(() => verifyNapiBoundaries( '#[napi(ts_args_type = "Array<(string)>", catch_unwind)]\n#[allow(dead_code)]\npub fn guarded() {}', @@ -49,8 +53,19 @@ function verifyNapiBoundaries(source, entry) { ); let itemIndex = index + 1; + let inBlockComment = false; while (itemIndex < lines.length) { const line = lines[itemIndex].trim(); + if (inBlockComment) { + inBlockComment = !line.includes('*/'); + itemIndex++; + continue; + } + if (line.startsWith('/*')) { + inBlockComment = !line.includes('*/'); + itemIndex++; + continue; + } if (line === '' || line.startsWith('//')) { itemIndex++; continue; diff --git a/test/native.test.mjs b/test/native.test.mjs index 2e81473..adb3d74 100644 --- a/test/native.test.mjs +++ b/test/native.test.mjs @@ -17,16 +17,18 @@ test('loads the artifact for the executing platform', async () => { }); test('turns a panic into a coded terminal error', async () => { - const firstHandle = new (loadAddon().TestHandle)(); - const secondHandle = new (loadAddon().TestHandle)(); + const addon = loadAddon(); + assert(addon.__testCreateHandle && addon.__testPanic && addon.__testCheck); + const firstHandle = addon.__testCreateHandle(); + const secondHandle = addon.__testCreateHandle(); assert.throws( - () => firstHandle.panic(), + () => addon.__testPanic(firstHandle), (error) => normalizeNativeError(error).code === 'E_NATIVE_PANIC', ); assert.throws( - () => firstHandle.check(), + () => addon.__testCheck(firstHandle), (error) => normalizeNativeError(error).code === 'E_POISONED', ); - assert.strictEqual(secondHandle.check(), true); + assert.strictEqual(addon.__testCheck(secondHandle), true); await assert.doesNotReject(runtimeInfo()); }); diff --git a/test/package.release.test.mjs b/test/package.release.test.mjs index 9ff05c1..7298ed4 100644 --- a/test/package.release.test.mjs +++ b/test/package.release.test.mjs @@ -35,7 +35,7 @@ test('the packed package loads without consumer lifecycle scripts', (context) => assert(includedPaths.includes('dist/native.js')); assert(includedPaths.some((file) => /^fulltext\..+\.node$/.test(file))); assert(!includedPaths.some((file) => file.startsWith('src/') || file === 'ts/addon.d.ts')); - assert.doesNotMatch(readFileSync(new URL('../ts/addon.d.ts', import.meta.url), 'utf8'), /TestHandle/); + assert.doesNotMatch(readFileSync(new URL('../ts/addon.d.ts', import.meta.url), 'utf8'), /__test/); const projectDirectory = path.join(temporaryDirectory, 'consumer'); mkdirSync(projectDirectory); @@ -64,7 +64,7 @@ test('the packed package loads without consumer lifecycle scripts', (context) => const artifact = includedPaths.find((file) => /^fulltext\..+\.node$/.test(file)); const require = createRequire(import.meta.url); const installedAddon = require(path.join(projectDirectory, 'node_modules/@harperfast/fulltext', artifact)); - assert(!('TestHandle' in installedAddon)); + assert(!('__testCreateHandle' in installedAddon)); const output = execFileSync( process.execPath, [ diff --git a/ts/load-addon.ts b/ts/load-addon.ts index 5940a02..a3c4e07 100644 --- a/ts/load-addon.ts +++ b/ts/load-addon.ts @@ -11,14 +11,11 @@ interface NativeRuntimeInfo { storageBackends: Array; } -interface NativeTestHandle { - panic(): void; - check(): boolean; -} - interface NativeAddonApi { runtimeInfo(): NativeRuntimeInfo; - TestHandle?: new () => NativeTestHandle; + __testCreateHandle?(): number; + __testPanic?(id: number): void; + __testCheck?(id: number): boolean; } const require = createRequire(import.meta.url); From 4df0a1620da82ef5126dd06ef70ba3914a190024 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sat, 5 Sep 2026 00:36:09 -0600 Subject: [PATCH 6/9] test: strengthen scaffold contract checks --- docs/scaffold-design.md | 11 +++-- package.json | 2 +- src/directory_harness.rs | 90 +++++++++++++++++++++++++++++++++------- src/lib.rs | 6 +-- test/native.test.mjs | 24 ++++++++++- 5 files changed, 108 insertions(+), 25 deletions(-) diff --git a/docs/scaffold-design.md b/docs/scaffold-design.md index e42b6ac..df36387 100644 --- a/docs/scaffold-design.md +++ b/docs/scaffold-design.md @@ -203,16 +203,19 @@ Every introduced source module has a direct test. The scaffold gates: 3. TypeScript type checking; 4. native addon build; 5. a Node smoke test that imports the public native entry point, awaits typed capability reporting, - and proves a test-only native panic becomes a stable coded JavaScript error; + verifies package and Tantivy versions against their manifests, and proves a test-only native + panic becomes a stable coded JavaScript error; 6. a backend-parameterized Rust `Directory` baseline harness, initially run against Tantivy `MmapDirectory`, covering concurrent atomic metadata visibility, missing-file error variants, in-process writer exclusion, backend-neutral open-handle deletion semantics, synchronous and - asynchronous boundary reads, write termination, `meta.json` watch notification, and + asynchronous boundary reads, write termination, content-correlated `meta.json` watch + notification, and `sync_directory`; logical read-call and requested-byte counters prove the instrumentation and a fixed baseline for the harness's own operations, while the Rocks adapter adds physical fetched/copied-byte accounting around real indexing and search; -7. negative controls proving the harness rejects always-successful locks, failed partial metadata, - and partial metadata exposed during a successful replacement; +7. negative controls proving the harness rejects always-successful locks, partial metadata exposed + and then restored by a failed replacement, and partial metadata exposed during a successful + replacement; 8. a smoke test installed from `npm pack` output rather than the repository tree, proving the exports map, packaged files, addon resolution, stable errors, and platform artifact together; 9. package-content and loaded-addon inspection proving private generated bindings, unintended diff --git a/package.json b/package.json index 8730779..9ff9e4a 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "prepack": "npm run build:native && npm run build:typescript && npm run verify:package-artifacts", "test": "cargo test --locked --all-features && npm run test:node", "test:rust": "cargo test --locked --all-features", - "test:node": "npm run build:test-native && npm run build:typescript && node scripts/run-node-tests.mjs test && npm run build:native && npm run build:typescript && node scripts/run-node-tests.mjs release", + "test:node": "npm run build:test-native && npm run build:typescript && node scripts/run-node-tests.mjs test && node scripts/run-node-tests.mjs release", "verify:package-artifacts": "node scripts/verify-package-artifacts.mjs" }, "engines": { diff --git a/src/directory_harness.rs b/src/directory_harness.rs index a875d3a..6d33492 100644 --- a/src/directory_harness.rs +++ b/src/directory_harness.rs @@ -151,12 +151,12 @@ where D: Directory + Clone, { verify_concurrent_atomic_visibility(&directory)?; + verify_watch(&directory)?; let directory = InstrumentedDirectory::new(directory); verify_missing_file_errors(&directory)?; verify_write_read_delete(&directory)?; verify_atomic_metadata(&directory)?; verify_in_process_writer_exclusion(&directory)?; - verify_watch(&directory)?; directory.sync_directory().map_err(|error| error.to_string())?; Ok(directory.logical_read_metrics()) } @@ -302,15 +302,48 @@ fn block_on(future: F) -> F::Output { } } -pub fn verify_failed_atomic_replacement( - directory: &dyn Directory, +pub fn verify_failed_atomic_replacement( + directory: &D, replacement: impl FnOnce() -> io::Result<()>, -) -> Result<(), String> { +) -> Result<(), String> +where + D: Directory + Clone, +{ let path = Path::new("meta.json"); let before = directory.atomic_read(path).map_err(|error| error.to_string())?; - if replacement().is_ok() { + let observer_directory = directory.clone(); + let observer_before = before.clone(); + let (ready_sender, ready_receiver) = mpsc::sync_channel(0); + let (stop_sender, stop_receiver) = mpsc::channel(); + let observer = thread::spawn(move || -> Result<(), String> { + ready_sender.send(()).map_err(|error| error.to_string())?; + loop { + match stop_receiver.try_recv() { + Ok(()) | Err(mpsc::TryRecvError::Disconnected) => break, + Err(mpsc::TryRecvError::Empty) => {} + } + let observed = observer_directory + .atomic_read(path) + .map_err(|error| error.to_string())?; + if observed != observer_before { + return Err("failed metadata replacement became observable".to_owned()); + } + thread::yield_now(); + } + Ok(()) + }); + ready_receiver + .recv_timeout(Duration::from_secs(2)) + .map_err(|error| error.to_string())?; + let replacement_result = replacement(); + let _ = stop_sender.send(()); + let observer_result = observer + .join() + .map_err(|_| "failed metadata observer panicked".to_owned())?; + if replacement_result.is_ok() { return Err("fault injection did not fail the metadata replacement".to_owned()); } + observer_result?; let after = directory.atomic_read(path).map_err(|error| error.to_string())?; if before != after { return Err("failed metadata replacement became observable".to_owned()); @@ -356,19 +389,30 @@ where Ok(()) } -fn verify_watch(directory: &dyn Directory) -> Result<(), String> { +fn verify_watch(directory: &D) -> Result<(), String> +where + D: Directory + Clone, +{ let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let watched_directory = directory.clone(); let _handle = directory .watch(WatchCallback::new(move || { - let _ = sender.try_send(()); + let observed = watched_directory.atomic_read(Path::new("meta.json")).ok(); + let _ = sender.try_send(observed); })) .map_err(|error| error.to_string())?; directory .atomic_write(Path::new("meta.json"), b"watched") .map_err(|error| error.to_string())?; - receiver - .recv_timeout(Duration::from_secs(2)) - .map_err(|_| "meta.json watch did not fire".to_owned()) + let deadline = std::time::Instant::now() + Duration::from_secs(2); + loop { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + match receiver.recv_timeout(remaining) { + Ok(Some(bytes)) if bytes == b"watched" => return Ok(()), + Ok(_) => {} + Err(_) => return Err("meta.json watch did not observe the tested write".to_owned()), + } + } } #[cfg(test)] @@ -479,10 +523,23 @@ mod tests { .insert(path.to_path_buf(), data.to_vec()); return Ok(()); } + let before = self.atomic_files.read().unwrap().get(path).cloned().unwrap_or_default(); + self.non_atomic_write_active.store(true, Ordering::Release); + self.partial_write_observed.store(false, Ordering::Release); self.atomic_files .write() .unwrap() .insert(path.to_path_buf(), data[..data.len() / 2].to_vec()); + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !self.partial_write_observed.load(Ordering::Acquire) { + if std::time::Instant::now() >= deadline { + self.non_atomic_write_active.store(false, Ordering::Release); + return Err(io::Error::other("observer did not inspect failed partial metadata")); + } + thread::yield_now(); + } + self.atomic_files.write().unwrap().insert(path.to_path_buf(), before); + self.non_atomic_write_active.store(false, Ordering::Release); Err(io::Error::other("injected partial write")) } @@ -519,12 +576,15 @@ mod tests { #[test] fn harness_rejects_visible_partial_metadata() { let directory = BrokenDirectory::new(false); - directory.atomic_write(Path::new("meta.json"), b"stable").unwrap(); + let before = vec![b'a'; 4096]; + let after = vec![b'b'; 4096]; + directory.atomic_write(Path::new("meta.json"), &before).unwrap(); directory.fail_next_atomic_write(); - assert!(verify_failed_atomic_replacement(&directory, || { - directory.atomic_write(Path::new("meta.json"), b"replacement") - }) - .is_err()); + assert_eq!( + verify_failed_atomic_replacement(&directory, || directory.atomic_write(Path::new("meta.json"), &after)) + .unwrap_err(), + "failed metadata replacement became observable" + ); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 127f130..3fad0cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,13 +72,13 @@ pub fn test_check(id: u32) -> boundary::Result { #[cfg(feature = "test-panic")] fn test_handle(id: u32) -> boundary::Result> { - boundary::run_stateless(|| { + let handle = boundary::run_stateless(|| { TEST_HANDLES .get_or_init(Default::default) .lock() .unwrap() .get(&id) .cloned() - .expect("unknown test handle") - }) + })?; + handle.ok_or_else(|| napi::Error::new("E_NATIVE_FAILURE", "unknown test handle")) } diff --git a/test/native.test.mjs b/test/native.test.mjs index adb3d74..1878c76 100644 --- a/test/native.test.mjs +++ b/test/native.test.mjs @@ -1,18 +1,25 @@ import assert from 'node:assert'; +import { readFileSync } from 'node:fs'; import test from 'node:test'; import { runtimeInfo } from '@harperfast/fulltext/native'; import { normalizeNativeError } from '../dist/errors.js'; import { loadAddon, platformTriple } from '../dist/load-addon.js'; +const packageManifest = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const cargoManifest = readFileSync(new URL('../Cargo.toml', import.meta.url), 'utf8'); +const cargoPackageVersion = /^version\s*=\s*"([^"]+)"/m.exec(tomlSection(cargoManifest, 'package'))?.[1]; +const tantivyVersion = /^tantivy\s*=\s*"=([^"]+)"/m.exec(tomlSection(cargoManifest, 'dependencies'))?.[1]; + test('loads the artifact for the executing platform', async () => { const info = await runtimeInfo(); assert.deepStrictEqual(info, { - packageVersion: '0.0.0', - tantivyVersion: '0.26.1', + packageVersion: packageManifest.version, + tantivyVersion, nativeAbiVersion: 1, storageBackends: ['native'], }); + assert.strictEqual(cargoPackageVersion, packageManifest.version); assert.match(platformTriple(), /^(darwin|linux|win32)-(arm64|x64)(-(gnu|musl|msvc))?$/); }); @@ -21,6 +28,11 @@ test('turns a panic into a coded terminal error', async () => { assert(addon.__testCreateHandle && addon.__testPanic && addon.__testCheck); const firstHandle = addon.__testCreateHandle(); const secondHandle = addon.__testCreateHandle(); + assert.throws( + () => addon.__testCheck(0), + (error) => normalizeNativeError(error).code === 'E_NATIVE_FAILURE', + ); + assert.strictEqual(addon.__testCheck(secondHandle), true); assert.throws( () => addon.__testPanic(firstHandle), (error) => normalizeNativeError(error).code === 'E_NATIVE_PANIC', @@ -32,3 +44,11 @@ test('turns a panic into a coded terminal error', async () => { assert.strictEqual(addon.__testCheck(secondHandle), true); await assert.doesNotReject(runtimeInfo()); }); + +function tomlSection(manifest, name) { + const sectionStart = manifest.indexOf(`[${name}]`); + assert.notStrictEqual(sectionStart, -1, `Cargo.toml is missing [${name}]`); + const bodyStart = sectionStart + name.length + 2; + const nextSection = manifest.indexOf('\n[', bodyStart); + return manifest.slice(bodyStart, nextSection === -1 ? undefined : nextSection); +} From 28f7adee22ae038f037ac39230378b8534ff281c Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sat, 5 Sep 2026 00:46:36 -0600 Subject: [PATCH 7/9] test: avoid reentrant watch reads --- src/directory_harness.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/directory_harness.rs b/src/directory_harness.rs index 6d33492..4b3d200 100644 --- a/src/directory_harness.rs +++ b/src/directory_harness.rs @@ -4,7 +4,7 @@ use std::io::{self, Write}; use std::ops::Range; use std::path::Path; use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{mpsc, Arc}; use std::task::{Context, Poll, Wake, Waker}; use std::thread; @@ -389,18 +389,18 @@ where Ok(()) } -fn verify_watch(directory: &D) -> Result<(), String> -where - D: Directory + Clone, -{ +fn verify_watch(directory: &dyn Directory) -> Result<(), String> { let (sender, receiver) = std::sync::mpsc::sync_channel(1); - let watched_directory = directory.clone(); + let armed = Arc::new(AtomicBool::new(false)); + let callback_armed = armed.clone(); let _handle = directory .watch(WatchCallback::new(move || { - let observed = watched_directory.atomic_read(Path::new("meta.json")).ok(); - let _ = sender.try_send(observed); + if callback_armed.load(Ordering::Acquire) { + let _ = sender.try_send(()); + } })) .map_err(|error| error.to_string())?; + armed.store(true, Ordering::Release); directory .atomic_write(Path::new("meta.json"), b"watched") .map_err(|error| error.to_string())?; @@ -408,8 +408,14 @@ where loop { let remaining = deadline.saturating_duration_since(std::time::Instant::now()); match receiver.recv_timeout(remaining) { - Ok(Some(bytes)) if bytes == b"watched" => return Ok(()), - Ok(_) => {} + Ok(()) => { + let observed = directory + .atomic_read(Path::new("meta.json")) + .map_err(|error| error.to_string())?; + if observed == b"watched" { + return Ok(()); + } + } Err(_) => return Err("meta.json watch did not observe the tested write".to_owned()), } } From 7ebadec6047b8e4a437ed203939404f2868da0c7 Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sat, 5 Sep 2026 12:44:55 -0600 Subject: [PATCH 8/9] fix: normalize text files for Windows CI --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf From fd89c05bde04d4170b2894b9820ee93379de542c Mon Sep 17 00:00:00 2001 From: Kyle Bernhardy Date: Sat, 5 Sep 2026 12:56:30 -0600 Subject: [PATCH 9/9] fix: harden cross-platform qualification --- .github/workflows/ci.yml | 2 +- src/directory_harness.rs | 16 ++++++++-------- test/package.release.test.mjs | 8 +++----- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6a6d48..a26d39a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: node: '22.18.0' - os: ubuntu-latest node: '24' - - os: macos-14 + - os: macos-15 node: '24' - os: windows-latest node: '24' diff --git a/src/directory_harness.rs b/src/directory_harness.rs index 4b3d200..afeb324 100644 --- a/src/directory_harness.rs +++ b/src/directory_harness.rs @@ -242,7 +242,6 @@ where let observer_before = before.to_vec(); let observer_after = after.to_vec(); let (ready_sender, ready_receiver) = mpsc::sync_channel(0); - let (stop_sender, stop_receiver) = mpsc::channel(); let observer = thread::spawn(move || -> Result<(), String> { let initial = observer_directory .atomic_read(path) @@ -251,26 +250,27 @@ where return Err("atomic metadata observer did not read the initial value".to_owned()); } ready_sender.send(()).map_err(|error| error.to_string())?; + let deadline = std::time::Instant::now() + Duration::from_secs(2); loop { - match stop_receiver.try_recv() { - Ok(()) | Err(mpsc::TryRecvError::Disconnected) => break, - Err(mpsc::TryRecvError::Empty) => {} - } let observed = observer_directory .atomic_read(path) .map_err(|error| error.to_string())?; - if observed != observer_before && observed != observer_after { + if observed == observer_after { + return Ok(()); + } + if observed != observer_before { return Err("atomic metadata observer saw a partial value".to_owned()); } + if std::time::Instant::now() >= deadline { + return Err("atomic metadata observer did not observe the replacement".to_owned()); + } thread::yield_now(); } - Ok(()) }); ready_receiver .recv_timeout(Duration::from_secs(2)) .map_err(|error| error.to_string())?; let write_result = directory.atomic_write(path, after).map_err(|error| error.to_string()); - let _ = stop_sender.send(()); let observer_result = observer .join() .map_err(|_| "atomic metadata observer panicked".to_owned())?; diff --git a/test/package.release.test.mjs b/test/package.release.test.mjs index 7298ed4..aff2f03 100644 --- a/test/package.release.test.mjs +++ b/test/package.release.test.mjs @@ -1,7 +1,6 @@ import assert from 'node:assert'; import { execFileSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import path from 'node:path'; import test from 'node:test'; @@ -62,15 +61,14 @@ test('the packed package loads without consumer lifecycle scripts', (context) => assert(!installedManifest.scripts?.[lifecycle], `${lifecycle} must not run in a consumer installation`); } const artifact = includedPaths.find((file) => /^fulltext\..+\.node$/.test(file)); - const require = createRequire(import.meta.url); - const installedAddon = require(path.join(projectDirectory, 'node_modules/@harperfast/fulltext', artifact)); - assert(!('__testCreateHandle' in installedAddon)); + const installedAddonPath = path.join(projectDirectory, 'node_modules/@harperfast/fulltext', artifact); const output = execFileSync( process.execPath, [ '--input-type=module', '--eval', - "import('@harperfast/fulltext/native').then(x => x.runtimeInfo()).then(console.log)", + "import { createRequire } from 'node:module'; const addon = createRequire(import.meta.url)(process.argv[1]); if ('__testCreateHandle' in addon) process.exit(1); console.log(await import('@harperfast/fulltext/native').then(x => x.runtimeInfo()));", + installedAddonPath, ], { cwd: projectDirectory, encoding: 'utf8' }, );