diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 92e4af7325..e428c2d6d4 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -133,3 +133,47 @@ jobs: - name: Run unit tests timeout-minutes: 30 run: npm run test:unit:windows + + # The vectorIndexPlane suite self-skips when the optional native package is absent, so the + # load probe below is what keeps that skip from reading as a green run. The crate's own + # tests belong to HarperFast/hnsw's CI, not here: this job runs the published prebuild, so + # testing the crate source would grade something other than the binary under test. + hnsw-plane: + name: HNSW native plane (Node.js v24) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout code + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - name: Setup Node.js 24 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: 24 + package-manager-cache: false + + - name: Install dependencies + run: npm install --ignore-scripts + + # strict here, unlike the other jobs: the plane suite runs against dist/, so a tolerated + # build failure would grade a stale dist and still report green + - name: Build + run: npm run build + + - name: Verify the published native binding loads + run: node -e "if (typeof require('@harperfast/hnsw').Plane?.open !== 'function') throw new Error('@harperfast/hnsw loaded without a Plane constructor');" + + - name: Setup Harper + env: + DEFAULTS_MODE: 'dev' + HDB_ADMIN_USERNAME: 'admin' + HDB_ADMIN_PASSWORD: 'password' + ROOTPATH: '/tmp/hdb' + NODE_HOSTNAME: 'localhost' + LOGGING_LEVEL: 'info' + run: node --enable-source-maps ./dist/bin/harper.js install + + - name: File-primary native plane tests + env: + HNSW_NATIVE_REBUILD_BENCHMARK: '1' + run: npx mocha unitTests/resources/vectorIndexPlane.test.js diff --git a/dependencies.md b/dependencies.md index 566ef59c29..3156e697ad 100644 --- a/dependencies.md +++ b/dependencies.md @@ -243,3 +243,15 @@ This is the inverse of the entries below — a dependency we take deliberate ste - Security: Microsoft-maintained TypeScript compiler, same publisher/package as the 5.x devDependency. - Overlap: Complements, does not replace, the `typescript` devDependency — TypeScript 7.0 ships no compiler API yet (planned for 7.1), so `@typescript-eslint/parser` still needs `typescript` 5.x. The 5.x and 7.x versions never coexist in `node_modules` at once: 5.x is the installed devDependency, 7.x is fetched on-demand by `npx` purely for `typecheck:fast`. - Eventual removal: Once TypeScript 7 stabilizes as the primary `typescript` devDependency (post-7.1's compiler API), this becomes redundant and `typecheck:fast` can be dropped. + +## @harperfast/hnsw (optional dependency) + +- Need for usage: Supplies the file-primary memory-mapped HNSW index used only by indexes that opt in with `nativePlane: true`. Harper keeps primary-key mappings and replay cursors in RocksDB; graph nodes and adjacency exist only in the native file. +- Size/memory cost: The JS/package metadata is about 250 KB unpacked plus one platform-specific native binary. Runtime mapped-file size is approximately 1,344 bytes per 768-dimensional int8 node at layer-0 cap 128; mappings are shared by the OS page cache across workers. +- Security: First-party Apache-2.0 Harper package. It runs native code in-process, so Harper exact-pins the package and its own manifest exact-pins every platform prebuild to the same version. The dedicated CI job loads the registry prebuild and tests Harper against it. +- Environment interaction: Lazily loaded only when an eligible index enables `nativePlane`; it creates a memory-mapped `.hnsw` derived-index file next to the index store and may create a `.stale` invalidation sidecar. It does not modify globals or install polyfills. +- Overlap: None for an opted-in index. Ordinary HNSW indexes continue to use the JS/RocksDB graph; `nativePlane` indexes use native insert, mutation, and search through the shared post-commit derived-index runtime. +- Transitive dependencies: Only exact-version, platform-specific optional prebuild packages; no JS runtime dependency tree. +- Binary compilation: Supported Linux glibc x64/arm64, macOS arm64, and Windows x64 targets use prebuilds. Other targets attempt a Rust source build. Because the root package is optional Harper still installs if that build fails, but opted-in indexes remain unavailable until the module is present. +- Can be deferred: Yes for ordinary HNSW indexes. The adapter loads lazily; an opted-in index returns 503 and retries rebuild when the native module is unavailable. +- Eventual removal: Disable `nativePlane`, allow the schema reindex to rebuild the ordinary JS/CF graph, then remove the optional dependency and adapter integration. diff --git a/hnsw-native-plane.md b/hnsw-native-plane.md new file mode 100644 index 0000000000..d69667cbbd --- /dev/null +++ b/hnsw-native-plane.md @@ -0,0 +1,807 @@ +# HNSW native traversal plane + +Design for moving HNSW graph storage and traversal into a native (Rust/napi-rs) module over a +memory-mapped fixed-slot file, replacing the RocksDB index column family as the home of graph +nodes. Companion to the scaling analysis in `DESIGN.md` ("efConstruction and the search-ef +ceiling both auto-scale with the graph") and issues #693, #711, #895, #2182. + +## 1. Motivation — measured, not estimated + +Per-visit cost decomposition at 5M nodes / ef 512 (768-d int8, `benchmarks/hnsw-scale.js` +corpus, 22.18 ms p50 / 5,107 visits): + +| Component | Cost | Share of a warm visit | +| ------------------------------------------- | ------- | --------------------- | +| Total per visited node | 4.34 µs | 100% | +| int8 asymmetric cosine, 768-d, JS | 0.43 µs | 10% | +| msgpackr decode of one node (VT-cache miss) | 5.57 µs | +128% when cold | +| Neighbour iteration + visited-set ops | 0.21 µs | 5% | + +~85% of a warm visit is JS object bookkeeping — candidate heap, visited `Set`, property access, +allocation, GC — not distance math and not I/O. Three consequences: + +1. **A native distance kernel is worth ~nothing.** Distance is 10% of the visit; a NAPI crossing + costs 0.1–0.5 µs. The win requires the whole search loop native, over a native data layout, + with one boundary crossing per query. +2. **The fetch path decides the ceiling.** A warm RocksDB `Get` is ~1–2 µs even called natively + (block-cache lookup, block parse, value memcpy) — 20–40× the SIMD distance it feeds. Direct + slot addressing (`base + id × SLOT_SIZE`) into a resident mapping is ~100–200 ns. Traversal + over RocksDB caps at ~3–5× improvement; traversal over a fixed-slot mapping reaches the + full ceiling. +3. **Estimated native budget: ~0.25–0.4 µs/visit** (SIMD int8 dot ~50 ns + streaming 768 + contiguous bytes ~150 ns + bitset/heap ops ~50 ns) → **~10–15× on the search path** + (22 ms → ~1.5–2 ms at 5M/ef 512), with the JS event loop untouched. + +This is also the enabling dependency for same-node index slicing (parallel slice searches need +off-loop execution) and changes cluster QPS arithmetic by the same factor. + +## 2. Goals / non-goals + +Goals: + +- Search traversal fully native, off the JS event loop, one NAPI crossing per query. +- Graph nodes in a memory-mapped fixed-slot file — **the file is the index**: the maintained + primary of the derived data, updated in place on every commit, not a cache of RocksDB. +- Incremental maintenance preserved: insert/update/delete keep working exactly as today from + the application's view. +- Relaxed transactional adherence (deliberate): HNSW results are approximate by contract, and + the existing post-load exact rescore + MVCC record lookup already filter stale/wrong + candidates. No cross-slot atomicity. +- Node-id reuse via an in-file freelist — structurally fixes the #2182 lifetime high-water + ef over-provisioning. +- Slicing-ready: one file per slice; native merge of per-slice top-k (C2 hook). + +Non-goals (this phase): + +- Binary quantization / Matryoshka truncation (benchmark-gated per the Reflex study; the format + reserves a quantization-mode field so a binary plane is a format v2, not a redesign). +- Native batch insertion; version 0.2.1 supplies the single-record native insert used here. +- Cross-node ANN protocol. Out of scope entirely. +- Lexical/BM25 anything. + +## 3. Architecture + +``` + JS (worker threads) native (Rust, napi-rs) + ┌─────────────────────────────────────────┐ ┌─────────────────────────────────────┐ + │ DerivedIndexBackend + HNSW adapter │ │ @harperfast/hnsw 0.2.1 │ + │ • pk↔nodeId mapping (RocksDB) │ │ • mmap'd graph file (per index) │ + │ • per-origin audit cursor (RocksDB) │ │ • insert/remove (seqlocked) │ + │ • committed-log replay + rebuild ├──►│ • search(query, k, ef, filter) → │ + │ • record load + exact rescore │◄──┤ top-k ids, libuv worker pool │ + │ • bounded keyed wake-up queues │ │ • TSFN batch filter callback │ + └─────────────────────────────────────────┘ └─────────────────────────────────────┘ +``` + +What stays in RocksDB: the primary records, pk↔nodeId mappings, durable replay cursors, and all +other indexes. The mappings are post-commit derived state and are recoverable from records plus +the retained transaction log. Node vectors, per-layer adjacency, entry point, id allocator, and +freelist live only in the native file. + +## 4. File format (v1) + +One file per index (per slice, once C2 lands): `.hnsw`. + +**Header (4 KB page):** + +| Field | Type | Notes | +| --------------------------------- | ---------- | ------------------------------------------------------------ | +| magic + format version | u32 + u32 | rebuild required on version mismatch (accepted contract) | +| dims, quantization mode | u16 + u8 | v1: int8 asymmetric; f32 supported for `quantization:"none"` | +| slot_size, layer0_cap, upper_cap | u16 ×3 | derived from M/optimizeRouting at creation | +| entry_point_id, entry_point_level | u32 + u8 | atomically updated | +| id_high_water | u64 atomic | replaces the shared Atomics BigInt64Array incrementer | +| freelist_head | u64 atomic | CAS push/pop; ABA-guarded with a 32-bit tag | +| txn_watermark | u64 | last durably indexed transaction; advanced by msync cadence | +| clean_shutdown flag | u8 | torn-state detection on open | + +**Main region — layer-0 slots**, addressed `4096 + id × slot_size`: + +| Field | Size (768-d int8, cap 64) | +| ------------------------------- | ----------------------------------- | +| seq (seqlock) | 4 B | +| flags (valid/deleted) + level | 2 B | +| scale (f32) + invMag (f32) | 8 B | +| degree | 2 B | +| vector (int8 × 768) | 768 B (padded to a 4-byte boundary) | +| neighbor ids (u32 × layer0_cap) | 256 B | +| **total, padded** | **1,040 B → 1 KB-aligned 1,088 B** | + +The vector's trailing pad keeps the neighbor array 4-aligned for every `dims`, so the search +hot path reads each neighbor id as one aligned volatile `u32`. Upper-layer id lists are padded +the same way (`degree u16 + pad u16 + ids`). + +At 100M nodes: ~109 GB (int8). A binary-code v2 slot (96 B codes + ids) is ~384 B → ~38 GB. +For comparison, today's encoding averages 1,425 B/node _plus_ RocksDB overhead — so v1 is +already ~25% smaller while being fixed-offset addressable, because per-edge cached float64 +distances are dropped (recomputing a distance costs ~50 ns native; storing it costs 8 B and +~40% of today's node bytes). + +**Upper-layer region** (append-allocated, compacted on rebuild): only ~6% of nodes have +level > 0, and upper layers hold neighbor id lists only (vectors live in the main slot). Each +entry: `node_id, level, [degree, ids × upper_cap] × level`. Kept fully resident; a few hundred +MB at 100M nodes. + +**Degree cap decision.** Today layer-0 caps at `M<<1` then `<<2` under `optimizeRouting` = 128, +with transient overshoot to 160 before pruning; measured mean degree is ~37. Sizing slots at +cap 128 doubles the file for a tail. v1 policy: **hard prune-to-cap-64 on write** — the insert +path's in-memory candidate selection can overshoot as today, but what is written is pruned to +64 by the same routing-aware selection that currently prunes at 160→128. Transient overshoot +never touches the file. Recall impact must be measured in the validation phase (§9); the cap is +a header field, so revising it is a rebuild, not a format change. + +## 5. Concurrency + +- **Per-slot lock with owner identity.** The lock word is a u32: bit 31 = locked, low bits = + the owner's pid; unlocked values are generations, validated seqlock-style by readers. A lock + whose value stays unchanged for a 20 ms window AND whose owner pid is dead (ESRCH) is taken + over by the waiter, which SANITIZES the slot (marks it invalid — a dead writer's payload is + half-written; invisible-until-rewritten, never spliced-but-valid). Elapsed time alone never + robs a lock: a live writer descheduled by CFS throttling or a page-fault storm keeps its + lock until rescheduled. On platforms without a liveness check, readers degrade to + treat-as-absent after the window and writers wait. +- **No cross-slot atomicity.** An insert updates the new node's slot plus ~M neighbors' + back-edge lists, each independently. A traversal may observe the half-linked state: an edge + to a slot whose valid flag is not yet set → skip (HNSW tolerates missing edges); a + just-deleted neighbor → skip via flags. Wrong-candidate leakage is filtered by the existing + exact rescore + MVCC record load, which is why relaxed adherence is safe _here_ and not a + general storage pattern. +- **Writers.** Multiple worker threads insert concurrently today (distinct records); the same + holds: id allocation is one atomic fetch_add on the header, freelist pop is CAS, slot writes + are seqlocked. Two inserts updating the same neighbor's edge list serialize on that slot's + seqlock (a Rust-side per-slot spinlock on the odd state). +- **Id reuse & ABA.** Delete pushes the id onto the freelist; a traversal holding the old id may + read the reused slot and score the wrong vector — acceptable under the relaxed contract + (rescore/record-load rejects it). The freelist head itself is tag-guarded against ABA. + +## 6. Durability & crash recovery + +The file is `msync`'d on a cadence (default: every N seconds or M mutated slots, configurable), +**not** per commit. The header watermark records the last transaction whose index mutations are +known durable; it advances only after a completed msync barrier. + +On open: + +- Clean-shutdown flag set → map and serve. +- Torn state → replay records from `txn_watermark` through the existing `runIndexing` re-feed + path (which already treats a re-fed already-indexed record as an update — the exact semantics + needed). This anchors today's heuristic crash re-feed to a precise watermark. +- Format-version mismatch or corruption (header checksum) → full rebuild from records. Explicit + contract: **format upgrades require reindex** (accepted). + +Note the asymmetry with today: RocksDB gave the graph per-commit durability; the file gives it +bounded-lag durability with deterministic catch-up. For an approximate index whose source of +truth (records + pk→nodeId) remains fully transactional, bounded lag is the right trade — it +buys the entire performance model. + +**Backup/copy-db/reseed:** the file is node-local derived state. Backup either includes it +(consistent-enough after an msync barrier) or marks the index rebuild-on-restore. Replica +reseed = rebuild from records (C5 bulk construction makes this fast; until then, the existing +per-row path). + +## 7. Search path & NAPI surface + +```ts +// one crossing per query; executes on the module's own thread pool +search(sliceHandles, queryVector: Float32Array, k, ef, filter?): Promise<{ids, distances}> +``` + +- Asymmetric distance as today: float query × int8 stored, cached invMag, SIMD (AVX2/VNNI on + x86, NEON on ARM; `std::arch` intrinsics with a scalar fallback). +- Visited set: epoch-stamped u32 array (one per pool thread, reused across queries — no + allocation per query). Candidate heap: fixed-capacity binary heap of (dist, id) pairs. +- Auto-ef / auto-efC read the node count from the header high-water minus freelist length — + same semantics as today, minus the #2182 inflation (freed ids return to the pool). + +**Filtering** (predicate-aware / ACORN, `filteredSearch = true` today): + +1. **Bitset fast path.** RBAC allow-lists and companion-condition candidate sets are computed + before the query and passed as a roaring/plain bitset over node ids. Zero callbacks. This + covers the dominant production filter shapes. +2. **Pipelined TSFN batch path** for arbitrary JS predicates. Traversal batches candidate ids + (64–256) through a ThreadsafeFunction to a JS evaluator and **continues expanding in + distance order while verdicts are in flight**; verdicts merge in to steer selection and + gate results. The existing `filterExpansion` visit budget bounds speculative overshoot. + Traversal never blocks on the event loop — that would re-import the p99 problem this + design exists to remove. Worst case (loop saturated): budget exhausts, return what passed — + the same contract as today's budget-bound filtered search. +3. TSFN lifecycle: shutdown-while-query-in-flight is a first-class test (see rocksdb-js #665's + TSFN teardown SIGSEGV). napi-rs `ThreadsafeFunction` + explicit abort on env teardown. + +## 8. Write path phasing + +- **Phase 1 prototype — dual-write, search cutover (superseded before merge).** Insert/update/delete logic stayed in JS + (`HierarchicalNavigableSmallWorld.ts` unchanged algorithmically); mutations persist to BOTH + the index CF (as today) and the file via native slot-write calls. Search runs native from the + file. Validation = compare native results against the JS path on the same graph; rollback = + flip search back to JS, drop the file. The double-write cost is bounded (index writes are + a fraction of insert cost) and temporary. + + This implementation was removed when phase 2 landed on the same draft PR. It never shipped, so + the PR has no deployed dual-write format to preserve or migrate in place. + +- **Phase 2 — shared post-commit delivery, file-primary (current implementation).** Implement #2489's + `DerivedIndexBackend` runtime rather than an HNSW-specific commit callback. This remains opt-in + behind `nativePlane: true`; ordinary HNSW indexes keep the RocksDB graph and do not acquire an + audit dependency. A table with an opted-in backend must explicitly declare `audit: true`; inheriting + either true or false from the global setting is rejected so a vector-index option cannot silently + expand the audit-readable security surface. Auditing adds a durable full-record audit entry to + every commit on the table, including commits that do not change the vector, for the configured + retention window. This storage and data-retention cost is part of opting in. + Enablement warns that the audit API can now expose full table history for that window. + + The pre-commit index hook compares the old and new projection, validates it, and stages only the + changed backend target on the transaction; unrelated field changes never enqueue an HNSW update. + Direct attributes use reference/length equality first and compare numeric components only when + identities differ; resolver-backed projections necessarily pay the component comparison. + Pending work is coalesced by `(index, primaryKey)`: rapid updates replace that key's queued target + with one reconciliation against current primary state. The runtime retains the log-position tickets + that caused each wake-up, while replay advances each origin cursor through the audit log only after + the native durability barrier. Before commit it checks the aggregate distinct-key and position-ticket + depth already published by every worker. A bounded number of simultaneously committing transactions + can pass the same check, so the configured limit is a scheduling threshold rather than a strict memory + ceiling. Once the observed depth reaches either threshold, a new vector-changing write receives a + retryable 503 until the backend catches up; unrelated writes continue. This admission control is + required because accepting unique-key load above native insert throughput and then rebuilding at + that same throughput cannot converge. + The initial limit is 65,536 changed records or 262,144 queued log-position tickets (approximately + 16 MiB of identifiers/metadata), whichever comes first; current depth and a process-wide rejection + count are logged at exponentially spaced rejection counts so the fixed limit can be tuned without + flooding logs. + + `RocksTransactionLogStore.aftercommit` only schedules the staged targets and returns. Each worker + has a queue, but one cross-thread per-index writer lock serializes complete batches, and each queue + is FIFO, so two updates to one key cannot apply in reverse order. The stored mapping also carries + the last reconciled primary-record version and discards an older observation. If applying or + flushing an already-committed entry fails, it drops the complete in-flight batch and memory queue, + leaves the durable cursor before the failed entry, logs the failure with index identity, marks the + index unavailable/rebuild-required, and rebuilds from current + records. It cannot crash the record writer, and no rejected promise escapes the detached drain. + Persistent rebuild failures retry with capped exponential backoff (1 second through 5 minutes), + keep search unavailable, and emit one error per failed retry rather than spinning. + + The backend advances a durable cursor for each origin log only after its own durability barrier. + On open, a cursor older than `logging.auditRetention` causes a full rebuild; replay never advances + across a retained gap or a corrupt frame. A whole-table reload marker (used by replica snapshot copy) + also forces a rebuild because its copied rows deliberately have no individual audit entries. The + default retention means a node unavailable beyond that window takes the + measured rebuild path and returns 503 for its duration. Operators can size retention for the + expected outage, but correctness does not depend on doing so: expiry changes recovery cost, not + outcome. During replay or rebuild, + searches return the existing index-in-progress 503 rather than reading a partial file. Cleanup + may delete old segments; the recovery contract is rebuild rather than pinning audit indefinitely. + + HNSW uses the package's standalone `insert`/`remove` API. The RocksDB index CF retains only + `primaryKey ↔ nativeNodeId` identity and per-origin replay cursors; graph nodes and adjacency + exist only in the mmap file. A changed mapping is marked pending, hidden from search, and published + only after the mmap flush succeeds; the cursor advances after the mapping publication. A crash can + therefore leave replay extra work, but cannot leave a published mapping to an undurable node. + Hot delivery and replay both re-read the current authoritative record + after commit. This gives multi-origin/source-resolution writes the same reconciliation rule and + makes an old entry idempotent as "delete current native id, then add the current value". The + audit object's in-memory record is an allowed optimization only when its version is still the + primary store's current version. Rebuild records the oldest retained physical entry in each log, + scans current records into a fresh file, and replays from those entries before the index becomes + queryable. This avoids both a timestamp race with already-open transactions and a synchronous scan + to discover the log tail. Existing phase-1 graph CFs are migrated by this rebuild. + + The current package fixes standalone construction at M=16, efConstruction=200, mL=1/ln(16), + and optimizeRouting=0.5, so file-primary mode accepts only that geometry. Its sparse reservation + is fixed at create time; `nativePlaneMaxNodes` is therefore a structural option (16M default), + and exhaustion makes the index unavailable until it is enlarged and rebuilt. The native file + is an approximate derived index: concurrent CRDT/source-resolution arrivals are not promised to + reproduce a single total order. Exact record load and rescore still reject stale candidates. + A crash between native allocation and identity persistence can leave an unreachable native node; + replay restores the live record and exact filtering hides the orphan, while rebuild reclaims it. + A schema requesting `nativePlane: true` with non-native construction geometry is rejected rather + than silently changing M/efConstruction/mL/optimizeRouting during upgrade. + + Phase 1 has not shipped: this PR is draft, so no deployed index is silently migrated from its + configurable JS geometry. Version 0.2.1 already implements the insertion search and graph mutation + inside the native `insert()` call; phase 2 uses that path for rebuild as well as incremental writes, + rather than the ~263 inserts/s JS anchor. The native CI job runs a gated 100k-record rebuild-insertion + benchmark, publishes progress (records, rate, ETA), and requires at least 1,000 inserts/s. The local + verification for this revision sustained 4,625 inserts/s at 100k including mapping writes and flush + barriers. That gate is a regression floor at 100k, not a rebuild-duration guarantee: insert rate falls + as the graph grows (8,482/s at 30k → 4,625/s at 100k here; §12's crate anchor is 1,242/s at 1M), and + nothing above 1M is measured. A 16M rebuild is therefore at least ~3.6 hours at the 1M rate and in + practice longer, so a deployment sizing above the default reservation is accepting a 503 recovery + window of that order until a batch API exists. + Worker shutdown is graceful and waits for a synchronous N-API insert to return; a worker is never + force-terminated in the middle of a plane mutation while the process survives. + +- **Phase 3 — native batch insert.** Add a bulk API so rebuild can cross N-API once per chunk and + report progress while native code owns the insertion loop. Single-record insertion search and + graph mutation already run natively in 0.2.1. + +### What `nativePlane: true` requires, and what it does not promise + +Removing the RocksDB graph moves several phase-1 conveniences into hard requirements. All of them +are enforced or surfaced in code, not left as advice. + +| Requirement | Enforcement | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| The table declares `audit: true` explicitly. Inheriting either value from the global setting is rejected, so a vector-index option cannot silently widen the audit-readable surface. | `resources/databases.ts` `table()` and `attachDerivedIndexBackends()` both throw a `ClientError`; enabling one logs once that the audit API now retains full record history for the retention window. | +| The RocksDB storage engine. | `ClientError` at index construction. | +| `M=16`, `efConstruction=200`, `mL=1/ln(16)`, `optimizeRouting=0.5`, int8-quantized cosine. Standalone `insert` in 0.2.1 fixes this geometry, so a schema asking for another one is rejected rather than silently rebuilt under native defaults. | `ClientError` at index construction. | +| `nativePlaneMaxNodes` (16M default) is structural — the sparse reservation is fixed at file create. Exhausting it makes the index unavailable until the value is raised and the index rebuilt. | Reservation is a create-time header field; growth is a possible later enhancement (§10). | +| `@harperfast/hnsw` must load on the platform. There is no JS graph to fall back to, so an absent or failing native module means the index is unavailable, not degraded. | Search returns 503; delivery throws a 503 `ServerError`. Ordinary HNSW indexes are unaffected. | +| The audit log must retain entries back to each origin cursor. A cursor before retention, a corrupt frame, or a whole-table reload marker forces a full rebuild from current records. | Detected in `reconcile`/`replay`; the index reports 503 for the rebuild's duration. | +| Vector-changing writes are admissible only while the shared backlog is under its bound. | The pre-commit hook returns a retryable 503 once the aggregate pending-key or log-position depth reaches its threshold; unrelated writes continue. | + +Not promised: + +- **A single total order across concurrent CRDT or source-resolution arrivals.** Hot delivery and + replay both re-read the current authoritative record, so the index converges on whatever the + primary store resolved, but two origins landing concurrently are not promised to produce the + index state a single serial order would. This is an approximate nearest-neighbour index and the + exact record load plus rescore on the read path already rejects stale candidates, so the + divergence is bounded by candidate selection, not by returned data. +- **Byte-identical graphs across nodes or across rebuilds.** Insertion order and the concurrent + prune both affect edge selection; only recall is held to a baseline (§9). +- **In-place format upgrades.** A plane whose format version does not match is rejected at open + and reindexed (§4). + +### Approaches considered for phase 2 + +**Chosen: transaction log plus rebuild on retention gap.** Correctness: record and audit entry +commit atomically; post-commit work cannot leak an aborted write; per-origin cursors advance after +the plane flush; retained gaps force rebuild; replay reconciles against current records. Performance: +the record transaction adds no marker or mmap work, but opting in does add the required audit-log +write to every table commit; projection comparison prevents unrelated commits from reaching HNSW, +rapid same-key changes coalesce, and changed-vector writes are admitted only while the bounded queue +has capacity. The commit hook only schedules work and the backend batches its durability barrier. +Operational complexity: it reuses Harper's existing audit log, +aftercommit stream, retention setting, and rebuild path. Scope: the runtime and interface are shared +with future Tantivy work, while this PR supplies only the HNSW backend. + +**Rejected: transactional dirty-key outbox.** Correctness is attractive because a marker cannot be +lost to audit retention and current-state reconciliation is naturally idempotent. It adds a second +durable delivery fact beside the transaction log, however, plus marker fencing, reclamation, and a +write to every indexed record transaction. That contradicts #2489's single protocol and raises hot +write amplification for every backend. + +**Rejected: keep the phase-1 CF graph as authority.** This preserves rollback and package fallback +and makes native state disposable. It retains the duplicate graph writes and graph decode/storage +cost this phase exists to remove, so write throughput and storage continue to scale with two graphs. + +**Rejected: ship post-commit delivery first and retain the CF graph until a later cutover.** This is +the safer rollout when phase 1 is already deployed, but phase 1 exists only on this draft PR and has +no production population to protect. The 0.2.1 dependency already moved insertion native, and the +opt-in flag plus 503-during-build behavior remains the rollout gate. Shipping another temporary disk +format would add a migration and prolong the duplicate graph cost without gathering compatibility +evidence from any existing deployment. File-primary therefore lands before this PR's first merge. + +**Rejected: synchronous native mutation inside the record transaction.** This removes the CF graph +with little new runtime code. An aborted transaction can still publish an mmap mutation, and native +construction remains on request latency. Recovery then needs the same log protocol anyway. + +**Rejected: build on `transactionBroadcast`'s subscription registry.** That module supplies the +scheduling and transaction-grouping precedent, but its cursors are live-subscription state +(`lastTxnTime` and one shared thread-local position), not durable per derived index and origin. Its +same-thread Rocks path also drains synchronously while holding `thread-local-writes`, so it cannot +contain an async native durability barrier. Adapting it would couple client-subscription lifetime to +index availability and still require the cursor, queue bound, gap detection, and rebuild machinery. +The derived runtime therefore subscribes directly to the same lower-level `aftercommit` event. + +**Chosen for overload: coalesce by key, then admission-control unique work.** Applying every +intermediate vector wastes construction work because delivery always reconciles against current +primary state. A bounded keyed queue replaces pending work for the same key while retaining ordered +position tickets for cursor-prefix accounting. Per-key rate limiting is unnecessary after coalescing; +serving the removed CF graph would reintroduce dual storage and cannot include new keys. If distinct +keys or tickets still fill the queue, accepting an unbounded deficit violates eventual convergence +and dropping work forces rebuild at the same inadequate rate, so the pre-commit hook returns a +retryable 503 before that vector-changing record commits. This couples that record to its declared +index, as ordinary transactional indexes already do, while unrelated writes remain independent. + +## 9. Validation plan + +Baselines exist in `benchmarks/hnsw-scale.js` output (1M/2M/5M anchors, e.g. 1M efC-200: +p50 7.2 ms / recall@10-set 0.997 @ ef 512). Acceptance for phase 1: + +1. **Parity:** native search over a dual-written graph returns identical candidate sets to the + JS path at equal ef (modulo seqlock-retry races under concurrent write load — measured as a + bounded divergence rate, not exact equality under churn). +2. **Recall:** cap-64 prune vs cap-128 measured at 1M and 5M; accept if recall@10 delta ≤ 0.5 pt + at equal ef, else revisit the cap (header field — rebuild, not redesign). +3. **Latency:** ≥8× p50 improvement at 5M/ef 512 (22.2 ms → ≤2.8 ms), p99 within 2× p50 under + concurrent insert load (the metric that motivates off-loop execution). +4. **Crash:** kill -9 during sustained ingest → reopen → watermark replay → graph passes + connectivity + recall checks (extend the #1712 repair test harness). +5. **Churn:** delete/reinsert cycles hold node count stable (freelist reuse; #2182 regression + test). + +Phase-2 acceptance: + +1. An aborted record transaction produces no native mutation; committed insert/update/delete reaches + native search only after `aftercommit`. +2. An opted-in table without an explicit `audit: true` is rejected, while an ordinary HNSW index + keeps inheriting the global audit setting. The schema error explains that the audit API retains + full table history for the configured window. +3. Restart replays retained entries from each origin cursor; a cursor before retention rebuilds and + search stays unavailable until publication. +4. Concurrent multi-origin delivery re-reads the winning primary record, independent of delivery + order. +5. An unrelated-attribute commit schedules zero plane operations. Repeated writes to one key + coalesce while retaining a correct contiguous cursor; saturating with distinct keys rejects a + changed-vector write with a retryable 503 before commit and still permits an unrelated write. +6. A forced apply/flush failure retains the earlier cursor, produces no unhandled rejection, logs + once for that episode, marks the index unavailable, and converges by rebuild. +7. A full rebuild and retained-log replay both meet the same recall@k baseline as a from-scratch JS + graph on the same deterministic corpus; duplicate and deleted ids are absent from returned keys. +8. A non-default M/efConstruction/mL/optimizeRouting with `nativePlane: true` is rejected during + schema setup, not silently rebuilt under native defaults. +9. CI's existing `HNSW native plane` job installs 0.2.1 and runs a separate load probe before mocha; + absence already fails the job instead of producing a skipped green suite (verified on 87e0fd71). +10. Two rapid updates to one key across workers cannot apply in reverse version order, and graceful + worker recycle waits until an in-flight native mutation has returned. +11. The gated 100k-record native rebuild benchmark reports progress and enforces at least 1,000 + inserts/s in the Linux native-plane CI job. +12. A soak combines concurrent search, queue admission, retry, and restart during delivery; after + recovery its result quality meets the same deterministic recall baseline. + +## 10. Decisions & open questions + +Decided (Kris, 2026-08-31): + +- **Degree cap: 128 for the int8 plane** (revised 2026-08-31 after measurement). The original + cap-64 preference assumed 128 doubles the file; it does not for int8 slots — the 768 B vector + dominates, so 128 costs +23.5% (1,344 vs 1,088 B slots). Measured at 1M: cap-64 loses 2.2 pts + of recall (0.975 vs 0.996, where JS = 0.997) at equal ef and equal latency. +24% bytes for + full recall parity is the right trade. The cap stays a header field; the **binary-code v2 + plane reopens the question** (cap-64 ≈ 352 B vs cap-128 ≈ 608 B slots, +73% — there a + diversity-preserving prune at lower cap is worth engineering). +- **Platform policy.** Performance is a Linux target only. macOS must work (mmap/msync semantics + differ — msync alone is a weaker barrier there; an `F_FULLFSYNC` pass is a known follow-up, + and sparse-file behavior varies by filesystem — functional, not optimized). Windows is supported + through the package prebuild. If the optional native package is unavailable on any platform, + `nativePlane` indexes stay unavailable; ordinary HNSW indexes continue to use the JS implementation. +- **Packaging: independent open-source package.** The core has zero Harper coupling — the crate + compiles standalone and its NAPI surface is generic (create/open plane, insert(vector), + remove(id), search(query, k, ef, filter), watermark get/set). Harper-specific glue — the + pk→nodeId mapping, #2489's `DerivedIndexBackend` delivery, txnlog-anchored replay, auto-ef + policy constants — stays in Harper regardless of packaging. Published as the exact-pinned + optional dependency `@harperfast/hnsw` 0.2.1 (Apache-2.0, HarperFast/hnsw), with platform + prebuilds and a source-build fallback; the Harper adapter owns availability and integration + policy. The pitch as a community package: a persistent, + incrementally-maintained, concurrently-searchable HNSW for Node — hnswlib-node has no durable + incremental persistence, no off-loop batched filtering, no seqlock concurrency. + +Open: + +- **Atomic slot payloads.** Fields a concurrent reader acts on (flags, level, degree, scale, + invMag, neighbor and upper ids) are read through aligned `read_volatile`, which forbids the + reload/split/sink across the seqlock's validating fence that `lto = true, codegen-units = 1` + otherwise licenses. That is not the same as being race-free under Rust's memory model: only + making those fields `AtomicU8`/`AtomicU16`/`AtomicU32` in the slot layout would be, and that + is a format change deferred past phase 1. The stored vector stays an ordinary load on + purpose — `cosine_int8_raw` must keep autovectorizing, and a torn vector only perturbs a + distance the generation check discards. +- **msync cadence default** — bounded-lag durability window vs write amplification; needs a + workload measurement, not a guess. +- **Workers have no shared readiness signal during reconstruction.** `isIndexing` is per-worker, + and the two gaps that follow both come from that. Reconstruction removes the file and then + clears the mappings that prove non-emptiness, so a query on a different worker between the + clear and the first re-inserted record sees neither and returns no results; and in the shorter + window before the clear, that worker sees mappings with no file and requests a reconstruction + of its own. The writer lock keeps two workers from reconstructing concurrently, but it does + not bound recurrence: each rebuild re-opens the same unlink-then-clear window on the other + worker, so whether the sequence stops depends on query timing rather than on a guarantee. + A shared flag beside the existing `getUserSharedBuffer` depth counters would close both. +- **A new origin's first local write forces a full rebuild.** An origin log that gains entries + while the index holds no cursor for it cannot be replayed, so `reconcile()` reconstructs from + primary records. Replaying that origin's retained log from its oldest entry would converge + just as well — delivery reconciles against current record state — and would avoid taking + vector search offline the first time each cluster peer writes to the table. +- **An interior corrupt audit frame can loop reconstruction.** `replay()` throws on + `corruptFrameStop.breaks`, and `rebuild()`'s own catch-up replay starts at the oldest retained + entry, so it meets the same frame and throws again: the backoff retries reconstruction + indefinitely and the index stays 503 until retention removes the corruption. Bounding this + needs the same safely-captured physical boundary as the item below — a rebuild that starts + after the break has no reason to read it. +- **Replay does not coalesce native mutations for one key.** 128 queued updates to a single key + replay its final vector 128 times, because a mapping still pending publication bypasses the + signature shortcut. Deduplicating within a batch is not free: `applyDerivedValue` reads current + record state at first touch, so a change committing later in the same batch would be skipped + and its entry then passed by the advancing cursor. A correct version re-reads deduplicated keys + at the batch boundary. +- **An undecodable audit header is skipped, not escalated.** Replay matches on + `tableId`/`recordId`; an entry that decodes to a sentinel carries neither, so it is passed over + while the cursor advances past it. If that entry was a record's last vector-changing commit the + index stays stale indefinitely. Decoder sentinels should force reconstruction instead. +- **Rebuild replays the whole retained log, not just the scan window.** After the primary-store + scan, `rebuild()` sets each origin's cursor to the _oldest_ retained entry rather than the log + tail at scan start, so the follow-up `replay()` re-reads and re-hashes the entire retention + window before the index leaves 503. It is more conservative than it needs to be, and §13.6 has + the reading of rocksdb-js that says so: a committed-only query is bounded at one physical offset + and reads every byte up to it, so nothing sits behind the tail it yields and that tail is a safe + anchor. Tightening it is work for the shared runtime's rebuild phase; until then rebuild cost + scales with retention rather than with scan duration. +- **f32 (quantization:"none") slot variant** — 3,072 B vectors → 3.4 KB slots; supported by the + format (dims × mode in header) but int8 is the default and the optimization target. +- ~~Upper-layer region persistence~~ — done (format v2): fixed-entry region in the same file, + per-entry seqlocks, reserved for max_nodes/8. Upper entries leak on delete (bounded by the + 2x-headroom reserve); an upper freelist is the remaining nicety. +- ~~Reservation growth~~ — decided (Kris, 2026-08-31): a generous sparse reservation at create + is the model; mremap-based growth is a possible later enhancement, not a requirement. + +## 11. Phase-1 findings resolved by phase 2 + +The phase-1 review found two integration constraints. Phase 2 resolves the first and retains the +second as part of the custom-index search contract: + +- **Pre-commit mmap writes:** resolved. Transactions only stage changed primary keys; native mutation + starts from `RocksTransactionLogStore.aftercommit`, and abort coverage verifies that no node is + allocated for a rolled-back record write. +- **The async custom-index search contract** (`resources/search.ts`): a plane-backed search + returns a promise-backed, async-only iterable; synchronous consumers of custom-index + results would throw. Harper's search paths tolerate MaybePromise, and one full-stack test + covers the async path; widening coverage of other consumers is follow-up. + +## 12. Prototype measurements (kzyp Linux box, 768-d int8, ef 512, cap 64) + +Gaussian-mixture corpus matching `benchmarks/hnsw-scale.js` calibration (intra-cos 0.75, +clusters = N/500). JS baseline for scale: 4.34 µs/visit; 1M efC-200 anchor: p50 7.2 ms, +recall@10-set 0.997, ~3,110 visits. + +| N | cap | p50 | p95 | visits/query | µs/visit | recall@10 (set) | build rate | +| ------------------- | --- | ------- | ------- | ------------ | -------- | --------------- | --------------- | +| 100K | 64 | 0.28 ms | 0.46 ms | 1,395 | 0.201 | 1.000 | 5,583 inserts/s | +| 1M | 64 | 0.81 ms | 1.60 ms | 2,279 | 0.353 | 0.975 | 1,670 inserts/s | +| 1M | 128 | 0.75 ms | 1.48 ms | 2,309 | 0.324 | **0.996** | 1,242 inserts/s | +| 1M (fmt v2) | 128 | 0.83 ms | 1.61 ms | 2,309 | 0.359 | 0.996 | 1,346 inserts/s | +| 1M (coverage prune) | 128 | 0.75 ms | 1.52 ms | 2,279 | 0.327 | **0.999** | 1,359 inserts/s | + +Concurrency (same 1M graph): **6,345 QPS aggregate** across 8 searcher threads (p50 1.03 ms, +worst-thread p99 3.84 ms) while a background writer sustained **1,102 inserts/s** — the QPS +input §9 of the Reflex study lacked. Reverse-edge overflow eviction is coverage-aware +(evict the far member provably reachable via a kept nearer one; bounded 16×16 checks): the +concurrent torture test caught closest-keep eviction orphaning nodes in near-duplicate +clusters (~1-in-4 runs), and the fix also raised 1M recall from 0.996 to 0.999 at equal +build cost. +| 1M JS anchor | 128 | 7.2 ms | 12.0 ms | ~3,110 | 4.34 | 0.997 | ~263 inserts/s | + +At the 1M anchor with cap 128: **9.6× p50, 12.9× per-visit, 4.7× build rate, at JS-equal +recall.** The µs/visit rise from 100K (0.20) to 1M (0.32–0.35) is the working set leaving L3 — +the memory-hierarchy term; it is the number that holds at 60–100M. An ef-1024 sweep on a +reopened cap-64 plane without its hierarchy (pre-sidecar) still reached 0.985 at p50 2.47 ms — +layer-0 beam is robust to a missing hierarchy, at ~3.4× the visits. + +Milestones: zero-copy seqlock reads + AVX2 kernels took per-visit cost from 0.440 µs (first +scalar prototype) to ~0.1–0.35 µs, beating the 0.25–0.4 µs design budget. The +optimizeRouting-parity insert (including the recomputed neighbor↔neighbor distances) restored +recall from 0.49 (placeholder insert) to JS parity. Uniform-random 768-d corpora produce +meaningless recall numbers (the JS benchmark's own calibration note: a corpus "no ANN can +index") — all comparisons use the mixture corpus. + +## 13. Convergence onto the shared derived-index runtime (#2533 / #2535) + +[#2533](https://github.com/HarperFast/harper/pull/2533) implements #2489's shared transaction-log +runtime (`resources/derivedIndexRuntime.ts`); [#2535](https://github.com/HarperFast/harper/pull/2535) +is stacked on it and adds a RocksDB storage adapter for native derived indexes. +`resources/DerivedIndexBackend.ts` on this branch is a second, HNSW-shaped implementation of the +same protocol. One runtime is kept — #2533's — and HNSW becomes a backend on it. + +### 13.1 What the measurements decide + +`unitTests/resources/hnswDerivedIngest.bench.js` runs a real audited table with a file-primary +HNSW index through three shapes. Measured at 384 dimensions, `@harperfast/hnsw` 0.2.1, one worker: + +| shape | graph ≈3,000 | graph ≈25,000 | +| -------------------------------------------- | ------------------------------- | ------------------------------- | +| foreground `put` | 0.320 ms (3,125/s), p99 10.1 ms | 0.509 ms (1,964/s), p99 17.0 ms | +| `applyDerivedValue` | 0.214 ms/call | 0.389 ms/call | +| `flushDerived` | 3.59 ms × 50 calls | 4.18 ms × 129 calls | +| backend share of the write-and-index wall | 66.9–94.9% of 641 ms | 76.2–97.2% of 2,554 ms | +| end-to-end indexed rate | 3,121/s | 1,957/s | +| event loop, max/p99 **while writing** | 19.9 / 18.0 ms | 31.9 / 25.7 ms | +| event loop, max/p99 **while draining alone** | 0.0 / 0.0 ms | 2.6 / 2.6 ms | +| serialized single writes | 4.97 ms/record, ≤88.8% barrier | 4.05 ms/record, ≤79.1% barrier | +| 50 keys × 20 rounds: repeated `apply` calls | 95% | 95% | + +Package cost in isolation (N = 10,000): insert 208 µs (128-d) / 315 µs (384-d) / 835 µs (1536-d); +update (remove + insert) 287 / 468 / 1,399 µs. `flushAsync` scales with the **dirty set**, not with +index size, and has a floor: after a single insert it costs 2.3 / 3.4 / 4.3 ms, and after 10,000 +inserts at 384-d it costs 181.7 ms — 3.4 ms per record against 18 µs per record, a ~190× spread +that is the whole case for a cadence. + +What the instrumentation does and does not separate: the meter wraps whole backend methods, so +`applyDerivedValue` carries the vector hash and the RocksDB mapping writes as well as the native +insert, and `flushDerived` carries publishing those mappings as well as the msync. Those rows +therefore bound the _backend's_ share, not the native share; the isolated package numbers above are +what establish the native term inside it. The backend-share row is a range for a second reason: +`applyDerivedValue` is synchronous, so its time is exact and is the floor, while `flushDerived` is +timed across an `await` and so charges the backend for anything the loop ran during the barrier — +the ceiling. The serialized row's barrier share is bounded the same way, for the same reason. Two +rows also measure less than they look like: + +- The serialized row awaits full drain between writes, so nothing is available to combine. It + bounds the cost of one isolated write — one barrier per record, ~80–90% of it — and does **not** + measure what a flush cadence could amortize. Arrivals paced independently of the drain are the + missing experiment. +- The 95% is repeated keys across the whole run, not within one delivery window. Batch coalescing + removes only the repeats that land in the same batch, so 95% is the ceiling, not the saving. + +Three conclusions do hold. + +1. **Per-transaction bookkeeping is not what decides HNSW throughput.** The backend's synchronous + apply alone is 67–76% of the wall clock of a write-and-index cycle, and adding the barrier's + wall-clock time — an over-count, since it spans an await — reaches 95–97%. Even at the floor, + everything else, foreground write work and runtime bookkeeping together, has at most a third of + the wall to share. The concern that #2533's collect/resolve + path is less optimized than this branch's is real in the small but cannot pay for a second + runtime: both implementations decode the same log entries and read the same authoritative + record. +2. **This branch forces a barrier per drain; #2533 permits amortization but does not schedule it.** + `replay()` awaits `flushDerived()` at every origin's final cursor advance. #2533 separates + _offered_ from _durable_ progress and lets a backend accept up to `maxAcceptedBatchesAhead` + batches before its barrier — but that is a ceiling, not a scheduler. A backend that flushes each + accepted batch keeps this branch's cost, and one that waits only for the ceiling can leave a lone + write undurable indefinitely. The cadence has to be specified, not inherited. +3. **The drain's blocking lands on the foreground write path, and neither runtime bounds it.** + Draining alone barely touches the event loop (0.0–2.6 ms), but while writes are in flight the + drain runs inside their awaits and the loop blocks for 20–32 ms, showing up as a `put` p99 of + 10–17 ms against a 0.06 ms median. `drain()` applies up to 128 keys between awaits, so a turn is + up to 128 × 0.39 ms of synchronous native work. #2533 is not better placed: `#collectBatch` + completes a whole transaction before checking any budget, `#resolveTransactions` then resolves + every distinct key and projection outside that check, and `deliver()` runs after it — so + `maxTransactionsPerTurn: 256` and `maxMillisecondsPerTurn: 5` bound neither one large transaction + nor the applied cost of a batch. A queue-and-accept backend converts the applied cost into queue + depth, which is why §13.2 bounds collection and resolved payload rather than application alone. + +### 13.2 Changes the shared runtime needs (stacked on #2535) + +- **Coalesced delivery view.** Add a batch-level, last-write-wins view over distinct + `(tableId, recordId)` alongside the existing `transactions` array, keeping `writeKeyId` identity + and the whole `through` cursor vector intact. `#resolveTransactions` already resolves each key + once and hands every occurrence the same resolved object; what repeats is the mutation wrapper, + which for a backend costing 0.2–1.4 ms per mutation is the expensive part. The coalesced entry + keeps the last `logVersion` in batch order, the only field that differs between occurrences. +- **Bounded collection, resolution and delivery.** Make the budget cover the work that is actually + unbounded: incremental collection with no cursor publication until a complete transaction is + covered, or an explicit oversized-transaction policy; payload accounting that includes pending, + deferred and accepted bytes, with a stated way to estimate projection size without serializing it + twice; and time-sliced yielding rather than one `setImmediate` per mutation. Alongside it, make + `maxTransactionsPerTurn`, `maxBytesPerTurn`, `maxMillisecondsPerTurn` and + `maxAcceptedBatchesAhead` settable per registration — a vector backend and a full-text backend + want different values — and correct the Stage 1 sentence claiming `deliver()` is "included in the + runner's wall-time budget", which the code does not enforce. +- **An explicit durability cadence.** Specify what obliges a backend to flush: a maximum flush age + so an isolated write becomes durable promptly, work and byte thresholds so a burst amortizes, idle + completion, and shutdown behaviour. Without it, "adopt #2533" buys the _permission_ to amortize + and none of the amortization. The thresholds are not free choices: the barrier grows with the + dirty set (§13.1), so a cadence that waits for a large one trades per-record cost for a longer + single stall, and both ends need a stated bound. +- **Rebuild as a runtime phase, on the conservative boundary.** #2533 stops at `needs-rebuild`: + `#needsRebuild()` logs, drops the iterator, releases the lock, and the index stays unavailable. + Reset storage → scan the primary store → project → deliver in bounded batches → replay → resume is + identical for HNSW and full-text and belongs in the runtime. Anchor catch-up on the tail of a + committed read taken at scan start — §13.6 shows why that is safe and why this branch's + oldest-retained anchor is a cost with no correctness return — and keep the exact-boundary + validation and fail-closed behaviour when retention or corruption prevents proof. +- **Generation fencing and cancellation.** Asynchronous acceptance makes ownership handoff unsafe + without it: worker A can accept a batch, release the runner lock, and later run a scheduled + mutation or a flush completion after worker B has reset the index, publishing A's mappings or + readiness into B's generation. The backend interface needs a queue-drain/cancellation handshake; + the runtime needs epoch checks before mutation and after every await, queue shutdown ordered + before lock release, `rebuilding` published before any destructive reset, and `ready` published + only after scan, catch-up and the final barrier. +- **Shared cross-worker readiness.** `indexStore.isIndexing` is per-worker and `getStatus()` is only + meaningful on the owner, so a non-owning worker can answer from a partially built index. Publish + readiness and its reason in a shared buffer beside the owner-epoch counter, and fence peer readers + with it — this is the same mechanism the fencing above needs, and it closes the two cross-worker + rebuild-window residuals in §10. +- **A lag policy, or an explicit decision not to have one.** Indexing capacity is one insert per + changed vector plus the barrier — about 2,000 records/s per index at 384 dimensions, falling with + dimension and graph size — while the foreground work of a `put` is 0.06 ms. A write stream that + does not share the owning worker's event loop, which is any peer worker or any client pipelined + across workers, therefore exceeds indexing capacity by a wide margin. The single-worker benchmark + cannot show that gap directly, because there the writer and the drain contend for one loop and + both land near 2,000/s. Deleting this branch's runtime deletes its retryable 503, and #2533 has no + replacement, so sustained overload runs the cursor past audit retention, rebuilds, and falls + behind again. Preserve the admission behaviour or approve the changed availability contract + explicitly, with observable lag; queue memory pressure and retention lag are separate signals. + +### 13.3 What #2430 becomes + +`resources/DerivedIndexBackend.ts` is deleted. `HierarchicalNavigableSmallWorld` implements #2533's +`DerivedIndexBackend` interface: `deliver()` queues and returns accepted, an applier drains it in +bounded time slices, `flushAsync()` is the barrier, and the cursor vector is stored with the plane +generation. HNSW does **not** use #2535's `RocksDerivedIndexStorage`: its durability barrier is a +database-wide `flushSync({ allowWriteStall: true })`, whereas an mmap index needs only `msync` of +its own file. + +Three constraints the async shape imposes, none of which exist in today's synchronous path: + +- **The flush cut must be immutable.** `flushDerived()` awaits the plane barrier and then publishes + the live `pendingDerivedMappings` map. Once application can continue during that await, a later + mutation's mapping enters that map and is published by an earlier barrier, so a crash can leave a + published mapping naming native state the barrier never covered. Snapshot the pending set at + barrier entry, or serialize application against flushing. +- **`msync` is not the whole durability unit.** The graph, both mapping directions, the generation + metadata and the offered cursor vector all have to have a stated persistence order, and `msync` + alone does not establish durability of the RocksDB half. Restart tests belong at each publication + boundary; an ordinary database reset does not stand in for power-loss ordering. +- **Cursor migration is versioned.** Per-origin `Symbol.for('derived-index-cursor:…')` keys become + the shared cursor vector. An old cursor read against a fresh plane generation must force + reconstruction rather than resume. + +### 13.4 Approaches considered + +**Invariant:** one delivery/recovery implementation, in which a published cursor certifies a +complete durable prefix for that index generation and every committed change outside that prefix +stays replayable. + +**Different layer.** Keep both runtimes and share only the transaction-log reader changes. Rejected: +ownership election, cursor validation and recovery stay duplicated, which is the failure #2489 +exists to prevent. + +**Deeper cause.** Move HNSW insertion into the package's own thread pool so delivery cost stops +being event-loop cost. Genuinely the deeper fix for the event-loop term, but 0.2.1 exposes only a +synchronous `insert`, it removes neither the duplicated runtime nor the repeated-key work, and it is +phase-3. + +**Do less.** Adopt #2533 unchanged and put coalescing, chunking and rebuild inside the HNSW backend. +Rejected for rebuild, which is not backend-specific and whose duplication is how two runtimes came +to exist. The coalescing half of this argument is narrower than it first looked: #2533 already +resolves each key once, so backend-private coalescing would discard repeated mutation wrappers, not +repeated primary reads — still worth doing in the runtime, but as an allocation and dispatch saving. + +**Chosen (revised after planning review).** One stacked PR on #2535 adds the coalesced view, bounded +collection/resolution/delivery, an explicit durability cadence, the rebuild phase, generation +fencing with cancellation, and shared readiness. #2430 then rebases onto it and becomes an HNSW +backend only. + +The first draft of this plan also put a tighter rebuild boundary in the same PR — the earliest +position uncommitted at scan start, derived from `readUncommitted` or from per-worker staged +timestamps. The planning review disqualified it on a fact rather than a preference: the shared +runner resumes _after_ a complete transaction at its exact cursor, so installing the oldest staged +transaction A as the cursor skips A's own transaction entirely, and if A aborts, that exact +committed boundary may never exist and cursor validation fails. A correct version needs a distinct +_inclusive_ rebuild anchor with its own transition into a durable cursor, plus synchronization +between capture, staging publication, commit visibility, abort, worker replacement and retention — +a correctness-sensitive change to every audited writer, in exchange for an unmeasured rebuild-cost +optimization. Reading the pinned rocksdb-js afterwards showed the whole construction to be +unnecessary rather than merely misplaced — a committed read is a contiguous byte prefix, so the tail +is already a safe anchor (§13.6). The rejected alternative was right that the proposal did not +belong in this PR; it was solving a problem that is not there. + +### 13.5 Sequencing + +1. Stacked PR on `codex/fulltext-storage-adapter` with §13.2. Its gate is #2533's existing + derived-index suite plus a fake backend carrying a synthetic per-mutation cost, exercising one + oversized transaction, ownership handoff while an apply is scheduled and while a flush is + pending, and independently paced arrivals for the cadence. +2. Rebase #2430 onto that branch; delete `DerivedIndexBackend.ts`; port `vectorIndexPlane.test.js` + alongside the runtime suite and require native availability in the designated gate rather than + accepting a skip. Targets are numeric — write latency, indexed throughput, queue memory and + maximum durability age — because a falling flush _share_ can also mean unrelated work got slower. +3. #2533 and #2535 merge first; #2430 stays draft until then. + +### 13.6 The rebuild anchor, resolved + +The conservative anchor makes rebuild replay the whole retention window (§10). The first draft of +this plan proposed tightening it with a boundary derived from staged/uncommitted positions, the +planning review disqualified that construction, and it was deferred to the storage layer. Reading +the pinned rocksdb-js retires the whole line of work: no storage change is needed, and the tail is +already safe. + +`TransactionLog.query()` resolves its end through `loadLastPosition()`, which decodes +`_lastCommittedPosition` into a single `{ logId, size }` physical offset; the iterator then walks +`while (position < size)`. A committed-only read is therefore a **contiguous byte prefix** of the +log, not a per-transaction filter over a sparse range. Nothing can sit physically behind the last +entry it yields, so anchoring rebuild catch-up on that entry cannot skip anything — the hazard the +conservative anchor exists to avoid does not exist. + +One question is left, and it is native-side: whether `_lastCommittedPosition` advances only to a +contiguous committed frontier, or to the end of whichever transaction committed most recently. Under +the second reading a committed read can return entries of a transaction that has not committed yet, +and would return entries of one that later aborts. That is not a rebuild-anchor problem — it is a +property every committed reader in Harper already has, replay and replication included — and this +protocol absorbs it: delivery re-reads the authoritative record rather than applying the log entry's +body (§8), so a phantom entry resolves to current state and is idempotent. + +So the shared runtime's rebuild phase (§13.2) should anchor on the tail of a committed read taken at +scan start, and this branch's oldest-retained anchor is a cost with no correctness return. It is not +changed here because this runtime is the one being deleted; the tail anchor belongs in the +replacement, where it is a few lines rather than a storage-layer project. diff --git a/package-lock.json b/package-lock.json index b24510e024..b7387ec43c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -141,6 +141,7 @@ "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { + "@harperfast/hnsw": "0.2.1", "bufferutil": "4.1.0", "segfault-handler": "1.3.0", "utf-8-validate": "5.0.10" @@ -1017,11 +1018,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-darwin-x64": { "version": "2.2.2", @@ -1030,11 +1033,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-arm": { "version": "2.2.2", @@ -1043,11 +1048,13 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-arm64": { "version": "2.2.2", @@ -1056,11 +1063,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-linux-x64": { "version": "2.2.2", @@ -1069,11 +1078,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@cbor-extract/cbor-extract-win32-x64": { "version": "2.2.2", @@ -1082,11 +1093,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@colors/colors": { "version": "1.5.0", @@ -2454,6 +2467,87 @@ "version": "1.0.3", "license": "Apache-2.0" }, + "node_modules/@harperfast/hnsw": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@harperfast/hnsw/-/hnsw-0.2.1.tgz", + "integrity": "sha512-ryUJYE9p7secerYYelVCfhc4kChBv6TZDvDFQUU7NQ99TC2dcPIXQjpOVxne2P6mQPQfh/WP5YmOYNjLn0UNbQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "optional": true, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@harperfast/hnsw-darwin-arm64": "0.2.1", + "@harperfast/hnsw-linux-arm64-glibc": "0.2.1", + "@harperfast/hnsw-linux-x64-glibc": "0.2.1", + "@harperfast/hnsw-win32-x64": "0.2.1" + } + }, + "node_modules/@harperfast/hnsw-darwin-arm64": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@harperfast/hnsw-darwin-arm64/-/hnsw-darwin-arm64-0.2.1.tgz", + "integrity": "sha512-dCSrj+eTWyD0qvaN6zHGqo1cpwTOxyfXxP6Kl+EqpuiEcModb9gl1lG+dI6jzCBMJakjmWTgI8WiWrLSWm9p5w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@harperfast/hnsw-linux-arm64-glibc": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@harperfast/hnsw-linux-arm64-glibc/-/hnsw-linux-arm64-glibc-0.2.1.tgz", + "integrity": "sha512-scLouN0oKR7s4Gv6h1RMzCfS7E0Lca/Tw4MeaLFD5rjQQh2YWvAfhwzPDh/HkPAiaUxEIgCXbc5KlRIzV8jZBQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@harperfast/hnsw-linux-x64-glibc": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@harperfast/hnsw-linux-x64-glibc/-/hnsw-linux-x64-glibc-0.2.1.tgz", + "integrity": "sha512-yAReLrNpdRrs0HpPnG/nykSjz4ycMgJOAjMPMVyjkDCg9q2nO+4utn8mSBu59rNXAGdfqUHOYzXvIHrPc2E0Rg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@harperfast/hnsw-win32-x64": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@harperfast/hnsw-win32-x64/-/hnsw-win32-x64-0.2.1.tgz", + "integrity": "sha512-TXbJhvg/7wIYrPFwGnjsOsvKcnevMXEctbDifZEiWLTUgSsepIHYKREaQXIIt+d7l08E3KGdfUl3vNaSZjGqjg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@harperfast/integration-testing": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/@harperfast/integration-testing/-/integration-testing-0.7.1.tgz", @@ -2560,9 +2654,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2579,9 +2670,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2598,9 +2686,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2617,9 +2702,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -3336,11 +3418,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { "version": "3.0.4", @@ -3349,11 +3433,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "darwin" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { "version": "3.0.4", @@ -3362,11 +3448,13 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { "version": "3.0.4", @@ -3375,11 +3463,13 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { "version": "3.0.4", @@ -3388,11 +3478,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "peer": true }, "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { "version": "3.0.4", @@ -3401,11 +3493,13 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "peer": true }, "node_modules/@noble/hashes": { "version": "1.8.0", @@ -3574,9 +3668,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3594,9 +3685,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3614,9 +3702,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3634,9 +3719,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3654,9 +3736,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3674,9 +3753,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3694,9 +3770,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3714,9 +3787,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ diff --git a/package.json b/package.json index c8f8435a37..743177876b 100644 --- a/package.json +++ b/package.json @@ -262,6 +262,7 @@ } }, "optionalDependencies": { + "@harperfast/hnsw": "0.2.1", "bufferutil": "4.1.0", "segfault-handler": "1.3.0", "utf-8-validate": "5.0.10" diff --git a/resources/DerivedIndexBackend.ts b/resources/DerivedIndexBackend.ts new file mode 100644 index 0000000000..a3b12eedff --- /dev/null +++ b/resources/DerivedIndexBackend.ts @@ -0,0 +1,486 @@ +import { ClientError, ServerError } from '../utility/errors/hdbError.ts'; +import { loggerWithTag } from '../utility/logging/logger.ts'; +import { getWorkerCount, getWorkerIndex } from '../server/threads/manageThreads.js'; +import type { AuditRecord } from './auditStore.ts'; + +const logger = loggerWithTag('DerivedIndex'); +const MAX_PENDING_KEYS = 65_536; +const MAX_PENDING_TICKETS = 262_144; +const APPLY_BATCH_SIZE = 128; +const REBUILD_PROGRESS_INTERVAL = 10_000; +const MAX_REBUILD_BACKOFF = 300_000; +const MAX_WORKER_SLOTS = 256; +const DEPTH_VALUES_PER_WORKER = 3; // pending keys, position tickets, rejected writes +const warnedAuditIndexes = new Set(); + +type Position = { nodeId: number; txnLogKey: number }; +type Ticket = Position & { done: boolean }; +type Pending = { id: any; tickets: Ticket[] }; +type StagedTarget = { runtime: DerivedIndexRuntime; id: any }; + +export function derivedIndexCursorKey(indexName: string, nodeId: number): symbol { + return Symbol.for(`derived-index-cursor:${indexName}:${nodeId}`); +} + +export interface DerivedIndexBackend { + readonly postCommit: true; + attachDerivedRuntime(runtime: DerivedIndexRuntime): void; + applyDerivedValue(id: any, value: any, version?: number): void; + flushDerived(watermark?: number): Promise; + resetDerivedStorage(): void; + hasDerivedStorage(): boolean; +} + +export function valuesEqual(a: any, b: any): boolean { + if (a === b) return true; + // Element-wise comparison is only defined for the array-like projections backends index. + // Anything else that already failed `===` counts as changed rather than silently unstaged. + if (a == null || b == null || typeof a.length !== 'number' || a.length !== b.length) return false; + for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false; + return true; +} + +async function lock(store: any, key: string, callback: () => Promise): Promise { + while (true) { + let unlocked: () => void; + const released = new Promise((resolve) => (unlocked = resolve)); + if (store.tryLock(key, unlocked!)) break; + await released; + } + try { + await callback(); + } finally { + store.unlock(key); + } +} + +export class DerivedIndexRuntime { + readonly backend: DerivedIndexBackend; + readonly indexStore: any; + readonly table: any; + readonly attribute: any; + readonly lockKey: string; + private sharedDepth: Int32Array; + private sharedDepthOffset: number; + private pending = new Map(); + private ticketsByOrigin = new Map(); + private pendingTickets = 0; + private drainScheduled = false; + private draining = false; + private closed = false; + private ready = false; + private retryScheduled = false; + private forceRebuild = false; + private rebuildBackoff = 1_000; + private listener: (entries: AuditRecord[], targets?: StagedTarget[]) => void; + + constructor(table: any, attribute: any, indexStore: any, backend: DerivedIndexBackend) { + this.table = table; + this.attribute = attribute; + this.indexStore = indexStore; + this.backend = backend; + this.lockKey = `derived-index-writer:${indexStore.name}`; + const workerIndex = getWorkerIndex() ?? 0; + this.sharedDepth = new Int32Array( + indexStore.getUserSharedBuffer( + `derived-index-depth:${indexStore.name}`, + new ArrayBuffer(MAX_WORKER_SLOTS * DEPTH_VALUES_PER_WORKER * Int32Array.BYTES_PER_ELEMENT) + ) + ); + this.sharedDepthOffset = workerIndex * DEPTH_VALUES_PER_WORKER; + if (this.sharedDepthOffset + DEPTH_VALUES_PER_WORKER - 1 >= this.sharedDepth.length) { + throw new Error(`Derived indexes support worker indexes below ${MAX_WORKER_SLOTS}`); + } + // A replacement worker reuses the dead worker's slot. Resetting only this slot removes + // stale admission pressure without disturbing live peers. + Atomics.store(this.sharedDepth, this.sharedDepthOffset, 0); + Atomics.store(this.sharedDepth, this.sharedDepthOffset + 1, 0); + Atomics.store(this.sharedDepth, this.sharedDepthOffset + 2, 0); + backend.attachDerivedRuntime(this); + this.listener = (entries, targets) => this.committed(entries, targets); + table.auditStore.on('aftercommit', this.listener); + indexStore.isIndexing = true; + void this.initialize(); + } + + stage(transaction: any, id: any, value: any, existingValue: any): void { + if (valuesEqual(value, existingValue)) return; + let totalKeys = 0; + let totalTickets = 0; + let rejectedWrites = 0; + const activeDepthLength = Math.min( + this.sharedDepth.length, + Math.max(1, getWorkerCount() ?? 1) * DEPTH_VALUES_PER_WORKER + ); + for (let offset = 0; offset < activeDepthLength; offset += DEPTH_VALUES_PER_WORKER) { + totalKeys += Atomics.load(this.sharedDepth, offset); + totalTickets += Atomics.load(this.sharedDepth, offset + 1); + rejectedWrites += Atomics.load(this.sharedDepth, offset + 2); + } + if ((!this.pending.has(id) && totalKeys >= MAX_PENDING_KEYS) || totalTickets >= MAX_PENDING_TICKETS) { + const localRejections = Atomics.add(this.sharedDepth, this.sharedDepthOffset + 2, 1) + 1; + if ((localRejections & (localRejections - 1)) === 0) { + logger.warn?.( + `Derived index ${this.indexStore.name} rejected ${rejectedWrites + 1} writes while ${totalKeys} keys and ${totalTickets} log positions were pending` + ); + } + throw new ServerError( + `Derived index ${this.indexStore.name} is catching up; retry this vector-changing write`, + 503 + ); + } + const targets: StagedTarget[] = (transaction.derivedIndexTargets ??= []); + const targetsByRuntime: Map> = (transaction.derivedIndexTargetIds ??= new Map()); + let ids = targetsByRuntime.get(this); + if (!ids) targetsByRuntime.set(this, (ids = new Set())); + if (!ids.has(id)) { + ids.add(id); + targets.push({ runtime: this, id }); + } + } + + private committed(entries: AuditRecord[], targets?: StagedTarget[]): void { + if (this.closed) return; + if (entries.some((entry) => entry.tableId === this.table.tableId && entry.type === 'reload')) { + this.requestRebuild(new Error(`Table ${this.table.tableName} received a whole-table reload`)); + return; + } + if (!targets) return; + const positions = new Map(); + for (const entry of entries) { + if (entry.tableId === this.table.tableId && entry.recordId != null) positions.set(entry.recordId, entry); + } + for (const target of targets) { + if (target.runtime !== this) continue; + const entry = positions.get(target.id); + if (!entry?.txnLogKey) { + this.fail(new Error(`Committed derived-index target ${String(target.id)} has no transaction-log entry`)); + return; + } + this.enqueue(target.id, { nodeId: entry.nodeId ?? 0, txnLogKey: entry.txnLogKey }); + } + this.scheduleDrain(); + } + + private enqueue(id: any, position: Position): void { + const ticket: Ticket = { ...position, done: false }; + let pending = this.pending.get(id); + if (!pending) { + this.pending.set(id, (pending = { id, tickets: [] })); + Atomics.add(this.sharedDepth, this.sharedDepthOffset, 1); + } + pending.tickets.push(ticket); + let originTickets = this.ticketsByOrigin.get(position.nodeId); + if (!originTickets) this.ticketsByOrigin.set(position.nodeId, (originTickets = [])); + originTickets.push(ticket); + this.pendingTickets++; + Atomics.add(this.sharedDepth, this.sharedDepthOffset + 1, 1); + } + + private scheduleDrain(): void { + if (this.closed || !this.ready || this.draining || this.drainScheduled || this.pending.size === 0) return; + this.drainScheduled = true; + setImmediate(() => { + this.drainScheduled = false; + void this.drain(); + }); + } + + private async drain(): Promise { + if (this.closed || this.draining || !this.ready) return; + this.draining = true; + try { + await lock(this.indexStore, this.lockKey, async () => { + if (this.closed) return; + const batch: Array<{ pending: Pending; ticketCount: number }> = []; + for (const pending of this.pending.values()) { + batch.push({ pending, ticketCount: pending.tickets.length }); + if (batch.length === APPLY_BATCH_SIZE) break; + } + await this.reconcile(this.table.auditStore.loadLogs?.() ?? []); + if (!this.ready || this.closed) return; + for (const { pending, ticketCount } of batch) { + for (const ticket of pending.tickets.slice(0, ticketCount)) ticket.done = true; + pending.tickets.splice(0, ticketCount); + if (pending.tickets.length === 0) { + this.pending.delete(pending.id); + Atomics.sub(this.sharedDepth, this.sharedDepthOffset, 1); + } + } + this.discardCompletedTickets(); + }); + } catch (error) { + this.fail(error); + } finally { + this.draining = false; + this.scheduleDrain(); + } + } + + private cursorKey(nodeId: number): symbol { + return derivedIndexCursorKey(this.indexStore.name, nodeId); + } + + private discardCompletedTickets(): void { + for (const [nodeId, tickets] of this.ticketsByOrigin) { + let completed = 0; + while (completed < tickets.length && tickets[completed].done) completed++; + if (completed > 0) { + tickets.splice(0, completed); + this.pendingTickets -= completed; + Atomics.sub(this.sharedDepth, this.sharedDepthOffset + 1, completed); + } + if (tickets.length === 0) this.ticketsByOrigin.delete(nodeId); + } + } + + private async initialize(): Promise { + if (this.closed) return; + try { + await this.table.indexingOperation; + if (this.closed) return; + // runIndexing clears this flag when its schema scan completes. Reassert ownership + // before replay/rebuild so queries cannot observe a partial native generation. + this.indexStore.isIndexing = true; + await lock(this.indexStore, this.lockKey, async () => { + if (this.closed) return; + const logs = this.table.auditStore.loadLogs?.() ?? []; + await this.reconcile(logs); + }); + if (this.closed) return; + this.ready = true; + this.indexStore.isIndexing = false; + this.rebuildBackoff = 1_000; + this.scheduleDrain(); + } catch (error) { + this.fail(error); + } + } + + private originHasEntries(nodeId: number): boolean { + for (const _entry of this.table.auditStore.getRange({ start: 0, log: nodeId })) return true; + return false; + } + + private cursorExists(nodeId: number, cursor: number): boolean { + for (const entry of this.table.auditStore.getRange({ start: cursor, exactStart: true, log: nodeId })) { + return entry.txnLogKey === cursor; + } + return false; + } + + private async reconcile(logs: any[]): Promise { + if (this.forceRebuild || !this.backend.hasDerivedStorage()) { + await this.rebuild(logs); + this.forceRebuild = false; + return; + } + for (let nodeId = 0; nodeId < logs.length; nodeId++) { + if (!logs[nodeId]) continue; + const cursor = this.indexStore.getSync(this.cursorKey(nodeId)); + // A replica that only ever received remote writes keeps an empty local log, and rebuild + // writes no cursor for it. Without this exemption every reconciliation would read that + // missing cursor as a retention gap and reconstruct the whole index again. + if (!cursor && !this.originHasEntries(nodeId)) continue; + if (!this.cursorExists(nodeId, cursor)) { + await this.rebuild(logs); + return; + } + } + await this.replay(logs); + } + + private async replay(logs: any[], ignoreReloadsBefore?: number): Promise { + let skipped = 0; + for (let nodeId = 0; nodeId < logs.length; nodeId++) { + if (!logs[nodeId]) continue; + const cursor = this.indexStore.getSync(this.cursorKey(nodeId)); + if (!cursor && !this.originHasEntries(nodeId)) continue; + if (!cursor || !this.cursorExists(nodeId, cursor)) { + throw new Error(`Derived-index cursor for origin ${nodeId} is outside audit retention`); + } + let latest = cursor; + let applied = 0; + const entries = this.table.auditStore.getRange({ + start: cursor, + exactStart: true, + exclusiveStart: true, + log: nodeId, + }); + for (const entry of entries) { + latest = entry.txnLogKey; + if (entry.tableId === this.table.tableId && entry.type === 'reload') { + if (ignoreReloadsBefore !== undefined && entry.txnLogKey < ignoreReloadsBefore) continue; + throw new Error(`Table ${this.table.tableName} requires reconstruction after a whole-table reload`); + } + if (entry.tableId === this.table.tableId && entry.recordId != null) { + const current = this.table.primaryStore.getEntry(entry.recordId); + const record = current?.value; + const value = + record && (this.attribute.resolve ? this.attribute.resolve(record) : record[this.attribute.name]); + try { + this.backend.applyDerivedValue(entry.recordId, value, current?.localTime ?? current?.version); + } catch (error) { + // Same rule as the rebuild scan: a record the backend rejects is unindexable, not a + // delivery failure. Without this the two disagree — rebuild ends by replaying from + // the oldest retained entry, so every record its scan skipped is met again here and + // rethrown, and reconstruction can never finish while that entry is retained. + if (!(error instanceof ClientError)) throw error; + if (skipped++ === 0) logger.warn?.(`${this.indexStore.name} skipped a record it cannot index`, error); + continue; + } + if (++applied % APPLY_BATCH_SIZE === 0) { + await this.backend.flushDerived(latest); + if (this.closed) return; + } + } + } + if (entries.corruptFrameStop.breaks) { + throw new Error(`Audit log ${nodeId} ended at a corrupt frame while updating ${this.indexStore.name}`); + } + // exactStart follows physical log order after locating the cursor. A transaction that + // began earlier can commit later with a numerically smaller timestamp, so inequality, + // rather than a numeric greater-than comparison, identifies forward progress. + if (latest !== cursor) { + await this.backend.flushDerived(latest); + // A retired runtime must not advance a durable cursor: its replacement may already + // have reset the generation this position describes. + if (this.closed) return; + this.indexStore.putSync(this.cursorKey(nodeId), latest); + } + } + } + + private async rebuild(logs: any[]): Promise { + const startedAt = Date.now(); + this.ready = false; + this.indexStore.isIndexing = true; + // A committed read is bounded at one physical offset and returns every byte up to it, so the + // tail it yields is already a safe anchor; the oldest retained entry is chosen only so this + // runtime, which is being replaced, is not the place that changes recovery semantics. + const boundaries: Array = []; + for (let nodeId = 0; nodeId < logs.length; nodeId++) { + if (!logs[nodeId]) continue; + for (const entry of this.table.auditStore.getRange({ start: 0, log: nodeId })) { + boundaries[nodeId] = entry.txnLogKey; + break; + } + } + this.backend.resetDerivedStorage(); + await this.indexStore.clear(); + let indexed = 0; + let skipped = 0; + const total = this.table.primaryStore.getKeysCount?.() ?? 0; + for (const { key, value, version, localTime } of this.table.primaryStore.getRange({ + versions: true, + snapshot: false, + })) { + if (!value) continue; + const projected = this.attribute.resolve ? this.attribute.resolve(value) : value[this.attribute.name]; + try { + this.backend.applyDerivedValue(key, projected, localTime ?? version); + } catch (error) { + // A record the backend rejects (a malformed vector stored before this index existed) + // would otherwise abort every reconstruction attempt at the same record, leaving the + // index permanently unavailable. Skip it and report the count. + if (!(error instanceof ClientError)) throw error; + if (skipped++ === 0) logger.warn?.(`${this.indexStore.name} skipped a record it cannot index`, error); + continue; + } + indexed++; + if (indexed % REBUILD_PROGRESS_INTERVAL === 0) { + await this.backend.flushDerived(); + const elapsedSeconds = Math.max((Date.now() - startedAt) / 1_000, 0.001); + const rate = Math.round(indexed / elapsedSeconds); + const etaSeconds = rate > 0 ? Math.max(0, Math.ceil((total - indexed) / rate)) : undefined; + logger.info?.(`Rebuilding ${this.indexStore.name}: ${indexed}/${total} records, ${rate}/s, ETA ${etaSeconds}s`); + } + if (indexed % APPLY_BATCH_SIZE === 0) { + await new Promise((resolve) => setImmediate(resolve)); + if (this.closed) return; + } + } + await this.backend.flushDerived(Math.max(1, ...boundaries.filter((value) => value != null))); + if (this.closed) return; + for (let nodeId = 0; nodeId < logs.length; nodeId++) { + const boundary = boundaries[nodeId]; + if (boundary) this.indexStore.putSync(this.cursorKey(nodeId), boundary); + } + await this.replay(logs, startedAt); + // A retired runtime must not publish readiness: on the table() redefine path its replacement + // has already set isIndexing, and clearing it here would let this worker serve searches from + // the half-built generation the replacement is still filling. + if (this.closed) return; + this.ready = true; + this.indexStore.isIndexing = false; + if (indexed) { + const elapsedSeconds = Math.max((Date.now() - startedAt) / 1_000, 0.001); + logger.info?.( + `Rebuilt ${this.indexStore.name} from ${indexed} records at ${Math.round(indexed / elapsedSeconds)}/s` + + (skipped ? `, skipping ${skipped} it cannot index` : '') + ); + } + } + + private fail(error: unknown): void { + if (this.closed || this.retryScheduled) return; + this.ready = false; + this.forceRebuild = true; + this.indexStore.isIndexing = true; + Atomics.sub(this.sharedDepth, this.sharedDepthOffset, this.pending.size); + Atomics.sub(this.sharedDepth, this.sharedDepthOffset + 1, this.pendingTickets); + this.pending.clear(); + this.ticketsByOrigin.clear(); + this.pendingTickets = 0; + logger.error?.(`Derived index ${this.indexStore.name} is unavailable and will rebuild`, error); + this.retryScheduled = true; + setTimeout(() => { + this.retryScheduled = false; + void this.initialize(); + }, this.rebuildBackoff).unref(); + this.rebuildBackoff = Math.min(this.rebuildBackoff * 2, MAX_REBUILD_BACKOFF); + } + + requestRebuild(error: unknown): void { + this.forceRebuild = true; + this.fail(error); + } + + close(): void { + if (this.closed) return; + this.closed = true; + Atomics.sub(this.sharedDepth, this.sharedDepthOffset, this.pending.size); + Atomics.sub(this.sharedDepth, this.sharedDepthOffset + 1, this.pendingTickets); + this.pending.clear(); + this.ticketsByOrigin.clear(); + this.pendingTickets = 0; + this.table.auditStore.removeListener('aftercommit', this.listener); + } +} + +export function attachDerivedIndexBackends(table: any): { close(): void } | undefined { + const runtimes: DerivedIndexRuntime[] = []; + const attachedStores = new Set(); + for (const attribute of table.attributes) { + const indexStore = table.indices[attribute.name]; + const backend = indexStore?.customIndex as DerivedIndexBackend; + if (backend?.postCommit && !attachedStores.has(indexStore)) { + if (table.audit !== true) { + throw new ClientError( + `Table '${table.databaseName}.${table.tableName}' must enable audit logging before using a post-commit derived index` + ); + } + attachedStores.add(indexStore); + const warningKey = `${table.databaseName}.${table.tableName}.${indexStore.name}`; + if (!warnedAuditIndexes.has(warningKey)) { + warnedAuditIndexes.add(warningKey); + logger.warn?.( + `Derived index ${indexStore.name} requires auditing; the audit API retains full record history for the configured retention window` + ); + } + runtimes.push(new DerivedIndexRuntime(table, attribute, indexStore, backend)); + } + } + if (runtimes.length === 0) return; + return { close: () => runtimes.forEach((runtime) => runtime.close()) }; +} diff --git a/resources/RocksTransactionLogStore.ts b/resources/RocksTransactionLogStore.ts index a692687377..e2b4e003fd 100644 --- a/resources/RocksTransactionLogStore.ts +++ b/resources/RocksTransactionLogStore.ts @@ -152,7 +152,7 @@ export class RocksTransactionLogStore extends EventEmitter { } } const entries = options.transaction.logEntries; - if (entries) this.emit('aftercommit', entries); + if (entries) this.emit('aftercommit', entries, options.transaction.derivedIndexTargets); }; } log.addEntry(entryBinary, options.transaction.id); @@ -242,6 +242,7 @@ export class RocksTransactionLogStore extends EventEmitter { getRange(options: { start?: number; exactStart?: boolean; + exclusiveStart?: boolean; end?: number; log?: string | number; excludeLogs?: string[]; diff --git a/resources/Table.ts b/resources/Table.ts index 0b9b74853b..dfe5abde77 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -780,6 +780,7 @@ export function makeTable(options) { static tableName = tableName; static tableId = tableId; static indices = indices; + static derivedIndexRuntime: { close(): void } | undefined; static audit = audit; static databasePath = databasePath; static databaseName = databaseName; @@ -1621,6 +1622,11 @@ export function makeTable(options) { static async dropTable() { TableResource.assertSchemaMutable('drop a table'); + // Retire post-commit derived-index delivery before any destructive work. Its aftercommit + // listener and rebuild retry outlive the stores otherwise, and a same-name recreate would + // leave the orphan racing the new table's runtime on the same derived file and mappings. + TableResource.derivedIndexRuntime?.close(); + TableResource.derivedIndexRuntime = undefined; const rootStore = primaryStore.rootStore; if (databaseName === databasePath) { // Persist a drop tombstone on the primary catalog entry BEFORE any @@ -1757,6 +1763,7 @@ export function makeTable(options) { const index = indices[attribute.name]; if (index) try { + index.customIndex?.resetDerivedStorage?.(); index.dropSync(); } catch (error) { ignoreAlreadyDropped(error); @@ -1777,7 +1784,10 @@ export function makeTable(options) { const drops = []; for (const attribute of attributes) { const index = indices[attribute.name]; - if (index) drops.push(index.drop().catch(ignoreAlreadyDropped)); + if (index) { + index.customIndex?.resetDerivedStorage?.(); + drops.push(index.drop().catch(ignoreAlreadyDropped)); + } } drops.push(primaryStore.drop().catch(ignoreAlreadyDropped)); await Promise.all(drops); @@ -6179,6 +6189,7 @@ export function makeTable(options) { const promises = [primaryStore.clear()]; for (const key in indices) { const index = indices[key]; + index.customIndex?.resetDerivedStorage?.(); promises.push(index.clearAsync ? index.clearAsync() : index.clear()); } return Promise.all(promises); @@ -6186,6 +6197,7 @@ export function makeTable(options) { /** Release everything makeTable() registered process-wide; the class must not be used afterwards. */ static cleanup() { disposed = true; + TableResource.derivedIndexRuntime?.close(); clearTimeout(cleanupTimer); settlePendingCleanup(); clearInterval(recordExpirationInterval); diff --git a/resources/databases.ts b/resources/databases.ts index 45d154fdf7..9daaf4f62e 100644 --- a/resources/databases.ts +++ b/resources/databases.ts @@ -3,7 +3,16 @@ import { initSync, getHdbBasePath, get as envGet } from '../utility/environment/ import { INTERNAL_DBIS_NAME } from '../utility/lmdb/terms.ts'; import { open, compareKeys, type Database, type RootDatabase } from 'lmdb'; import { join, extname, basename } from 'path'; -import { existsSync, mkdirSync, readFileSync, readdirSync, realpathSync } from 'node:fs'; +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + readdirSync, + realpathSync, + unlinkSync, +} from 'node:fs'; import { unlink } from 'node:fs/promises'; import { getBaseSchemaPath, @@ -34,10 +43,12 @@ import { databasePaths, deleteRootBlobPathsForDB } from './blob.ts'; import { removeStorageReclamation } from '../server/storageReclamation.ts'; import { commonValidators, schemaRegex } from '../validation/common_validators.ts'; import { CUSTOM_INDEXES } from './indexes/customIndexes.ts'; +import { planeFilePathFor, planeStalePathFor } from './indexes/hnswPlaneBinding.ts'; import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts'; import { RocksDatabase, supportedCompression, type RocksDatabaseOptions } from '@harperfast/rocksdb-js'; import { PrimaryRocksDatabase } from './PrimaryRocksDatabase.ts'; import { replayLogs } from './replayLogs.ts'; +import { attachDerivedIndexBackends } from './DerivedIndexBackend.ts'; import { totalmem } from 'node:os'; import { RocksIndexStore } from './RocksIndexStore.ts'; import { when } from '../utility/when.ts'; @@ -1301,6 +1312,8 @@ function initStores( table.schemaVersion = 1; if (!destination) databaseEventsEmitter.emit('updateTable', table); } + table.derivedIndexRuntime?.close(); + table.derivedIndexRuntime = attachDerivedIndexBackends(table); if (Array.isArray(primaryAttribute.relationships)) { relationshipsToHydrate.push({ table, databaseName, tableName, definitions: primaryAttribute.relationships }); } else if (primaryAttribute.relationships !== undefined) { @@ -2203,6 +2216,9 @@ function openIndex(dbiKey: string, rootStore: RootDatabaseKind, attribute: any) const CustomIndex = CUSTOM_INDEXES[attribute.indexed.type]; if (CustomIndex) { indexStore.customIndex = new CustomIndex(indexStore, attribute.indexed); + // derived state whose maintaining option is now off must not linger to be adopted + // stale on a later re-enable + indexStore.customIndex.cleanupDisabledPlane?.(); } else { logger.error(`The indexing type '${attribute.indexed.type}' is unknown`); } @@ -2278,6 +2294,18 @@ export function table(tableDefinition: TableDefinition): Tabl // flag must be left as-is. Only an explicit value can re-assert on the existing-Table branch. const schemaDefinedExplicit = tableDefinition.schemaDefined !== undefined; if (schemaDefined == undefined) schemaDefined = true; + if ( + attributes.some((attribute) => attribute.indexed?.type === 'HNSW' && attribute.indexed.nativePlane) && + audit !== true && + // An explicit false must fail here even for an already-audited Table. Nothing clears the + // static, so the runtime would stay attached while the descriptor persists audit: false, + // and the next process start would fail catalog load on the derived-index attach. + (audit === false || Table?.audit !== true) + ) { + throw new ClientError( + `Table '${databaseName}.${tableName}' must explicitly enable audit logging before using nativePlane because its transaction log is the derived-index recovery source` + ); + } const relationshipDefinitions = schemaRelationshipsDefined ? normalizeRelationships(attributes) : undefined; const internalDbiInit = createOpenDBIObject(false); @@ -2853,6 +2881,8 @@ export function table(tableDefinition: TableDefinition): Tabl signalling.signalSchemaChange( new SchemaEventMsg(process.pid, 'schema-change', Table.databaseName, Table.tableName) ); + Table.derivedIndexRuntime?.close(); + Table.derivedIndexRuntime = attachDerivedIndexBackends(Table); Table.origin = origin; if (hasChanges || refreshRelationshipAttributes) { @@ -2985,6 +3015,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { ); let lastResolution; for (const index of indicesToRemove) { + index.customIndex?.resetDerivedStorage?.(); lastResolution = index.drop(); } let interrupted; @@ -3000,6 +3031,7 @@ async function runIndexing(Table, attributes, indicesToRemove) { if (compareKeys(attribute.lastIndexedKey, start) < 0) start = attribute.lastIndexedKey; if (attribute.lastIndexedKey == undefined) { // if we are starting from the beginning, clear out any previous index entries since we are rewriting + attribute.dbi.customIndex?.resetDerivedStorage?.(); if (attribute.dbi.clearAsync) { // LMDB, note that we don't need to wait for this to complete, just gets enqueued in front of the other writes attribute.dbi.clearAsync(); @@ -3213,6 +3245,24 @@ function completeInterruptedDrop(rootStore, attributesDbi, databaseName: string, } finally { columnStore.close(); } + // derived HNSW plane files live next to the store; the normal drop path removes + // them through the custom index, but this recovery path drops raw column stores, + // and a same-name recreate must never open a stale plane over a fresh CF + try { + unlinkSync(planeFilePathFor(rootStore.path, columnName)); + } catch (error: any) { + // a stale plane left behind (e.g. Windows EBUSY while still mapped) would be + // opened over a fresh same-name CF, resolving another graph's node ids + // against it — tombstone it so no attach ever adopts it + if (error?.code !== 'ENOENT') { + logger.warn(`could not delete the HNSW plane file for ${columnName}; tombstoning it as stale`, error); + try { + closeSync(openSync(planeStalePathFor(planeFilePathFor(rootStore.path, columnName)), 'w')); + } catch (tombstoneError) { + logger.warn(`could not tombstone the stale HNSW plane file for ${columnName}`, tombstoneError); + } + } + } } } } else { diff --git a/resources/indexes/HierarchicalNavigableSmallWorld.ts b/resources/indexes/HierarchicalNavigableSmallWorld.ts index 6ee044285b..90f9d4cb7a 100644 --- a/resources/indexes/HierarchicalNavigableSmallWorld.ts +++ b/resources/indexes/HierarchicalNavigableSmallWorld.ts @@ -1,9 +1,22 @@ +import { closeSync, existsSync, openSync, rmSync, statSync, unlinkSync } from 'node:fs'; import { cosineDistance, euclideanDistance, dotProductDistance } from './vector.ts'; import { FLOAT32_OPTIONS } from 'msgpackr'; import { loggerWithTag } from '../../utility/logging/logger.ts'; -import { ClientError } from '../../utility/errors/hdbError.ts'; +import { ClientError, ServerError } from '../../utility/errors/hdbError.ts'; import type { Id } from '../../resources/ResourceInterface.ts'; +import { valuesEqual as derivedValuesEqual } from '../DerivedIndexBackend.ts'; +import type { DerivedIndexRuntime } from '../DerivedIndexBackend.ts'; import { SKIP } from '@harperfast/extended-iterable'; +import { RocksDatabase } from '@harperfast/rocksdb-js'; +import { createHash } from 'node:crypto'; +import { + getPlaneBinding, + invalidatePlaneFile as invalidateHnswPlaneFile, + planeFilePathFor, + planeStalePathFor, + PLANE_NO_ID, + type HnswPlane, +} from './hnswPlaneBinding.ts'; const logger = loggerWithTag('HNSW'); @@ -105,6 +118,22 @@ function autoScaleEfConstruction(nodeCount: number): number { // this only has to be short enough that a table growing from empty picks up a larger ef promptly. const NODE_COUNT_TTL = 10_000; +// Native traversal-plane geometry (see hnsw-native-plane.md). The layer-0 cap is derived from +// M/optimizeRouting at creation to cover the JS graph's effective cap (grace overshoot above it +// truncates by distance); a configuration deriving past this ceiling is refused as ineligible +// rather than silently truncated. maxNodes is a fixed sparse reservation — pages materialize on +// write — and ids at or past it are rejected by the crate, which disables the plane. +const PLANE_LAYER0_CAP_MAX = 1024; +const PLANE_MAX_NODES = 1 << 24; +// An existing plane file that cannot be opened is normally another worker mid-create (retry); +// past this age it is a crashed create and is deleted so the audit-backed runtime can rebuild it. +const PLANE_STALE_CREATE_MS = 60_000; +// Retry cadence while another worker holds the create; its header lands shortly after exclusive open. +const PLANE_ATTACH_RETRY_MS = 250; +// Marks an error thrown by an app-supplied filter during a plane search: the caller re-raises +// it as an ordinary query failure instead of disabling the (healthy) plane. +const NOT_A_PLANE_FAILURE = Symbol('notAPlaneFailure'); + class MinHeap { private data: Candidate[] = []; get size() { @@ -235,6 +264,22 @@ export class HierarchicalNavigableSmallWorld { private convertedNodes = new WeakMap(); private nodeCount = 0; private nodeCountAt = 0; + // Native file-primary index. The RocksDB index store holds identity mappings and replay cursors; + // graph nodes and adjacency exist only in this file. + // undefined = not yet attached (may retry), null = unavailable or disabled for this process. + private plane: HnswPlane | null | undefined; + private planeEligible = false; + private planeReady = false; + private planeRetryAt = 0; + private planeDisabledLogged = false; + private filePrimary = false; + private nativePlaneMaxNodes = PLANE_MAX_NODES; + private derivedRuntime?: DerivedIndexRuntime; + private pendingDerivedMappings = new Map< + Id, + { id?: number; signature?: string; version?: number; pending?: boolean } + >(); + postCommit?: true; constructor(indexStore: any, options: any) { this.indexStore = indexStore; if (indexStore) { @@ -265,8 +310,489 @@ export class HierarchicalNavigableSmallWorld { if (options.optimizeRouting !== undefined) this.optimizeRouting = options.optimizeRouting; if (options.filterExpansion !== undefined) this.filterExpansion = options.filterExpansion; } + if (options?.nativePlane) { + if (!(indexStore?.rootStore instanceof RocksDatabase)) { + throw new ClientError('nativePlane requires the RocksDB storage engine'); + } + const nativeML = 1 / Math.log(16); + if ( + (options.M !== undefined && options.M !== 16) || + (options.efConstruction !== undefined && options.efConstruction !== 200) || + (options.mL !== undefined && options.mL !== nativeML) || + (options.optimizeRouting !== undefined && options.optimizeRouting !== 0.5) + ) { + throw new ClientError('nativePlane requires M=16, efConstruction=200, mL=1/ln(16), and optimizeRouting=0.5'); + } + this.efConstruction = options.efConstruction ?? 200; + this.nativePlaneMaxNodes = options.nativePlaneMaxNodes ?? PLANE_MAX_NODES; + if ( + !Number.isSafeInteger(this.nativePlaneMaxNodes) || + this.nativePlaneMaxNodes < 1 || + this.nativePlaneMaxNodes >= PLANE_NO_ID + ) { + throw new ClientError('nativePlaneMaxNodes must be a positive integer below 2^32-1'); + } + // The plane stores int8 bins and computes asymmetric cosine only, so the flag is a + // no-op for float (quantization: "none") and non-cosine indexes; a graph whose derived + // layer-0 cap exceeds the plane maximum is refused rather than silently truncated. + this.planeEligible = + this.int8 && this.distance === cosineDistance && this.planeLayer0Cap() <= PLANE_LAYER0_CAP_MAX; + if (!this.planeEligible) { + throw new ClientError('nativePlane requires an int8-quantized cosine HNSW index'); + } + this.filePrimary = true; + this.postCommit = true; + } + } + + /** Remove derived native state after the option is disabled. */ + cleanupDisabledPlane(): void { + if (this.planeEligible) return; + const filePath = this.planeFilePath(); + if (!filePath) return; + try { + if (existsSync(filePath)) this.invalidatePlaneFile(filePath, this.plane); + unlinkSync(filePath); + logger.info?.('deleted the HNSW plane file of an index no longer using nativePlane'); + } catch (error: any) { + if (error?.code !== 'ENOENT') { + // A later re-enable must not adopt a file that missed mutations while disabled. + logger.warn?.('could not delete the HNSW plane file; marking it stale', error); + this.invalidatePlaneFile(filePath); + } + } + } + + /** Absolute path of this index's plane file, or undefined when the store exposes no path. */ + planeFilePath(): string | undefined { + const storePath = this.indexStore?.path; + const storeName = this.indexStore?.name; + if (typeof storePath !== 'string' || typeof storeName !== 'string') return undefined; + return planeFilePathFor(storePath, storeName); + } + + /** + * Open or lazily create the native file. An exclusive create resolves multi-worker races; the + * derived-index runtime owns population, replay, and publication. + */ + private getPlane(dims?: number, dimsFromVector = false): HnswPlane | null { + if (this.plane !== undefined) return this.plane; + if (!this.planeEligible) return (this.plane = null); + const now = Date.now(); + if (now < this.planeRetryAt) return null; + const Plane = getPlaneBinding(); + if (!Plane) return (this.plane = null); // the loader warned once already + const filePath = this.planeFilePath(); + if (!filePath) { + this.disablePlane(new Error('the index store exposes no path to place the plane file next to')); + return null; + } + try { + // a tombstone marks a plane a previous unlink could not remove (Windows EBUSY while + // mapped): the file is stale and must never be opened over a fresh graph + const stalePath = planeStalePathFor(filePath); + if (existsSync(stalePath)) { + try { + // force: either artifact may already be gone (the documented rollback deletes the + // plane file by hand), and an ENOENT here disables the plane on every later attach + rmSync(filePath, { force: true }); + rmSync(stalePath, { force: true }); + } catch { + this.planeRetryAt = now + NODE_COUNT_TTL; + return null; + } + } + if (existsSync(filePath)) { + try { + // Crash recovery is per-slot inside the crate. The clean flag is advisory; + // another worker may still be constructing this shared file. + return (this.plane = Plane.open(filePath)); + } catch (openError) { + if (now - statSync(filePath).mtimeMs <= PLANE_STALE_CREATE_MS) { + // another worker is between its exclusive create and the header write + this.planeRetryAt = now + PLANE_ATTACH_RETRY_MS; + return null; + } + logger.warn?.('deleting an unopenable HNSW plane file left by an interrupted create', openError); + unlinkSync(filePath); + if (this.filePrimary) { + // The surviving mappings still name node ids from the file just removed. + // Creating a fresh plane here would resolve them against an empty graph; + // only reconstruction, which clears the mappings first, is safe. + this.planeRetryAt = now + PLANE_ATTACH_RETRY_MS; + return null; + } + } + } + if (!dims) return null; // open-only call and no file: nothing to attach yet + // A search target must not pin an empty index's dimensionality: creation is deferred to + // the first committed vector or to the rebuild scan. Return before the exclusive create + // rather than creating and unlinking — a concurrent insert that saw the empty file would + // read it as another worker's in-progress create and 503 for PLANE_STALE_CREATE_MS. + if (!dimsFromVector) return null; + let fd: number; + try { + fd = openSync(filePath, 'wx'); + } catch { + // another worker won the create race; its header lands within moments + this.planeRetryAt = now + PLANE_ATTACH_RETRY_MS; + return null; + } + closeSync(fd); + try { + return (this.plane = Plane.create(filePath, dims, this.planeLayer0Cap(), this.nativePlaneMaxNodes)); + } catch (createError) { + // Never leave a partial file that a later process could trust as current. + try { + unlinkSync(filePath); + } catch { + // the disable below already forces the JS path for this process + } + this.disablePlane(createError); + return null; + } + } catch (error) { + this.planeRetryAt = now + NODE_COUNT_TTL; + logger.warn?.('could not attach the HNSW plane file; will retry', error); + return null; + } + } + + /** True only after rebuild/replay has crossed a native durability barrier. */ + private planeSearchReady(plane: HnswPlane): boolean { + if (plane.invalidated()) { + this.plane = undefined; + this.planeReady = false; + return false; + } + if (this.indexStore.isIndexing) return false; + if (this.planeReady) return true; + if (plane.getWatermark() > 0) { + this.planeReady = true; + return true; + } + return false; + } + + /** The JS graph's effective layer-0 cap for this configuration; sizes the plane's slots. */ + private planeLayer0Cap(): number { + return this.optimizeRouting ? this.M << 3 : this.M << 1; + } + + /** + * Disable the file-primary index for this process. Invalidation makes the old mapping unusable + * in peer workers; the runtime keeps search unavailable until a rebuild succeeds. + */ + private disablePlane(error: unknown): void { + const attached = this.plane; + this.plane = null; + this.planeReady = false; + const filePath = this.planeFilePath(); + if (filePath) { + if (this.filePrimary && existsSync(filePath)) this.invalidatePlaneFile(filePath, attached); + try { + unlinkSync(filePath); + } catch (unlinkError: any) { + if (unlinkError?.code !== 'ENOENT') { + logger.warn?.('could not delete the disabled HNSW plane file; marking it stale', unlinkError); + this.invalidatePlaneFile(filePath, attached); + } + } + } + if (!this.planeDisabledLogged) { + this.planeDisabledLogged = true; + logger.error?.( + this.filePrimary + ? 'disabling the HNSW native index until it is rebuilt' + : 'disabling the HNSW native plane for this index (falling back to the JS path)', + error + ); + } + if (this.filePrimary) this.derivedRuntime?.requestRebuild(error); + } + + /** + * Delete the derived plane state before an audit-backed reconstruction. Path invalidation + * makes peers stop using an old mapping even when unlink leaves their mmap inode alive. + */ + resetDerivedStorage(): void { + this.pendingDerivedMappings.clear(); + const attached = this.plane; + this.plane = undefined; + this.planeReady = false; + this.planeRetryAt = 0; + const filePath = this.planeFilePath(); + if (!filePath) return; + if (this.filePrimary && existsSync(filePath)) this.invalidatePlaneFile(filePath, attached); + try { + unlinkSync(filePath); + } catch (error: any) { + if (error?.code !== 'ENOENT') { + // a stale file that cannot be deleted (e.g. Windows EBUSY while mapped) must not + // be reopened as if current — mark it so no process ever adopts it + this.plane = null; + logger.warn?.('could not delete the HNSW plane file; marking it stale', error); + this.invalidatePlaneFile(filePath, attached); + } + } + } + + /** True while any node-id mapping survives, which is the only proof this index holds nodes. */ + private hasNodeMappings(): boolean { + // Node ids are the store's numeric keys, so bounding the probe to that key space keeps this + // off a full scan of the primary-key mappings beside them. + for (const { key } of this.indexStore.getRange({ start: 0, end: Number.MAX_SAFE_INTEGER })) { + if (typeof key === 'number') return true; + } + return false; + } + + hasDerivedStorage(): boolean { + const filePath = this.planeFilePath(); + if (!filePath || !existsSync(filePath)) return false; + // A peer's rebuild invalidates the old inode and creates a replacement. This process's + // cached handle still maps the old one, so answering from it reports storage as absent and + // makes every worker rebuild over the last worker's replacement. Reattach first. + if (this.plane?.invalidated()) { + this.plane = undefined; + this.planeReady = false; + this.planeRetryAt = 0; + } + const plane = this.getPlane(); + return Boolean(plane && !plane.invalidated()); + } + + /** Make an undeletable plane unadoptable before this process releases it. */ + private invalidatePlaneFile(filePath: string, attached?: HnswPlane | null): void { + try { + invalidateHnswPlaneFile(filePath, attached); + } catch (error) { + logger.warn?.('could not invalidate the stale HNSW plane file', error); + } } + + /** + * Native search over the plane: one NAPI crossing, traversal on the libuv pool, promise + * resolution maps node ids back to primary keys through the existing pk resolution. The + * predicate adapter runs on this thread's event loop (batched over a ThreadsafeFunction), so + * this promise must never be awaited by code the predicate itself blocks on; the normal + * request path awaits it safely. + */ + private searchPlane( + plane: HnswPlane, + target: number[], + ef: number, + filter: ((primaryKey: Id) => boolean) | undefined, + filterState: FilterState | undefined, + options: any + ): Promise { + const query = Float32Array.from(target); + let resultPromise: Promise<{ id: number; distance: number }[]>; + let predicateError: unknown; + if (filter && filterState) { + const predicate = (ids: number[]): Uint8Array => { + const verdicts = new Uint8Array(ids.length); + if (predicateError !== undefined) return verdicts; // already failed — deny remaining batches cheaply + try { + for (let i = 0; i < ids.length; i++) { + const primaryKey = this.safeGetSync(ids[i], options)?.primaryKey; + if (primaryKey !== undefined && this.admit(filter, filterState, primaryKey)) verdicts[i] = 1; + } + } catch (error) { + // an app-supplied filter threw: deny the batch and surface the error once the + // traversal resolves — the same query failure the JS path raises — instead of + // letting it escape into the fatal-strategy ThreadsafeFunction callback + predicateError ??= error; + } + return verdicts; + }; + // pass the already-resolved JS visit budget verbatim so both paths stop at the same count + resultPromise = plane.searchWithPredicate(query, ef, ef, predicate, undefined, filterState.maxVisits); + } else { + resultPromise = plane.search(query, ef, ef); + } + return resultPromise.then((hits) => { + if (predicateError !== undefined) { + // the plane itself is healthy; mark the failure as the application's so the caller + // re-raises it rather than disabling the plane and retrying + try { + (predicateError as any)[NOT_A_PLANE_FAILURE] = true; + } catch { + // a frozen/primitive throw still propagates, it just also disables the plane + } + throw predicateError; + } + const entries: any[] = []; + try { + for (const hit of hits) { + const mapping = this.safeGetSync(hit.id, options); + if (mapping?.pending) continue; + const primaryKey = mapping?.primaryKey; + if (primaryKey === undefined) continue; // deleted/reused id raced the search + entries.push({ key: primaryKey, distance: hit.distance }); + } + } catch (error) { + // The traversal already succeeded; this is a RocksDB read of the id mappings, which + // can fail transiently or because the store closed under an in-flight search. Tagging + // it keeps the caller from reading it as a plane failure and unlinking a healthy file, + // which costs a full reconstruction. + try { + (error as any)[NOT_A_PLANE_FAILURE] = true; + } catch { + // a frozen/primitive throw still propagates, it just also disables the plane + } + throw error; + } + // nodesVisited stays 0 here: layer-0 visits happen inside the native traversal + // (filterEvaluations is still counted by the predicate adapter) + return withStats(entries, filterState); + }); + } + + attachDerivedRuntime(runtime: DerivedIndexRuntime): void { + this.derivedRuntime = runtime; + } + + prepareCommitted(primaryKey: Id, vector: number[], existingVector: number[], options: any): void { + // Validation is O(dims) and runs on the commit path of every write to the table, so skip it + // when the projection did not change — an unchanged vector was validated when it was first + // written. It must still precede stage(): a rejected write that leaves a target behind makes + // committed() find no audit entry for the id and escalate to a full rebuild. + if (derivedValuesEqual(vector, existingVector)) return; + this.validateVector(primaryKey, vector); + this.derivedRuntime?.stage(options.transaction, primaryKey, vector, existingVector); + } + + private validateVector(primaryKey: Id, vector?: number[]): void { + if (!vector) return; + this.assertPlaneVector(vector, `Vector for attribute "${String(primaryKey)}"`); + } + + /** + * Everything reaching the plane — a committed record's projection or a query target — has to + * convert to the f32 it stores, and to a representable magnitude. What fails here would instead + * throw out of `Float32Array.from` or out of the crate, as an error neither the search path nor + * reconstruction can attribute to the record: a query would unlink a healthy file, and a record + * would abort every rebuild attempt at the same entry. + */ + private assertPlaneVector(vector: number[], label: string): void { + // A positive integer length, not merely a numeric one: a negative or fractional length skips + // the component loop and the emptiness check, and reaches Plane.create with it. + const length = (vector as any)?.length; + if (!Number.isInteger(length) || length < 1) { + throw new ClientError(`${label} must be an array of at least one number.`); + } + let sumOfSquares = 0; + for (let i = 0; i < vector.length; i++) { + const component = vector[i]; + // The type check precedes Math.fround, which throws a TypeError on the BigInt a msgpackr + // or cbor-x decode produces for a large int64. + if (typeof component !== 'number' || !Number.isFinite(Math.fround(component))) { + throw new ClientError( + `${label} has a component at index ${i} that is not a finite 32-bit float: ${String(component)}.` + ); + } + const asFloat32 = Math.fround(component); + sumOfSquares += asFloat32 * asFloat32; + } + // Components can each be f32-finite while their squares are not, which stores invMag 0 and + // makes every distance involving that node NaN. + if (!Number.isFinite(Math.fround(sumOfSquares))) { + throw new ClientError(`${label} has a magnitude too large to represent in 32-bit floats.`); + } + // The plane's dimensionality is fixed at create time by the first committed vector. + const dims = this.plane?.dims; + if (dims !== undefined && vector.length !== dims) { + throw new ClientError(`${label} has ${vector.length} components, but this index stores ${dims}.`); + } + } + + applyDerivedValue(primaryKey: Id, vector: number[], version?: number): void { + this.validateVector(primaryKey, vector); + const safeKey = typeof primaryKey === 'number' ? [KEY_PREFIX, primaryKey] : primaryKey; + const pendingMapping = this.pendingDerivedMappings.get(primaryKey); + const storedMapping = pendingMapping ?? this.indexStore.getSync(safeKey); + const oldNodeId = typeof storedMapping === 'number' ? storedMapping : storedMapping?.id; + if (storedMapping?.version != null && version != null && storedMapping.version > version) return; + const nativeVector = vector ? Float32Array.from(vector) : undefined; + const signature = nativeVector + ? createHash('sha256') + .update(Buffer.from(nativeVector.buffer, nativeVector.byteOffset, nativeVector.byteLength)) + .digest('base64url') + : undefined; + if ( + oldNodeId != null && + signature && + storedMapping.signature === signature && + !pendingMapping && + !storedMapping.pending + ) { + this.indexStore.putSync(safeKey, { id: oldNodeId, signature, version }); + this.indexStore.putSync(oldNodeId, { primaryKey, version }); + return; + } + let plane = vector ? this.getPlane(vector.length, true) : this.getPlane(); + if (plane?.invalidated()) { + this.plane = undefined; + plane = vector ? this.getPlane(vector.length, true) : this.getPlane(); + } + // The pre-commit check above ran before this worker had a plane to compare against, so it + // cannot have caught a mismatch. Reject before the removal below, or the record loses its + // old node on the way to failing. + if (vector && plane && vector.length !== plane.dims) { + throw new ClientError( + `Vector for attribute "${String(primaryKey)}" has ${vector.length} components, but this index stores ${plane.dims}.` + ); + } + if (oldNodeId != null) { + plane?.remove(oldNodeId); + this.indexStore.removeSync(oldNodeId); + } + if (!vector) { + this.indexStore.removeSync(safeKey); + this.pendingDerivedMappings.set(primaryKey, { version }); + return; + } + if (!plane) throw new ServerError('The native HNSW module is unavailable for a file-primary index', 503); + const nodeId = plane.insert(nativeVector!); + this.indexStore.putSync(safeKey, { id: nodeId, signature, version, pending: true }); + this.indexStore.putSync(nodeId, { primaryKey, version, pending: true }); + this.pendingDerivedMappings.set(primaryKey, { id: nodeId, signature, version, pending: true }); + } + + async flushDerived(watermark?: number): Promise { + const plane = this.getPlane(); + if (!plane) { + if (!getPlaneBinding()) throw new ServerError('The native HNSW module is unavailable', 503); + return this.publishDerivedMappings(); + } + await plane.flushAsync(watermark); + this.publishDerivedMappings(); + this.planeReady = true; + } + + private publishDerivedMappings(): void { + for (const [primaryKey, mapping] of this.pendingDerivedMappings) { + const safeKey = typeof primaryKey === 'number' ? [KEY_PREFIX, primaryKey] : primaryKey; + if (mapping.id === undefined) { + this.indexStore.removeSync(safeKey); + } else { + const published = { id: mapping.id, signature: mapping.signature, version: mapping.version }; + this.indexStore.putSync(safeKey, published); + this.indexStore.putSync(mapping.id, { primaryKey, version: mapping.version }); + } + } + this.pendingDerivedMappings.clear(); + } + index(primaryKey: Id, vector: number[], existingVector?: number[], options: any = {}) { + if (this.filePrimary) { + if (options.transaction) return this.prepareCommitted(primaryKey, vector, existingVector, options); + // runIndexing invokes custom indexes without a transaction. The shared runtime waits for + // that schema scan, then owns one primary-record rebuild plus log replay; populating here + // would duplicate native construction and race the runtime's generation reset. + return; + } // Reject non-finite components before touching the graph. NaN in particular poisons // bisectInsert (arr[mid].distance <= NaN is always false → returns 0, pinning the // candidate to rank 1 of every future search). Infinity causes analogous ordering @@ -508,18 +1034,15 @@ export class HierarchicalNavigableSmallWorld { } // Store the new element - this.indexStore.put( - nodeId, - { - vector: storedVector, - scale: storedScale, - invMag, - level, - primaryKey, - ...connections, - }, - options - ); + const storedNode = { + vector: storedVector, + scale: storedScale, + invMag, + level, + primaryKey, + ...connections, + }; + this.indexStore.put(nodeId, storedNode, options); } else { // removal of this node, but first make sure we have a valid entry point if (entryPointId === nodeId) { @@ -1163,6 +1686,55 @@ export class HierarchicalNavigableSmallWorld { filterEvaluations: 0, } : undefined; + if (this.filePrimary && distanceFunction !== this.distance) { + throw new ClientError('A nativePlane index only supports its configured cosine distance'); + } + // The plane traverses the index's own metric (cosine — the eligibility requirement), so a + // query overriding `distance` has to take the JS path: rescoreResults only corrects the + // reported distances of whatever candidates came back, not which candidates the beam kept. + if (this.planeEligible && distanceFunction === this.distance) { + const plane = this.getPlane(target.length, false); + // A file-primary index has no JS path to fall through to, so a target the plane cannot + // accept has to fail as the client error it is. Otherwise it throws out of + // Float32Array.from or the traversal, is read as plane corruption, and unlinks a healthy + // file — one malformed query costing a full reconstruction. + if (this.filePrimary && plane) this.assertPlaneVector(target, 'Search target'); + // a non-file-primary query whose dimensionality differs from the graph's takes the JS + // path, which tolerates the mismatch, rather than disabling the healthy plane + if (plane && plane.dims === target.length && this.planeSearchReady(plane)) { + try { + return this.searchPlane(plane, target, effectiveEf, filter, filterState, options).catch((error) => { + // the query failed for a reason outside the traversal: re-raise instead of disabling the file + if (error?.[NOT_A_PLANE_FAILURE]) throw error; + // There is no JS graph behind a file-primary index: it stays unavailable until + // its audit-backed rebuild succeeds. + this.disablePlane(error); + throw new ServerError('The native HNSW index is rebuilding', 503); + }); + } catch (error) { + // Handle a throw raised before the asynchronous native search returns its promise. + this.disablePlane(error); + throw new ServerError('The native HNSW index is rebuilding', 503); + } + } + } + if (this.filePrimary) { + const planePath = this.planeFilePath(); + if (this.indexStore?.isIndexing || (planePath && existsSync(planePath))) { + throw new ServerError('The native HNSW index is rebuilding', 503); + } + // The absence of a file is not proof of an empty index — it is also the state just after + // any process removes an unopenable one. Only the surviving node mappings prove it. + if (this.hasNodeMappings()) { + // A table taking no further writes never drains, so a query is the only thing left + // that can notice the graph is gone and ask for it back. + this.derivedRuntime?.requestRebuild( + new Error(`${this.indexStore.name} lost its native file while its node mappings survive`) + ); + throw new ServerError('The native HNSW index is rebuilding', 503); + } + return withStats([], filterState); + } let entryPoint = this.getEntryPoint(options); if (!entryPoint) return withStats([], filterState); let entryPointId = entryPoint.id; diff --git a/resources/indexes/hnswPlaneBinding.ts b/resources/indexes/hnswPlaneBinding.ts new file mode 100644 index 0000000000..d11a256803 --- /dev/null +++ b/resources/indexes/hnswPlaneBinding.ts @@ -0,0 +1,156 @@ +import { closeSync, fsyncSync, openSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { loggerWithTag } from '../../utility/logging/logger.ts'; + +const logger = loggerWithTag('HNSW'); + +export interface PlaneSearchHit { + id: number; + distance: number; +} + +/** NAPI surface of the native file-primary HNSW index (`@harperfast/hnsw`). */ +export interface HnswPlane { + readonly dims: number; + readonly layer0Cap: number; + insert(vector: Float32Array): number; + remove(id: number): void; + writeNodeRaw( + id: number, + level: number, + vector: Buffer, + scale: number, + invMag: number, + neighbors: Uint32Array, + upper: Uint32Array[] | null + ): void; + clearNode(id: number): void; + setEntryPoint(id: number, level: number): void; + getEntryPoint(): number[]; + search( + vector: Float32Array, + k: number, + ef: number, + filter?: Uint8Array | null, + filterExpansion?: number | null + ): Promise; + searchWithPredicate( + vector: Float32Array, + k: number, + ef: number, + predicate: (ids: number[]) => Uint8Array, + filterExpansion?: number | null, + visitBudget?: number | null + ): Promise; + searchSync(vector: Float32Array, k: number, ef: number): PlaneSearchHit[]; + writeNodeRawIfAbsent( + id: number, + level: number, + vector: Buffer, + scale: number, + invMag: number, + neighbors: Uint32Array, + upper: Uint32Array[] | null + ): boolean; + openedClean(): boolean; + idHighWater(): number; + getWatermark(): number; + setWatermark(txn: number): void; + flush(watermark?: number): void; + flushAsync(watermark?: number): Promise; + invalidateFile(): PlaneInvalidationOutcome; + invalidated(): boolean; +} + +export interface HnswPlaneConstructor { + create(path: string, dims: number, layer0Cap: number, maxNodes: number): HnswPlane; + open(path: string): HnswPlane; +} + +export interface PlaneInvalidationOutcome { + inBand: boolean; + sidecar: boolean; + inBandError?: string; + sidecarError?: string; +} + +interface HnswPlanePackage { + Plane: HnswPlaneConstructor; + invalidatePlane(path: string): PlaneInvalidationOutcome; + stalePathFor(path: string): string; +} + +/** Entry-point id meaning "none" (u32::MAX in the plane header). */ +export const PLANE_NO_ID = 0xffffffff; + +/** + * Where an index's plane file lives: next to its store, named by the dbiKey + * (`table/attribute`, flattened to a single file name). Exposed separately from the index + * instance so crash-recovery drop paths can remove the file without opening the index. + */ +export function planeFilePathFor(storePath: string, storeName: string): string { + // encodeURIComponent is injective and never emits a path separator: table `a` attribute + // `b.c` and table `a.b` attribute `c` must not share a plane file (dot-flattening let two + // indexes serve each other's node ids as their own primary keys) + return join(storePath, `${encodeURIComponent(storeName)}.hnsw`); +} + +/** + * Tombstone marking a plane file that could not be deleted (e.g. Windows EBUSY while still + * mapped). Its presence means the plane file is STALE: never open it — delete both when + * possible and rebuild. + */ +export function planeStalePathFor(planePath: string): string { + // the package owns the convention; the literal covers the calls that precede its load + return binding?.stalePathFor(planePath) ?? `${planePath}.stale`; +} + +let binding: HnswPlanePackage | null | undefined; + +function getHnswPackage(): HnswPlanePackage | null { + if (binding !== undefined) return binding; + try { + binding = require('@harperfast/hnsw') as HnswPlanePackage; + } catch (error) { + binding = null; + logger.warn?.( + `The @harperfast/hnsw native module is not available (${(error as Error).message}); ` + + 'indexes with nativePlane enabled will remain unavailable until the module can load' + ); + } + return binding; +} + +/** The native plane constructor, or null when the compiled artifact is unavailable (warns once). */ +export function getPlaneBinding(): HnswPlaneConstructor | null { + return getHnswPackage()?.Plane ?? null; +} + +/** Make a derived plane unadoptable before it is replaced or removed. */ +export function invalidatePlaneFile(filePath: string, attached?: HnswPlane | null): PlaneInvalidationOutcome { + if (attached) return attached.invalidateFile(); + const hnswPackage = getHnswPackage(); + if (hnswPackage) return hnswPackage.invalidatePlane(filePath); + // A plane can outlive the package that made it (uninstall, or a prebuild that stopped + // loading), and a reinstall would then adopt it. Only the sidecar is reachable without the + // package, and it has to survive a power loss to be worth writing, so fsync it and the + // directory entry that names it — the same durability the package's own sidecar has. + const fd = openSync(planeStalePathFor(filePath), 'w'); + try { + fsyncSync(fd); + } finally { + closeSync(fd); + } + try { + const dirFd = openSync(dirname(filePath), 'r'); + try { + fsyncSync(dirFd); + } finally { + closeSync(dirFd); + } + } catch { + // Windows cannot open a directory as a file; it also does not need this — the metadata + // journal already orders the create ahead of anything that could read the sidecar + } + return { inBand: false, sidecar: true }; +} diff --git a/resources/search.ts b/resources/search.ts index e4ec82d7b5..9f978b215b 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -550,26 +550,64 @@ export function searchByIndex( // exploring until it has enough MATCHING results, rather than post-filtering an under-filled // candidate set. Only indexes that opt in (filteredSearch) receive it; others post-filter as before. const recordFilter = index.customIndex.filteredSearch ? searchCondition.recordFilter : undefined; - const loaded = index.customIndex.search(searchCondition, context, recordFilter, minResults).map((entry) => { - // if the custom index returns an entry with metadata, merge it with the loaded entry - if (typeof entry === 'object' && entry) { - const { key, ...otherProps } = entry; - if (key == null) return SKIP; // primaryKey missing from HNSW node — skip rather than crash - const loadedEntry = Table.primaryStore.getEntry(key, { - transaction: context && Table._readTxnForContext(context), - }); - if (!loadedEntry) return SKIP; // record was deleted/expired or not yet visible - freezeRecord(loadedEntry?.value); - recordRead(loadedEntry); - return { ...otherProps, ...loadedEntry }; + const searched = index.customIndex.search(searchCondition, context, recordFilter, minResults); + const processEntries = (entries: any[]) => { + const loaded = entries + .map((entry) => { + // if the custom index returns an entry with metadata, merge it with the loaded entry + if (typeof entry === 'object' && entry) { + const { key, ...otherProps } = entry; + if (key == null) return SKIP; // primaryKey missing from HNSW node — skip rather than crash + const loadedEntry = Table.primaryStore.getEntry(key, { + transaction: context && Table._readTxnForContext(context), + }); + if (!loadedEntry) return SKIP; // record was deleted/expired or not yet visible + freezeRecord(loadedEntry?.value); + recordRead(loadedEntry); + return { ...otherProps, ...loadedEntry }; + } + return entry; + }) + .filter((entry) => entry !== SKIP); + if (index.customIndex.rescoreResults) { + const rescored = index.customIndex.rescoreResults(loaded, searchCondition, comparator, attribute_name); + if (rescored != null) return rescored as any; } - return entry; - }); - if (index.customIndex.rescoreResults) { - const rescored = index.customIndex.rescoreResults(loaded, searchCondition, comparator, attribute_name); - if (rescored != null) return rescored as any; + return loaded; + }; + if (typeof (searched as any)?.then === 'function') { + const pending = (searched as Promise).then(processEntries); + // A consumer may abandon this lazy iterable without calling next(). + pending.catch(() => {}); + const results: any = new ExtendedIterable(); + results.iterate = (options?: { async?: boolean }) => { + if (!options?.async) { + throw new Error( + 'This index resolves search results asynchronously; the results must be consumed with async iteration' + ); + } + // Overlapping next() calls share one cursor and cannot duplicate its first entry. + const iteratorPromise = pending.then((entries) => entries[Symbol.iterator]()); + iteratorPromise.catch(() => {}); + let closed = false; + return { + next() { + if (closed) return Promise.resolve({ done: true, value: undefined }); + return iteratorPromise.then((inner) => (closed ? { done: true, value: undefined } : inner.next())); + }, + return(value?: any) { + closed = true; + iteratorPromise.then( + (inner) => (inner as any).return?.(value), + () => {} + ); + return Promise.resolve({ done: true, value }); + }, + }; + }; + return results; } - return loaded; + return processEntries(searched); } const scanned = index.getRange(rangeOptions).map( filter diff --git a/unitTests/resources/hnswDerivedIngest.bench.js b/unitTests/resources/hnswDerivedIngest.bench.js new file mode 100644 index 0000000000..27bbb96697 --- /dev/null +++ b/unitTests/resources/hnswDerivedIngest.bench.js @@ -0,0 +1,244 @@ +// Integrated ingest benchmark for the file-primary HNSW derived index. Excluded from +// `test:unit:resources` by its `.bench.js` name; run it directly: +// +// HOME= npx mocha unitTests/resources/hnswDerivedIngest.bench.js +// +// The meter wraps whole backend methods: `applyDerivedValue` includes the vector hash and the +// RocksDB mapping writes as well as the native insert, and `flushDerived` includes publishing +// those mappings as well as the msync. It therefore bounds the backend's share, and does not +// separate native from JS inside it — the isolated package numbers do that. +// +// The serialized case awaits full drain between writes, so it measures the cost of one isolated +// write, not what a flush cadence could amortize. The repeated-key case counts distinct keys +// across the whole run, not within one delivery window. +require('../testUtils'); +const assert = require('node:assert'); +const { monitorEventLoopDelay } = require('node:perf_hooks'); +const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); +const { table } = require('#src/resources/databases'); +const { getPlaneBinding } = require('#src/resources/indexes/hnswPlaneBinding'); +const { derivedIndexCursorKey } = require('#src/resources/DerivedIndexBackend'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +const DIMS = Number(process.env.BENCH_DIMS ?? 384); +const SEED = Number(process.env.BENCH_SEED ?? 1000); +const BURST = Number(process.env.BENCH_BURST ?? 2000); +const TRICKLE = Number(process.env.BENCH_TRICKLE ?? 40); +const HOT_KEYS = Number(process.env.BENCH_HOT_KEYS ?? 50); +const HOT_ROUNDS = Number(process.env.BENCH_HOT_ROUNDS ?? 20); +const DB = 'vector-ingest-bench'; + +let seedState = 42; +function rand() { + seedState = (seedState * 1103515245 + 12345) % 2147483648; + return seedState / 2147483648; +} +function makeVector() { + const vector = new Array(DIMS); + for (let i = 0; i < DIMS; i++) vector[i] = rand() * 2 - 1; + return vector; +} + +function percentile(samples, fraction) { + if (samples.length === 0) return 0; + const sorted = [...samples].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))]; +} + +describe('HNSW file-primary ingest cost', function () { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; + if (!getPlaneBinding()) { + it.skip('skipped: @harperfast/hnsw native module is unavailable', () => {}); + return; + } + this.timeout(20 * 60_000); + let Bench; + const meter = { applyCount: 0, applyMs: 0, flushCount: 0, flushMs: 0 }; + + function customIndex() { + return Bench.indices.vector.customIndex; + } + + function instrument() { + const index = customIndex(); + const apply = index.applyDerivedValue.bind(index); + const flush = index.flushDerived.bind(index); + index.applyDerivedValue = (key, vector, version) => { + const started = process.hrtime.bigint(); + try { + return apply(key, vector, version); + } finally { + meter.applyMs += Number(process.hrtime.bigint() - started) / 1e6; + meter.applyCount++; + } + }; + // flushDerived awaits the native barrier, so this spans an await and charges anything the + // loop ran meanwhile to the backend. It is an upper bound on barrier cost, and the reason + // the reported backend share is printed as a range with applyMs — which is synchronous and + // exact — as its floor. + index.flushDerived = async (watermark) => { + const started = process.hrtime.bigint(); + try { + return await flush(watermark); + } finally { + meter.flushMs += Number(process.hrtime.bigint() - started) / 1e6; + meter.flushCount++; + } + }; + } + + function resetMeter() { + meter.applyCount = 0; + meter.applyMs = 0; + meter.flushCount = 0; + meter.flushMs = 0; + } + + // Scanning the audit log to find the tail costs more than the drain being measured, so callers + // read the tail before starting a clock and pass it to drained(), which only polls cursors. + function auditTails() { + const tails = []; + const logs = Bench.auditStore.loadLogs(); + for (let nodeId = 0; nodeId < logs.length; nodeId++) { + if (!logs[nodeId]) continue; + let latest; + for (const entry of Bench.auditStore.getRange({ start: 0, log: nodeId })) latest = entry.txnLogKey; + if (latest !== undefined) tails.push([nodeId, latest]); + } + return tails; + } + + const POLL_MS = 1; + + async function drained(tails = auditTails()) { + return waitFor( + () => + tails.every( + ([nodeId, latest]) => + Bench.indices.vector.getSync(derivedIndexCursorKey(Bench.indices.vector.name, nodeId)) === latest + ), + { timeout: 15 * 60_000, interval: POLL_MS, message: 'derived index did not drain' } + ); + } + + before(async () => { + setupTestDBPath(); + setMainIsWorker(true); + Bench = table({ + table: 'IngestBench', + database: DB, + audit: true, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { + name: 'vector', + indexed: { type: 'HNSW', nativePlane: true, efConstruction: 200 }, + type: 'Array', + }, + ], + }); + await Bench.indexingOperation; + await Bench.put(0, { vector: makeVector() }); + await drained(); + instrument(); + // Seed a graph large enough that insert cost reflects a populated index rather than + // the near-empty one a fresh table would measure. + for (let id = 1; id <= SEED; id++) await Bench.put(id, { vector: makeVector() }); + await drained(); + }); + + it('burst ingest: foreground cost, drain cost, and event-loop occupancy', async () => { + resetMeter(); + const loop = monitorEventLoopDelay({ resolution: 1 }); + const putSamples = []; + loop.enable(); + const startedWrites = process.hrtime.bigint(); + for (let id = SEED + 1; id <= SEED + BURST; id++) { + const started = process.hrtime.bigint(); + await Bench.put(id, { vector: makeVector() }); + putSamples.push(Number(process.hrtime.bigint() - started) / 1e6); + } + const writeMs = Number(process.hrtime.bigint() - startedWrites) / 1e6; + loop.disable(); + const writeLoopMax = loop.max / 1e6; + const writeLoopP99 = loop.percentile(99) / 1e6; + // Read the tail before the drain clock starts: scanning the retained log costs more than + // the drain, and inside the window it would be charged to the runtime. + const tails = auditTails(); + loop.reset(); + loop.enable(); + const startedDrain = process.hrtime.bigint(); + await drained(tails); + const drainMs = Number(process.hrtime.bigint() - startedDrain) / 1e6; + const totalMs = writeMs + drainMs; + loop.disable(); + + console.log(`\n== burst ${BURST} puts, dims=${DIMS}, graph≈${SEED + BURST} ==`); + console.log( + ` foreground put: ${(writeMs / BURST).toFixed(3)} ms/put (${Math.round(BURST / (writeMs / 1000))}/s)` + ); + console.log( + ` put p50/p99: ${percentile(putSamples, 0.5).toFixed(3)} / ${percentile(putSamples, 0.99).toFixed(3)} ms` + ); + console.log(` write+drain wall: ${totalMs.toFixed(0)} ms (${Math.round(BURST / (totalMs / 1000))} indexed/s)`); + console.log( + ` applyDerivedValue: ${meter.applyCount} calls, ${meter.applyMs.toFixed(0)} ms total, ${(meter.applyMs / Math.max(1, meter.applyCount)).toFixed(3)} ms/call` + ); + console.log( + ` flushDerived: ${meter.flushCount} calls, ${meter.flushMs.toFixed(0)} ms total, ${(meter.flushMs / Math.max(1, meter.flushCount)).toFixed(3)} ms/call` + ); + console.log( + ` flush share of drain:${((meter.flushMs / Math.max(1, meter.applyMs + meter.flushMs)) * 100).toFixed(1)} %` + ); + console.log( + ` backend share of wall:${((meter.applyMs / Math.max(1, totalMs)) * 100).toFixed(1)}–${(((meter.applyMs + meter.flushMs) / Math.max(1, totalMs)) * 100).toFixed(1)} % of ${totalMs.toFixed(0)} ms (floor = synchronous apply ${meter.applyMs.toFixed(0)} ms; ceiling adds the barrier's ${meter.flushMs.toFixed(0)} ms, timed across an await)` + ); + console.log(` loop max/p99 writing:${writeLoopMax.toFixed(1)} / ${writeLoopP99.toFixed(1)} ms`); + console.log( + ` loop max/p99 draining:${(loop.max / 1e6).toFixed(1)} / ${(loop.percentile(99) / 1e6).toFixed(1)} ms` + ); + assert.ok(meter.applyCount >= BURST); + }); + + it('serialized writes: freshness cost of one isolated write', async () => { + resetMeter(); + const base = SEED + BURST + 1; + for (let n = 0; n < TRICKLE; n++) { + await Bench.put(base + n, { vector: makeVector() }); + await drained(); + } + + // The wall clock here would carry a retained-log scan and a poll interval per record, both + // larger than the work; the meter is what this case is for. + console.log(`\n== serialized ${TRICKLE} puts, full drain between each ==`); + console.log(` indexing work per record: ${((meter.applyMs + meter.flushMs) / TRICKLE).toFixed(2)} ms`); + console.log( + ` applyDerivedValue: ${meter.applyCount} calls, ${(meter.applyMs / Math.max(1, meter.applyCount)).toFixed(3)} ms/call` + ); + console.log( + ` flushDerived: ${meter.flushCount} calls, ${(meter.flushMs / Math.max(1, meter.flushCount)).toFixed(3)} ms/call` + ); + console.log( + ` flush share: ${((meter.flushMs / Math.max(1, meter.applyMs + meter.flushMs)) * 100).toFixed(1)} %` + ); + console.log(` flushes per record: ${(meter.flushCount / TRICKLE).toFixed(2)}`); + }); + + it('repeated keys: upper bound on what per-key coalescing could remove', async () => { + resetMeter(); + const first = SEED + BURST + TRICKLE + 10; + for (let round = 0; round < HOT_ROUNDS; round++) { + for (let k = 0; k < HOT_KEYS; k++) await Bench.put(first + k, { vector: makeVector() }); + } + await drained(); + const writes = HOT_KEYS * HOT_ROUNDS; + console.log(`\n== ${HOT_KEYS} keys × ${HOT_ROUNDS} rounds = ${writes} commits ==`); + console.log(` applyDerivedValue: ${meter.applyCount} calls (${HOT_KEYS} distinct keys)`); + console.log(` time in apply: ${meter.applyMs.toFixed(0)} ms`); + console.log( + ` repeated keys: ${(((meter.applyCount - HOT_KEYS) / Math.max(1, meter.applyCount)) * 100).toFixed(1)} % of apply calls` + ); + console.log(` flushDerived: ${meter.flushCount} calls, ${meter.flushMs.toFixed(0)} ms total`); + }); +}); diff --git a/unitTests/resources/vectorIndexPlane-thread.js b/unitTests/resources/vectorIndexPlane-thread.js new file mode 100644 index 0000000000..c9dcb86949 --- /dev/null +++ b/unitTests/resources/vectorIndexPlane-thread.js @@ -0,0 +1,52 @@ +require('../testUtils'); +const { parentPort } = require('node:worker_threads'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +if (parentPort) { + setupTestDBPath(); + setMainIsWorker(true); + const PlaneTest = table({ + table: 'PlaneTest', + database: 'vector-plane', + audit: true, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'name', indexed: true }, + { + name: 'vector', + indexed: { type: 'HNSW', nativePlane: true, efConstruction: 200 }, + type: 'Array', + }, + ], + }); + + async function waitUntilReady() { + await PlaneTest.indexingOperation; + while (PlaneTest.indices.vector.isIndexing) await new Promise((resolve) => setTimeout(resolve, 10)); + } + + void waitUntilReady().then(() => + parentPort.postMessage({ + type: 'ready', + retainedMarker: PlaneTest.indices.vector.getSync('__native-plane-reopen-marker__'), + }) + ); + parentPort.on('message', async (message) => { + if (message.type === 'shutdown') process.exit(0); + try { + if (message.type === 'commitAndBlock') { + await PlaneTest.put(message.record.id, message.record); + parentPort.postMessage({ type: 'committed' }); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 60_000); + return; + } + if (message.type !== 'put') return; + for (const record of message.records) await PlaneTest.put(record.id, record); + parentPort.postMessage({ type: 'done' }); + } catch (error) { + parentPort.postMessage({ type: 'error', message: error.message, stack: error.stack }); + } + }); +} diff --git a/unitTests/resources/vectorIndexPlane.test.js b/unitTests/resources/vectorIndexPlane.test.js new file mode 100644 index 0000000000..cf14f42fdb --- /dev/null +++ b/unitTests/resources/vectorIndexPlane.test.js @@ -0,0 +1,671 @@ +require('../testUtils'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const { Worker } = require('node:worker_threads'); +const { setupTestDBPath } = require('../testUtils'); +const { waitFor } = require('../waitFor'); +const { table, resetDatabases } = require('#src/resources/databases'); +const { DatabaseTransaction } = require('#src/resources/DatabaseTransaction'); +const { getPlaneBinding } = require('#src/resources/indexes/hnswPlaneBinding'); +const { derivedIndexCursorKey } = require('#src/resources/DerivedIndexBackend'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +async function fromAsync(iterable) { + const out = []; + for await (const value of iterable) out.push(value); + return out; +} + +const DIMS = 24; +const N = 500; +const EF = 200; +const DB = 'vector-plane'; +const RETAINED_MARKER = '__native-plane-reopen-marker__'; +let seedState = 42; +function rand() { + seedState = (seedState * 1103515245 + 12345) % 2147483648; + return seedState / 2147483648; +} +const centers = Array.from({ length: 20 }, () => Array.from({ length: DIMS }, () => rand() * 2 - 1)); +function makeVector(i) { + const center = centers[i % centers.length]; + return center.map((value) => value + (rand() - 0.5) * 0.2); +} +function cosineDistance(a, b) { + let dot = 0; + let aa = 0; + let bb = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + aa += a[i] * a[i]; + bb += b[i] * b[i]; + } + return 1 - dot / Math.sqrt(aa * bb); +} + +describe('HNSW native plane file-primary delivery', function () { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; + if (!getPlaneBinding()) { + it.skip('skipped: @harperfast/hnsw native module is unavailable', () => {}); + return; + } + this.timeout(30_000); + let PlaneTest; + const vectors = new Map(); + + function defineTable() { + return table({ + table: 'PlaneTest', + database: DB, + audit: true, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'name', indexed: true }, + { + name: 'vector', + indexed: { type: 'HNSW', nativePlane: true, efConstruction: 200 }, + type: 'Array', + }, + ], + }); + } + function customIndex() { + return PlaneTest.indices.vector.customIndex; + } + async function nativeSearch(target, filter) { + return await customIndex().search( + { target, comparator: 'sort', distance: 'cosine', ef: EF }, + { transaction: undefined }, + filter + ); + } + async function readySearch(target, filter) { + return waitFor( + async () => { + if (PlaneTest.indices.vector.isIndexing) return false; + try { + return await nativeSearch(target, filter); + } catch (error) { + if (/rebuilding/.test(error.message)) return false; + throw error; + } + }, + { timeout: 15_000, message: 'native plane did not become searchable' } + ); + } + async function waitForKey(id, target) { + return waitFor( + async () => { + const results = await readySearch(target); + return results.some((entry) => entry.key === id); + }, + { timeout: 15_000, message: `native plane did not index record ${id}` } + ); + } + async function waitForCursors() { + return waitFor( + () => { + const logs = PlaneTest.auditStore.loadLogs(); + for (let nodeId = 0; nodeId < logs.length; nodeId++) { + if (!logs[nodeId]) continue; + let latest; + for (const entry of PlaneTest.auditStore.getRange({ start: 0, log: nodeId })) latest = entry.txnLogKey; + if ( + latest !== undefined && + PlaneTest.indices.vector.getSync(derivedIndexCursorKey(PlaneTest.indices.vector.name, nodeId)) !== latest + ) + return false; + } + return true; + }, + { timeout: 15_000, message: 'native plane cursors did not reach the retained audit-log tail' } + ); + } + + before(async () => { + setupTestDBPath(); + setMainIsWorker(true); + PlaneTest = defineTable(); + await PlaneTest.indexingOperation; + const firstVector = makeVector(0); + vectors.set(0, firstVector); + await PlaneTest.put(0, { name: 'rec0', vector: firstVector }); + await waitForKey(0, firstVector); + await waitForCursors(); + for (let i = 1; i < N; i++) { + const vector = makeVector(i); + vectors.set(i, vector); + await PlaneTest.put(i, { name: `rec${i}`, vector }); + } + await waitForKey(3, vectors.get(3)); + await waitFor( + () => { + let mappings = 0; + for (const { key } of PlaneTest.indices.vector.getRange()) if (typeof key === 'number') mappings++; + return mappings >= N; + }, + { timeout: 15_000, message: 'post-commit native delivery did not drain' } + ); + await waitForCursors(); + }); + + it('stores only primary-key mappings and cursors in RocksDB', () => { + assert.ok(fs.existsSync(customIndex().planeFilePath())); + let mappings = 0; + for (const { key, value } of PlaneTest.indices.vector.getRange()) { + if (typeof key !== 'number') continue; + mappings++; + assert.equal(value.level, undefined, 'the CF must not retain HNSW graph nodes'); + assert.equal(value.vector, undefined, 'the CF must not retain graph vectors'); + assert.equal(value.pending, undefined, 'published mappings must follow the native durability barrier'); + assert.notEqual(value.primaryKey, undefined, 'numeric entries are native-id to primary-key mappings'); + } + assert.ok(mappings >= N); + }); + + it('builds searchable native state with deterministic recall', async () => { + for (const probe of [3, 77, 300]) { + const entries = await nativeSearch(vectors.get(probe)); + assert.equal(entries[0].key, probe); + const expected = [...vectors] + .sort((a, b) => cosineDistance(vectors.get(probe), a[1]) - cosineDistance(vectors.get(probe), b[1])) + .slice(0, 10) + .map(([id]) => id); + const returned = new Set(entries.slice(0, 20).map((entry) => entry.key)); + assert.ok( + expected.filter((id) => returned.has(id)).length >= 9, + 'recall@10 in the first 20 must be at least 0.9' + ); + } + }); + + it('does not publish an aborted transaction to the plane', async () => { + const vector = makeVector(50_000); + const plane = customIndex().getPlane(); + const highWater = plane.idHighWater(); + const context = { transaction: new DatabaseTransaction() }; + await PlaneTest.put(50_000, { name: 'aborted', vector }, context); + context.transaction.abort(); + assert.equal(await PlaneTest.get(50_000), undefined); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(plane.idHighWater(), highWater, 'an aborted write must allocate no native node'); + }); + + it('keeps repeated same-key replay mappings pending until the native flush', async () => { + const id = 50_001; + const vector = makeVector(id); + const index = customIndex(); + index.applyDerivedValue(id, vector, 1); + index.applyDerivedValue(id, vector, 2); + assert.ok(!(await nativeSearch(vector)).some((entry) => entry.key === id)); + await index.flushDerived(); + assert.ok((await nativeSearch(vector)).some((entry) => entry.key === id)); + index.applyDerivedValue(id, undefined, 3); + await index.flushDerived(); + }); + + it('ignores unrelated field changes and applies committed update/delete', async () => { + const plane = customIndex().getPlane(); + const highWater = plane.idHighWater(); + await PlaneTest.put(3, { name: 'renamed', vector: vectors.get(3) }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(plane.idHighWater(), highWater, 'an unchanged vector must schedule no native insert'); + + const updated = makeVector(60_003); + vectors.set(3, updated); + await PlaneTest.put(3, { name: 'renamed', vector: updated }); + await waitForKey(3, updated); + await PlaneTest.delete(77); + vectors.delete(77); + await waitFor(async () => !(await readySearch(updated)).some((entry) => entry.key === 77), { + timeout: 10_000, + message: 'deleted native id remained searchable', + }); + }); + + it('applies predicates and full-stack exact rescoring', async () => { + const filtered = await readySearch(vectors.get(21), (id) => id % 3 === 0); + assert.ok(filtered.length > 0); + for (const entry of filtered) assert.equal(entry.key % 3, 0); + const target = vectors.get(42); + const results = await fromAsync( + PlaneTest.search({ + sort: { attribute: 'vector', target, distance: 'cosine' }, + select: ['id', '$distance'], + limit: 10, + }) + ); + assert.equal(results[0].id, 42); + for (let i = 1; i < results.length; i++) assert.ok(results[i].$distance >= results[i - 1].$distance); + const withCondition = await fromAsync( + PlaneTest.search({ + sort: { attribute: 'vector', target, distance: 'cosine' }, + conditions: [{ attribute: 'name', comparator: 'gt', value: 'rec9' }], + select: ['id', 'name'], + limit: 20, + }) + ); + assert.ok(withCondition.length > 0); + for (const record of withCondition) assert.ok(record.name > 'rec9'); + const within = await fromAsync( + PlaneTest.search({ + conditions: [{ attribute: 'vector', comparator: 'le', value: 0.05, target }], + select: ['id', '$distance'], + }) + ); + assert.ok(within.length > 0, 'le threshold query should return nearby records'); + for (const record of within) assert.ok(record.$distance <= 0.05, `distance ${record.$distance} exceeds threshold`); + }); + + it('surfaces a throwing app filter without disabling the native plane', async () => { + const target = vectors.get(3); + await assert.rejects( + nativeSearch(target, () => { + throw new Error('filter boom'); + }), + /filter boom/ + ); + assert.ok((await readySearch(target)).length > 0); + }); + + it('rejects a wrong-dimension write and query as client errors, not index failures', async () => { + const wrong = makeVector(0).concat(1); + await assert.rejects( + async () => PlaneTest.put(90_001, { name: 'wrong-dims', vector: wrong }), + (error) => { + assert.match(error.message, /components, but this index stores/); + assert.equal(error.statusCode, 400, 'a wrong-length vector is the caller mistake, not a 503'); + return true; + } + ); + // The native insert would throw an unclassifiable error instead, which reconstruction + // rethrows — one such record would abort every rebuild and strand the index at 503. + assert.equal(await PlaneTest.get(90_001), undefined, 'the rejected write must not commit'); + await assert.rejects( + async () => nativeSearch(wrong), + (error) => { + assert.match(error.message, /Search target has/); + assert.equal(error.statusCode, 400); + return true; + } + ); + // A BigInt component would throw out of Float32Array.from, which the caller reads as plane + // corruption and answers by unlinking a healthy file. + const bigintTarget = vectors.get(3).slice(); + bigintTarget[0] = 1n; + await assert.rejects(async () => nativeSearch(bigintTarget), /not a finite 32-bit float/); + assert.ok(fs.existsSync(customIndex().planeFilePath()), 'a malformed query must not unlink the plane'); + assert.ok((await readySearch(vectors.get(3))).length > 0, 'the index stays healthy'); + }); + + it('rejects a component outside the f32 range instead of stranding reconstruction', async () => { + const overflowing = makeVector(0).slice(); + overflowing[0] = 1e39; // finite as a double, Infinity as the f32 the plane stores + await assert.rejects( + async () => PlaneTest.put(90_002, { name: 'f32-overflow', vector: overflowing }), + (error) => { + assert.match(error.message, /not a finite 32-bit float/); + assert.equal(error.statusCode, 400); + return true; + } + ); + assert.equal(await PlaneTest.get(90_002), undefined, 'the rejected write must not commit'); + + // f32-finite components whose squares are not: invMag would store 0 and every distance + // involving the node would be NaN. + const huge = makeVector(0).map(() => 1e20); + await assert.rejects( + async () => PlaneTest.put(90_003, { name: 'f32-magnitude', vector: huge }), + /magnitude too large/ + ); + + // Math.fround throws a TypeError on a BigInt, which is not a ClientError and would strand + // reconstruction exactly as the native throw did. + const bigint = makeVector(0).slice(); + bigint[0] = 1n; + await assert.rejects( + async () => PlaneTest.put(90_004, { name: 'f32-bigint', vector: bigint }), + /not a finite 32-bit float/ + ); + + assert.ok((await readySearch(vectors.get(3))).length > 0, 'the index stays healthy'); + }); + + it('rejects synchronous iteration of asynchronous plane-backed results', () => { + const results = PlaneTest.search({ + sort: { attribute: 'vector', target: vectors.get(42), distance: 'cosine' }, + select: ['id'], + limit: 5, + }); + assert.throws(() => [...results], /async/i, 'sync iteration must throw instead of spinning'); + }); + + it('serializes overlapping next calls on one plane-backed result cursor', async () => { + const query = () => ({ + sort: { attribute: 'vector', target: vectors.get(42), distance: 'cosine' }, + select: ['id'], + limit: 5, + }); + const sequential = (await fromAsync(PlaneTest.search(query()))).map((record) => record.id); + assert.ok(sequential.length > 2, 'need several results to detect a duplicate or skip'); + const iterator = PlaneTest.search(query()).iterate({ async: true }); + const [first, second] = await Promise.all([iterator.next(), iterator.next()]); + const seen = [first.value.id, second.value.id]; + for (let next = await iterator.next(); !next.done; next = await iterator.next()) seen.push(next.value.id); + assert.deepEqual(seen, sequential, 'overlapping next calls must yield the sequential order exactly once'); + }); + + it('does not raise an unhandled rejection when a plane-backed iterable is abandoned', async () => { + const index = customIndex(); + const unhandled = []; + const onUnhandled = (reason) => unhandled.push(reason); + process.on('unhandledRejection', onUnhandled); + try { + index.rescoreResults = () => { + throw new Error('rescore boom'); + }; + const results = PlaneTest.search({ + sort: { attribute: 'vector', target: vectors.get(42), distance: 'cosine' }, + select: ['id'], + limit: 5, + }); + await results.iterate({ async: true }).return(); + await new Promise((resolve) => setTimeout(resolve, 100)); + } finally { + delete index.rescoreResults; + process.off('unhandledRejection', onUnhandled); + } + assert.deepEqual( + unhandled.map((reason) => String(reason?.message ?? reason)), + [], + 'an unobserved rejection here exits the process under Node default policy' + ); + }); + + it('serializes post-commit delivery from two workers into one native file', async () => { + PlaneTest.indices.vector.putSync(RETAINED_MARKER, true); + const worker = new Worker(require.resolve('./vectorIndexPlane-thread.js'), { + workerData: { workerIndex: 1, workerCount: 2 }, + }); + const nextMessage = () => + new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); + try { + const ready = await nextMessage(); + assert.equal(ready.type, 'ready'); + assert.equal(ready.retainedMarker, true, 'a peer worker should reopen a current plane without rebuilding it'); + const workerRecords = []; + for (let i = 0; i < 20; i++) { + const id = 2_000 + i; + const vector = makeVector(id); + vectors.set(id, vector); + workerRecords.push({ id, name: `worker${i}`, vector }); + } + const workerDone = nextMessage(); + worker.postMessage({ type: 'put', records: workerRecords }); + for (let i = 0; i < 20; i++) { + const id = 3_000 + i; + const vector = makeVector(id); + vectors.set(id, vector); + await PlaneTest.put(id, { name: `main${i}`, vector }); + } + const result = await workerDone; + assert.equal(result.type, 'done', result.stack ?? result.message); + await waitForKey(2_003, vectors.get(2_003)); + await waitForKey(3_003, vectors.get(3_003)); + } finally { + await worker.terminate(); + } + const replacement = new Worker(require.resolve('./vectorIndexPlane-thread.js'), { + workerData: { workerIndex: 1, workerCount: 2 }, + }); + try { + const ready = await new Promise((resolve, reject) => { + replacement.once('message', resolve); + replacement.once('error', reject); + }); + assert.equal(ready.type, 'ready'); + assert.equal(ready.retainedMarker, true, 'a replacement worker should reopen rather than rebuild the plane'); + await waitForCursors(); + } finally { + await replacement.terminate(); + } + }); + + it('replays a committed write after its worker terminates before delivery', async () => { + const id = 4_000; + const vector = makeVector(id); + vectors.set(id, vector); + const worker = new Worker(require.resolve('./vectorIndexPlane-thread.js'), { + workerData: { workerIndex: 1, workerCount: 2 }, + }); + try { + const ready = await new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); + assert.equal(ready.type, 'ready'); + const committed = new Promise((resolve, reject) => { + worker.once('message', resolve); + worker.once('error', reject); + }); + worker.postMessage({ type: 'commitAndBlock', record: { id, name: 'crash-window', vector } }); + assert.equal((await committed).type, 'committed'); + } finally { + await worker.terminate(); + } + + const replacement = new Worker(require.resolve('./vectorIndexPlane-thread.js'), { + workerData: { workerIndex: 1, workerCount: 2 }, + }); + try { + const ready = await new Promise((resolve, reject) => { + replacement.once('message', resolve); + replacement.once('error', reject); + }); + assert.equal(ready.type, 'ready'); + await waitForKey(id, vector); + } finally { + await replacement.terminate(); + } + }); + + it('rebuilds after a whole-table snapshot reload marker', async () => { + PlaneTest.indices.vector.putSync(999_998, { primaryKey: 'stale-snapshot-mapping' }); + await PlaneTest.writeReloadMarker(); + await waitFor( + () => !PlaneTest.indices.vector.isIndexing && PlaneTest.indices.vector.getSync(999_998) === undefined, + { timeout: 15_000, message: 'whole-table reload marker did not replace derived mappings' } + ); + await waitForKey(42, vectors.get(42)); + }); + + it('rebuilds when a replacement worker replays a snapshot reload marker', async () => { + PlaneTest.indices.vector.putSync(999_997, { primaryKey: 'stale-offline-snapshot-mapping' }); + PlaneTest.derivedIndexRuntime.close(); + await PlaneTest.writeReloadMarker(); + resetDatabases(); + PlaneTest = defineTable(); + await waitFor( + () => !PlaneTest.indices.vector.isIndexing && PlaneTest.indices.vector.getSync(999_997) === undefined, + { timeout: 15_000, message: 'replayed reload marker did not replace derived mappings' } + ); + await waitForKey(42, vectors.get(42)); + }); + + it('recovers searchable native state across a database reset', async () => { + const planePath = customIndex().planeFilePath(); + resetDatabases(); + PlaneTest = defineTable(); + await waitForKey(42, vectors.get(42)); + assert.ok(fs.existsSync(planePath)); + }); + + it('rebuilds from primary records when its durable cursor is outside audit retention', async () => { + const index = customIndex(); + const planePath = index.planeFilePath(); + PlaneTest.indices.vector.putSync(999_999, { primaryKey: 'stale-derived-mapping' }); + PlaneTest.indices.vector.putSync(derivedIndexCursorKey(PlaneTest.indices.vector.name, 0), 1); + + resetDatabases(); + PlaneTest = defineTable(); + await waitForKey(42, vectors.get(42)); + assert.equal( + PlaneTest.indices.vector.getSync(999_999), + undefined, + 'the retention gap must replace stale derived mappings from primary records' + ); + assert.ok(fs.existsSync(planePath)); + }); + + it('answers an empty index with no results instead of a rebuilding 503', async () => { + const EmptyTable = table({ + table: 'PlaneEmpty', + database: DB, + audit: true, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', nativePlane: true, efConstruction: 200 }, type: 'Array' }, + ], + }); + const index = EmptyTable.indices.vector.customIndex; + await waitFor(() => !EmptyTable.indices.vector.isIndexing, { + timeout: 15_000, + message: 'the empty derived runtime never became ready', + }); + const results = await index.search( + { target: makeVector(0), comparator: 'sort', distance: 'cosine', ef: EF }, + { transaction: undefined } + ); + assert.deepEqual([...results], []); + // The search target must not publish a placeholder file: a concurrent first insert would + // read it as another worker's in-progress create and reject the write for a full minute. + assert.ok(!fs.existsSync(index.planeFilePath())); + await EmptyTable.dropTable(); + }); + + it('reports 503 rather than no results when a populated index has lost its file', async () => { + const LostFile = table({ + table: 'PlaneLostFile', + database: DB, + audit: true, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', nativePlane: true, efConstruction: 200 }, type: 'Array' }, + ], + }); + await LostFile.indexingOperation; + const index = LostFile.indices.vector.customIndex; + const probe = makeVector(5); + await LostFile.put(1, { vector: probe }); + await waitFor( + async () => { + if (LostFile.indices.vector.isIndexing) return false; + const hits = await index.search( + { target: probe, comparator: 'sort', distance: 'cosine', ef: EF }, + { + transaction: undefined, + } + ); + return [...hits].some((entry) => entry.key === 1); + }, + { timeout: 15_000, message: 'the populated index never became searchable' } + ); + index.plane = undefined; + fs.rmSync(index.planeFilePath(), { force: true }); + // The node mappings survive the file, so emptiness is not proven: answering [] here would + // hide every indexed vector behind a query that looks successful. + assert.throws( + () => index.search({ target: probe, comparator: 'sort', distance: 'cosine', ef: EF }, { transaction: undefined }), + /rebuilding/ + ); + await LostFile.dropTable(); + }); + + it('requires audit logging and native construction geometry', () => { + assert.throws( + () => + table({ + table: 'PlaneNoAudit', + database: DB, + audit: false, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', nativePlane: true }, type: 'Array' }, + ], + }), + /audit logging/ + ); + assert.throws( + () => + table({ + table: 'PlaneBadGeometry', + database: DB, + audit: true, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', nativePlane: true, M: 8 }, type: 'Array' }, + ], + }), + /requires M=16/ + ); + for (const bad of [[], { length: -1 }, { length: 1.5 }, {}]) { + assert.throws( + () => customIndex().prepareCommitted('bad-vector', bad, undefined, { transaction: {} }), + /must be an array of at least one number/, + `${JSON.stringify(bad)} must not reach the plane` + ); + } + // An already-audited table must not be able to turn auditing off underneath a nativePlane + // index: nothing clears Table.audit, so the descriptor would persist audit: false and the + // next process start would fail catalog load on the derived-index attach. + assert.throws( + () => + table({ + table: 'PlaneTest', + database: DB, + audit: false, + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'name', indexed: true }, + { + name: 'vector', + indexed: { type: 'HNSW', nativePlane: true, efConstruction: 200 }, + type: 'Array', + }, + ], + }), + /audit logging/ + ); + }); + + (process.env.HNSW_NATIVE_REBUILD_BENCHMARK ? it : it.skip)( + 'sustains the native rebuild insertion floor for 100k records', + async function () { + this.timeout(180_000); + const total = 100_000; + const index = customIndex(); + const startedAt = Date.now(); + for (let id = 100_000; id < 100_000 + total; id++) { + index.applyDerivedValue(id, makeVector(id), id); + if (id % 10_000 === 9_999) { + await index.flushDerived(id); + const indexed = id - 100_000 + 1; + const rate = Math.round(indexed / Math.max((Date.now() - startedAt) / 1_000, 0.001)); + const eta = Math.ceil((total - indexed) / Math.max(rate, 1)); + console.log(`Native rebuild benchmark: ${indexed}/${total} records, ${rate}/s, ETA ${eta}s`); + } + } + const rate = total / Math.max((Date.now() - startedAt) / 1_000, 0.001); + assert.ok(rate >= 1_000, `native rebuild rate ${Math.round(rate)}/s is below the 1,000/s floor`); + } + ); + + it('removes the native file when the table is dropped', async () => { + const planePath = customIndex().planeFilePath(); + await PlaneTest.dropTable(); + assert.ok(!fs.existsSync(planePath)); + }); +});