From a098e64429012d3790821e9e41a1ac0881e1c0cb Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 2 Sep 2026 17:37:58 -0600 Subject: [PATCH] Size tables for query planning with the O(1) RocksDB key estimate estimatedEntryCount() called RocksDatabase.getKeysCount(), a synchronous native iteration of the entire key space, once per store per 10-second memo window. On a 28.6M-row table that is ~11s on the main thread, and the memo never took effect there: `now` is sampled before the call, so `now + 10000` is already in the past when it returns. The query planner calls it once per additional condition through intersectionEstimate and the range heuristics, so an N-condition query paid N-1 full scans -- a 10-condition operation measured ~100s of main-thread stall with the Operations API unresponsive throughout. Read `rocksdb.estimate-num-keys` instead, which is O(1) and returned the identical count on the affected table. estimate-num-keys skews high on overwrite/delete-heavy data until compaction; every consumer is a relative-ordering or explicitly-estimated path, so the accuracy-per-cost trade is deliberate. It also reports 0 for a *populated* table once accumulated tombstones reach its non-deletions, which getKeysCount() never did -- there a 0 meant a genuinely empty store. Zero is still the truthful count, and Table.ts reports it verbatim as an estimated pagination total, so it is left alone and the two arithmetic sites that cannot take it are guarded instead: - the AND-group branch of estimateConditionForTable divides by it, yielding Infinity; - the `ne null` branch subtracts the index's null count from it, yielding a negative estimate. The exact count could not underrun getValuesCount(null); a 0 estimate can. The divisor matters to planning: the adaptive filter/index switch derives thresholdRemainingMisses as `estimated_count >> 4`, and Infinity >> 4 is 0 rather than NaN, so the isNaN check passes it and the first non-matching record flips the request onto searchByIndex plus a full Set build. Guarded, the same group estimates 200 and the threshold is a usable 12. It uses the `|| 1` idiom already applied to the sibling relationship divisor two branches below. The subtraction does not affect planning -- -32 and 0 both trip the switch on the first miss -- but a negative cardinality is not a value to hand to a caller: on main Table.ts reports it verbatim as an estimated pagination total. It is clamped with Math.max at 0, not 1, since `|| 1` does not catch a negative and a table whose rows are all null has a true `ne null` count of zero. Backport of the estimatedEntryCount portion of #2163, whose helper and intersectionEstimate are taken byte-identical; the two arithmetic guards are the addition, and go to main separately. getEstimatedKeyCount() has been present since rocksdb-js 2.4.x, so this needs no dependency bump; the statistical range-estimate portion of #2163, which does require 2.8.0, is deliberately not included. Co-Authored-By: Claude Opus 5 --- resources/search.ts | 18 ++-- .../resources/estimatedEntryCount.test.js | 96 +++++++++++++++++++ 2 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 unitTests/resources/estimatedEntryCount.test.js diff --git a/resources/search.ts b/resources/search.ts index c1f7dcafdf..46cf6d1272 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -1131,7 +1131,7 @@ export function estimateCondition(table) { for (const subCondition of condition.conditions) { estimateConditionForTable(subCondition); estimatedCount = isFinite(estimatedCount) - ? (estimatedCount * subCondition.estimated_count) / estimatedEntryCount(table.primaryStore) + ? (estimatedCount * subCondition.estimated_count) / (estimatedEntryCount(table.primaryStore) || 1) : subCondition.estimated_count; } } @@ -1174,8 +1174,10 @@ export function estimateCondition(table) { const attribute_name = condition[0] ?? condition.attribute; const index = table.indices[attribute_name]; if (condition.value === null && searchType === 'ne') { - condition.estimated_count = - estimatedEntryCount(table.primaryStore) - (index ? index.getValuesCount(null) : 0); + condition.estimated_count = Math.max( + estimatedEntryCount(table.primaryStore) - (index ? index.getValuesCount(null) : 0), + 0 + ); } else condition.estimated_count = Infinity; } else if (searchType === 'in') { const attribute_name = condition[0] ?? condition.attribute; @@ -1619,16 +1621,18 @@ export function flattenKey(key) { return key; } -function estimatedEntryCount(store) { +export function estimatedEntryCount(store) { const now = Date.now(); if ((store.estimatedEntryCountExpires || 0) < now) { - // use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own - store.estimatedEntryCount = store instanceof RocksDatabase ? store.getKeysCount() : store.getStats().entryCount; + // getStats is the LMDB fast path; for RocksDB, estimate-num-keys is O(1) where an exact + // getKeysCount() would iterate the entire store + store.estimatedEntryCount = + store instanceof RocksDatabase ? store.getEstimatedKeyCount() : store.getStats().entryCount; store.estimatedEntryCountExpires = now + 10000; } return store.estimatedEntryCount; } export function intersectionEstimate(store, left, right) { - return (left * right) / estimatedEntryCount(store); + return (left * right) / Math.max(estimatedEntryCount(store), 1); } diff --git a/unitTests/resources/estimatedEntryCount.test.js b/unitTests/resources/estimatedEntryCount.test.js new file mode 100644 index 0000000000..14a200f531 --- /dev/null +++ b/unitTests/resources/estimatedEntryCount.test.js @@ -0,0 +1,96 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { RocksDatabase } = require('@harperfast/rocksdb-js'); +const { estimateCondition, estimatedEntryCount, intersectionEstimate } = require('#src/resources/search'); + +describe('estimatedEntryCount', () => { + const { setupTestDBPath } = require('../testUtils'); + const { table } = require('#src/resources/databases'); + const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + const N = 2000; + let T; + + before(async function () { + this.timeout(120000); + setupTestDBPath(); + setMainIsWorker(true); + T = table({ + table: 'EntryCountTest', + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }], + }); + const written = []; + for (let i = 0; i < N; i++) { + written.push(T.put({ id: i })); + } + await Promise.all(written); + if (typeof T.primaryStore.flush === 'function') await T.primaryStore.flush(); + }); + + it('reads the O(1) key-count estimate rather than iterating the store', function () { + const store = T.primaryStore; + // gate on the engine, not on the method: under RocksDB a renamed or dropped + // getEstimatedKeyCount must fail this guard rather than silently skip it + if (!(store instanceof RocksDatabase)) return this.skip(); + store.estimatedEntryCountExpires = 0; + const getKeysCount = store.getKeysCount; + store.getKeysCount = () => assert.fail('estimatedEntryCount must not iterate the whole key space'); + let estimate; + try { + estimate = estimatedEntryCount(store); + } finally { + store.getKeysCount = getKeysCount; + } + assert.ok(estimate >= N / 2 && estimate <= N * 2, `estimate ${estimate} is not in range for ${N} rows`); + }); + + it('memoizes the estimate for 10 seconds', () => { + const store = T.primaryStore; + store.estimatedEntryCountExpires = 0; + const first = estimatedEntryCount(store); + const { getEstimatedKeyCount, getStats } = store; + const reprobed = () => assert.fail('memoized estimate must not re-probe the store'); + store.getEstimatedKeyCount = reprobed; + store.getStats = reprobed; + try { + assert.strictEqual(estimatedEntryCount(store), first); + } finally { + store.getEstimatedKeyCount = getEstimatedKeyCount; + store.getStats = getStats; + } + }); +}); + +describe('a store estimating zero entries', () => { + // estimate-num-keys reports this for a populated table whose tombstones reach its non-deletions + const churned = { getStats: () => ({ entryCount: 0 }) }; + + it('is reported as zero rather than floored, so callers see the real count', () => { + assert.strictEqual(estimatedEntryCount(churned), 0); + }); + + it('keeps intersectionEstimate finite', () => { + churned.estimatedEntryCountExpires = 0; + assert.strictEqual(intersectionEstimate(churned, 5, 7), 35); + }); + + it('keeps a `ne null` estimate non-negative, so an estimated total is never below zero', () => { + churned.estimatedEntryCountExpires = 0; + const table = { + primaryKey: 'id', + primaryStore: churned, + indices: { attr: { getValuesCount: () => 500 } }, + attributes: [], + }; + const estimated = estimateCondition(table)({ attribute: 'attr', comparator: 'ne', value: null }); + assert.strictEqual(estimated, 0); + }); + + it("keeps an AND group's estimate finite, so the adaptive index guard still applies", () => { + churned.estimatedEntryCountExpires = 0; + const table = { primaryKey: 'id', primaryStore: churned, indices: {}, attributes: [] }; + const condition = { operator: 'and', conditions: [{ estimated_count: 10 }, { estimated_count: 20 }] }; + const estimated = estimateCondition(table)(condition); + assert.strictEqual(estimated, 200); + }); +});