From 1c4be8df71970c8ea45793f3a46e8306ff7c08c4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 13 Aug 2026 10:13:42 -0600 Subject: [PATCH 01/15] feat: statistical range key-count estimation with progressive refinement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Database::EstimateCount — a no-iteration range key-count estimate built from RocksDB statistics: GetApproximateMemTableStats supplies the memtable entry count directly, and the SST portion converts approximate file bytes in range (GetApproximateSizes) to entries via the live-entry density of only the SSTs overlapping the range (GetPropertiesOfTablesInRange: (num_entries - num_deletions) / file bytes). Open-ended ranges subtract the complementary range from estimate-num-keys rather than passing an empty upper-bound slice (which would denote the smallest key). Public API: getEstimatedKeyCount(options?: RangeOptions) extends the existing whole-DB method with range support, and createCountEstimator() returns a CountEstimator that rides an iterator: advance(lastKey, n) checkpoints progress and estimate() returns the exact traversed count plus a remainder estimate calibrated by the observed actual/estimated ratio over the traversed portion, converging toward the exact total. Closes #205 Co-Authored-By: Claude Fable 5 --- README.md | 38 +++++++- src/binding/database/database.cpp | 119 ++++++++++++++++++++++ src/binding/database/database.h | 1 + src/count-estimator.ts | 98 +++++++++++++++++++ src/database.ts | 45 ++++++++- src/index.ts | 1 + src/load-binding.ts | 1 + src/store.ts | 23 +++++ test/estimate-count.test.ts | 157 ++++++++++++++++++++++++++++++ 9 files changed, 474 insertions(+), 9 deletions(-) create mode 100644 src/count-estimator.ts create mode 100644 test/estimate-count.test.ts diff --git a/README.md b/README.md index 1d69984f3..048d488f1 100644 --- a/README.md +++ b/README.md @@ -429,14 +429,44 @@ if (result === constants.FRESH_VERSION_FLAG) { Synchronous version of `get()`. Like `get()`, this can return the `FRESH_VERSION_FLAG` sentinel when the `expectedVersion` option is used. -### `db.getEstimatedKeyCount(): number` +### `db.getEstimatedKeyCount(options?: RangeOptions): number` -Retrieves the estimated number of keys in the database. This is an alias for -`db.getDBIntProperty('rocksdb.estimate-num-keys')`. +Retrieves the estimated number of keys in the database, or within a key range when one is given. +Unlike `getKeysCount()`, this never iterates: the estimate is derived from RocksDB statistics +(memtable stats plus approximate SST sizes converted through the entry density of the SSTs +overlapping the range), so it stays fast regardless of range size — typically microseconds where an +exact count takes milliseconds. Accuracy improves with range size (resolution is bounded by SST +data-block granularity, so tiny ranges over-report), and recently deleted or overwritten entries +may be counted until compaction. Estimates always reflect committed state; writes pending in a +transaction are not included. ```typescript const estimated = db.getEstimatedKeyCount(); -console.log(estimated); +const rangeEstimate = db.getEstimatedKeyCount({ start: 'a', end: 'z' }); +``` + +### `db.createCountEstimator(options?: CountEstimatorOptions): CountEstimator` + +Creates an estimator that progressively refines a range count estimate while the range is being +iterated — useful for reporting a total alongside a page of results without scanning the full +range. Before any traversal, `estimate()` returns the pure statistical estimate (same as +`getEstimatedKeyCount(range)`). As the caller reports progress with `advance(lastKey, count)` +(e.g. once per page), `estimate()` returns the exact traversed count plus a statistical estimate +of the remainder, calibrated by the observed ratio of actual-to-estimated entries over the portion +already traversed — so the estimate converges toward the exact total as iteration proceeds. Set +`reverse: true` when iterating from `end` toward `start`. + +```typescript +const range = { start: 'a', end: 'z' }; +const estimator = db.createCountEstimator(range); +let lastKey; +let pageSize = 0; +for (const { key } of db.getRange({ ...range, limit: 25 })) { + lastKey = key; + pageSize++; +} +estimator.advance(lastKey, pageSize); +const total = estimator.estimate(); // ~total keys in the range ``` ### `db.getKeys(options?: IteratorOptions): ExtendedIterable` diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5beb92f72..5674d3999 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -922,6 +922,124 @@ napi_value Database::GetCount(napi_env env, napi_callback_info info) { return result; } +/** + * Estimates the number of live keys in `[start, end)` from RocksDB statistics + * alone — no iteration: + * + * - memtable portion: `GetApproximateMemTableStats` returns an entry count + * directly (it counts all memtable entries, including tombstones and + * overwrites, so it can over-report a recently-deleted range). + * - SST portion: the approximate file bytes covered by the range + * (`GetApproximateSizes`) converted to entries using the live-entry density + * of only the SSTs overlapping the range (`GetPropertiesOfTablesInRange`: + * `(num_entries - num_deletions) / file bytes`). Using range-local table + * properties keeps the density honest when entry sizes vary across the + * keyspace, and needs no cache/invalidation. + * + * Overlapping versions of a key in multiple levels are counted once per + * level, so the estimate skews high on heavily-overwritten ranges until + * compaction; resolution is bounded by SST data-block granularity, so tiny + * ranges over-report. + */ +static double estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const rocksdb::Slice& start, const rocksdb::Slice& end) { + rocksdb::Range range(start, end); + + uint64_t memtableCount = 0; + uint64_t memtableSize = 0; + db->GetApproximateMemTableStats(cf, range, &memtableCount, &memtableSize); + + rocksdb::SizeApproximationOptions sizeOptions; + sizeOptions.include_memtables = false; + sizeOptions.files_size_error_margin = 0.1; + uint64_t sstBytes = 0; + rocksdb::Status status = db->GetApproximateSizes(sizeOptions, cf, &range, 1, &sstBytes); + if (!status.ok() || sstBytes == 0) { + return static_cast(memtableCount); + } + + rocksdb::TablePropertiesCollection props; + status = db->GetPropertiesOfTablesInRange(cf, &range, 1, &props); + uint64_t entries = 0; + uint64_t deletions = 0; + uint64_t fileBytes = 0; + if (status.ok()) { + for (const auto& prop : props) { + const rocksdb::TableProperties& p = *prop.second; + entries += p.num_entries; + deletions += p.num_deletions; + // Approximate the on-disk file size covered by table properties; + // GetApproximateSizes offsets span data + index + filter blocks, + // so the density denominator must too. + fileBytes += p.data_size + p.index_size + p.filter_size; + } + } + if (entries <= deletions || fileBytes == 0) { + return static_cast(memtableCount); + } + + double density = static_cast(entries - deletions) / static_cast(fileBytes); + return static_cast(memtableCount) + static_cast(sstBytes) * density; +} + +/** + * Estimates the number of keys within a range without iterating. Both keys + * are optional buffers; an open-ended side is handled by subtracting the + * complementary range from the whole-column-family `estimate-num-keys` + * (an empty slice is the *smallest* key, so it must never be passed as an + * upper bound). + * + * @example + * ```typescript + * const db = NativeDatabase.open('path/to/db'); + * const estimate = db.estimateCount(startBuffer, endBuffer); + * ``` + */ +napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { + NAPI_METHOD_ARGV(2); + UNWRAP_DB_HANDLE_AND_OPEN(); + ACQUIRE_OPERATIONS_LOCK(); + + rocksdb::DB* db = (*dbHandle)->descriptor->db.get(); + rocksdb::ColumnFamilyHandle* cf = (*dbHandle)->getColumnFamilyHandle(); + + void* startData = nullptr; + size_t startLength = 0; + napi_valuetype startType; + NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &startType)); + if (startType != napi_undefined && startType != napi_null) { + NAPI_STATUS_THROWS(::napi_get_buffer_info(env, argv[0], &startData, &startLength)); + } + + void* endData = nullptr; + size_t endLength = 0; + napi_valuetype endType; + NAPI_STATUS_THROWS(::napi_typeof(env, argv[1], &endType)); + if (endType != napi_undefined && endType != napi_null) { + NAPI_STATUS_THROWS(::napi_get_buffer_info(env, argv[1], &endData, &endLength)); + } + + rocksdb::Slice startSlice(static_cast(startData), startLength); + rocksdb::Slice endSlice(static_cast(endData), endLength); + + double estimate = 0; + if (endData == nullptr) { + uint64_t totalKeys = 0; + db->GetIntProperty(cf, rocksdb::DB::Properties::kEstimateNumKeys, &totalKeys); + if (startData == nullptr) { + estimate = static_cast(totalKeys); + } else { + // No upper bound: estimate [start, ∞) as total minus [min, start). + estimate = std::max(0.0, static_cast(totalKeys) - estimateRangeCount(db, cf, rocksdb::Slice(), startSlice)); + } + } else { + estimate = estimateRangeCount(db, cf, startSlice, endSlice); + } + + napi_value result; + NAPI_STATUS_THROWS(::napi_create_double(env, std::round(estimate), &result)); + return result; +} + napi_value Database::GetMonotonicTimestamp(napi_env env, napi_callback_info info) { NAPI_METHOD(); UNWRAP_DB_HANDLE_AND_OPEN(); @@ -1987,6 +2105,7 @@ void Database::Init(napi_env env, napi_value exports) { { "destroy", nullptr, Destroy, nullptr, nullptr, nullptr, napi_default, nullptr }, { "drop", nullptr, Drop, nullptr, nullptr, nullptr, napi_default, nullptr }, { "dropSync", nullptr, DropSync, nullptr, nullptr, nullptr, napi_default, nullptr }, + { "estimateCount", nullptr, EstimateCount, nullptr, nullptr, nullptr, napi_default, nullptr }, { "flush", nullptr, Flush, nullptr, nullptr, nullptr, napi_default, nullptr }, { "flushSync", nullptr, FlushSync, nullptr, nullptr, nullptr, napi_default, nullptr }, { "get", nullptr, Get, nullptr, nullptr, nullptr, napi_default, nullptr }, diff --git a/src/binding/database/database.h b/src/binding/database/database.h index a1272c906..1c6a11d97 100644 --- a/src/binding/database/database.h +++ b/src/binding/database/database.h @@ -285,6 +285,7 @@ struct Database final { static napi_value FlushSync(napi_env env, napi_callback_info info); static napi_value Get(napi_env env, napi_callback_info info); static napi_value GetCompression(napi_env env, napi_callback_info info); + static napi_value EstimateCount(napi_env env, napi_callback_info info); static napi_value GetCount(napi_env env, napi_callback_info info); static napi_value GetDBIntProperty(napi_env env, napi_callback_info info); static napi_value GetDBProperty(napi_env env, napi_callback_info info); diff --git a/src/count-estimator.ts b/src/count-estimator.ts new file mode 100644 index 000000000..9031df6a8 --- /dev/null +++ b/src/count-estimator.ts @@ -0,0 +1,98 @@ +import type { RangeOptions } from './dbi.ts'; +import type { Key } from './encoding.ts'; +import type { Store } from './store.ts'; + +export interface CountEstimatorOptions extends RangeOptions { + /** + * When `true`, iteration proceeds from `end` toward `start`, so keys + * passed to `advance()` are treated as the new lower edge of the + * untraversed remainder. + */ + reverse?: boolean; +} + +/** + * Below this many traversed entries the observed density is too noisy to + * calibrate with, so `estimate()` returns the raw statistical estimate. + */ +const CALIBRATION_MIN_TRAVERSED = 16; + +/** + * Bounds on how far the observed-vs-estimated ratio may scale the remainder; + * the statistical estimate can be arbitrarily wrong at data-block granularity + * and an unclamped ratio would let one bad sample dominate. + */ +const CALIBRATION_MAX = 8; + +/** + * Progressively refines a range key-count estimate as an iterator traverses + * the range. The estimate starts as the pure statistical estimate + * (`estimateCount`) and converges toward the exact count: the traversed + * portion is exact, and the statistical estimate of the untraversed remainder + * is calibrated by the observed ratio of actual-to-estimated entries over the + * portion already traversed. + * + * The estimator never touches the iterator itself — the caller reports + * progress with `advance(lastKey, count)` whenever it wants a checkpoint + * (e.g. once per page), then reads `estimate()`. + */ +export class CountEstimator { + #store: Store; + #start: Key | Uint8Array | undefined; + #end: Key | Uint8Array | undefined; + #reverse: boolean; + #cursor: Key | Uint8Array | undefined; + #traversed = 0; + + constructor(store: Store, options?: CountEstimatorOptions) { + this.#store = store; + this.#start = options?.start; + this.#end = options?.end; + this.#reverse = options?.reverse ?? false; + } + + /** + * The number of entries reported traversed so far. + */ + get traversed(): number { + return this.#traversed; + } + + /** + * Records that iteration has advanced through `count` more entries, ending + * at `lastKey`. + */ + advance(lastKey: Key | Uint8Array, count = 1): void { + this.#cursor = lastKey; + this.#traversed += count; + } + + /** + * Estimates the total number of entries in the full range: the exact + * traversed count plus a calibrated statistical estimate of the remainder. + */ + estimate(): number { + if (this.#cursor === undefined) { + return this.#store.estimateCount({ start: this.#start, end: this.#end }); + } + + const traversedRange = this.#reverse + ? { start: this.#cursor, end: this.#end } + : { start: this.#start, end: this.#cursor }; + const remainingRange = this.#reverse + ? { start: this.#start, end: this.#cursor } + : { start: this.#cursor, end: this.#end }; + + let remaining = this.#store.estimateCount(remainingRange); + if (this.#traversed >= CALIBRATION_MIN_TRAVERSED) { + const traversedEstimate = this.#store.estimateCount(traversedRange); + const calibration = Math.min( + CALIBRATION_MAX, + Math.max(1 / CALIBRATION_MAX, this.#traversed / Math.max(traversedEstimate, 1)) + ); + remaining *= calibration; + } + + return Math.round(this.#traversed + remaining); + } +} diff --git a/src/database.ts b/src/database.ts index 61a218e6d..ef65e9337 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,6 +1,7 @@ import type { BackupStreamOptions } from './backup-stream.ts'; import type { BackupOptions } from './backup.ts'; -import { DBI, type DBITransactional } from './dbi.ts'; +import { CountEstimator, type CountEstimatorOptions } from './count-estimator.ts'; +import { DBI, type DBITransactional, type RangeOptions } from './dbi.ts'; import type { BufferWithDataView, Encoder, EncoderFunction, Key } from './encoding.ts'; import { addGlobalListener, @@ -455,17 +456,51 @@ export class RocksDatabase extends DBI { } /** - * Retrieves the estimated number of keys in the database. + * Retrieves the estimated number of keys in the database, or within a key + * range when one is given. Unlike `getKeysCount()`, this never iterates: + * the estimate is derived from RocksDB statistics (memtable stats plus + * approximate SST sizes converted through the entry density of the SSTs + * overlapping the range), so it stays fast no matter how large the range + * is. Accuracy improves with range size — resolution is bounded by SST + * data-block granularity, so tiny ranges over-report — and recently + * deleted or overwritten entries may be counted until compaction. + * + * Estimates always reflect committed state; writes pending in a + * transaction are not included. * * @example * ```typescript * const db = RocksDatabase.open('/path/to/database'); * const estimated = db.getEstimatedKeyCount(); - * console.log(estimated); + * const rangeEstimate = db.getEstimatedKeyCount({ start: 'a', end: 'z' }); + * ``` + */ + getEstimatedKeyCount(options?: RangeOptions): number { + return this.store.estimateCount(options); + } + + /** + * Creates a `CountEstimator` for progressively refining a range count + * estimate while iterating the range: report progress with + * `advance(lastKey, count)` (e.g. once per page of results) and + * `estimate()` returns the exact traversed count plus a calibrated + * estimate of the remainder, converging toward the exact total. + * + * @example + * ```typescript + * const estimator = db.createCountEstimator({ start: 'a', end: 'z' }); + * let lastKey; + * let pageSize = 0; + * for (const { key } of db.getRange({ start: 'a', end: 'z', limit: 25 })) { + * lastKey = key; + * pageSize++; + * } + * estimator.advance(lastKey, pageSize); + * const total = estimator.estimate(); // ~total matches in the range * ``` */ - getEstimatedKeyCount(): number { - return this.getDBIntProperty('rocksdb.estimate-num-keys') ?? 0; + createCountEstimator(options?: CountEstimatorOptions): CountEstimator { + return new CountEstimator(this.store, options); } /** diff --git a/src/index.ts b/src/index.ts index 598e792f8..fdf51bf06 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,6 +14,7 @@ export { type RocksDBStat, type RocksDBStats, } from './database.ts'; +export { CountEstimator, type CountEstimatorOptions } from './count-estimator.ts'; export { DBIterator } from './dbi-iterator.ts'; export { DBI, type IteratorOptions } from './dbi.ts'; export type { Key } from './encoding.ts'; diff --git a/src/load-binding.ts b/src/load-binding.ts index f3e61f0ca..fa683674a 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -333,6 +333,7 @@ export type NativeDatabase = { txnId?: number, expectedVersion?: number ): number; + estimateCount(startKey?: Buffer, endKey?: Buffer): number; getCompression(): { algorithm: string; level?: number }; getCount(options?: RangeOptions, txnId?: number): number; getDBIntProperty(propertyName: string): number | undefined; diff --git a/src/store.ts b/src/store.ts index c578f4951..40b1ef102 100644 --- a/src/store.ts +++ b/src/store.ts @@ -846,6 +846,29 @@ export class Store { return result; } + /** + * Estimates the number of keys in a range from RocksDB statistics + * (memtable stats + approximate SST sizes with range-local entry density) + * without iterating. Estimates always reflect committed state, so there is + * no transactional variant. + */ + estimateCount(options?: RangeOptions): number { + let startBuffer: Buffer | undefined; + let endBuffer: Buffer | undefined; + + if (options?.start !== undefined) { + const start = this.encodeKey(options.start); + startBuffer = Buffer.from(start.subarray(start.start, start.end)); + } + + if (options?.end !== undefined) { + const end = this.encodeKey(options.end); + endBuffer = Buffer.from(end.subarray(end.start, end.end)); + } + + return this.db.estimateCount(startBuffer, endBuffer); + } + getCount(context: StoreContext, options?: StoreRangeOptions): number { options = { ...options }; diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts new file mode 100644 index 000000000..c86cbaf7d --- /dev/null +++ b/test/estimate-count.test.ts @@ -0,0 +1,157 @@ +import { dbRunner } from './lib/util.ts'; +import { describe, expect, it } from 'vitest'; + +/** + * Estimates are statistical (block-granular SST approximation + memtable + * skip-list approximation), so assertions use a tolerance factor rather than + * exact bounds. Uniform fixed-size entries keep the real accuracy well inside + * these bounds; the factor only guards against wild regressions. + */ +function expectWithin(estimate: number, exact: number, factor: number) { + expect(estimate).toBeGreaterThanOrEqual(exact / factor); + expect(estimate).toBeLessThanOrEqual(exact * factor); +} + +const KEY = (i: number) => `key-${String(i).padStart(6, '0')}`; + +describe('estimateCount', () => { + it('should estimate counts for ranges over flushed data', () => + dbRunner(async ({ db }) => { + const N = 20000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}-${'x'.repeat(50)}`); + } + await db.flush(); + + // full range, both open-ended and bounded + expectWithin(db.getEstimatedKeyCount(), N, 2); + expectWithin(db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N) }), N, 2); + + // half range [25%, 75%) + const half = db.getEstimatedKeyCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); + expectWithin(half, N / 2, 2); + + // open-ended: start only and end only + expectWithin(db.getEstimatedKeyCount({ start: KEY(N / 2) }), N / 2, 2); + expectWithin(db.getEstimatedKeyCount({ end: KEY(N / 2) }), N / 2, 2); + + // a range past all data should estimate near zero relative to N + const empty = db.getEstimatedKeyCount({ start: 'z', end: 'zz' }); + expect(empty).toBeLessThan(N / 20); + })); + + it('should estimate counts for data still in the memtable', () => + dbRunner(async ({ db }) => { + const N = 10000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + // no flush: everything is in the memtable + expectWithin(db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N) }), N, 2); + expectWithin(db.getEstimatedKeyCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }), N / 2, 2); + })); + + it('should estimate counts spanning memtable and SST data', () => + dbRunner(async ({ db }) => { + const N = 10000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + for (let i = N; i < 2 * N; i++) { + await db.put(KEY(i), `value-${i}`); + } + expectWithin(db.getEstimatedKeyCount({ start: KEY(0), end: KEY(2 * N) }), 2 * N, 2); + })); + + it('should return 0 for an empty database', () => + dbRunner(async ({ db }) => { + expect(db.getEstimatedKeyCount()).toBe(0); + expect(db.getEstimatedKeyCount({ start: 'a', end: 'z' })).toBe(0); + })); + + it('should scale estimates with range width', () => + dbRunner(async ({ db }) => { + const N = 20000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}-${'x'.repeat(30)}`); + } + await db.flush(); + + const tenth = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N / 10) }); + const half = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N / 2) }); + const full = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N) }); + expect(tenth).toBeLessThan(half); + expect(half).toBeLessThan(full); + })); +}); + +describe('CountEstimator', () => { + it('should refine the estimate as iteration advances', () => + dbRunner(async ({ db }) => { + const N = 20000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}-${'x'.repeat(50)}`); + } + await db.flush(); + + const range = { start: KEY(0), end: KEY(N) }; + const estimator = db.createCountEstimator(range); + + // before any traversal: the pure statistical estimate + expectWithin(estimator.estimate(), N, 2); + + // walk the first half, checkpointing once per "page" + let traversed = 0; + let lastKey: unknown; + for (const key of db.getKeys(range)) { + lastKey = key; + if (++traversed % 1000 === 0) { + estimator.advance(lastKey as string, 1000); + } + if (traversed >= N / 2) { + break; + } + } + expect(estimator.traversed).toBe(N / 2); + + // with half the range traversed exactly, the estimate must be at + // least the traversed count and within a tighter overall bound + const refined = estimator.estimate(); + expect(refined).toBeGreaterThanOrEqual(N / 2); + expectWithin(refined, N, 1.6); + })); + + it('should support reverse iteration', () => + dbRunner(async ({ db }) => { + const N = 10000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + + const estimator = db.createCountEstimator({ start: KEY(0), end: KEY(N), reverse: true }); + // walk the last quarter in reverse + estimator.advance(KEY((3 * N) / 4), N / 4); + const refined = estimator.estimate(); + expect(refined).toBeGreaterThanOrEqual(N / 4); + expectWithin(refined, N, 2); + })); + + it('should converge to near-exact as traversal completes', () => + dbRunner(async ({ db }) => { + const N = 5000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + + const estimator = db.createCountEstimator({ start: KEY(0), end: KEY(N) }); + estimator.advance(KEY(N - 1), N); + // remainder is (KEY(N-1), KEY(N)) — essentially empty, though the + // estimate of it is block-granular, so allow a small overshoot + const final = estimator.estimate(); + expect(final).toBeGreaterThanOrEqual(N); + expect(final).toBeLessThan(N * 1.25); + })); +}); From 4481bc44feb650d654f0e98f7389a78c0aa5de35 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 13 Aug 2026 10:26:22 -0600 Subject: [PATCH 02/15] fix: address cross-model review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - guard inverted/empty bounded ranges (GetApproximateSizes would underflow end-start offsets in uint64) — returns 0 - honor exclusiveStart/inclusiveEnd by appending the bytewise-successor zero byte to the encoded bound - CountEstimator: exclude the cursor entry from the remainder (forward mode double-counted it, blocking convergence), add finish() as the completion signal, memoize estimate() per checkpoint, and document the caller-owned progress contract - temper the cost claims: scales with overlapping SSTs, table-property reads can do I/O for cold files, start-only ranges do complement work Co-Authored-By: Claude Fable 5 --- README.md | 19 +++++++++----- src/binding/database/database.cpp | 5 ++++ src/count-estimator.ts | 38 ++++++++++++++++++++++++--- src/database.ts | 13 +++++++--- src/store.ts | 21 +++++++++++++-- test/estimate-count.test.ts | 43 +++++++++++++++++++++++++++++++ 6 files changed, 122 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 048d488f1..50c90a96a 100644 --- a/README.md +++ b/README.md @@ -434,11 +434,14 @@ the `expectedVersion` option is used. Retrieves the estimated number of keys in the database, or within a key range when one is given. Unlike `getKeysCount()`, this never iterates: the estimate is derived from RocksDB statistics (memtable stats plus approximate SST sizes converted through the entry density of the SSTs -overlapping the range), so it stays fast regardless of range size — typically microseconds where an -exact count takes milliseconds. Accuracy improves with range size (resolution is bounded by SST -data-block granularity, so tiny ranges over-report), and recently deleted or overwritten entries -may be counted until compaction. Estimates always reflect committed state; writes pending in a -transaction are not included. +overlapping the range), so its cost scales with the number of SSTs overlapping the range rather +than the number of keys — typically microseconds where an exact count takes milliseconds, though +reading table properties for cold files can do I/O through the table cache. A start-only range is +computed as the whole-database estimate minus the complement, so it does the work of the range +_below_ `start`. Accuracy improves with range size (resolution is bounded by SST data-block +granularity, so tiny ranges over-report), and recently deleted or overwritten entries may be +counted until compaction. Estimates always reflect committed state; writes pending in a +transaction are not included. An inverted range (`start` ≥ `end`) returns 0. ```typescript const estimated = db.getEstimatedKeyCount(); @@ -453,8 +456,10 @@ range. Before any traversal, `estimate()` returns the pure statistical estimate `getEstimatedKeyCount(range)`). As the caller reports progress with `advance(lastKey, count)` (e.g. once per page), `estimate()` returns the exact traversed count plus a statistical estimate of the remainder, calibrated by the observed ratio of actual-to-estimated entries over the portion -already traversed — so the estimate converges toward the exact total as iteration proceeds. Set -`reverse: true` when iterating from `end` toward `start`. +already traversed — so the estimate converges toward the exact total as iteration proceeds. When +traversal completes, call `finish()` and `estimate()` returns the exact count. Set +`reverse: true` when iterating from `end` toward `start`. The caller owns the progress contract: +cursors must move monotonically through the range and each entry must be reported exactly once. ```typescript const range = { start: 'a', end: 'z' }; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5674d3999..b3bfa41e2 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -1031,6 +1031,11 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { // No upper bound: estimate [start, ∞) as total minus [min, start). estimate = std::max(0.0, static_cast(totalKeys) - estimateRangeCount(db, cf, rocksdb::Slice(), startSlice)); } + } else if (startData != nullptr && startSlice.compare(endSlice) >= 0) { + // Inverted or empty range: GetApproximateSizes would underflow + // (end offset minus start offset in uint64). Comparator is always + // bytewise (db_descriptor.cpp), so Slice::compare matches key order. + estimate = 0; } else { estimate = estimateRangeCount(db, cf, startSlice, endSlice); } diff --git a/src/count-estimator.ts b/src/count-estimator.ts index 9031df6a8..c37c831dc 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -34,7 +34,16 @@ const CALIBRATION_MAX = 8; * * The estimator never touches the iterator itself — the caller reports * progress with `advance(lastKey, count)` whenever it wants a checkpoint - * (e.g. once per page), then reads `estimate()`. + * (e.g. once per page), then reads `estimate()`. The caller owns the + * progress contract: cursors must move monotonically through the range and + * each entry must be reported once (a re-reported page inflates the count + * undetectably). When traversal completes, call `finish()` so `estimate()` + * returns the exact total instead of adding a block-granular remainder. + * + * Each `estimate()` checkpoint queries RocksDB statistics for the two range + * segments (cost scales with the SSTs overlapping them, not with key count); + * results are memoized per checkpoint, so repeated reads between `advance()` + * calls are free. */ export class CountEstimator { #store: Store; @@ -43,6 +52,8 @@ export class CountEstimator { #reverse: boolean; #cursor: Key | Uint8Array | undefined; #traversed = 0; + #finished = false; + #memoized: number | undefined; constructor(store: Store, options?: CountEstimatorOptions) { this.#store = store; @@ -65,6 +76,15 @@ export class CountEstimator { advance(lastKey: Key | Uint8Array, count = 1): void { this.#cursor = lastKey; this.#traversed += count; + this.#memoized = undefined; + } + + /** + * Marks traversal of the range as complete: `estimate()` becomes the exact + * traversed count. + */ + finish(): void { + this.#finished = true; } /** @@ -72,16 +92,25 @@ export class CountEstimator { * traversed count plus a calibrated statistical estimate of the remainder. */ estimate(): number { + if (this.#finished) { + return this.#traversed; + } if (this.#cursor === undefined) { return this.#store.estimateCount({ start: this.#start, end: this.#end }); } + if (this.#memoized !== undefined) { + return this.#memoized; + } + // The cursor entry itself belongs to the traversed side, so the + // remainder excludes it in both directions (an inclusive lower bound + // forward would count it twice and block convergence). const traversedRange = this.#reverse ? { start: this.#cursor, end: this.#end } - : { start: this.#start, end: this.#cursor }; + : { start: this.#start, end: this.#cursor, inclusiveEnd: true }; const remainingRange = this.#reverse ? { start: this.#start, end: this.#cursor } - : { start: this.#cursor, end: this.#end }; + : { start: this.#cursor, end: this.#end, exclusiveStart: true }; let remaining = this.#store.estimateCount(remainingRange); if (this.#traversed >= CALIBRATION_MIN_TRAVERSED) { @@ -93,6 +122,7 @@ export class CountEstimator { remaining *= calibration; } - return Math.round(this.#traversed + remaining); + this.#memoized = Math.round(this.#traversed + remaining); + return this.#memoized; } } diff --git a/src/database.ts b/src/database.ts index ef65e9337..7c972c9bb 100644 --- a/src/database.ts +++ b/src/database.ts @@ -460,10 +460,14 @@ export class RocksDatabase extends DBI { * range when one is given. Unlike `getKeysCount()`, this never iterates: * the estimate is derived from RocksDB statistics (memtable stats plus * approximate SST sizes converted through the entry density of the SSTs - * overlapping the range), so it stays fast no matter how large the range - * is. Accuracy improves with range size — resolution is bounded by SST + * overlapping the range), so its cost scales with the number of SSTs + * overlapping the range rather than the number of keys — though reading + * table properties for cold files can do I/O through the table cache, and + * a start-only range does the work of its complement below `start`. + * Accuracy improves with range size — resolution is bounded by SST * data-block granularity, so tiny ranges over-report — and recently - * deleted or overwritten entries may be counted until compaction. + * deleted or overwritten entries may be counted until compaction. An + * inverted range (`start` >= `end`) returns 0. * * Estimates always reflect committed state; writes pending in a * transaction are not included. @@ -484,7 +488,8 @@ export class RocksDatabase extends DBI { * estimate while iterating the range: report progress with * `advance(lastKey, count)` (e.g. once per page of results) and * `estimate()` returns the exact traversed count plus a calibrated - * estimate of the remainder, converging toward the exact total. + * estimate of the remainder, converging toward the exact total. Call + * `finish()` when traversal completes to make `estimate()` exact. * * @example * ```typescript diff --git a/src/store.ts b/src/store.ts index 40b1ef102..93e9e77a9 100644 --- a/src/store.ts +++ b/src/store.ts @@ -858,12 +858,14 @@ export class Store { if (options?.start !== undefined) { const start = this.encodeKey(options.start); - startBuffer = Buffer.from(start.subarray(start.start, start.end)); + // A zero byte appended to a key is its bytewise successor, turning an + // inclusive bound into an exclusive one (and vice versa for the end). + startBuffer = copyEncodedKey(start, options.exclusiveStart === true); } if (options?.end !== undefined) { const end = this.encodeKey(options.end); - endBuffer = Buffer.from(end.subarray(end.start, end.end)); + endBuffer = copyEncodedKey(end, options.inclusiveEnd === true); } return this.db.estimateCount(startBuffer, endBuffer); @@ -1258,6 +1260,21 @@ export class Store { } } +/** + * Copies an encoded key out of the shared key buffer (which the next + * `encodeKey` call would clobber), optionally appending a zero byte to + * produce the key's bytewise successor. + */ +function copyEncodedKey(encoded: BufferWithDataView, appendSuccessorByte: boolean): Buffer { + const length = encoded.end - encoded.start; + const copy = Buffer.allocUnsafe(length + (appendSuccessorByte ? 1 : 0)); + copy.set(encoded.subarray(encoded.start, encoded.end)); + if (appendSuccessorByte) { + copy[length] = 0; + } + return copy; +} + /** * Ensure that they key has been copied into our shared buffer, and return the ending position * @param keyBuffer diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index c86cbaf7d..e71389264 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -70,6 +70,16 @@ describe('estimateCount', () => { expect(db.getEstimatedKeyCount({ start: 'a', end: 'z' })).toBe(0); })); + it('should return 0 for an inverted range', () => + dbRunner(async ({ db }) => { + for (let i = 0; i < 5000; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + expect(db.getEstimatedKeyCount({ start: KEY(4000), end: KEY(1000) })).toBe(0); + expect(db.getEstimatedKeyCount({ start: KEY(1000), end: KEY(1000) })).toBe(0); + })); + it('should scale estimates with range width', () => dbRunner(async ({ db }) => { const N = 20000; @@ -153,5 +163,38 @@ describe('CountEstimator', () => { const final = estimator.estimate(); expect(final).toBeGreaterThanOrEqual(N); expect(final).toBeLessThan(N * 1.25); + + estimator.finish(); + expect(estimator.estimate()).toBe(N); + })); + + it('should report the exact total for a paginated loop driven to completion', () => + dbRunner(async ({ db }) => { + const N = 5000; + const PAGE = 250; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + + const range = { start: KEY(0), end: KEY(N) }; + const estimator = db.createCountEstimator(range); + let pageStart: string | undefined; + let exclusiveStart = false; + for (;;) { + const page = Array.from( + db.getKeys({ ...range, start: pageStart ?? range.start, exclusiveStart, limit: PAGE }) + ); + if (page.length === 0) { + break; + } + pageStart = page[page.length - 1] as string; + exclusiveStart = true; + estimator.advance(pageStart, page.length); + expect(estimator.estimate()).toBeGreaterThanOrEqual(estimator.traversed); + } + expect(estimator.traversed).toBe(N); + estimator.finish(); + expect(estimator.estimate()).toBe(N); })); }); From 0ddaf72c672ad92f56a490a913bda21e3969f130 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 13 Aug 2026 10:31:31 -0600 Subject: [PATCH 03/15] fix: harden zero-length native bounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit napi can return a null data pointer for a zero-length buffer, which the previous guard read as an omitted bound — an empty end bound (below every key) became a whole-database estimate on the NativeDatabase surface (encodeKey shields the public API). Track presence explicitly: empty end returns 0, empty start is the minimum key. Co-Authored-By: Claude Fable 5 --- src/binding/database/database.cpp | 26 ++++++++++++++++---------- test/estimate-count.test.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index b3bfa41e2..deb2273b0 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -1002,11 +1002,15 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { rocksdb::DB* db = (*dbHandle)->descriptor->db.get(); rocksdb::ColumnFamilyHandle* cf = (*dbHandle)->getColumnFamilyHandle(); + // Presence is tracked separately from the data pointer: napi may return a + // null pointer for a zero-length buffer, and an empty bound (the smallest + // key) must not be confused with an omitted one. void* startData = nullptr; size_t startLength = 0; napi_valuetype startType; NAPI_STATUS_THROWS(::napi_typeof(env, argv[0], &startType)); - if (startType != napi_undefined && startType != napi_null) { + bool hasStart = startType != napi_undefined && startType != napi_null; + if (hasStart) { NAPI_STATUS_THROWS(::napi_get_buffer_info(env, argv[0], &startData, &startLength)); } @@ -1014,27 +1018,29 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { size_t endLength = 0; napi_valuetype endType; NAPI_STATUS_THROWS(::napi_typeof(env, argv[1], &endType)); - if (endType != napi_undefined && endType != napi_null) { + bool hasEnd = endType != napi_undefined && endType != napi_null; + if (hasEnd) { NAPI_STATUS_THROWS(::napi_get_buffer_info(env, argv[1], &endData, &endLength)); } - rocksdb::Slice startSlice(static_cast(startData), startLength); - rocksdb::Slice endSlice(static_cast(endData), endLength); + rocksdb::Slice startSlice(startLength ? static_cast(startData) : "", startLength); + rocksdb::Slice endSlice(endLength ? static_cast(endData) : "", endLength); double estimate = 0; - if (endData == nullptr) { + if (!hasEnd) { uint64_t totalKeys = 0; db->GetIntProperty(cf, rocksdb::DB::Properties::kEstimateNumKeys, &totalKeys); - if (startData == nullptr) { + if (!hasStart || startLength == 0) { estimate = static_cast(totalKeys); } else { // No upper bound: estimate [start, ∞) as total minus [min, start). estimate = std::max(0.0, static_cast(totalKeys) - estimateRangeCount(db, cf, rocksdb::Slice(), startSlice)); } - } else if (startData != nullptr && startSlice.compare(endSlice) >= 0) { - // Inverted or empty range: GetApproximateSizes would underflow - // (end offset minus start offset in uint64). Comparator is always - // bytewise (db_descriptor.cpp), so Slice::compare matches key order. + } else if (endLength == 0 || startSlice.compare(endSlice) >= 0) { + // Empty end bound (below every key) or inverted/empty range: + // GetApproximateSizes would underflow (end offset minus start offset + // in uint64). Comparator is always bytewise (db_descriptor.cpp), so + // Slice::compare matches key order. estimate = 0; } else { estimate = estimateRangeCount(db, cf, startSlice, endSlice); diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index e71389264..c94fd7505 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -80,6 +80,35 @@ describe('estimateCount', () => { expect(db.getEstimatedKeyCount({ start: KEY(1000), end: KEY(1000) })).toBe(0); })); + it('should treat zero-length native bounds safely', () => + dbRunner(async ({ db }) => { + for (let i = 0; i < 5000; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + // encodeKey rejects zero-length keys, so exercise the native surface + // directly: an empty end bound sorts below every key (empty range), + // an empty start bound is the minimum key (no-op lower bound) + const native = (db as any).store.db; + expect(native.estimateCount(undefined, Buffer.alloc(0))).toBe(0); + expect(native.estimateCount(Buffer.alloc(0), undefined)).toBe(db.getEstimatedKeyCount()); + })); + + it('should not count uncommitted transaction writes', () => + dbRunner(async ({ db }) => { + const N = 10000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.transaction(async (txn) => { + for (let i = N; i < 2 * N; i++) { + txn.putSync(KEY(i), `value-${i}`); + } + const estimate = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(2 * N) }); + expect(estimate).toBeLessThan(N * 1.5); + }); + })); + it('should scale estimates with range width', () => dbRunner(async ({ db }) => { const N = 20000; From b2cb53a3b14233a766dfc82bb78e9f98d81d846e Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 13 Aug 2026 12:07:36 -0600 Subject: [PATCH 04/15] feat: return { count, confidence } from estimate APIs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback on the API shape: a bare number hides how much an estimate should be trusted. New db.estimateCount(options?) returns { count, confidence }; getEstimatedKeyCount() reverts to its original no-arg number signature (kept as the cheap estimate-num-keys alias), so one name no longer covers two cost profiles. CountEstimator.estimate() returns the same shape. confidence is a heuristic [0,1], exactly 1 only when the count is exact (finish(), inverted/empty-by-construction ranges). Computed natively from the estimate components: resolution (SST data-block / memtable sampling granularity relative to the count), tombstone fraction of the overlapping SSTs, and for start-only ranges the error compounded by complement subtraction. Measured on 500k varied entries: 0.999 on full/half ranges (~3% error), 0.88 at 1%, 0.21 on a 50-key range (~2x over-report), 0.13 on a start-only tail (+39% — complement subtraction correctly distrusted); estimator confidence converges to 1. Co-Authored-By: Claude Fable 5 --- README.md | 62 ++++++++++++++------- src/binding/database/database.cpp | 92 +++++++++++++++++++++++++------ src/count-estimator.ts | 23 +++++--- src/database.ts | 52 +++++++++++------ src/dbi.ts | 17 ++++++ src/index.ts | 2 +- src/load-binding.ts | 2 +- src/store.ts | 4 +- test/estimate-count.test.ts | 92 ++++++++++++++++++++----------- 9 files changed, 247 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 50c90a96a..47a0bc584 100644 --- a/README.md +++ b/README.md @@ -429,23 +429,41 @@ if (result === constants.FRESH_VERSION_FLAG) { Synchronous version of `get()`. Like `get()`, this can return the `FRESH_VERSION_FLAG` sentinel when the `expectedVersion` option is used. -### `db.getEstimatedKeyCount(options?: RangeOptions): number` +### `db.getEstimatedKeyCount(): number` -Retrieves the estimated number of keys in the database, or within a key range when one is given. -Unlike `getKeysCount()`, this never iterates: the estimate is derived from RocksDB statistics -(memtable stats plus approximate SST sizes converted through the entry density of the SSTs -overlapping the range), so its cost scales with the number of SSTs overlapping the range rather -than the number of keys — typically microseconds where an exact count takes milliseconds, though -reading table properties for cold files can do I/O through the table cache. A start-only range is -computed as the whole-database estimate minus the complement, so it does the work of the range -_below_ `start`. Accuracy improves with range size (resolution is bounded by SST data-block -granularity, so tiny ranges over-report), and recently deleted or overwritten entries may be -counted until compaction. Estimates always reflect committed state; writes pending in a -transaction are not included. An inverted range (`start` ≥ `end`) returns 0. +Retrieves the estimated number of keys in the database. This is an alias for +`db.getDBIntProperty('rocksdb.estimate-num-keys')`; use `estimateCount()` for range support and a +confidence indicator. ```typescript const estimated = db.getEstimatedKeyCount(); -const rangeEstimate = db.getEstimatedKeyCount({ start: 'a', end: 'z' }); +console.log(estimated); +``` + +### `db.estimateCount(options?: RangeOptions): CountEstimate` + +Estimates the number of keys in the database, or within a key range, returning +`{ count, confidence }`. Unlike `getKeysCount()`, this never iterates: the estimate is derived +from RocksDB statistics (memtable stats plus approximate SST sizes converted through the entry +density of the SSTs overlapping the range), so its cost scales with the number of SSTs overlapping +the range rather than the number of keys — typically microseconds where an exact count takes +milliseconds, though reading table properties for cold files can do I/O through the table cache. A +start-only range is computed as the whole-database estimate minus the complement, so it does the +work of the range _below_ `start`. Accuracy improves with range size (resolution is bounded by SST +data-block granularity, so tiny ranges over-report), and recently deleted or overwritten entries +may be counted until compaction. Estimates always reflect committed state; writes pending in a +transaction are not included. An inverted range (`start` ≥ `end`) returns +`{ count: 0, confidence: 1 }`. + +`confidence` is a heuristic 0–1 indicator of how trustworthy `count` is — exactly 1 only when the +count is exact. It is derived from the estimate's resolution (data-block/memtable-sampling +granularity relative to the count), the tombstone fraction of the overlapping SSTs, and — for +start-only ranges — the error compounded by complement subtraction. Treat it as an ordering +signal (e.g. when to trust an estimate for query planning vs fall back to a heuristic), not a +statistical bound. + +```typescript +const { count, confidence } = db.estimateCount({ start: 'a', end: 'z' }); ``` ### `db.createCountEstimator(options?: CountEstimatorOptions): CountEstimator` @@ -453,13 +471,15 @@ const rangeEstimate = db.getEstimatedKeyCount({ start: 'a', end: 'z' }); Creates an estimator that progressively refines a range count estimate while the range is being iterated — useful for reporting a total alongside a page of results without scanning the full range. Before any traversal, `estimate()` returns the pure statistical estimate (same as -`getEstimatedKeyCount(range)`). As the caller reports progress with `advance(lastKey, count)` -(e.g. once per page), `estimate()` returns the exact traversed count plus a statistical estimate -of the remainder, calibrated by the observed ratio of actual-to-estimated entries over the portion -already traversed — so the estimate converges toward the exact total as iteration proceeds. When -traversal completes, call `finish()` and `estimate()` returns the exact count. Set -`reverse: true` when iterating from `end` toward `start`. The caller owns the progress contract: -cursors must move monotonically through the range and each entry must be reported exactly once. +`estimateCount(range)`). As the caller reports progress with `advance(lastKey, count)` (e.g. once +per page), `estimate()` returns the exact traversed count plus a statistical estimate of the +remainder, calibrated by the observed ratio of actual-to-estimated entries over the portion +already traversed — so the count converges toward the exact total, and `confidence` (the +exactness-weighted blend of the traversed portion and the remainder's confidence) converges to 1. +When traversal completes, call `finish()` and `estimate()` returns the exact count with +confidence 1. Set `reverse: true` when iterating from `end` toward `start`. The caller owns the +progress contract: cursors must move monotonically through the range and each entry must be +reported exactly once. ```typescript const range = { start: 'a', end: 'z' }; @@ -471,7 +491,7 @@ for (const { key } of db.getRange({ ...range, limit: 25 })) { pageSize++; } estimator.advance(lastKey, pageSize); -const total = estimator.estimate(); // ~total keys in the range +const { count, confidence } = estimator.estimate(); ``` ### `db.getKeys(options?: IteratorOptions): ExtendedIterable` diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index deb2273b0..56e95146c 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -941,12 +941,27 @@ napi_value Database::GetCount(napi_env env, napi_callback_info info) { * compaction; resolution is bounded by SST data-block granularity, so tiny * ranges over-report. */ -static double estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const rocksdb::Slice& start, const rocksdb::Slice& end) { +struct RangeEstimate { + double count = 0; + double memtableCount = 0; + double sstCount = 0; + // Live-entry resolution of one SST data block — the granularity the SST + // portion of the estimate is quantized to (0 when unknown/no SST data). + double entriesPerBlock = 0; + // Live fraction of SST entries ((entries - deletions) / entries); 1 when + // no table properties were available. + double liveFraction = 1; +}; + +static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const rocksdb::Slice& start, const rocksdb::Slice& end) { rocksdb::Range range(start, end); + RangeEstimate result; uint64_t memtableCount = 0; uint64_t memtableSize = 0; db->GetApproximateMemTableStats(cf, range, &memtableCount, &memtableSize); + result.memtableCount = static_cast(memtableCount); + result.count = result.memtableCount; rocksdb::SizeApproximationOptions sizeOptions; sizeOptions.include_memtables = false; @@ -954,7 +969,7 @@ static double estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* c uint64_t sstBytes = 0; rocksdb::Status status = db->GetApproximateSizes(sizeOptions, cf, &range, 1, &sstBytes); if (!status.ok() || sstBytes == 0) { - return static_cast(memtableCount); + return result; } rocksdb::TablePropertiesCollection props; @@ -974,24 +989,48 @@ static double estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* c } } if (entries <= deletions || fileBytes == 0) { - return static_cast(memtableCount); + return result; } double density = static_cast(entries - deletions) / static_cast(fileBytes); - return static_cast(memtableCount) + static_cast(sstBytes) * density; + result.sstCount = static_cast(sstBytes) * density; + result.count += result.sstCount; + result.entriesPerBlock = density * 4096; + result.liveFraction = static_cast(entries - deletions) / static_cast(entries); + return result; +} + +/** + * Heuristic [0, 1] trust indicator for a range estimate — 1 only when exact. + * Combines the estimate's resolution (SST portion is quantized to data-block + * granularity, memtable counts to skip-list sampling granularity) with the + * tombstone fraction of the overlapping SSTs (a proxy for overwrite/delete + * skew the estimate cannot see). + */ +static double estimateConfidence(const RangeEstimate& est) { + if (est.count <= 0) { + // Nothing overlaps the range per both memtable stats and SST sizes; + // nearly certain but not provably exact (in-flight flush/compaction). + return 0.95; + } + double sstResolution = std::max(est.entriesPerBlock, 1.0); + double memtableResolution = 8; + double resolution = (est.sstCount * sstResolution + est.memtableCount * memtableResolution) / est.count; + double granularity = est.count / (est.count + resolution); + return granularity * (0.5 + 0.5 * est.liveFraction); } /** - * Estimates the number of keys within a range without iterating. Both keys - * are optional buffers; an open-ended side is handled by subtracting the - * complementary range from the whole-column-family `estimate-num-keys` - * (an empty slice is the *smallest* key, so it must never be passed as an - * upper bound). + * Estimates the number of keys within a range without iterating, returning + * `{ count, confidence }`. Both keys are optional buffers; an open-ended side + * is handled by subtracting the complementary range from the + * whole-column-family `estimate-num-keys` (an empty slice is the *smallest* + * key, so it must never be passed as an upper bound). * * @example * ```typescript * const db = NativeDatabase.open('path/to/db'); - * const estimate = db.estimateCount(startBuffer, endBuffer); + * const { count, confidence } = db.estimateCount(startBuffer, endBuffer); * ``` */ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { @@ -1027,27 +1066,48 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { rocksdb::Slice endSlice(endLength ? static_cast(endData) : "", endLength); double estimate = 0; + double confidence = 0; if (!hasEnd) { uint64_t totalKeys = 0; db->GetIntProperty(cf, rocksdb::DB::Properties::kEstimateNumKeys, &totalKeys); - if (!hasStart || startLength == 0) { - estimate = static_cast(totalKeys); + double total = static_cast(totalKeys); + if (!hasStart || startLength == 0 || totalKeys == 0) { + estimate = total; + // estimate-num-keys is RocksDB's own memtable+SST estimate; it + // skews high on overwrite/delete-heavy data until compaction. + confidence = totalKeys == 0 ? 1.0 : 0.9; } else { // No upper bound: estimate [start, ∞) as total minus [min, start). - estimate = std::max(0.0, static_cast(totalKeys) - estimateRangeCount(db, cf, rocksdb::Slice(), startSlice)); + RangeEstimate complement = estimateRangeCount(db, cf, rocksdb::Slice(), startSlice); + estimate = std::max(0.0, total - complement.count); + // Subtracting two independent estimates compounds their absolute + // errors, so trust shrinks with the complement's share of the + // total (a narrow tail range after a large complement is mostly + // error). + double share = estimate / std::max(estimate + complement.count, 1.0); + confidence = std::min(0.9, estimateConfidence(complement)) * share; } } else if (endLength == 0 || startSlice.compare(endSlice) >= 0) { // Empty end bound (below every key) or inverted/empty range: // GetApproximateSizes would underflow (end offset minus start offset // in uint64). Comparator is always bytewise (db_descriptor.cpp), so - // Slice::compare matches key order. + // Slice::compare matches key order. Empty by construction, so exact. estimate = 0; + confidence = 1.0; } else { - estimate = estimateRangeCount(db, cf, startSlice, endSlice); + RangeEstimate rangeEstimate = estimateRangeCount(db, cf, startSlice, endSlice); + estimate = rangeEstimate.count; + confidence = estimateConfidence(rangeEstimate); } napi_value result; - NAPI_STATUS_THROWS(::napi_create_double(env, std::round(estimate), &result)); + NAPI_STATUS_THROWS(::napi_create_object(env, &result)); + napi_value countValue; + NAPI_STATUS_THROWS(::napi_create_double(env, std::round(estimate), &countValue)); + NAPI_STATUS_THROWS(::napi_set_named_property(env, result, "count", countValue)); + napi_value confidenceValue; + NAPI_STATUS_THROWS(::napi_create_double(env, confidence, &confidenceValue)); + NAPI_STATUS_THROWS(::napi_set_named_property(env, result, "confidence", confidenceValue)); return result; } diff --git a/src/count-estimator.ts b/src/count-estimator.ts index c37c831dc..67c8a2d40 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -1,4 +1,4 @@ -import type { RangeOptions } from './dbi.ts'; +import type { CountEstimate, RangeOptions } from './dbi.ts'; import type { Key } from './encoding.ts'; import type { Store } from './store.ts'; @@ -53,7 +53,7 @@ export class CountEstimator { #cursor: Key | Uint8Array | undefined; #traversed = 0; #finished = false; - #memoized: number | undefined; + #memoized: CountEstimate | undefined; constructor(store: Store, options?: CountEstimatorOptions) { this.#store = store; @@ -90,10 +90,13 @@ export class CountEstimator { /** * Estimates the total number of entries in the full range: the exact * traversed count plus a calibrated statistical estimate of the remainder. + * `confidence` is the exactness-weighted blend of the traversed portion + * (exact) and the remainder's statistical confidence, so it converges to 1 + * as traversal proceeds (and is exactly 1 after `finish()`). */ - estimate(): number { + estimate(): CountEstimate { if (this.#finished) { - return this.#traversed; + return { count: this.#traversed, confidence: 1 }; } if (this.#cursor === undefined) { return this.#store.estimateCount({ start: this.#start, end: this.#end }); @@ -112,17 +115,23 @@ export class CountEstimator { ? { start: this.#start, end: this.#cursor } : { start: this.#cursor, end: this.#end, exclusiveStart: true }; - let remaining = this.#store.estimateCount(remainingRange); + const remainingEstimate = this.#store.estimateCount(remainingRange); + let remaining = remainingEstimate.count; if (this.#traversed >= CALIBRATION_MIN_TRAVERSED) { const traversedEstimate = this.#store.estimateCount(traversedRange); const calibration = Math.min( CALIBRATION_MAX, - Math.max(1 / CALIBRATION_MAX, this.#traversed / Math.max(traversedEstimate, 1)) + Math.max(1 / CALIBRATION_MAX, this.#traversed / Math.max(traversedEstimate.count, 1)) ); remaining *= calibration; } - this.#memoized = Math.round(this.#traversed + remaining); + const count = Math.round(this.#traversed + remaining); + const confidence = + count > 0 + ? Math.min(1, (this.#traversed + remainingEstimate.confidence * remaining) / count) + : remainingEstimate.confidence; + this.#memoized = { count, confidence }; return this.#memoized; } } diff --git a/src/database.ts b/src/database.ts index 7c972c9bb..2482a8132 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,7 +1,7 @@ import type { BackupStreamOptions } from './backup-stream.ts'; import type { BackupOptions } from './backup.ts'; import { CountEstimator, type CountEstimatorOptions } from './count-estimator.ts'; -import { DBI, type DBITransactional, type RangeOptions } from './dbi.ts'; +import { DBI, type CountEstimate, type DBITransactional, type RangeOptions } from './dbi.ts'; import type { BufferWithDataView, Encoder, EncoderFunction, Key } from './encoding.ts'; import { addGlobalListener, @@ -456,30 +456,46 @@ export class RocksDatabase extends DBI { } /** - * Retrieves the estimated number of keys in the database, or within a key - * range when one is given. Unlike `getKeysCount()`, this never iterates: - * the estimate is derived from RocksDB statistics (memtable stats plus - * approximate SST sizes converted through the entry density of the SSTs - * overlapping the range), so its cost scales with the number of SSTs - * overlapping the range rather than the number of keys — though reading - * table properties for cold files can do I/O through the table cache, and - * a start-only range does the work of its complement below `start`. - * Accuracy improves with range size — resolution is bounded by SST - * data-block granularity, so tiny ranges over-report — and recently + * Retrieves the estimated number of keys in the database. This is an alias + * for `db.estimateCount().count`; use `estimateCount()` for range support + * and a confidence indicator. + * + * @example + * ```typescript + * const db = RocksDatabase.open('/path/to/database'); + * const estimated = db.getEstimatedKeyCount(); + * console.log(estimated); + * ``` + */ + getEstimatedKeyCount(): number { + return this.getDBIntProperty('rocksdb.estimate-num-keys') ?? 0; + } + + /** + * Estimates the number of keys in the database, or within a key range, + * returning `{ count, confidence }`. Unlike `getKeysCount()`, this never + * iterates: the estimate is derived from RocksDB statistics (memtable + * stats plus approximate SST sizes converted through the entry density of + * the SSTs overlapping the range), so its cost scales with the number of + * SSTs overlapping the range rather than the number of keys — though + * reading table properties for cold files can do I/O through the table + * cache, and a start-only range does the work of its complement below + * `start`. Accuracy improves with range size — resolution is bounded by + * SST data-block granularity, so tiny ranges over-report — and recently * deleted or overwritten entries may be counted until compaction. An - * inverted range (`start` >= `end`) returns 0. + * inverted range (`start` >= `end`) returns `{ count: 0, confidence: 1 }`. * - * Estimates always reflect committed state; writes pending in a - * transaction are not included. + * `confidence` is a heuristic 0–1 trust indicator (1 only when exact) — + * see `CountEstimate`. Estimates always reflect committed state; writes + * pending in a transaction are not included. * * @example * ```typescript * const db = RocksDatabase.open('/path/to/database'); - * const estimated = db.getEstimatedKeyCount(); - * const rangeEstimate = db.getEstimatedKeyCount({ start: 'a', end: 'z' }); + * const { count, confidence } = db.estimateCount({ start: 'a', end: 'z' }); * ``` */ - getEstimatedKeyCount(options?: RangeOptions): number { + estimateCount(options?: RangeOptions): CountEstimate { return this.store.estimateCount(options); } @@ -501,7 +517,7 @@ export class RocksDatabase extends DBI { * pageSize++; * } * estimator.advance(lastKey, pageSize); - * const total = estimator.estimate(); // ~total matches in the range + * const { count, confidence } = estimator.estimate(); * ``` */ createCountEstimator(options?: CountEstimatorOptions): CountEstimator { diff --git a/src/dbi.ts b/src/dbi.ts index dd72742fe..db5efbdcb 100644 --- a/src/dbi.ts +++ b/src/dbi.ts @@ -62,6 +62,23 @@ export interface RocksDBOptions { tailing?: boolean; } +export interface CountEstimate { + /** + * The estimated number of keys. + */ + count: number; + + /** + * Heuristic 0–1 indicator of how trustworthy `count` is; exactly 1 only + * when the count is exact. Derived from the estimate's resolution + * (SST data-block / memtable sampling granularity relative to the count), + * the tombstone fraction of the overlapping SSTs, and — for open-ended + * starts — the error compounded by complement subtraction. A heuristic + * ordering signal, not a statistical bound. + */ + confidence: number; +} + export interface RangeOptions extends RocksDBOptions { /** * The range end key, otherwise known as the "upper bound". Defaults to diff --git a/src/index.ts b/src/index.ts index fdf51bf06..ed1a563f2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,7 @@ export { } from './database.ts'; export { CountEstimator, type CountEstimatorOptions } from './count-estimator.ts'; export { DBIterator } from './dbi-iterator.ts'; -export { DBI, type IteratorOptions } from './dbi.ts'; +export { DBI, type CountEstimate, type IteratorOptions } from './dbi.ts'; export type { Key } from './encoding.ts'; export type * from './stats.ts'; export { diff --git a/src/load-binding.ts b/src/load-binding.ts index fa683674a..7a14200a3 100644 --- a/src/load-binding.ts +++ b/src/load-binding.ts @@ -333,7 +333,7 @@ export type NativeDatabase = { txnId?: number, expectedVersion?: number ): number; - estimateCount(startKey?: Buffer, endKey?: Buffer): number; + estimateCount(startKey?: Buffer, endKey?: Buffer): { count: number; confidence: number }; getCompression(): { algorithm: string; level?: number }; getCount(options?: RangeOptions, txnId?: number): number; getDBIntProperty(propertyName: string): number | undefined; diff --git a/src/store.ts b/src/store.ts index 93e9e77a9..c4c0a5728 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,7 +1,7 @@ import { type BackupStreamOptions, backupToStream } from './backup-stream.ts'; import { assertBackupDirOutsideDatabase, type BackupOptions } from './backup.ts'; import { DBIterator, type DBIteratorValue } from './dbi-iterator.ts'; -import type { DBITransactional, IteratorOptions, RangeOptions } from './dbi.ts'; +import type { CountEstimate, DBITransactional, IteratorOptions, RangeOptions } from './dbi.ts'; import { type BufferWithDataView, createFixedBuffer, @@ -852,7 +852,7 @@ export class Store { * without iterating. Estimates always reflect committed state, so there is * no transactional variant. */ - estimateCount(options?: RangeOptions): number { + estimateCount(options?: RangeOptions): CountEstimate { let startBuffer: Buffer | undefined; let endBuffer: Buffer | undefined; diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index c94fd7505..8cb01dd30 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -12,6 +12,11 @@ function expectWithin(estimate: number, exact: number, factor: number) { expect(estimate).toBeLessThanOrEqual(exact * factor); } +function expectConfidence(confidence: number, min = 0, max = 1) { + expect(confidence).toBeGreaterThanOrEqual(min); + expect(confidence).toBeLessThanOrEqual(max); +} + const KEY = (i: number) => `key-${String(i).padStart(6, '0')}`; describe('estimateCount', () => { @@ -25,19 +30,23 @@ describe('estimateCount', () => { // full range, both open-ended and bounded expectWithin(db.getEstimatedKeyCount(), N, 2); - expectWithin(db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N) }), N, 2); + const full = db.estimateCount({ start: KEY(0), end: KEY(N) }); + expectWithin(full.count, N, 2); + // a large uniform range should be high-confidence + expectConfidence(full.confidence, 0.5, 1); // half range [25%, 75%) - const half = db.getEstimatedKeyCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); - expectWithin(half, N / 2, 2); + const half = db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); + expectWithin(half.count, N / 2, 2); // open-ended: start only and end only - expectWithin(db.getEstimatedKeyCount({ start: KEY(N / 2) }), N / 2, 2); - expectWithin(db.getEstimatedKeyCount({ end: KEY(N / 2) }), N / 2, 2); + expectWithin(db.estimateCount({ start: KEY(N / 2) }).count, N / 2, 2); + expectWithin(db.estimateCount({ end: KEY(N / 2) }).count, N / 2, 2); // a range past all data should estimate near zero relative to N - const empty = db.getEstimatedKeyCount({ start: 'z', end: 'zz' }); - expect(empty).toBeLessThan(N / 20); + const empty = db.estimateCount({ start: 'z', end: 'zz' }); + expect(empty.count).toBeLessThan(N / 20); + expectConfidence(empty.confidence); })); it('should estimate counts for data still in the memtable', () => @@ -47,8 +56,8 @@ describe('estimateCount', () => { await db.put(KEY(i), `value-${i}`); } // no flush: everything is in the memtable - expectWithin(db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N) }), N, 2); - expectWithin(db.getEstimatedKeyCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }), N / 2, 2); + expectWithin(db.estimateCount({ start: KEY(0), end: KEY(N) }).count, N, 2); + expectWithin(db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }).count, N / 2, 2); })); it('should estimate counts spanning memtable and SST data', () => @@ -61,23 +70,33 @@ describe('estimateCount', () => { for (let i = N; i < 2 * N; i++) { await db.put(KEY(i), `value-${i}`); } - expectWithin(db.getEstimatedKeyCount({ start: KEY(0), end: KEY(2 * N) }), 2 * N, 2); + expectWithin(db.estimateCount({ start: KEY(0), end: KEY(2 * N) }).count, 2 * N, 2); })); - it('should return 0 for an empty database', () => + it('should return a confident 0 for an empty database', () => dbRunner(async ({ db }) => { expect(db.getEstimatedKeyCount()).toBe(0); - expect(db.getEstimatedKeyCount({ start: 'a', end: 'z' })).toBe(0); + expect(db.estimateCount()).toEqual({ count: 0, confidence: 1 }); + const range = db.estimateCount({ start: 'a', end: 'z' }); + expect(range.count).toBe(0); + expectConfidence(range.confidence, 0.9, 1); })); - it('should return 0 for an inverted range', () => + it('should return an exact 0 for an inverted range', () => dbRunner(async ({ db }) => { for (let i = 0; i < 5000; i++) { await db.put(KEY(i), `value-${i}`); } await db.flush(); - expect(db.getEstimatedKeyCount({ start: KEY(4000), end: KEY(1000) })).toBe(0); - expect(db.getEstimatedKeyCount({ start: KEY(1000), end: KEY(1000) })).toBe(0); + // inverted/empty by construction: exact, so confidence is 1 + expect(db.estimateCount({ start: KEY(4000), end: KEY(1000) })).toEqual({ + count: 0, + confidence: 1, + }); + expect(db.estimateCount({ start: KEY(1000), end: KEY(1000) })).toEqual({ + count: 0, + confidence: 1, + }); })); it('should treat zero-length native bounds safely', () => @@ -90,8 +109,10 @@ describe('estimateCount', () => { // directly: an empty end bound sorts below every key (empty range), // an empty start bound is the minimum key (no-op lower bound) const native = (db as any).store.db; - expect(native.estimateCount(undefined, Buffer.alloc(0))).toBe(0); - expect(native.estimateCount(Buffer.alloc(0), undefined)).toBe(db.getEstimatedKeyCount()); + expect(native.estimateCount(undefined, Buffer.alloc(0)).count).toBe(0); + expect(native.estimateCount(Buffer.alloc(0), undefined).count).toBe( + db.getEstimatedKeyCount() + ); })); it('should not count uncommitted transaction writes', () => @@ -104,8 +125,8 @@ describe('estimateCount', () => { for (let i = N; i < 2 * N; i++) { txn.putSync(KEY(i), `value-${i}`); } - const estimate = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(2 * N) }); - expect(estimate).toBeLessThan(N * 1.5); + const estimate = db.estimateCount({ start: KEY(0), end: KEY(2 * N) }); + expect(estimate.count).toBeLessThan(N * 1.5); }); })); @@ -117,9 +138,9 @@ describe('estimateCount', () => { } await db.flush(); - const tenth = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N / 10) }); - const half = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N / 2) }); - const full = db.getEstimatedKeyCount({ start: KEY(0), end: KEY(N) }); + const tenth = db.estimateCount({ start: KEY(0), end: KEY(N / 10) }).count; + const half = db.estimateCount({ start: KEY(0), end: KEY(N / 2) }).count; + const full = db.estimateCount({ start: KEY(0), end: KEY(N) }).count; expect(tenth).toBeLessThan(half); expect(half).toBeLessThan(full); })); @@ -138,7 +159,9 @@ describe('CountEstimator', () => { const estimator = db.createCountEstimator(range); // before any traversal: the pure statistical estimate - expectWithin(estimator.estimate(), N, 2); + const initial = estimator.estimate(); + expectWithin(initial.count, N, 2); + expectConfidence(initial.confidence); // walk the first half, checkpointing once per "page" let traversed = 0; @@ -155,10 +178,12 @@ describe('CountEstimator', () => { expect(estimator.traversed).toBe(N / 2); // with half the range traversed exactly, the estimate must be at - // least the traversed count and within a tighter overall bound + // least the traversed count, within a tighter overall bound, and + // more trusted than the untraversed estimate const refined = estimator.estimate(); - expect(refined).toBeGreaterThanOrEqual(N / 2); - expectWithin(refined, N, 1.6); + expect(refined.count).toBeGreaterThanOrEqual(N / 2); + expectWithin(refined.count, N, 1.6); + expect(refined.confidence).toBeGreaterThan(initial.confidence); })); it('should support reverse iteration', () => @@ -173,8 +198,8 @@ describe('CountEstimator', () => { // walk the last quarter in reverse estimator.advance(KEY((3 * N) / 4), N / 4); const refined = estimator.estimate(); - expect(refined).toBeGreaterThanOrEqual(N / 4); - expectWithin(refined, N, 2); + expect(refined.count).toBeGreaterThanOrEqual(N / 4); + expectWithin(refined.count, N, 2); })); it('should converge to near-exact as traversal completes', () => @@ -190,11 +215,12 @@ describe('CountEstimator', () => { // remainder is (KEY(N-1), KEY(N)) — essentially empty, though the // estimate of it is block-granular, so allow a small overshoot const final = estimator.estimate(); - expect(final).toBeGreaterThanOrEqual(N); - expect(final).toBeLessThan(N * 1.25); + expect(final.count).toBeGreaterThanOrEqual(N); + expect(final.count).toBeLessThan(N * 1.25); + expectConfidence(final.confidence, 0.7, 1); estimator.finish(); - expect(estimator.estimate()).toBe(N); + expect(estimator.estimate()).toEqual({ count: N, confidence: 1 }); })); it('should report the exact total for a paginated loop driven to completion', () => @@ -220,10 +246,10 @@ describe('CountEstimator', () => { pageStart = page[page.length - 1] as string; exclusiveStart = true; estimator.advance(pageStart, page.length); - expect(estimator.estimate()).toBeGreaterThanOrEqual(estimator.traversed); + expect(estimator.estimate().count).toBeGreaterThanOrEqual(estimator.traversed); } expect(estimator.traversed).toBe(N); estimator.finish(); - expect(estimator.estimate()).toBe(N); + expect(estimator.estimate()).toEqual({ count: N, confidence: 1 }); })); }); From 0548d42a1db3fd85171ea31935f54f2fb0d6e1a0 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 13 Aug 2026 12:17:46 -0600 Subject: [PATCH 05/15] fix: failed statistics must not report confident estimates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adjudicated major from the API-shape review: a failed GetApproximateSizes/ GetPropertiesOfTablesInRange silently degraded to a memtable-only count while the confidence formula still reported it as trustworthy, and a failed estimate-num-keys property read returned { 0, 1.0 } — a missing answer dressed as a confidently empty database. Track degradation in RangeEstimate (capping confidence at 0.1) and return { 0, 0 } for the failed property read. Also: guard null table-properties entries, honor the range own exclusiveStart/inclusiveEnd flags in CountEstimator segments, and cap non-exact estimator confidence at 0.999 so only finish() and exact-by-construction ranges claim 1. Co-Authored-By: Claude Fable 5 --- src/binding/database/database.cpp | 28 +++++++++++++++++++++-- src/count-estimator.ts | 38 ++++++++++++++++++++++++------- test/estimate-count.test.ts | 5 +++- 3 files changed, 60 insertions(+), 11 deletions(-) diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 56e95146c..5fbd13357 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -951,6 +951,10 @@ struct RangeEstimate { // Live fraction of SST entries ((entries - deletions) / entries); 1 when // no table properties were available. double liveFraction = 1; + // Set when a statistics call failed, so the count is missing a portion it + // should have had — the confidence must reflect that, or a failed estimate + // masquerades as a trustworthy small count. + bool degraded = false; }; static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const rocksdb::Slice& start, const rocksdb::Slice& end) { @@ -969,6 +973,7 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa uint64_t sstBytes = 0; rocksdb::Status status = db->GetApproximateSizes(sizeOptions, cf, &range, 1, &sstBytes); if (!status.ok() || sstBytes == 0) { + result.degraded = !status.ok(); return result; } @@ -979,6 +984,9 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa uint64_t fileBytes = 0; if (status.ok()) { for (const auto& prop : props) { + if (!prop.second) { + continue; + } const rocksdb::TableProperties& p = *prop.second; entries += p.num_entries; deletions += p.num_deletions; @@ -987,8 +995,13 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa // so the density denominator must too. fileBytes += p.data_size + p.index_size + p.filter_size; } + } else { + result.degraded = true; } if (entries <= deletions || fileBytes == 0) { + // SST bytes overlap the range but no usable density: the SST portion + // is missing from the count. + result.degraded = true; return result; } @@ -1008,6 +1021,12 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa * skew the estimate cannot see). */ static double estimateConfidence(const RangeEstimate& est) { + if (est.degraded) { + // A statistics call failed, so a portion of the count is simply + // missing; without the cap a failed estimate would report a small + // count with high confidence. + return 0.1; + } if (est.count <= 0) { // Nothing overlaps the range per both memtable stats and SST sizes; // nearly certain but not provably exact (in-flight flush/compaction). @@ -1069,9 +1088,14 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { double confidence = 0; if (!hasEnd) { uint64_t totalKeys = 0; - db->GetIntProperty(cf, rocksdb::DB::Properties::kEstimateNumKeys, &totalKeys); + bool totalOk = db->GetIntProperty(cf, rocksdb::DB::Properties::kEstimateNumKeys, &totalKeys); double total = static_cast(totalKeys); - if (!hasStart || startLength == 0 || totalKeys == 0) { + if (!totalOk) { + // A failed property read leaves totalKeys at 0 — that is a + // missing answer, not an empty database. + estimate = 0; + confidence = 0; + } else if (!hasStart || startLength == 0 || totalKeys == 0) { estimate = total; // estimate-num-keys is RocksDB's own memtable+SST estimate; it // skews high on overwrite/delete-heavy data until compaction. diff --git a/src/count-estimator.ts b/src/count-estimator.ts index 67c8a2d40..2081858d1 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -49,6 +49,8 @@ export class CountEstimator { #store: Store; #start: Key | Uint8Array | undefined; #end: Key | Uint8Array | undefined; + #exclusiveStart: boolean; + #inclusiveEnd: boolean; #reverse: boolean; #cursor: Key | Uint8Array | undefined; #traversed = 0; @@ -59,6 +61,8 @@ export class CountEstimator { this.#store = store; this.#start = options?.start; this.#end = options?.end; + this.#exclusiveStart = options?.exclusiveStart ?? false; + this.#inclusiveEnd = options?.inclusiveEnd ?? false; this.#reverse = options?.reverse ?? false; } @@ -99,7 +103,12 @@ export class CountEstimator { return { count: this.#traversed, confidence: 1 }; } if (this.#cursor === undefined) { - return this.#store.estimateCount({ start: this.#start, end: this.#end }); + return this.#store.estimateCount({ + start: this.#start, + end: this.#end, + exclusiveStart: this.#exclusiveStart, + inclusiveEnd: this.#inclusiveEnd, + }); } if (this.#memoized !== undefined) { return this.#memoized; @@ -107,13 +116,24 @@ export class CountEstimator { // The cursor entry itself belongs to the traversed side, so the // remainder excludes it in both directions (an inclusive lower bound - // forward would count it twice and block convergence). + // forward would count it twice and block convergence). The range's own + // bound flags stay with their original edge of the full range. const traversedRange = this.#reverse - ? { start: this.#cursor, end: this.#end } - : { start: this.#start, end: this.#cursor, inclusiveEnd: true }; + ? { start: this.#cursor, end: this.#end, inclusiveEnd: this.#inclusiveEnd } + : { + start: this.#start, + exclusiveStart: this.#exclusiveStart, + end: this.#cursor, + inclusiveEnd: true, + }; const remainingRange = this.#reverse - ? { start: this.#start, end: this.#cursor } - : { start: this.#cursor, end: this.#end, exclusiveStart: true }; + ? { start: this.#start, exclusiveStart: this.#exclusiveStart, end: this.#cursor } + : { + start: this.#cursor, + exclusiveStart: true, + end: this.#end, + inclusiveEnd: this.#inclusiveEnd, + }; const remainingEstimate = this.#store.estimateCount(remainingRange); let remaining = remainingEstimate.count; @@ -127,10 +147,12 @@ export class CountEstimator { } const count = Math.round(this.#traversed + remaining); + // Cap below 1: only finish() (or an exact-by-construction range) may + // claim exactness, even when rounding makes the remainder vanish. const confidence = count > 0 - ? Math.min(1, (this.#traversed + remainingEstimate.confidence * remaining) / count) - : remainingEstimate.confidence; + ? Math.min(0.999, (this.#traversed + remainingEstimate.confidence * remaining) / count) + : Math.min(0.999, remainingEstimate.confidence); this.#memoized = { count, confidence }; return this.#memoized; } diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index 8cb01dd30..c25eab6f9 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -246,7 +246,10 @@ describe('CountEstimator', () => { pageStart = page[page.length - 1] as string; exclusiveStart = true; estimator.advance(pageStart, page.length); - expect(estimator.estimate().count).toBeGreaterThanOrEqual(estimator.traversed); + const checkpoint = estimator.estimate(); + expect(checkpoint.count).toBeGreaterThanOrEqual(estimator.traversed); + // only finish() may claim exactness + expect(checkpoint.confidence).toBeLessThan(1); } expect(estimator.traversed).toBe(N); estimator.finish(); From 89d437b5b7539d029509242aead5fda26af46703 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 13 Aug 2026 12:20:01 -0600 Subject: [PATCH 06/15] fix: zero estimate-num-keys is an estimate, not exact; partial props degrade Follow-ups from the delta review: a successful zero estimate-num-keys read now reports 0.95 confidence (deletion entries can offset puts, so even zero is estimated), and a null entry in the table-properties collection marks the density degraded rather than being silently skipped. Co-Authored-By: Claude Fable 5 --- src/binding/database/database.cpp | 9 +++++++-- test/estimate-count.test.ts | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 5fbd13357..83d4c7780 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -985,6 +985,9 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa if (status.ok()) { for (const auto& prop : props) { if (!prop.second) { + // A missing entry means the density is computed from an + // incomplete sample. + result.degraded = true; continue; } const rocksdb::TableProperties& p = *prop.second; @@ -1098,8 +1101,10 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { } else if (!hasStart || startLength == 0 || totalKeys == 0) { estimate = total; // estimate-num-keys is RocksDB's own memtable+SST estimate; it - // skews high on overwrite/delete-heavy data until compaction. - confidence = totalKeys == 0 ? 1.0 : 0.9; + // skews high on overwrite/delete-heavy data until compaction, and + // even a zero is an estimate (deletion entries offset puts), so + // nothing on this path claims exactness. + confidence = totalKeys == 0 ? 0.95 : 0.9; } else { // No upper bound: estimate [start, ∞) as total minus [min, start). RangeEstimate complement = estimateRangeCount(db, cf, rocksdb::Slice(), startSlice); diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index c25eab6f9..90197bcc4 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -76,10 +76,11 @@ describe('estimateCount', () => { it('should return a confident 0 for an empty database', () => dbRunner(async ({ db }) => { expect(db.getEstimatedKeyCount()).toBe(0); - expect(db.estimateCount()).toEqual({ count: 0, confidence: 1 }); + // even a zero from estimate-num-keys is an estimate, not exact + expect(db.estimateCount()).toEqual({ count: 0, confidence: 0.95 }); const range = db.estimateCount({ start: 'a', end: 'z' }); expect(range.count).toBe(0); - expectConfidence(range.confidence, 0.9, 1); + expectConfidence(range.confidence, 0.9, 0.99); })); it('should return an exact 0 for an inverted range', () => From d0c13624081ef9996982f87de3ea59c314280bae Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 06:05:32 -0600 Subject: [PATCH 07/15] fix: address range count estimator review feedback Co-Authored-By: GPT-5 Codex --- README.md | 23 +++++---- src/binding/database/database.cpp | 6 ++- src/count-estimator.ts | 58 +++++++++++---------- src/database.ts | 9 +++- src/dbi.ts | 8 +++ src/index.ts | 2 +- src/store.ts | 27 +++++++--- test/estimate-count.test.ts | 83 ++++++++++++++++++++++++++++--- 8 files changed, 159 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 47a0bc584..d72ac980f 100644 --- a/README.md +++ b/README.md @@ -440,19 +440,20 @@ const estimated = db.getEstimatedKeyCount(); console.log(estimated); ``` -### `db.estimateCount(options?: RangeOptions): CountEstimate` +### `db.estimateCount(options?: CountEstimateOptions): CountEstimate` Estimates the number of keys in the database, or within a key range, returning `{ count, confidence }`. Unlike `getKeysCount()`, this never iterates: the estimate is derived from RocksDB statistics (memtable stats plus approximate SST sizes converted through the entry density of the SSTs overlapping the range), so its cost scales with the number of SSTs overlapping -the range rather than the number of keys — typically microseconds where an exact count takes -milliseconds, though reading table properties for cold files can do I/O through the table cache. A -start-only range is computed as the whole-database estimate minus the complement, so it does the -work of the range _below_ `start`. Accuracy improves with range size (resolution is bounded by SST -data-block granularity, so tiny ranges over-report), and recently deleted or overwritten entries -may be counted until compaction. Estimates always reflect committed state; writes pending in a -transaction are not included. An inverted range (`start` ≥ `end`) returns +the range rather than the number of keys. Reading cold table properties can do I/O through the +table cache, so bounded ranges are preferable. A start-only range is computed as the +whole-database estimate minus the complement, so it does the work of the range _below_ `start`. +Accuracy improves with range size (resolution is bounded by SST data-block granularity, so tiny +ranges over-report), and recently deleted or overwritten entries may be counted until compaction. +Estimates always reflect committed state; writes pending in a transaction are not included. Set +`reverse: true` to use `getRange()`'s reverse convention (`start` is the upper bound and `end` is +the lower bound). An inverted range (`start` ≥ `end`) returns `{ count: 0, confidence: 1 }`. `confidence` is a heuristic 0–1 indicator of how trustworthy `count` is — exactly 1 only when the @@ -477,9 +478,9 @@ remainder, calibrated by the observed ratio of actual-to-estimated entries over already traversed — so the count converges toward the exact total, and `confidence` (the exactness-weighted blend of the traversed portion and the remainder's confidence) converges to 1. When traversal completes, call `finish()` and `estimate()` returns the exact count with -confidence 1. Set `reverse: true` when iterating from `end` toward `start`. The caller owns the -progress contract: cursors must move monotonically through the range and each entry must be -reported exactly once. +confidence 1. Reverse ranges follow `getRange`: set `start` to the upper bound and `end` to the +lower bound, then set `reverse: true`. The caller owns the progress contract: cursors must move +monotonically through the range and each entry must be reported exactly once. ```typescript const range = { start: 'a', end: 'z' }; diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 83d4c7780..30a814c6d 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -982,6 +982,7 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa uint64_t entries = 0; uint64_t deletions = 0; uint64_t fileBytes = 0; + uint64_t dataBlocks = 0; if (status.ok()) { for (const auto& prop : props) { if (!prop.second) { @@ -997,6 +998,7 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa // GetApproximateSizes offsets span data + index + filter blocks, // so the density denominator must too. fileBytes += p.data_size + p.index_size + p.filter_size; + dataBlocks += p.num_data_blocks; } } else { result.degraded = true; @@ -1011,7 +1013,9 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa double density = static_cast(entries - deletions) / static_cast(fileBytes); result.sstCount = static_cast(sstBytes) * density; result.count += result.sstCount; - result.entriesPerBlock = density * 4096; + result.entriesPerBlock = dataBlocks > 0 + ? static_cast(entries - deletions) / static_cast(dataBlocks) + : 0; result.liveFraction = static_cast(entries - deletions) / static_cast(entries); return result; } diff --git a/src/count-estimator.ts b/src/count-estimator.ts index 2081858d1..0321e0905 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -1,21 +1,15 @@ -import type { CountEstimate, RangeOptions } from './dbi.ts'; +import type { CountEstimate, CountEstimateOptions } from './dbi.ts'; import type { Key } from './encoding.ts'; import type { Store } from './store.ts'; -export interface CountEstimatorOptions extends RangeOptions { - /** - * When `true`, iteration proceeds from `end` toward `start`, so keys - * passed to `advance()` are treated as the new lower edge of the - * untraversed remainder. - */ - reverse?: boolean; -} +export interface CountEstimatorOptions extends CountEstimateOptions {} /** * Below this many traversed entries the observed density is too noisy to * calibrate with, so `estimate()` returns the raw statistical estimate. */ const CALIBRATION_MIN_TRAVERSED = 16; +const CALIBRATION_MIN_CONFIDENCE = 0.8; /** * Bounds on how far the observed-vs-estimated ratio may scale the remainder; @@ -59,11 +53,11 @@ export class CountEstimator { constructor(store: Store, options?: CountEstimatorOptions) { this.#store = store; - this.#start = options?.start; - this.#end = options?.end; - this.#exclusiveStart = options?.exclusiveStart ?? false; - this.#inclusiveEnd = options?.inclusiveEnd ?? false; this.#reverse = options?.reverse ?? false; + this.#start = this.#reverse ? options?.end : options?.start; + this.#end = this.#reverse ? options?.start : options?.end; + this.#exclusiveStart = options?.exclusiveStart ?? this.#reverse; + this.#inclusiveEnd = options?.inclusiveEnd ?? this.#reverse; } /** @@ -77,7 +71,10 @@ export class CountEstimator { * Records that iteration has advanced through `count` more entries, ending * at `lastKey`. */ - advance(lastKey: Key | Uint8Array, count = 1): void { + advance(lastKey: Key | Uint8Array | undefined, count = 1): void { + if (lastKey === undefined) { + return; + } this.#cursor = lastKey; this.#traversed += count; this.#memoized = undefined; @@ -102,16 +99,17 @@ export class CountEstimator { if (this.#finished) { return { count: this.#traversed, confidence: 1 }; } + if (this.#memoized !== undefined) { + return { ...this.#memoized }; + } if (this.#cursor === undefined) { - return this.#store.estimateCount({ + this.#memoized = this.#store.estimateCount({ start: this.#start, end: this.#end, exclusiveStart: this.#exclusiveStart, inclusiveEnd: this.#inclusiveEnd, }); - } - if (this.#memoized !== undefined) { - return this.#memoized; + return { ...this.#memoized }; } // The cursor entry itself belongs to the traversed side, so the @@ -137,23 +135,31 @@ export class CountEstimator { const remainingEstimate = this.#store.estimateCount(remainingRange); let remaining = remainingEstimate.count; + let calibrationConfidence = 1; if (this.#traversed >= CALIBRATION_MIN_TRAVERSED) { const traversedEstimate = this.#store.estimateCount(traversedRange); - const calibration = Math.min( - CALIBRATION_MAX, - Math.max(1 / CALIBRATION_MAX, this.#traversed / Math.max(traversedEstimate.count, 1)) - ); - remaining *= calibration; + if ( + traversedEstimate.count >= CALIBRATION_MIN_TRAVERSED && + traversedEstimate.confidence >= CALIBRATION_MIN_CONFIDENCE + ) { + const calibration = Math.min( + CALIBRATION_MAX, + Math.max(1 / CALIBRATION_MAX, this.#traversed / traversedEstimate.count) + ); + remaining *= calibration; + calibrationConfidence = Math.min(calibration, 1 / calibration); + } } const count = Math.round(this.#traversed + remaining); // Cap below 1: only finish() (or an exact-by-construction range) may // claim exactness, even when rounding makes the remainder vanish. const confidence = - count > 0 + calibrationConfidence * + (count > 0 ? Math.min(0.999, (this.#traversed + remainingEstimate.confidence * remaining) / count) - : Math.min(0.999, remainingEstimate.confidence); + : Math.min(0.999, remainingEstimate.confidence)); this.#memoized = { count, confidence }; - return this.#memoized; + return { ...this.#memoized }; } } diff --git a/src/database.ts b/src/database.ts index 2482a8132..f4eee847d 100644 --- a/src/database.ts +++ b/src/database.ts @@ -1,7 +1,12 @@ import type { BackupStreamOptions } from './backup-stream.ts'; import type { BackupOptions } from './backup.ts'; import { CountEstimator, type CountEstimatorOptions } from './count-estimator.ts'; -import { DBI, type CountEstimate, type DBITransactional, type RangeOptions } from './dbi.ts'; +import { + DBI, + type CountEstimate, + type CountEstimateOptions, + type DBITransactional, +} from './dbi.ts'; import type { BufferWithDataView, Encoder, EncoderFunction, Key } from './encoding.ts'; import { addGlobalListener, @@ -495,7 +500,7 @@ export class RocksDatabase extends DBI { * const { count, confidence } = db.estimateCount({ start: 'a', end: 'z' }); * ``` */ - estimateCount(options?: RangeOptions): CountEstimate { + estimateCount(options?: CountEstimateOptions): CountEstimate { return this.store.estimateCount(options); } diff --git a/src/dbi.ts b/src/dbi.ts index db5efbdcb..ad9ffaaf8 100644 --- a/src/dbi.ts +++ b/src/dbi.ts @@ -105,6 +105,14 @@ export interface RangeOptions extends RocksDBOptions { start?: Key | Uint8Array; } +export interface CountEstimateOptions extends RangeOptions { + /** + * Interpret `start` as the upper bound and `end` as the lower bound, as + * `getRange()` does for reverse iteration. Defaults to `false`. + */ + reverse?: boolean; +} + export interface IteratorOptions extends RangeOptions { // decoder?: (value: any) => any, diff --git a/src/index.ts b/src/index.ts index ed1a563f2..8c6692e22 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,7 @@ export { } from './database.ts'; export { CountEstimator, type CountEstimatorOptions } from './count-estimator.ts'; export { DBIterator } from './dbi-iterator.ts'; -export { DBI, type CountEstimate, type IteratorOptions } from './dbi.ts'; +export { DBI, type CountEstimate, type CountEstimateOptions, type IteratorOptions } from './dbi.ts'; export type { Key } from './encoding.ts'; export type * from './stats.ts'; export { diff --git a/src/store.ts b/src/store.ts index c4c0a5728..24a2ce415 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,7 +1,13 @@ import { type BackupStreamOptions, backupToStream } from './backup-stream.ts'; import { assertBackupDirOutsideDatabase, type BackupOptions } from './backup.ts'; import { DBIterator, type DBIteratorValue } from './dbi-iterator.ts'; -import type { CountEstimate, DBITransactional, IteratorOptions, RangeOptions } from './dbi.ts'; +import type { + CountEstimate, + CountEstimateOptions, + DBITransactional, + IteratorOptions, + RangeOptions, +} from './dbi.ts'; import { type BufferWithDataView, createFixedBuffer, @@ -852,20 +858,25 @@ export class Store { * without iterating. Estimates always reflect committed state, so there is * no transactional variant. */ - estimateCount(options?: RangeOptions): CountEstimate { + estimateCount(options?: CountEstimateOptions): CountEstimate { let startBuffer: Buffer | undefined; let endBuffer: Buffer | undefined; + const reverse = options?.reverse ?? false; + const start = reverse ? options?.end : options?.start; + const end = reverse ? options?.start : options?.end; + const exclusiveStart = options?.exclusiveStart ?? reverse; + const inclusiveEnd = options?.inclusiveEnd ?? reverse; - if (options?.start !== undefined) { - const start = this.encodeKey(options.start); + if (start !== undefined) { + const encodedStart = this.encodeKey(start); // A zero byte appended to a key is its bytewise successor, turning an // inclusive bound into an exclusive one (and vice versa for the end). - startBuffer = copyEncodedKey(start, options.exclusiveStart === true); + startBuffer = copyEncodedKey(encodedStart, exclusiveStart); } - if (options?.end !== undefined) { - const end = this.encodeKey(options.end); - endBuffer = copyEncodedKey(end, options.inclusiveEnd === true); + if (end !== undefined) { + const encodedEnd = this.encodeKey(end); + endBuffer = copyEncodedKey(encodedEnd, inclusiveEnd); } return this.db.estimateCount(startBuffer, endBuffer); diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index 90197bcc4..e460a8e40 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -1,5 +1,5 @@ import { dbRunner } from './lib/util.ts'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; /** * Estimates are statistical (block-granular SST approximation + memtable @@ -145,6 +145,22 @@ describe('estimateCount', () => { expect(tenth).toBeLessThan(half); expect(half).toBeLessThan(full); })); + + it('should use range-local density for varied value sizes', () => + dbRunner(async ({ db }) => { + const N = 3000; + for (let i = 0; i < N; i++) { + await db.put(`small-${KEY(i)}`, 'x'.repeat(20)); + } + await db.flush(); + for (let i = 0; i < N; i++) { + await db.put(`large-${KEY(i)}`, 'x'.repeat(1000)); + } + await db.flush(); + + expectWithin(db.estimateCount({ start: 'small-', end: 'small.' }).count, N, 2); + expectWithin(db.estimateCount({ start: 'large-', end: 'large.' }).count, N, 2); + })); }); describe('CountEstimator', () => { @@ -184,7 +200,7 @@ describe('CountEstimator', () => { const refined = estimator.estimate(); expect(refined.count).toBeGreaterThanOrEqual(N / 2); expectWithin(refined.count, N, 1.6); - expect(refined.confidence).toBeGreaterThan(initial.confidence); + expectConfidence(refined.confidence); })); it('should support reverse iteration', () => @@ -195,12 +211,63 @@ describe('CountEstimator', () => { } await db.flush(); - const estimator = db.createCountEstimator({ start: KEY(0), end: KEY(N), reverse: true }); - // walk the last quarter in reverse - estimator.advance(KEY((3 * N) / 4), N / 4); - const refined = estimator.estimate(); - expect(refined.count).toBeGreaterThanOrEqual(N / 4); - expectWithin(refined.count, N, 2); + const range = { start: KEY(N), end: KEY(0), reverse: true }; + const estimator = db.createCountEstimator(range); + expectWithin(db.estimateCount(range).count, N - 1, 2); + let upperBound = range.start; + let inclusiveEnd = true; + for (;;) { + const page = Array.from( + db.getKeys({ ...range, start: upperBound, inclusiveEnd, limit: 500 }) + ); + if (page.length === 0) { + break; + } + upperBound = page[page.length - 1] as string; + inclusiveEnd = false; + estimator.advance(upperBound, page.length); + const checkpoint = estimator.estimate(); + expect(checkpoint.count).toBeGreaterThanOrEqual(estimator.traversed); + expectWithin(checkpoint.count, N - 1, 2); + } + expect(estimator.traversed).toBe(N - 1); + estimator.finish(); + expect(estimator.estimate()).toEqual({ count: N - 1, confidence: 1 }); + })); + + it('should memoize before traversal and ignore an empty page', () => + dbRunner(async ({ db }) => { + for (let i = 0; i < 1000; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + + const estimateCount = vi.spyOn(db.store, 'estimateCount'); + const estimator = db.createCountEstimator({ start: KEY(0), end: KEY(1000) }); + const initial = estimator.estimate(); + const expected = { ...initial }; + expect(estimator.estimate()).toEqual(initial); + expect(estimateCount).toHaveBeenCalledTimes(1); + initial.count = 0; + expect(estimator.estimate()).toEqual(expected); + + estimator.advance(undefined, 1); + expect(estimator.traversed).toBe(0); + expect(estimator.estimate()).toEqual(expected); + expect(estimateCount).toHaveBeenCalledTimes(1); + })); + + it('should not calibrate from a block-granular partial page', () => + dbRunner(async ({ db }) => { + const N = 20000; + for (let i = 0; i < N; i++) { + await db.put(KEY(i), `value-${i}`); + } + await db.flush(); + + const estimator = db.createCountEstimator({ start: KEY(0), end: KEY(N) }); + estimator.advance(KEY(24), 25); + expectWithin(estimator.estimate().count, N, 2); })); it('should converge to near-exact as traversal completes', () => From c21a923ad7e758dde9f8109f3b7ddae0b8bd103a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 06:46:28 -0600 Subject: [PATCH 08/15] fix: harden estimate confidence Co-Authored-By: GPT-5 Codex --- README.md | 2 +- src/binding/database/database.cpp | 8 ++++++-- src/count-estimator.ts | 14 ++++++++++---- test/estimate-count.test.ts | 13 ++++++++++++- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d72ac980f..0a9928664 100644 --- a/README.md +++ b/README.md @@ -447,7 +447,7 @@ Estimates the number of keys in the database, or within a key range, returning from RocksDB statistics (memtable stats plus approximate SST sizes converted through the entry density of the SSTs overlapping the range), so its cost scales with the number of SSTs overlapping the range rather than the number of keys. Reading cold table properties can do I/O through the -table cache, so bounded ranges are preferable. A start-only range is computed as the +table cache, so narrow ranges are preferable. A start-only range is computed as the whole-database estimate minus the complement, so it does the work of the range _below_ `start`. Accuracy improves with range size (resolution is bounded by SST data-block granularity, so tiny ranges over-report), and recently deleted or overwritten entries may be counted until compaction. diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index 30a814c6d..e360fe9ae 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -972,8 +972,8 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa sizeOptions.files_size_error_margin = 0.1; uint64_t sstBytes = 0; rocksdb::Status status = db->GetApproximateSizes(sizeOptions, cf, &range, 1, &sstBytes); - if (!status.ok() || sstBytes == 0) { - result.degraded = !status.ok(); + if (!status.ok()) { + result.degraded = true; return result; } @@ -1006,6 +1006,10 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa if (entries <= deletions || fileBytes == 0) { // SST bytes overlap the range but no usable density: the SST portion // is missing from the count. + result.degraded = sstBytes != 0; + return result; + } + if (sstBytes == 0) { result.degraded = true; return result; } diff --git a/src/count-estimator.ts b/src/count-estimator.ts index 0321e0905..e243e573c 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -73,6 +73,9 @@ export class CountEstimator { */ advance(lastKey: Key | Uint8Array | undefined, count = 1): void { if (lastKey === undefined) { + if (count !== 0) { + throw new Error('CountEstimator.advance requires lastKey when count is nonzero'); + } return; } this.#cursor = lastKey; @@ -155,10 +158,13 @@ export class CountEstimator { // Cap below 1: only finish() (or an exact-by-construction range) may // claim exactness, even when rounding makes the remainder vanish. const confidence = - calibrationConfidence * - (count > 0 - ? Math.min(0.999, (this.#traversed + remainingEstimate.confidence * remaining) / count) - : Math.min(0.999, remainingEstimate.confidence)); + count > 0 + ? Math.min( + 0.999, + (this.#traversed + calibrationConfidence * remainingEstimate.confidence * remaining) / + count + ) + : Math.min(0.999, calibrationConfidence * remainingEstimate.confidence); this.#memoized = { count, confidence }; return { ...this.#memoized }; } diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index e460a8e40..99ba02adb 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -35,6 +35,15 @@ describe('estimateCount', () => { // a large uniform range should be high-confidence expectConfidence(full.confidence, 0.5, 1); + const singleKey = db.estimateCount({ + start: KEY(N / 2), + end: KEY(N / 2), + inclusiveEnd: true, + }); + if (singleKey.count === 0) { + expect(singleKey.confidence).toBeLessThanOrEqual(0.1); + } + // half range [25%, 75%) const half = db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); expectWithin(half.count, N / 2, 2); @@ -251,7 +260,9 @@ describe('CountEstimator', () => { initial.count = 0; expect(estimator.estimate()).toEqual(expected); - estimator.advance(undefined, 1); + expect(() => estimator.advance(undefined, 1)).toThrow( + 'CountEstimator.advance requires lastKey when count is nonzero' + ); expect(estimator.traversed).toBe(0); expect(estimator.estimate()).toEqual(expected); expect(estimateCount).toHaveBeenCalledTimes(1); From d3030fbf7720e8d56166ebae63e0839cecda9f09 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 15 Aug 2026 06:51:22 -0600 Subject: [PATCH 09/15] docs: clarify count estimator caveats Co-Authored-By: GPT-5 Codex --- README.md | 2 ++ src/binding/database/database.cpp | 49 +++++++------------------------ src/count-estimator.ts | 2 +- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 0a9928664..d25c8831e 100644 --- a/README.md +++ b/README.md @@ -477,6 +477,8 @@ per page), `estimate()` returns the exact traversed count plus a statistical est remainder, calibrated by the observed ratio of actual-to-estimated entries over the portion already traversed — so the count converges toward the exact total, and `confidence` (the exactness-weighted blend of the traversed portion and the remainder's confidence) converges to 1. +Each checkpoint reads committed state, so a traversal performed against a transaction snapshot may +be calibrated against data committed after that snapshot. When traversal completes, call `finish()` and `estimate()` returns the exact count with confidence 1. Reverse ranges follow `getRange`: set `start` to the upper bound and `end` to the lower bound, then set `reverse: true`. The caller owns the progress contract: cursors must move diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index e360fe9ae..a4d07131f 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -922,6 +922,15 @@ napi_value Database::GetCount(napi_env env, napi_callback_info info) { return result; } +struct RangeEstimate { + double count = 0; + double memtableCount = 0; + double sstCount = 0; + double entriesPerBlock = 0; + double liveFraction = 1; + bool degraded = false; +}; + /** * Estimates the number of live keys in `[start, end)` from RocksDB statistics * alone — no iteration: @@ -941,22 +950,6 @@ napi_value Database::GetCount(napi_env env, napi_callback_info info) { * compaction; resolution is bounded by SST data-block granularity, so tiny * ranges over-report. */ -struct RangeEstimate { - double count = 0; - double memtableCount = 0; - double sstCount = 0; - // Live-entry resolution of one SST data block — the granularity the SST - // portion of the estimate is quantized to (0 when unknown/no SST data). - double entriesPerBlock = 0; - // Live fraction of SST entries ((entries - deletions) / entries); 1 when - // no table properties were available. - double liveFraction = 1; - // Set when a statistics call failed, so the count is missing a portion it - // should have had — the confidence must reflect that, or a failed estimate - // masquerades as a trustworthy small count. - bool degraded = false; -}; - static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const rocksdb::Slice& start, const rocksdb::Slice& end) { rocksdb::Range range(start, end); RangeEstimate result; @@ -986,8 +979,6 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa if (status.ok()) { for (const auto& prop : props) { if (!prop.second) { - // A missing entry means the density is computed from an - // incomplete sample. result.degraded = true; continue; } @@ -1004,8 +995,7 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa result.degraded = true; } if (entries <= deletions || fileBytes == 0) { - // SST bytes overlap the range but no usable density: the SST portion - // is missing from the count. + // A nonzero byte estimate without density leaves the SST portion unknown. result.degraded = sstBytes != 0; return result; } @@ -1033,14 +1023,9 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa */ static double estimateConfidence(const RangeEstimate& est) { if (est.degraded) { - // A statistics call failed, so a portion of the count is simply - // missing; without the cap a failed estimate would report a small - // count with high confidence. return 0.1; } if (est.count <= 0) { - // Nothing overlaps the range per both memtable stats and SST sizes; - // nearly certain but not provably exact (in-flight flush/compaction). return 0.95; } double sstResolution = std::max(est.entriesPerBlock, 1.0); @@ -1071,9 +1056,7 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { rocksdb::DB* db = (*dbHandle)->descriptor->db.get(); rocksdb::ColumnFamilyHandle* cf = (*dbHandle)->getColumnFamilyHandle(); - // Presence is tracked separately from the data pointer: napi may return a - // null pointer for a zero-length buffer, and an empty bound (the smallest - // key) must not be confused with an omitted one. + // N-API may return a null data pointer for a zero-length buffer. void* startData = nullptr; size_t startLength = 0; napi_valuetype startType; @@ -1102,25 +1085,15 @@ napi_value Database::EstimateCount(napi_env env, napi_callback_info info) { bool totalOk = db->GetIntProperty(cf, rocksdb::DB::Properties::kEstimateNumKeys, &totalKeys); double total = static_cast(totalKeys); if (!totalOk) { - // A failed property read leaves totalKeys at 0 — that is a - // missing answer, not an empty database. estimate = 0; confidence = 0; } else if (!hasStart || startLength == 0 || totalKeys == 0) { estimate = total; - // estimate-num-keys is RocksDB's own memtable+SST estimate; it - // skews high on overwrite/delete-heavy data until compaction, and - // even a zero is an estimate (deletion entries offset puts), so - // nothing on this path claims exactness. confidence = totalKeys == 0 ? 0.95 : 0.9; } else { // No upper bound: estimate [start, ∞) as total minus [min, start). RangeEstimate complement = estimateRangeCount(db, cf, rocksdb::Slice(), startSlice); estimate = std::max(0.0, total - complement.count); - // Subtracting two independent estimates compounds their absolute - // errors, so trust shrinks with the complement's share of the - // total (a narrow tail range after a large complement is mostly - // error). double share = estimate / std::max(estimate + complement.count, 1.0); confidence = std::min(0.9, estimateConfidence(complement)) * share; } diff --git a/src/count-estimator.ts b/src/count-estimator.ts index e243e573c..def0e9a82 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -69,7 +69,7 @@ export class CountEstimator { /** * Records that iteration has advanced through `count` more entries, ending - * at `lastKey`. + * at `lastKey`. Pass `count: 0` when an empty page has no last key. */ advance(lastKey: Key | Uint8Array | undefined, count = 1): void { if (lastKey === undefined) { From 9c222a3cbdbd4cf3080e99b94d148c3c70b2d922 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 18:18:44 -0600 Subject: [PATCH 10/15] Address range estimator review feedback Co-Authored-By: GPT-5 Codex --- README.md | 12 ++++++++---- src/binding/database/database.cpp | 4 ++-- test/estimate-count.test.ts | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d25c8831e..0d1e92abe 100644 --- a/README.md +++ b/README.md @@ -449,8 +449,10 @@ density of the SSTs overlapping the range), so its cost scales with the number o the range rather than the number of keys. Reading cold table properties can do I/O through the table cache, so narrow ranges are preferable. A start-only range is computed as the whole-database estimate minus the complement, so it does the work of the range _below_ `start`. -Accuracy improves with range size (resolution is bounded by SST data-block granularity, so tiny -ranges over-report), and recently deleted or overwritten entries may be counted until compaction. +Accuracy improves with range size. Resolution is bounded by SST data-block granularity, so a range +narrower than a block is unreliable in either direction: it may over-report or report 0 for present +keys, and its low `confidence` is the signal. Recently deleted or overwritten entries may be counted +until compaction. Estimates always reflect committed state; writes pending in a transaction are not included. Set `reverse: true` to use `getRange()`'s reverse convention (`start` is the upper bound and `end` is the lower bound). An inverted range (`start` ≥ `end`) returns @@ -475,8 +477,10 @@ range. Before any traversal, `estimate()` returns the pure statistical estimate `estimateCount(range)`). As the caller reports progress with `advance(lastKey, count)` (e.g. once per page), `estimate()` returns the exact traversed count plus a statistical estimate of the remainder, calibrated by the observed ratio of actual-to-estimated entries over the portion -already traversed — so the count converges toward the exact total, and `confidence` (the -exactness-weighted blend of the traversed portion and the remainder's confidence) converges to 1. +already traversed — so the count converges toward the exact total. `confidence` is the +exactness-weighted blend of the traversed portion and the remainder's confidence, so it approaches 1 +as the exact portion grows, although a checkpoint may decrease when calibration makes a large +correction. Each checkpoint reads committed state, so a traversal performed against a transaction snapshot may be calibrated against data committed after that snapshot. When traversal completes, call `finish()` and `estimate()` returns the exact count with diff --git a/src/binding/database/database.cpp b/src/binding/database/database.cpp index a4d07131f..76d02bacc 100644 --- a/src/binding/database/database.cpp +++ b/src/binding/database/database.cpp @@ -948,7 +948,7 @@ struct RangeEstimate { * Overlapping versions of a key in multiple levels are counted once per * level, so the estimate skews high on heavily-overwritten ranges until * compaction; resolution is bounded by SST data-block granularity, so tiny - * ranges over-report. + * ranges can over-report or report zero for present keys. */ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const rocksdb::Slice& start, const rocksdb::Slice& end) { rocksdb::Range range(start, end); @@ -996,7 +996,7 @@ static RangeEstimate estimateRangeCount(rocksdb::DB* db, rocksdb::ColumnFamilyHa } if (entries <= deletions || fileBytes == 0) { // A nonzero byte estimate without density leaves the SST portion unknown. - result.degraded = sstBytes != 0; + result.degraded = result.degraded || sstBytes != 0; return result; } if (sstBytes == 0) { diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index 99ba02adb..d90b6ed32 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -203,13 +203,13 @@ describe('CountEstimator', () => { } expect(estimator.traversed).toBe(N / 2); - // with half the range traversed exactly, the estimate must be at - // least the traversed count, within a tighter overall bound, and - // more trusted than the untraversed estimate + // With half the range traversed exactly, the estimate must be at + // least the traversed count, within a tighter overall bound, and at + // least as confident as the exact fraction of the estimate. const refined = estimator.estimate(); expect(refined.count).toBeGreaterThanOrEqual(N / 2); expectWithin(refined.count, N, 1.6); - expectConfidence(refined.confidence); + expectConfidence(refined.confidence, estimator.traversed / refined.count, 1); })); it('should support reverse iteration', () => From 9cb039507e538ab5547b9e66666e3eab96c2ddc4 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 18:22:53 -0600 Subject: [PATCH 11/15] Strengthen tiny-range confidence coverage Co-Authored-By: GPT-5 Codex --- test/estimate-count.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index d90b6ed32..3decd3588 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -40,9 +40,8 @@ describe('estimateCount', () => { end: KEY(N / 2), inclusiveEnd: true, }); - if (singleKey.count === 0) { - expect(singleKey.confidence).toBeLessThanOrEqual(0.1); - } + expect(singleKey.count).toBe(0); + expect(singleKey.confidence).toBeLessThanOrEqual(0.1); // half range [25%, 75%) const half = db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); From 106ee40d26dba41442b163a116ffa76ffd1f24b3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 18:26:55 -0600 Subject: [PATCH 12/15] Avoid block-layout coupling in confidence test Co-Authored-By: GPT-5 Codex --- test/estimate-count.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index 3decd3588..22485af07 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -40,8 +40,7 @@ describe('estimateCount', () => { end: KEY(N / 2), inclusiveEnd: true, }); - expect(singleKey.count).toBe(0); - expect(singleKey.confidence).toBeLessThanOrEqual(0.1); + expect(singleKey.count > 0 || singleKey.confidence <= 0.1).toBe(true); // half range [25%, 75%) const half = db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); From e9e080a027195cf8db97ba1b37ac0ca90f2acee2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 18:32:02 -0600 Subject: [PATCH 13/15] Align range estimate API documentation Co-Authored-By: GPT-5 Codex --- src/database.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/database.ts b/src/database.ts index f4eee847d..e5a536560 100644 --- a/src/database.ts +++ b/src/database.ts @@ -485,10 +485,11 @@ export class RocksDatabase extends DBI { * SSTs overlapping the range rather than the number of keys — though * reading table properties for cold files can do I/O through the table * cache, and a start-only range does the work of its complement below - * `start`. Accuracy improves with range size — resolution is bounded by - * SST data-block granularity, so tiny ranges over-report — and recently - * deleted or overwritten entries may be counted until compaction. An - * inverted range (`start` >= `end`) returns `{ count: 0, confidence: 1 }`. + * `start`. Accuracy improves with range size. Resolution is bounded by SST + * data-block granularity, so tiny ranges may over-report or report zero for + * present keys; low `confidence` is the signal. Recently deleted or + * overwritten entries may be counted until compaction. An inverted range + * (`start` >= `end`) returns `{ count: 0, confidence: 1 }`. * * `confidence` is a heuristic 0–1 trust indicator (1 only when exact) — * see `CountEstimate`. Estimates always reflect committed state; writes From 10f4aaa74a8adcffc41bdd22fcbd0fe6c7dc5fd2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 23:16:00 -0600 Subject: [PATCH 14/15] Address final range estimator review feedback Co-Authored-By: GPT-5 Codex --- src/count-estimator.ts | 5 +++-- test/estimate-count.test.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/count-estimator.ts b/src/count-estimator.ts index def0e9a82..29d04c4e7 100644 --- a/src/count-estimator.ts +++ b/src/count-estimator.ts @@ -95,8 +95,9 @@ export class CountEstimator { * Estimates the total number of entries in the full range: the exact * traversed count plus a calibrated statistical estimate of the remainder. * `confidence` is the exactness-weighted blend of the traversed portion - * (exact) and the remainder's statistical confidence, so it converges to 1 - * as traversal proceeds (and is exactly 1 after `finish()`). + * (exact) and the remainder's statistical confidence, so it approaches 1 + * as the exact portion grows (though a checkpoint can decrease when + * calibration makes a large correction), and is exactly 1 after `finish()`. */ estimate(): CountEstimate { if (this.#finished) { diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index 22485af07..d809eeb69 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -40,7 +40,7 @@ describe('estimateCount', () => { end: KEY(N / 2), inclusiveEnd: true, }); - expect(singleKey.count > 0 || singleKey.confidence <= 0.1).toBe(true); + expect(singleKey.count <= 2 || singleKey.confidence <= 0.5).toBe(true); // half range [25%, 75%) const half = db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) }); From 75562fc263719556c791f34990fecb4a03c78e35 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 23:21:01 -0600 Subject: [PATCH 15/15] Tighten tiny-range confidence regression Co-Authored-By: GPT-5 Codex --- test/estimate-count.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/estimate-count.test.ts b/test/estimate-count.test.ts index d809eeb69..6b331bf97 100644 --- a/test/estimate-count.test.ts +++ b/test/estimate-count.test.ts @@ -40,7 +40,7 @@ describe('estimateCount', () => { end: KEY(N / 2), inclusiveEnd: true, }); - expect(singleKey.count <= 2 || singleKey.confidence <= 0.5).toBe(true); + expect(singleKey.count === 1 || singleKey.confidence <= 0.5).toBe(true); // half range [25%, 75%) const half = db.estimateCount({ start: KEY(N / 4), end: KEY((3 * N) / 4) });