Use storage-level statistical range estimates in the query planner

DESIGN.md

@@ -1272,3 +1272,34 @@ half-written file into a valid-looking env-only config. Its three outcomes are d
 one matters: a usable read withdraws the file's give-up report and restores the budget; giving up
 restores the budget (the write that repairs the file can itself be read mid-write) but leaves the
 report standing, since it is shared with every other watcher of that file; closing is terminal.
+
+## Query-plan range estimation blends statistical estimates by confidence (`search.ts`)
+
+`estimateCondition` estimates range comparators (`starts_with`/`prefix`, the `between` family,
+`lt`/`le`/`gt`/`ge`) via the store's `estimateCount({start, end, …}) → { count, confidence }`
+(rocksdb-js ≥ 2.8.0) instead of flat table fractions, blended as
+`round(confidence × count + (1 − confidence) × fraction-heuristic)` so a low-confidence estimate
+degrades to the historical behavior rather than replacing it. Invariants that are easy to break:
+
+- **Capability is feature-detected per store** (`typeof store.estimateCount === 'function'`)
+  because LMDB-backed and custom index stores do not implement it. The result shape is validated
+  (`Number.isFinite(count)`, `0 ≤ confidence ≤ 1`) and the native call is try/caught, so a store
+  that answers differently — or one closing concurrently — degrades the plan to the fraction
+  heuristic instead of NaN-poisoning condition ordering.
+- **The estimated range must be the executed range.** Construction mirrors `searchByIndex`'s
+  comparator switch; bounds longer than `MAX_SEARCH_KEY_LENGTH` fall back entirely because
+  execution truncates + filters (wider range than the estimable one). Two ways this has already
+  been got wrong: `lt`/`le` need `searchByIndex`'s `start: true` lower bound, or the estimate
+  counts the `[null, primaryKey]` entries an `indexNulls` index holds and execution skips (`true`
+  sorts above `null`) — measured at 21× inflation on an index that is 99% nulls, which is worse
+  than the flat heuristic it replaces; and `RocksIndexStore` must widen value-space bounds to
+  `[value, MAXIMUM_KEY]` composite bounds, because the base implementation's byte-successor
+  semantics exclude the wrong entries on composite `[value, primaryKey]` keys. `getRange` and
+  `estimateCount` therefore share one `translateIndexBounds` helper rather than two copies.
+- **Negated conditions estimate `Infinity` at the root** (`estimateConditionForTable`), following
+  the filter-only convention (`contains`/`ends_with`): the negated flag always forces
+  `needFullScan`, so `estimated_count` here is execution-cost ordering, not result cardinality —
+  a narrow negated range must never look selective enough to become the driving condition.
+- `estimatedEntryCount` reads `estimate-num-keys` (O(1)) rather than iterating; it skews high on
+  overwrite/delete-heavy data until compaction, which is acceptable for the relative-ordering and
+  explicitly-estimated consumers it feeds (and it is a divisor — keep the ≥1 floor).

package.json

@@ -179,7 +179,7 @@
 		"@fastify/cors": "^11.2.0",
 		"@fastify/static": "^9.1.3",
 		"@harperfast/extended-iterable": "1.0.3",
-		"@harperfast/rocksdb-js": "2.7.1",
+		"@harperfast/rocksdb-js": "2.8.0",
 		"@harperfast/skills": "^1.10.8",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
@@ -223,7 +223,7 @@
 		"minimist": "1.2.8",
 		"moment": "2.30.1",
 		"mqtt-packet": "~9.0.1",
-		"msgpackr": "2.0.5",
+		"msgpackr": "2.0.6",
 		"needle": "3.5.0",
 		"node-forge": "^1.3.1",
 		"node-stream-zip": "1.16.0",

resources/RocksIndexStore.ts

@@ -1,4 +1,6 @@
 import {
+	type CountEstimate,
+	type CountEstimateOptions,
 	DBI,
 	type StoreIteratorOptions,
 	type StorePutOptions,
@@ -14,6 +16,25 @@ declare module '@harperfast/rocksdb-js' {
 	}
 }
 
+/**
+ * Widen value-space bounds to `[value, MAXIMUM_KEY]` composite bounds. The base implementation's
+ * encoded-byte successor of the bare indexed value would exclude the wrong entries on composite
+ * `[indexedValue, primaryKey]` keys.
+ */
+function translateIndexBounds<
+	T extends { start?: any; end?: any; exclusiveStart?: boolean; inclusiveEnd?: boolean; reverse?: boolean },
+>(options: T): T {
+	let { start, end } = options;
+	const { exclusiveStart, inclusiveEnd, reverse } = options;
+	if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) {
+		start = [start, MAXIMUM_KEY];
+	}
+	if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) {
+		end = [end, MAXIMUM_KEY];
+	}
+	return { ...options, start, end };
+}
+
 /**
  * A specialized RocksDB-based index store that maintains indexed references to primary keys.
  * This store uses composite keys consisting of indexed values and primary keys, enabling
@@ -27,19 +48,19 @@ export class RocksIndexStore extends RocksDatabase {
 	 * @param options
 	 */
 	getRange(options: StoreIteratorOptions): Iterable<any> {
-		let { start, end, exclusiveStart, inclusiveEnd, reverse } = options;
-		if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) {
-			start = [start, MAXIMUM_KEY];
-		}
-		if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) {
-			end = [end, MAXIMUM_KEY];
-		}
-		const translatedOptions = { ...options, start, end };
-		return super.getRange(translatedOptions).map(({ key }) => {
+		return super.getRange(translateIndexBounds(options)).map(({ key }) => {
 			return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] };
 		});
 	}
 
+	/**
+	 * Estimate the entry count of a range of indexed values. Shares `getRange`'s bound translation
+	 * so a planner estimate always covers exactly the range execution would iterate.
+	 */
+	estimateCount(options?: CountEstimateOptions): CountEstimate {
+		return super.estimateCount(translateIndexBounds(options ?? {}));
+	}
+
 	/**
 	 * Translate a put with indexed value and primary key to an underlying put
 	 * @param indexedValue - ignored, only used by LMDB