diff --git a/DESIGN.md b/DESIGN.md index ecc83e5cc0..ab8642afd8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1272,3 +1272,34 @@ half-written file into a valid-looking env-only config. Its three outcomes are d one matters: a usable read withdraws the file's give-up report and restores the budget; giving up restores the budget (the write that repairs the file can itself be read mid-write) but leaves the report standing, since it is shared with every other watcher of that file; closing is terminal. + +## Query-plan range estimation blends statistical estimates by confidence (`search.ts`) + +`estimateCondition` estimates range comparators (`starts_with`/`prefix`, the `between` family, +`lt`/`le`/`gt`/`ge`) via the store's `estimateCount({start, end, …}) → { count, confidence }` +(rocksdb-js ≥ 2.8.0) instead of flat table fractions, blended as +`round(confidence × count + (1 − confidence) × fraction-heuristic)` so a low-confidence estimate +degrades to the historical behavior rather than replacing it. Invariants that are easy to break: + +- **Capability is feature-detected per store** (`typeof store.estimateCount === 'function'`) + because LMDB-backed and custom index stores do not implement it. The result shape is validated + (`Number.isFinite(count)`, `0 ≤ confidence ≤ 1`) and the native call is try/caught, so a store + that answers differently — or one closing concurrently — degrades the plan to the fraction + heuristic instead of NaN-poisoning condition ordering. +- **The estimated range must be the executed range.** Construction mirrors `searchByIndex`'s + comparator switch; bounds longer than `MAX_SEARCH_KEY_LENGTH` fall back entirely because + execution truncates + filters (wider range than the estimable one). Two ways this has already + been got wrong: `lt`/`le` need `searchByIndex`'s `start: true` lower bound, or the estimate + counts the `[null, primaryKey]` entries an `indexNulls` index holds and execution skips (`true` + sorts above `null`) — measured at 21× inflation on an index that is 99% nulls, which is worse + than the flat heuristic it replaces; and `RocksIndexStore` must widen value-space bounds to + `[value, MAXIMUM_KEY]` composite bounds, because the base implementation's byte-successor + semantics exclude the wrong entries on composite `[value, primaryKey]` keys. `getRange` and + `estimateCount` therefore share one `translateIndexBounds` helper rather than two copies. +- **Negated conditions estimate `Infinity` at the root** (`estimateConditionForTable`), following + the filter-only convention (`contains`/`ends_with`): the negated flag always forces + `needFullScan`, so `estimated_count` here is execution-cost ordering, not result cardinality — + a narrow negated range must never look selective enough to become the driving condition. +- `estimatedEntryCount` reads `estimate-num-keys` (O(1)) rather than iterating; it skews high on + overwrite/delete-heavy data until compaction, which is acceptable for the relative-ordering and + explicitly-estimated consumers it feeds (and it is a divisor — keep the ≥1 floor). diff --git a/package-lock.json b/package-lock.json index b1f75a82ca..db96d3044a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@fastify/cors": "^11.2.0", "@fastify/static": "^9.1.3", "@harperfast/extended-iterable": "1.0.3", - "@harperfast/rocksdb-js": "2.7.1", + "@harperfast/rocksdb-js": "2.8.0", "@harperfast/skills": "^1.10.8", "@turf/area": "6.5.0", "@turf/boolean-contains": "6.5.0", @@ -61,7 +61,7 @@ "minimist": "1.2.8", "moment": "2.30.1", "mqtt-packet": "~9.0.1", - "msgpackr": "2.0.5", + "msgpackr": "2.0.6", "needle": "3.5.0", "node-forge": "^1.3.1", "node-stream-zip": "1.16.0", @@ -2495,13 +2495,13 @@ } }, "node_modules/@harperfast/rocksdb-js": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-2.7.1.tgz", - "integrity": "sha512-Fs+Ki/9ysu4w0oGl4NN+6LWlKlQtUpw0RUZlOewbAXStEzJ4087Dk1vHKM2YqnAqNYoZb0UOz2XeoM8YkelMww==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-2.8.0.tgz", + "integrity": "sha512-czswCG+1KCRMYe6XQcsh5u28IQ6EbIx9eXPQd0kmjiLbtoMjiszYE92756bZJSbDykbR2OJLOazMZg7+qfFSJA==", "license": "Apache-2.0", "dependencies": { "@harperfast/extended-iterable": "1.0.3", - "msgpackr": "2.0.5", + "msgpackr": "2.0.6", "ordered-binary": "1.6.1" }, "bin": { @@ -2511,20 +2511,20 @@ "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { - "@harperfast/rocksdb-js-darwin-arm64": "2.7.1", - "@harperfast/rocksdb-js-darwin-x64": "2.7.1", - "@harperfast/rocksdb-js-linux-arm64-glibc": "2.7.1", - "@harperfast/rocksdb-js-linux-arm64-musl": "2.7.1", - "@harperfast/rocksdb-js-linux-x64-glibc": "2.7.1", - "@harperfast/rocksdb-js-linux-x64-musl": "2.7.1", - "@harperfast/rocksdb-js-win32-arm64": "2.7.1", - "@harperfast/rocksdb-js-win32-x64": "2.7.1" + "@harperfast/rocksdb-js-darwin-arm64": "2.8.0", + "@harperfast/rocksdb-js-darwin-x64": "2.8.0", + "@harperfast/rocksdb-js-linux-arm64-glibc": "2.8.0", + "@harperfast/rocksdb-js-linux-arm64-musl": "2.8.0", + "@harperfast/rocksdb-js-linux-x64-glibc": "2.8.0", + "@harperfast/rocksdb-js-linux-x64-musl": "2.8.0", + "@harperfast/rocksdb-js-win32-arm64": "2.8.0", + "@harperfast/rocksdb-js-win32-x64": "2.8.0" } }, "node_modules/@harperfast/rocksdb-js-darwin-arm64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-2.7.1.tgz", - "integrity": "sha512-H0aEOziU6WFaVUjoRvsWVUoIEAZ6ylLRYTa4z4R7SRVUt2pzpjiUMW1mHBTmIRAue/Fwsfm8SCM95WQj5goFKA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-2.8.0.tgz", + "integrity": "sha512-v7cj3bGpRptZHXSbhVzYJ0k+jyMD0Z0kX0u8YNZH6Xq+ifZ6zBkBbyoSdvQ2waGLJXxBORylUxTgYMvTZWiQBA==", "cpu": [ "arm64" ], @@ -2538,9 +2538,9 @@ } }, "node_modules/@harperfast/rocksdb-js-darwin-x64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-2.7.1.tgz", - "integrity": "sha512-YDPVGmfFyg9sCu3C+uulJa68o4B06pJ8MDWSgSqm+5uNXh4QNWuMSE1x2wLZYOt1hxE7xptHXzhMFFuSqIcvtg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-2.8.0.tgz", + "integrity": "sha512-9mMPPvRhxjLorE7u1gN3n+InzT4XHXQkHYdNKE6qN4iwZy0vM8iHoQLwIiN9GpzUiueEqBEilAP/9mIOHhMgBw==", "cpu": [ "x64" ], @@ -2554,9 +2554,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-arm64-glibc": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-2.7.1.tgz", - "integrity": "sha512-8LoPDc4muFb4U09VMWC1BJZRA/CxYBer1eIRW6Td6EfB8lOiBgZlpQvmc44GilYgrH4/C1at8lMK+G24U2CDyA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-2.8.0.tgz", + "integrity": "sha512-H1nNimVbIB4/6Sxv5DrxmP1asHLbxTCHuZ2ZN/zudqPkp/bTTSs5RsczOH6JeKTEUxUsJDTXM4nGRcn6LwFaOg==", "cpu": [ "arm64" ], @@ -2573,9 +2573,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-arm64-musl": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-2.7.1.tgz", - "integrity": "sha512-vqeH0JM/FUbQfx1Q5n21aBCiLUukN/qMXKa5d3YUQt7hH1fFA4d9wQhnRqwV9VbIjxm6i3drYAy5HaFkU2TV1g==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-2.8.0.tgz", + "integrity": "sha512-EqG4lkYptNviIcPPaQjK+X4ufjqzSImIvVxP89GHFHrbOJkGgEwRpiya0gWfGu3vJ1iGL1XL1nsU/kEjeDOugg==", "cpu": [ "arm64" ], @@ -2592,9 +2592,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-x64-glibc": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-2.7.1.tgz", - "integrity": "sha512-2pbbjq36Ln+ln31a3oDf2ic5IQNvrXNP9ND6LM7lMYUjuG/7tV3tWK79x2AyATym6iq0Kmz5Sp0b8tnUL2nQNg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-2.8.0.tgz", + "integrity": "sha512-MZPKZ6gF8kCy/KXTav4tc7T2uq+iOunxDOYq7nifZkCCy+0+C9U7g9CZx0NErORRQRP29x3z0zEKkCQ/go5c6Q==", "cpu": [ "x64" ], @@ -2611,9 +2611,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-x64-musl": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-2.7.1.tgz", - "integrity": "sha512-hrk3BvNjjrWV0j0XjDZ80r3XIQdLnP2nH/WNy+KFU484XEkdXONmj0zH0hsOJVItTv3JShEECWOk6BXlR3osaw==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-2.8.0.tgz", + "integrity": "sha512-PnMn1f/UxGOZKxlayR82tzsFCGoaMu0mBGKMmP/r2rbxzX5T89T0c+aVhuhpgNrFlWXTdawlA10Oh/YjfT/2qQ==", "cpu": [ "x64" ], @@ -2630,9 +2630,9 @@ } }, "node_modules/@harperfast/rocksdb-js-win32-arm64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-2.7.1.tgz", - "integrity": "sha512-E4Y725WvIcJjfSidOJt/gee4ogKWaFxj8exX/D6Kw2+v5RxuVjMgyKyOIxXQ+nLd18XGXTKj6HtgYzWOJfbEGg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-2.8.0.tgz", + "integrity": "sha512-4khqmGbIebiCwY3FVIgcUXEmMapgWP8f97aNrbThPDCIkuJscB3yoAG4A8OJo+Jln/TdjJZOgy8e2ir9F926dA==", "cpu": [ "arm64" ], @@ -2646,9 +2646,9 @@ } }, "node_modules/@harperfast/rocksdb-js-win32-x64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-2.7.1.tgz", - "integrity": "sha512-7TcSJ3gXgknoccLe+Tc0nqAxOA1kt5FE30GFrrTs7MyZczdO1rioRiccqhgcRA4UvW8Xc29GwJPd9ehmK8yYHg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-2.8.0.tgz", + "integrity": "sha512-E5HpBAnRwotITq++534WCinMvYidkkPoaAD5uwBNHiQbNt6YYvNZllJAIh/bukXD+uO6DL6qUMmVkeBh3kWK5A==", "cpu": [ "x64" ], @@ -11557,9 +11557,9 @@ "license": "MIT" }, "node_modules/msgpackr": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.5.tgz", - "integrity": "sha512-cef05H/dSYpLpqp3sj/qyZh5vhUYCalnaLO7j1yOmpsR0y/XwLVtK7r5gn+U/F7CTEfMowcGhlUQJDLcLf7jcA==", + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.6.tgz", + "integrity": "sha512-plGul/tqjt9vqWFR9zyqyLZls6gb5KTLvnsRO2B9+TZ8tNiXiVI/CR8LiDWlAX4IUeVkQ9gsmzKA6voC2QOciw==", "license": "MIT", "optionalDependencies": { "msgpackr-extract": "^3.0.4" diff --git a/package.json b/package.json index f2e6dedab1..1f4e711481 100644 --- a/package.json +++ b/package.json @@ -179,7 +179,7 @@ "@fastify/cors": "^11.2.0", "@fastify/static": "^9.1.3", "@harperfast/extended-iterable": "1.0.3", - "@harperfast/rocksdb-js": "2.7.1", + "@harperfast/rocksdb-js": "2.8.0", "@harperfast/skills": "^1.10.8", "@turf/area": "6.5.0", "@turf/boolean-contains": "6.5.0", @@ -223,7 +223,7 @@ "minimist": "1.2.8", "moment": "2.30.1", "mqtt-packet": "~9.0.1", - "msgpackr": "2.0.5", + "msgpackr": "2.0.6", "needle": "3.5.0", "node-forge": "^1.3.1", "node-stream-zip": "1.16.0", diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index c223f84090..5696c4d862 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -1,4 +1,6 @@ import { + type CountEstimate, + type CountEstimateOptions, DBI, type StoreIteratorOptions, type StorePutOptions, @@ -14,6 +16,25 @@ declare module '@harperfast/rocksdb-js' { } } +/** + * Widen value-space bounds to `[value, MAXIMUM_KEY]` composite bounds. The base implementation's + * encoded-byte successor of the bare indexed value would exclude the wrong entries on composite + * `[indexedValue, primaryKey]` keys. + */ +function translateIndexBounds< + T extends { start?: any; end?: any; exclusiveStart?: boolean; inclusiveEnd?: boolean; reverse?: boolean }, +>(options: T): T { + let { start, end } = options; + const { exclusiveStart, inclusiveEnd, reverse } = options; + if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) { + start = [start, MAXIMUM_KEY]; + } + if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) { + end = [end, MAXIMUM_KEY]; + } + return { ...options, start, end }; +} + /** * A specialized RocksDB-based index store that maintains indexed references to primary keys. * This store uses composite keys consisting of indexed values and primary keys, enabling @@ -27,19 +48,19 @@ export class RocksIndexStore extends RocksDatabase { * @param options */ getRange(options: StoreIteratorOptions): Iterable { - let { start, end, exclusiveStart, inclusiveEnd, reverse } = options; - if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) { - start = [start, MAXIMUM_KEY]; - } - if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) { - end = [end, MAXIMUM_KEY]; - } - const translatedOptions = { ...options, start, end }; - return super.getRange(translatedOptions).map(({ key }) => { + return super.getRange(translateIndexBounds(options)).map(({ key }) => { return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] }; }); } + /** + * Estimate the entry count of a range of indexed values. Shares `getRange`'s bound translation + * so a planner estimate always covers exactly the range execution would iterate. + */ + estimateCount(options?: CountEstimateOptions): CountEstimate { + return super.estimateCount(translateIndexBounds(options ?? {})); + } + /** * Translate a put with indexed value and primary key to an underlying put * @param indexedValue - ignored, only used by LMDB diff --git a/resources/search.ts b/resources/search.ts index c1f7dcafdf..52576db747 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -1112,6 +1112,91 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar } } +/** + * Estimates a range comparator's entry count from the store's statistical estimate, blended with + * the table-fraction heuristic by the estimate's confidence so low-confidence estimates degrade + * to the old behavior. Returns undefined (caller falls back to the heuristic) when the store + * cannot estimate, the shape is unexpected, or the executed range wouldn't match this one. + */ +function estimateRangeCondition(table, condition, searchType, fraction) { + const attributeName = condition[0] ?? condition.attribute; + const isPrimaryKey = attributeName === table.primaryKey; + const store = isPrimaryKey ? table.primaryStore : table.indices[attributeName]; + // LMDB-backed and custom index stores do not implement estimateCount + if (typeof store?.estimateCount !== 'function') return undefined; + let value = condition[1] ?? condition.value; + if (value instanceof Date) value = value.getTime(); + let range; + switch (searchType) { + case 'lt': + // `start: true` mirrors searchByIndex: `true` sorts above `null`, so an indexNulls + // index's `[null, primaryKey]` entries are outside the executed range + range = { start: true, end: value }; + break; + case 'le': + range = { start: true, end: value, inclusiveEnd: true }; + break; + case 'gt': + range = { start: value, exclusiveStart: true }; + break; + case 'ge': + range = { start: value }; + break; + case 'between': + case 'gele': + case 'gelt': + case 'gtlt': + case 'gtle': { + if (!Array.isArray(value)) return undefined; + let [start, end] = value; + if (start instanceof Date) start = start.getTime(); + if (end instanceof Date) end = end.getTime(); + range = { + start, + end, + inclusiveEnd: searchType === 'between' || searchType === 'gele' || searchType === 'gtle', + exclusiveStart: searchType === 'gtlt' || searchType === 'gtle', + }; + break; + } + case 'starts_with': { + const prefix = value?.toString(); + if (!prefix) return undefined; + range = { start: prefix, end: getStringPrefixUpperBound(prefix) }; + break; + } + case 'prefix': { + let start = Array.isArray(value) ? value : [value, null]; + if (start[start.length - 1] != null) start = start.concat(null); + const end = start.slice(0); + end[end.length - 1] = MAXIMUM_KEY; + range = { start, end }; + break; + } + default: + return undefined; + } + // Long string bounds get truncated + filtered at execution (searchByIndex), so the + // executed range is wider than this one; don't estimate what won't be iterated. + if ( + (typeof range.start === 'string' && range.start.length > MAX_SEARCH_KEY_LENGTH) || + (typeof range.end === 'string' && range.end.length > MAX_SEARCH_KEY_LENGTH) + ) { + return undefined; + } + let count, confidence; + try { + ({ count, confidence } = store.estimateCount(range) ?? {}); + } catch { + // a concurrently closing/dropped store must degrade the plan, not fail the query + return undefined; + } + if (!Number.isFinite(count) || count < 0 || !Number.isFinite(confidence) || confidence < 0 || confidence > 1) + return undefined; + const heuristic = fraction * estimatedEntryCount(table.primaryStore) + 1; + return Math.max(1, Math.round(confidence * count + (1 - confidence) * heuristic)); +} + export function estimateCondition(table) { function estimateConditionForTable(condition) { if (condition.estimated_count === undefined) { @@ -1141,7 +1226,14 @@ export function estimateCondition(table) { // skip if it is cached let searchType = condition.comparator || condition.search_type; searchType = ALTERNATE_COMPARATOR_NAMES[searchType] || searchType; - if (searchType === SEARCH_TYPES.EQUALS || !searchType) { + if (condition.negated) { + // a negated condition always executes as a full scan (searchByIndex forces + // needFullScan), so follow the filter-only convention used by contains/ends_with: + // estimate Infinity so its positive-range estimate can never win the driving-condition + // ordering. Short-circuit here rather than computing (and discarding) a positive-range + // estimate that would cross the native FFI boundary for nothing. + condition.estimated_count = Infinity; + } else if (searchType === SEARCH_TYPES.EQUALS || !searchType) { const attribute_name = condition[0] ?? condition.attribute; if (attribute_name == null || attribute_name === table.primaryKey) condition.estimated_count = 1; else if (Array.isArray(attribute_name) && attribute_name.length > 1) { @@ -1190,11 +1282,14 @@ export function estimateCondition(table) { } else if (Array.isArray(condition.value)) { condition.estimated_count = Infinity; } else condition.estimated_count = Infinity; - // for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess } else if (searchType === 'starts_with' || searchType === 'prefix') - condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; + condition.estimated_count = + estimateRangeCondition(table, condition, searchType, STARTS_WITH_ESTIMATE) ?? + STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; else if (searchType === 'between') - condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; + condition.estimated_count = + estimateRangeCondition(table, condition, searchType, BETWEEN_ESTIMATE) ?? + BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; else if (searchType === 'sort') { const attribute_name = condition[0] ?? condition.attribute; const index = table.indices[attribute_name]; @@ -1209,7 +1304,10 @@ export function estimateCondition(table) { if (index?.customIndex?.estimateCount) // allow custom index to define its own estimation of counts condition.estimated_count = index.customIndex.estimateCount(condition.value); - else condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; + else + condition.estimated_count = + estimateRangeCondition(table, condition, searchType, OPEN_RANGE_ESTIMATE) ?? + OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; } // we give a condition significantly more weight/preference if we will be ordering by it if (typeof condition.descending === 'boolean') condition.estimated_count /= 2; @@ -1622,13 +1720,15 @@ export function flattenKey(key) { function estimatedEntryCount(store) { const now = Date.now(); if ((store.estimatedEntryCountExpires || 0) < now) { - // use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own - store.estimatedEntryCount = store instanceof RocksDatabase ? store.getKeysCount() : store.getStats().entryCount; + // getStats is the LMDB fast path; for RocksDB, estimate-num-keys is O(1) where an exact + // getKeysCount() would iterate the entire store + store.estimatedEntryCount = + store instanceof RocksDatabase ? store.getEstimatedKeyCount() : store.getStats().entryCount; store.estimatedEntryCountExpires = now + 10000; } return store.estimatedEntryCount; } export function intersectionEstimate(store, left, right) { - return (left * right) / estimatedEntryCount(store); + return (left * right) / Math.max(estimatedEntryCount(store), 1); } diff --git a/unitTests/resources/estimateRangeCondition.test.js b/unitTests/resources/estimateRangeCondition.test.js new file mode 100644 index 0000000000..54f453ebe5 --- /dev/null +++ b/unitTests/resources/estimateRangeCondition.test.js @@ -0,0 +1,337 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { estimateCondition } = require('#src/resources/search'); +const { MAXIMUM_KEY } = require('ordered-binary'); + +const ENTRY_COUNT = 1000; + +function makeStore(estimate) { + return { + calls: [], + getStats: () => ({ entryCount: ENTRY_COUNT }), + estimateCount(range) { + this.calls.push(range); + return estimate; + }, + }; +} + +function makeTable({ indexEstimate, primaryEstimate } = {}) { + const primaryStore = { + getStats: () => ({ entryCount: ENTRY_COUNT }), + }; + if (primaryEstimate) { + primaryStore.calls = []; + primaryStore.estimateCount = function (range) { + this.calls.push(range); + return primaryEstimate; + }; + } + const index = + indexEstimate === undefined ? { getStats: () => ({ entryCount: ENTRY_COUNT }) } : makeStore(indexEstimate); + return { + primaryKey: 'id', + primaryStore, + indices: { attr: index }, + attributes: [], + }; +} + +function estimate(table, condition) { + return estimateCondition(table)(condition); +} + +describe('estimateCondition range estimates', () => { + it('uses the statistical estimate outright at confidence 1', () => { + const table = makeTable({ indexEstimate: { count: 120, confidence: 1 } }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }); + assert.strictEqual(estimated, 120); + assert.deepStrictEqual(table.indices.attr.calls[0], { + start: 5, + end: 10, + inclusiveEnd: true, + exclusiveStart: false, + }); + }); + + it('falls back to the fraction heuristic at confidence 0', () => { + const table = makeTable({ indexEstimate: { count: 120, confidence: 0 } }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }); + assert.strictEqual(estimated, 0.1 * ENTRY_COUNT + 1); + }); + + it('blends estimate and heuristic by confidence', () => { + const table = makeTable({ indexEstimate: { count: 120, confidence: 0.5 } }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }); + assert.strictEqual(estimated, Math.round(0.5 * 120 + 0.5 * (0.1 * ENTRY_COUNT + 1))); + }); + + it('constructs a prefix upper bound for starts_with', () => { + const table = makeTable({ indexEstimate: { count: 7, confidence: 1 } }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'starts_with', value: 'ab' }); + assert.strictEqual(estimated, 7); + const range = table.indices.attr.calls[0]; + assert.strictEqual(range.start, 'ab'); + assert.ok(range.end instanceof Uint8Array); + }); + + it('constructs composite bounds for prefix', () => { + const table = makeTable({ indexEstimate: { count: 9, confidence: 1 } }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'prefix', value: 'a' }); + assert.strictEqual(estimated, 9); + assert.deepStrictEqual(table.indices.attr.calls[0], { start: ['a', null], end: ['a', MAXIMUM_KEY] }); + }); + + it('constructs open ranges for gt/lt', () => { + const table = makeTable({ indexEstimate: { count: 40, confidence: 1 } }); + assert.strictEqual(estimate(table, { attribute: 'attr', comparator: 'gt', value: 5 }), 40); + assert.deepStrictEqual(table.indices.attr.calls[0], { start: 5, exclusiveStart: true }); + + // searchByIndex bounds lt/le at `start: true` to skip an indexNulls index's [null, id] entries + const table2 = makeTable({ indexEstimate: { count: 40, confidence: 1 } }); + assert.strictEqual(estimate(table2, { attribute: 'attr', comparator: 'lt', value: 5 }), 40); + assert.deepStrictEqual(table2.indices.attr.calls[0], { start: true, end: 5 }); + + const table3 = makeTable({ indexEstimate: { count: 40, confidence: 1 } }); + assert.strictEqual(estimate(table3, { attribute: 'attr', comparator: 'le', value: 5 }), 40); + assert.deepStrictEqual(table3.indices.attr.calls[0], { start: true, end: 5, inclusiveEnd: true }); + }); + + it('estimates primary-key ranges against the primary store', () => { + const table = makeTable({ primaryEstimate: { count: 33, confidence: 1 } }); + const estimated = estimate(table, { attribute: 'id', comparator: 'ge', value: 100 }); + assert.strictEqual(estimated, 33); + assert.deepStrictEqual(table.primaryStore.calls[0], { start: 100 }); + }); + + it('keeps the fraction heuristics when the store cannot estimate', () => { + const table = makeTable(); + assert.strictEqual( + estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }), + 0.1 * ENTRY_COUNT + 1 + ); + assert.strictEqual( + estimate(table, { attribute: 'attr', comparator: 'starts_with', value: 'ab' }), + 0.05 * ENTRY_COUNT + 1 + ); + assert.strictEqual(estimate(table, { attribute: 'attr', comparator: 'gt', value: 5 }), 0.3 * ENTRY_COUNT + 1); + }); + + it('never estimates below 1', () => { + const table = makeTable({ indexEstimate: { count: 0, confidence: 1 } }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }); + assert.strictEqual(estimated, 1); + }); + + it('estimates negated conditions as Infinity (they always full-scan)', () => { + // a narrow negated range must not look highly selective — the full-scan + // convention (contains/ends_with) keeps it out of the driving-condition slot + const table = makeTable({ indexEstimate: { count: 10, confidence: 1 } }); + const estimated = estimate(table, { + attribute: 'attr', + comparator: 'between', + value: [5, 10], + negated: true, + }); + assert.strictEqual(estimated, Infinity); + + // pre-existing defect: negated equals estimated its (possibly tiny) positive count + const eqTable = makeTable(); + eqTable.indices.attr.getValuesCount = () => 3; + const negatedEquals = estimate(eqTable, { + attribute: 'attr', + comparator: 'equals', + value: 'x', + negated: true, + }); + assert.strictEqual(negatedEquals, Infinity); + }); + + it('falls back when the estimate shape is unexpected', () => { + for (const bad of [ + 42, + { count: NaN, confidence: 1 }, + { count: -1, confidence: 1 }, + { count: 10 }, + { count: 10, confidence: 2 }, + { count: 10, confidence: NaN }, + { count: 10, confidence: -0.1 }, + null, + ]) { + const table = makeTable({ indexEstimate: bad }); + const estimated = estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }); + assert.strictEqual(estimated, 0.1 * ENTRY_COUNT + 1, `shape ${JSON.stringify(bad)} must fall back`); + } + }); + + it('falls back when estimateCount throws', () => { + const table = makeTable({ indexEstimate: { count: 1, confidence: 1 } }); + table.indices.attr.estimateCount = () => { + throw new Error('store closed'); + }; + const estimated = estimate(table, { attribute: 'attr', comparator: 'between', value: [5, 10] }); + assert.strictEqual(estimated, 0.1 * ENTRY_COUNT + 1); + }); + + it('falls back for over-length string bounds (executed range is truncated + filtered)', () => { + const table = makeTable({ indexEstimate: { count: 1, confidence: 1 } }); + const estimated = estimate(table, { + attribute: 'attr', + comparator: 'starts_with', + value: 'x'.repeat(2000), + }); + assert.strictEqual(estimated, 0.05 * ENTRY_COUNT + 1); + assert.strictEqual(table.indices.attr.calls.length, 0); + }); +}); + +describe('RocksIndexStore.estimateCount composite bound translation', () => { + const { RocksIndexStore } = require('#src/resources/RocksIndexStore'); + const { RocksDatabase } = require('@harperfast/rocksdb-js'); + + // Both methods must translate bounds identically or the planner estimates a range execution + // never iterates. + function captureBounds(method, options) { + const original = RocksDatabase.prototype[method]; + let captured; + RocksDatabase.prototype[method] = function (received) { + captured = received; + return method === 'estimateCount' ? { count: 0, confidence: 1 } : []; + }; + try { + RocksIndexStore.prototype[method].call(Object.create(RocksIndexStore.prototype), options); + } finally { + RocksDatabase.prototype[method] = original; + } + return captured; + } + + const cases = [ + ['bare bounds pass through', { start: 5, end: 10 }], + ['exclusiveStart widens the lower bound', { start: 5, end: 10, exclusiveStart: true }], + ['inclusiveEnd widens the upper bound', { start: 5, end: 10, inclusiveEnd: true }], + ['reverse flips which bound widens', { start: 10, end: 5, reverse: true }], + ['reverse with explicit flags', { start: 10, end: 5, reverse: true, exclusiveStart: true, inclusiveEnd: true }], + ]; + + for (const [name, options] of cases) { + it(`${name} the same way getRange does`, () => { + const estimated = captureBounds('estimateCount', options); + const iterated = captureBounds('getRange', options); + assert.deepStrictEqual( + { start: estimated.start, end: estimated.end }, + { start: iterated.start, end: iterated.end } + ); + }); + } + + it('widens to [value, MAXIMUM_KEY] rather than the bare indexed value', () => { + const { start, end } = captureBounds('estimateCount', { + start: 5, + end: 10, + exclusiveStart: true, + inclusiveEnd: true, + }); + assert.deepStrictEqual(start, [5, MAXIMUM_KEY]); + assert.deepStrictEqual(end, [10, MAXIMUM_KEY]); + }); +}); + +describe('estimateCondition range estimates (real stores)', () => { + const { setupTestDBPath } = require('../testUtils'); + const { table } = require('#src/resources/databases'); + const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + const N = 20000; + const SPARSE_COUNT = 200; + let T; + + before(async function () { + this.timeout(120000); + setupTestDBPath(); + setMainIsWorker(true); + T = table({ + table: 'EstimateTest', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'score', type: 'Int', indexed: true }, + { name: 'name', indexed: true }, + { name: 'sparse', type: 'Int', indexed: true }, + ], + }); + // HARPER_STORAGE_ENGINE=lmdb runs this same suite against index stores with no + // estimateCount, where every range falls back to the flat heuristic + if (typeof T.indices.score.estimateCount !== 'function') return this.skip(); + let last; + for (let i = 0; i < N; i++) { + last = T.put({ + id: i, + score: i, + name: `name-${String(i).padStart(6, '0')}`, + // explicit nulls are indexed as [null, primaryKey]; a missing attribute is not + sparse: i < SPARSE_COUNT ? i : null, + }); + } + await last; + await T.primaryStore.flush(); + await T.indices.score.flush(); + await T.indices.name.flush(); + await T.indices.sparse.flush(); + }); + + it('scales between estimates with the real range width', () => { + const est = estimateCondition(T); + const narrow = est({ attribute: 'score', comparator: 'between', value: [1000, 1100] }); + const wide = est({ attribute: 'score', comparator: 'between', value: [1000, 11000] }); + assert.ok(narrow < wide, `narrow (${narrow}) should be < wide (${wide})`); + assert.ok(wide > 0.15 * N, `wide (${wide}) should exceed the flat between heuristic`); + }); + + it('estimates starts_with from the real prefix range', () => { + const est = estimateCondition(T); + const broad = est({ attribute: 'name', comparator: 'starts_with', value: 'name-0' }); + const narrow = est({ attribute: 'name', comparator: 'starts_with', value: 'name-000' }); + assert.ok(narrow < broad, `narrow (${narrow}) should be < broad (${broad})`); + }); + + it('orders open-range estimates by real range width', () => { + // absolute accuracy at this scale is block-granular (tiny index entries, few data + // blocks), so assert ordering, which is what condition planning consumes + const est = estimateCondition(T); + const tail10 = est({ attribute: 'score', comparator: 'gt', value: N - 2000 }); + const tail50 = est({ attribute: 'score', comparator: 'gt', value: N / 2 }); + assert.ok(tail10 < tail50, `10% tail (${tail10}) should be < 50% tail (${tail50})`); + assert.ok(tail10 <= 0.3 * N + 1, `10% tail (${tail10}) should not exceed the flat heuristic`); + }); + + // The estimate is only worth anything if it tracks what the query returns; the assertions above + // compare estimates to each other, this one compares them to reality. + for (const [name, condition] of [ + ['secondary index between', { attribute: 'score', comparator: 'between', value: [1000, 3000] }], + ['secondary index lt', { attribute: 'score', comparator: 'lt', value: 4000 }], + ['primary key lt', { attribute: 'id', comparator: 'lt', value: 1000 }], + ]) { + it(`estimates ${name} within an order of magnitude of the executed result count`, async () => { + const estimated = estimateCondition(T)(condition); + let actual = 0; + for await (const _ of T.search({ conditions: [condition], select: ['id'] })) actual++; + assert.ok(actual > 0, `precondition: ${name} must match rows, got ${actual}`); + assert.ok( + estimated > actual / 10 && estimated < actual * 10, + `${name}: estimated ${estimated} is not within 10x of the executed ${actual}` + ); + }); + } + + it('excludes null index entries from lt/le, as execution does', () => { + // `sparse` is set on 200 of 20000 rows, so the index holds 19800 [null, id] entries that + // searchByIndex's `start: true` skips. An unbounded lower bound counts them all and makes + // a 200-row condition look like a 20000-row one — worse than the heuristic it replaces. + const est = estimateCondition(T); + const belowAll = est({ attribute: 'sparse', comparator: 'lt', value: SPARSE_COUNT }); + assert.ok( + belowAll < N / 2, + `lt over all ${SPARSE_COUNT} non-null values (${belowAll}) must not count the ${N - SPARSE_COUNT} null entries` + ); + }); +});