From eaa6cf353ae5c4c0391e77e9489e9bd9e8ddf276 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Mon, 10 Aug 2026 10:14:02 -0500 Subject: [PATCH 01/14] feat(rest): total-count pagination via `Prefer: count=` (Content-Range) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds opt-in total-record-count for REST collection queries so a client can paginate ("1–25 of 1,234") without a second round-trip or a custom resource. - `Prefer: count=exact` — Table.search drains the full matched set once, windowing the requested page in the same pass (O(matched) filter evals, O(limit) memory), bounded by MAX_EXACT_COUNT_SCAN so a page fetch can't turn into an unbounded scan. - `Prefer: count=estimated` — returns just the page plus a cheap planner/table estimate (estimateCondition / estimatedEntryCount, now exported), no full scan. - No default: without the header nothing is computed and no header is emitted. - REST emits `Content-Range: items -/` (200, not 206), `Range-Unit: items`, and `Preference-Applied: count=exact|estimated|none`, and adds them to `Access-Control-Expose-Headers` so browser (CORS) clients can read them. HEAD returns the headers with no body — a cheap "how many match?" pre-flight. Tests: resources-level unit (exact/estimated/window/filtered/default streaming) and REST integration (Content-Range/Range-Unit/Preference-Applied/CORS/HEAD/opt-in). Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/rest.test.mjs | 64 +++++++++++++++++ resources/RequestTarget.ts | 2 + resources/Table.ts | 54 +++++++++++++-- resources/search.ts | 2 +- server/REST.ts | 37 ++++++++++ unitTests/resources/queryCount.test.js | 91 +++++++++++++++++++++++++ 6 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 unitTests/resources/queryCount.test.js diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index a5e0e99d78..403547e9c0 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -244,4 +244,68 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { ) .expect(200); }); + + // `Prefer: count=` (pagination total-count) — emits Content-Range/Range-Unit/Preference-Applied. + test('[rest] count=exact emits an exact Content-Range', () => { + return client + .reqRest('/Related/?sort(id)&limit(2)') + .set('Prefer', 'count=exact') + .expect('Range-Unit', 'items') + .expect('Content-Range', 'items 0-1/5') + .expect('Preference-Applied', 'count=exact') + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect((r) => + assert.ok( + (r.headers['access-control-expose-headers'] || '').includes('Content-Range'), + `expected Content-Range to be exposed for CORS, got: ${r.headers['access-control-expose-headers']}` + ) + ) + .expect(200); + }); + + test('[rest] count=exact reflects the offset window but a total independent of it', () => { + return client + .reqRest('/Related/?sort(id)&limit(1,3)') // offset 1, 2 rows + .set('Prefer', 'count=exact') + .expect('Content-Range', 'items 1-2/5') + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + test('[rest] count=exact on a filtered query counts only matches', () => { + return client + .reqRest('/Related/?name==name-2&limit(10)') + .set('Prefer', 'count=exact') + .expect('Content-Range', 'items 0-0/1') + .expect('Preference-Applied', 'count=exact') + .expect(200); + }); + + test('[rest] count=estimated emits a numeric total flagged estimated', () => { + return client + .reqRest('/Related/?sort(id)&limit(2)') + .set('Prefer', 'count=estimated') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.match(r.headers['content-range'], /^items 0-1\/\d+$/, r.text)) + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + test('[rest] no Prefer header means no Content-Range (opt-in only)', () => { + return client + .reqRest('/Related/?sort(id)&limit(2)') + .expect((r) => assert.equal(r.headers['content-range'], undefined, r.text)) + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + test('[rest] HEAD with count=exact returns the count header and no body', () => { + return request(client.restURL) + .head('/Related/?sort(id)&limit(2)') + .set(client.headers) + .set('Prefer', 'count=exact') + .expect('Content-Range', 'items 0-1/5') + .expect((r) => assert.ok(!r.body || Object.keys(r.body).length === 0, r.text)) + .expect(200); + }); }); diff --git a/resources/RequestTarget.ts b/resources/RequestTarget.ts index ef9d3a145b..22a140fdfe 100644 --- a/resources/RequestTarget.ts +++ b/resources/RequestTarget.ts @@ -36,6 +36,8 @@ export class RequestTarget extends URLSearchParams { declare select?: Select; /** Return an explanation of the query order */ declare explain?: boolean; + /** Request a total count of matching records for pagination (REST `Prefer: count=exact|estimated`). */ + declare count?: 'exact' | 'estimated'; /** Force the query to be executed in the order of conditions */ declare enforceExecutionOrder?: boolean; declare lazy?: boolean; diff --git a/resources/Table.ts b/resources/Table.ts index 08d11c78e3..9a7b14ef03 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -55,6 +55,7 @@ import { searchByIndex, findAttribute, estimateCondition, + estimatedEntryCount, flattenKey, COERCIBLE_OPERATORS, executeConditions, @@ -128,6 +129,9 @@ const EVICTION_BATCH_SIZE = 100; // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); +// Guardrail for `Prefer: count=exact`: cap how many rows an exact count scan visits before giving up +// (reporting an unknown total) so a paginated read can't turn into an unbounded full-table scan. +const MAX_EXACT_COUNT_SCAN = 1_000_000; envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; @@ -3405,6 +3409,7 @@ export function makeTable(options) { } } const select = target.select; + const hasUserConditions = conditions.length > 0; if (conditions.length === 0) { conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }]; } @@ -3487,12 +3492,51 @@ export function makeTable(options) { readTxn, transformToRecord ); + const offset = target.offset || 0; + const end = target.limit !== undefined ? offset + (target.limit as number) : undefined; + // `Prefer: count=` (REST pagination): materialize the requested page and attach a total record + // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once, + // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/ + // table estimate. Opt-in only — the default streaming path below is untouched. + if (target.count) { + const wantExact = target.count === 'exact'; + return (async () => { + const page: any = []; + let scanned = 0; + let exact = true; + for await (const record of results) { + if (scanned >= offset && (end === undefined || scanned < end)) page.push(record); + scanned++; + // `estimated` only needs the page; `exact` keeps counting the whole match set, bounded + // so a pathological table can't turn a page fetch into an unbounded scan. + if (!wantExact && end !== undefined && scanned >= end) break; + if (wantExact && scanned > MAX_EXACT_COUNT_SCAN) { + exact = false; + break; + } + } + txn.doneReadTxn(); + let total: number | null; + if (wantExact) { + total = exact ? scanned : null; + } else if (!hasUserConditions) { + total = estimatedEntryCount(primaryStore); + } else { + const est = estimateCondition(TableResource)({ + conditions, + operator: operator ? String(operator).toLowerCase() : 'and', + }); + total = isFinite(est) ? Math.round(est) : null; + } + page.recordCount = total; + page.recordCountExact = wantExact && exact; + page.selectApplied = true; + page.getColumns = getColumns; + return page; + })() as any; + } // apply any offset/limit after all the sorting and filtering - if (target.offset || target.limit !== undefined) - results = results.slice( - target.offset, - target.limit !== undefined ? (target.offset || 0) + target.limit : undefined - ); + if (target.offset || target.limit !== undefined) results = results.slice(offset, end); results.onDone = () => { results.onDone = null; // ensure that it isn't called twice txn.doneReadTxn(); diff --git a/resources/search.ts b/resources/search.ts index 5e240c8356..aad3216042 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -1614,7 +1614,7 @@ export function flattenKey(key) { return key; } -function estimatedEntryCount(store) { +export function estimatedEntryCount(store) { const now = Date.now(); if ((store.estimatedEntryCountExpires || 0) < now) { // use getStats for LMDB because it is fast path, otherwise RocksDB can handle fast path on its own diff --git a/server/REST.ts b/server/REST.ts index 49e0c7be68..394326592f 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -117,6 +117,24 @@ async function findInactiveComponent(url: string): Promise { } } +/** + * Emit RFC 7233-style pagination headers for a `Prefer: count=` request. Table.search returns the page + * with a `recordCount` (total matching records, or null when an exact scan hit its guardrail) and a + * `recordCountExact` flag. `Content-Range: items -/` lets a client paginate; + * `Preference-Applied` echoes whether the total is exact, estimated, or unavailable. All three are added + * to `Access-Control-Expose-Headers` so a browser can read them cross-origin (they aren't safelisted). + */ +function setCountHeaders(headers: Headers, offset: number, page: any) { + const total = page.recordCount; + const len = Array.isArray(page) ? page.length : 0; + const range = len > 0 ? `${offset}-${offset + len - 1}` : '*'; + const totalStr = typeof total === 'number' ? String(total) : '*'; + headers.set('Range-Unit', 'items'); + headers.set('Content-Range', `items ${range}/${totalStr}`); + headers.set('Preference-Applied', `count=${total == null ? 'none' : page.recordCountExact ? 'exact' : 'estimated'}`); + headers.set('Access-Control-Expose-Headers', 'Content-Range, Range-Unit, Preference-Applied'); +} + async function http(request: Request, nextHandler, resources: Resources, httpOptions: any) { const headersObject = request.headers.asObject; const isSse = headersObject.accept === 'text/event-stream'; @@ -165,6 +183,18 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt (target as any).async = true; resource = entry.Resource; + // Pagination total-count opt-in (no default): `Prefer: count=exact|estimated`. Table.search + // reads target.count to compute the total emitted as Content-Range below. + const prefer = headersObject['prefer']; + if (prefer) { + for (const pref of parseHeaderValue(prefer as any)) { + const mode = (pref?.value as string | undefined)?.toLowerCase(); + if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { + (target as any).count = mode; + break; + } + } + } } if ((resource as any)?.isCaching) { const cacheControl = headersObject['cache-control']; @@ -348,6 +378,13 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt } // TODO: Handle 201 Created if (responseData !== undefined) { + if ( + (target as any)?.count && + (method === 'GET' || method === 'HEAD') && + responseData.recordCount !== undefined + ) { + setCountHeaders(headers, (target as any).offset || 0, responseData); + } responseObject.body = serialize(responseData, request, responseObject); if (method === 'HEAD') responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body } diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js new file mode 100644 index 0000000000..3e0157301f --- /dev/null +++ b/unitTests/resources/queryCount.test.js @@ -0,0 +1,91 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +// Covers the `Prefer: count=` pagination support in Table.search: a `count` target returns the +// requested page (offset/limit window) as an array carrying `recordCount` (total matching records) +// and `recordCountExact`, instead of the default lazy streaming iterable. +describe('Table.search count (REST pagination total-count)', () => { + let CountTable; + const TOTAL = 20; + const GROUP_A = 12; // ids 0..11 + const GROUP_B = TOTAL - GROUP_A; // ids 12..19 + + before(async function () { + setupTestDBPath(); + setMainIsWorker(true); + CountTable = table({ + table: 'QueryCountTable', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'group', indexed: true }, + { name: 'name' }, + ], + }); + let last; + for (let i = 0; i < TOTAL; i++) { + last = CountTable.put({ id: i, group: i < GROUP_A ? 'a' : 'b', name: 'n-' + i }); + } + await last; + }); + + it('exact: whole-collection page carries the exact total', async function () { + const page = await CountTable.search({ limit: 5, offset: 0, count: 'exact' }); + assert.ok(Array.isArray(page)); + assert.strictEqual(page.length, 5); + assert.strictEqual(page.recordCount, TOTAL); + assert.strictEqual(page.recordCountExact, true); + }); + + it('exact: total is independent of the offset/limit window', async function () { + const nearEnd = await CountTable.search({ limit: 5, offset: 18, count: 'exact' }); + assert.strictEqual(nearEnd.length, 2); // only ids 18,19 remain + assert.strictEqual(nearEnd.recordCount, TOTAL); + + const pastEnd = await CountTable.search({ limit: 5, offset: 25, count: 'exact' }); + assert.strictEqual(pastEnd.length, 0); + assert.strictEqual(pastEnd.recordCount, TOTAL); + assert.strictEqual(pastEnd.recordCountExact, true); + }); + + it('exact: a filtered query counts only matching records', async function () { + const paged = await CountTable.search({ conditions: [{ attribute: 'group', value: 'a' }], limit: 5, count: 'exact' }); + assert.strictEqual(paged.length, 5); + assert.strictEqual(paged.recordCount, GROUP_A); + assert.strictEqual(paged.recordCountExact, true); + + // no limit: the page is the whole matched set and the count agrees with it + const all = await CountTable.search({ conditions: [{ attribute: 'group', value: 'b' }], count: 'exact' }); + assert.strictEqual(all.length, GROUP_B); + assert.strictEqual(all.recordCount, GROUP_B); + }); + + it('estimated: returns the page plus a positive estimate, flagged non-exact', async function () { + const page = await CountTable.search({ limit: 5, count: 'estimated' }); + assert.strictEqual(page.length, 5); + assert.strictEqual(typeof page.recordCount, 'number'); + assert.ok(page.recordCount > 0, `expected a positive estimate, got ${page.recordCount}`); + assert.strictEqual(page.recordCountExact, false); + + const filtered = await CountTable.search({ + conditions: [{ attribute: 'group', value: 'a' }], + limit: 3, + count: 'estimated', + }); + assert.strictEqual(filtered.length, 3); + assert.ok(filtered.recordCount > 0); + assert.strictEqual(filtered.recordCountExact, false); + }); + + it('default (no count): still returns the lazy streaming iterable, not a materialized page', async function () { + const results = CountTable.search({ limit: 5 }); + assert.ok(!Array.isArray(results), 'default search must not materialize an array'); + assert.strictEqual(results.recordCount, undefined); + let n = 0; + for await (const _ of results) n++; + assert.strictEqual(n, 5); + }); +}); From 1520757939d4ea1c8e85cf1249ae27d1380def09 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Mon, 10 Aug 2026 21:42:44 -0500 Subject: [PATCH 02/14] fix(rest): address code-review findings on count pagination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-model review (Codex) of the count feature surfaced several correctness, resource, and disclosure issues, all fixed here: - Read-txn leak: the count drain now releases the read transaction in a `finally`, so a throw mid-iteration (record load, rowFilter policy error) can't leak a pinned snapshot. - Guardrail no longer truncates the page: the requested [offset, end) window is always collected in full; the row cap only abandons the running total. Added a wall-clock budget (MAX_EXACT_COUNT_MS) alongside the row cap so an exact count of a large match set can't run unbounded — on exhaustion the total is reported unknown (Content-Range .../*), never a short page. - Estimated totals no longer corrupted by the planner's synthetic `sort` pseudo-condition: hasUserConditions now reads the raw request conditions, and the estimate drops `sort` pseudo-conditions. A clamp keeps a non-empty page's Content-Range valid when an estimate undershoots (exact totals stay authoritative). - Estimated totals return unknown (null -> .../*) when an opaque rowFilter/vectorFilter participates, instead of a misleading estimate that could disclose hidden cardinality. - Spurious headers: the REST gate now requires an array result, so a single-record GET whose record carries a `recordCount` attribute can't be mistaken for a count page. - CORS: Access-Control-Expose-Headers is appended (not overwritten), preserving a resource's own exposed headers. Adds regression tests for the sorted-estimate, filter-aware estimate, and filtered-exact paths. Resources unit 8 passing; REST integration 21 passing. Co-Authored-By: Claude Opus 4.8 (1M context) --- resources/Table.ts | 57 +++++++++++++++++++------- server/REST.ts | 14 ++++++- unitTests/resources/queryCount.test.js | 30 ++++++++++++++ 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 9a7b14ef03..7f4c42f151 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -129,9 +129,13 @@ const EVICTION_BATCH_SIZE = 100; // letting an unbounded number of open transactions (and their snapshots) accumulate. const MAX_INFLIGHT_EVICTION_BATCHES = 4; const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]); -// Guardrail for `Prefer: count=exact`: cap how many rows an exact count scan visits before giving up -// (reporting an unknown total) so a paginated read can't turn into an unbounded full-table scan. +// Guardrails for `Prefer: count=exact`: once the requested page has been collected, counting the rest +// of the match set is bounded by BOTH a row cap and a wall-clock budget, so a paginated read can't turn +// into an unbounded scan. Exceeding either reports an unknown total (Content-Range `.../*`) rather than +// truncating the page. These bound the count tail, not the page itself; a genuinely expensive query +// (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure. const MAX_EXACT_COUNT_SCAN = 1_000_000; +const MAX_EXACT_COUNT_MS = 1_000; envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; @@ -3409,7 +3413,10 @@ export function makeTable(options) { } } const select = target.select; - const hasUserConditions = conditions.length > 0; + // Whether the caller supplied real filter conditions — read from the raw request, NOT the + // planner-augmented `conditions` (which by now may carry a synthetic `sort` pseudo-condition and + // injected full-scan condition). Used to pick the count-estimate source below. + const hasUserConditions = Array.isArray(target.conditions) && target.conditions.length > 0; if (conditions.length === 0) { conditions = [{ attribute: primaryKey, comparator: 'greater_than', value: true }]; } @@ -3500,34 +3507,56 @@ export function makeTable(options) { // table estimate. Opt-in only — the default streaming path below is untouched. if (target.count) { const wantExact = target.count === 'exact'; + const countStart = performance.now(); return (async () => { const page: any = []; let scanned = 0; let exact = true; - for await (const record of results) { - if (scanned >= offset && (end === undefined || scanned < end)) page.push(record); - scanned++; - // `estimated` only needs the page; `exact` keeps counting the whole match set, bounded - // so a pathological table can't turn a page fetch into an unbounded scan. - if (!wantExact && end !== undefined && scanned >= end) break; - if (wantExact && scanned > MAX_EXACT_COUNT_SCAN) { - exact = false; - break; + try { + for await (const record of results) { + if (scanned >= offset && (end === undefined || scanned < end)) page.push(record); + scanned++; + // The page window [offset, end) is always collected in full first — the guardrail + // only ever abandons the running TOTAL, never truncates the page body. + if (end !== undefined && scanned >= end) { + if (!wantExact) break; // `estimated` needs nothing past the page + // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a + // large match set can't turn a bounded page fetch into an unbounded scan. + if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) { + exact = false; + break; + } + } } + } finally { + // We own the iteration here (no results.onDone consumer), so release the read + // transaction unconditionally — including when the drain throws — or the snapshot leaks. + txn.doneReadTxn(); } - txn.doneReadTxn(); let total: number | null; if (wantExact) { total = exact ? scanned : null; + } else if (boundRowFilter || typeof target.vectorFilter === 'function') { + // An opaque row/vector filter shapes the result but isn't reflected in the index/condition + // estimate; guessing would both mislead and disclose cardinality the filter hides. + total = null; } else if (!hasUserConditions) { total = estimatedEntryCount(primaryStore); } else { + // Estimate from the real conditions only — drop the planner's synthetic `sort` + // pseudo-condition, which otherwise contributes a bogus (entryCount/2) cardinality. const est = estimateCondition(TableResource)({ - conditions, + conditions: conditions.filter((c: any) => c.comparator !== 'sort'), operator: operator ? String(operator).toLowerCase() : 'and', }); total = isFinite(est) ? Math.round(est) : null; } + // For an estimate, never report a total below the last row actually returned — keeps the + // Content-Range valid (start-end/total) when an estimate undershoots a non-empty page. + // Exact totals are authoritative (and an empty page past the end must not be clamped up). + if (!wantExact && total != null && page.length > 0 && total < offset + page.length) { + total = offset + page.length; + } page.recordCount = total; page.recordCountExact = wantExact && exact; page.selectApplied = true; diff --git a/server/REST.ts b/server/REST.ts index 394326592f..21b23e5973 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -132,7 +132,13 @@ function setCountHeaders(headers: Headers, offset: number, page: any) { headers.set('Range-Unit', 'items'); headers.set('Content-Range', `items ${range}/${totalStr}`); headers.set('Preference-Applied', `count=${total == null ? 'none' : page.recordCountExact ? 'exact' : 'estimated'}`); - headers.set('Access-Control-Expose-Headers', 'Content-Range, Range-Unit, Preference-Applied'); + // Append (don't overwrite) so a resource that already exposed its own headers keeps them. + const exposed = headers.get('Access-Control-Expose-Headers'); + for (const name of ['Content-Range', 'Range-Unit', 'Preference-Applied']) { + if (!exposed || !String(exposed).toLowerCase().includes(name.toLowerCase())) { + headers.append('Access-Control-Expose-Headers', name, true); + } + } } async function http(request: Request, nextHandler, resources: Resources, httpOptions: any) { @@ -381,8 +387,12 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt if ( (target as any)?.count && (method === 'GET' || method === 'HEAD') && - responseData.recordCount !== undefined + Array.isArray(responseData) && + (responseData as any).recordCount !== undefined ) { + // Array.isArray guards the single-record path: a record that happens to carry a + // `recordCount` attribute must not be mistaken for a count page (Table.search only ever + // returns the count as an array). setCountHeaders(headers, (target as any).offset || 0, responseData); } responseObject.body = serialize(responseData, request, responseObject); diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index 3e0157301f..4fdb439f36 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -88,4 +88,34 @@ describe('Table.search count (REST pagination total-count)', () => { for await (const _ of results) n++; assert.strictEqual(n, 5); }); + + it('estimated: a sorted whole-collection estimate is not halved by the planner sort condition', async function () { + // Regression: the synthetic `sort` pseudo-condition used to flip hasUserConditions and feed + // estimateCondition, yielding ~entryCount/2 and impossible ranges (e.g. items 3-4/3). + const page = await CountTable.search({ sort: { attribute: 'id' }, offset: 3, limit: 3, count: 'estimated' }); + assert.strictEqual(page.length, 3); + assert.ok( + page.recordCount >= 3 + page.length, + `range must be valid: total ${page.recordCount} vs page end ${3 + page.length}` + ); + assert.ok( + page.recordCount >= TOTAL * 0.75, + `sorted estimate ${page.recordCount} should track table size ${TOTAL}, not half it` + ); + }); + + it('estimated: an opaque rowFilter yields an unknown total (null), not a misleading estimate', async function () { + const page = await CountTable.search({ rowFilter: (r) => r.group === 'a', limit: 3, count: 'estimated' }); + assert.ok(page.length <= 3); + assert.ok(page.every((r) => r.group === 'a'), 'page must honor the rowFilter'); + assert.strictEqual(page.recordCount, null); + assert.strictEqual(page.recordCountExact, false); + }); + + it('exact: honors a rowFilter in both the page and the count', async function () { + const page = await CountTable.search({ rowFilter: (r) => r.group === 'b', count: 'exact' }); + assert.strictEqual(page.length, GROUP_B); + assert.strictEqual(page.recordCount, GROUP_B); + assert.strictEqual(page.recordCountExact, true); + }); }); From 8d213792e827546e8f097efb3239bc18e21be231 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Tue, 11 Aug 2026 01:26:03 -0500 Subject: [PATCH 03/14] feat(rest): per-mount `exactCount` config gate for count=exact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an operator control for the expensive exact-count scan: `rest: { exactCount: false }` on a REST mount serves a `Prefer: count=exact` request as a cheap estimate instead (signaled back via `Preference-Applied: count=estimated`), rather than rejecting it. Default enabled. Read from httpOptions in the same per-mount way as the existing `includeExpensiveRecordCountEstimates` option. This is the operator-facing half of the DoS mitigation for exact counts: the in-code guardrails (row cap + time budget) bound a single request, and this lets a deployment turn exact counts off entirely on a sensitive/public mount. It is a per-REST-mount policy — components exporting at the shared root path share one mount's options. Integration: a dedicated suite (its own instance, since a gated component would otherwise share the root mount with the main suite) verifies count=exact downgrades to estimated while count=estimated is unchanged. 23 REST integration tests passing. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/rest.test.mjs | 72 +++++++++++++++++++++++++ server/REST.ts | 6 ++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index 403547e9c0..cb3f039863 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -68,6 +68,27 @@ const SUBOBJECT_ROWS = [ { id: '5', relatedId: '5', any: 'any-5' }, ]; +// Second component whose REST mount disables the expensive exact scan. +const SCHEMA_GATE_GRAPHQL = ` +type GatedWidget @table @export(rest: true, mqtt: false) { + id: ID @primaryKey + name: String @indexed +} +`; + +const CONFIG_GATE_YAML = `rest: + exactCount: false +graphqlSchema: + files: '*.graphql' +graphql: true +`; + +const GATE_ROWS = [ + { id: '1', name: 'w-1' }, + { id: '2', name: 'w-2' }, + { id: '3', name: 'w-3' }, +]; + const skipSuite = process.platform === 'win32'; suite('REST query syntax', { skip: skipSuite }, (ctx) => { @@ -308,4 +329,55 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { .expect((r) => assert.ok(!r.body || Object.keys(r.body).length === 0, r.text)) .expect(200); }); + +}); + +// exactCount is a per-REST-mount policy, so it needs its own instance: two components exporting at +// the root path share one mount (the handler dedupes), and the gated config would otherwise bleed +// onto the main suite's routes. +suite('REST count exactCount gate', { skip: skipSuite }, (ctx) => { + let client; + + before(async () => { + await startHarper(ctx, { config: {}, env: {} }); + client = createApiClient(ctx.harper); + + await installAppComponent(client, { + project: 'appCountGate', + files: { 'schema.graphql': SCHEMA_GATE_GRAPHQL, 'config.yaml': CONFIG_GATE_YAML }, + probePath: '/GatedWidget/', + restartTimeoutMs: 120000, + }); + + await client + .req() + .send({ operation: 'insert', table: 'GatedWidget', records: GATE_ROWS }) + .expect((r) => assert.ok(r.body.message.includes('inserted 3 of 3 records'), r.text)) + .expect(200); + }); + + after(async () => { + await teardownHarper(ctx); + }); + + // `rest: { exactCount: false }` serves count=exact as a cheap estimate instead. + test('[rest] exactCount:false downgrades count=exact to estimated', () => { + return client + .reqRest('/GatedWidget/?sort(id)&limit(2)') + .set('Prefer', 'count=exact') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.match(r.headers['content-range'], /^items 0-1\/\d+$/, r.text)) + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); + + // estimated still works normally on a gated mount. + test('[rest] exactCount:false leaves count=estimated unchanged', () => { + return client + .reqRest('/GatedWidget/?sort(id)&limit(2)') + .set('Prefer', 'count=estimated') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.equal(r.body.length, 2, r.text)) + .expect(200); + }); }); diff --git a/server/REST.ts b/server/REST.ts index 21b23e5973..696ed33eb8 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -196,7 +196,11 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt for (const pref of parseHeaderValue(prefer as any)) { const mode = (pref?.value as string | undefined)?.toLowerCase(); if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { - (target as any).count = mode; + // A mount can disable the expensive exact scan with `rest: { exactCount: false }` + // (default enabled); a count=exact request is then served as a cheap estimate, + // signaled back to the client via Preference-Applied. + (target as any).count = + mode === 'exact' && (httpOptions as any).exactCount === false ? 'estimated' : mode; break; } } From 8e502cf13e3c00b351ce38350c0b0db608668479 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Tue, 11 Aug 2026 02:08:30 -0500 Subject: [PATCH 04/14] test(rest): guard count-page Bytes against read-buffer aliasing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A code-review concern held that the count path releases its read transaction before the page is serialized, so a Bytes/Blob field decoded as a zero-copy view of the read buffer could be corrupted by later reads/writes. Verified it does NOT occur: the count drain reads every record eagerly while the txn is open and returns owned copies (Bytes come back as standalone Buffers, byteOffset 0), so releasing before serialize is safe — unlike the streaming path, which reads lazily during serialization and must hold the txn. This test churns writes/reads after an exact count and asserts the returned Bytes are unchanged, on both storage engines. Co-Authored-By: Claude Opus 4.8 (1M context) --- unitTests/resources/queryCountBytes.test.js | 69 +++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 unitTests/resources/queryCountBytes.test.js diff --git a/unitTests/resources/queryCountBytes.test.js b/unitTests/resources/queryCountBytes.test.js new file mode 100644 index 0000000000..8482d07320 --- /dev/null +++ b/unitTests/resources/queryCountBytes.test.js @@ -0,0 +1,69 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +// Verifies that a `Prefer: count=` page — which is materialized and returned AFTER Table.search +// releases its read transaction — does not hand back Bytes fields that alias the (now-released) read +// buffer. If decoded Bytes are zero-copy views into the read snapshot, churning reads/writes after the +// count would mutate the already-returned page. +describe('Table.search count with Bytes columns (read-buffer safety)', () => { + let BytesTable; + const N = 5; + const LEN = 64; + + before(async function () { + setupTestDBPath(); + setMainIsWorker(true); + BytesTable = table({ + table: 'BytesCountTable', + database: 'test', + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'data', type: 'Bytes' }], + }); + let last; + for (let i = 0; i < N; i++) { + last = BytesTable.put({ id: i, data: new Uint8Array(LEN).fill(i + 1) }); + } + await last; + }); + + it('exact count returns intact Bytes that survive post-release buffer churn', async function () { + const page = await BytesTable.search({ limit: N, count: 'exact' }); + assert.strictEqual(page.recordCount, N); + assert.strictEqual(page.length, N); + + // correctness immediately after the count released its read txn + for (const rec of page) { + assert.ok(rec.data && rec.data.length === LEN, `record ${rec.id} missing bytes`); + assert.ok( + [...rec.data].every((b) => b === rec.id + 1), + `record ${rec.id} bytes wrong right after count: ${[...rec.data.slice(0, 4)]}` + ); + } + + // Snapshot the returned bytes, then churn writes + reads to reuse read buffers. + const snapshots = page.map((r) => [...r.data]); + for (let r = 0; r < 60; r++) { + await BytesTable.put({ id: 100 + r, data: new Uint8Array(LEN).fill(150 + (r % 100)) }); + } + for (let round = 0; round < 5; round++) { + // eslint-disable-next-line no-unused-vars + for await (const _ of BytesTable.search({ limit: 500 })) { + } + } + + // The already-returned page must be unchanged — no aliasing of the released read buffer. + page.forEach((rec, i) => { + assert.deepStrictEqual( + [...rec.data], + snapshots[i], + `record ${rec.id} bytes changed after churn — count page aliased the released read buffer` + ); + assert.ok( + [...rec.data].every((b) => b === rec.id + 1), + `record ${rec.id} bytes corrupted after churn: ${[...rec.data.slice(0, 4)]}` + ); + }); + }); +}); From b3d2e58c029da25861c89946c5d0136a04d84f22 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Tue, 11 Aug 2026 02:14:03 -0500 Subject: [PATCH 05/14] fix(rest): echo requested count mode on unavailable totals; robust exactCount gate Addresses two review findings: - #2: Preference-Applied now echoes the count mode the server applied (exact|estimated, after any per-mount downgrade) instead of `count=none` when the total is unavailable. A `Content-Range: items x-y/*` now reads as "that mode was applied but the total is unavailable" (guardrail hit, or an estimate suppressed by an opaque filter / Infinity estimate) rather than "no count was requested". Added an integration case: a `ne` condition (Infinity estimate) yields items 0-.../* with count=estimated. - #4: the exactCount disable check also accepts the string "false", since not every config source coerces to a boolean. 24 REST integration tests passing. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/rest.test.mjs | 11 ++++++++++ server/REST.ts | 29 +++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index cb3f039863..33ba27a11c 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -312,6 +312,17 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { .expect(200); }); + test('[rest] an uncomputable total reports items .../* but still echoes the requested mode', () => { + // `name != x` estimates to Infinity, so the total is unavailable. The header must still say + // count=estimated (the mode applied), not count=none — the client asked, it just can't be given. + return client + .reqRest('/Related/?name!=name-2&limit(2)') + .set('Prefer', 'count=estimated') + .expect('Preference-Applied', 'count=estimated') + .expect((r) => assert.match(r.headers['content-range'], /^items 0-\d+\/\*$/, r.text)) + .expect(200); + }); + test('[rest] no Prefer header means no Content-Range (opt-in only)', () => { return client .reqRest('/Related/?sort(id)&limit(2)') diff --git a/server/REST.ts b/server/REST.ts index 696ed33eb8..bd5af1d094 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -119,19 +119,22 @@ async function findInactiveComponent(url: string): Promise { /** * Emit RFC 7233-style pagination headers for a `Prefer: count=` request. Table.search returns the page - * with a `recordCount` (total matching records, or null when an exact scan hit its guardrail) and a - * `recordCountExact` flag. `Content-Range: items -/` lets a client paginate; - * `Preference-Applied` echoes whether the total is exact, estimated, or unavailable. All three are added - * to `Access-Control-Expose-Headers` so a browser can read them cross-origin (they aren't safelisted). + * with a `recordCount` (total matching records, or null when an exact scan hit its guardrail or an + * estimate was suppressed by an opaque filter). `Content-Range: items -/` lets a + * client paginate — `` is `*` when unavailable. `Preference-Applied` echoes the count mode the + * server actually applied (`exact` or `estimated`, after any per-mount downgrade), so a `.../*` total + * reads as "that mode was applied but the total is unavailable" rather than "no count was requested". + * All three headers are added to `Access-Control-Expose-Headers` so a browser can read them cross-origin + * (they aren't safelisted). */ -function setCountHeaders(headers: Headers, offset: number, page: any) { +function setCountHeaders(headers: Headers, offset: number, mode: string, page: any) { const total = page.recordCount; const len = Array.isArray(page) ? page.length : 0; const range = len > 0 ? `${offset}-${offset + len - 1}` : '*'; const totalStr = typeof total === 'number' ? String(total) : '*'; headers.set('Range-Unit', 'items'); headers.set('Content-Range', `items ${range}/${totalStr}`); - headers.set('Preference-Applied', `count=${total == null ? 'none' : page.recordCountExact ? 'exact' : 'estimated'}`); + headers.set('Preference-Applied', `count=${mode}`); // Append (don't overwrite) so a resource that already exposed its own headers keeps them. const exposed = headers.get('Access-Control-Expose-Headers'); for (const name of ['Content-Range', 'Range-Unit', 'Preference-Applied']) { @@ -193,14 +196,16 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt // reads target.count to compute the total emitted as Content-Range below. const prefer = headersObject['prefer']; if (prefer) { + // A mount can disable the expensive exact scan with `rest: { exactCount: false }` (default + // enabled); a count=exact request is then served as a cheap estimate. Accept a string + // `"false"` too, since not every config source coerces to a boolean. + const exactDisabled = + (httpOptions as any).exactCount === false || (httpOptions as any).exactCount === 'false'; for (const pref of parseHeaderValue(prefer as any)) { const mode = (pref?.value as string | undefined)?.toLowerCase(); if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { - // A mount can disable the expensive exact scan with `rest: { exactCount: false }` - // (default enabled); a count=exact request is then served as a cheap estimate, - // signaled back to the client via Preference-Applied. - (target as any).count = - mode === 'exact' && (httpOptions as any).exactCount === false ? 'estimated' : mode; + // Downgrade is signaled back to the client via Preference-Applied. + (target as any).count = mode === 'exact' && exactDisabled ? 'estimated' : mode; break; } } @@ -397,7 +402,7 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt // Array.isArray guards the single-record path: a record that happens to carry a // `recordCount` attribute must not be mistaken for a count page (Table.search only ever // returns the count as an array). - setCountHeaders(headers, (target as any).offset || 0, responseData); + setCountHeaders(headers, (target as any).offset || 0, (target as any).count, responseData); } responseObject.body = serialize(responseData, request, responseObject); if (method === 'HEAD') responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body From 0a124651382a4ff92ecb7354d349fe0dcbf48edb Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Tue, 11 Aug 2026 18:26:55 -0500 Subject: [PATCH 06/14] Formatting --- integrationTests/apiTests/rest.test.mjs | 1 - server/REST.ts | 3 +-- unitTests/resources/queryCount.test.js | 17 ++++++++++------- unitTests/resources/queryCountBytes.test.js | 5 ++++- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index 33ba27a11c..a610474c49 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -340,7 +340,6 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { .expect((r) => assert.ok(!r.body || Object.keys(r.body).length === 0, r.text)) .expect(200); }); - }); // exactCount is a per-REST-mount policy, so it needs its own instance: two components exporting at diff --git a/server/REST.ts b/server/REST.ts index bd5af1d094..d4a10902dc 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -199,8 +199,7 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt // A mount can disable the expensive exact scan with `rest: { exactCount: false }` (default // enabled); a count=exact request is then served as a cheap estimate. Accept a string // `"false"` too, since not every config source coerces to a boolean. - const exactDisabled = - (httpOptions as any).exactCount === false || (httpOptions as any).exactCount === 'false'; + const exactDisabled = (httpOptions as any).exactCount === false || (httpOptions as any).exactCount === 'false'; for (const pref of parseHeaderValue(prefer as any)) { const mode = (pref?.value as string | undefined)?.toLowerCase(); if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index 4fdb439f36..22584fff1a 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -19,11 +19,7 @@ describe('Table.search count (REST pagination total-count)', () => { CountTable = table({ table: 'QueryCountTable', database: 'test', - attributes: [ - { name: 'id', isPrimaryKey: true }, - { name: 'group', indexed: true }, - { name: 'name' }, - ], + attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'group', indexed: true }, { name: 'name' }], }); let last; for (let i = 0; i < TOTAL; i++) { @@ -52,7 +48,11 @@ describe('Table.search count (REST pagination total-count)', () => { }); it('exact: a filtered query counts only matching records', async function () { - const paged = await CountTable.search({ conditions: [{ attribute: 'group', value: 'a' }], limit: 5, count: 'exact' }); + const paged = await CountTable.search({ + conditions: [{ attribute: 'group', value: 'a' }], + limit: 5, + count: 'exact', + }); assert.strictEqual(paged.length, 5); assert.strictEqual(paged.recordCount, GROUP_A); assert.strictEqual(paged.recordCountExact, true); @@ -107,7 +107,10 @@ describe('Table.search count (REST pagination total-count)', () => { it('estimated: an opaque rowFilter yields an unknown total (null), not a misleading estimate', async function () { const page = await CountTable.search({ rowFilter: (r) => r.group === 'a', limit: 3, count: 'estimated' }); assert.ok(page.length <= 3); - assert.ok(page.every((r) => r.group === 'a'), 'page must honor the rowFilter'); + assert.ok( + page.every((r) => r.group === 'a'), + 'page must honor the rowFilter' + ); assert.strictEqual(page.recordCount, null); assert.strictEqual(page.recordCountExact, false); }); diff --git a/unitTests/resources/queryCountBytes.test.js b/unitTests/resources/queryCountBytes.test.js index 8482d07320..c35292c9cd 100644 --- a/unitTests/resources/queryCountBytes.test.js +++ b/unitTests/resources/queryCountBytes.test.js @@ -19,7 +19,10 @@ describe('Table.search count with Bytes columns (read-buffer safety)', () => { BytesTable = table({ table: 'BytesCountTable', database: 'test', - attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'data', type: 'Bytes' }], + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'data', type: 'Bytes' }, + ], }); let last; for (let i = 0; i < N; i++) { From cac3ae224713560d76b31e811a76470a3bf4cd86 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Tue, 11 Aug 2026 18:34:02 -0500 Subject: [PATCH 07/14] fix(rest): require a limit for count so the guardrail can't be bypassed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review (claude[bot] on #2147) found the exact-count guardrail (row cap + time budget) and the estimated early-exit only applied when the request included a limit(): both live inside `if (end !== undefined ...)`. A count=exact/estimated request with no limit() therefore drained AND materialized the entire matched set with no cap — the exact unbounded-scan/-memory DoS the guardrail was built to prevent, on the most likely-hit path (a bare collection GET), and it bypassed the exactCount gate too. Counting is a pagination feature, so it now requires a limit(): a count request without one falls through to the normal streaming path (no count emitted), which keeps the guardrail always applied to a bounded page. Updated the unit test that documented the no-limit drain as intentional, and added a test asserting a no-limit count streams (does not materialize). Co-Authored-By: Claude Opus 4.8 (1M context) --- resources/Table.ts | 15 +++++++++++---- unitTests/resources/queryCount.test.js | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 7f4c42f151..b8a6548eab 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -3505,8 +3505,15 @@ export function makeTable(options) { // count so the HTTP layer can emit a Content-Range. `exact` drains the full matched set once, // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/ // table estimate. Opt-in only — the default streaming path below is untouched. - if (target.count) { + // + // Requires a limit. Counting is a pagination feature, and with no limit the "page" is the + // entire matched set — materializing/draining it would be an unbounded operation on the most + // likely-hit path (a bare collection GET). A count request without a limit therefore falls + // through to the normal streaming path (no count emitted), so the guardrail below always + // applies to a bounded page rather than being skipped when `end` is undefined. + if (target.count && target.limit !== undefined) { const wantExact = target.count === 'exact'; + const pageEnd = offset + (target.limit as number); const countStart = performance.now(); return (async () => { const page: any = []; @@ -3514,11 +3521,11 @@ export function makeTable(options) { let exact = true; try { for await (const record of results) { - if (scanned >= offset && (end === undefined || scanned < end)) page.push(record); + if (scanned >= offset && scanned < pageEnd) page.push(record); scanned++; - // The page window [offset, end) is always collected in full first — the guardrail + // The page window [offset, pageEnd) is always collected in full first — the guardrail // only ever abandons the running TOTAL, never truncates the page body. - if (end !== undefined && scanned >= end) { + if (scanned >= pageEnd) { if (!wantExact) break; // `estimated` needs nothing past the page // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a // large match set can't turn a bounded page fetch into an unbounded scan. diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index 22584fff1a..ddf42e6651 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -57,12 +57,23 @@ describe('Table.search count (REST pagination total-count)', () => { assert.strictEqual(paged.recordCount, GROUP_A); assert.strictEqual(paged.recordCountExact, true); - // no limit: the page is the whole matched set and the count agrees with it - const all = await CountTable.search({ conditions: [{ attribute: 'group', value: 'b' }], count: 'exact' }); + // a limit wide enough to cover the whole matched set gives page == matches, count == matches + const all = await CountTable.search({ conditions: [{ attribute: 'group', value: 'b' }], limit: 100, count: 'exact' }); assert.strictEqual(all.length, GROUP_B); assert.strictEqual(all.recordCount, GROUP_B); }); + it('count without a limit falls through to streaming (no unbounded drain)', async function () { + // A count is a pagination feature; without a limit the page would be the entire matched set, so the + // request is served by the normal streaming path with no count instead of materializing everything. + const results = CountTable.search({ conditions: [{ attribute: 'group', value: 'b' }], count: 'exact' }); + assert.ok(!Array.isArray(results), 'count without a limit must not materialize a page'); + assert.strictEqual(results.recordCount, undefined); + let n = 0; + for await (const _ of results) n++; + assert.strictEqual(n, GROUP_B); // still returns every matching row, just no count + }); + it('estimated: returns the page plus a positive estimate, flagged non-exact', async function () { const page = await CountTable.search({ limit: 5, count: 'estimated' }); assert.strictEqual(page.length, 5); @@ -116,7 +127,7 @@ describe('Table.search count (REST pagination total-count)', () => { }); it('exact: honors a rowFilter in both the page and the count', async function () { - const page = await CountTable.search({ rowFilter: (r) => r.group === 'b', count: 'exact' }); + const page = await CountTable.search({ rowFilter: (r) => r.group === 'b', limit: 100, count: 'exact' }); assert.strictEqual(page.length, GROUP_B); assert.strictEqual(page.recordCount, GROUP_B); assert.strictEqual(page.recordCountExact, true); From 289bfdfb6470bab613cbba9e2dc6fae61616d075 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Tue, 11 Aug 2026 18:36:25 -0500 Subject: [PATCH 08/14] chore: prettier format queryCount test Co-Authored-By: Claude Opus 4.8 (1M context) --- unitTests/resources/queryCount.test.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index ddf42e6651..89918ac507 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -58,7 +58,11 @@ describe('Table.search count (REST pagination total-count)', () => { assert.strictEqual(paged.recordCountExact, true); // a limit wide enough to cover the whole matched set gives page == matches, count == matches - const all = await CountTable.search({ conditions: [{ attribute: 'group', value: 'b' }], limit: 100, count: 'exact' }); + const all = await CountTable.search({ + conditions: [{ attribute: 'group', value: 'b' }], + limit: 100, + count: 'exact', + }); assert.strictEqual(all.length, GROUP_B); assert.strictEqual(all.recordCount, GROUP_B); }); From 5ec0d38a9ab7b13e04ca1a954c59351ff3074d62 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Thu, 13 Aug 2026 11:41:32 -0500 Subject: [PATCH 09/14] =?UTF-8?q?fix(rest):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20bound=20count=20pages,=20opt-in=20exact,=20GET/HEAD=20only,?= =?UTF-8?q?=20Vary/CORS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses kriszyp's review on #2147: - Bound the count page: the count path now requires a finite, non-negative integer limit no larger than MAX_COUNT_PAGE (10k). limit(Infinity), limit(foo)->NaN, a negative, or an oversized limit fall through to streaming with no count, so a count request can't be coerced into materializing an unbounded page. - Exact counting is now opt-in per mount (`rest: { exactCount: true }`, default off); count=exact is otherwise served as an estimate. Estimated stays the safe default, removing the default worker-saturation surface on public tables. - Only honor Prefer: count on GET/HEAD. It was set for every method, so a collection DELETE carrying limit()+Prefer received a materialized array from search() (declared AsyncIterable) and threw instead of deleting. - Emit `Vary: Prefer` on collection reads (after serialize, which resets Vary) so a shared cache can't serve count headers to a request that didn't ask, or a cached non-count response to one that did. - Compare Access-Control-Expose-Headers as case-insensitive comma tokens, not substrings, so an unrelated existing token (e.g. X-Content-Range-Metadata) no longer suppresses the real Content-Range token. Tests: unit 11 passing (added invalid/oversized-limit fall-through); integration 27 passing (oversized-limit fall-through, Vary: Prefer, DELETE-not-misrouted, and the new opt-in default via exactCount: true / default-off suites). Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/rest.test.mjs | 58 ++++++++++++++++++++----- resources/Table.ts | 19 +++++--- server/REST.ts | 40 +++++++++++------ unitTests/resources/queryCount.test.js | 11 +++++ 4 files changed, 97 insertions(+), 31 deletions(-) diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index a610474c49..2ef43f57ce 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -45,7 +45,8 @@ type SubObject @table(audit: false) @export { } `; -const CONFIG_YAML = `rest: true +const CONFIG_YAML = `rest: + exactCount: true graphqlSchema: files: '*.graphql' graphql: true @@ -68,7 +69,7 @@ const SUBOBJECT_ROWS = [ { id: '5', relatedId: '5', any: 'any-5' }, ]; -// Second component whose REST mount disables the expensive exact scan. +// Second component whose REST mount uses the default (exact counting NOT opted in). const SCHEMA_GATE_GRAPHQL = ` type GatedWidget @table @export(rest: true, mqtt: false) { id: ID @primaryKey @@ -76,8 +77,7 @@ type GatedWidget @table @export(rest: true, mqtt: false) { } `; -const CONFIG_GATE_YAML = `rest: - exactCount: false +const CONFIG_GATE_YAML = `rest: true graphqlSchema: files: '*.graphql' graphql: true @@ -340,12 +340,48 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { .expect((r) => assert.ok(!r.body || Object.keys(r.body).length === 0, r.text)) .expect(200); }); + + test('[rest] an oversized page limit falls through to streaming with no count', () => { + // A limit past the max count-page size must not materialize a count page — the request is served + // normally (all rows) with no Content-Range, rather than buffering an unbounded page. + return client + .reqRest('/Related/?sort(id)&limit(0,20000)') + .set('Prefer', 'count=exact') + .expect((r) => assert.equal(r.headers['content-range'], undefined, r.text)) + .expect((r) => assert.equal(r.body.length, 5, r.text)) + .expect(200); + }); + + test('[rest] a collection read declares Vary: Prefer', () => { + // So a shared cache keys on Prefer and never serves count headers to a request that did not ask. + return client + .reqRest('/Related/?sort(id)&limit(2)') + .expect((r) => assert.match(r.headers['vary'] || '', /\bPrefer\b/i, r.text)) + .expect(200); + }); + + test('[rest] DELETE with a limit and Prefer: count is not misrouted to the count path', () => { + // Regression: the count preference is GET/HEAD-only. A DELETE that also carried limit()+Prefer used + // to receive a materialized array from search() and throw instead of deleting. + return client + .req() + .send({ operation: 'insert', table: 'Related', records: [{ id: 'del-me', name: 'to-delete' }] }) + .expect(200) + .then(() => + request(client.restURL) + .delete('/Related/?id==del-me&limit(10)') + .set(client.headers) + .set('Prefer', 'count=exact') + .expect((r) => assert.ok(r.status >= 200 && r.status < 300, `expected 2xx, got ${r.status}: ${r.text}`)) + ) + .then(() => client.reqRest('/Related/?id==del-me').expect((r) => assert.equal(r.body.length, 0, r.text))); + }); }); // exactCount is a per-REST-mount policy, so it needs its own instance: two components exporting at -// the root path share one mount (the handler dedupes), and the gated config would otherwise bleed -// onto the main suite's routes. -suite('REST count exactCount gate', { skip: skipSuite }, (ctx) => { +// the root path share one mount (the handler dedupes), and this mount's default config would otherwise +// bleed onto the main suite's routes (which opt in with exactCount: true). +suite('REST count default (exact not opted in)', { skip: skipSuite }, (ctx) => { let client; before(async () => { @@ -370,8 +406,8 @@ suite('REST count exactCount gate', { skip: skipSuite }, (ctx) => { await teardownHarper(ctx); }); - // `rest: { exactCount: false }` serves count=exact as a cheap estimate instead. - test('[rest] exactCount:false downgrades count=exact to estimated', () => { + // Default (no exactCount opt-in): count=exact is served as a cheap estimate instead. + test('[rest] default downgrades count=exact to estimated', () => { return client .reqRest('/GatedWidget/?sort(id)&limit(2)') .set('Prefer', 'count=exact') @@ -381,8 +417,8 @@ suite('REST count exactCount gate', { skip: skipSuite }, (ctx) => { .expect(200); }); - // estimated still works normally on a gated mount. - test('[rest] exactCount:false leaves count=estimated unchanged', () => { + // estimated still works normally on a default mount. + test('[rest] default leaves count=estimated unchanged', () => { return client .reqRest('/GatedWidget/?sort(id)&limit(2)') .set('Prefer', 'count=estimated') diff --git a/resources/Table.ts b/resources/Table.ts index b8a6548eab..f9e8d6cc4f 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -136,6 +136,10 @@ const CACHEABLE_STATUS_CODES = new Set([200, 203, 204, 206, 300, 301, 308, 404, // (large filtered full-scan, in-memory sort) should still be gated by config before broad exposure. const MAX_EXACT_COUNT_SCAN = 1_000_000; const MAX_EXACT_COUNT_MS = 1_000; +// Largest page a `Prefer: count=` request will materialize. A request whose limit exceeds this (or is +// not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal +// streaming path with no count, so a count request can't be coerced into buffering an unbounded page. +const MAX_COUNT_PAGE = 10_000; envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; @@ -3506,14 +3510,15 @@ export function makeTable(options) { // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/ // table estimate. Opt-in only — the default streaming path below is untouched. // - // Requires a limit. Counting is a pagination feature, and with no limit the "page" is the - // entire matched set — materializing/draining it would be an unbounded operation on the most - // likely-hit path (a bare collection GET). A count request without a limit therefore falls - // through to the normal streaming path (no count emitted), so the guardrail below always - // applies to a bounded page rather than being skipped when `end` is undefined. - if (target.count && target.limit !== undefined) { + // Requires a bounded page. Counting is a pagination feature; the page limit must be a finite, + // non-negative integer no larger than MAX_COUNT_PAGE. A missing limit (a bare collection GET), a + // non-finite/negative/non-integer limit (limit(Infinity), limit(foo)), or an oversized one all + // fall through to the normal streaming path with no count, so a count request can't materialize + // an unbounded page before the guardrail below applies. + const pageLimit = target.limit as number; + if (target.count && Number.isInteger(pageLimit) && pageLimit >= 0 && pageLimit <= MAX_COUNT_PAGE) { const wantExact = target.count === 'exact'; - const pageEnd = offset + (target.limit as number); + const pageEnd = offset + pageLimit; const countStart = performance.now(); return (async () => { const page: any = []; diff --git a/server/REST.ts b/server/REST.ts index d4a10902dc..6ce1435807 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -8,7 +8,7 @@ import { Resources } from '../resources/Resources.ts'; import { Resource, missingMethod, allowedMethods } from '../resources/Resource.ts'; import { IterableEventQueue } from '../resources/IterableEventQueue.ts'; import { transaction } from '../resources/transaction.ts'; -import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts'; +import { Headers, mergeHeaders, addVaryHeader } from '../server/serverHelpers/Headers.ts'; import { generateJsonApi } from '../resources/openApi.ts'; import { getConfigPath } from '../config/configUtils.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; @@ -135,12 +135,18 @@ function setCountHeaders(headers: Headers, offset: number, mode: string, page: a headers.set('Range-Unit', 'items'); headers.set('Content-Range', `items ${range}/${totalStr}`); headers.set('Preference-Applied', `count=${mode}`); - // Append (don't overwrite) so a resource that already exposed its own headers keeps them. + // Append (don't overwrite) so a resource that already exposed its own headers keeps them. Compare + // case-insensitive comma tokens, not substrings, so an unrelated existing token (e.g. + // `X-Content-Range-Metadata`) doesn't suppress the real `Content-Range` token. const exposed = headers.get('Access-Control-Expose-Headers'); + const existing = new Set( + (Array.isArray(exposed) ? exposed.join(',') : exposed || '') + .split(',') + .map((token) => token.trim().toLowerCase()) + .filter(Boolean) + ); for (const name of ['Content-Range', 'Range-Unit', 'Preference-Applied']) { - if (!exposed || !String(exposed).toLowerCase().includes(name.toLowerCase())) { - headers.append('Access-Control-Expose-Headers', name, true); - } + if (!existing.has(name.toLowerCase())) headers.append('Access-Control-Expose-Headers', name, true); } } @@ -193,18 +199,22 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt (target as any).async = true; resource = entry.Resource; // Pagination total-count opt-in (no default): `Prefer: count=exact|estimated`. Table.search - // reads target.count to compute the total emitted as Content-Range below. + // reads target.count to compute the total emitted as Content-Range below. Only honored on + // GET/HEAD reads: setting it for other methods would hand their `search()` a materialized array + // instead of the AsyncIterable they iterate (e.g. a collection DELETE at Table.ts). const prefer = headersObject['prefer']; - if (prefer) { - // A mount can disable the expensive exact scan with `rest: { exactCount: false }` (default - // enabled); a count=exact request is then served as a cheap estimate. Accept a string - // `"false"` too, since not every config source coerces to a boolean. - const exactDisabled = (httpOptions as any).exactCount === false || (httpOptions as any).exactCount === 'false'; + if (prefer && (method === 'GET' || method === 'HEAD')) { + // Exact counting scans the full matched set, so it is opt-in per mount via + // `rest: { exactCount: true }` (default off); count=exact is otherwise served as a cheap + // estimate. Estimated is always available. Accept a string `"true"` too, since not every + // config source coerces to a boolean. + const exactEnabled = (httpOptions as any).exactCount === true || (httpOptions as any).exactCount === 'true'; for (const pref of parseHeaderValue(prefer as any)) { const mode = (pref?.value as string | undefined)?.toLowerCase(); if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { - // Downgrade is signaled back to the client via Preference-Applied. - (target as any).count = mode === 'exact' && exactDisabled ? 'estimated' : mode; + // A count=exact request on a mount that hasn't opted in is downgraded to estimated, + // signaled back to the client via Preference-Applied. + (target as any).count = mode === 'exact' && !exactEnabled ? 'estimated' : mode; break; } } @@ -406,6 +416,10 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt responseObject.body = serialize(responseData, request, responseObject); if (method === 'HEAD') responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body } + // A collection read's count headers vary by the request's `Prefer` value; serialize() just reset + // `Vary`, so declare it here (after serialization) — otherwise a shared cache could serve count + // headers to a request that didn't ask, or a cached non-count response to one that did. + if ((method === 'GET' || method === 'HEAD') && (target as any)?.isCollection) addVaryHeader(headers, 'Prefer'); return responseObject; } catch (error) { error ??= new Error('Unknown error occurred'); diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index 89918ac507..d01f5793e2 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -78,6 +78,17 @@ describe('Table.search count (REST pagination total-count)', () => { assert.strictEqual(n, GROUP_B); // still returns every matching row, just no count }); + it('a non-finite, negative, or oversized limit falls through to streaming (no count)', function () { + // The count page must be a finite, non-negative integer no larger than the max count-page size; + // anything else (limit(Infinity)/limit(foo)->NaN, a negative, or an oversized limit) must not + // materialize a count page. + for (const limit of [Infinity, NaN, -1, 20000]) { + const results = CountTable.search({ limit, count: 'exact' }); + assert.ok(!Array.isArray(results), `limit=${limit} must not materialize a count page`); + assert.strictEqual(results.recordCount, undefined); + } + }); + it('estimated: returns the page plus a positive estimate, flagged non-exact', async function () { const page = await CountTable.search({ limit: 5, count: 'estimated' }); assert.strictEqual(page.length, 5); From 57caa5ec3e08932d245c07dbbd429dec9f031229 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Thu, 13 Aug 2026 17:24:52 -0500 Subject: [PATCH 10/14] fix(rest): validate count offset and total window, not just the limit Follow-up to kriszyp's review on #2147: the bounded-page check validated the limit but accepted any offset. A negative offset (limit(-5,10)) diverged from the normal slice path, and an arbitrarily large offset postponed the exact-count guardrails (which engage only past the page window) until that offset had been scanned. The count path now also requires the offset to be a finite, non-negative integer and the window (offset + limit) to be within MAX_EXACT_COUNT_SCAN; anything else (a negative offset, or a deep-page window past the scan budget) falls through to streaming with no count. Adds unit + integration coverage for negative and oversized-window offsets. Co-Authored-By: Claude Opus 4.8 (1M context) --- integrationTests/apiTests/rest.test.mjs | 10 ++++++++++ resources/Table.ts | 22 ++++++++++++++++------ unitTests/resources/queryCount.test.js | 13 +++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/integrationTests/apiTests/rest.test.mjs b/integrationTests/apiTests/rest.test.mjs index 2ef43f57ce..829dd93256 100644 --- a/integrationTests/apiTests/rest.test.mjs +++ b/integrationTests/apiTests/rest.test.mjs @@ -352,6 +352,16 @@ suite('REST query syntax', { skip: skipSuite }, (ctx) => { .expect(200); }); + test('[rest] a deep-page offset past the scan budget falls through with no count', () => { + // limit(start,end) with a huge start is a huge offset; the count path must not iterate an unbounded + // offset before its guardrail engages, so the request falls through with no Content-Range. + return client + .reqRest('/Related/?sort(id)&limit(2000000,2000010)') + .set('Prefer', 'count=exact') + .expect((r) => assert.equal(r.headers['content-range'], undefined, r.text)) + .expect(200); + }); + test('[rest] a collection read declares Vary: Prefer', () => { // So a shared cache keys on Prefer and never serves count headers to a request that did not ask. return client diff --git a/resources/Table.ts b/resources/Table.ts index f9e8d6cc4f..84dcbd7d23 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -3510,13 +3510,23 @@ export function makeTable(options) { // windowing the page in the same pass; `estimated` returns just the page plus a cheap planner/ // table estimate. Opt-in only — the default streaming path below is untouched. // - // Requires a bounded page. Counting is a pagination feature; the page limit must be a finite, - // non-negative integer no larger than MAX_COUNT_PAGE. A missing limit (a bare collection GET), a - // non-finite/negative/non-integer limit (limit(Infinity), limit(foo)), or an oversized one all - // fall through to the normal streaming path with no count, so a count request can't materialize - // an unbounded page before the guardrail below applies. + // Requires a bounded page AND window. Counting is a pagination feature; both the limit and the + // offset must be finite, non-negative integers, the limit no larger than MAX_COUNT_PAGE, and the + // window (offset + limit) no larger than MAX_EXACT_COUNT_SCAN. Anything else — a missing/ + // oversized/non-finite/negative limit or offset (a bare collection GET, limit(Infinity), + // limit(foo), limit(-5,10)) or a deep-page window past the scan budget — falls through to the + // normal streaming path with no count. This bounds the offset too: without it a huge offset would + // postpone the exact guardrail (which only engages past the page) until that offset was scanned. const pageLimit = target.limit as number; - if (target.count && Number.isInteger(pageLimit) && pageLimit >= 0 && pageLimit <= MAX_COUNT_PAGE) { + if ( + target.count && + Number.isInteger(pageLimit) && + pageLimit >= 0 && + pageLimit <= MAX_COUNT_PAGE && + Number.isInteger(offset) && + offset >= 0 && + offset + pageLimit <= MAX_EXACT_COUNT_SCAN + ) { const wantExact = target.count === 'exact'; const pageEnd = offset + pageLimit; const countStart = performance.now(); diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index d01f5793e2..92448bbf13 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -89,6 +89,19 @@ describe('Table.search count (REST pagination total-count)', () => { } }); + it('a negative or oversized-window offset falls through to streaming (no count)', function () { + // The offset must be a finite non-negative integer, and offset+limit must be within the scan + // budget — otherwise a huge offset would postpone the exact guardrail until it had been scanned. + for (const t of [ + { offset: -5, limit: 10 }, + { offset: 2_000_000, limit: 10 }, + ]) { + const results = CountTable.search({ ...t, count: 'exact' }); + assert.ok(!Array.isArray(results), `offset=${t.offset} must not materialize a count page`); + assert.strictEqual(results.recordCount, undefined); + } + }); + it('estimated: returns the page plus a positive estimate, flagged non-exact', async function () { const page = await CountTable.search({ limit: 5, count: 'estimated' }); assert.strictEqual(page.length, 5); From 6bcd6f8e32fb63cbf74293a9c8c9e022f4df65b4 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Fri, 28 Aug 2026 13:24:46 -0500 Subject: [PATCH 11/14] feat(rest): use rocksdb-js estimateCount for range count estimates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps @harperfast/rocksdb-js to 2.8.0 and wires its new statistical range key-count estimator (`estimateCount`) into the query planner's range estimates, which the `Prefer: count=estimated` pagination path reads. Range comparators (between, starts_with, greater/less, open ranges) previously returned an arbitrary fixed fraction of the table size, because the storage layer could not estimate a range's cardinality. `estimateCondition` now asks the engine for a real range estimate on RocksDB, and only falls back to the old heuristic when unavailable (LMDB engine, non-indexed / custom-indexed attribute, an unbounded/degenerate range, or a zero-confidence — failed-statistics — read). Equals still uses the exact per-value index count; this only replaces the range guesses. Because the REST estimated-count path funnels through `estimateCondition`, `count=estimated` now reports a range-aware total, and the planner picks indexes for range queries from real selectivity rather than a constant. `RocksIndexStore.estimateCount` overrides the inherited estimator to apply the same `[indexedValue, primaryKey]` composite-key rewrite as its `getRange`, so a secondary-index range estimate covers exactly the keys the scan would visit (otherwise an inclusive end / exclusive start would miss the value's bucket). Tests: resources unit adds primary-key and secondary-index range-estimate cases proving the total tracks range width (a narrow tail estimates fewer than the whole table/index) where the old range-blind heuristic would tie. Existing count, planner, and search suites green (1803 resources tests passing); tsc and lint clean. Behavior on LMDB is unchanged (falls back to the prior heuristic). Co-Authored-By: Claude Opus 4.8 (1M context) --- package-lock.json | 83 +++++++++++++----------- package.json | 2 +- resources/RocksIndexStore.ts | 20 ++++++ resources/search.ts | 89 ++++++++++++++++++++++++-- unitTests/resources/queryCount.test.js | 68 +++++++++++++++++++- 5 files changed, 217 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index fc1937654c..b310e0386f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@fastify/cors": "^11.2.0", "@fastify/static": "^9.1.3", "@harperfast/extended-iterable": "1.0.3", - "@harperfast/rocksdb-js": "2.7.1", + "@harperfast/rocksdb-js": "2.8.0", "@harperfast/skills": "^1.10.8", "@turf/area": "6.5.0", "@turf/boolean-contains": "6.5.0", @@ -2495,13 +2495,13 @@ } }, "node_modules/@harperfast/rocksdb-js": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-2.7.1.tgz", - "integrity": "sha512-Fs+Ki/9ysu4w0oGl4NN+6LWlKlQtUpw0RUZlOewbAXStEzJ4087Dk1vHKM2YqnAqNYoZb0UOz2XeoM8YkelMww==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js/-/rocksdb-js-2.8.0.tgz", + "integrity": "sha512-czswCG+1KCRMYe6XQcsh5u28IQ6EbIx9eXPQd0kmjiLbtoMjiszYE92756bZJSbDykbR2OJLOazMZg7+qfFSJA==", "license": "Apache-2.0", "dependencies": { "@harperfast/extended-iterable": "1.0.3", - "msgpackr": "2.0.5", + "msgpackr": "2.0.6", "ordered-binary": "1.6.1" }, "bin": { @@ -2511,20 +2511,20 @@ "node": "^22.18.0 || >=24.0.0" }, "optionalDependencies": { - "@harperfast/rocksdb-js-darwin-arm64": "2.7.1", - "@harperfast/rocksdb-js-darwin-x64": "2.7.1", - "@harperfast/rocksdb-js-linux-arm64-glibc": "2.7.1", - "@harperfast/rocksdb-js-linux-arm64-musl": "2.7.1", - "@harperfast/rocksdb-js-linux-x64-glibc": "2.7.1", - "@harperfast/rocksdb-js-linux-x64-musl": "2.7.1", - "@harperfast/rocksdb-js-win32-arm64": "2.7.1", - "@harperfast/rocksdb-js-win32-x64": "2.7.1" + "@harperfast/rocksdb-js-darwin-arm64": "2.8.0", + "@harperfast/rocksdb-js-darwin-x64": "2.8.0", + "@harperfast/rocksdb-js-linux-arm64-glibc": "2.8.0", + "@harperfast/rocksdb-js-linux-arm64-musl": "2.8.0", + "@harperfast/rocksdb-js-linux-x64-glibc": "2.8.0", + "@harperfast/rocksdb-js-linux-x64-musl": "2.8.0", + "@harperfast/rocksdb-js-win32-arm64": "2.8.0", + "@harperfast/rocksdb-js-win32-x64": "2.8.0" } }, "node_modules/@harperfast/rocksdb-js-darwin-arm64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-2.7.1.tgz", - "integrity": "sha512-H0aEOziU6WFaVUjoRvsWVUoIEAZ6ylLRYTa4z4R7SRVUt2pzpjiUMW1mHBTmIRAue/Fwsfm8SCM95WQj5goFKA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-arm64/-/rocksdb-js-darwin-arm64-2.8.0.tgz", + "integrity": "sha512-v7cj3bGpRptZHXSbhVzYJ0k+jyMD0Z0kX0u8YNZH6Xq+ifZ6zBkBbyoSdvQ2waGLJXxBORylUxTgYMvTZWiQBA==", "cpu": [ "arm64" ], @@ -2538,9 +2538,9 @@ } }, "node_modules/@harperfast/rocksdb-js-darwin-x64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-2.7.1.tgz", - "integrity": "sha512-YDPVGmfFyg9sCu3C+uulJa68o4B06pJ8MDWSgSqm+5uNXh4QNWuMSE1x2wLZYOt1hxE7xptHXzhMFFuSqIcvtg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-darwin-x64/-/rocksdb-js-darwin-x64-2.8.0.tgz", + "integrity": "sha512-9mMPPvRhxjLorE7u1gN3n+InzT4XHXQkHYdNKE6qN4iwZy0vM8iHoQLwIiN9GpzUiueEqBEilAP/9mIOHhMgBw==", "cpu": [ "x64" ], @@ -2554,9 +2554,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-arm64-glibc": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-2.7.1.tgz", - "integrity": "sha512-8LoPDc4muFb4U09VMWC1BJZRA/CxYBer1eIRW6Td6EfB8lOiBgZlpQvmc44GilYgrH4/C1at8lMK+G24U2CDyA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-glibc/-/rocksdb-js-linux-arm64-glibc-2.8.0.tgz", + "integrity": "sha512-H1nNimVbIB4/6Sxv5DrxmP1asHLbxTCHuZ2ZN/zudqPkp/bTTSs5RsczOH6JeKTEUxUsJDTXM4nGRcn6LwFaOg==", "cpu": [ "arm64" ], @@ -2573,9 +2573,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-arm64-musl": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-2.7.1.tgz", - "integrity": "sha512-vqeH0JM/FUbQfx1Q5n21aBCiLUukN/qMXKa5d3YUQt7hH1fFA4d9wQhnRqwV9VbIjxm6i3drYAy5HaFkU2TV1g==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-arm64-musl/-/rocksdb-js-linux-arm64-musl-2.8.0.tgz", + "integrity": "sha512-EqG4lkYptNviIcPPaQjK+X4ufjqzSImIvVxP89GHFHrbOJkGgEwRpiya0gWfGu3vJ1iGL1XL1nsU/kEjeDOugg==", "cpu": [ "arm64" ], @@ -2592,9 +2592,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-x64-glibc": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-2.7.1.tgz", - "integrity": "sha512-2pbbjq36Ln+ln31a3oDf2ic5IQNvrXNP9ND6LM7lMYUjuG/7tV3tWK79x2AyATym6iq0Kmz5Sp0b8tnUL2nQNg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-glibc/-/rocksdb-js-linux-x64-glibc-2.8.0.tgz", + "integrity": "sha512-MZPKZ6gF8kCy/KXTav4tc7T2uq+iOunxDOYq7nifZkCCy+0+C9U7g9CZx0NErORRQRP29x3z0zEKkCQ/go5c6Q==", "cpu": [ "x64" ], @@ -2611,9 +2611,9 @@ } }, "node_modules/@harperfast/rocksdb-js-linux-x64-musl": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-2.7.1.tgz", - "integrity": "sha512-hrk3BvNjjrWV0j0XjDZ80r3XIQdLnP2nH/WNy+KFU484XEkdXONmj0zH0hsOJVItTv3JShEECWOk6BXlR3osaw==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-linux-x64-musl/-/rocksdb-js-linux-x64-musl-2.8.0.tgz", + "integrity": "sha512-PnMn1f/UxGOZKxlayR82tzsFCGoaMu0mBGKMmP/r2rbxzX5T89T0c+aVhuhpgNrFlWXTdawlA10Oh/YjfT/2qQ==", "cpu": [ "x64" ], @@ -2630,9 +2630,9 @@ } }, "node_modules/@harperfast/rocksdb-js-win32-arm64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-2.7.1.tgz", - "integrity": "sha512-E4Y725WvIcJjfSidOJt/gee4ogKWaFxj8exX/D6Kw2+v5RxuVjMgyKyOIxXQ+nLd18XGXTKj6HtgYzWOJfbEGg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-arm64/-/rocksdb-js-win32-arm64-2.8.0.tgz", + "integrity": "sha512-4khqmGbIebiCwY3FVIgcUXEmMapgWP8f97aNrbThPDCIkuJscB3yoAG4A8OJo+Jln/TdjJZOgy8e2ir9F926dA==", "cpu": [ "arm64" ], @@ -2646,9 +2646,9 @@ } }, "node_modules/@harperfast/rocksdb-js-win32-x64": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-2.7.1.tgz", - "integrity": "sha512-7TcSJ3gXgknoccLe+Tc0nqAxOA1kt5FE30GFrrTs7MyZczdO1rioRiccqhgcRA4UvW8Xc29GwJPd9ehmK8yYHg==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@harperfast/rocksdb-js-win32-x64/-/rocksdb-js-win32-x64-2.8.0.tgz", + "integrity": "sha512-E5HpBAnRwotITq++534WCinMvYidkkPoaAD5uwBNHiQbNt6YYvNZllJAIh/bukXD+uO6DL6qUMmVkeBh3kWK5A==", "cpu": [ "x64" ], @@ -2661,6 +2661,15 @@ "node": "^22.18.0 || >=24.0.0" } }, + "node_modules/@harperfast/rocksdb-js/node_modules/msgpackr": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-2.0.6.tgz", + "integrity": "sha512-plGul/tqjt9vqWFR9zyqyLZls6gb5KTLvnsRO2B9+TZ8tNiXiVI/CR8LiDWlAX4IUeVkQ9gsmzKA6voC2QOciw==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.4" + } + }, "node_modules/@harperfast/skills": { "version": "1.12.1", "resolved": "https://registry.npmjs.org/@harperfast/skills/-/skills-1.12.1.tgz", diff --git a/package.json b/package.json index 418e1f13a3..d827ab68f1 100644 --- a/package.json +++ b/package.json @@ -179,7 +179,7 @@ "@fastify/cors": "^11.2.0", "@fastify/static": "^9.1.3", "@harperfast/extended-iterable": "1.0.3", - "@harperfast/rocksdb-js": "2.7.1", + "@harperfast/rocksdb-js": "2.8.0", "@harperfast/skills": "^1.10.8", "@turf/area": "6.5.0", "@turf/boolean-contains": "6.5.0", diff --git a/resources/RocksIndexStore.ts b/resources/RocksIndexStore.ts index c223f84090..ad36651b11 100644 --- a/resources/RocksIndexStore.ts +++ b/resources/RocksIndexStore.ts @@ -1,5 +1,7 @@ import { DBI, + type CountEstimate, + type CountEstimateOptions, type StoreIteratorOptions, type StorePutOptions, type StoreRemoveOptions, @@ -40,6 +42,24 @@ export class RocksIndexStore extends RocksDatabase { }); } + /** + * Estimate the number of index entries in a range. The bounds are bare indexed values, so they get the + * same composite-key rewrite as {@link getRange} — otherwise an inclusive end or exclusive start would + * miss the value's `[value, primaryKey]` bucket, and the estimate would cover a different key range than + * the scan it is meant to describe. + */ + estimateCount(options?: CountEstimateOptions): CountEstimate { + if (!options) return super.estimateCount(options); + let { start, end, exclusiveStart, inclusiveEnd, reverse } = options; + if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) { + start = [start, MAXIMUM_KEY]; + } + if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) { + end = [end, MAXIMUM_KEY]; + } + return super.estimateCount({ ...options, start, end }); + } + /** * Translate a put with indexed value and primary key to an underlying put * @param indexedValue - ignored, only used by LMDB diff --git a/resources/search.ts b/resources/search.ts index 07eb4d4abe..1b30fdf435 100644 --- a/resources/search.ts +++ b/resources/search.ts @@ -1112,6 +1112,76 @@ export function filterByType(searchCondition, Table, context, filtered, isPrimar } } +// Maps a range comparator + value to the key-range bounds (`{ start, end, inclusiveEnd, exclusiveStart }`) +// that a scan would visit, in the exact shape `estimateCount()`/`getRange()` accept. Kept in sync with the +// comparator switch in `searchByIndex` (the source of truth for how a scan is bounded) so a range estimate +// covers precisely the keys the scan would. Returns undefined for anything that isn't a cleanly bounded +// range (empty prefix, malformed between, or a full-scan comparator) so the caller keeps its heuristic. +function rangeBoundsForEstimate(comparator, value) { + if (value instanceof Date) value = value.getTime(); + switch (ALTERNATE_COMPARATOR_NAMES[comparator] || comparator) { + case 'lt': + return { end: value, inclusiveEnd: false }; + case 'le': + return { end: value, inclusiveEnd: true }; + case 'gt': + return { start: value, exclusiveStart: true }; + case 'ge': + return { start: value, exclusiveStart: false }; + case 'between': + case 'gele': + case 'gelt': + case 'gtlt': + case 'gtle': { + if (!Array.isArray(value) || value.length < 2) return undefined; + let start = value[0]; + if (start instanceof Date) start = start.getTime(); + let end = value[1]; + if (end instanceof Date) end = end.getTime(); + return { + start, + end, + inclusiveEnd: comparator === 'gele' || comparator === 'gtle' || comparator === 'between', + exclusiveStart: comparator === 'gtlt' || comparator === 'gtle', + }; + } + case 'starts_with': { + const start = value?.toString() ?? ''; + if (start.length === 0) return undefined; // empty prefix is a full scan, not a range + return { start, end: getStringPrefixUpperBound(start), inclusiveEnd: false }; + } + case 'prefix': { + // multi-part key prefix: [prefix, null] .. [prefix, MAXIMUM_KEY] + let start = Array.isArray(value) ? value : [value, null]; + if (start[start.length - 1] != null) start = start.concat(null); + const end = start.slice(0); + end[end.length - 1] = MAXIMUM_KEY; + return { start, end, inclusiveEnd: true }; + } + default: + return undefined; + } +} + +// A statistical key-count estimate over the range a range condition scans, from the storage engine's +// range estimator (rocksdb-js `estimateCount`). Returns undefined when unavailable — the LMDB engine (no +// range estimator), a non-indexed / custom-indexed attribute, an unbounded/degenerate range, or a +// zero-confidence estimate (a failed statistics read) — so the caller falls back to a heuristic fraction. +function estimateRangeCount(table, condition, comparator) { + const attribute_name = condition[0] ?? condition.attribute; + const isPrimaryKey = attribute_name == null || attribute_name === table.primaryKey; + const store = isPrimaryKey ? table.primaryStore : table.indices[attribute_name]; + // estimateCount is a RocksDB-only capability. A custom index's store is a plain object store keyed by + // primary key (not [indexedValue, primaryKey]), so a value-range estimate over it would be meaningless — + // leave those (and LMDB stores) to the heuristic / the custom index's own estimation. + if (!(store instanceof RocksDatabase) || (store as any).customIndex) return undefined; + const bounds = rangeBoundsForEstimate(comparator, condition[1] ?? condition.value); + if (!bounds) return undefined; + const { count, confidence } = store.estimateCount(bounds); + // confidence 0 marks a failed/degenerate estimate — don't prefer it over the heuristic. + return confidence > 0 ? count : undefined; +} + export function estimateCondition(table) { function estimateConditionForTable(condition) { if (condition.estimated_count === undefined) { @@ -1166,7 +1236,8 @@ export function estimateCondition(table) { : estimate); } } else { - // we only attempt to estimate count on equals operator because that's really all that LMDB supports (some other key-value stores like libmdbx could be considered if we need to do estimated counts of ranges at some point) + // Equals is an exact per-value index count on any engine; range comparators fall to the + // range branches below, which get a statistical estimate on RocksDB (see estimateRangeCount). const index = table.indices[attribute_name]; condition.estimated_count = index ? index.getValuesCount(condition[1] ?? condition.value) : Infinity; } @@ -1190,11 +1261,16 @@ export function estimateCondition(table) { } else if (Array.isArray(condition.value)) { condition.estimated_count = Infinity; } else condition.estimated_count = Infinity; - // for range queries (betweens, startsWith, greater, etc.), just arbitrarily guess + // Range queries (between, starts_with, greater/less, open ranges): ask the storage engine for a + // statistical range estimate; only fall back to an arbitrary fraction of the table when it can't. } else if (searchType === 'starts_with' || searchType === 'prefix') - condition.estimated_count = STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; + condition.estimated_count = + estimateRangeCount(table, condition, searchType) ?? + STARTS_WITH_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; else if (searchType === 'between') - condition.estimated_count = BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; + condition.estimated_count = + estimateRangeCount(table, condition, searchType) ?? + BETWEEN_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; else if (searchType === 'sort') { const attribute_name = condition[0] ?? condition.attribute; const index = table.indices[attribute_name]; @@ -1209,7 +1285,10 @@ export function estimateCondition(table) { if (index?.customIndex?.estimateCount) // allow custom index to define its own estimation of counts condition.estimated_count = index.customIndex.estimateCount(condition.value); - else condition.estimated_count = OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; + else + condition.estimated_count = + estimateRangeCount(table, condition, searchType) ?? + OPEN_RANGE_ESTIMATE * estimatedEntryCount(table.primaryStore) + 1; } // we give a condition significantly more weight/preference if we will be ordering by it if (typeof condition.descending === 'boolean') condition.estimated_count /= 2; diff --git a/unitTests/resources/queryCount.test.js b/unitTests/resources/queryCount.test.js index 92448bbf13..7e4ec03a49 100644 --- a/unitTests/resources/queryCount.test.js +++ b/unitTests/resources/queryCount.test.js @@ -19,11 +19,16 @@ describe('Table.search count (REST pagination total-count)', () => { CountTable = table({ table: 'QueryCountTable', database: 'test', - attributes: [{ name: 'id', isPrimaryKey: true }, { name: 'group', indexed: true }, { name: 'name' }], + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'group', indexed: true }, + { name: 'score', indexed: true }, + { name: 'name' }, + ], }); let last; for (let i = 0; i < TOTAL; i++) { - last = CountTable.put({ id: i, group: i < GROUP_A ? 'a' : 'b', name: 'n-' + i }); + last = CountTable.put({ id: i, group: i < GROUP_A ? 'a' : 'b', score: i, name: 'n-' + i }); } await last; }); @@ -119,6 +124,65 @@ describe('Table.search count (REST pagination total-count)', () => { assert.strictEqual(filtered.recordCountExact, false); }); + it('estimated: a range query returns a valid, non-exact total that reflects the range on RocksDB', async function () { + // A range comparator gets a statistical range estimate from the storage engine (rocksdb-js + // estimateCount) and falls back to a heuristic fraction on LMDB. Either way the count target must + // return a valid, non-exact total no smaller than the page. + const wide = await CountTable.search({ + conditions: [{ attribute: 'id', comparator: 'greater_than_equal', value: 0 }], + limit: 4, + count: 'estimated', + }); + assert.strictEqual(wide.length, 4); + assert.strictEqual(typeof wide.recordCount, 'number'); + assert.strictEqual(wide.recordCountExact, false); + assert.ok(wide.recordCount >= wide.length, `total ${wide.recordCount} must be >= page length ${wide.length}`); + + // On RocksDB the range estimator makes the total track the actual key range: a narrow tail must + // estimate fewer rows than the whole table. The old heuristic was range-blind (a fixed fraction of + // the table size), so narrow and wide would tie — this is what proves estimateCount is in play. + const { RocksDatabase } = require('@harperfast/rocksdb-js'); + if (CountTable.primaryStore instanceof RocksDatabase) { + const narrow = await CountTable.search({ + conditions: [{ attribute: 'id', comparator: 'greater_than_equal', value: 15 }], + limit: 4, + count: 'estimated', + }); + assert.ok( + narrow.recordCount < wide.recordCount, + `narrow tail (${narrow.recordCount}) should estimate fewer than the whole table (${wide.recordCount})` + ); + } + }); + + it('estimated: a secondary-index range estimate ranges over the composite index keyspace', async function () { + // A range on an indexed attribute estimates over the index store (RocksIndexStore), whose keys are + // [indexedValue, primaryKey]. Its estimateCount must apply the same composite-key rewrite as its + // getRange, so the estimate still tracks the range width — a narrow tail below the whole index. + const wide = await CountTable.search({ + conditions: [{ attribute: 'score', comparator: 'greater_than_equal', value: 0 }], + limit: 4, + count: 'estimated', + }); + assert.strictEqual(wide.length, 4); + assert.strictEqual(typeof wide.recordCount, 'number'); + assert.strictEqual(wide.recordCountExact, false); + assert.ok(wide.recordCount >= wide.length, `total ${wide.recordCount} must be >= page length ${wide.length}`); + + const { RocksDatabase } = require('@harperfast/rocksdb-js'); + if (CountTable.indices?.score instanceof RocksDatabase) { + const narrow = await CountTable.search({ + conditions: [{ attribute: 'score', comparator: 'greater_than_equal', value: 15 }], + limit: 4, + count: 'estimated', + }); + assert.ok( + narrow.recordCount < wide.recordCount, + `narrow index tail (${narrow.recordCount}) should estimate fewer than the whole index (${wide.recordCount})` + ); + } + }); + it('default (no count): still returns the lazy streaming iterable, not a materialized page', async function () { const results = CountTable.search({ limit: 5 }); assert.ok(!Array.isArray(results), 'default search must not materialize an array'); From 9e44500d7912340e2581182afb9a9b065fad378f Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Mon, 31 Aug 2026 09:54:55 -0500 Subject: [PATCH 12/14] fix(rest): don't advertise an HNSW/vector-sorted count as exact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `Prefer: count=exact` request sorted by a vector/HNSW attribute (or shaped by a `vectorFilter`) drained `scanned` rows and reported that as the exact total. An HNSW traversal returns a bounded, approximate candidate set whose size is chosen from `minResults` (offset + limit), so `scanned` tracks the requested page size, not the true match count — the same query at limit(5) vs limit(200) could advertise two different `count=exact` totals. The count path now detects an approximate (vector-sorted or vector-filtered) result set and reports the total as unavailable (`recordCount` null, `recordCountExact` false → `Content-Range: items x-y/*`) instead of a page-size-dependent number, mirroring how the estimated branch already bails to null for an opaque row/vector filter. Pages still materialize normally; only the untrustworthy total is withheld. Regression test (`queryCountVector.test.js`): the same cosine-sorted query at limit(5) and limit(40) must report the total unavailable at both, not two different exact numbers. Full resources suite green (1883). Addresses the standing review blocker raised across rounds (2026-08-20..31). Co-Authored-By: Claude Opus 4.8 (1M context) --- resources/Table.ts | 16 +++++- unitTests/resources/queryCountVector.test.js | 59 ++++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 unitTests/resources/queryCountVector.test.js diff --git a/resources/Table.ts b/resources/Table.ts index 95cb76588b..1d8cd9e256 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -3745,6 +3745,16 @@ export function makeTable(options) { const wantExact = target.count === 'exact'; const pageEnd = offset + pageLimit; const countStart = performance.now(); + // A vector/HNSW-driven result set — sorted by a custom index, or shaped by a vector filter — + // is a bounded, approximate candidate set whose size is chosen from `minResults` (offset + + // limit), so `scanned` tracks the requested page size, not the true match count. It is not an + // authoritative total: the same query at limit(5) vs limit(200) would otherwise advertise two + // different `count=exact` totals. Report the total as unavailable instead (mirroring how the + // estimated branch below already bails to null for a vector/row filter). + let approximateResultSet = typeof target.vectorFilter === 'function'; + for (let order = sort; !approximateResultSet && order; order = order.next) { + if (typeof order.attribute === 'string' && indices[order.attribute]?.customIndex) approximateResultSet = true; + } return (async () => { const page: any = []; let scanned = 0; @@ -3772,7 +3782,9 @@ export function makeTable(options) { } let total: number | null; if (wantExact) { - total = exact ? scanned : null; + // `scanned` is only an authoritative total when the iteration was exhaustive and deterministic; + // an approximate (vector/HNSW) result set is neither, so report the total as unavailable. + total = exact && !approximateResultSet ? scanned : null; } else if (boundRowFilter || typeof target.vectorFilter === 'function') { // An opaque row/vector filter shapes the result but isn't reflected in the index/condition // estimate; guessing would both mislead and disclose cardinality the filter hides. @@ -3795,7 +3807,7 @@ export function makeTable(options) { total = offset + page.length; } page.recordCount = total; - page.recordCountExact = wantExact && exact; + page.recordCountExact = wantExact && exact && !approximateResultSet; page.selectApplied = true; page.getColumns = getColumns; return page; diff --git a/unitTests/resources/queryCountVector.test.js b/unitTests/resources/queryCountVector.test.js new file mode 100644 index 0000000000..f698368eba --- /dev/null +++ b/unitTests/resources/queryCountVector.test.js @@ -0,0 +1,59 @@ +require('../testUtils'); +const assert = require('node:assert'); +const { setupTestDBPath } = require('../testUtils'); +const { table } = require('#src/resources/databases'); +const { setMainIsWorker } = require('#js/server/threads/manageThreads'); + +// Regression for the `Prefer: count=exact` pagination path on vector/HNSW-sorted queries. An HNSW sort +// returns a bounded, approximate candidate set whose size is chosen from the requested page (minResults = +// offset + limit), so the number of rows drained is NOT the true match count and must never be advertised +// as `count=exact`. The count path reports the total as unavailable (recordCount null, recordCountExact +// false) for such queries instead of a page-size-dependent number. +describe('Table.search count on vector/HNSW-sorted queries (approximate totals)', () => { + if (process.env.HARPER_STORAGE_ENGINE === 'lmdb') return; // HNSW is a RocksDB-only custom index + let VectorCount; + const TARGET = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; + + before(async function () { + setupTestDBPath(); + setMainIsWorker(true); + VectorCount = table({ + table: 'VectorCountTable', + database: 'test', + attributes: [ + { name: 'id', isPrimaryKey: true }, + { name: 'vector', indexed: { type: 'HNSW', optimizeRouting: 0.6 }, type: 'Array' }, + ], + }); + let last; + for (let i = 0; i < 80; i++) { + const v = [i % 2, i % 3, i % 4, i % 5, i % 6, i % 7, i % 8, i % 9, i % 10, i % 11]; + last = VectorCount.put(i, { vector: v }); + } + await last; + }); + + const vectorSearch = (limit) => + VectorCount.search({ + sort: { attribute: 'vector', target: TARGET, distance: 'cosine' }, + select: ['id', '$distance'], + limit, + count: 'exact', + }); + + it('exact: a vector-sorted total is reported unavailable, not a page-size-dependent number', async function () { + const small = await vectorSearch(5); + const large = await vectorSearch(40); + + // Pages still materialize (the feature works); it's only the total that can't be trusted as exact. + assert.strictEqual(small.length, 5); + assert.ok(large.length > small.length, `expected a larger page, got ${large.length}`); + + // The total must be reported unavailable rather than advertising two different `count=exact` numbers + // (scanned tracks the requested page size for an approximate candidate set). + assert.strictEqual(small.recordCount, null, `small total ${small.recordCount} must be unavailable`); + assert.strictEqual(large.recordCount, null, `large total ${large.recordCount} must be unavailable`); + assert.strictEqual(small.recordCountExact, false); + assert.strictEqual(large.recordCountExact, false); + }); +}); From 088e8c141110fee3b9920b13fb05012d2d2a122d Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Mon, 31 Aug 2026 10:46:00 -0500 Subject: [PATCH 13/14] fix(rest): treat any custom-index (vector) traversal as an approximate count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Broadens the approximate-result detection from the previous fix: an HNSW `lt`/`le` threshold filter drives the same bounded, minResults-widened traversal as a vector sort (HierarchicalNavigableSmallWorld handles `lt`/`le` in the same switch as `sort`), and can be the driving condition with no `sort` clause at all — so `count=exact` over it advertised a page-size-dependent `scanned` as authoritative, the same defect the sort fix addressed, reached through a sibling comparator. Detection now walks the executing `conditions` (recursively, through OR groups) for any attribute backed by a custom index, plus the `vectorFilter` check. This is both broader (catches the threshold-filter path) and more precise than the sort-chain walk: a vector sort applied as in-memory post-ordering leaves no custom-index condition in `conditions`, so it correctly stays exact rather than being over-flagged. Regression test adds the `lt` threshold-filter case (no sort) at two page sizes; verified it reports `scanned` as exact under the old sort-only detection and unavailable under this one. Addresses the follow-up review blocker on the HNSW fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- resources/Table.ts | 27 ++++++++++++-------- unitTests/resources/queryCountVector.test.js | 18 +++++++++++++ 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 1d8cd9e256..2b77276181 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -3745,16 +3745,23 @@ export function makeTable(options) { const wantExact = target.count === 'exact'; const pageEnd = offset + pageLimit; const countStart = performance.now(); - // A vector/HNSW-driven result set — sorted by a custom index, or shaped by a vector filter — - // is a bounded, approximate candidate set whose size is chosen from `minResults` (offset + - // limit), so `scanned` tracks the requested page size, not the true match count. It is not an - // authoritative total: the same query at limit(5) vs limit(200) would otherwise advertise two - // different `count=exact` totals. Report the total as unavailable instead (mirroring how the - // estimated branch below already bails to null for a vector/row filter). - let approximateResultSet = typeof target.vectorFilter === 'function'; - for (let order = sort; !approximateResultSet && order; order = order.next) { - if (typeof order.attribute === 'string' && indices[order.attribute]?.customIndex) approximateResultSet = true; - } + // A custom-index (vector/HNSW) traversal returns a bounded, approximate candidate set whose size is + // chosen from `minResults` (offset + limit), so `scanned` over it tracks the requested page size, not + // the true match count — the same query at limit(5) vs limit(200) would otherwise advertise two + // different `count=exact` totals. Any query whose execution touches a custom index is affected: a + // custom-index sort (its aligned pseudo-condition lands in `conditions`), a custom-index threshold + // filter (an HNSW `lt`/`le` is the same minResults-widened traversal as a sort), or an opaque vector + // filter. Report the total as unavailable for those rather than advertising it as count=exact + // (mirroring how the estimated branch below bails to null for an opaque row/vector filter). A vector + // sort applied as in-memory post-ordering leaves no custom-index condition here and stays exact. + const touchesCustomIndex = (conds: any[]): boolean => + conds.some((c: any) => { + if (!c) return false; + if (c.conditions) return touchesCustomIndex(c.conditions); + const attr = Array.isArray(c.attribute) ? c.attribute[0] : (c.attribute ?? c[0]); + return typeof attr === 'string' && Boolean(indices[attr]?.customIndex); + }); + const approximateResultSet = typeof target.vectorFilter === 'function' || touchesCustomIndex(conditions); return (async () => { const page: any = []; let scanned = 0; diff --git a/unitTests/resources/queryCountVector.test.js b/unitTests/resources/queryCountVector.test.js index f698368eba..b28fed3c23 100644 --- a/unitTests/resources/queryCountVector.test.js +++ b/unitTests/resources/queryCountVector.test.js @@ -56,4 +56,22 @@ describe('Table.search count on vector/HNSW-sorted queries (approximate totals)' assert.strictEqual(small.recordCountExact, false); assert.strictEqual(large.recordCountExact, false); }); + + it('exact: a vector threshold filter (lt/le, no sort) is also reported unavailable', async function () { + // An HNSW `lt`/`le` threshold query drives the same bounded, minResults-widened traversal as a + // vector sort — no `sort` clause needed — so it must be treated as approximate too. + const thresholdSearch = (limit) => + VectorCount.search({ + conditions: [{ attribute: 'vector', comparator: 'lt', value: 0.5, target: TARGET, distance: 'cosine' }], + select: ['id', '$distance'], + limit, + count: 'exact', + }); + const small = await thresholdSearch(5); + const large = await thresholdSearch(40); + assert.strictEqual(small.recordCount, null, `small total ${small.recordCount} must be unavailable`); + assert.strictEqual(large.recordCount, null, `large total ${large.recordCount} must be unavailable`); + assert.strictEqual(small.recordCountExact, false); + assert.strictEqual(large.recordCountExact, false); + }); }); From d7d4356b60c986164a298c21d8e87e9c44f00bc0 Mon Sep 17 00:00:00 2001 From: Chris Barber Date: Mon, 31 Aug 2026 11:44:18 -0500 Subject: [PATCH 14/14] perf(rest): keep the exact-count drain from blocking the event loop; harden Prefer parse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-model review (Gemini + Cursor-Grok) findings on the count path: - The exact-count drain iterated the store's async iterator with no macrotask yield. On a store whose iterator settles synchronously (the common indexed-scan case) a large `count=exact` scan ran as one uninterrupted microtask burst, blocking the event loop for up to the whole MAX_EXACT_COUNT_MS budget and starving concurrent requests. Yield to the macrotask queue every COUNT_YIELD_INTERVAL rows so I/O and other requests keep progressing — covering the page-window scan too, not just the tail past it. - For an approximate (vector/HNSW) result set, `count=exact` now stops at the page window instead of draining the tail: the total is reported unavailable anyway, so the tail work produced a number that was never published. - REST Prefer parse hardened against malformed input: optional-chain `httpOptions` (a programmatic mount may pass none) and String()-coerce the preference value before lower-casing (a bare `Prefer: count` with no `=` yields a non-string). Not changed — an unadjudicated Cursor-Grok "async allowRead breaks count" blocker was investigated and did NOT reproduce: driving `get()` with an async allowRead and `Prefer: count=` returns a proper page array (recordCount intact, iterates cleanly); the Table-level async-authorization branch isn't reached for the count path (auth resolves at the Resource layer first). No fix shipped for a non-issue. The 8 pre-existing resources-suite failures (transaction-log/snapshot/audit/reload) reproduce identically on a clean tree and are unrelated to this change. Co-Authored-By: Claude Opus 4.8 (1M context) --- resources/Table.ts | 12 +++++++++++- server/REST.ts | 6 ++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/resources/Table.ts b/resources/Table.ts index 2b77276181..b905ff55a3 100644 --- a/resources/Table.ts +++ b/resources/Table.ts @@ -157,6 +157,9 @@ const MAX_EXACT_COUNT_MS = 1_000; // not a finite, non-negative integer, e.g. `limit(Infinity)`/`limit(foo)`) falls through to the normal // streaming path with no count, so a count request can't be coerced into buffering an unbounded page. const MAX_COUNT_PAGE = 10_000; +// How often the exact-count drain yields to the macrotask queue (must be a power of two for the bit-mask +// check). Keeps a large scan from monopolizing the event loop without adding a yield per row. +const COUNT_YIELD_INTERVAL = 2_048; envMngr.initSync(); const LMDB_PREFETCH_WRITES = envMngr.get(CONFIG_PARAMS.STORAGE_PREFETCHWRITES); const LOCK_TIMEOUT = 10000; @@ -3770,10 +3773,17 @@ export function makeTable(options) { for await (const record of results) { if (scanned >= offset && scanned < pageEnd) page.push(record); scanned++; + // A store whose async iterator settles synchronously (the common indexed-scan case) would + // otherwise let this drain spin as one uninterrupted microtask run, blocking the event loop + // for the whole count. Yield to the macrotask queue periodically so concurrent requests and + // I/O still make progress during a large exact scan. + if ((scanned & (COUNT_YIELD_INTERVAL - 1)) === 0) await new Promise((resolve) => setImmediate(resolve)); // The page window [offset, pageEnd) is always collected in full first — the guardrail // only ever abandons the running TOTAL, never truncates the page body. if (scanned >= pageEnd) { - if (!wantExact) break; // `estimated` needs nothing past the page + // `estimated` needs nothing past the page; an approximate (vector) exact total is going to + // be reported unavailable anyway, so don't drain its tail for a number we won't publish. + if (!wantExact || approximateResultSet) break; // `exact` keeps counting the tail, bounded by a row cap AND a time budget so a // large match set can't turn a bounded page fetch into an unbounded scan. if (scanned > MAX_EXACT_COUNT_SCAN || performance.now() - countStart > MAX_EXACT_COUNT_MS) { diff --git a/server/REST.ts b/server/REST.ts index 6ce1435807..e9b76af516 100644 --- a/server/REST.ts +++ b/server/REST.ts @@ -208,9 +208,11 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt // `rest: { exactCount: true }` (default off); count=exact is otherwise served as a cheap // estimate. Estimated is always available. Accept a string `"true"` too, since not every // config source coerces to a boolean. - const exactEnabled = (httpOptions as any).exactCount === true || (httpOptions as any).exactCount === 'true'; + const exactEnabled = (httpOptions as any)?.exactCount === true || (httpOptions as any)?.exactCount === 'true'; for (const pref of parseHeaderValue(prefer as any)) { - const mode = (pref?.value as string | undefined)?.toLowerCase(); + // The header parser can hand back a non-string value (e.g. a bare `Prefer: count` with no + // `=`), so coerce before lower-casing rather than calling `.toLowerCase()` on a boolean. + const mode = String(pref?.value ?? '').toLowerCase(); if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) { // A count=exact request on a mount that hasn't opted in is downgraded to estimated, // signaled back to the client via Preference-Applied.