Measure k: count the page's own script calls on both sides of net offload (browser v1.22.0, plugin v0.65.0, console v0.13.0)

packages/browser/README.md

@@ -385,6 +385,32 @@ Two things worth checking before adding a rule: an attribute may be **load-beari
 `<astro-island>` on hydration, so stripping `ssr` would destroy the only marker distinguishing a
 healthy snapshot from an un-hydrated one. Strip what is inert, not what is merely non-visual.
 
+### What the page asks of its origin — the `subrequests` tally (v1.22.0)
+
+Every result posts a count of the same-origin requests the page made beyond the document, judged by
+whether a **shared cache** in front of the origin would have answered them (RFC 9111 rules):
+
+```
+subrequests: { sameOrigin, cacheable, uncacheable, unspecified, blocked }
+scriptsStripped: <postProcess.stripScripts>
+```
+
+`uncacheable` is the per-page factor the plugin's offload figure needs — the XHR/API calls a
+crawler that executes JavaScript would send to the origin when it runs the same page, which never
+pass through the plugin and so cannot be counted anywhere else (prerender-plugin#153). It is
+explicit-only: non-GET, an uncacheable status, `Set-Cookie`, `no-store` / `private` / `no-cache`, a
+zero max-age, `Vary: *`, or an expired `Expires`. `cacheable` is explicit positive freshness;
+`unspecified` had no freshness information at all and depends on the CDN's defaults, so it is
+reported and counted on neither side. `blocked` is the same-origin requests this fleet's block list
+aborted before any response — requests a crawler would make, of unknown class — the visible bound on
+the undercount; blocked images, fonts and media are left out of it, since a CDN caches those as a
+matter of course and they say nothing about k. `scriptsStripped` says whether the stored snapshot can make any of these calls when
+a crawler runs it: with scripts stripped the factor is a **saving** at serve time, not a cost.
+
+Cost: header checks on a response hook that already fires per response; no body reads, nothing
+awaited. The per-window stats line carries the totals as `subrequestsUncacheable` and
+`subrequestsUnspecified`.
+
 ## Custom renderer
 
 A renderer receives the Puppeteer `page` and the `RenderJob` and returns the serialized HTML (or

packages/browser/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@harperfast/prerender-browser",
-	"version": "1.21.0",
+	"version": "1.22.0",
 	"type": "module",
 	"description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.",
 	"keywords": [

packages/browser/src/RenderJob.ts

@@ -2,6 +2,7 @@ import { setTimeout as sleep } from 'timers/promises';
 import { request } from './external/http.js';
 import logger from './util/Logger.js';
 import { settings } from './settings.js';
+import type { SubrequestTally } from './subrequests.js';
 import { encode } from './util/encoder.js';
 import { getHostHealth, parseRetryAfter } from './HostHealth.js';
 import { renderPhaseOf } from './util/renderPhase.js';
@@ -66,6 +67,13 @@ type RenderAttempt = {
 	 * request produces no response.
 	 */
 	subresourceErrors?: number;
+	/**
+	 * Every same-origin response the page provoked beyond the document, by shared-cache verdict
+	 * (src/subrequests.ts). `uncacheable` is the per-page factor the plugin applies at serve time
+	 * to count the origin calls a script-running crawler's page-view makes — on both sides of the
+	 * offload ledger (HarperFast/prerender-plugin#153).
+	 */
+	subrequests?: SubrequestTally;
 };
 
 type OriginHttpResponse = {
@@ -223,6 +231,14 @@ export default class RenderJob {
 			redirectedTo: this.redirectedTo,
 			isIndexable: this.isIndexable,
 			structuredOffers: this.structuredOffers,
+			// The page's own origin traffic, and whether the snapshot being posted can still make it.
+			// Posted on EVERY result that had an attempt (a redirect or an error saw a partial tally
+			// and that is still what the page asked for), and `scriptsStripped` is this fleet's
+			// setting rather than a per-page fact — a snapshot stored with its scripts removed cannot
+			// hydrate anything when a crawler runs it, which is what turns the factor from a cost
+			// into a saving. An older plugin ignores both fields.
+			subrequests: this.latestAttempt?.subrequests,
+			scriptsStripped: settings.config.postProcess.stripScripts,
 			outcome: this.outcome,
 			// One slug for WHY there is no content (see the field doc). The redirect/error
 			// fallbacks are derived here so every no-content result carries a reason without

packages/browser/src/Worker.ts

@@ -109,6 +109,12 @@ export default class RenderWorker {
 			// cache is affected); `subresourceErrors` is the total refused assets (how badly).
 			rendersDegraded: 0,
 			subresourceErrors: 0,
+			// Same-origin requests the pages made that no shared cache would serve — the origin load a
+			// script-running crawler's page-view carries, summed over the window (RenderAttempt.subrequests).
+			// `subrequestsUnspecified` is the part with no freshness information at all, whose fate depends
+			// on the CDN's defaults — reported so the uncacheable figure can be read as the bound it is.
+			subrequestsUncacheable: 0,
+			subrequestsUnspecified: 0,
 			renderTimes: [] as number[],
 			// Per-phase wall-clock samples (ms), drained into percentiles by logStats. Attribute
 			// the render time to network-wait (navTtfb/navTotal) vs in-browser work (settle/postProcess).
@@ -269,6 +275,9 @@ export default class RenderWorker {
 				// render "succeeded" — treat it like a failure count, not a curiosity.
 				rendersDegraded: s.rendersDegraded,
 				subresourceErrors: s.subresourceErrors,
+				// Per-window totals; divide by `completed` for the per-page factor the plugin applies.
+				subrequestsUncacheable: s.subrequestsUncacheable,
+				subrequestsUnspecified: s.subrequestsUnspecified,
 				fromSitemap: s.fromSitemap,
 				failures: failuresTotal,
 				failuresByType: s.failures,
@@ -499,6 +508,10 @@ export default class RenderWorker {
 			this.stats.rendersDegraded++;
 			this.stats.subresourceErrors += attempt.subresourceErrors;
 		}
+		if (attempt?.subrequests) {
+			this.stats.subrequestsUncacheable += attempt.subrequests.uncacheable;
+			this.stats.subrequestsUnspecified += attempt.subrequests.unspecified;
+		}
 		const t = attempt?.timings;
 		if (t) {
 			if (t.navTtfb !== undefined) this.stats.navTtfb.push(t.navTtfb);

packages/browser/src/renderer.ts

@@ -2,6 +2,7 @@ import { Renderer } from './Worker.js';
 import type { RenderTimings } from './RenderJob.js';
 import { settings } from './settings.js';
 import { CACHE_REPLAY_HEADER, getResourceCache } from './ResourceCache.js';
+import { countsAsBlocked, emptyTally, tallySubresponse } from './subrequests.js';
 import type { PostProcessConfig } from './config.js';
 import { canonicalizeUrl, canonicalVerdict } from './util/url.js';
 import { markRenderPhase } from './util/renderPhase.js';
@@ -52,6 +53,12 @@ const renderer: Renderer = async (page, job) => {
 	const timings: RenderTimings = {};
 	if (job.latestAttempt) job.latestAttempt.timings = timings;
 	let navStart = 0;
+	// What the page asked of its own origin beyond the document, by shared-cache verdict — the
+	// per-page factor the offload figure downstream needs (src/subrequests.ts). Same in-place
+	// discipline as `timings`: the attempt holds the reference, so a partial tally survives an
+	// early return.
+	const subrequests = emptyTally();
+	if (job.latestAttempt) job.latestAttempt.subrequests = subrequests;
 
 	const blockedResourceTypes = new Set(config.block.resourceTypes);
 	const blockedUrlPatterns = config.block.urlPatterns;
@@ -117,10 +124,14 @@ const renderer: Renderer = async (page, job) => {
 				return;
 			}
 			if (isBlockedUrl(req.url())) {
+				// A same-origin request this fleet refuses to make is one a crawler WOULD make, of a
+				// class nobody can judge without a response — counted so the undercount is visible.
+				if (isSameOrigin(req.url()) && countsAsBlocked(req.resourceType())) subrequests.blocked++;
 				req.abort().catch(noop);
 				return;
 			}
 			if (blockedResourceTypes.has(req.resourceType())) {
+				if (isSameOrigin(req.url()) && countsAsBlocked(req.resourceType())) subrequests.blocked++;
 				// Stub blocked images (vs abort) so lazy-loaders keep their real src URLs.
 				if (config.block.stubImages && req.resourceType() === 'image') {
 					req.respond(STUB_IMAGE_RESPONSE).catch(noop);
@@ -178,16 +189,27 @@ const renderer: Renderer = async (page, job) => {
 				return;
 			}
 
+			const sameOrigin = isSameOrigin(res.url());
 			// A same-origin asset the origin refused while the document succeeded. Recorded on the
 			// attempt (mutated in place, like `timings`, so it survives an early return) and
 			// aggregated by the worker — a render whose scripts all 403 otherwise reports as a
 			// clean success.
-			if (res.status() >= 400 && isSameOrigin(res.url()) && job.latestAttempt) {
+			if (res.status() >= 400 && sameOrigin && job.latestAttempt) {
 				job.latestAttempt.subresourceErrors = (job.latestAttempt.subresourceErrors ?? 0) + 1;
 			}
 
-			if (!cache || !cache.isCacheableRequest(req)) return;
 			const resHeaders = res.headers();
+			// Every same-origin response the page provoked, judged by whether a shared cache in front
+			// of the origin would have answered it. Sub-frame documents count too: they are not the
+			// navigation, and a crawler's renderer loads them the same way. Header reads only — no
+			// body, no await — on a hook that already fires per response.
+			if (sameOrigin) {
+				tallySubresponse(subrequests, req.method(), res.status(), resHeaders, {
+					replayedFromOwnCache: Boolean(resHeaders[CACHE_REPLAY_HEADER]),
+				});
+			}
+
+			if (!cache || !cache.isCacheableRequest(req)) return;
 			// Skip responses we just synthesized from our own cache.
 			if (resHeaders[CACHE_REPLAY_HEADER]) return;
 			const policy = cache.getCachePolicy(res);

packages/browser/src/subrequests.ts

@@ -0,0 +1,133 @@
+/**
+ * What a page load asks of its own origin beyond the document — and how much of it a shared cache
+ * would have absorbed.
+ *
+ * WHY THIS IS MEASURED HERE. The prerender system's offload figure counts documents: bot requests
+ * the origin never saw, less the renders/probes/sitemap fetches this system made. It cannot see
+ * the other half of a page-view — the XHR/fetch calls the page's OWN scripts make once a crawler
+ * that executes JavaScript runs it — because those go crawler → CDN → origin and never pass
+ * through the plugin. But this process runs the same page in the same kind of browser, watches
+ * every request it makes, and reads every response's cache headers. So the per-page factor is
+ * measurable exactly once, here, and posted with the render for the plugin to apply at serve time
+ * (HarperFast/prerender-plugin#153). The arithmetic downstream needs one number per page — how
+ * many same-origin requests reach the origin whoever runs the page — which is the `uncacheable`
+ * count below.
+ *
+ * THE CLASSIFICATION IS ABOUT A SHARED CACHE, NOT OUR OWN. `ResourceCache.getCachePolicy` decides
+ * what THIS fleet may replay across renders and is deliberately narrow (scripts and stylesheets,
+ * private refused). The question here is what a CDN in front of the origin would serve without
+ * an origin round trip, for any request the page makes, so it follows RFC 9111's shared-cache
+ * rules and reports three verdicts rather than two:
+ *
+ *   uncacheable  — explicitly reaches the origin every time: a non-GET method, a status a cache
+ *                  may not store, `Set-Cookie`, `no-store` / `private` / `no-cache`, a zero max-age,
+ *                  `Vary: *`, or an `Expires` already in the past. Only this class is counted as
+ *                  origin load.
+ *   cacheable    — explicit positive freshness (`s-maxage`, `max-age`, or a future `Expires`).
+ *   unspecified  — no freshness information at all. A CDN may apply heuristic freshness or a
+ *                  configured default TTL, or may not; that is a deployment fact this process
+ *                  cannot see, so these are reported and counted on neither side.
+ *
+ * Nothing about the REQUEST's cookies or authorization is consulted: this fleet sends a bypass
+ * token the crawler would not, and a crawler's renderer starts with no cookies — the response is
+ * what decides shared cacheability, and `Set-Cookie` is the request-specific case it covers.
+ *
+ * Blocked requests are counted separately. `block.resourceTypes` / `block.urlPatterns` abort a
+ * request before any response exists, so a same-origin request this fleet refused to make is a
+ * request a crawler WOULD make whose class is unknown — the visible bound on the undercount. Static
+ * media (images, fonts, audio/video) is left out of that count: a fleet blocks those by the
+ * hundred, a CDN caches them as a matter of course, and counting them would make the bound read as
+ * an alarm about requests that say nothing about k. What remains — blocked scripts, XHR/fetch,
+ * documents, "other" — is exactly the class that might.
+ */
+
+export type SubrequestClass = 'uncacheable' | 'cacheable' | 'unspecified';
+
+export type SubrequestTally = {
+	/** Responses from the navigation origin, the document itself excluded. */
+	sameOrigin: number;
+	cacheable: number;
+	uncacheable: number;
+	unspecified: number;
+	/** Same-origin requests this fleet's block list aborted before a response existed — static media excluded. */
+	blocked: number;
+};
+
+// Resource types whose blocked requests are NOT counted as an unknown: static media a CDN caches.
+const STATIC_MEDIA = new Set(['image', 'font', 'media']);
+
+/** Does a blocked same-origin request of this resource type count toward the `blocked` bound? */
+export const countsAsBlocked = (resourceType: string): boolean => !STATIC_MEDIA.has(resourceType);
+
+export const emptyTally = (): SubrequestTally => ({
+	sameOrigin: 0,
+	cacheable: 0,
+	uncacheable: 0,
+	unspecified: 0,
+	blocked: 0,
+});
+
+// Status codes a cache may store without explicit freshness (RFC 9110 §15.1 "heuristically
+// cacheable"), plus 308. Anything else — every 5xx, 401/403, 429 — reached the origin and will
+// again.
+const CACHEABLE_STATUSES = new Set([200, 203, 204, 206, 300, 301, 308, 404, 405, 410, 414, 501]);
+
+// Lenient on purpose: the ABNF has no whitespace around `=` and no quotes on a delta-seconds value,
+// but origins emit both (`max-age = 60`, `max-age="60"`) and the caches this classifier stands in
+// for accept them. Strictness here would push a response a CDN happily caches into `unspecified`.
+const directiveSeconds = (cc: string, name: string): number | null => {
+	const m = cc.match(new RegExp(`(?:^|[,\\s])${name}\\s*=\\s*"?(\\d+)"?`));
+	return m ? parseInt(m[1], 10) : null;
+};
+
+/**
+ * Would a shared cache have answered this response without going to the origin?
+ *
+ * `headers` as puppeteer's `HTTPResponse.headers()` hands them: lower-cased keys, multi-valued
+ * headers joined. Pure, so the rules above are testable without a browser.
+ */
+export function classifySubresponse(method: string, status: number, headers: Record<string, string>): SubrequestClass {
+	const verb = method.toUpperCase();
+	if (verb !== 'GET' && verb !== 'HEAD') return 'uncacheable';
+	if (!CACHEABLE_STATUSES.has(status)) return 'uncacheable';
+	if (headers['set-cookie']) return 'uncacheable';
+
+	const cc = (headers['cache-control'] ?? '').toLowerCase();
+	if (/(?:^|[,\s])(?:no-store|private|no-cache)(?:$|[,\s=])/.test(cc)) return 'uncacheable';
+	if ((headers['vary'] ?? '').trim() === '*') return 'uncacheable';
+
+	// Shared caches honour s-maxage over max-age; a zero in either is "stale on arrival", i.e. a
+	// revalidation against the origin on every use — origin load, however the response is labelled.
+	const sMaxAge = directiveSeconds(cc, 's-maxage');
+	if (sMaxAge !== null) return sMaxAge > 0 ? 'cacheable' : 'uncacheable';
+	const maxAge = directiveSeconds(cc, 'max-age');
+	if (maxAge !== null) return maxAge > 0 ? 'cacheable' : 'uncacheable';
+
+	if (headers['expires'] !== undefined) {
+		// Measured against the origin's own clock when it says what time it is; an unparseable
+		// Expires is "already expired" by specification.
+		const expires = Date.parse(headers['expires']);
+		if (Number.isNaN(expires)) return 'uncacheable';
+		const dateHeader = headers['date'] !== undefined ? Date.parse(headers['date']) : NaN;
+		const now = Number.isNaN(dateHeader) ? Date.now() : dateHeader;
+		return expires > now ? 'cacheable' : 'uncacheable';
+	}
+
+	return 'unspecified';
+}
+
+/** Record one same-origin, non-navigation response in the tally. Mutates in place. */
+export function tallySubresponse(
+	tally: SubrequestTally,
+	method: string,
+	status: number,
+	headers: Record<string, string>,
+	{ replayedFromOwnCache = false } = {}
+): void {
+	tally.sameOrigin++;
+	// A response this fleet replayed from its own resource cache passed getCachePolicy, which is
+	// stricter than the shared-cache rules here — so it is cacheable by construction, and its
+	// replayed headers (some stripped) are not the evidence to re-judge it on.
+	const verdict = replayedFromOwnCache ? 'cacheable' : classifySubresponse(method, status, headers);
+	tally[verdict]++;
+}

packages/browser/test/jobResult.test.ts

@@ -124,3 +124,32 @@ test('a failed render posts outcome=error with the attempt error and derived rea
 		{ name: 'Error', message: 'Navigation timeout of 30000 ms exceeded' }
 	);
 });
+
+// The per-page origin factor (HarperFast/prerender-plugin#153). The plugin reads an ABSENT
+// `subrequests` as "this renderer predates the measurement", so a tally must ride every result
+// that had an attempt — including one that never reached content — and `scriptsStripped` must
+// say what this fleet does to the snapshot, since that is what turns the factor from a cost into
+// a saving at serve time.
+test('a render posts its same-origin subrequest tally and whether the snapshot keeps its scripts', async () => {
+	const job = makeJob();
+	const attempt = job.attemptStarted();
+	attempt.subrequests = { sameOrigin: 12, cacheable: 7, uncacheable: 4, unspecified: 1, blocked: 3 };
+	job.httpResponse = { statusCode: 200, headers: {} };
+	job.attemptEnded(undefined, '<html>ok</html>');
+
+	const meta = await send(job);
+	assert.deepEqual(meta.subrequests, { sameOrigin: 12, cacheable: 7, uncacheable: 4, unspecified: 1, blocked: 3 });
+	// The default config strips scripts; the field is the fleet's setting, posted as a boolean.
+	assert.equal(meta.scriptsStripped, true);
+});
+
+test('a result that never reached content still carries the partial tally', async () => {
+	const job = makeJob();
+	const attempt = job.attemptStarted();
+	attempt.subrequests = { sameOrigin: 2, cacheable: 1, uncacheable: 1, unspecified: 0, blocked: 0 };
+	job.attemptEnded(new Error('Navigation timeout of 30000 ms exceeded'), undefined);
+
+	const meta = await send(job);
+	assert.equal(meta.outcome, 'error');
+	assert.deepEqual(meta.subrequests, { sameOrigin: 2, cacheable: 1, uncacheable: 1, unspecified: 0, blocked: 0 });
+});