From ef3e3a4c834d82fe27c50fb0631f46a38bcb60e0 Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:18:08 -0400 Subject: [PATCH 1/7] feat: resolve entity datasets through the STAC catalog Entity datasets could only be opened by root CID, so reaching GHCND meant knowing a CID the catalog already held. The catalog marks which kind an item is with `dclimate:layout`, but nothing read it. Adds `client.loadEntities({ request: { collection, dataset } })`, the entity counterpart to `loadDataset`, reusing the same `resolveDatasetDetails` lookup. Separate method rather than a layout branch inside `loadDataset`: the two return different types with different query surfaces -- `EntityDataset` has no `point()` and its `nearest()` is async and can find nothing -- so folding them together would widen `loadDataset`'s return to a union and make every existing gridded caller narrow before calling a Zarr method. The layout guard only rejects an item that positively declares itself something else, so items published before the convention still open. Metadata carries `commitId`/`streamId` through, so a caller can re-resolve the exact snapshot a query ran against rather than whatever is newest later. `columnKey` now defaults to upper-casing, which is what this catalog's datasets publish; without it `elements("TMAX")` is unknown on a dataset every document describes that way. `entities.load({ cid })` is unchanged and stays the way to pin an exact snapshot. Verified against the live catalog: noaa_ghcnd resolves and opens in ~830ms, findNearestEntity in ~224ms, and a 10-day TMAX series returns 10 rows. --- README.md | 27 ++++-- src/client.ts | 127 ++++++++++++++++++++++++++- src/constants.ts | 6 ++ src/entities/entities-client.ts | 11 ++- src/stac/stac-catalog.ts | 10 +++ src/types.ts | 34 ++++++++ tests/entities.test.ts | 5 +- tests/load-entities.test.ts | 148 ++++++++++++++++++++++++++++++++ 8 files changed, 353 insertions(+), 15 deletions(-) create mode 100644 tests/load-entities.test.ts diff --git a/README.md b/README.md index 66bd204..f8ef8cd 100644 --- a/README.md +++ b/README.md @@ -83,15 +83,23 @@ degrees, ISO timestamps, chained selections. in GHCND, a buoy in NDBC. ```typescript -// `columnKey` maps schema field names to the column names queries use. It is a -// property of the dataset's publishing profile, not of the stored blocks, so -// the caller states it: GHCND stores `tmax` and publishes `TMAX`. Without it, -// column names default to the schema's own (lowercase, for GHCND). -const entities = await client.entities.load({ - cid: "bafyr4i...", - columnKey: (field) => field.name.toUpperCase(), // GHCND's mapping +// Addressed through the STAC catalog, like `loadDataset`. Separate from it +// because the two return different types: `EntityDataset` has no `point()`, and +// its `nearest()` is async and can find nothing. +const [entities, metadata] = await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, }); +// `metadata.commitId` identifies the snapshot this read ran against, so the +// same one can be re-resolved later instead of whatever is newest then. + +// To pin an exact snapshot, or to read a dataset that is not in the catalog, +// `client.entities.load({ cid })` remains available. It takes `columnKey`, +// which maps schema field names to published column names -- a property of the +// dataset's profile, not the stored blocks: GHCND stores `tmax`, publishes +// `TMAX`. `loadEntities` defaults it to upper case, which is what this catalog +// publishes; the direct form defaults to the schema's own names. + // Every entity, with position and coverage window. for (const e of await entities.listEntities()) { console.log(e.entityId, e.latitude, e.longitude, e.start, e.end); @@ -167,8 +175,9 @@ const rows = await range.rows(); // may be empty *because* of the gap above ``` Reads go over the IPFS HTTP gateway, so no local daemon is required and the same -code runs in a browser. Resolution is by CID for now; STAC catalog support will -follow. +code runs in a browser. Datasets resolve through the STAC catalog via +`loadEntities`; `entities.load({ cid })` stays available for pinning an exact +snapshot. ### Dataset version history diff --git a/src/client.ts b/src/client.ts index a68c1ac..19c0285 100644 --- a/src/client.ts +++ b/src/client.ts @@ -6,10 +6,16 @@ import { DatasetRequest, DatasetVersionRequest, DatasetVersionsRequest, + EntityDatasetRequest, GeoSelectionOptions, LoadDatasetOptions, + LoadEntitiesOptions, } from "./types.js"; -import { DEFAULT_IPFS_GATEWAY } from "./constants.js"; +import { DEFAULT_IPFS_GATEWAY, ENTITY_DATASET_LAYOUT } from "./constants.js"; +// Type-only, like `TableField` in types.ts: naming `EntityDataset` as a return +// type must not statically pull tabular's reader into every consumer's bundle, +// which is the whole reason `entities.load` imports it dynamically. +import type { EntityDataset, TableField } from "@dclimate/tabular/reader"; import { openDatasetFromCid, IpfsElements } from "./ipfs/open-dataset.js"; import { DatasetNotFoundError, @@ -102,6 +108,16 @@ function resolveZarrSelection( return {}; } +/** + * Published column names are upper case across this catalog's entity datasets + * while their schema fields are lower case (`tmax` stored, `TMAX` published). + * Hoisted to module scope rather than inlined so every call shares one function + * identity -- the reader keys nothing on it today, but a per-call closure would + * be a needless difference if it ever did. + */ +const defaultEntityColumnKey = (field: TableField): string => + field.name.toUpperCase(); + export class DClimateClient { private gatewayUrl: string; private stacServerUrl: string | null; @@ -277,6 +293,115 @@ export class DClimateClient { return this.entitiesClient; } + /** + * Open an entity (point-observation) dataset by catalog name. + * + * The entity counterpart to `loadDataset`, and deliberately a separate method + * rather than a layout branch inside it. The two return different types with + * different query surfaces -- `EntityDataset` has no `point()`, its `nearest()` + * is async and can find nothing -- so folding them together would widen + * `loadDataset`'s return to a union and make every existing gridded caller + * narrow before it could call a Zarr method. Callers know which kind they + * want; the entry point says so. + * + * Resolution is the same STAC lookup `loadDataset` performs, so a dataset is + * addressed the same way here as anywhere else in the library, and the release + * metadata comes back alongside it. That matters more for entity data than for + * gridded: `commitId` and `streamId` identify the exact snapshot a query ran + * against, which is what lets a caller re-resolve it later rather than + * silently getting whatever is newest. + * + * @throws {DatasetNotFoundError} if the resolved item is not tabular -- a + * gridded collection named here is a caller mistake worth reporting in terms + * of the fix, not a manifest parse failure deep inside the reader. + */ + async loadEntities({ + request, + options = {}, + }: { + request: EntityDatasetRequest; + options?: LoadEntitiesOptions; + }): Promise<[EntityDataset, DatasetMetadata]> { + const gatewayUrl = options.gatewayUrl ?? this.gatewayUrl; + + const resolved = await this.resolveDatasetDetails( + { + collection: request.collection, + dataset: request.dataset, + ...(request.variant ? { variant: request.variant } : {}), + ...(request.organization + ? { organization: request.organization } + : {}), + }, + gatewayUrl + ); + + // Checked rather than assumed: the catalog holds both kinds, and the CID of + // a Zarr store handed to the entity reader fails as a corrupt-dataset error + // that says nothing about the actual mistake. `layout` is absent on items + // published before the convention, so this only rejects an item that + // positively declares itself something else. + if (resolved.layout && resolved.layout !== ENTITY_DATASET_LAYOUT) { + throw new DatasetNotFoundError( + `${request.collection}/${request.dataset} is a '${resolved.layout}' dataset, not an entity dataset. Use loadDataset() for gridded data.` + ); + } + + const metadataVariant = resolved.variant || ""; + const dataset = await this.entities.load({ + cid: resolved.cid, + gatewayUrl, + // Published column names are a property of the dataset's profile, not of + // the stored blocks: the schema field is `tmax` and the published column + // is `TMAX`. The reader's default is the identity, so without this the + // published names are unreachable -- `elements("TMAX")` is an unknown + // column on a dataset every document describes that way. Every dataset + // this catalog serves is published upper case, so it is the default here + // rather than a rule each caller has to know; an override stays available + // for a profile that does otherwise. + columnKey: request.columnKey ?? defaultEntityColumnKey, + }); + + const pathParts = [ + resolved.collectionId, + resolved.dataset, + metadataVariant, + ].filter(Boolean); + + const metadata: DatasetMetadata = { + dataset: resolved.dataset, + collection: resolved.collectionId, + variant: metadataVariant, + path: pathParts.join("-"), + cid: resolved.cid, + source: "stac", + fetchedAt: new Date(), + ...(resolved.organizationId + ? { organization: resolved.organizationId } + : {}), + ...(resolved.versionsApi ? { versionsApi: resolved.versionsApi } : {}), + ...(resolved.provenanceApi + ? { provenanceApi: resolved.provenanceApi } + : {}), + ...(resolved.citationApi ? { citationApi: resolved.citationApi } : {}), + ...(resolved.streamId ? { streamId: resolved.streamId } : {}), + ...(resolved.commitId ? { commitId: resolved.commitId } : {}), + ...(resolved.versionLabel ? { versionLabel: resolved.versionLabel } : {}), + ...(resolved.isCitable !== undefined + ? { isCitable: resolved.isCitable } + : {}), + ...(resolved.retentionClass + ? { retentionClass: resolved.retentionClass } + : {}), + }; + + if (!metadata.organization && metadata.collection?.includes("_")) { + metadata.organization = metadata.collection.split("_")[0]; + } + + return [dataset, metadata]; + } + async loadDataset({ request, options = { diff --git a/src/constants.ts b/src/constants.ts index ef305fd..3853cb9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,2 +1,8 @@ export const DEFAULT_IPFS_GATEWAY = "https://ipfs-gateway.dclimate.net"; export const DEFAULT_SIREN_API_URL = "https://production-api-siren.dclimate.net/api"; + +/** + * The `dclimate:layout` value marking a STAC item as entity (point-observation) + * data rather than a gridded Zarr store. + */ +export const ENTITY_DATASET_LAYOUT = "tabular"; diff --git a/src/entities/entities-client.ts b/src/entities/entities-client.ts index 1f7c703..fef1da5 100644 --- a/src/entities/entities-client.ts +++ b/src/entities/entities-client.ts @@ -20,9 +20,12 @@ import { DatasetNotFoundError } from "../errors.js"; * ISO timestamps, chained selections. `EntityDataset` provides that surface, so * this namespace stays thin: resolve a root, hand back the dataset. * - * Resolution is by CID only for now. There is no STAC equivalent for entity - * data yet; when there is, `load` grows a `{ collection, dataset }` form - * alongside the CID and the rest of this file is unaffected. + * Resolution here is by CID only. Catalog resolution lives one level up, in + * `DClimateClient.loadEntities`, which resolves a collection/dataset through + * STAC and then calls this. Keeping it there rather than adding a second form + * here leaves this class with one job, and leaves the direct-CID path as the + * way to pin an exact snapshot rather than taking whatever the catalog calls + * latest. */ export interface EntitiesClientOptions { gatewayUrl: string; @@ -90,7 +93,7 @@ export class EntitiesClient { async load(request: LoadEntitiesRequest): Promise { if (!request.cid) { throw new DatasetNotFoundError( - "An entity dataset CID is required. Catalog resolution is not available yet." + "An entity dataset CID is required. To address a dataset by name instead, use client.loadEntities({ request: { collection, dataset } })." ); } diff --git a/src/stac/stac-catalog.ts b/src/stac/stac-catalog.ts index 6ebd347..697493a 100644 --- a/src/stac/stac-catalog.ts +++ b/src/stac/stac-catalog.ts @@ -150,6 +150,15 @@ function getBooleanProperty( } export interface StacReleaseMetadata { + /** + * Storage layout the item advertises: "tabular" for entity (point-observation) + * datasets, absent or anything else for the gridded Zarr datasets that were + * the only kind when this type was written. + * + * This is what lets `loadEntities` refuse a Zarr collection rather than + * handing its CID to a reader that will fail deep inside a manifest parse. + */ + layout?: string; versionsApi?: string; provenanceApi?: string; citationApi?: string; @@ -164,6 +173,7 @@ export function getStacReleaseMetadata( properties: Record | undefined ): StacReleaseMetadata { return { + layout: getStringProperty(properties, "dclimate:layout"), versionsApi: getStringProperty(properties, "dclimate:versions_api"), provenanceApi: getStringProperty(properties, "dclimate:provenance_api"), citationApi: getStringProperty(properties, "dclimate:citation_api"), diff --git a/src/types.ts b/src/types.ts index ff455b2..eea1629 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,9 @@ import type { IPFSELEMENTS_INTERFACE } from "@dclimate/jaxray"; import type { SirenOptions } from "./siren/types.js"; import type { VersionFilters } from "./versions/types.js"; +// Type-only: erased at compile time, so this does not statically chain the main +// entry to tabular's reader stack the way a value import would. +import type { TableField } from "@dclimate/tabular/reader"; export type IpfsElements = IPFSELEMENTS_INTERFACE; @@ -151,3 +154,34 @@ export interface DatasetObject { dims: string[]; sizes: Record; } + +/** + * Address an entity dataset in the STAC catalog. + * + * Deliberately not `DatasetRequest`: that type carries `cid` and `resolution`, + * and neither applies here. A direct CID is `client.entities.load({ cid })`, + * which stays the escape hatch for pinning an exact snapshot, and resolution is + * a property of a grid that entity data does not have. + */ +export interface EntityDatasetRequest { + /** Catalog collection id, e.g. "noaa_ghcnd". */ + collection: string; + /** Dataset id within the collection, e.g. "station_observations". */ + dataset: string; + /** Variant to select; defaults to the catalog's preferred one. */ + variant?: string; + /** Organization id, when the collection is not already prefixed with it. */ + organization?: string; + /** + * Override how schema fields map to published column names. + * + * Defaults to upper-casing, which is what every dataset this catalog serves + * publishes. Pass this only for a profile that does otherwise. + */ + columnKey?: (field: TableField) => string; +} + +export interface LoadEntitiesOptions { + /** Override the client's IPFS gateway for this dataset only. */ + gatewayUrl?: string; +} diff --git a/tests/entities.test.ts b/tests/entities.test.ts index b1527c5..45ff55a 100644 --- a/tests/entities.test.ts +++ b/tests/entities.test.ts @@ -52,7 +52,10 @@ describe("client.entities", () => { expect(client.entities).toBeDefined(); }); - it("requires a CID until catalog resolution exists", async () => { + // `client.loadEntities({ collection, dataset })` is the catalog-addressed + // entry point; `entities.load` stays the direct-CID escape hatch and still + // requires one. + it("requires a CID when addressed directly rather than by catalog", async () => { const client = new DClimateClient({ gatewayUrl: "http://127.0.0.1:8080" }); await expect( client.entities.load({ cid: "" }) diff --git a/tests/load-entities.test.ts b/tests/load-entities.test.ts new file mode 100644 index 0000000..fbda071 --- /dev/null +++ b/tests/load-entities.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it, vi } from "vitest"; +import { DClimateClient } from "../src/index.js"; +import { DatasetNotFoundError } from "../src/errors.js"; + +/** + * `loadEntities` is the entity counterpart to `loadDataset`: it resolves a + * collection/dataset through STAC and opens the result as an `EntityDataset`. + * + * These tests stub `resolveDatasetDetails` rather than the network. What is + * being tested is the dispatch this method adds -- the layout guard, the + * `columnKey` default, and the metadata it assembles -- not STAC resolution, + * which `stac-server.test.ts` already covers against the live server. + */ +const resolved = (over: Record = {}) => ({ + cid: "bafyr4ieoihgvnl5rvu6eh2fqduapjtz7wjp3e7kdtfxjospmavi5lgkoq4", + collectionId: "noaa_ghcnd", + dataset: "station_observations", + variant: "default", + zarrResolutions: [], + layout: "tabular", + commitId: "k1commit", + streamId: "kjstream", + versionLabel: "2026-08-26", + ...over, +}); + +const stubResolution = (client: DClimateClient, over?: Record) => + vi + .spyOn( + client as unknown as { + resolveDatasetDetails: (...a: unknown[]) => unknown; + }, + "resolveDatasetDetails" + ) + .mockResolvedValue(resolved(over)); + +describe("client.loadEntities", () => { + it("refuses a gridded dataset instead of handing its CID to the entity reader", async () => { + // The failure mode this prevents: a Zarr CID opened as an entity dataset + // dies inside a manifest parse as a corruption error, which sends the + // caller looking at the publisher rather than at their own call. + const client = new DClimateClient(); + stubResolution(client, { layout: "zarr" }); + + await expect( + client.loadEntities({ + request: { collection: "ecmwf_era5", dataset: "reanalysis" }, + }) + ).rejects.toThrow(/not an entity dataset.*loadDataset/s); + await expect( + client.loadEntities({ + request: { collection: "ecmwf_era5", dataset: "reanalysis" }, + }) + ).rejects.toThrow(DatasetNotFoundError); + }); + + it("accepts an item published before the layout convention", async () => { + // Absent is not "some other layout": items predating `dclimate:layout` + // should still open rather than being rejected for a field they cannot + // have carried. + const client = new DClimateClient(); + stubResolution(client, { layout: undefined }); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + }); + expect(load).toHaveBeenCalledOnce(); + }); + + it("defaults columnKey to the upper-case published names", async () => { + // Without this the reader defaults to the schema's own lower-case field + // names and `TMAX` -- the name every GHCND document uses -- is unknown. + const client = new DClimateClient(); + stubResolution(client); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + }); + + const { columnKey } = load.mock.calls[0]![0]!; + expect(columnKey?.({ name: "tmax" } as never)).toBe("TMAX"); + }); + + it("lets a caller override columnKey for a profile that differs", async () => { + const client = new DClimateClient(); + stubResolution(client); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { + collection: "noaa_ndbc", + dataset: "buoy_observations", + columnKey: (field) => field.name, + }, + }); + + const { columnKey } = load.mock.calls[0]![0]!; + expect(columnKey?.({ name: "SwH" } as never)).toBe("SwH"); + }); + + it("returns the snapshot identity a caller needs to re-resolve this exact read", async () => { + // The reason entity metadata carries these at all: a settlement or citation + // has to be able to say which snapshot it ran against, not just "whatever + // was newest that day". + const client = new DClimateClient(); + stubResolution(client); + vi.spyOn(client.entities, "load").mockResolvedValue({} as never); + + const [, metadata] = await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + }); + + expect(metadata).toMatchObject({ + collection: "noaa_ghcnd", + dataset: "station_observations", + variant: "default", + organization: "noaa", + source: "stac", + commitId: "k1commit", + streamId: "kjstream", + versionLabel: "2026-08-26", + }); + expect(metadata.path).toBe("noaa_ghcnd-station_observations-default"); + }); + + it("passes a per-request gateway through to the reader", async () => { + const client = new DClimateClient({ gatewayUrl: "https://default.example" }); + stubResolution(client); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + options: { gatewayUrl: "https://override.example" }, + }); + + expect(load.mock.calls[0]![0]!.gatewayUrl).toBe("https://override.example"); + }); +}); From 8b68e0e5ba1ab50e91fff21e1923e396eef539e9 Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:24:35 -0400 Subject: [PATCH 2/7] fix: version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9e540c5..3e23d42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dclimate/dclimate-client-js", - "version": "0.9.1", + "version": "0.10.0", "description": "JavaScript client for dClimate datasets using jaxray and IPFS stores", "type": "module", "engines": { From c842ebc552c77fdf59d3b0d11303844657a09d60 Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:34:45 -0400 Subject: [PATCH 3/7] fix: require a positive tabular layout before opening entities The guard accepted a missing `dclimate:layout` as an entity dataset. That is backwards: entity support postdates the field, so an item without it is a gridded one from before the convention -- there is no legacy entity item to accommodate. The permissive branch protected an empty set while admitting exactly the Zarr items the guard exists to catch, producing the misleading manifest failure it was written to prevent. All 64 items in the live catalogue carry the field (60 zarr, 4 tabular), so nothing published today relies on the absent case either way. Reported by automated review. --- src/client.ts | 15 ++++++++++----- tests/load-entities.test.ts | 20 ++++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/client.ts b/src/client.ts index 19c0285..abfb99c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -338,12 +338,17 @@ export class DClimateClient { // Checked rather than assumed: the catalog holds both kinds, and the CID of // a Zarr store handed to the entity reader fails as a corrupt-dataset error - // that says nothing about the actual mistake. `layout` is absent on items - // published before the convention, so this only rejects an item that - // positively declares itself something else. - if (resolved.layout && resolved.layout !== ENTITY_DATASET_LAYOUT) { + // that says nothing about the actual mistake. + // + // Positive match, not "absent or tabular". Entity support postdates + // `dclimate:layout`, so an item without the field is a gridded one from + // before the convention -- treating absence as permission would admit + // exactly the Zarr items this guard exists to catch, in order to accommodate + // legacy entity items that cannot exist. + if (resolved.layout !== ENTITY_DATASET_LAYOUT) { + const found = resolved.layout ?? "gridded"; throw new DatasetNotFoundError( - `${request.collection}/${request.dataset} is a '${resolved.layout}' dataset, not an entity dataset. Use loadDataset() for gridded data.` + `${request.collection}/${request.dataset} is a '${found}' dataset, not an entity dataset. Use loadDataset() for gridded data.` ); } diff --git a/tests/load-entities.test.ts b/tests/load-entities.test.ts index fbda071..80ce821 100644 --- a/tests/load-entities.test.ts +++ b/tests/load-entities.test.ts @@ -54,20 +54,24 @@ describe("client.loadEntities", () => { ).rejects.toThrow(DatasetNotFoundError); }); - it("accepts an item published before the layout convention", async () => { - // Absent is not "some other layout": items predating `dclimate:layout` - // should still open rather than being rejected for a field they cannot - // have carried. + it("refuses an item that declares no layout at all", async () => { + // Absence is not permission. Entity support postdates `dclimate:layout`, so + // an item without the field is a gridded one from before the convention -- + // there is no such thing as a legacy entity dataset to accommodate. Opening + // it would hand a Zarr CID to the entity reader, which is the exact + // misleading failure this guard exists to prevent. const client = new DClimateClient(); stubResolution(client, { layout: undefined }); const load = vi .spyOn(client.entities, "load") .mockResolvedValue({} as never); - await client.loadEntities({ - request: { collection: "noaa_ghcnd", dataset: "station_observations" }, - }); - expect(load).toHaveBeenCalledOnce(); + await expect( + client.loadEntities({ + request: { collection: "ecmwf_era5", dataset: "reanalysis" }, + }) + ).rejects.toThrow(/is a 'gridded' dataset.*loadDataset/s); + expect(load).not.toHaveBeenCalled(); }); it("defaults columnKey to the upper-case published names", async () => { From 57393af3da9f884229ce1d65450ca2c02bed5f30 Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:54:07 -0400 Subject: [PATCH 4/7] fix: paginate /collections so later collections keep their titles `listAvailableDatasetsFromStacServer` fetched `/collections` once and used whatever came back. The endpoint paginates with a default page size of 10 and the catalogue now publishes 14, so the request returned a well-formed but short list -- numberMatched 14, numberReturned 10 -- and the four collections past the first page arrived with no title or organization, since that endpoint is their only source. Item search still found them, so they appeared in the catalogue looking complete apart from the missing fields. That is what the list-datasets-parity test was reporting: prism_prism is the 11th collection, so the IPFS walker had a title for it and the STAC path did not. The failure was real, not a flaky live-data disagreement. Follows rel="next" rather than sending a larger limit: a limit only moves the cliff to whenever the catalogue outgrows it, silently, again. Reuses the page cap and repeat-detection the item search already uses, and refuses a next link that leaves the configured origin. Fixes the last failing test; the suite is now green. --- src/stac/stac-server.ts | 78 +++++++++++++++++++++++++++++------ tests/stac-server.test.ts | 85 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 12 deletions(-) diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index 016b414..e4ab789 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -472,6 +472,70 @@ function stripIpfsScheme(cid: string | undefined): string | undefined { * - Search pagination is bounded and repeated requests are detected to avoid * looping on malformed `next` links. */ +/** + * Fetch every page of `/collections`. + * + * The endpoint paginates and defaults to a page size smaller than the number of + * collections published, so a single unpaged request silently returns a prefix: + * `numberMatched` exceeds `numberReturned` and the tail is simply absent. That + * is invisible at the call site -- the response is a well-formed list, just a + * short one -- and the collections it drops lose the title and organization + * this endpoint is the only source of, leaving them to fall back to whatever the + * item search alone can say. + * + * Follows `rel="next"` rather than passing a large `limit`, because a limit only + * moves the cliff: it is a guess about how many collections will exist later, + * and the day the catalogue outgrows it the truncation returns silently. The + * link is what the server itself says comes next. + * + * Shares `MAX_STAC_SEARCH_PAGES` and the repeat-detection of the item search + * above: a server that returns a `next` pointing at the page just fetched would + * otherwise spin forever. + */ +async function fetchAllCollections( + resolvedServerUrl: string +): Promise { + const collections: StacServerCollectionsResponse["collections"] = []; + const seen = new Set(); + let url: string | undefined = `${resolvedServerUrl.replace( + /\/+$/, + "" + )}/collections`; + + for (let page = 0; page < MAX_STAC_SEARCH_PAGES; page++) { + if (!url) break; + if (seen.has(url)) { + throw new Error( + "STAC server /collections pagination repeated a request; results truncated" + ); + } + seen.add(url); + + const response: Response = await fetch(url, { redirect: "manual" }); + if (!response.ok) { + const text = await response.text(); + throw new Error( + `STAC server /collections error ${response.status}: ${text}` + ); + } + const body = (await response.json()) as StacServerCollectionsResponse & { + links?: Array<{ rel?: string; href?: string }>; + }; + collections.push(...(body.collections ?? [])); + + const next = body.links?.find((link) => link.rel === "next")?.href; + // Resolved against the current page so a relative `next` works, and dropped + // if it points at another origin -- following that would be an open redirect + // out of the configured server. + url = next ? new URL(next, url).toString() : undefined; + if (url && new URL(url).origin !== new URL(resolvedServerUrl).origin) { + url = undefined; + } + } + + return { collections }; +} + export async function listAvailableDatasetsFromStacServer( serverUrl: string = DEFAULT_STAC_SERVER_URL ): Promise { @@ -486,21 +550,11 @@ export async function listAvailableDatasetsFromStacServer( } return features; })(); - const [collectionsResp, searchFeatures] = await Promise.all([ - fetch(`${resolvedServerUrl.replace(/\/+$/, "")}/collections`, { - redirect: "manual", - }), + const [collectionsBody, searchFeatures] = await Promise.all([ + fetchAllCollections(resolvedServerUrl), searchFeaturesPromise, ]); - if (!collectionsResp.ok) { - const text = await collectionsResp.text(); - throw new Error( - `STAC server /collections error ${collectionsResp.status}: ${text}` - ); - } - const collectionsBody = (await collectionsResp.json()) as StacServerCollectionsResponse; - interface CollectionAccumulator { title?: string; organization?: string; diff --git a/tests/stac-server.test.ts b/tests/stac-server.test.ts index 5e5f7d5..dfc2c14 100644 --- a/tests/stac-server.test.ts +++ b/tests/stac-server.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { resolveCidFromStacServer, resolveDatasetCidFromStacServer, + listAvailableDatasetsFromStacServer, DEFAULT_STAC_SERVER_URL, } from "../src/stac/stac-server.js"; @@ -402,3 +403,87 @@ describe("STAC Server", () => { }); }); }); + +describe("listAvailableDatasetsFromStacServer pagination", () => { + it("follows rel=next instead of keeping only the first page", async () => { + // The failure this guards: `/collections` paginates, and a single unpaged + // request returns a well-formed but short list. The collections past the + // first page are not missing anything obvious -- they simply arrive with no + // title or organization, because this endpoint is the only source of both. + // That surfaced as a parity test failing on the 11th collection of 14. + const pages: Record = { + "https://stac.example/collections": { + collections: [{ id: "a_one", title: "One" }], + links: [{ rel: "next", href: "https://stac.example/collections?offset=1" }], + numberMatched: 2, + numberReturned: 1, + }, + "https://stac.example/collections?offset=1": { + collections: [{ id: "b_two", title: "Two" }], + links: [], + numberMatched: 2, + numberReturned: 1, + }, + }; + const seen: string[] = []; + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input instanceof Request ? input.url : input); + seen.push(url); + if (url in pages) { + return new Response(JSON.stringify(pages[url]), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + // The concurrent item search; empty is enough to reach the assertions. + return new Response(JSON.stringify({ features: [], links: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }); + + try { + await listAvailableDatasetsFromStacServer("https://stac.example"); + expect(seen).toContain("https://stac.example/collections"); + expect(seen).toContain("https://stac.example/collections?offset=1"); + } finally { + fetchMock.mockRestore(); + } + }); + + it("stops rather than following rel=next to another origin", async () => { + // A `next` pointing off-origin would walk the client out of the server it + // was configured with; stopping loses a page, following loses the boundary. + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input instanceof Request ? input.url : input); + if (url === "https://stac.example/collections") { + return new Response( + JSON.stringify({ + collections: [{ id: "a_one", title: "One" }], + links: [{ rel: "next", href: "https://evil.example/collections" }], + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + } + if (url.startsWith("https://evil.example")) { + throw new Error(`followed next off-origin: ${url}`); + } + return new Response(JSON.stringify({ features: [], links: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }); + + try { + await expect( + listAvailableDatasetsFromStacServer("https://stac.example") + ).resolves.toBeDefined(); + } finally { + fetchMock.mockRestore(); + } + }); +}); From 5ab41b7a7b2ea89324982adcb3b88a09422664f6 Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:17:20 -0400 Subject: [PATCH 5/7] fix: do not guess a columnKey for entity datasets `loadEntities` defaulted `columnKey` to upper-casing. That was a guess at each dataset's publishing profile dressed up as a catalogue-wide convention, and it is only right by coincidence: GHCND does publish upper case, and NDBC, SCAN and SNOTEL happen to already be upper case. NDBC's `.spec` feed publishes `SwH`, `SwP` and `STEEPNESS`, so the day that dataset is catalogued the default would silently respell its columns -- the exact renaming the ETL's own profile comments say must not happen. The justification was wrong too. `columnKey` renames columns; it does not gate access to them. Without one every column is still readable under the schema's own field names, which are what the dataset stores and so are never wrong -- `rows()` returns all of them either way. So the default bought a cosmetic match with the docs and paid for it with silent mis-naming. Nor is the profile derivable here: tabular deliberately stores the schema's names rather than the writer's rendering, precisely so a reader is not bound to one profile's casing (roots before 0.5.0 did the latter, and a cold reader's `entityColumns` returned names its own `query` then rejected), and the STAC item does not carry the mapping. Publishing it into STAC would fix this properly; until then it is the caller's to pass, as it already is for `entities.load`. Reported by automated review. --- README.md | 14 ++++++++----- src/client.ts | 41 +++++++++++++++++++------------------ tests/load-entities.test.ts | 14 +++++++------ 3 files changed, 38 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f8ef8cd..a9b7934 100644 --- a/README.md +++ b/README.md @@ -93,12 +93,16 @@ const [entities, metadata] = await client.loadEntities({ // `metadata.commitId` identifies the snapshot this read ran against, so the // same one can be re-resolved later instead of whatever is newest then. +// Columns are named by the schema's own field names. `columnKey` renames them +// to a dataset's published spelling -- GHCND stores `tmax` and publishes +// `TMAX` -- but it only changes the spelling: every column is readable either +// way. Neither the stored dataset nor STAC states which profile a dataset uses, +// so this is the caller's to pass rather than something to guess: +// +// loadEntities({ request: { ..., columnKey: (f) => f.name.toUpperCase() } }) +// // To pin an exact snapshot, or to read a dataset that is not in the catalog, -// `client.entities.load({ cid })` remains available. It takes `columnKey`, -// which maps schema field names to published column names -- a property of the -// dataset's profile, not the stored blocks: GHCND stores `tmax`, publishes -// `TMAX`. `loadEntities` defaults it to upper case, which is what this catalog -// publishes; the direct form defaults to the schema's own names. +// `client.entities.load({ cid })` remains available and takes the same option. // Every entity, with position and coverage window. for (const e of await entities.listEntities()) { diff --git a/src/client.ts b/src/client.ts index abfb99c..ad36137 100644 --- a/src/client.ts +++ b/src/client.ts @@ -15,7 +15,7 @@ import { DEFAULT_IPFS_GATEWAY, ENTITY_DATASET_LAYOUT } from "./constants.js"; // Type-only, like `TableField` in types.ts: naming `EntityDataset` as a return // type must not statically pull tabular's reader into every consumer's bundle, // which is the whole reason `entities.load` imports it dynamically. -import type { EntityDataset, TableField } from "@dclimate/tabular/reader"; +import type { EntityDataset } from "@dclimate/tabular/reader"; import { openDatasetFromCid, IpfsElements } from "./ipfs/open-dataset.js"; import { DatasetNotFoundError, @@ -108,16 +108,6 @@ function resolveZarrSelection( return {}; } -/** - * Published column names are upper case across this catalog's entity datasets - * while their schema fields are lower case (`tmax` stored, `TMAX` published). - * Hoisted to module scope rather than inlined so every call shares one function - * identity -- the reader keys nothing on it today, but a per-call closure would - * be a needless difference if it ever did. - */ -const defaultEntityColumnKey = (field: TableField): string => - field.name.toUpperCase(); - export class DClimateClient { private gatewayUrl: string; private stacServerUrl: string | null; @@ -311,6 +301,11 @@ export class DClimateClient { * against, which is what lets a caller re-resolve it later rather than * silently getting whatever is newest. * + * Columns are named by the schema's own field names unless `columnKey` is + * given. That mapping is a property of a dataset's publishing profile (GHCND + * stores `tmax` and publishes `TMAX`), and nothing readable from here states + * it, so it is the caller's to pass rather than this method's to guess. + * * @throws {DatasetNotFoundError} if the resolved item is not tabular -- a * gridded collection named here is a caller mistake worth reporting in terms * of the fix, not a manifest parse failure deep inside the reader. @@ -356,15 +351,21 @@ export class DClimateClient { const dataset = await this.entities.load({ cid: resolved.cid, gatewayUrl, - // Published column names are a property of the dataset's profile, not of - // the stored blocks: the schema field is `tmax` and the published column - // is `TMAX`. The reader's default is the identity, so without this the - // published names are unreachable -- `elements("TMAX")` is an unknown - // column on a dataset every document describes that way. Every dataset - // this catalog serves is published upper case, so it is the default here - // rather than a rule each caller has to know; an override stays available - // for a profile that does otherwise. - columnKey: request.columnKey ?? defaultEntityColumnKey, + // Forwarded only when given, so the reader's identity default stands. + // + // No default is supplied here, deliberately. `columnKey` renames columns; + // it does not gate access to them. Without one, every column is still + // readable under the schema's own field names -- which are what the + // dataset actually stores, so they are never wrong. Supplying a default + // would only be guessing at the spelling a dataset's publishing profile + // uses, and a wrong guess renames columns silently rather than failing. + // + // The profile is not derivable here either: tabular deliberately stores + // the schema's names rather than the writer's rendering, precisely so a + // reader is not bound to one profile's casing, and STAC does not carry + // the mapping. So a caller wanting the published spelling passes it, the + // same as with `entities.load({ cid })`. + ...(request.columnKey ? { columnKey: request.columnKey } : {}), }); const pathParts = [ diff --git a/tests/load-entities.test.ts b/tests/load-entities.test.ts index 80ce821..9cc23f2 100644 --- a/tests/load-entities.test.ts +++ b/tests/load-entities.test.ts @@ -74,9 +74,12 @@ describe("client.loadEntities", () => { expect(load).not.toHaveBeenCalled(); }); - it("defaults columnKey to the upper-case published names", async () => { - // Without this the reader defaults to the schema's own lower-case field - // names and `TMAX` -- the name every GHCND document uses -- is unknown. + it("supplies no columnKey of its own", async () => { + // `columnKey` renames columns; it does not gate access to them. Without one + // every column is still readable under the schema's own field names, which + // are what the dataset stores and so are never wrong. A default here would + // be a guess at a dataset's publishing profile -- right for GHCND, silently + // wrong for a profile like NDBC's `.spec` feed that publishes `SwH`. const client = new DClimateClient(); stubResolution(client); const load = vi @@ -87,11 +90,10 @@ describe("client.loadEntities", () => { request: { collection: "noaa_ghcnd", dataset: "station_observations" }, }); - const { columnKey } = load.mock.calls[0]![0]!; - expect(columnKey?.({ name: "tmax" } as never)).toBe("TMAX"); + expect(load.mock.calls[0]![0]!).not.toHaveProperty("columnKey"); }); - it("lets a caller override columnKey for a profile that differs", async () => { + it("forwards a caller's columnKey", async () => { const client = new DClimateClient(); stubResolution(client); const load = vi From 8ceba42fb3053fa15c5d3dc7497ccf208e9c139f Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:34:54 -0400 Subject: [PATCH 6/7] fix: bugs --- src/stac/stac-server.ts | 10 ++++ src/types.ts | 10 +++- .../stac-server-pagination.test.ts | 56 ++++++++++++++++++- 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index e4ab789..e4b66e8 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -533,6 +533,16 @@ async function fetchAllCollections( } } + // Reached only with a live `next` still in hand: the loop ran out of budget + // rather than out of pages. Returning here would hand back a catalogue that + // looks complete, and the collections missing from it would surface later as + // untitled or unknown datasets rather than as this failure. + if (url) { + throw new Error( + `STAC server /collections pagination exceeded ${MAX_STAC_SEARCH_PAGES} pages; results truncated` + ); + } + return { collections }; } diff --git a/src/types.ts b/src/types.ts index eea1629..c4b7a63 100644 --- a/src/types.ts +++ b/src/types.ts @@ -175,8 +175,14 @@ export interface EntityDatasetRequest { /** * Override how schema fields map to published column names. * - * Defaults to upper-casing, which is what every dataset this catalog serves - * publishes. Pass this only for a profile that does otherwise. + * Without this, columns keep the schema's own field names -- what the dataset + * actually stores, so never wrong, but not always what its docs call them. + * The mapping is a property of a dataset's publishing profile (GHCND stores + * `tmax` and publishes `TMAX`, NDBC preserves mixed case like `SwH`) and + * nothing readable from the catalog states it, so no default is guessed: + * a wrong guess would rename columns silently rather than fail. + * + * For GHCND, pass `(field) => field.name.toUpperCase()`. */ columnKey?: (field: TableField) => string; } diff --git a/tests/review-fixes/stac-server-pagination.test.ts b/tests/review-fixes/stac-server-pagination.test.ts index c817562..2ce1bfd 100644 --- a/tests/review-fixes/stac-server-pagination.test.ts +++ b/tests/review-fixes/stac-server-pagination.test.ts @@ -1,5 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { resolveCidFromStacServer } from "../../src/stac/stac-server.js"; +import { + listAvailableDatasetsFromStacServer, + resolveCidFromStacServer, +} from "../../src/stac/stac-server.js"; const serverUrl = "https://paginated-stac.test"; const collection = "bigcoll"; @@ -277,3 +280,54 @@ describe("resolveCidFromStacServer pagination", () => { expect(postBodies[1]).toEqual({ limit: 100, token }); }); }); + +describe("listAvailableDatasetsFromStacServer /collections pagination", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("surfaces truncation instead of silently returning a partial catalogue", async () => { + // Every /collections page advertises another next link, so the walk can + // never terminate naturally. Returning what it had would hand back a + // catalogue that looks complete, with the missing collections showing up + // later as untitled or unknown datasets rather than as this failure. + let collectionsRequests = 0; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.includes("/collections")) { + collectionsRequests += 1; + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + collections: [{ id: collection, title: "Big Collection" }], + links: [ + { + rel: "next", + href: `${serverUrl}/collections?page=${collectionsRequests + 1}`, + }, + ], + }), + text: async () => "", + } as Response; + } + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + type: "FeatureCollection", + features: [feature(0)], + links: [], + }), + text: async () => "", + } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + listAvailableDatasetsFromStacServer(serverUrl), + ).rejects.toThrow(/truncated/); + }); +}); From 51b13b87f07860485d51a8f56b8a19891a2e27fc Mon Sep 17 00:00:00 2001 From: TheGreatAlgo <37487508+TheGreatAlgo@users.noreply.github.com> Date: Thu, 27 Aug 2026 11:58:02 -0400 Subject: [PATCH 7/7] fix: this --- src/stac/stac-server.ts | 39 +++++++++++---- .../stac-server-pagination.test.ts | 48 +++++++++++++++++++ tests/stac-server.test.ts | 18 +++++-- 3 files changed, 94 insertions(+), 11 deletions(-) diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index e4b66e8..9719430 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -488,9 +488,12 @@ function stripIpfsScheme(cid: string | undefined): string | undefined { * and the day the catalogue outgrows it the truncation returns silently. The * link is what the server itself says comes next. * - * Shares `MAX_STAC_SEARCH_PAGES` and the repeat-detection of the item search - * above: a server that returns a `next` pointing at the page just fetched would - * otherwise spin forever. + * Shares `MAX_STAC_SEARCH_PAGES`, the repeat-detection, and the origin check of + * the item search above: a server that returns a `next` pointing at the page + * just fetched would otherwise spin forever, and one pointing off-origin is + * refused rather than followed. Every one of those ends the walk by throwing, + * because each leaves the catalogue incomplete -- and an incomplete catalogue + * that returns normally is indistinguishable from a complete one. */ async function fetchAllCollections( resolvedServerUrl: string @@ -524,13 +527,33 @@ async function fetchAllCollections( collections.push(...(body.collections ?? [])); const next = body.links?.find((link) => link.rel === "next")?.href; - // Resolved against the current page so a relative `next` works, and dropped - // if it points at another origin -- following that would be an open redirect - // out of the configured server. - url = next ? new URL(next, url).toString() : undefined; - if (url && new URL(url).origin !== new URL(resolvedServerUrl).origin) { + if (!next) { + // Cleared before leaving so a surviving `url` below means exactly one + // thing: the page budget ran out with another page still to fetch. A + // clean finish must not look like a truncated one. url = undefined; + break; } + + // Resolved against the current page so a relative `next` works. + const parsedNext: URL = new URL(next, url); + // Following an off-origin link would be an open redirect out of the + // configured server, so it is refused -- but refused loudly. Dropping it + // silently would end the walk exactly like a server that said it was + // finished, handing back a catalogue that looks complete while the + // collections past this point go missing. Same rule, and same reason, as + // the item search's pagination-link check. + if ( + normalizedOrigin(parsedNext.toString()) !== + normalizedOrigin(resolvedServerUrl) || + parsedNext.username !== "" || + parsedNext.password !== "" + ) { + throw new Error( + `STAC server /collections pagination link must use the configured server origin ${normalizedOrigin(resolvedServerUrl)}: ${sanitizedUrl(parsedNext)}` + ); + } + url = parsedNext.toString(); } // Reached only with a live `next` still in hand: the loop ran out of budget diff --git a/tests/review-fixes/stac-server-pagination.test.ts b/tests/review-fixes/stac-server-pagination.test.ts index 2ce1bfd..87bac4b 100644 --- a/tests/review-fixes/stac-server-pagination.test.ts +++ b/tests/review-fixes/stac-server-pagination.test.ts @@ -330,4 +330,52 @@ describe("listAvailableDatasetsFromStacServer /collections pagination", () => { listAvailableDatasetsFromStacServer(serverUrl), ).rejects.toThrow(/truncated/); }); + + it("follows a next link whose host carries a fully-qualified trailing dot", async () => { + // `https://host./x` addresses the same server as `https://host/x`, but + // `URL.origin` compares them unequal. Normalizing is what keeps an + // in-bounds link from being refused as if it left the server. + const seenUrls: string[] = []; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.includes("/collections")) { + seenUrls.push(url); + const first = !url.includes("page=2"); + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + collections: [{ id: collection, title: first ? "One" : "Two" }], + links: first + ? [ + { + rel: "next", + href: `https://paginated-stac.test./collections?page=2`, + }, + ] + : [], + }), + text: async () => "", + } as Response; + } + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + type: "FeatureCollection", + features: [feature(0)], + links: [], + }), + text: async () => "", + } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + listAvailableDatasetsFromStacServer(serverUrl), + ).resolves.toBeDefined(); + expect(seenUrls.some((url) => url.includes("page=2"))).toBe(true); + }); }); diff --git a/tests/stac-server.test.ts b/tests/stac-server.test.ts index dfc2c14..b1b42b6 100644 --- a/tests/stac-server.test.ts +++ b/tests/stac-server.test.ts @@ -453,9 +453,12 @@ describe("listAvailableDatasetsFromStacServer pagination", () => { } }); - it("stops rather than following rel=next to another origin", async () => { + it("throws rather than following rel=next to another origin", async () => { // A `next` pointing off-origin would walk the client out of the server it - // was configured with; stopping loses a page, following loses the boundary. + // was configured with, so it is never followed. But it is not dropped + // silently either: that would end the walk exactly like a server saying it + // was finished, returning a truncated catalogue that looks whole. Refusing + // the link keeps the boundary; throwing keeps the truncation visible. const fetchMock = vi .spyOn(globalThis, "fetch") .mockImplementation(async (input: RequestInfo | URL) => { @@ -481,7 +484,16 @@ describe("listAvailableDatasetsFromStacServer pagination", () => { try { await expect( listAvailableDatasetsFromStacServer("https://stac.example") - ).resolves.toBeDefined(); + ).rejects.toThrow(/configured server origin/); + // The boundary held: the off-origin href was reported, never fetched. + // (Had it been requested, the mock would have thrown its own error.) + expect( + fetchMock.mock.calls.some((call) => + String(call[0] instanceof Request ? call[0].url : call[0]).startsWith( + "https://evil.example" + ) + ) + ).toBe(false); } finally { fetchMock.mockRestore(); }