diff --git a/README.md b/README.md index 66bd204..a9b7934 100644 --- a/README.md +++ b/README.md @@ -83,15 +83,27 @@ degrees, ISO timestamps, chained selections. in GHCND, a buoy in NDBC. ```typescript -// `columnKey` maps schema field names to the column names queries use. It is a -// property of the dataset's publishing profile, not of the stored blocks, so -// the caller states it: GHCND stores `tmax` and publishes `TMAX`. Without it, -// column names default to the schema's own (lowercase, for GHCND). -const entities = await client.entities.load({ - cid: "bafyr4i...", - columnKey: (field) => field.name.toUpperCase(), // GHCND's mapping +// Addressed through the STAC catalog, like `loadDataset`. Separate from it +// because the two return different types: `EntityDataset` has no `point()`, and +// its `nearest()` is async and can find nothing. +const [entities, metadata] = await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, }); +// `metadata.commitId` identifies the snapshot this read ran against, so the +// same one can be re-resolved later instead of whatever is newest then. + +// Columns are named by the schema's own field names. `columnKey` renames them +// to a dataset's published spelling -- GHCND stores `tmax` and publishes +// `TMAX` -- but it only changes the spelling: every column is readable either +// way. Neither the stored dataset nor STAC states which profile a dataset uses, +// so this is the caller's to pass rather than something to guess: +// +// loadEntities({ request: { ..., columnKey: (f) => f.name.toUpperCase() } }) +// +// To pin an exact snapshot, or to read a dataset that is not in the catalog, +// `client.entities.load({ cid })` remains available and takes the same option. + // Every entity, with position and coverage window. for (const e of await entities.listEntities()) { console.log(e.entityId, e.latitude, e.longitude, e.start, e.end); @@ -167,8 +179,9 @@ const rows = await range.rows(); // may be empty *because* of the gap above ``` Reads go over the IPFS HTTP gateway, so no local daemon is required and the same -code runs in a browser. Resolution is by CID for now; STAC catalog support will -follow. +code runs in a browser. Datasets resolve through the STAC catalog via +`loadEntities`; `entities.load({ cid })` stays available for pinning an exact +snapshot. ### Dataset version history diff --git a/package.json b/package.json index 9e540c5..3e23d42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dclimate/dclimate-client-js", - "version": "0.9.1", + "version": "0.10.0", "description": "JavaScript client for dClimate datasets using jaxray and IPFS stores", "type": "module", "engines": { diff --git a/src/client.ts b/src/client.ts index a68c1ac..ad36137 100644 --- a/src/client.ts +++ b/src/client.ts @@ -6,10 +6,16 @@ import { DatasetRequest, DatasetVersionRequest, DatasetVersionsRequest, + EntityDatasetRequest, GeoSelectionOptions, LoadDatasetOptions, + LoadEntitiesOptions, } from "./types.js"; -import { DEFAULT_IPFS_GATEWAY } from "./constants.js"; +import { DEFAULT_IPFS_GATEWAY, ENTITY_DATASET_LAYOUT } from "./constants.js"; +// Type-only, like `TableField` in types.ts: naming `EntityDataset` as a return +// type must not statically pull tabular's reader into every consumer's bundle, +// which is the whole reason `entities.load` imports it dynamically. +import type { EntityDataset } from "@dclimate/tabular/reader"; import { openDatasetFromCid, IpfsElements } from "./ipfs/open-dataset.js"; import { DatasetNotFoundError, @@ -277,6 +283,131 @@ export class DClimateClient { return this.entitiesClient; } + /** + * Open an entity (point-observation) dataset by catalog name. + * + * The entity counterpart to `loadDataset`, and deliberately a separate method + * rather than a layout branch inside it. The two return different types with + * different query surfaces -- `EntityDataset` has no `point()`, its `nearest()` + * is async and can find nothing -- so folding them together would widen + * `loadDataset`'s return to a union and make every existing gridded caller + * narrow before it could call a Zarr method. Callers know which kind they + * want; the entry point says so. + * + * Resolution is the same STAC lookup `loadDataset` performs, so a dataset is + * addressed the same way here as anywhere else in the library, and the release + * metadata comes back alongside it. That matters more for entity data than for + * gridded: `commitId` and `streamId` identify the exact snapshot a query ran + * against, which is what lets a caller re-resolve it later rather than + * silently getting whatever is newest. + * + * Columns are named by the schema's own field names unless `columnKey` is + * given. That mapping is a property of a dataset's publishing profile (GHCND + * stores `tmax` and publishes `TMAX`), and nothing readable from here states + * it, so it is the caller's to pass rather than this method's to guess. + * + * @throws {DatasetNotFoundError} if the resolved item is not tabular -- a + * gridded collection named here is a caller mistake worth reporting in terms + * of the fix, not a manifest parse failure deep inside the reader. + */ + async loadEntities({ + request, + options = {}, + }: { + request: EntityDatasetRequest; + options?: LoadEntitiesOptions; + }): Promise<[EntityDataset, DatasetMetadata]> { + const gatewayUrl = options.gatewayUrl ?? this.gatewayUrl; + + const resolved = await this.resolveDatasetDetails( + { + collection: request.collection, + dataset: request.dataset, + ...(request.variant ? { variant: request.variant } : {}), + ...(request.organization + ? { organization: request.organization } + : {}), + }, + gatewayUrl + ); + + // Checked rather than assumed: the catalog holds both kinds, and the CID of + // a Zarr store handed to the entity reader fails as a corrupt-dataset error + // that says nothing about the actual mistake. + // + // Positive match, not "absent or tabular". Entity support postdates + // `dclimate:layout`, so an item without the field is a gridded one from + // before the convention -- treating absence as permission would admit + // exactly the Zarr items this guard exists to catch, in order to accommodate + // legacy entity items that cannot exist. + if (resolved.layout !== ENTITY_DATASET_LAYOUT) { + const found = resolved.layout ?? "gridded"; + throw new DatasetNotFoundError( + `${request.collection}/${request.dataset} is a '${found}' dataset, not an entity dataset. Use loadDataset() for gridded data.` + ); + } + + const metadataVariant = resolved.variant || ""; + const dataset = await this.entities.load({ + cid: resolved.cid, + gatewayUrl, + // Forwarded only when given, so the reader's identity default stands. + // + // No default is supplied here, deliberately. `columnKey` renames columns; + // it does not gate access to them. Without one, every column is still + // readable under the schema's own field names -- which are what the + // dataset actually stores, so they are never wrong. Supplying a default + // would only be guessing at the spelling a dataset's publishing profile + // uses, and a wrong guess renames columns silently rather than failing. + // + // The profile is not derivable here either: tabular deliberately stores + // the schema's names rather than the writer's rendering, precisely so a + // reader is not bound to one profile's casing, and STAC does not carry + // the mapping. So a caller wanting the published spelling passes it, the + // same as with `entities.load({ cid })`. + ...(request.columnKey ? { columnKey: request.columnKey } : {}), + }); + + const pathParts = [ + resolved.collectionId, + resolved.dataset, + metadataVariant, + ].filter(Boolean); + + const metadata: DatasetMetadata = { + dataset: resolved.dataset, + collection: resolved.collectionId, + variant: metadataVariant, + path: pathParts.join("-"), + cid: resolved.cid, + source: "stac", + fetchedAt: new Date(), + ...(resolved.organizationId + ? { organization: resolved.organizationId } + : {}), + ...(resolved.versionsApi ? { versionsApi: resolved.versionsApi } : {}), + ...(resolved.provenanceApi + ? { provenanceApi: resolved.provenanceApi } + : {}), + ...(resolved.citationApi ? { citationApi: resolved.citationApi } : {}), + ...(resolved.streamId ? { streamId: resolved.streamId } : {}), + ...(resolved.commitId ? { commitId: resolved.commitId } : {}), + ...(resolved.versionLabel ? { versionLabel: resolved.versionLabel } : {}), + ...(resolved.isCitable !== undefined + ? { isCitable: resolved.isCitable } + : {}), + ...(resolved.retentionClass + ? { retentionClass: resolved.retentionClass } + : {}), + }; + + if (!metadata.organization && metadata.collection?.includes("_")) { + metadata.organization = metadata.collection.split("_")[0]; + } + + return [dataset, metadata]; + } + async loadDataset({ request, options = { diff --git a/src/constants.ts b/src/constants.ts index ef305fd..3853cb9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,2 +1,8 @@ export const DEFAULT_IPFS_GATEWAY = "https://ipfs-gateway.dclimate.net"; export const DEFAULT_SIREN_API_URL = "https://production-api-siren.dclimate.net/api"; + +/** + * The `dclimate:layout` value marking a STAC item as entity (point-observation) + * data rather than a gridded Zarr store. + */ +export const ENTITY_DATASET_LAYOUT = "tabular"; diff --git a/src/entities/entities-client.ts b/src/entities/entities-client.ts index 1f7c703..fef1da5 100644 --- a/src/entities/entities-client.ts +++ b/src/entities/entities-client.ts @@ -20,9 +20,12 @@ import { DatasetNotFoundError } from "../errors.js"; * ISO timestamps, chained selections. `EntityDataset` provides that surface, so * this namespace stays thin: resolve a root, hand back the dataset. * - * Resolution is by CID only for now. There is no STAC equivalent for entity - * data yet; when there is, `load` grows a `{ collection, dataset }` form - * alongside the CID and the rest of this file is unaffected. + * Resolution here is by CID only. Catalog resolution lives one level up, in + * `DClimateClient.loadEntities`, which resolves a collection/dataset through + * STAC and then calls this. Keeping it there rather than adding a second form + * here leaves this class with one job, and leaves the direct-CID path as the + * way to pin an exact snapshot rather than taking whatever the catalog calls + * latest. */ export interface EntitiesClientOptions { gatewayUrl: string; @@ -90,7 +93,7 @@ export class EntitiesClient { async load(request: LoadEntitiesRequest): Promise { if (!request.cid) { throw new DatasetNotFoundError( - "An entity dataset CID is required. Catalog resolution is not available yet." + "An entity dataset CID is required. To address a dataset by name instead, use client.loadEntities({ request: { collection, dataset } })." ); } diff --git a/src/stac/stac-catalog.ts b/src/stac/stac-catalog.ts index 6ebd347..697493a 100644 --- a/src/stac/stac-catalog.ts +++ b/src/stac/stac-catalog.ts @@ -150,6 +150,15 @@ function getBooleanProperty( } export interface StacReleaseMetadata { + /** + * Storage layout the item advertises: "tabular" for entity (point-observation) + * datasets, absent or anything else for the gridded Zarr datasets that were + * the only kind when this type was written. + * + * This is what lets `loadEntities` refuse a Zarr collection rather than + * handing its CID to a reader that will fail deep inside a manifest parse. + */ + layout?: string; versionsApi?: string; provenanceApi?: string; citationApi?: string; @@ -164,6 +173,7 @@ export function getStacReleaseMetadata( properties: Record | undefined ): StacReleaseMetadata { return { + layout: getStringProperty(properties, "dclimate:layout"), versionsApi: getStringProperty(properties, "dclimate:versions_api"), provenanceApi: getStringProperty(properties, "dclimate:provenance_api"), citationApi: getStringProperty(properties, "dclimate:citation_api"), diff --git a/src/stac/stac-server.ts b/src/stac/stac-server.ts index 016b414..9719430 100644 --- a/src/stac/stac-server.ts +++ b/src/stac/stac-server.ts @@ -472,6 +472,103 @@ function stripIpfsScheme(cid: string | undefined): string | undefined { * - Search pagination is bounded and repeated requests are detected to avoid * looping on malformed `next` links. */ +/** + * Fetch every page of `/collections`. + * + * The endpoint paginates and defaults to a page size smaller than the number of + * collections published, so a single unpaged request silently returns a prefix: + * `numberMatched` exceeds `numberReturned` and the tail is simply absent. That + * is invisible at the call site -- the response is a well-formed list, just a + * short one -- and the collections it drops lose the title and organization + * this endpoint is the only source of, leaving them to fall back to whatever the + * item search alone can say. + * + * Follows `rel="next"` rather than passing a large `limit`, because a limit only + * moves the cliff: it is a guess about how many collections will exist later, + * and the day the catalogue outgrows it the truncation returns silently. The + * link is what the server itself says comes next. + * + * Shares `MAX_STAC_SEARCH_PAGES`, the repeat-detection, and the origin check of + * the item search above: a server that returns a `next` pointing at the page + * just fetched would otherwise spin forever, and one pointing off-origin is + * refused rather than followed. Every one of those ends the walk by throwing, + * because each leaves the catalogue incomplete -- and an incomplete catalogue + * that returns normally is indistinguishable from a complete one. + */ +async function fetchAllCollections( + resolvedServerUrl: string +): Promise { + const collections: StacServerCollectionsResponse["collections"] = []; + const seen = new Set(); + let url: string | undefined = `${resolvedServerUrl.replace( + /\/+$/, + "" + )}/collections`; + + for (let page = 0; page < MAX_STAC_SEARCH_PAGES; page++) { + if (!url) break; + if (seen.has(url)) { + throw new Error( + "STAC server /collections pagination repeated a request; results truncated" + ); + } + seen.add(url); + + const response: Response = await fetch(url, { redirect: "manual" }); + if (!response.ok) { + const text = await response.text(); + throw new Error( + `STAC server /collections error ${response.status}: ${text}` + ); + } + const body = (await response.json()) as StacServerCollectionsResponse & { + links?: Array<{ rel?: string; href?: string }>; + }; + collections.push(...(body.collections ?? [])); + + const next = body.links?.find((link) => link.rel === "next")?.href; + if (!next) { + // Cleared before leaving so a surviving `url` below means exactly one + // thing: the page budget ran out with another page still to fetch. A + // clean finish must not look like a truncated one. + url = undefined; + break; + } + + // Resolved against the current page so a relative `next` works. + const parsedNext: URL = new URL(next, url); + // Following an off-origin link would be an open redirect out of the + // configured server, so it is refused -- but refused loudly. Dropping it + // silently would end the walk exactly like a server that said it was + // finished, handing back a catalogue that looks complete while the + // collections past this point go missing. Same rule, and same reason, as + // the item search's pagination-link check. + if ( + normalizedOrigin(parsedNext.toString()) !== + normalizedOrigin(resolvedServerUrl) || + parsedNext.username !== "" || + parsedNext.password !== "" + ) { + throw new Error( + `STAC server /collections pagination link must use the configured server origin ${normalizedOrigin(resolvedServerUrl)}: ${sanitizedUrl(parsedNext)}` + ); + } + url = parsedNext.toString(); + } + + // Reached only with a live `next` still in hand: the loop ran out of budget + // rather than out of pages. Returning here would hand back a catalogue that + // looks complete, and the collections missing from it would surface later as + // untitled or unknown datasets rather than as this failure. + if (url) { + throw new Error( + `STAC server /collections pagination exceeded ${MAX_STAC_SEARCH_PAGES} pages; results truncated` + ); + } + + return { collections }; +} + export async function listAvailableDatasetsFromStacServer( serverUrl: string = DEFAULT_STAC_SERVER_URL ): Promise { @@ -486,21 +583,11 @@ export async function listAvailableDatasetsFromStacServer( } return features; })(); - const [collectionsResp, searchFeatures] = await Promise.all([ - fetch(`${resolvedServerUrl.replace(/\/+$/, "")}/collections`, { - redirect: "manual", - }), + const [collectionsBody, searchFeatures] = await Promise.all([ + fetchAllCollections(resolvedServerUrl), searchFeaturesPromise, ]); - if (!collectionsResp.ok) { - const text = await collectionsResp.text(); - throw new Error( - `STAC server /collections error ${collectionsResp.status}: ${text}` - ); - } - const collectionsBody = (await collectionsResp.json()) as StacServerCollectionsResponse; - interface CollectionAccumulator { title?: string; organization?: string; diff --git a/src/types.ts b/src/types.ts index ff455b2..c4b7a63 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,9 @@ import type { IPFSELEMENTS_INTERFACE } from "@dclimate/jaxray"; import type { SirenOptions } from "./siren/types.js"; import type { VersionFilters } from "./versions/types.js"; +// Type-only: erased at compile time, so this does not statically chain the main +// entry to tabular's reader stack the way a value import would. +import type { TableField } from "@dclimate/tabular/reader"; export type IpfsElements = IPFSELEMENTS_INTERFACE; @@ -151,3 +154,40 @@ export interface DatasetObject { dims: string[]; sizes: Record; } + +/** + * Address an entity dataset in the STAC catalog. + * + * Deliberately not `DatasetRequest`: that type carries `cid` and `resolution`, + * and neither applies here. A direct CID is `client.entities.load({ cid })`, + * which stays the escape hatch for pinning an exact snapshot, and resolution is + * a property of a grid that entity data does not have. + */ +export interface EntityDatasetRequest { + /** Catalog collection id, e.g. "noaa_ghcnd". */ + collection: string; + /** Dataset id within the collection, e.g. "station_observations". */ + dataset: string; + /** Variant to select; defaults to the catalog's preferred one. */ + variant?: string; + /** Organization id, when the collection is not already prefixed with it. */ + organization?: string; + /** + * Override how schema fields map to published column names. + * + * Without this, columns keep the schema's own field names -- what the dataset + * actually stores, so never wrong, but not always what its docs call them. + * The mapping is a property of a dataset's publishing profile (GHCND stores + * `tmax` and publishes `TMAX`, NDBC preserves mixed case like `SwH`) and + * nothing readable from the catalog states it, so no default is guessed: + * a wrong guess would rename columns silently rather than fail. + * + * For GHCND, pass `(field) => field.name.toUpperCase()`. + */ + columnKey?: (field: TableField) => string; +} + +export interface LoadEntitiesOptions { + /** Override the client's IPFS gateway for this dataset only. */ + gatewayUrl?: string; +} diff --git a/tests/entities.test.ts b/tests/entities.test.ts index b1527c5..45ff55a 100644 --- a/tests/entities.test.ts +++ b/tests/entities.test.ts @@ -52,7 +52,10 @@ describe("client.entities", () => { expect(client.entities).toBeDefined(); }); - it("requires a CID until catalog resolution exists", async () => { + // `client.loadEntities({ collection, dataset })` is the catalog-addressed + // entry point; `entities.load` stays the direct-CID escape hatch and still + // requires one. + it("requires a CID when addressed directly rather than by catalog", async () => { const client = new DClimateClient({ gatewayUrl: "http://127.0.0.1:8080" }); await expect( client.entities.load({ cid: "" }) diff --git a/tests/load-entities.test.ts b/tests/load-entities.test.ts new file mode 100644 index 0000000..9cc23f2 --- /dev/null +++ b/tests/load-entities.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it, vi } from "vitest"; +import { DClimateClient } from "../src/index.js"; +import { DatasetNotFoundError } from "../src/errors.js"; + +/** + * `loadEntities` is the entity counterpart to `loadDataset`: it resolves a + * collection/dataset through STAC and opens the result as an `EntityDataset`. + * + * These tests stub `resolveDatasetDetails` rather than the network. What is + * being tested is the dispatch this method adds -- the layout guard, the + * `columnKey` default, and the metadata it assembles -- not STAC resolution, + * which `stac-server.test.ts` already covers against the live server. + */ +const resolved = (over: Record = {}) => ({ + cid: "bafyr4ieoihgvnl5rvu6eh2fqduapjtz7wjp3e7kdtfxjospmavi5lgkoq4", + collectionId: "noaa_ghcnd", + dataset: "station_observations", + variant: "default", + zarrResolutions: [], + layout: "tabular", + commitId: "k1commit", + streamId: "kjstream", + versionLabel: "2026-08-26", + ...over, +}); + +const stubResolution = (client: DClimateClient, over?: Record) => + vi + .spyOn( + client as unknown as { + resolveDatasetDetails: (...a: unknown[]) => unknown; + }, + "resolveDatasetDetails" + ) + .mockResolvedValue(resolved(over)); + +describe("client.loadEntities", () => { + it("refuses a gridded dataset instead of handing its CID to the entity reader", async () => { + // The failure mode this prevents: a Zarr CID opened as an entity dataset + // dies inside a manifest parse as a corruption error, which sends the + // caller looking at the publisher rather than at their own call. + const client = new DClimateClient(); + stubResolution(client, { layout: "zarr" }); + + await expect( + client.loadEntities({ + request: { collection: "ecmwf_era5", dataset: "reanalysis" }, + }) + ).rejects.toThrow(/not an entity dataset.*loadDataset/s); + await expect( + client.loadEntities({ + request: { collection: "ecmwf_era5", dataset: "reanalysis" }, + }) + ).rejects.toThrow(DatasetNotFoundError); + }); + + it("refuses an item that declares no layout at all", async () => { + // Absence is not permission. Entity support postdates `dclimate:layout`, so + // an item without the field is a gridded one from before the convention -- + // there is no such thing as a legacy entity dataset to accommodate. Opening + // it would hand a Zarr CID to the entity reader, which is the exact + // misleading failure this guard exists to prevent. + const client = new DClimateClient(); + stubResolution(client, { layout: undefined }); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await expect( + client.loadEntities({ + request: { collection: "ecmwf_era5", dataset: "reanalysis" }, + }) + ).rejects.toThrow(/is a 'gridded' dataset.*loadDataset/s); + expect(load).not.toHaveBeenCalled(); + }); + + it("supplies no columnKey of its own", async () => { + // `columnKey` renames columns; it does not gate access to them. Without one + // every column is still readable under the schema's own field names, which + // are what the dataset stores and so are never wrong. A default here would + // be a guess at a dataset's publishing profile -- right for GHCND, silently + // wrong for a profile like NDBC's `.spec` feed that publishes `SwH`. + const client = new DClimateClient(); + stubResolution(client); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + }); + + expect(load.mock.calls[0]![0]!).not.toHaveProperty("columnKey"); + }); + + it("forwards a caller's columnKey", async () => { + const client = new DClimateClient(); + stubResolution(client); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { + collection: "noaa_ndbc", + dataset: "buoy_observations", + columnKey: (field) => field.name, + }, + }); + + const { columnKey } = load.mock.calls[0]![0]!; + expect(columnKey?.({ name: "SwH" } as never)).toBe("SwH"); + }); + + it("returns the snapshot identity a caller needs to re-resolve this exact read", async () => { + // The reason entity metadata carries these at all: a settlement or citation + // has to be able to say which snapshot it ran against, not just "whatever + // was newest that day". + const client = new DClimateClient(); + stubResolution(client); + vi.spyOn(client.entities, "load").mockResolvedValue({} as never); + + const [, metadata] = await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + }); + + expect(metadata).toMatchObject({ + collection: "noaa_ghcnd", + dataset: "station_observations", + variant: "default", + organization: "noaa", + source: "stac", + commitId: "k1commit", + streamId: "kjstream", + versionLabel: "2026-08-26", + }); + expect(metadata.path).toBe("noaa_ghcnd-station_observations-default"); + }); + + it("passes a per-request gateway through to the reader", async () => { + const client = new DClimateClient({ gatewayUrl: "https://default.example" }); + stubResolution(client); + const load = vi + .spyOn(client.entities, "load") + .mockResolvedValue({} as never); + + await client.loadEntities({ + request: { collection: "noaa_ghcnd", dataset: "station_observations" }, + options: { gatewayUrl: "https://override.example" }, + }); + + expect(load.mock.calls[0]![0]!.gatewayUrl).toBe("https://override.example"); + }); +}); diff --git a/tests/review-fixes/stac-server-pagination.test.ts b/tests/review-fixes/stac-server-pagination.test.ts index c817562..87bac4b 100644 --- a/tests/review-fixes/stac-server-pagination.test.ts +++ b/tests/review-fixes/stac-server-pagination.test.ts @@ -1,5 +1,8 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { resolveCidFromStacServer } from "../../src/stac/stac-server.js"; +import { + listAvailableDatasetsFromStacServer, + resolveCidFromStacServer, +} from "../../src/stac/stac-server.js"; const serverUrl = "https://paginated-stac.test"; const collection = "bigcoll"; @@ -277,3 +280,102 @@ describe("resolveCidFromStacServer pagination", () => { expect(postBodies[1]).toEqual({ limit: 100, token }); }); }); + +describe("listAvailableDatasetsFromStacServer /collections pagination", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("surfaces truncation instead of silently returning a partial catalogue", async () => { + // Every /collections page advertises another next link, so the walk can + // never terminate naturally. Returning what it had would hand back a + // catalogue that looks complete, with the missing collections showing up + // later as untitled or unknown datasets rather than as this failure. + let collectionsRequests = 0; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.includes("/collections")) { + collectionsRequests += 1; + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + collections: [{ id: collection, title: "Big Collection" }], + links: [ + { + rel: "next", + href: `${serverUrl}/collections?page=${collectionsRequests + 1}`, + }, + ], + }), + text: async () => "", + } as Response; + } + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + type: "FeatureCollection", + features: [feature(0)], + links: [], + }), + text: async () => "", + } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + listAvailableDatasetsFromStacServer(serverUrl), + ).rejects.toThrow(/truncated/); + }); + + it("follows a next link whose host carries a fully-qualified trailing dot", async () => { + // `https://host./x` addresses the same server as `https://host/x`, but + // `URL.origin` compares them unequal. Normalizing is what keeps an + // in-bounds link from being refused as if it left the server. + const seenUrls: string[] = []; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.includes("/collections")) { + seenUrls.push(url); + const first = !url.includes("page=2"); + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + collections: [{ id: collection, title: first ? "One" : "Two" }], + links: first + ? [ + { + rel: "next", + href: `https://paginated-stac.test./collections?page=2`, + }, + ] + : [], + }), + text: async () => "", + } as Response; + } + return { + ok: true, + status: 200, + statusText: "OK", + json: async () => ({ + type: "FeatureCollection", + features: [feature(0)], + links: [], + }), + text: async () => "", + } as Response; + }); + vi.stubGlobal("fetch", fetchMock); + + await expect( + listAvailableDatasetsFromStacServer(serverUrl), + ).resolves.toBeDefined(); + expect(seenUrls.some((url) => url.includes("page=2"))).toBe(true); + }); +}); diff --git a/tests/stac-server.test.ts b/tests/stac-server.test.ts index 5e5f7d5..b1b42b6 100644 --- a/tests/stac-server.test.ts +++ b/tests/stac-server.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { resolveCidFromStacServer, resolveDatasetCidFromStacServer, + listAvailableDatasetsFromStacServer, DEFAULT_STAC_SERVER_URL, } from "../src/stac/stac-server.js"; @@ -402,3 +403,99 @@ describe("STAC Server", () => { }); }); }); + +describe("listAvailableDatasetsFromStacServer pagination", () => { + it("follows rel=next instead of keeping only the first page", async () => { + // The failure this guards: `/collections` paginates, and a single unpaged + // request returns a well-formed but short list. The collections past the + // first page are not missing anything obvious -- they simply arrive with no + // title or organization, because this endpoint is the only source of both. + // That surfaced as a parity test failing on the 11th collection of 14. + const pages: Record = { + "https://stac.example/collections": { + collections: [{ id: "a_one", title: "One" }], + links: [{ rel: "next", href: "https://stac.example/collections?offset=1" }], + numberMatched: 2, + numberReturned: 1, + }, + "https://stac.example/collections?offset=1": { + collections: [{ id: "b_two", title: "Two" }], + links: [], + numberMatched: 2, + numberReturned: 1, + }, + }; + const seen: string[] = []; + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input instanceof Request ? input.url : input); + seen.push(url); + if (url in pages) { + return new Response(JSON.stringify(pages[url]), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + // The concurrent item search; empty is enough to reach the assertions. + return new Response(JSON.stringify({ features: [], links: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }); + + try { + await listAvailableDatasetsFromStacServer("https://stac.example"); + expect(seen).toContain("https://stac.example/collections"); + expect(seen).toContain("https://stac.example/collections?offset=1"); + } finally { + fetchMock.mockRestore(); + } + }); + + it("throws rather than following rel=next to another origin", async () => { + // A `next` pointing off-origin would walk the client out of the server it + // was configured with, so it is never followed. But it is not dropped + // silently either: that would end the walk exactly like a server saying it + // was finished, returning a truncated catalogue that looks whole. Refusing + // the link keeps the boundary; throwing keeps the truncation visible. + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input instanceof Request ? input.url : input); + if (url === "https://stac.example/collections") { + return new Response( + JSON.stringify({ + collections: [{ id: "a_one", title: "One" }], + links: [{ rel: "next", href: "https://evil.example/collections" }], + }), + { status: 200, headers: { "Content-Type": "application/json" } } + ); + } + if (url.startsWith("https://evil.example")) { + throw new Error(`followed next off-origin: ${url}`); + } + return new Response(JSON.stringify({ features: [], links: [] }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }); + + try { + await expect( + listAvailableDatasetsFromStacServer("https://stac.example") + ).rejects.toThrow(/configured server origin/); + // The boundary held: the off-origin href was reported, never fetched. + // (Had it been requested, the mock would have thrown its own error.) + expect( + fetchMock.mock.calls.some((call) => + String(call[0] instanceof Request ? call[0].url : call[0]).startsWith( + "https://evil.example" + ) + ) + ).toBe(false); + } finally { + fetchMock.mockRestore(); + } + }); +});