Feat/load entities from stac

README.md

@@ -83,15 +83,27 @@ degrees, ISO timestamps, chained selections.
 in GHCND, a buoy in NDBC.
 
 ```typescript
-// `columnKey` maps schema field names to the column names queries use. It is a
-// property of the dataset's publishing profile, not of the stored blocks, so
-// the caller states it: GHCND stores `tmax` and publishes `TMAX`. Without it,
-// column names default to the schema's own (lowercase, for GHCND).
-const entities = await client.entities.load({
-  cid: "bafyr4i...",
-  columnKey: (field) => field.name.toUpperCase(), // GHCND's mapping
+// Addressed through the STAC catalog, like `loadDataset`. Separate from it
+// because the two return different types: `EntityDataset` has no `point()`, and
+// its `nearest()` is async and can find nothing.
+const [entities, metadata] = await client.loadEntities({
+  request: { collection: "noaa_ghcnd", dataset: "station_observations" },
 });
 
+// `metadata.commitId` identifies the snapshot this read ran against, so the
+// same one can be re-resolved later instead of whatever is newest then.
+
+// Columns are named by the schema's own field names. `columnKey` renames them
+// to a dataset's published spelling -- GHCND stores `tmax` and publishes
+// `TMAX` -- but it only changes the spelling: every column is readable either
+// way. Neither the stored dataset nor STAC states which profile a dataset uses,
+// so this is the caller's to pass rather than something to guess:
+//
+//   loadEntities({ request: { ..., columnKey: (f) => f.name.toUpperCase() } })
+//
+// To pin an exact snapshot, or to read a dataset that is not in the catalog,
+// `client.entities.load({ cid })` remains available and takes the same option.
+
 // Every entity, with position and coverage window.
 for (const e of await entities.listEntities()) {
   console.log(e.entityId, e.latitude, e.longitude, e.start, e.end);
@@ -167,8 +179,9 @@ const rows = await range.rows(); // may be empty *because* of the gap above
 ```
 
 Reads go over the IPFS HTTP gateway, so no local daemon is required and the same
-code runs in a browser. Resolution is by CID for now; STAC catalog support will
-follow.
+code runs in a browser. Datasets resolve through the STAC catalog via
+`loadEntities`; `entities.load({ cid })` stays available for pinning an exact
+snapshot.
 
 ### Dataset version history
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dclimate/dclimate-client-js",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "description": "JavaScript client for dClimate datasets using jaxray and IPFS stores",
   "type": "module",
   "engines": {

src/client.ts

@@ -6,10 +6,16 @@ import {
   DatasetRequest,
   DatasetVersionRequest,
   DatasetVersionsRequest,
+  EntityDatasetRequest,
   GeoSelectionOptions,
   LoadDatasetOptions,
+  LoadEntitiesOptions,
 } from "./types.js";
-import { DEFAULT_IPFS_GATEWAY } from "./constants.js";
+import { DEFAULT_IPFS_GATEWAY, ENTITY_DATASET_LAYOUT } from "./constants.js";
+// Type-only, like `TableField` in types.ts: naming `EntityDataset` as a return
+// type must not statically pull tabular's reader into every consumer's bundle,
+// which is the whole reason `entities.load` imports it dynamically.
+import type { EntityDataset } from "@dclimate/tabular/reader";
 import { openDatasetFromCid, IpfsElements } from "./ipfs/open-dataset.js";
 import {
   DatasetNotFoundError,
@@ -277,6 +283,131 @@ export class DClimateClient {
     return this.entitiesClient;
   }
 
+  /**
+   * Open an entity (point-observation) dataset by catalog name.
+   *
+   * The entity counterpart to `loadDataset`, and deliberately a separate method
+   * rather than a layout branch inside it. The two return different types with
+   * different query surfaces -- `EntityDataset` has no `point()`, its `nearest()`
+   * is async and can find nothing -- so folding them together would widen
+   * `loadDataset`'s return to a union and make every existing gridded caller
+   * narrow before it could call a Zarr method. Callers know which kind they
+   * want; the entry point says so.
+   *
+   * Resolution is the same STAC lookup `loadDataset` performs, so a dataset is
+   * addressed the same way here as anywhere else in the library, and the release
+   * metadata comes back alongside it. That matters more for entity data than for
+   * gridded: `commitId` and `streamId` identify the exact snapshot a query ran
+   * against, which is what lets a caller re-resolve it later rather than
+   * silently getting whatever is newest.
+   *
+   * Columns are named by the schema's own field names unless `columnKey` is
+   * given. That mapping is a property of a dataset's publishing profile (GHCND
+   * stores `tmax` and publishes `TMAX`), and nothing readable from here states
+   * it, so it is the caller's to pass rather than this method's to guess.
+   *
+   * @throws {DatasetNotFoundError} if the resolved item is not tabular -- a
+   * gridded collection named here is a caller mistake worth reporting in terms
+   * of the fix, not a manifest parse failure deep inside the reader.
+   */
+  async loadEntities({
+    request,
+    options = {},
+  }: {
+    request: EntityDatasetRequest;
+    options?: LoadEntitiesOptions;
+  }): Promise<[EntityDataset, DatasetMetadata]> {
+    const gatewayUrl = options.gatewayUrl ?? this.gatewayUrl;
+
+    const resolved = await this.resolveDatasetDetails(
+      {
+        collection: request.collection,
+        dataset: request.dataset,
+        ...(request.variant ? { variant: request.variant } : {}),
+        ...(request.organization
+          ? { organization: request.organization }
+          : {}),
+      },
+      gatewayUrl
+    );
+
+    // Checked rather than assumed: the catalog holds both kinds, and the CID of
+    // a Zarr store handed to the entity reader fails as a corrupt-dataset error
+    // that says nothing about the actual mistake.
+    //
+    // Positive match, not "absent or tabular". Entity support postdates
+    // `dclimate:layout`, so an item without the field is a gridded one from
+    // before the convention -- treating absence as permission would admit
+    // exactly the Zarr items this guard exists to catch, in order to accommodate
+    // legacy entity items that cannot exist.
+    if (resolved.layout !== ENTITY_DATASET_LAYOUT) {
+      const found = resolved.layout ?? "gridded";
+      throw new DatasetNotFoundError(
+        `${request.collection}/${request.dataset} is a '${found}' dataset, not an entity dataset. Use loadDataset() for gridded data.`
+      );
+    }
+
+    const metadataVariant = resolved.variant || "";
+    const dataset = await this.entities.load({
+      cid: resolved.cid,
+      gatewayUrl,
+      // Forwarded only when given, so the reader's identity default stands.
+      //
+      // No default is supplied here, deliberately. `columnKey` renames columns;
+      // it does not gate access to them. Without one, every column is still
+      // readable under the schema's own field names -- which are what the
+      // dataset actually stores, so they are never wrong. Supplying a default
+      // would only be guessing at the spelling a dataset's publishing profile
+      // uses, and a wrong guess renames columns silently rather than failing.
+      //
+      // The profile is not derivable here either: tabular deliberately stores
+      // the schema's names rather than the writer's rendering, precisely so a
+      // reader is not bound to one profile's casing, and STAC does not carry
+      // the mapping. So a caller wanting the published spelling passes it, the
+      // same as with `entities.load({ cid })`.
+      ...(request.columnKey ? { columnKey: request.columnKey } : {}),
+    });
+
+    const pathParts = [
+      resolved.collectionId,
+      resolved.dataset,
+      metadataVariant,
+    ].filter(Boolean);
+
+    const metadata: DatasetMetadata = {
+      dataset: resolved.dataset,
+      collection: resolved.collectionId,
+      variant: metadataVariant,
+      path: pathParts.join("-"),
+      cid: resolved.cid,
+      source: "stac",
+      fetchedAt: new Date(),
+      ...(resolved.organizationId
+        ? { organization: resolved.organizationId }
+        : {}),
+      ...(resolved.versionsApi ? { versionsApi: resolved.versionsApi } : {}),
+      ...(resolved.provenanceApi
+        ? { provenanceApi: resolved.provenanceApi }
+        : {}),
+      ...(resolved.citationApi ? { citationApi: resolved.citationApi } : {}),
+      ...(resolved.streamId ? { streamId: resolved.streamId } : {}),
+      ...(resolved.commitId ? { commitId: resolved.commitId } : {}),
+      ...(resolved.versionLabel ? { versionLabel: resolved.versionLabel } : {}),
+      ...(resolved.isCitable !== undefined
+        ? { isCitable: resolved.isCitable }
+        : {}),
+      ...(resolved.retentionClass
+        ? { retentionClass: resolved.retentionClass }
+        : {}),
+    };
+
+    if (!metadata.organization && metadata.collection?.includes("_")) {
+      metadata.organization = metadata.collection.split("_")[0];
+    }
+
+    return [dataset, metadata];
+  }
+
   async loadDataset({
     request,
     options = {

src/constants.ts

@@ -1,2 +1,8 @@
 export const DEFAULT_IPFS_GATEWAY = "https://ipfs-gateway.dclimate.net";
 export const DEFAULT_SIREN_API_URL = "https://production-api-siren.dclimate.net/api";
+
+/**
+ * The `dclimate:layout` value marking a STAC item as entity (point-observation)
+ * data rather than a gridded Zarr store.
+ */
+export const ENTITY_DATASET_LAYOUT = "tabular";

src/entities/entities-client.ts

@@ -20,9 +20,12 @@ import { DatasetNotFoundError } from "../errors.js";
  * ISO timestamps, chained selections. `EntityDataset` provides that surface, so
  * this namespace stays thin: resolve a root, hand back the dataset.
  *
- * Resolution is by CID only for now. There is no STAC equivalent for entity
- * data yet; when there is, `load` grows a `{ collection, dataset }` form
- * alongside the CID and the rest of this file is unaffected.
+ * Resolution here is by CID only. Catalog resolution lives one level up, in
+ * `DClimateClient.loadEntities`, which resolves a collection/dataset through
+ * STAC and then calls this. Keeping it there rather than adding a second form
+ * here leaves this class with one job, and leaves the direct-CID path as the
+ * way to pin an exact snapshot rather than taking whatever the catalog calls
+ * latest.
  */
 export interface EntitiesClientOptions {
   gatewayUrl: string;
@@ -90,7 +93,7 @@ export class EntitiesClient {
   async load(request: LoadEntitiesRequest): Promise<EntityDataset> {
     if (!request.cid) {
       throw new DatasetNotFoundError(
-        "An entity dataset CID is required. Catalog resolution is not available yet."
+        "An entity dataset CID is required. To address a dataset by name instead, use client.loadEntities({ request: { collection, dataset } })."
       );
     }
 

src/stac/stac-catalog.ts

@@ -150,6 +150,15 @@ function getBooleanProperty(
 }
 
 export interface StacReleaseMetadata {
+  /**
+   * Storage layout the item advertises: "tabular" for entity (point-observation)
+   * datasets, absent or anything else for the gridded Zarr datasets that were
+   * the only kind when this type was written.
+   *
+   * This is what lets `loadEntities` refuse a Zarr collection rather than
+   * handing its CID to a reader that will fail deep inside a manifest parse.
+   */
+  layout?: string;
   versionsApi?: string;
   provenanceApi?: string;
   citationApi?: string;
@@ -164,6 +173,7 @@ export function getStacReleaseMetadata(
   properties: Record<string, unknown> | undefined
 ): StacReleaseMetadata {
   return {
+    layout: getStringProperty(properties, "dclimate:layout"),
     versionsApi: getStringProperty(properties, "dclimate:versions_api"),
     provenanceApi: getStringProperty(properties, "dclimate:provenance_api"),
     citationApi: getStringProperty(properties, "dclimate:citation_api"),

src/stac/stac-server.ts

@@ -472,6 +472,103 @@ function stripIpfsScheme(cid: string | undefined): string | undefined {
  *   - Search pagination is bounded and repeated requests are detected to avoid
  *     looping on malformed `next` links.
  */
+/**
+ * Fetch every page of `/collections`.
+ *
+ * The endpoint paginates and defaults to a page size smaller than the number of
+ * collections published, so a single unpaged request silently returns a prefix:
+ * `numberMatched` exceeds `numberReturned` and the tail is simply absent. That
+ * is invisible at the call site -- the response is a well-formed list, just a
+ * short one -- and the collections it drops lose the title and organization
+ * this endpoint is the only source of, leaving them to fall back to whatever the
+ * item search alone can say.
+ *
+ * Follows `rel="next"` rather than passing a large `limit`, because a limit only
+ * moves the cliff: it is a guess about how many collections will exist later,
+ * and the day the catalogue outgrows it the truncation returns silently. The
+ * link is what the server itself says comes next.
+ *
+ * Shares `MAX_STAC_SEARCH_PAGES`, the repeat-detection, and the origin check of
+ * the item search above: a server that returns a `next` pointing at the page
+ * just fetched would otherwise spin forever, and one pointing off-origin is
+ * refused rather than followed. Every one of those ends the walk by throwing,
+ * because each leaves the catalogue incomplete -- and an incomplete catalogue
+ * that returns normally is indistinguishable from a complete one.
+ */
+async function fetchAllCollections(
+  resolvedServerUrl: string
+): Promise<StacServerCollectionsResponse> {
+  const collections: StacServerCollectionsResponse["collections"] = [];
+  const seen = new Set<string>();
+  let url: string | undefined = `${resolvedServerUrl.replace(
+    /\/+$/,
+    ""
+  )}/collections`;
+
+  for (let page = 0; page < MAX_STAC_SEARCH_PAGES; page++) {
+    if (!url) break;
+    if (seen.has(url)) {
+      throw new Error(
+        "STAC server /collections pagination repeated a request; results truncated"
+      );
+    }
+    seen.add(url);
+
+    const response: Response = await fetch(url, { redirect: "manual" });
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(
+        `STAC server /collections error ${response.status}: ${text}`
+      );
+    }
+    const body = (await response.json()) as StacServerCollectionsResponse & {
+      links?: Array<{ rel?: string; href?: string }>;
+    };
+    collections.push(...(body.collections ?? []));
+
+    const next = body.links?.find((link) => link.rel === "next")?.href;
+    if (!next) {
+      // Cleared before leaving so a surviving `url` below means exactly one
+      // thing: the page budget ran out with another page still to fetch. A
+      // clean finish must not look like a truncated one.
+      url = undefined;
+      break;
+    }
+
+    // Resolved against the current page so a relative `next` works.
+    const parsedNext: URL = new URL(next, url);
+    // Following an off-origin link would be an open redirect out of the
+    // configured server, so it is refused -- but refused loudly. Dropping it
+    // silently would end the walk exactly like a server that said it was
+    // finished, handing back a catalogue that looks complete while the
+    // collections past this point go missing. Same rule, and same reason, as
+    // the item search's pagination-link check.
+    if (
+      normalizedOrigin(parsedNext.toString()) !==
+        normalizedOrigin(resolvedServerUrl) ||
+      parsedNext.username !== "" ||
+      parsedNext.password !== ""
+    ) {
+      throw new Error(
+        `STAC server /collections pagination link must use the configured server origin ${normalizedOrigin(resolvedServerUrl)}: ${sanitizedUrl(parsedNext)}`
+      );
+    }
+    url = parsedNext.toString();
+  }
+
+  // Reached only with a live `next` still in hand: the loop ran out of budget
+  // rather than out of pages. Returning here would hand back a catalogue that
+  // looks complete, and the collections missing from it would surface later as
+  // untitled or unknown datasets rather than as this failure.
+  if (url) {
+    throw new Error(
+      `STAC server /collections pagination exceeded ${MAX_STAC_SEARCH_PAGES} pages; results truncated`
+    );
+  }
+
+  return { collections };
+}
+
 export async function listAvailableDatasetsFromStacServer(
   serverUrl: string = DEFAULT_STAC_SERVER_URL
 ): Promise<DatasetCatalog> {
@@ -486,21 +583,11 @@ export async function listAvailableDatasetsFromStacServer(
     }
     return features;
   })();
-  const [collectionsResp, searchFeatures] = await Promise.all([
-    fetch(`${resolvedServerUrl.replace(/\/+$/, "")}/collections`, {
-      redirect: "manual",
-    }),
+  const [collectionsBody, searchFeatures] = await Promise.all([
+    fetchAllCollections(resolvedServerUrl),
     searchFeaturesPromise,
   ]);
 
-  if (!collectionsResp.ok) {
-    const text = await collectionsResp.text();
-    throw new Error(
-      `STAC server /collections error ${collectionsResp.status}: ${text}`
-    );
-  }
-  const collectionsBody = (await collectionsResp.json()) as StacServerCollectionsResponse;
-
   interface CollectionAccumulator {
     title?: string;
     organization?: string;