Updating branch

.github/workflows/deploy.yml

@@ -17,7 +17,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 24
           cache: npm
 
       - name: Install dependencies

.github/workflows/test-deploy.yml

@@ -17,10 +17,12 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-node@v4
         with:
-          node-version: 20
+          node-version: 24
           cache: npm
 
       - name: Install dependencies
         run: npm ci
+      - name: Run unit tests
+        run: npm run test:unit
       - name: Test build website
         run: npm run build

.nvmrc

@@ -0,0 +1 @@
+24

docs/index.mdx

@@ -30,11 +30,6 @@ To install the latest release of AgML, run the following command:
 pip install agml
 ```
 
-**_NOTE:_** Some features of AgML, such as synthetic data generation, require GUI applications. When running AgML through
-Windows Subsystem for Linux (WSL), it may be necessary to configure your WSL environment to utilize these features. Please
-follow the [Microsoft documentation](https://learn.microsoft.com/en-us/windows/wsl/tutorials/gui-apps) to install all
-necessary prerequisites and update WSL. The latest version of WSL includes built-in support for running Linux GUI applications.
-
 ## Quick Start
 
 AgML datasets are hosted on the [Hugging Face Hub](https://huggingface.co/Project-AgML) under the `Project-AgML`

package.json

@@ -3,8 +3,8 @@
   "version": "0.0.0",
   "private": true,
   "scripts": {
-    "prebuild": "npm run generate-datasets",
-    "prestart": "npm run generate-datasets",
+    "prebuild": "npm run generate-datasets && npm run generate-embeddings",
+    "prestart": "npm run generate-datasets && npm run generate-embeddings",
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
     "build": "docusaurus build",
@@ -15,13 +15,18 @@
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids",
     "typecheck": "tsc",
-    "generate-datasets": "node scripts/generate-datasets.mjs"
+    "generate-datasets": "node scripts/generate-datasets.mjs",
+    "generate-embeddings": "node scripts/generate-embeddings.mjs",
+    "test": "node --import tsx --test tests/*.test.mjs tests/*.test.ts",
+    "test:unit": "node --import tsx --test tests/generate-embeddings.test.mjs tests/semantic-search-index.test.ts"
   },
   "dependencies": {
     "@docusaurus/core": "3.10.1",
     "@docusaurus/faster": "3.10.1",
     "@docusaurus/preset-classic": "3.10.1",
+    "@huggingface/transformers": "4.2.0",
     "@mdx-js/react": "^3.0.0",
+    "@orama/orama": "^3.1.18",
     "clsx": "^2.0.0",
     "prism-react-renderer": "^2.3.0",
     "react": "^19.0.0",
@@ -32,6 +37,7 @@
     "@docusaurus/tsconfig": "3.10.1",
     "@docusaurus/types": "3.10.1",
     "@types/react": "^19.0.0",
+    "tsx": "^4.23.5",
     "typescript": "~6.0.2"
   },
   "browserslist": {
@@ -47,6 +53,6 @@
     ]
   },
   "engines": {
-    "node": ">=20.0"
+    "node": ">=24.0"
   }
 }

scripts/generate-embeddings.mjs

@@ -0,0 +1,191 @@
+import fs from 'fs';
+import path from 'path';
+import crypto from 'crypto';
+import { fileURLToPath } from 'url';
+import { pipeline } from '@huggingface/transformers';
+import { MODEL_ID, DIM, DTYPE } from '../src/lib/embeddingModel.mjs';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const projectRoot = path.resolve(__dirname, '..');
+const staticDataDir = path.join(projectRoot, 'static', 'data');
+const datasetsPath = path.join(staticDataDir, 'datasets.json');
+const hfDatasetsPath = path.join(staticDataDir, 'hf_datasets.json');
+const embeddingsDir = path.join(staticDataDir, 'embeddings');
+const vectorsPath = path.join(embeddingsDir, 'vectors.bin');
+const metaPath = path.join(embeddingsDir, 'meta.json');
+
+const BATCH_SIZE = 64;
+
+function readJson(filePath) {
+  if (!fs.existsSync(filePath)) return null;
+  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+}
+
+function hasValue(value) {
+  return Array.isArray(value) ? value.length > 0 : value != null && String(value).trim() !== '';
+}
+
+function pickFirstDefined(a, b, field) {
+  return hasValue(a?.[field]) ? a[field] : (b?.[field] ?? null);
+}
+
+// Mirrors firstString() in src/lib/datasets.ts.
+function firstString(...values) {
+  for (const value of values) {
+    if (typeof value === 'string') {
+      const trimmed = value.trim();
+      if (trimmed) return trimmed;
+    }
+  }
+  return null;
+}
+
+// Fields the embedding text template reads (see buildEmbeddingText below). Mirrors the subset
+// of the `Dataset` interface in src/lib/datasets.ts that carries semantic meaning — keep the
+// two in sync if either changes. This is a narrower, name-keyed coalesce merge than
+// mergeDataset()/loadDatasets() in that file (which merges the full Dataset shape); it only
+// needs to cover the fields below.
+const TEMPLATE_FIELDS = [
+  'machine_learning_task',
+  'agricultural_task',
+  'crop_types',
+  'location',
+  'environment',
+  'sensor_modality',
+  'platform',
+  'real_or_synthetic',
+  'classes',
+];
+
+function mergeForEmbedding(datasetsRaw, hfDatasetsRaw) {
+  const byName = new Map();
+  for (const raw of [...datasetsRaw, ...hfDatasetsRaw]) {
+    // Mirrors normalizeDataset()'s name fallback chain in src/lib/datasets.ts, so a record keyed
+    // by e.g. `slug` instead of `name` still gets a corpus entry instead of being silently
+    // dropped by the `if (!name) continue` below.
+    const name = firstString(raw?.name, raw?.dataset, raw?.slug, raw?.id, raw?.key);
+    if (!name) continue;
+    const existing = byName.get(name);
+    if (!existing) {
+      byName.set(name, { ...raw, name });
+      continue;
+    }
+    const merged = { name };
+    for (const field of TEMPLATE_FIELDS) merged[field] = pickFirstDefined(existing, raw, field);
+    byName.set(name, merged);
+  }
+  return Array.from(byName.values()).sort((a, b) => a.name.localeCompare(b.name));
+}
+
+function humanize(value) {
+  return String(value).replace(/_/g, ' ').trim();
+}
+
+// "iNatAg-mini/abelmoschus_esculentus" -> "abelmoschus esculentus (iNatAg-mini species image dataset)"
+// The ~5,900 iNatAg(-mini) child records share near-identical task/location/platform/sensor
+// metadata — the species slug in the name is the only real distinguishing signal for them.
+function buildNameText(name) {
+  const slash = name.indexOf('/');
+  if (slash === -1) return humanize(name);
+  const parent = name.slice(0, slash);
+  const species = humanize(name.slice(slash + 1));
+  return `${species} (${parent} species image dataset)`;
+}
+
+// Field order matters: the tokenizer truncates from the end, so the most identifying fields
+// come first and the noisiest/longest field (classes) comes last.
+function buildEmbeddingText(record) {
+  const parts = [buildNameText(record.name)];
+  if (record.machine_learning_task) parts.push(`Task: ${humanize(record.machine_learning_task)}`);
+  if (record.agricultural_task && record.agricultural_task !== record.machine_learning_task) {
+    parts.push(`Agricultural task: ${humanize(record.agricultural_task)}`);
+  }
+  const crops = Array.isArray(record.crop_types) ? record.crop_types : record.crop_types ? [record.crop_types] : [];
+  if (crops.length) parts.push(`Crops: ${crops.map(humanize).join(', ')}`);
+  const location = Array.isArray(record.location) ? record.location.join(', ') : record.location;
+  if (location) parts.push(`Location: ${location}`);
+  if (record.environment) parts.push(`Environment: ${humanize(record.environment)}`);
+  if (record.sensor_modality) parts.push(`Sensor: ${humanize(record.sensor_modality)}`);
+  if (record.platform) parts.push(`Platform: ${humanize(record.platform)}`);
+  if (record.real_or_synthetic) parts.push(`Data: ${humanize(record.real_or_synthetic)}`);
+  // Joined with ', ' (not the bare `String()` of an array, which uses Array.prototype.toString's
+  // comma-with-no-space) so this matches src/lib/datasets.ts's toText()/mergeDataset() output
+  // that src/lib/semanticSearchIndex.ts's buildIndexRow() truncates on the client side — otherwise
+  // the two 300-char cutoffs land at different content offsets for the same dataset.
+  const classesText = Array.isArray(record.classes) ? record.classes.join(', ') : record.classes;
+  if (hasValue(classesText)) parts.push(`Classes: ${String(classesText).slice(0, 300)}`);
+  return parts.join('. ');
+}
+
+// Includes MODEL_ID/DTYPE so a future model or quantization change is treated as a cache miss
+// even if no dataset record changed — otherwise generateEmbeddings()'s skip-if-unchanged check
+// below would silently keep stale vectors embedded with the old model/dtype.
+function hashTexts(texts) {
+  const hash = crypto.createHash('sha256');
+  hash.update(MODEL_ID).update('\n').update(DTYPE).update('\n');
+  for (const text of texts) hash.update(text).update('\n');
+  return hash.digest('hex');
+}
+
+async function embedAll(texts) {
+  // dtype MUST match what the browser uses (src/lib/semanticSearch.ts) — both must embed with
+  // the same quantized weights so build-time corpus vectors and client-time query vectors live
+  // in the same space. device: 'cpu' uses onnxruntime-node here; that's a build-time-only
+  // native dependency and never ships to the browser bundle.
+  const extractor = await pipeline('feature-extraction', MODEL_ID, { dtype: DTYPE, device: 'cpu' });
+  const vectors = new Float32Array(texts.length * DIM);
+  for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+    const batch = texts.slice(i, i + BATCH_SIZE);
+    const output = await extractor(batch, { pooling: 'mean', normalize: true });
+    vectors.set(output.data, i * DIM);
+    console.log(`Embedded ${Math.min(i + BATCH_SIZE, texts.length)}/${texts.length}`);
+  }
+  return vectors;
+}
+
+async function generateEmbeddings() {
+  const datasetsRaw = readJson(datasetsPath) ?? [];
+  const hfDatasetsRaw = readJson(hfDatasetsPath) ?? [];
+  const merged = mergeForEmbedding(
+    Array.isArray(datasetsRaw) ? datasetsRaw : Object.values(datasetsRaw),
+    Array.isArray(hfDatasetsRaw) ? hfDatasetsRaw : Object.values(hfDatasetsRaw)
+  );
+  const texts = merged.map(buildEmbeddingText);
+  const contentHash = hashTexts(texts);
+
+  const existingMeta = readJson(metaPath);
+  if (existingMeta?.contentHash === contentHash && fs.existsSync(vectorsPath)) {
+    console.log('generate-embeddings: content unchanged, skipping regeneration.');
+    return;
+  }
+
+  const vectors = await embedAll(texts);
+
+  fs.mkdirSync(embeddingsDir, { recursive: true });
+  fs.writeFileSync(vectorsPath, Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength));
+  fs.writeFileSync(
+    metaPath,
+    JSON.stringify({
+      model: MODEL_ID,
+      dtype: DTYPE,
+      dim: DIM,
+      count: merged.length,
+      generatedAt: new Date().toISOString(),
+      contentHash,
+      names: merged.map((d) => d.name),
+    })
+  );
+  console.log('Wrote', vectorsPath, `(${vectors.byteLength} bytes)`, 'and', metaPath, `(${merged.length} records)`);
+}
+
+// Guarded so tests can import this module's pure functions (mergeForEmbedding,
+// buildEmbeddingText, hashTexts, embedAll, ...) without triggering a full run against the
+// real static/data files as a side effect of the import.
+if (import.meta.url === `file://${process.argv[1]}`) {
+  generateEmbeddings().catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+}
+
+export { mergeForEmbedding, buildEmbeddingText, buildNameText, hashTexts, embedAll, generateEmbeddings, MODEL_ID, DIM };

src/components/DatasetMetadataModal.module.css

@@ -134,6 +134,53 @@
 	overflow-wrap: anywhere;
 }
 
+.coordinate {
+	font-family: var(--ifm-code-font-family);
+	font-size: 0.74rem;
+	color: var(--agml-muted);
+	white-space: nowrap;
+}
+
+.locationColumns {
+	display: grid;
+	grid-template-columns: 1fr 1fr;
+	gap: 1.25rem;
+	margin: 1.25rem 0 1.75rem;
+}
+
+.locationColumn {
+	display: flex;
+	flex-direction: column;
+	gap: 0.85rem;
+	min-width: 0;
+}
+
+.siteList {
+	list-style: none;
+	margin: 0;
+	padding: 0;
+	display: flex;
+	flex-direction: column;
+	gap: 0.2rem;
+}
+
+.siteRow {
+	display: flex;
+	flex-wrap: wrap;
+	align-items: baseline;
+	justify-content: space-between;
+	gap: 0.5rem;
+	font-size: 0.8rem;
+	line-height: 1.35;
+	color: var(--agml-text);
+	overflow-wrap: anywhere;
+	padding: 0.15rem 0;
+}
+
+.siteRow:not(:last-child) {
+	border-bottom: 1px solid var(--agml-border);
+}
+
 .section {
 	margin-bottom: 1.75rem;
 }
@@ -222,6 +269,17 @@
 	white-space: pre;
 }
 
+.citationCode {
+	font-family: var(--ifm-code-font-family);
+	font-size: 0.78rem;
+	line-height: 1.6;
+	color: var(--agml-text);
+	overflow-x: auto;
+	white-space: pre-wrap;
+	overflow-wrap: anywhere;
+	margin: 0;
+}
+
 .snippetCopyButton {
 	flex-shrink: 0;
 	align-self: center;
@@ -542,4 +600,12 @@
 	.detailGrid {
 		grid-template-columns: repeat(2, 1fr) !important;
 	}
+
+	.locationColumns {
+		grid-template-columns: 1fr;
+	}
+
+	.locationColumn .detailGrid {
+		grid-template-columns: 1fr !important;
+	}
 }