diff --git a/package-lock.json b/package-lock.json index 8f8349b..abbf9ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,7 +28,7 @@ "typescript": "~6.0.2" }, "engines": { - "node": ">=20.0" + "node": ">=24.0" } }, "node_modules/@algolia/abtesting": { diff --git a/src/components/DatasetMetadataModal.module.css b/src/components/DatasetMetadataModal.module.css index e4686c1..4fc8e7c 100644 --- a/src/components/DatasetMetadataModal.module.css +++ b/src/components/DatasetMetadataModal.module.css @@ -134,6 +134,53 @@ overflow-wrap: anywhere; } +.coordinate { + font-family: var(--ifm-code-font-family); + font-size: 0.74rem; + color: var(--agml-muted); + white-space: nowrap; +} + +.locationColumns { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 1.25rem; + margin: 1.25rem 0 1.75rem; +} + +.locationColumn { + display: flex; + flex-direction: column; + gap: 0.85rem; + min-width: 0; +} + +.siteList { + list-style: none; + margin: 0; + padding: 0; + display: flex; + flex-direction: column; + gap: 0.2rem; +} + +.siteRow { + display: flex; + flex-wrap: wrap; + align-items: baseline; + justify-content: space-between; + gap: 0.5rem; + font-size: 0.8rem; + line-height: 1.35; + color: var(--agml-text); + overflow-wrap: anywhere; + padding: 0.15rem 0; +} + +.siteRow:not(:last-child) { + border-bottom: 1px solid var(--agml-border); +} + .section { margin-bottom: 1.75rem; } @@ -222,6 +269,17 @@ white-space: pre; } +.citationCode { + font-family: var(--ifm-code-font-family); + font-size: 0.78rem; + line-height: 1.6; + color: var(--agml-text); + overflow-x: auto; + white-space: pre-wrap; + overflow-wrap: anywhere; + margin: 0; +} + .snippetCopyButton { flex-shrink: 0; align-self: center; @@ -542,4 +600,12 @@ .detailGrid { grid-template-columns: repeat(2, 1fr) !important; } + + .locationColumns { + grid-template-columns: 1fr; + } + + .locationColumn .detailGrid { + grid-template-columns: 1fr !important; + } } diff --git a/src/components/DatasetMetadataModal.tsx b/src/components/DatasetMetadataModal.tsx index e7bebe8..a9be387 100644 --- a/src/components/DatasetMetadataModal.tsx +++ b/src/components/DatasetMetadataModal.tsx @@ -1,6 +1,6 @@ import { Fragment, useEffect, useMemo, useState } from 'react'; import type { Dataset } from '../lib/datasets'; -import { formatDisplayLocation } from '../lib/datasets'; +import { formatCoordinateList, formatDisplayLocation, formatLocationList } from '../lib/datasets'; import { METRIC_CATEGORY_LABELS, useDatasetPerformance } from '../lib/performance'; import type { MetricCategory, PerformanceEntry } from '../lib/performance'; import styles from './DatasetMetadataModal.module.css'; @@ -234,6 +234,7 @@ export function DatasetMetadataModal({ const [cropsExpanded, setCropsExpanded] = useState(false); const [classesExpanded, setClassesExpanded] = useState(false); const [copied, setCopied] = useState(false); + const [citationCopied, setCitationCopied] = useState(false); const [expandedRowKeys, setExpandedRowKeys] = useState>(new Set()); const toggleExpandedRow = (key: string) => { @@ -249,7 +250,6 @@ export function DatasetMetadataModal({ if (!open || dataset == null) return null; const metadataRows = [ - ['Location', formatDisplayLocation(dataset.location)], ['Sensor modality', formatValue(dataset.sensor_modality)], ['Platform', formatValue(dataset.platform)], ['Number of images', formatImageCount(dataset.num_images)], @@ -264,6 +264,14 @@ export function DatasetMetadataModal({ const loader = formatLoaderInstructions(dataset); const cropList = dataset.crop_types ?? []; const classList = dataset.classes ? dataset.classes.split(', ').filter(Boolean) : []; + const locationList = formatLocationList(dataset); + const coordinateList = formatCoordinateList(dataset.lat_lon); + // Pairs each location with the coordinate at the same index (same shape the source data uses + // for both fields) into single compact rows, instead of two separate full lists. + const siteRows = Array.from({ length: Math.max(locationList?.length ?? 0, coordinateList?.length ?? 0) }, (_, index) => ({ + location: locationList?.[index] ?? null, + coordinate: coordinateList?.[index] ?? null, + })); return (
@@ -287,6 +295,7 @@ export function DatasetMetadataModal({ )} {dataset.agricultural_task && {formatValue(dataset.agricultural_task)}} {dataset.real_or_synthetic && {formatValue(dataset.real_or_synthetic)}} + {dataset.license && {formatValue(dataset.license)}}
+ + + )} ); diff --git a/src/lib/datasets.ts b/src/lib/datasets.ts index 7ef07e5..9a21a2a 100644 --- a/src/lib/datasets.ts +++ b/src/lib/datasets.ts @@ -7,6 +7,13 @@ export interface Dataset { machine_learning_task: string | null; agricultural_task: string | null; location: string | string[] | null; + country: string | string[] | null; + // Derived: `country`, falling back to `location` when country wasn't extracted. Used for the + // search page's location filter/display so it groups by country instead of full addresses. + display_location: string | string[] | null; + lat_lon: string | string[] | null; + imaging_equipment: string | string[] | null; + collection_period: string | string[] | null; environment: string | null; augmented_counterpart: string | null; crop_types: string[] | null; @@ -115,12 +122,20 @@ function normalizeDataset(raw: unknown): Dataset | null { const augmentedCounterpart = firstString(raw.augmented_counterpart) ?? (augmentedNumImages != null ? 'yes' : 'no'); + const location = normalizeLocation(raw.location); + const country = normalizeLocation(raw.country); + return { name, machine_learning_task: firstString(raw.machine_learning_task, raw.ml_task, raw.task), source: firstString(raw.source), agricultural_task: firstString(raw.agricultural_task, raw.ag_task), - location: normalizeLocation(raw.location), + location, + country, + display_location: country ?? location, + lat_lon: normalizeLocation(raw.lat_lon), + imaging_equipment: normalizeLocation(raw.imaging_equipment), + collection_period: normalizeLocation(raw.collection_period), environment: firstString(raw.environment, raw.env, raw.image_environment)?.toLowerCase() ?? null, augmented_counterpart: augmentedCounterpart, crop_types: toStringArray(raw.crop_types ?? raw.cropType ?? raw.crop_type)?.map((c) => c.toLowerCase()) ?? null, @@ -152,6 +167,11 @@ function mergeDataset(current: Dataset, incoming: Dataset): Dataset { source: current.source ?? incoming.source, agricultural_task: current.agricultural_task ?? incoming.agricultural_task, location: current.location ?? incoming.location, + country: current.country ?? incoming.country, + display_location: current.display_location ?? incoming.display_location, + lat_lon: current.lat_lon ?? incoming.lat_lon, + imaging_equipment: current.imaging_equipment ?? incoming.imaging_equipment, + collection_period: current.collection_period ?? incoming.collection_period, environment: current.environment ?? incoming.environment, augmented_counterpart: current.augmented_counterpart ?? incoming.augmented_counterpart, crop_types: current.crop_types ?? incoming.crop_types, @@ -283,6 +303,9 @@ export function filterDatasets( ? d.location.some((entry) => entry.toLowerCase().includes(q)) : d.location?.toLowerCase().includes(q) ?? false; const cropMatches = d.crop_types?.some((entry) => entry.toLowerCase().includes(q)) ?? false; + const countryMatches = Array.isArray(d.country) + ? d.country.some((entry) => entry.toLowerCase().includes(q)) + : d.country?.toLowerCase().includes(q) ?? false; return ( d.name.toLowerCase().includes(q) || d.agricultural_task?.toLowerCase().includes(q) || @@ -290,6 +313,7 @@ export function filterDatasets( d.environment?.toLowerCase().includes(q) || d.augmented_counterpart?.toLowerCase().includes(q) || d.platform?.toLowerCase().includes(q) || + countryMatches || locationMatches || cropMatches ); @@ -333,7 +357,7 @@ export function useDatasetOptions(data: Dataset[]) { environments: unique(data.map((d) => d.environment)), augmentedCounterparts: unique(data.map((d) => d.augmented_counterpart)), cropTypes: unique(data.flatMap((d) => d.crop_types ?? [])), - locations: unique(data.map((d) => d.location)), + locations: unique(data.map((d) => d.display_location)), platforms: unique(data.map((d) => d.platform)), realOptions: unique(data.map((d) => d.real_or_synthetic)), }; @@ -345,6 +369,106 @@ export function formatDisplayLocation(value: string | string[] | null | undefine return Array.isArray(value) ? value.join(', ') : value; } +// Primary location label used on cards/badges: prefer the structured `country` field, +// falling back to the free-text `location` field when country wasn't extracted. +export function formatPrimaryLocation(dataset: Pick): string | null { + if (dataset.display_location == null) return null; + const formatted = formatDisplayLocation(dataset.display_location); + return formatted === '—' ? null : formatted; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +// Collapses `country` down to a single name when every entry agrees (handles both the plain +// string case and an array like ["Bangladesh", "Bangladesh"]); returns null when the dataset +// spans more than one country, since there's then no single name to drop from the location text. +function getSingleCountry(country: string | string[] | null): string | null { + if (country == null) return null; + const list = Array.isArray(country) ? country : [country]; + const unique = Array.from(new Set(list.map((entry) => entry.trim()).filter(Boolean))); + return unique.length === 1 ? unique[0] : null; +} + +function stripTrailingCountry(location: string, country: string): string { + const suffix = new RegExp(`\\s*,?\\s*${escapeRegExp(country)}\\s*$`, 'i'); + const stripped = location.replace(suffix, '').trim(); + return stripped || location; +} + +// One entry per `location` string, each with the trailing ", " removed when the whole +// dataset is confined to a single country (redundant once that country is already shown elsewhere). +export function formatLocationList(dataset: Pick): string[] | null { + const locations = Array.isArray(dataset.location) ? dataset.location : dataset.location ? [dataset.location] : []; + if (locations.length === 0) return null; + + const singleCountry = getSingleCountry(dataset.country); + if (!singleCountry) return locations; + + return locations.map((entry) => stripTrailingCountry(entry, singleCountry)); +} + +// `lat_lon` entries come from source metadata in either decimal ("23.7654, 90.4449") or +// DMS ("23 46 17.652, 90 22 30.2514") form, optionally with a trailing hemisphere letter. +function parseCoordPart(part: string): number | null { + const trimmed = part.trim(); + + const dms = trimmed.match(/^(-?\d+(?:\.\d+)?)\s+(\d+(?:\.\d+)?)\s+(\d+(?:\.\d+)?)\s*([NSEW])?$/i); + if (dms) { + const [, degStr, minStr, secStr, dir] = dms; + const deg = Math.abs(Number(degStr)); + const value = deg + Number(minStr) / 60 + Number(secStr) / 3600; + // Compare the raw text rather than the parsed number: Number("-0") < 0 is false in JS, + // which would silently drop the sign on values like "-0 28 59" (just south of the equator). + const negative = degStr.trim().startsWith('-') || (dir != null && /[SW]/i.test(dir)); + return Number.isFinite(value) ? (negative ? -value : value) : null; + } + + const decimal = trimmed.match(/^(-?\d+(?:\.\d+)?)\s*([NSEW])?$/i); + if (decimal) { + const [, numStr, dir] = decimal; + const value = Number(numStr); + if (!Number.isFinite(value)) return null; + return dir != null && /[SW]/i.test(dir) ? -Math.abs(value) : value; + } + + return null; +} + +function parseLatLon(raw: string): { lat: number; lon: number } | null { + const parts = raw.split(','); + if (parts.length !== 2) return null; + const lat = parseCoordPart(parts[0]); + const lon = parseCoordPart(parts[1]); + if (lat == null || lon == null) return null; + if (Math.abs(lat) > 90 || Math.abs(lon) > 180) return null; + return { lat, lon }; +} + +function toDMS(value: number, positiveLetter: string, negativeLetter: string): string { + const dir = value < 0 ? negativeLetter : positiveLetter; + const abs = Math.abs(value); + const deg = Math.floor(abs); + const minFloat = (abs - deg) * 60; + const min = Math.floor(minFloat); + const sec = (minFloat - min) * 60; + return `${deg}°${min}'${sec.toFixed(1)}"${dir}`; +} + +// Renders each `lat_lon` entry as a readable "12°50'26.2"N, 80°09'12.2"E" string. Entries that +// don't parse fall back to the raw source text rather than being dropped. +export function formatCoordinateList(value: string | string[] | null): string[] | null { + const entries = Array.isArray(value) ? value : value ? [value] : []; + if (entries.length === 0) return null; + + return entries.map((entry) => { + const parsed = parseLatLon(entry); + if (!parsed) return entry; + return `${toDMS(parsed.lat, 'N', 'S')}, ${toDMS(parsed.lon, 'E', 'W')}`; + }); +} + export interface DatasetStats { datasetCount: number; imageCount: number; diff --git a/src/pages/datasets/index.tsx b/src/pages/datasets/index.tsx index aa33a41..e4c3cc0 100644 --- a/src/pages/datasets/index.tsx +++ b/src/pages/datasets/index.tsx @@ -4,7 +4,7 @@ import { useHistory, useLocation } from '@docusaurus/router'; import { DatasetMetadataModal } from '../../components/DatasetMetadataModal'; import { MultiSelectDropdown } from '../../components/MultiSelectDropdown'; import styles from './index.module.css'; -import { computeDatasetStats, filterDatasets, formatDisplayLocation, useDatasets } from '../../lib/datasets'; +import { computeDatasetStats, filterDatasets, formatPrimaryLocation, useDatasets } from '../../lib/datasets'; import { useSemanticDatasetSearch } from '../../lib/semanticSearch'; type FilterKind = 'checkbox' | 'dropdown'; @@ -44,7 +44,7 @@ const DATASET_FILTERS: DatasetFilterConfig[] = [ { key: 'location', label: 'Location', - field: 'location', + field: 'display_location', kind: 'dropdown', mode: 'containsAny', formatOption: (value) => value, @@ -190,11 +190,11 @@ function DatasetCard({ zip_size_bytes, augmented_num_images, augmented_zip_size_bytes, - location, } = dataset; const fileSize = formatBytesDecimal(zip_size_bytes); const augmentedFileSize = formatBytesDecimal(augmented_zip_size_bytes); const hasAugmented = augmented_num_images != null; + const primaryLocation = formatPrimaryLocation(dataset); return (