diff --git a/.gitignore b/.gitignore index d322cf5..1f252dc 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,6 @@ npm-debug.log* yarn-debug.log* yarn-error.log* -.external \ No newline at end of file +.external + +performance_staging/ \ No newline at end of file diff --git a/docs/contributing-results.mdx b/docs/contributing-results.mdx new file mode 100644 index 0000000..e3baa70 --- /dev/null +++ b/docs/contributing-results.mdx @@ -0,0 +1,146 @@ +--- +title: Contributing Leaderboard Results +sidebar_position: 5 +--- + +## Contributing Leaderboard Results + +The [leaderboard](/leaderboard) is built from per-dataset JSON files under +[`static/data/performance/`](https://github.com/Project-AgML/project-agml.github.io/tree/main/static/data/performance) +— one file per dataset, named after the dataset (e.g. `ash_gourd_disease_classification.json`), each holding a +JSON array of raw benchmark run records. `index.json` and `global.json` in that same directory are generated +manifests, not something you edit by hand — see [Regenerating the manifests](#regenerating-the-manifests) below. + +### The Run Record Format + +Each element of a dataset's JSON array describes a single model evaluated on that dataset. For example: + +```json +{ + "benchmark_id": 1, + "timestamp": "2026-07-18T23:27:54.133075+00:00", + "dataset": "Project-AgML/ash_gourd_disease_classification", + "dataset_config": null, + "split": "train", + "model": "openai/clip-vit-base-patch32", + "task": "classification", + "result_type": "zero-shot", + "optimized": "no", + "train_pct": 0.0, + "test_pct": 100.0, + "val_pct": 0.0, + "num_samples": 2676, + "device": "cuda", + "inference_time_seconds": 70.37, + "inference_time_seconds_per_image": 0.026298, + "train_time_seconds": 0.0, + "train_time_seconds_per_image": 0.0, + "metrics": { + "f1": 0.152805, + "precision": 0.139378, + "recall": 0.199623 + }, + "notes": "" +} +``` + +| Field | Required | Description | +| :--- | :--- | :--- | +| `benchmark_id` | Recommended | Numeric ID grouping runs from the same benchmark sweep. | +| `timestamp` | Recommended | ISO-8601 timestamp of when the run was produced. Its first 10 characters (`YYYY-MM-DD`) are shown as the result's date. | +| `dataset` | Recommended | The Hugging Face dataset path (e.g. `Project-AgML/`). | +| `dataset_config` | Optional | The dataset config the model was evaluated on (e.g. `"augmented"`). Omit or `null` for the raw config. | +| `split` | Optional | The data split evaluated, e.g. `"train"` or `"test"`. | +| `model` | **Yes** | The model identifier, e.g. `"openai/clip-vit-base-patch32"`. Rows are grouped by this value. | +| `task` | **Yes** | One of `classification`, `detection`, `segmentation` — determines which `metrics` key is used as the row's primary score (see below). | +| `result_type` | Optional | Free-text label such as `"zero-shot"`; informational only — whether a run is treated as zero-shot or fine-tuned is actually determined by the presence of a `finetune` object (see [Fine-tuned runs](#fine-tuned-runs)). | +| `optimized` | Optional | `"yes"` or `"no"` (string, not boolean) — whether the model was run with inference optimizations. | +| `train_pct` / `test_pct` / `val_pct` | Optional | Informational split percentages; not used to compute `splitBreakdown` (that's derived from sample counts instead — see `num_samples` and `finetune.train_samples`/`val_samples`). | +| `num_samples` | Recommended | Number of samples the model was evaluated on. Shown in the row's notes and used as the "test" share of the split breakdown. | +| `device` | Recommended | The hardware platform the run used, e.g. `"cuda"`. Shown as the row's platform. | +| `inference_time_seconds` / `inference_time_seconds_per_image` | Optional | Timing info. | +| `train_time_seconds` / `train_time_seconds_per_image` | Optional | Timing info for fine-tuning runs. | +| `metrics` | **Yes** | Object of metric name → numeric value (see below). | +| `notes` | Optional | Free-text notes, e.g. the prompt used for a zero-shot classification run. | +| `finetune` | Only for fine-tuned runs | See [Fine-tuned runs](#fine-tuned-runs). | + +#### `task` and `metrics` + +`task` determines which key in `metrics` is picked as the row's primary sortable score, in priority order: + +| `task` | Metric keys tried, in order | +| :--- | :--- | +| `classification` | `f1`, `accuracy`, `top1_accuracy` | +| `detection` | `map`, `map_50`, `map50`, `mAP`, `mAP@0.5`, `f1_at_iou50` | +| `segmentation` | `miou`, `iou`, `mean_iou` | + +If none of those keys are present, the first metric with a finite numeric value is used instead. + +Separately from the primary score, the leaderboard's four sortable global columns (F1, mAP, Precision, Recall) are +computed from `metrics` directly by matching key name patterns — a `metrics.f1` key becomes the row's F1 score, an +`mAP@0.5`-style key becomes its mAP score, and so on. You don't need to add extra keys for this; reporting whichever +metrics you actually computed (`f1`, `precision`, `recall`, `map`, ...) is enough to populate the columns that apply. +Metrics you didn't compute are left blank rather than penalized. + +#### Fine-tuned runs + +A run is treated as fine-tuned (rather than zero-shot) when it includes a `finetune` object: + +```json +"finetune": { + "train_samples": 2000, + "val_samples": 300, + "epochs": 10, + "lr": 0.0001, + "weight_decay": 0.01, + "split_seed": 42, + "train_ratio": 0.8, + "training_time_seconds": 812.4, + "optimized": "no" +} +``` + +All fields are optional and used only to render notes and the split breakdown. If a model has both a zero-shot and a +fine-tuned run in the same file, the leaderboard shows the best result of each side by side rather than picking one. + +A model can appear multiple times per dataset (e.g. repeated runs). Only the single best zero-shot run and the +single best fine-tuned run per model are kept — you don't need to prune older/worse runs from the file yourself, +though doing so keeps the file smaller. + +### Adding Results + +1. Find (or create) the dataset's file at `static/data/performance/.json` — `` must + match the dataset's `name` in the catalog (`static/data/datasets.json` or `static/data/hf_datasets.json`). +2. Append your run record(s) to the JSON array, following the format above. +3. If you're adding results for a brand-new dataset (no existing file), just create the file with a JSON array + containing your record(s) — it will be picked up automatically the next time the manifests are regenerated. + +If you have a batch of new runs and don't want to hand-merge them into existing files, drop per-dataset JSON arrays +into a `performance_staging/` directory at the project root (same filenames as their target dataset files) and run: + +```bash +python3 scripts/apply_performance_staging.py +``` + +This appends each staging file's records onto the matching file in `static/data/performance/` (staging files are +left in place afterward). Pass `--dry-run` to preview the merge without writing anything. + +### Regenerating the Manifests + +`index.json` (the list of datasets with performance data) and `global.json` (the aggregated, percentile-ranked +leaderboard used by the site-wide leaderboard view) are both generated from the per-dataset files — never edit them +directly. Regenerate them with: + +```bash +npm run generate-datasets +``` + +This also runs automatically before `npm start` and `npm run build`. After regenerating, build or run the site +locally and check the [leaderboard page](/leaderboard) and the affected dataset's leaderboard tab to confirm your +results appear with the expected score, platform, and split breakdown. + +### Opening the Pull Request + +Once your run records are added and the manifests are regenerated, open a pull request against the +`project-agml.github.io` repository with the JSON changes (per-dataset files plus the regenerated `index.json` and +`global.json`). Include where the results came from (benchmark script, paper, etc.) in the PR description. diff --git a/docs/index.mdx b/docs/index.mdx index e36aeab..492b572 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -130,7 +130,8 @@ into a single training pipeline. Datasets on the Hugging Face Hub encode annotat We welcome contributions! If you would like to contribute a new feature, fix an issue that you've noticed, or even just mention a bug or feature that you would like to see implemented, please don't hesitate to use the *Issues* tab to bring it to our attention. -See the [contributing guidelines](/docs/development) for more information. +See the [contributing guidelines](/docs/development) for more information, or the +[guide to contributing leaderboard results](/docs/contributing-results) if you have benchmark results to add. ## Funding This project is partly funded by the [National AI Institute for Food Systems](https://aifs.ucdavis.edu). diff --git a/scripts/apply_performance_staging.py b/scripts/apply_performance_staging.py new file mode 100644 index 0000000..a46ff88 --- /dev/null +++ b/scripts/apply_performance_staging.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Appends run records from performance_staging/*.json into the matching +static/data/performance/.json files. Staging files are left in +place (not cleared) after merging. Run the start command afterward to +regenerate the index.json/global.json leaderboard manifests. + +Usage: python3 scripts/apply_performance_staging.py [--dry-run] +""" + +import json +import sys +from pathlib import Path + +project_root = Path(__file__).resolve().parent.parent +staging_dir = project_root / 'performance_staging' +performance_dir = project_root / 'static' / 'data' / 'performance' +dry_run = '--dry-run' in sys.argv[1:] + + +def read_json_array(file_path): + if not file_path.exists(): + return [] + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + if not isinstance(data, list): + raise ValueError(f'Expected a JSON array in {file_path}') + return data + + +def write_json_array(file_path, records): + file_path.parent.mkdir(parents=True, exist_ok=True) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(json.dumps(records, indent=2) + '\n') + + +def main(): + if not staging_dir.exists(): + print('No performance_staging directory found, nothing to do.') + return + + staging_files = sorted(f.name for f in staging_dir.iterdir() if f.suffix == '.json') + if not staging_files: + print('performance_staging is empty, nothing to do.') + return + + for file in staging_files: + staging_path = staging_dir / file + target_path = performance_dir / file + + new_records = read_json_array(staging_path) + existing_records = read_json_array(target_path) + merged = existing_records + new_records + + print( + f'{file}: {len(existing_records)} existing + {len(new_records)} staged = {len(merged)} total' + ) + + if not dry_run: + write_json_array(target_path, merged) + + if dry_run: + print('\nDry run - no files were changed.') + return + + print('\nDone.') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/generate-datasets.mjs b/scripts/generate-datasets.mjs index 9e41ef2..4a2b98d 100644 --- a/scripts/generate-datasets.mjs +++ b/scripts/generate-datasets.mjs @@ -58,7 +58,7 @@ function buildDatasetMetadataLookup(...manifests) { // rendering of the same files — keep the two in sync if the run schema changes. const TASK_METRIC_KEYS = { classification: ['f1', 'accuracy', 'top1_accuracy'], - detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5'], + detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5', 'f1_at_iou50'], segmentation: ['miou', 'iou', 'mean_iou'], }; @@ -82,16 +82,40 @@ function isMapMetricKey(key) { return /^m?ap([_@-]|$)/i.test(key); } +// Detection runs that can't produce a confidence-ranked mAP instead report f1/precision/recall +// "at IoU" (e.g. f1_at_iou50) — same metric families as classification, just suffixed. Matches +// the bare key too (e.g. "f1") so this subsumes the plain-key case. +function baseMetricKind(key) { + const normalized = key.toLowerCase(); + if (/^f1([_@-]|$)/.test(normalized)) return 'f1'; + if (/^(precision|prec)([_@-]|$)/.test(normalized)) return 'precision'; + if (/^(recall|rec)([_@-]|$)/.test(normalized)) return 'recall'; + return null; +} + function computeCategoryScores(metrics) { - const f1 = isFiniteNumber(metrics.f1) ? metrics.f1 : null; + let f1 = null; let map = null; + let precision = null; + let recall = null; for (const [key, value] of Object.entries(metrics)) { - if (isMapMetricKey(key) && isFiniteNumber(value)) { + if (!isFiniteNumber(value)) continue; + if (isMapMetricKey(key)) { map = map == null ? value : Math.max(map, value); + continue; + } + switch (baseMetricKind(key)) { + case 'f1': + f1 = f1 == null ? value : Math.max(f1, value); + break; + case 'precision': + precision = precision == null ? value : Math.max(precision, value); + break; + case 'recall': + recall = recall == null ? value : Math.max(recall, value); + break; } } - const precision = isFiniteNumber(metrics.precision) ? metrics.precision : null; - const recall = isFiniteNumber(metrics.recall) ? metrics.recall : null; return { f1, map, precision, recall }; } @@ -135,6 +159,7 @@ function makeLeaderboardRow(entry, metricKey, variant) { model: entry.model.trim(), score: entry.metrics[metricKey], categoryScores: computeCategoryScores(entry.metrics), + benchmarkId: isFiniteNumber(entry.benchmark_id) ? entry.benchmark_id : null, variant, date: typeof entry.timestamp === 'string' ? entry.timestamp.slice(0, 10) : null, submitted_by: null, @@ -252,6 +277,7 @@ function buildGlobalPerformanceRecords(performanceDatasets, metadataLookup) { scores: entry.categoryScores, crop_types: meta.crop_types, machine_learning_task: meta.machine_learning_task, + benchmarkId: entry.benchmarkId ?? null, variant: entry.variant === 'zero-shot' || entry.variant === 'fine-tuned' ? entry.variant : null, optimized: Boolean(entry.optimized), platform: typeof entry.platform === 'string' && entry.platform.trim() ? entry.platform.trim() : null, diff --git a/sidebars.ts b/sidebars.ts index aadad3e..eeacd74 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -19,7 +19,7 @@ const sidebars: SidebarsConfig = { label: 'Guide', collapsible: false, className: 'sidebarCategoryLabel', - items: ['index', 'development'], + items: ['index', 'development', 'contributing-results'], }, { type: 'category', diff --git a/src/components/DatasetMetadataModal.module.css b/src/components/DatasetMetadataModal.module.css index 4fc8e7c..bed1eea 100644 --- a/src/components/DatasetMetadataModal.module.css +++ b/src/components/DatasetMetadataModal.module.css @@ -574,6 +574,9 @@ color: var(--agml-text); line-height: 1.5; overflow-wrap: anywhere; + white-space: pre-wrap; + max-height: 220px; + overflow-y: auto; } .leaderboardTable a { diff --git a/src/components/LeaderboardDetailModal.tsx b/src/components/LeaderboardDetailModal.tsx index 2a5c0f4..361b34d 100644 --- a/src/components/LeaderboardDetailModal.tsx +++ b/src/components/LeaderboardDetailModal.tsx @@ -7,8 +7,25 @@ function toLabel(value: string) { return value.replace(/_/g, ' '); } +// 1st, 2nd, 3rd, 4th, ..., 11th-13th stay "th" (the exception the mod-10 rule alone gets wrong). +function ordinal(value: number): string { + const rounded = Math.round(value); + const mod100 = rounded % 100; + if (mod100 >= 11 && mod100 <= 13) return `${rounded}th`; + switch (rounded % 10) { + case 1: + return `${rounded}st`; + case 2: + return `${rounded}nd`; + case 3: + return `${rounded}rd`; + default: + return `${rounded}th`; + } +} + function formatPercentile(value: number | null) { - return value == null ? null : `${value.toFixed(0)}th pctl`; + return value == null ? null : `${ordinal(value)} pctl`; } function formatScore(value: number) { diff --git a/src/lib/performance.ts b/src/lib/performance.ts index 856042b..6c1379f 100644 --- a/src/lib/performance.ts +++ b/src/lib/performance.ts @@ -3,6 +3,7 @@ import useBaseUrl from '@docusaurus/useBaseUrl'; export interface PerformanceEntry { rank: number | null; + benchmarkId: number | null; model: string; score: number | null; submitted_by: string | null; @@ -75,6 +76,7 @@ function normalizeEntry(raw: unknown): PerformanceEntry | null { return { rank: toNumber(raw.rank), + benchmarkId: toNumber(raw.benchmark_id), model, score: toNumber(raw.score ?? raw.value ?? raw.metric_value), submitted_by: toText(raw.submitted_by ?? raw.submittedBy ?? raw.author), @@ -100,7 +102,7 @@ function normalizeEntry(raw: unknown): PerformanceEntry | null { // global.json from the same files at build time — keep the two in sync if the run schema changes. const TASK_METRIC_KEYS: Record = { classification: ['f1', 'accuracy', 'top1_accuracy'], - detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5'], + detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5', 'f1_at_iou50'], segmentation: ['miou', 'iou', 'mean_iou'], }; @@ -138,6 +140,17 @@ function isMapMetricKey(key: string): boolean { return /^m?ap([_@-]|$)/i.test(key); } +// Detection runs that can't produce a confidence-ranked mAP instead report f1/precision/recall +// "at IoU" (e.g. f1_at_iou50) — same metric families as classification, just suffixed. Matches +// the bare key too (e.g. "f1") so this subsumes the plain-key case. +function baseMetricKind(key: string): 'f1' | 'precision' | 'recall' | null { + const normalized = key.toLowerCase(); + if (/^f1([_@-]|$)/.test(normalized)) return 'f1'; + if (/^(precision|prec)([_@-]|$)/.test(normalized)) return 'precision'; + if (/^(recall|rec)([_@-]|$)/.test(normalized)) return 'recall'; + return null; +} + export interface CategoryScores { f1: number | null; map: number | null; @@ -151,15 +164,28 @@ export interface CategoryScores { // and recall are ranked/percentiled independently, not averaged into one blended score. // Mirrored in scripts/generate-datasets.mjs — keep in sync. function computeCategoryScores(metrics: Record): CategoryScores { - const f1 = isFiniteNumber(metrics.f1) ? metrics.f1 : null; + let f1: number | null = null; let map: number | null = null; + let precision: number | null = null; + let recall: number | null = null; for (const [key, value] of Object.entries(metrics)) { - if (isMapMetricKey(key) && isFiniteNumber(value)) { + if (!isFiniteNumber(value)) continue; + if (isMapMetricKey(key)) { map = map == null ? value : Math.max(map, value); + continue; + } + switch (baseMetricKind(key)) { + case 'f1': + f1 = f1 == null ? value : Math.max(f1, value); + break; + case 'precision': + precision = precision == null ? value : Math.max(precision, value); + break; + case 'recall': + recall = recall == null ? value : Math.max(recall, value); + break; } } - const precision = isFiniteNumber(metrics.precision) ? metrics.precision : null; - const recall = isFiniteNumber(metrics.recall) ? metrics.recall : null; return { f1, map, precision, recall }; } @@ -178,10 +204,27 @@ function formatMapMetricLabel(key: string): string { return `mAP@${lo}`; } +const BASE_METRIC_LABELS: Record<'f1' | 'precision' | 'recall', string> = { + f1: 'F1', + precision: 'Precision', + recall: 'Recall', +}; + +// "iou50" -> ".5", "iou75" -> ".75" — a bare fraction (no "IoU" text, no leading 0), e.g. +// "Precision@.5", matching how mAP thresholds are conventionally written in short form. +function formatIouFraction(rawValue: number): string { + return (rawValue / 100).toFixed(2).replace(/0$/, '').replace(/^0/, ''); +} + function labelForMetricKey(key: string): string | null { const normalized = key.toLowerCase(); if (isMapMetricKey(normalized)) return formatMapMetricLabel(normalized); - return NAMED_METRIC_LABELS[normalized] ?? null; + if (NAMED_METRIC_LABELS[normalized]) return NAMED_METRIC_LABELS[normalized]; + const kind = baseMetricKind(normalized); + if (!kind) return null; + const iouMatch = normalized.match(/iou[_@]?(\d{2,3})/); + if (iouMatch) return `${BASE_METRIC_LABELS[kind]}@${formatIouFraction(Number(iouMatch[1]))}`; + return BASE_METRIC_LABELS[kind]; } function collectMetrics(metrics: Record): MetricValue[] { @@ -195,13 +238,18 @@ function collectMetrics(metrics: Record): MetricValue[] { return results; } +// entry.notes carries the run's full methodology text (prompt used, parsing rules, metric +// definitions, etc.) straight from the benchmark script — surfaced in full below the generated +// summary line rather than discarded, since it's the only record of how the run was actually done. function buildRunNote(entry: Record): string | null { const parts: string[] = ['Zero-shot']; if (isFiniteNumber(entry.num_samples)) parts.push(`evaluated on ${entry.num_samples} images`); - return parts.join(' · '); + const summary = parts.join(' · '); + const detail = toText(entry.notes); + return detail ? `${summary}\n\n${detail}` : summary; } -function buildFinetuneNote(finetune: unknown): string | null { +function buildFinetuneNote(finetune: unknown, entryNotes?: unknown): string | null { if (!isRecord(finetune)) return null; const parts: string[] = ['Fine-tuned']; if (isFiniteNumber(finetune.train_samples)) parts.push(`trained on ${finetune.train_samples} images from this dataset`); @@ -211,7 +259,9 @@ function buildFinetuneNote(finetune: unknown): string | null { if (isFiniteNumber(finetune.weight_decay)) parts.push(`weight decay=${finetune.weight_decay}`); if (isFiniteNumber(finetune.split_seed)) parts.push(`seed=${finetune.split_seed}`); if (isFiniteNumber(finetune.train_ratio)) parts.push(`train ratio=${finetune.train_ratio}`); - return parts.join(' · '); + const summary = parts.join(' · '); + const detail = toText(entryNotes); + return detail ? `${summary}\n\n${detail}` : summary; } function computeTrainPercentage(entry: Record, finetune: unknown): number | null { @@ -257,11 +307,12 @@ function makeLeaderboardRow( return { model: (entry.model as string).trim(), score: metrics[metricKey] as number, + benchmarkId: isFiniteNumber(entry.benchmark_id) ? entry.benchmark_id : null, variant, date: toText(entry.timestamp)?.slice(0, 10) ?? null, submitted_by: null, link: null, - notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune) : buildRunNote(entry), + notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune, entry.notes) : buildRunNote(entry), optimized: isOptimized(entry.optimized) || (isRecord(finetune) && isOptimized(finetune.optimized)), splitBreakdown: buildSplitBreakdown(entry, finetune), trainPercentage: computeTrainPercentage(entry, finetune), @@ -355,6 +406,7 @@ export interface GlobalPerformanceRecord { scores: CategoryScores; crop_types: string[] | null; machine_learning_task: string | null; + benchmarkId: number | null; variant: 'zero-shot' | 'fine-tuned' | null; optimized: boolean; platform: string | null; @@ -441,6 +493,7 @@ function normalizeGlobalPerformanceRecord(raw: unknown): GlobalPerformanceRecord scores: normalizeCategoryScores(raw.scores), crop_types: cropTypes?.length ? cropTypes : null, machine_learning_task: toText(raw.machine_learning_task), + benchmarkId: toNumber(raw.benchmarkId), variant, optimized: Boolean(raw.optimized), platform: toText(raw.platform), @@ -468,10 +521,20 @@ export function computeGlobalLeaderboard( tuned?: ('tuned' | 'not-tuned')[]; optimizedValues?: ('optimized' | 'not-optimized')[]; platforms?: string[]; + datasets?: string[]; minAppearances?: number; } = {} ): GlobalLeaderboardEntry[] { - const { cropTypes = [], mlTasks = [], resultTypes = [], tuned = [], optimizedValues = [], platforms = [], minAppearances = 3 } = options; + const { + cropTypes = [], + mlTasks = [], + resultTypes = [], + tuned = [], + optimizedValues = [], + platforms = [], + datasets = [], + minAppearances = 3, + } = options; const stats = new Map< string, { @@ -504,6 +567,7 @@ export function computeGlobalLeaderboard( if (!optimizedValues.includes(optimizedKey)) continue; } if (platforms.length && !(record.platform && platforms.includes(record.platform))) continue; + if (datasets.length && !datasets.includes(record.dataset)) continue; const key = `${record.model}|||${record.machine_learning_task ?? ''}`; const entryStats = diff --git a/src/pages/datasets/index.tsx b/src/pages/datasets/index.tsx index e4c3cc0..d4e8770 100644 --- a/src/pages/datasets/index.tsx +++ b/src/pages/datasets/index.tsx @@ -21,7 +21,7 @@ type DatasetFilterConfig = { const DATASET_FILTERS: DatasetFilterConfig[] = [ { key: 'ml_task', - label: 'Task Type', + label: 'CV Task', field: 'machine_learning_task', kind: 'checkbox', formatOption: (value) => value.replace(/_/g, ' '), diff --git a/src/pages/leaderboard/index.module.css b/src/pages/leaderboard/index.module.css index a87caf8..9d1bef1 100644 --- a/src/pages/leaderboard/index.module.css +++ b/src/pages/leaderboard/index.module.css @@ -153,23 +153,64 @@ color: var(--agml-muted); } +.benchmarkHeader { + margin-bottom: 1.5rem; +} + +.tableHeading { + font-size: 0.95rem; + font-weight: 600; + color: var(--agml-text); + margin: 0; +} + +.tableSection { + margin-bottom: 2rem; +} + +.tableSection:last-child { + margin-bottom: 0; +} + .tableWrap { border: 1px solid var(--agml-border); border-radius: 8px; overflow-x: auto; } +.tableCaption { + padding: 0.65rem 1rem; + border-bottom: 1px solid var(--agml-border); + background: var(--agml-surface-soft); + border-radius: 8px 8px 0 0; +} + +.sectionHeading { + font-size: 0.8rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--agml-text); + margin: 0; +} + +.sectionDescription { + font-size: 0.78rem; + color: var(--agml-muted); + margin: 0.35rem 0 0; +} + .leaderboardTable { width: 100%; - min-width: 980px; + min-width: 760px; font-size: 0.85rem; } .tableRow { display: grid; grid-template-columns: - minmax(100px, 1.1fr) minmax(200px, 2.4fr) minmax(120px, 1.1fr) minmax(120px, 1.1fr) - minmax(120px, 1.1fr) minmax(120px, 1.1fr) minmax(90px, 0.8fr) minmax(78px, 0.4fr); + minmax(200px, 2.4fr) minmax(120px, 1.1fr) minmax(120px, 1.1fr) + minmax(120px, 1.1fr) minmax(90px, 0.8fr) minmax(78px, 0.4fr); gap: 0.6rem; align-items: center; } @@ -235,35 +276,6 @@ text-underline-offset: 3px; } -.taskBadge { - font-family: var(--ifm-code-font-family); - font-size: 0.68rem; - padding: 0.15rem 0.5rem; - border-radius: 4px; - display: inline-block; - white-space: nowrap; -} - -.badgeClassification { - background: var(--agml-badge-classification-bg); - color: var(--agml-badge-classification-fg); -} - -.badgeDetection { - background: var(--agml-badge-detection-bg); - color: var(--agml-badge-detection-fg); -} - -.badgeSegmentation { - background: var(--agml-badge-segmentation-bg); - color: var(--agml-badge-segmentation-fg); -} - -.badgeOther { - background: var(--agml-badge-other-bg); - color: var(--agml-badge-other-fg); -} - .metricCell { display: flex; align-items: center; diff --git a/src/pages/leaderboard/index.tsx b/src/pages/leaderboard/index.tsx index 5f3b539..a4a940d 100644 --- a/src/pages/leaderboard/index.tsx +++ b/src/pages/leaderboard/index.tsx @@ -7,17 +7,38 @@ import { import type { GlobalLeaderboardEntry } from '../../lib/performance'; import { MultiSelectDropdown } from '../../components/MultiSelectDropdown'; import { LeaderboardDetailModal } from '../../components/LeaderboardDetailModal'; +import { useDatasets } from '../../lib/datasets'; import styles from './index.module.css'; const MIN_APPEARANCES = 3; const PAGE_SIZE = 25; -type SortField = 'f1' | 'map' | 'precision' | 'recall'; +// The global leaderboard only ever sorts/displays F1, precision, and recall — mAP stays fully +// computed (GlobalLeaderboardEntry.avgMapPercentile, computeGlobalLeaderboard, etc.) since +// per-dataset detail views and future benchmarks still use it, it's just not a table column here. +type SortField = 'f1' | 'precision' | 'recall'; function toLabel(value: string) { return value.replace(/_/g, ' '); } +// 1st, 2nd, 3rd, 4th, ..., 11th-13th stay "th" (the exception the mod-10 rule alone gets wrong). +function ordinal(value: number): string { + const rounded = Math.round(value); + const mod100 = rounded % 100; + if (mod100 >= 11 && mod100 <= 13) return `${rounded}th`; + switch (rounded % 10) { + case 1: + return `${rounded}st`; + case 2: + return `${rounded}nd`; + case 3: + return `${rounded}rd`; + default: + return `${rounded}th`; + } +} + function shortTaskLabel(value: string) { const lower = value.toLowerCase(); if (lower.includes('classif')) return 'Classification'; @@ -30,8 +51,6 @@ function percentileValue(entry: GlobalLeaderboardEntry, field: SortField) { switch (field) { case 'f1': return entry.avgF1Percentile; - case 'map': - return entry.avgMapPercentile; case 'precision': return entry.avgPrecisionPercentile; case 'recall': @@ -39,14 +58,6 @@ function percentileValue(entry: GlobalLeaderboardEntry, field: SortField) { } } -function taskBadgeClass(task: string | null): string { - if (!task) return styles.badgeOther; - if (task.includes('classif')) return styles.badgeClassification; - if (task.includes('detect')) return styles.badgeDetection; - if (task.includes('segment')) return styles.badgeSegmentation; - return styles.badgeOther; -} - function CheckboxFilterGroup({ label, options, @@ -89,22 +100,145 @@ function PercentileCell({ value }: { value: number | null }) {
- {value.toFixed(0)}th + {ordinal(value)} +
+ ); +} + +function LeaderboardSection({ + title, + entries, + sortBy, + setSortBy, + page, + setPage, + onSelect, + description, +}: { + title: string; + entries: GlobalLeaderboardEntry[]; + sortBy: SortField; + setSortBy: (field: SortField) => void; + page: number; + setPage: (updater: (current: number) => number) => void; + onSelect: (key: string) => void; + description?: string; +}) { + const pageCount = Math.max(1, Math.ceil(entries.length / PAGE_SIZE)); + const currentPage = Math.min(page, pageCount); + const paged = entries.slice((currentPage - 1) * PAGE_SIZE, currentPage * PAGE_SIZE); + const sortHeaderClass = (field: SortField) => `${styles.sortHeader} ${sortBy === field ? styles.sortHeaderActive : ''}`; + + return ( +
+
+
+

{title}

+ {description &&

{description}

} +
+ {entries.length === 0 ? ( +

No models have at least {MIN_APPEARANCES} dataset appearances for the current filters.

+ ) : ( +
+
+ Model + + + + + + + + + + Result type + # Results +
+ {paged.map((entry) => { + const key = `${entry.model}|||${entry.machineLearningTask ?? ''}`; + return ( +
onSelect(key)} + onKeyDown={(event) => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + onSelect(key); + } + }} + > + + {entry.model} + + + + + + + + + + + {entry.resultType} + {entry.appearances} +
+ ); + })} +
+ )} +
+ {entries.length > 0 && pageCount > 1 && ( +
+ + + Page {currentPage} of {pageCount} + + +
+ )}
); } export default function GlobalLeaderboardPage() { const { data: records, loading, error } = useGlobalPerformance(); + const { data: datasets } = useDatasets(); const [search, setSearch] = useState(''); const searchDeferred = useDeferredValue(search); const [cropTypes, setCropTypes] = useState([]); const [mlTasks, setMlTasks] = useState([]); + const [agTasks, setAgTasks] = useState([]); const [tuned, setTuned] = useState<('tuned' | 'not-tuned')[]>([]); const [optimizedValues, setOptimizedValues] = useState<('optimized' | 'not-optimized')[]>([]); const [platforms, setPlatforms] = useState([]); - const [sortBy, setSortBy] = useState('map'); + const [sortBy, setSortBy] = useState('f1'); const { cropTypeOptions, mlTaskOptions, platformOptions } = useMemo(() => { const cropSet = new Set(); @@ -122,15 +256,42 @@ export default function GlobalLeaderboardPage() { }; }, [records]); + const { agTaskOptions, datasetsByAgTask } = useMemo(() => { + const agTaskSet = new Set(); + const byAgTask = new Map(); + for (const dataset of datasets) { + if (!dataset.agricultural_task) continue; + agTaskSet.add(dataset.agricultural_task); + const list = byAgTask.get(dataset.agricultural_task) ?? []; + list.push(dataset.name); + byAgTask.set(dataset.agricultural_task, list); + } + return { + agTaskOptions: Array.from(agTaskSet).sort((a, b) => a.localeCompare(b)), + datasetsByAgTask: byAgTask, + }; + }, [datasets]); + + const agTaskDatasets = useMemo(() => { + if (!agTasks.length) return []; + return agTasks.flatMap((task) => datasetsByAgTask.get(task) ?? []); + }, [agTasks, datasetsByAgTask]); + const toggleValue = (setter: (updater: (current: T[]) => T[]) => void, value: T) => { setter((current) => (current.includes(value) ? current.filter((v) => v !== value) : [...current, value])); }; const hasActiveFilters = - cropTypes.length > 0 || mlTasks.length > 0 || tuned.length > 0 || optimizedValues.length > 0 || platforms.length > 0; + cropTypes.length > 0 || + mlTasks.length > 0 || + agTasks.length > 0 || + tuned.length > 0 || + optimizedValues.length > 0 || + platforms.length > 0; const clearFilters = () => { setCropTypes([]); setMlTasks([]); + setAgTasks([]); setTuned([]); setOptimizedValues([]); setPlatforms([]); @@ -144,9 +305,10 @@ export default function GlobalLeaderboardPage() { tuned, optimizedValues, platforms, + datasets: agTaskDatasets, minAppearances: MIN_APPEARANCES, }), - [records, cropTypes, mlTasks, tuned, optimizedValues, platforms] + [records, cropTypes, mlTasks, tuned, optimizedValues, platforms, agTaskDatasets] ); const searched = useMemo(() => { @@ -166,24 +328,28 @@ export default function GlobalLeaderboardPage() { [searched, sortBy] ); - const [page, setPage] = useState(1); - useEffect(() => setPage(1), [cropTypes, mlTasks, tuned, optimizedValues, platforms, searchDeferred]); - - const pageCount = Math.max(1, Math.ceil(sorted.length / PAGE_SIZE)); - const currentPage = Math.min(page, pageCount); - const pagedLeaderboard = useMemo( - () => sorted.slice((currentPage - 1) * PAGE_SIZE, currentPage * PAGE_SIZE), - [sorted, currentPage] + const classificationEntries = useMemo( + () => sorted.filter((entry) => shortTaskLabel(entry.machineLearningTask ?? '') === 'Classification'), + [sorted] + ); + const detectionEntries = useMemo( + () => sorted.filter((entry) => shortTaskLabel(entry.machineLearningTask ?? '') === 'Detection'), + [sorted] ); + const [classificationPage, setClassificationPage] = useState(1); + const [detectionPage, setDetectionPage] = useState(1); + useEffect(() => { + setClassificationPage(1); + setDetectionPage(1); + }, [cropTypes, mlTasks, agTasks, tuned, optimizedValues, platforms, searchDeferred]); + const [selectedKey, setSelectedKey] = useState(null); const selectedEntry = useMemo( () => sorted.find((entry) => `${entry.model}|||${entry.machineLearningTask ?? ''}` === selectedKey) ?? null, [sorted, selectedKey] ); - const sortHeaderClass = (field: SortField) => `${styles.sortHeader} ${sortBy === field ? styles.sortHeaderActive : ''}`; - return (
@@ -210,6 +376,9 @@ export default function GlobalLeaderboardPage() {