updating results with gemma detection first run

.gitignore

@@ -19,4 +19,7 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
-.external
+.external
+
+# Dev
+performance_staging/

package.json

@@ -16,6 +16,7 @@
     "write-heading-ids": "docusaurus write-heading-ids",
     "typecheck": "tsc",
     "generate-datasets": "node scripts/generate-datasets.mjs",
+    "apply-performance-staging": "node scripts/apply-performance-staging.mjs"
     "generate-embeddings": "node scripts/generate-embeddings.mjs",
     "test": "node --import tsx --test tests/*.test.mjs tests/*.test.ts",
     "test:unit": "node --import tsx --test tests/generate-embeddings.test.mjs tests/semantic-search-index.test.ts"

scripts/apply_performance_staging.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Appends run records from performance_staging/*.json into the matching
+static/data/performance/<dataset>.json files. Staging files are left in
+place (not cleared) after merging. Run the start command afterward to
+regenerate the index.json/global.json leaderboard manifests.
+
+Usage: python3 scripts/apply_performance_staging.py [--dry-run]
+"""
+
+import json
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).resolve().parent.parent
+staging_dir = project_root / 'performance_staging'
+performance_dir = project_root / 'static' / 'data' / 'performance'
+dry_run = '--dry-run' in sys.argv[1:]
+
+
+def read_json_array(file_path):
+    if not file_path.exists():
+        return []
+    with open(file_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise ValueError(f'Expected a JSON array in {file_path}')
+    return data
+
+
+def write_json_array(file_path, records):
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(file_path, 'w', encoding='utf-8') as f:
+        f.write(json.dumps(records, indent=2) + '\n')
+
+
+def main():
+    if not staging_dir.exists():
+        print('No performance_staging directory found, nothing to do.')
+        return
+
+    staging_files = sorted(f.name for f in staging_dir.iterdir() if f.suffix == '.json')
+    if not staging_files:
+        print('performance_staging is empty, nothing to do.')
+        return
+
+    for file in staging_files:
+        staging_path = staging_dir / file
+        target_path = performance_dir / file
+
+        new_records = read_json_array(staging_path)
+        existing_records = read_json_array(target_path)
+        merged = existing_records + new_records
+
+        print(
+            f'{file}: {len(existing_records)} existing + {len(new_records)} staged = {len(merged)} total'
+        )
+
+        if not dry_run:
+            write_json_array(target_path, merged)
+
+    if dry_run:
+        print('\nDry run - no files were changed.')
+        return
+
+    print('\nDone.')
+
+
+if __name__ == '__main__':
+    main()

scripts/generate-datasets.mjs

@@ -58,7 +58,7 @@ function buildDatasetMetadataLookup(...manifests) {
 // rendering of the same files — keep the two in sync if the run schema changes.
 const TASK_METRIC_KEYS = {
   classification: ['f1', 'accuracy', 'top1_accuracy'],
-  detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5'],
+  detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5', 'f1_at_iou50', 'f1'],
   segmentation: ['miou', 'iou', 'mean_iou'],
 };
 
@@ -82,31 +82,53 @@ function isMapMetricKey(key) {
   return /^m?ap([_@-]|$)/i.test(key);
 }
 
+// Detection runs may report F1/precision/recall at a specific IoU match threshold rather than
+// (or alongside) the aggregate keys above, e.g. "f1_at_iou50", "precision_at_iou50" — mirrored
+// in src/lib/performance.ts, keep in sync.
+const IOU_SUFFIXED_METRIC_RE = /^(f1|precision|prec|recall|rec)(?:_at_iou(\d{2,3}))?$/i;
+
+function matchIouSuffixedMetricKey(key) {
+  const match = IOU_SUFFIXED_METRIC_RE.exec(key);
+  if (!match) return null;
+  const base = match[1].toLowerCase();
+  const normalizedBase = base === 'f1' ? 'f1' : base === 'precision' || base === 'prec' ? 'precision' : 'recall';
+  return { base: normalizedBase, iou: match[2] ?? null };
+}
+
 function computeCategoryScores(metrics) {
-  const f1 = isFiniteNumber(metrics.f1) ? metrics.f1 : null;
+  let f1 = null;
   let map = null;
+  let precision = null;
+  let recall = null;
   for (const [key, value] of Object.entries(metrics)) {
-    if (isMapMetricKey(key) && isFiniteNumber(value)) {
+    if (!isFiniteNumber(value)) continue;
+    if (isMapMetricKey(key)) {
       map = map == null ? value : Math.max(map, value);
+      continue;
     }
+    const iouMatch = matchIouSuffixedMetricKey(key);
+    if (!iouMatch) continue;
+    if (iouMatch.base === 'f1') f1 = f1 == null ? value : Math.max(f1, value);
+    else if (iouMatch.base === 'precision') precision = precision == null ? value : Math.max(precision, value);
+    else recall = recall == null ? value : Math.max(recall, value);
   }
-  const precision = isFiniteNumber(metrics.precision) ? metrics.precision : null;
-  const recall = isFiniteNumber(metrics.recall) ? metrics.recall : null;
   return { f1, map, precision, recall };
 }
 
 function buildRunNote(entry) {
   const parts = [];
   if (isFiniteNumber(entry.num_samples)) parts.push(`${entry.num_samples} samples`);
   if (typeof entry.device === 'string' && entry.device.trim()) parts.push(entry.device.trim());
+  if (typeof entry.notes === 'string' && entry.notes.trim()) parts.push(entry.notes.trim());
   return parts.length ? parts.join(' · ') : null;
 }
 
-function buildFinetuneNote(finetune) {
+function buildFinetuneNote(finetune, rawNotes) {
   if (!finetune || typeof finetune !== 'object') return null;
   const parts = [];
   if (isFiniteNumber(finetune.epochs)) parts.push(`${finetune.epochs} epochs`);
   if (isFiniteNumber(finetune.train_samples)) parts.push(`${finetune.train_samples} train samples`);
+  if (rawNotes) parts.push(rawNotes);
   return parts.length ? parts.join(' · ') : null;
 }
 
@@ -139,7 +161,9 @@ function makeLeaderboardRow(entry, metricKey, variant) {
     date: typeof entry.timestamp === 'string' ? entry.timestamp.slice(0, 10) : null,
     submitted_by: null,
     link: null,
-    notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune) : buildRunNote(entry),
+    notes: variant === 'fine-tuned'
+      ? buildFinetuneNote(finetune, typeof entry.notes === 'string' && entry.notes.trim() ? entry.notes.trim() : null)
+      : buildRunNote(entry),
     optimized: isOptimized(entry.optimized) || (finetune != null && typeof finetune === 'object' && isOptimized(finetune.optimized)),
     platform: typeof entry.device === 'string' && entry.device.trim() ? entry.device.trim() : null,
     splitBreakdown: buildSplitBreakdown(entry, finetune),

src/lib/performance.ts

@@ -31,9 +31,9 @@ export interface MetricValue {
 }
 
 export function classifyMetricLabel(label: string): MetricCategory {
-  if (label === 'F1') return 'f1';
+  if (label === 'F1' || label.startsWith('F1@')) return 'f1';
   if (label.startsWith('mAP')) return 'map';
-  if (label === 'Precision' || label === 'Recall') return 'precision_recall';
+  if (label === 'Precision' || label === 'Recall' || label.startsWith('P@') || label.startsWith('R@')) return 'precision_recall';
   return 'other';
 }
 
@@ -100,7 +100,7 @@ function normalizeEntry(raw: unknown): PerformanceEntry | null {
 // global.json from the same files at build time — keep the two in sync if the run schema changes.
 const TASK_METRIC_KEYS: Record<string, string[]> = {
   classification: ['f1', 'accuracy', 'top1_accuracy'],
-  detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5'],
+  detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5', 'f1_at_iou50', 'f1'],
   segmentation: ['miou', 'iou', 'mean_iou'],
 };
 
@@ -122,13 +122,8 @@ function resolveMetricKey(task: unknown, metrics: Record<string, unknown>): stri
 // particular may report mAP at several IoU thresholds (map_50, map_75, map_50_95, ...); each is
 // surfaced as its own labeled value rather than collapsed into one number.
 const NAMED_METRIC_LABELS: Record<string, string> = {
-  f1: 'F1',
   accuracy: 'Accuracy',
   top1_accuracy: 'Top-1 Accuracy',
-  precision: 'Precision',
-  prec: 'Precision',
-  recall: 'Recall',
-  rec: 'Recall',
   miou: 'mIoU',
   iou: 'IoU',
   mean_iou: 'mIoU',
@@ -138,6 +133,21 @@ function isMapMetricKey(key: string): boolean {
   return /^m?ap([_@-]|$)/i.test(key);
 }
 
+// Detection runs may report F1/precision/recall at a specific IoU match threshold rather than
+// (or alongside) the aggregate keys above, e.g. "f1_at_iou50", "precision_at_iou50" — Gemma's
+// zero-shot detection results are matched greedily per-class at IoU>=0.5 and report exactly this
+// shape (see static/data/performance/pomegranate_growth_detection.json). Mirrored in
+// scripts/generate-datasets.mjs — keep in sync.
+const IOU_SUFFIXED_METRIC_RE = /^(f1|precision|prec|recall|rec)(?:_at_iou(\d{2,3}))?$/i;
+
+function matchIouSuffixedMetricKey(key: string): { base: 'f1' | 'precision' | 'recall'; iou: string | null } | null {
+  const match = IOU_SUFFIXED_METRIC_RE.exec(key);
+  if (!match) return null;
+  const base = match[1].toLowerCase();
+  const normalizedBase = base === 'f1' ? 'f1' : base === 'precision' || base === 'prec' ? 'precision' : 'recall';
+  return { base: normalizedBase, iou: match[2] ?? null };
+}
+
 export interface CategoryScores {
   f1: number | null;
   map: number | null;
@@ -151,15 +161,22 @@ export interface CategoryScores {
 // and recall are ranked/percentiled independently, not averaged into one blended score.
 // Mirrored in scripts/generate-datasets.mjs — keep in sync.
 function computeCategoryScores(metrics: Record<string, unknown>): CategoryScores {
-  const f1 = isFiniteNumber(metrics.f1) ? metrics.f1 : null;
+  let f1: number | null = null;
   let map: number | null = null;
+  let precision: number | null = null;
+  let recall: number | null = null;
   for (const [key, value] of Object.entries(metrics)) {
-    if (isMapMetricKey(key) && isFiniteNumber(value)) {
+    if (!isFiniteNumber(value)) continue;
+    if (isMapMetricKey(key)) {
       map = map == null ? value : Math.max(map, value);
+      continue;
     }
+    const iouMatch = matchIouSuffixedMetricKey(key);
+    if (!iouMatch) continue;
+    if (iouMatch.base === 'f1') f1 = f1 == null ? value : Math.max(f1, value);
+    else if (iouMatch.base === 'precision') precision = precision == null ? value : Math.max(precision, value);
+    else recall = recall == null ? value : Math.max(recall, value);
   }
-  const precision = isFiniteNumber(metrics.precision) ? metrics.precision : null;
-  const recall = isFiniteNumber(metrics.recall) ? metrics.recall : null;
   return { f1, map, precision, recall };
 }
 
@@ -178,9 +195,25 @@ function formatMapMetricLabel(key: string): string {
   return `mAP@${lo}`;
 }
 
+// Trims a two-digit IoU suffix ("50", "75", "95") down to its bare fraction (".5", ".75", ".95")
+// for the compact P@.5 / R@.5 / F1@.5 metric labels below.
+function formatIouFraction(iou: string): string {
+  return (Number(iou) / 100)
+    .toFixed(2)
+    .replace(/^0/, '')
+    .replace(/0$/, '');
+}
+
 function labelForMetricKey(key: string): string | null {
   const normalized = key.toLowerCase();
   if (isMapMetricKey(normalized)) return formatMapMetricLabel(normalized);
+  const iouMatch = matchIouSuffixedMetricKey(normalized);
+  if (iouMatch) {
+    const baseLabel = iouMatch.base === 'f1' ? 'F1' : iouMatch.base === 'precision' ? 'Precision' : 'Recall';
+    if (!iouMatch.iou) return baseLabel;
+    const baseAbbrev = iouMatch.base === 'f1' ? 'F1' : iouMatch.base === 'precision' ? 'P' : 'R';
+    return `${baseAbbrev}@${formatIouFraction(iouMatch.iou)}`;
+  }
   return NAMED_METRIC_LABELS[normalized] ?? null;
 }
 
@@ -198,10 +231,12 @@ function collectMetrics(metrics: Record<string, unknown>): MetricValue[] {
 function buildRunNote(entry: Record<string, unknown>): string | null {
   const parts: string[] = ['Zero-shot'];
   if (isFiniteNumber(entry.num_samples)) parts.push(`evaluated on ${entry.num_samples} images`);
+  const rawNotes = toText(entry.notes);
+  if (rawNotes) parts.push(rawNotes);
   return parts.join(' · ');
 }
 
-function buildFinetuneNote(finetune: unknown): string | null {
+function buildFinetuneNote(finetune: unknown, rawNotes: string | null): string | null {
   if (!isRecord(finetune)) return null;
   const parts: string[] = ['Fine-tuned'];
   if (isFiniteNumber(finetune.train_samples)) parts.push(`trained on ${finetune.train_samples} images from this dataset`);
@@ -211,6 +246,7 @@ function buildFinetuneNote(finetune: unknown): string | null {
   if (isFiniteNumber(finetune.weight_decay)) parts.push(`weight decay=${finetune.weight_decay}`);
   if (isFiniteNumber(finetune.split_seed)) parts.push(`seed=${finetune.split_seed}`);
   if (isFiniteNumber(finetune.train_ratio)) parts.push(`train ratio=${finetune.train_ratio}`);
+  if (rawNotes) parts.push(rawNotes);
   return parts.join(' · ');
 }
 
@@ -261,7 +297,7 @@ function makeLeaderboardRow(
     date: toText(entry.timestamp)?.slice(0, 10) ?? null,
     submitted_by: null,
     link: null,
-    notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune) : buildRunNote(entry),
+    notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune, toText(entry.notes)) : buildRunNote(entry),
     optimized: isOptimized(entry.optimized) || (isRecord(finetune) && isOptimized(finetune.optimized)),
     splitBreakdown: buildSplitBreakdown(entry, finetune),
     trainPercentage: computeTrainPercentage(entry, finetune),

static/data/performance/Strawberry-DS_strawberry_detection.json

@@ -0,0 +1,30 @@
+[
+  {
+    "timestamp": "2026-08-03T21:17:26.048567+00:00",
+    "dataset": "Project-AgML/Strawberry-DS_strawberry_detection",
+    "dataset_config": null,
+    "split": "train",
+    "model": "google/gemma-4-12b-it",
+    "task": "detection",
+    "result_type": "zero-shot",
+    "optimized": "no",
+    "train_pct": 0,
+    "test_pct": 100,
+    "val_pct": 0,
+    "num_samples": 247,
+    "device": "cuda",
+    "inference_time_seconds": 156.12,
+    "inference_time_seconds_per_image": 0.632074,
+    "train_time_seconds": 0,
+    "train_time_seconds_per_image": 0,
+    "metrics": {
+      "precision_at_iou50": 0.312398,
+      "recall_at_iou50": 0.353647,
+      "f1_at_iou50": 0.331745,
+      "true_positives": 383,
+      "false_positives": 843,
+      "false_negatives": 700
+    },
+    "notes": "Gemma detection prompt: 'Detect the 2d bounding boxes of the Early-Turning, Green, Late-Turning, Red, Turning, White. Output a JSON list where each entry contains the 2D bounding box in \"box_2d\" and a text label in \"label\" (one of: Early-Turning, Green, Late-Turning, Red, Turning, White). The box_2d coordinates are [y_min, x_min, y_max, x_max], normalized to 0-1000.'. Generation sampling params (vLLM): {'temperature': 1.0, 'top_p': 0.95, 'top_k': 64}, max_tokens=512, batch_size=32. precision_at_iou50/recall_at_iou50/f1_at_iou50 come from greedy same-class box matching at IoU>=0.5 (see match_detections); no mAP is computed since Gemma emits no per-box confidence score to rank predictions by."
+  }
+]