From 98d84be7f45d0af0923a0290d52b86daf2a8ee5d Mon Sep 17 00:00:00 2001 From: Jared Smith Date: Mon, 3 Aug 2026 16:42:20 -0700 Subject: [PATCH] updating results with gemma detection first run --- .gitignore | 5 +- package.json | 3 +- scripts/apply_performance_staging.py | 69 +++++++++++++++++ scripts/generate-datasets.mjs | 38 ++++++++-- src/lib/performance.ts | 64 ++++++++++++---- .../Strawberry-DS_strawberry_detection.json | 30 ++++++++ static/data/performance/global.json | 75 +++++++++++++++++++ static/data/performance/index.json | 5 +- .../performance/mango_growth_detection.json | 30 ++++++++ .../pomegranate_growth_detection.json | 30 ++++++++ 10 files changed, 325 insertions(+), 24 deletions(-) create mode 100755 scripts/apply_performance_staging.py create mode 100644 static/data/performance/Strawberry-DS_strawberry_detection.json create mode 100644 static/data/performance/mango_growth_detection.json create mode 100644 static/data/performance/pomegranate_growth_detection.json diff --git a/.gitignore b/.gitignore index d322cf5..8ac5834 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,7 @@ npm-debug.log* yarn-debug.log* yarn-error.log* -.external \ No newline at end of file +.external + +# Dev +performance_staging/ \ No newline at end of file diff --git a/package.json b/package.json index 044d80b..e7c447e 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", "typecheck": "tsc", - "generate-datasets": "node scripts/generate-datasets.mjs" + "generate-datasets": "node scripts/generate-datasets.mjs", + "apply-performance-staging": "node scripts/apply-performance-staging.mjs" }, "dependencies": { "@docusaurus/core": "3.10.1", diff --git a/scripts/apply_performance_staging.py b/scripts/apply_performance_staging.py new file mode 100755 index 0000000..8e33f33 --- /dev/null +++ b/scripts/apply_performance_staging.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Appends run records from performance_staging/*.json into the matching +static/data/performance/.json files. Staging files are left in +place (not cleared) after merging. Run the start command afterward to +regenerate the index.json/global.json leaderboard manifests. + +Usage: python3 scripts/apply_performance_staging.py [--dry-run] +""" + +import json +import sys +from pathlib import Path + +project_root = Path(__file__).resolve().parent.parent +staging_dir = project_root / 'performance_staging' +performance_dir = project_root / 'static' / 'data' / 'performance' +dry_run = '--dry-run' in sys.argv[1:] + + +def read_json_array(file_path): + if not file_path.exists(): + return [] + with open(file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + if not isinstance(data, list): + raise ValueError(f'Expected a JSON array in {file_path}') + return data + + +def write_json_array(file_path, records): + file_path.parent.mkdir(parents=True, exist_ok=True) + with open(file_path, 'w', encoding='utf-8') as f: + f.write(json.dumps(records, indent=2) + '\n') + + +def main(): + if not staging_dir.exists(): + print('No performance_staging directory found, nothing to do.') + return + + staging_files = sorted(f.name for f in staging_dir.iterdir() if f.suffix == '.json') + if not staging_files: + print('performance_staging is empty, nothing to do.') + return + + for file in staging_files: + staging_path = staging_dir / file + target_path = performance_dir / file + + new_records = read_json_array(staging_path) + existing_records = read_json_array(target_path) + merged = existing_records + new_records + + print( + f'{file}: {len(existing_records)} existing + {len(new_records)} staged = {len(merged)} total' + ) + + if not dry_run: + write_json_array(target_path, merged) + + if dry_run: + print('\nDry run - no files were changed.') + return + + print('\nDone.') + + +if __name__ == '__main__': + main() diff --git a/scripts/generate-datasets.mjs b/scripts/generate-datasets.mjs index 9e41ef2..3a60614 100644 --- a/scripts/generate-datasets.mjs +++ b/scripts/generate-datasets.mjs @@ -58,7 +58,7 @@ function buildDatasetMetadataLookup(...manifests) { // rendering of the same files — keep the two in sync if the run schema changes. const TASK_METRIC_KEYS = { classification: ['f1', 'accuracy', 'top1_accuracy'], - detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5'], + detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5', 'f1_at_iou50', 'f1'], segmentation: ['miou', 'iou', 'mean_iou'], }; @@ -82,16 +82,36 @@ function isMapMetricKey(key) { return /^m?ap([_@-]|$)/i.test(key); } +// Detection runs may report F1/precision/recall at a specific IoU match threshold rather than +// (or alongside) the aggregate keys above, e.g. "f1_at_iou50", "precision_at_iou50" — mirrored +// in src/lib/performance.ts, keep in sync. +const IOU_SUFFIXED_METRIC_RE = /^(f1|precision|prec|recall|rec)(?:_at_iou(\d{2,3}))?$/i; + +function matchIouSuffixedMetricKey(key) { + const match = IOU_SUFFIXED_METRIC_RE.exec(key); + if (!match) return null; + const base = match[1].toLowerCase(); + const normalizedBase = base === 'f1' ? 'f1' : base === 'precision' || base === 'prec' ? 'precision' : 'recall'; + return { base: normalizedBase, iou: match[2] ?? null }; +} + function computeCategoryScores(metrics) { - const f1 = isFiniteNumber(metrics.f1) ? metrics.f1 : null; + let f1 = null; let map = null; + let precision = null; + let recall = null; for (const [key, value] of Object.entries(metrics)) { - if (isMapMetricKey(key) && isFiniteNumber(value)) { + if (!isFiniteNumber(value)) continue; + if (isMapMetricKey(key)) { map = map == null ? value : Math.max(map, value); + continue; } + const iouMatch = matchIouSuffixedMetricKey(key); + if (!iouMatch) continue; + if (iouMatch.base === 'f1') f1 = f1 == null ? value : Math.max(f1, value); + else if (iouMatch.base === 'precision') precision = precision == null ? value : Math.max(precision, value); + else recall = recall == null ? value : Math.max(recall, value); } - const precision = isFiniteNumber(metrics.precision) ? metrics.precision : null; - const recall = isFiniteNumber(metrics.recall) ? metrics.recall : null; return { f1, map, precision, recall }; } @@ -99,14 +119,16 @@ function buildRunNote(entry) { const parts = []; if (isFiniteNumber(entry.num_samples)) parts.push(`${entry.num_samples} samples`); if (typeof entry.device === 'string' && entry.device.trim()) parts.push(entry.device.trim()); + if (typeof entry.notes === 'string' && entry.notes.trim()) parts.push(entry.notes.trim()); return parts.length ? parts.join(' · ') : null; } -function buildFinetuneNote(finetune) { +function buildFinetuneNote(finetune, rawNotes) { if (!finetune || typeof finetune !== 'object') return null; const parts = []; if (isFiniteNumber(finetune.epochs)) parts.push(`${finetune.epochs} epochs`); if (isFiniteNumber(finetune.train_samples)) parts.push(`${finetune.train_samples} train samples`); + if (rawNotes) parts.push(rawNotes); return parts.length ? parts.join(' · ') : null; } @@ -139,7 +161,9 @@ function makeLeaderboardRow(entry, metricKey, variant) { date: typeof entry.timestamp === 'string' ? entry.timestamp.slice(0, 10) : null, submitted_by: null, link: null, - notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune) : buildRunNote(entry), + notes: variant === 'fine-tuned' + ? buildFinetuneNote(finetune, typeof entry.notes === 'string' && entry.notes.trim() ? entry.notes.trim() : null) + : buildRunNote(entry), optimized: isOptimized(entry.optimized) || (finetune != null && typeof finetune === 'object' && isOptimized(finetune.optimized)), platform: typeof entry.device === 'string' && entry.device.trim() ? entry.device.trim() : null, splitBreakdown: buildSplitBreakdown(entry, finetune), diff --git a/src/lib/performance.ts b/src/lib/performance.ts index 856042b..06e4840 100644 --- a/src/lib/performance.ts +++ b/src/lib/performance.ts @@ -31,9 +31,9 @@ export interface MetricValue { } export function classifyMetricLabel(label: string): MetricCategory { - if (label === 'F1') return 'f1'; + if (label === 'F1' || label.startsWith('F1@')) return 'f1'; if (label.startsWith('mAP')) return 'map'; - if (label === 'Precision' || label === 'Recall') return 'precision_recall'; + if (label === 'Precision' || label === 'Recall' || label.startsWith('P@') || label.startsWith('R@')) return 'precision_recall'; return 'other'; } @@ -100,7 +100,7 @@ function normalizeEntry(raw: unknown): PerformanceEntry | null { // global.json from the same files at build time — keep the two in sync if the run schema changes. const TASK_METRIC_KEYS: Record = { classification: ['f1', 'accuracy', 'top1_accuracy'], - detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5'], + detection: ['map', 'map_50', 'map50', 'mAP', 'mAP@0.5', 'f1_at_iou50', 'f1'], segmentation: ['miou', 'iou', 'mean_iou'], }; @@ -122,13 +122,8 @@ function resolveMetricKey(task: unknown, metrics: Record): stri // particular may report mAP at several IoU thresholds (map_50, map_75, map_50_95, ...); each is // surfaced as its own labeled value rather than collapsed into one number. const NAMED_METRIC_LABELS: Record = { - f1: 'F1', accuracy: 'Accuracy', top1_accuracy: 'Top-1 Accuracy', - precision: 'Precision', - prec: 'Precision', - recall: 'Recall', - rec: 'Recall', miou: 'mIoU', iou: 'IoU', mean_iou: 'mIoU', @@ -138,6 +133,21 @@ function isMapMetricKey(key: string): boolean { return /^m?ap([_@-]|$)/i.test(key); } +// Detection runs may report F1/precision/recall at a specific IoU match threshold rather than +// (or alongside) the aggregate keys above, e.g. "f1_at_iou50", "precision_at_iou50" — Gemma's +// zero-shot detection results are matched greedily per-class at IoU>=0.5 and report exactly this +// shape (see static/data/performance/pomegranate_growth_detection.json). Mirrored in +// scripts/generate-datasets.mjs — keep in sync. +const IOU_SUFFIXED_METRIC_RE = /^(f1|precision|prec|recall|rec)(?:_at_iou(\d{2,3}))?$/i; + +function matchIouSuffixedMetricKey(key: string): { base: 'f1' | 'precision' | 'recall'; iou: string | null } | null { + const match = IOU_SUFFIXED_METRIC_RE.exec(key); + if (!match) return null; + const base = match[1].toLowerCase(); + const normalizedBase = base === 'f1' ? 'f1' : base === 'precision' || base === 'prec' ? 'precision' : 'recall'; + return { base: normalizedBase, iou: match[2] ?? null }; +} + export interface CategoryScores { f1: number | null; map: number | null; @@ -151,15 +161,22 @@ export interface CategoryScores { // and recall are ranked/percentiled independently, not averaged into one blended score. // Mirrored in scripts/generate-datasets.mjs — keep in sync. function computeCategoryScores(metrics: Record): CategoryScores { - const f1 = isFiniteNumber(metrics.f1) ? metrics.f1 : null; + let f1: number | null = null; let map: number | null = null; + let precision: number | null = null; + let recall: number | null = null; for (const [key, value] of Object.entries(metrics)) { - if (isMapMetricKey(key) && isFiniteNumber(value)) { + if (!isFiniteNumber(value)) continue; + if (isMapMetricKey(key)) { map = map == null ? value : Math.max(map, value); + continue; } + const iouMatch = matchIouSuffixedMetricKey(key); + if (!iouMatch) continue; + if (iouMatch.base === 'f1') f1 = f1 == null ? value : Math.max(f1, value); + else if (iouMatch.base === 'precision') precision = precision == null ? value : Math.max(precision, value); + else recall = recall == null ? value : Math.max(recall, value); } - const precision = isFiniteNumber(metrics.precision) ? metrics.precision : null; - const recall = isFiniteNumber(metrics.recall) ? metrics.recall : null; return { f1, map, precision, recall }; } @@ -178,9 +195,25 @@ function formatMapMetricLabel(key: string): string { return `mAP@${lo}`; } +// Trims a two-digit IoU suffix ("50", "75", "95") down to its bare fraction (".5", ".75", ".95") +// for the compact P@.5 / R@.5 / F1@.5 metric labels below. +function formatIouFraction(iou: string): string { + return (Number(iou) / 100) + .toFixed(2) + .replace(/^0/, '') + .replace(/0$/, ''); +} + function labelForMetricKey(key: string): string | null { const normalized = key.toLowerCase(); if (isMapMetricKey(normalized)) return formatMapMetricLabel(normalized); + const iouMatch = matchIouSuffixedMetricKey(normalized); + if (iouMatch) { + const baseLabel = iouMatch.base === 'f1' ? 'F1' : iouMatch.base === 'precision' ? 'Precision' : 'Recall'; + if (!iouMatch.iou) return baseLabel; + const baseAbbrev = iouMatch.base === 'f1' ? 'F1' : iouMatch.base === 'precision' ? 'P' : 'R'; + return `${baseAbbrev}@${formatIouFraction(iouMatch.iou)}`; + } return NAMED_METRIC_LABELS[normalized] ?? null; } @@ -198,10 +231,12 @@ function collectMetrics(metrics: Record): MetricValue[] { function buildRunNote(entry: Record): string | null { const parts: string[] = ['Zero-shot']; if (isFiniteNumber(entry.num_samples)) parts.push(`evaluated on ${entry.num_samples} images`); + const rawNotes = toText(entry.notes); + if (rawNotes) parts.push(rawNotes); return parts.join(' · '); } -function buildFinetuneNote(finetune: unknown): string | null { +function buildFinetuneNote(finetune: unknown, rawNotes: string | null): string | null { if (!isRecord(finetune)) return null; const parts: string[] = ['Fine-tuned']; if (isFiniteNumber(finetune.train_samples)) parts.push(`trained on ${finetune.train_samples} images from this dataset`); @@ -211,6 +246,7 @@ function buildFinetuneNote(finetune: unknown): string | null { if (isFiniteNumber(finetune.weight_decay)) parts.push(`weight decay=${finetune.weight_decay}`); if (isFiniteNumber(finetune.split_seed)) parts.push(`seed=${finetune.split_seed}`); if (isFiniteNumber(finetune.train_ratio)) parts.push(`train ratio=${finetune.train_ratio}`); + if (rawNotes) parts.push(rawNotes); return parts.join(' · '); } @@ -261,7 +297,7 @@ function makeLeaderboardRow( date: toText(entry.timestamp)?.slice(0, 10) ?? null, submitted_by: null, link: null, - notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune) : buildRunNote(entry), + notes: variant === 'fine-tuned' ? buildFinetuneNote(finetune, toText(entry.notes)) : buildRunNote(entry), optimized: isOptimized(entry.optimized) || (isRecord(finetune) && isOptimized(finetune.optimized)), splitBreakdown: buildSplitBreakdown(entry, finetune), trainPercentage: computeTrainPercentage(entry, finetune), diff --git a/static/data/performance/Strawberry-DS_strawberry_detection.json b/static/data/performance/Strawberry-DS_strawberry_detection.json new file mode 100644 index 0000000..1400a9e --- /dev/null +++ b/static/data/performance/Strawberry-DS_strawberry_detection.json @@ -0,0 +1,30 @@ +[ + { + "timestamp": "2026-08-03T21:17:26.048567+00:00", + "dataset": "Project-AgML/Strawberry-DS_strawberry_detection", + "dataset_config": null, + "split": "train", + "model": "google/gemma-4-12b-it", + "task": "detection", + "result_type": "zero-shot", + "optimized": "no", + "train_pct": 0, + "test_pct": 100, + "val_pct": 0, + "num_samples": 247, + "device": "cuda", + "inference_time_seconds": 156.12, + "inference_time_seconds_per_image": 0.632074, + "train_time_seconds": 0, + "train_time_seconds_per_image": 0, + "metrics": { + "precision_at_iou50": 0.312398, + "recall_at_iou50": 0.353647, + "f1_at_iou50": 0.331745, + "true_positives": 383, + "false_positives": 843, + "false_negatives": 700 + }, + "notes": "Gemma detection prompt: 'Detect the 2d bounding boxes of the Early-Turning, Green, Late-Turning, Red, Turning, White. Output a JSON list where each entry contains the 2D bounding box in \"box_2d\" and a text label in \"label\" (one of: Early-Turning, Green, Late-Turning, Red, Turning, White). The box_2d coordinates are [y_min, x_min, y_max, x_max], normalized to 0-1000.'. Generation sampling params (vLLM): {'temperature': 1.0, 'top_p': 0.95, 'top_k': 64}, max_tokens=512, batch_size=32. precision_at_iou50/recall_at_iou50/f1_at_iou50 come from greedy same-class box matching at IoU>=0.5 (see match_detections); no mAP is computed since Gemma emits no per-box confidence score to rank predictions by." + } +] diff --git a/static/data/performance/global.json b/static/data/performance/global.json index b627dee..84a7857 100644 --- a/static/data/performance/global.json +++ b/static/data/performance/global.json @@ -74,6 +74,31 @@ "splitBreakdown": "-/100/-", "datasetConfig": "raw" }, + { + "model": "google/gemma-4-12b-it", + "dataset": "Strawberry-DS_strawberry_detection", + "percentiles": { + "f1": 100, + "map": null, + "precision": 100, + "recall": 100 + }, + "scores": { + "f1": 0.331745, + "map": null, + "precision": 0.312398, + "recall": 0.353647 + }, + "crop_types": [ + "strawberry" + ], + "machine_learning_task": "object_detection", + "variant": "zero-shot", + "optimized": false, + "platform": "cuda", + "splitBreakdown": "-/100/-", + "datasetConfig": "raw" + }, { "model": "google/siglip-base-patch16-224", "dataset": "ash_gourd_disease_classification", @@ -373,5 +398,55 @@ "platform": "cuda", "splitBreakdown": "-/100/-", "datasetConfig": "raw" + }, + { + "model": "google/gemma-4-12b-it", + "dataset": "mango_growth_detection", + "percentiles": { + "f1": 100, + "map": null, + "precision": 100, + "recall": 100 + }, + "scores": { + "f1": 0.301615, + "map": null, + "precision": 0.278044, + "recall": 0.329553 + }, + "crop_types": [ + "mango" + ], + "machine_learning_task": "object_detection", + "variant": "zero-shot", + "optimized": false, + "platform": "cuda", + "splitBreakdown": "-/100/-", + "datasetConfig": "raw" + }, + { + "model": "google/gemma-4-12b-it", + "dataset": "pomegranate_growth_detection", + "percentiles": { + "f1": 100, + "map": null, + "precision": 100, + "recall": 100 + }, + "scores": { + "f1": 0.478479, + "map": null, + "precision": 0.506368, + "recall": 0.453501 + }, + "crop_types": [ + "pomegranate" + ], + "machine_learning_task": "object_detection", + "variant": "zero-shot", + "optimized": false, + "platform": "cuda", + "splitBreakdown": "-/100/-", + "datasetConfig": "raw" } ] \ No newline at end of file diff --git a/static/data/performance/index.json b/static/data/performance/index.json index 16c7ae3..fcc7e45 100644 --- a/static/data/performance/index.json +++ b/static/data/performance/index.json @@ -1,7 +1,10 @@ [ "MoringaLeafNet_disease_classification", + "Strawberry-DS_strawberry_detection", "ash_gourd_disease_classification", "cotton_leaf_disease_classification", "eggplant_leaf_disease_classification", - "groundnut_leaf_disease_classification" + "groundnut_leaf_disease_classification", + "mango_growth_detection", + "pomegranate_growth_detection" ] \ No newline at end of file diff --git a/static/data/performance/mango_growth_detection.json b/static/data/performance/mango_growth_detection.json new file mode 100644 index 0000000..b22002f --- /dev/null +++ b/static/data/performance/mango_growth_detection.json @@ -0,0 +1,30 @@ +[ + { + "timestamp": "2026-08-03T23:31:33.978541+00:00", + "dataset": "Project-AgML/mango_growth_detection", + "dataset_config": null, + "split": "train", + "model": "google/gemma-4-12b-it", + "task": "detection", + "result_type": "zero-shot", + "optimized": "no", + "train_pct": 0.0, + "test_pct": 100.0, + "val_pct": 0.0, + "num_samples": 2000, + "device": "cuda", + "inference_time_seconds": 652.82, + "inference_time_seconds_per_image": 0.326409, + "train_time_seconds": 0.0, + "train_time_seconds_per_image": 0.0, + "metrics": { + "precision_at_iou50": 0.278044, + "recall_at_iou50": 0.329553, + "f1_at_iou50": 0.301615, + "true_positives": 1046, + "false_positives": 2716, + "false_negatives": 2128 + }, + "notes": "Gemma detection prompt: 'Detect the 2d bounding boxes of the Early-Fruit, Mature, Premature, Ripe. Output a JSON list where each entry contains the 2D bounding box in \"box_2d\" and a text label in \"label\" (one of: Early-Fruit, Mature, Premature, Ripe). The box_2d coordinates are [y_min, x_min, y_max, x_max], normalized to 0-1000.'. Generation sampling params (vLLM): {'temperature': 1.0, 'top_p': 0.95, 'top_k': 64}, max_tokens=512, batch_size=32. precision_at_iou50/recall_at_iou50/f1_at_iou50 come from greedy same-class box matching at IoU>=0.5 (see match_detections); no mAP is computed since Gemma emits no per-box confidence score to rank predictions by." + } +] diff --git a/static/data/performance/pomegranate_growth_detection.json b/static/data/performance/pomegranate_growth_detection.json new file mode 100644 index 0000000..f2178f5 --- /dev/null +++ b/static/data/performance/pomegranate_growth_detection.json @@ -0,0 +1,30 @@ +[ + { + "timestamp": "2026-08-03T21:45:21.667950+00:00", + "dataset": "Project-AgML/pomegranate_growth_detection", + "dataset_config": null, + "split": "train", + "model": "google/gemma-4-12b-it", + "task": "detection", + "result_type": "zero-shot", + "optimized": "no", + "train_pct": 0, + "test_pct": 100, + "val_pct": 0, + "num_samples": 5857, + "device": "cuda", + "inference_time_seconds": 1603.74, + "inference_time_seconds_per_image": 0.273815, + "train_time_seconds": 0, + "train_time_seconds_per_image": 0, + "metrics": { + "precision_at_iou50": 0.506368, + "recall_at_iou50": 0.453501, + "f1_at_iou50": 0.478479, + "true_positives": 5208, + "false_positives": 5077, + "false_negatives": 6276 + }, + "notes": "Gemma detection prompt: 'Detect the 2d bounding boxes of the bud, flower, early-fruit, mid-growth, mature. Output a JSON list where each entry contains the 2D bounding box in \"box_2d\" and a text label in \"label\" (one of: bud, flower, early-fruit, mid-growth, mature). The box_2d coordinates are [y_min, x_min, y_max, x_max], normalized to 0-1000.'. Generation sampling params (vLLM): {'temperature': 1.0, 'top_p': 0.95, 'top_k': 64}, max_tokens=512, batch_size=32. precision_at_iou50/recall_at_iou50/f1_at_iou50 come from greedy same-class box matching at IoU>=0.5 (see match_detections); no mAP is computed since Gemma emits no per-box confidence score to rank predictions by." + } +]