diff --git a/nemo_retriever/src/nemo_retriever/harness/portal/app.py b/nemo_retriever/src/nemo_retriever/harness/portal/app.py index 92eff5d708..81581c18e2 100644 --- a/nemo_retriever/src/nemo_retriever/harness/portal/app.py +++ b/nemo_retriever/src/nemo_retriever/harness/portal/app.py @@ -36,6 +36,13 @@ from nemo_retriever.harness import history from nemo_retriever.harness.config import VALID_EVALUATION_MODES +from nemo_retriever.harness.contracts import MODE_TO_RUN_MODE + +# Execution modes accepted by the current `retriever harness run --mode` surface +# (local, batch, service). Kept in sync with the harness contract so the portal +# never dispatches a job the runner cannot translate into a valid CLI invocation. +VALID_RUN_MODES = tuple(MODE_TO_RUN_MODE) +DEFAULT_RUN_MODE = "local" mimetypes.add_type("text/javascript", ".jsx") @@ -232,6 +239,9 @@ class TriggerRequest(BaseModel): git_commit: str | None = None nsys_profile: bool = False graph_id: int | None = None + # Execution mode for the harness `--mode` flag: "local", "batch", or + # "service". ``run_mode`` is kept as a legacy alias for older clients. + mode: str | None = None run_mode: str | None = None service_url: str | None = None service_max_concurrency: int | None = None @@ -1521,6 +1531,54 @@ async def get_yaml_config(): return {"datasets": {}, "presets": {}, "active": {}} +@app.get("/api/harness-info") +async def get_harness_info(): + """Expose the current harness run contract so the UI stays in sync. + + Reports the execution modes accepted by ``retriever harness run --mode`` + and the code-owned benchmark/runset registry. The registry import is + intentionally lightweight (no ingest/query modules) so this endpoint stays + cheap to call from the trigger UI. + """ + modes = [ + {"value": mode, "ingest_run_mode": MODE_TO_RUN_MODE[mode]} + for mode in sorted(MODE_TO_RUN_MODE) + ] + benchmarks: list[dict[str, Any]] = [] + runsets: list[dict[str, Any]] = [] + try: + from nemo_retriever.harness.benchmark_registry import list_benchmarks, list_runsets + + for spec in list_benchmarks(): + benchmarks.append( + { + "name": spec.name, + "dataset": spec.dataset, + "tags": list(spec.tags), + "description": spec.description, + } + ) + for runset in list_runsets(): + runsets.append( + { + "name": runset.name, + "runs": list(runset.runs), + "tags": list(runset.tags), + "description": runset.description, + } + ) + except Exception as exc: # pragma: no cover - registry import is best-effort + logger.warning("Failed to load harness benchmark registry: %s", exc) + + return { + "default_mode": DEFAULT_RUN_MODE, + "modes": modes, + "evaluation_modes": sorted(VALID_EVALUATION_MODES), + "benchmarks": benchmarks, + "runsets": runsets, + } + + # --------------------------------------------------------------------------- # Managed Dataset CRUD # --------------------------------------------------------------------------- @@ -2228,8 +2286,18 @@ async def trigger_run(req: TriggerRequest): req.git_commit, ) - if req.run_mode == "service": - merged_overrides["run_mode"] = "service" + # Resolve the execution mode against the harness contract. ``mode`` is the + # canonical field; ``run_mode`` is accepted as a legacy alias. Anything the + # current harness cannot map to ``retriever harness run --mode`` is rejected + # up front so a job never reaches a runner in an unrunnable state. + run_mode = (req.mode or req.run_mode or DEFAULT_RUN_MODE).strip().lower() + if run_mode not in MODE_TO_RUN_MODE: + raise HTTPException( + status_code=422, + detail=f"mode must be one of {sorted(MODE_TO_RUN_MODE)}, got {run_mode!r}", + ) + merged_overrides["run_mode"] = run_mode + if run_mode == "service": if req.service_url: merged_overrides["service_url"] = req.service_url if req.service_max_concurrency: diff --git a/nemo_retriever/src/nemo_retriever/harness/portal/mcp_tools.py b/nemo_retriever/src/nemo_retriever/harness/portal/mcp_tools.py index 696c4e4148..f8de1417f0 100644 --- a/nemo_retriever/src/nemo_retriever/harness/portal/mcp_tools.py +++ b/nemo_retriever/src/nemo_retriever/harness/portal/mcp_tools.py @@ -27,8 +27,9 @@ category="Jobs", description=( "Trigger a benchmark run on a dataset with an optional preset. " - "Returns the job ID and status. Use list_datasets and list_presets " - "first to discover valid names." + "Optionally set the execution mode ('local', 'batch', or 'service'; " + "defaults to 'local'). Returns the job ID and status. Use list_datasets " + "and list_presets first to discover valid names." ), tags=["write", "jobs"], ) @@ -37,32 +38,43 @@ def trigger_benchmark_run( preset: str | None = None, runner_id: int | None = None, tags: list[str] | None = None, + mode: str | None = None, ) -> dict[str, Any]: """Trigger a benchmark run.""" + from nemo_retriever.harness.contracts import MODE_TO_RUN_MODE from nemo_retriever.harness.portal.app import ( + DEFAULT_RUN_MODE, _resolve_dataset_config, _resolve_git_override, _resolve_preset_overrides, ) - dataset_path, dataset_overrides = _resolve_dataset_config(dataset) + run_mode = (mode or DEFAULT_RUN_MODE).strip().lower() + if run_mode not in MODE_TO_RUN_MODE: + raise ValueError(f"mode must be one of {sorted(MODE_TO_RUN_MODE)}, got {run_mode!r}") + + dataset_path, dataset_overrides, dataset_meta = _resolve_dataset_config(dataset) preset_overrides = _resolve_preset_overrides(preset) merged_overrides = {**(dataset_overrides or {}), **preset_overrides} + merged_overrides["run_mode"] = run_mode pinned_sha, pinned_ref = _resolve_git_override(None, None) - job = history.create_job( - { - "trigger_source": "mcp", - "dataset": dataset, - "dataset_path": dataset_path, - "dataset_overrides": merged_overrides if merged_overrides else None, - "preset": preset, - "assigned_runner_id": runner_id, - "git_commit": pinned_sha, - "git_ref": pinned_ref, - "tags": tags or ["mcp-triggered"], - } - ) + job_data: dict[str, Any] = { + "trigger_source": "mcp", + "dataset": dataset, + "dataset_path": dataset_path, + "dataset_overrides": merged_overrides if merged_overrides else None, + "preset": preset, + "assigned_runner_id": runner_id, + "git_commit": pinned_sha, + "git_ref": pinned_ref, + "tags": tags or ["mcp-triggered"], + } + if dataset_meta: + job_data["dataset_id"] = dataset_meta["dataset_id"] + job_data["dataset_config_hash"] = dataset_meta["dataset_config_hash"] + + job = history.create_job(job_data) return {"job_id": job["id"], "status": "pending"} diff --git a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/datasets.jsx b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/datasets.jsx index c1062539f9..0418f312da 100644 --- a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/datasets.jsx +++ b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/datasets.jsx @@ -201,7 +201,7 @@ function DatasetFormModal({ dataset, onClose, onSaved }) { beir_dataset_name: dataset?.beir_dataset_name || "", beir_split: dataset?.beir_split || "test", beir_query_language: dataset?.beir_query_language || "", - beir_doc_id_field: dataset?.beir_doc_id_field || "pdf_basename", + beir_doc_id_field: dataset?.beir_doc_id_field || "pdf_page", beir_ks: (dataset?.beir_ks || [1,3,5,10]).join(", "), embed_model_name: dataset?.embed_model_name || "", embed_modality: dataset?.embed_modality || "text", @@ -288,10 +288,14 @@ function DatasetFormModal({ dataset, onClose, onSaved }) {
@@ -362,6 +366,7 @@ function DatasetFormModal({ dataset, onClose, onSaved }) { diff --git a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runners.jsx b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runners.jsx index 3804191eb4..e49170775f 100644 --- a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runners.jsx +++ b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runners.jsx @@ -46,7 +46,7 @@ function RunnersView({ runners, loading, onRefresh, githubRepoUrl }) {
No runners registered
- Register a runner manually or use retriever harness runner start --manager-url <portal-url> + Register a runner with the button above, then point your runner agent at this portal so it can poll the runner work API for jobs.
) : pg.pageData.map(r => { diff --git a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runs.jsx b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runs.jsx index 8f4b6d6fa7..f6d6b77241 100644 --- a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runs.jsx +++ b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/runs.jsx @@ -424,7 +424,7 @@ function RunsView({ runs, datasets, loading, filterDataset, setFilterDataset, fi
No runs found
- Trigger a run or use retriever harness backfill to import existing results. + Trigger a run from the portal, or run retriever harness run <benchmark> on a runner to record results here.
) : pg.pageData.map(run => ( diff --git a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/trigger.jsx b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/trigger.jsx index eb1a12214a..8c9eac0180 100644 --- a/nemo_retriever/src/nemo_retriever/harness/portal/static/views/trigger.jsx +++ b/nemo_retriever/src/nemo_retriever/harness/portal/static/views/trigger.jsx @@ -7,6 +7,8 @@ function TriggerModal({ onClose, onTriggered }) { const [dataset, setDataset] = useState(""); const [preset, setPreset] = useState(""); const [pipelineMode, setPipelineMode] = useState("preset"); + const [runMode, setRunMode] = useState("local"); + const [modeOptions, setModeOptions] = useState(["local", "batch", "service"]); const [graphId, setGraphId] = useState(""); const [runnerId, setRunnerId] = useState(""); const [submitting, setSubmitting] = useState(false); @@ -30,6 +32,11 @@ function TriggerModal({ onClose, onTriggered }) { if (cfg.datasets?.length) setDataset(cfg.datasets[0]); if (cfg.presets?.length) setPreset(cfg.presets[0]); }); + fetch("/api/harness-info").then(r=>r.json()).then(info => { + const modes = (info.modes || []).map(m => m.value).filter(Boolean); + if (modes.length) setModeOptions(modes); + if (info.default_mode) setRunMode(info.default_mode); + }).catch(()=>{}); fetch("/api/runners").then(r=>r.json()).then(setRunners).catch(()=>{}); fetch("/api/graphs").then(r=>r.json()).then(list => { const arr = Array.isArray(list) ? list : []; @@ -61,12 +68,12 @@ function TriggerModal({ onClose, onTriggered }) { preset: pipelineMode === "preset" ? (preset || null) : null, runner_id: runnerId ? parseInt(runnerId, 10) : null, nsys_profile: nsysProfile, + mode: runMode, }; if (pipelineMode === "graph") { payload.graph_id = parseInt(graphId, 10); } - if (pipelineMode === "service") { - payload.run_mode = "service"; + if (runMode === "service") { payload.service_url = serviceUrl.trim(); payload.service_max_concurrency = serviceMaxConcurrency; } @@ -93,13 +100,20 @@ function TriggerModal({ onClose, onTriggered }) { const labelStyle = {display:'block',fontSize:'12px',fontWeight:500,color:'var(--nv-text-muted)',marginBottom:'6px',textTransform:'uppercase',letterSpacing:'0.04em'}; const hintStyle = {fontSize:'11px',color:'var(--nv-text-dim)',marginTop:'4px',lineHeight:'1.5'}; - const modeBtn = (id, label) => ({ + const toggleBtn = (active) => ({ fontSize:'11px',padding:'5px 12px',flex:1,justifyContent:'center',textAlign:'center', - background: pipelineMode===id ? 'rgba(118,185,0,0.12)' : 'transparent', - color: pipelineMode===id ? 'var(--nv-green)' : 'var(--nv-text-dim)', - border: `1px solid ${pipelineMode===id ? 'rgba(118,185,0,0.3)' : 'var(--nv-border)'}`, - cursor:'pointer', borderRadius:'6px', fontWeight: pipelineMode===id ? 600 : 400, + background: active ? 'rgba(118,185,0,0.12)' : 'transparent', + color: active ? 'var(--nv-green)' : 'var(--nv-text-dim)', + border: `1px solid ${active ? 'rgba(118,185,0,0.3)' : 'var(--nv-border)'}`, + cursor:'pointer', borderRadius:'6px', fontWeight: active ? 600 : 400, }); + const modeBtn = (id) => toggleBtn(pipelineMode===id); + const runModeLabels = { local: "Local", batch: "Batch", service: "Service" }; + const runModeHints = { + local: "In-process ingest/query on the runner. Best for smoke and small BEIR datasets.", + batch: "Ray-backed batch ingest. Use for large BEIR corpora (BO767, FinanceBench, Earnings, ViDoRe).", + service: "Runs against an already-deployed Retriever service. No GPU or Ray cluster needed on the runner.", + }; const selectedGraph = graphs.find(g => String(g.id) === graphId); @@ -119,6 +133,22 @@ function TriggerModal({ onClose, onTriggered }) {
+ {/* Execution Mode Toggle — maps to `retriever harness run --mode` */} +
+ +
+ {modeOptions.map(m => ( + + ))} +
+
{runModeHints[runMode] || ""}
+
+ {/* Pipeline Mode Toggle */}
@@ -127,18 +157,15 @@ function TriggerModal({ onClose, onTriggered }) { Preset -
{graphs.length === 0 && pipelineMode === "preset" && (
No saved graphs available. Create one in the Designer view to enable graph pipeline runs.
)} - {pipelineMode === "service" && ( -
Uploads documents to a running retriever service and measures ingestion throughput. No GPU or Ray cluster needed on the runner.
+ {runMode === "service" && ( +
Graph pipelines are not available in service mode; runs use the deployed service's ingest/query APIs.
)} @@ -168,7 +195,7 @@ function TriggerModal({ onClose, onTriggered }) { )} - {pipelineMode === "service" && ( + {runMode === "service" && (
@@ -307,8 +334,8 @@ function TriggerModal({ onClose, onTriggered }) {
-