fix: honour the agent override in official eval and test the release cut

.github/workflows/cut-release.yml

@@ -76,28 +76,12 @@ jobs:
 
       - name: Update CHANGELOG
         run: |
-          DATE=$(date +%Y-%m-%d)
-          if grep -q "## \[${{ env.version }}\]" CHANGELOG.md; then
-            echo "CHANGELOG already has [${{ env.version }}] entry — skipping insertion"
-            exit 0
-          fi
-          VERSION="${{ env.version }}" DATE="$DATE" python - <<'PY'
-          import os
-          import sys
-          from pathlib import Path
-
-          path = Path("CHANGELOG.md")
-          text = path.read_text(encoding="utf-8")
-          marker = "## [Unreleased]"
-          version = os.environ["VERSION"]
-          date = os.environ["DATE"]
-          if marker not in text:
-              print("::error::CHANGELOG.md is missing the ## [Unreleased] section.")
-              sys.exit(1)
-          replacement = f"{marker}\n\n## [{version}] - {date}"
-          path.write_text(text.replace(marker, replacement, 1), encoding="utf-8")
-          PY
-          echo "CHANGELOG updated with [${{ env.version }}] - $DATE"
+          # The insertion lives in scripts/check_changelog.py so that
+          # tests/unit/test_check_changelog.py exercises the real code path
+          # instead of a copy that would drift from this workflow.
+          python scripts/check_changelog.py cut \
+            --version "${{ env.version }}" \
+            --date "$(date +%Y-%m-%d)"
 
       - name: Sync plugin versions
         run: |

CHANGELOG.md

@@ -5,6 +5,27 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres
 
 ## [Unreleased]
 
+### Fixed
+- **The official evaluation runner now honours the agent version override.**
+  `prepare_official_eval` read the agent name and version straight from
+  `agentops.yaml` and ignored both the `--agent` flag and the
+  `AGENTOPS_AGENT` environment variable, so a pipeline that pinned a
+  specific agent version still evaluated whatever version the config
+  happened to carry. The override is now resolved in one place
+  (`resolve_agent_override`), applied by `official_eval.py`, and forwarded
+  by the generated GitHub Actions and Azure DevOps workflows. An
+  unexpanded CI token such as `$(AGENTOPS_AGENT)` is treated as absent
+  instead of being parsed as an agent name.
+
+### Changed
+- **The release cut logic moved out of `cut-release.yml` and into
+  `scripts/check_changelog.py cut`.** The workflow used to carry the
+  transformation as an inline Python heredoc, which no test could import,
+  so a regression in it was only visible when a release was already being
+  cut. That is exactly how the 0.8.6 cut broke. The same code now backs
+  both the workflow and the unit tests, and the subcommand is idempotent:
+  re-running it for a version already present in the file is a no-op.
+
 ## [0.8.6] - 2026-08-07
 
 ### Added

scripts/check_changelog.py

@@ -189,6 +189,27 @@ def unreleased_has_content(changelog: str) -> bool:
     return False
 
 
+def apply_release_cut(changelog: str, version: str, date: str) -> str:
+    """Return *changelog* with a `## [version] - date` heading opened below Unreleased.
+
+    This is the whole of the release cut: the workflow never moves text. It
+    inserts the new heading directly under `## [Unreleased]`, so everything that
+    was accumulating there is now attributed to the release, and `[Unreleased]`
+    becomes empty until the next merged change adds to it.
+
+    `cut-release.yml` calls this so a test can exercise the real implementation
+    instead of a copy that would silently drift.
+
+    Raises:
+        ValueError: when the `## [Unreleased]` heading is missing.
+    """
+
+    marker = "## [Unreleased]"
+    if marker not in changelog:
+        raise ValueError("CHANGELOG.md is missing the ## [Unreleased] section.")
+    return changelog.replace(marker, f"{marker}\n\n## [{version}] - {date}", 1)
+
+
 def added_line_numbers(diff: str) -> set[int]:
     """Return new-file line numbers of added lines in a unified diff."""
     added: set[int] = set()
@@ -351,6 +372,27 @@ def _cmd_check_unreleased(args: argparse.Namespace) -> int:
     )
 
 
+def _cmd_cut(args: argparse.Namespace) -> int:
+    path = Path(args.path) if args.path else REPO_ROOT / "CHANGELOG.md"
+    if not path.exists():
+        return _fail(f"{path} does not exist.")
+
+    changelog = path.read_text(encoding="utf-8")
+    if f"## [{args.version}]" in changelog:
+        print(f"CHANGELOG already has [{args.version}], skipping insertion")
+        return 0
+
+    try:
+        path.write_text(
+            apply_release_cut(changelog, args.version, args.date), encoding="utf-8"
+        )
+    except ValueError as exc:
+        return _fail(str(exc))
+
+    print(f"CHANGELOG updated with [{args.version}] - {args.date}")
+    return 0
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     sub = parser.add_subparsers(dest="command", required=True)
@@ -373,6 +415,15 @@ def main(argv: list[str] | None = None) -> int:
     unreleased.add_argument("--path", default=None, help="Path to CHANGELOG.md.")
     unreleased.set_defaults(func=_cmd_check_unreleased)
 
+    cut = sub.add_parser(
+        "cut",
+        help="Open a versioned heading below [Unreleased] (used by cut-release.yml).",
+    )
+    cut.add_argument("--version", required=True, help="Release version, e.g. 0.9.0.")
+    cut.add_argument("--date", required=True, help="Release date, e.g. 2026-01-31.")
+    cut.add_argument("--path", default=None, help="Path to CHANGELOG.md.")
+    cut.set_defaults(func=_cmd_cut)
+
     args = parser.parse_args(argv)
     return int(args.func(args))
 

src/agentops/core/agentops_config.py

@@ -26,10 +26,11 @@
 
 from __future__ import annotations
 
+import os
 import re
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Dict, List, Literal, Mapping, Optional
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
@@ -1327,6 +1328,47 @@ def apply_agent_version_override(agent: str, override: str) -> str:
     )
 
 
+def is_unexpanded_ci_token(value: str) -> bool:
+    """Return True when *value* is an unexpanded CI variable reference.
+
+    Azure DevOps leaves ``$(NAME)`` verbatim when a variable is undefined, and
+    GitHub expressions can leak through as ``${{ ... }}`` the same way. Callers
+    treat that as "no override" instead of trying to evaluate it as an agent
+    target.
+    """
+
+    candidate = value.strip()
+    return candidate.startswith("$(") or candidate.startswith("${{")
+
+
+def resolve_agent_override(
+    agent: str,
+    *,
+    explicit: str | None = None,
+    env: Mapping[str, str] | None = None,
+) -> str | None:
+    """Return the effective ``agent`` expression when an override applies.
+
+    *explicit* is a command-line value such as ``--agent``. When it is ``None``
+    the value of :data:`AGENT_OVERRIDE_ENV` in *env* (defaulting to the process
+    environment) is used instead. Unexpanded CI tokens and blank values are
+    ignored.
+
+    Returns ``None`` when no override applies, so callers can distinguish "keep
+    the configured agent" from "the override happened to match the config".
+    """
+
+    source = os.environ if env is None else env
+    requested = explicit if explicit is not None else source.get(AGENT_OVERRIDE_ENV)
+    if requested is None:
+        return None
+    if is_unexpanded_ci_token(requested):
+        return None
+    if not requested.strip():
+        return None
+    return apply_agent_version_override(agent, requested)
+
+
 def classify_agent(
     agent: str,
     protocol: Optional[Protocol] = None,

src/agentops/pipeline/official_eval.py

@@ -9,7 +9,12 @@
 from pathlib import Path
 from typing import Any, Iterable, Mapping, Sequence
 
-from agentops.core.agentops_config import AgentOpsConfig, classify_agent
+from agentops.core.agentops_config import (
+    AGENT_OVERRIDE_ENV,
+    AgentOpsConfig,
+    classify_agent,
+    resolve_agent_override,
+)
 from agentops.core.config_loader import load_agentops_config
 from agentops.core.evaluators import EvaluatorPreset, detect_dataset_shape, select_evaluators
 from agentops.utils.yaml import load_yaml
@@ -108,11 +113,15 @@ class _EvalPlan:
     warnings: tuple[str, ...]
 
 
-def analyze_official_eval_support(config_path: Path) -> OfficialEvalSupport:
+def analyze_official_eval_support(
+    config_path: Path,
+    *,
+    agent: str | None = None,
+) -> OfficialEvalSupport:
     """Report whether ``config_path`` can use the Microsoft Foundry eval runner."""
 
     try:
-        plan = _build_plan(config_path)
+        plan = _build_plan(config_path, agent=agent)
     except OfficialEvalUnsupported as exc:
         return OfficialEvalSupport(
             eligible=False,
@@ -182,10 +191,17 @@ def prepare_official_eval(
     output_path: Path,
     *,
     deployment_name: str | None = None,
+    agent: str | None = None,
 ) -> OfficialEvalPreparation:
-    """Convert AgentOps JSONL config into Microsoft Foundry AI Agent Evaluation JSON."""
+    """Convert AgentOps JSONL config into Microsoft Foundry AI Agent Evaluation JSON.
+
+    *agent* overrides the ``agent:`` target for this run. When it is ``None`` the
+    ``AGENTOPS_AGENT`` environment variable is honored, so a CI step that just
+    produced a new agent version scores that version instead of the one pinned in
+    ``agentops.yaml``.
+    """
 
-    plan = _build_plan(config_path)
+    plan = _build_plan(config_path, agent=agent)
     deployment = _resolve_deployment_name(deployment_name)
     payload = _build_payload(plan)
 
@@ -224,7 +240,7 @@ def prepare_official_eval(
     )
 
 
-def _build_plan(config_path: Path) -> _EvalPlan:
+def _build_plan(config_path: Path, *, agent: str | None = None) -> _EvalPlan:
     config_path = config_path.resolve()
     if not config_path.exists():
         raise OfficialEvalUnsupported(
@@ -233,7 +249,12 @@ def _build_plan(config_path: Path) -> _EvalPlan:
         )
 
     config = load_agentops_config(config_path)
-    target = classify_agent(config.agent, config.protocol)
+    try:
+        override = resolve_agent_override(config.agent, explicit=agent)
+    except ValueError as exc:
+        raise OfficialEvalUnsupported(str(exc)) from exc
+    agent_expression = override or config.agent
+    target = classify_agent(agent_expression, config.protocol)
     if target.kind != "foundry_prompt":
         raise OfficialEvalUnsupported(
             "Microsoft Foundry AI Agent Evaluation only evaluates Foundry prompt agents "
@@ -262,7 +283,7 @@ def _build_plan(config_path: Path) -> _EvalPlan:
         config=config,
         config_path=config_path,
         dataset_path=dataset_path,
-        agent_ids=config.agent,
+        agent_ids=agent_expression,
         official_evaluators=tuple(official_evaluators),
         skipped_agentops_evaluators=tuple(skipped),
         warnings=tuple(warnings),
@@ -410,6 +431,7 @@ def _command_prepare(args: argparse.Namespace) -> int:
         Path(args.config),
         Path(args.out),
         deployment_name=args.deployment_name,
+        agent=args.agent,
     )
     outputs = {
         "data_path": str(prepared.data_path),
@@ -436,6 +458,13 @@ def main(argv: Sequence[str] | None = None) -> int:
     prepare_parser.add_argument("--config", default="agentops.yaml")
     prepare_parser.add_argument("--out", required=True)
     prepare_parser.add_argument("--deployment-name")
+    prepare_parser.add_argument(
+        "--agent",
+        help=(
+            "Override the agent target for this run. Accepts a bare version or a "
+            f"full agent expression. Falls back to ${AGENT_OVERRIDE_ENV}."
+        ),
+    )
     prepare_parser.add_argument("--github-output")
     prepare_parser.add_argument("--ado-output", action="store_true")
     prepare_parser.add_argument("--print-json", action="store_true")

src/agentops/services/cicd.py

@@ -543,6 +543,9 @@ def _github_eval_substitutions(
           AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}}
           AZURE_OPENAI_MODEL_NAME: ${{{{ vars.AZURE_OPENAI_MODEL_NAME }}}}
           {OFFICIAL_EVAL_ACTION_ENV}: {official_action}
+          # The prepare step reads this and scores the overridden agent version.
+          # An unset variable expands to an empty string, which is ignored.
+          {AGENT_OVERRIDE_ENV}: ${{{{ env.{AGENT_OVERRIDE_ENV} || vars.{AGENT_OVERRIDE_ENV} }}}}
         run: |
           python -m agentops.pipeline.official_eval prepare \\
             --config \"{config_path}\" \\
@@ -745,6 +748,9 @@ def _ado_eval_substitutions(
     AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT)
     AZURE_OPENAI_MODEL_NAME: $(AZURE_OPENAI_MODEL_NAME)
     {OFFICIAL_EVAL_ADO_TASK_ENV}: {official_task}
+    # The prepare step reads this and scores the overridden agent version. An
+    # undefined variable stays literal as $(NAME) and is ignored.
+    {AGENT_OVERRIDE_ENV}: $({AGENT_OVERRIDE_ENV})
 
 - task: {official_task}
   displayName: Run official AI Agent Evaluation

src/agentops/templates/workflows/agentops-deploy-prod-azd.yml

@@ -83,6 +83,10 @@ __AILZ_PREFLIGHT_COMMAND__
             echo "Skipping azd provision for PROD. Use workflow_dispatch with provision=true for reviewed infra changes."
           fi
 
+  # No `environment:` here on purpose. This job is the gate that must run
+  # *before* the protected environment approval, so the reviewer sees the
+  # safety result when they approve. Adding an `environment:` would make the
+  # gate itself wait for the approval it is supposed to inform.
   safety-eval:
     name: Safety eval (gate)
     needs: provision

src/agentops/templates/workflows/agentops-deploy-prod.yml

@@ -43,6 +43,10 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  # No `environment:` here on purpose. This job is the gate that must run
+  # *before* the protected environment approval, so the reviewer sees the
+  # safety result when they approve. Adding an `environment:` would make the
+  # gate itself wait for the approval it is supposed to inform.
   safety-eval:
     name: Safety eval (gate)
     runs-on: ubuntu-latest