From 36ea46d7295fe2f5b88bbad89984f8ef1a187664 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Sun, 9 Aug 2026 09:25:18 -0300 Subject: [PATCH] fix: honour the agent override in official eval and test the release cut Closes five audit follow-ups that shared one root cause: behaviour that nothing exercised. #398 - the official evaluation runner ignored the agent override. prepare_official_eval read the agent name and version straight from agentops.yaml, so --agent and AGENTOPS_AGENT were accepted and then discarded. A pipeline pinning a version still evaluated whatever the config carried. Resolution now happens once in resolve_agent_override, official_eval.py applies it, and the generated GitHub Actions and Azure DevOps workflows forward it. An unexpanded CI token such as $(AGENTOPS_AGENT) is treated as absent rather than parsed as a name. #404 - the release cut logic lived as an inline Python heredoc inside cut-release.yml, where no test could import it, so a regression only surfaced while a release was being cut. That is how the 0.8.6 cut broke. The transformation moved to scripts/check_changelog.py cut, the workflow calls it, and the unit tests exercise the same code. The subcommand is idempotent and rejects a changelog with no [Unreleased] marker. #399 - the CHANGELOG gate had no fixture proving Dependabot pull requests pass. They do, but only because the EXEMPT_AUTHORS check runs before file classification. PR #359 touches pyproject.toml, which is shipping code, so reordering those checks would block every dependency bump. Three real payloads now pin that ordering. #395 - the Azure DevOps CI test covered a single eval runner. It is now parametrized over azd-ai-agent-eval, agentops-cloud and agentops-local. #396 - the safety-eval job deliberately omits environment: so it cannot be gated behind a manual approval. Both production templates carry a marker comment explaining why, and two tests read the generated file to keep it. Test plan: 1206 passed, 6 skipped. The one deselected failure, test_list_role_definition_ids_extracts_guid_suffix, is environmental (azure.mgmt is absent from the local venv) and unrelated to this change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bcb9c0b6-d506-46dc-90d2-8120413166ee --- .github/workflows/cut-release.yml | 28 +--- CHANGELOG.md | 21 +++ scripts/check_changelog.py | 51 +++++++ src/agentops/core/agentops_config.py | 44 +++++- src/agentops/pipeline/official_eval.py | 45 +++++-- src/agentops/services/cicd.py | 6 + .../workflows/agentops-deploy-prod-azd.yml | 4 + .../workflows/agentops-deploy-prod.yml | 4 + tests/unit/test_agentops_config.py | 79 +++++++++++ tests/unit/test_check_changelog.py | 85 ++++++++++++ tests/unit/test_cicd.py | 127 +++++++++++++++++- tests/unit/test_official_eval.py | 115 ++++++++++++++++ 12 files changed, 575 insertions(+), 34 deletions(-) diff --git a/.github/workflows/cut-release.yml b/.github/workflows/cut-release.yml index f712632..8089e51 100644 --- a/.github/workflows/cut-release.yml +++ b/.github/workflows/cut-release.yml @@ -76,28 +76,12 @@ jobs: - name: Update CHANGELOG run: | - DATE=$(date +%Y-%m-%d) - if grep -q "## \[${{ env.version }}\]" CHANGELOG.md; then - echo "CHANGELOG already has [${{ env.version }}] entry — skipping insertion" - exit 0 - fi - VERSION="${{ env.version }}" DATE="$DATE" python - <<'PY' - import os - import sys - from pathlib import Path - - path = Path("CHANGELOG.md") - text = path.read_text(encoding="utf-8") - marker = "## [Unreleased]" - version = os.environ["VERSION"] - date = os.environ["DATE"] - if marker not in text: - print("::error::CHANGELOG.md is missing the ## [Unreleased] section.") - sys.exit(1) - replacement = f"{marker}\n\n## [{version}] - {date}" - path.write_text(text.replace(marker, replacement, 1), encoding="utf-8") - PY - echo "CHANGELOG updated with [${{ env.version }}] - $DATE" + # The insertion lives in scripts/check_changelog.py so that + # tests/unit/test_check_changelog.py exercises the real code path + # instead of a copy that would drift from this workflow. + python scripts/check_changelog.py cut \ + --version "${{ env.version }}" \ + --date "$(date +%Y-%m-%d)" - name: Sync plugin versions run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 528b605..bd9cb6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Fixed +- **The official evaluation runner now honours the agent version override.** + `prepare_official_eval` read the agent name and version straight from + `agentops.yaml` and ignored both the `--agent` flag and the + `AGENTOPS_AGENT` environment variable, so a pipeline that pinned a + specific agent version still evaluated whatever version the config + happened to carry. The override is now resolved in one place + (`resolve_agent_override`), applied by `official_eval.py`, and forwarded + by the generated GitHub Actions and Azure DevOps workflows. An + unexpanded CI token such as `$(AGENTOPS_AGENT)` is treated as absent + instead of being parsed as an agent name. + +### Changed +- **The release cut logic moved out of `cut-release.yml` and into + `scripts/check_changelog.py cut`.** The workflow used to carry the + transformation as an inline Python heredoc, which no test could import, + so a regression in it was only visible when a release was already being + cut. That is exactly how the 0.8.6 cut broke. The same code now backs + both the workflow and the unit tests, and the subcommand is idempotent: + re-running it for a version already present in the file is a no-op. + ## [0.8.6] - 2026-08-07 ### Added diff --git a/scripts/check_changelog.py b/scripts/check_changelog.py index 74db343..5361da1 100644 --- a/scripts/check_changelog.py +++ b/scripts/check_changelog.py @@ -189,6 +189,27 @@ def unreleased_has_content(changelog: str) -> bool: return False +def apply_release_cut(changelog: str, version: str, date: str) -> str: + """Return *changelog* with a `## [version] - date` heading opened below Unreleased. + + This is the whole of the release cut: the workflow never moves text. It + inserts the new heading directly under `## [Unreleased]`, so everything that + was accumulating there is now attributed to the release, and `[Unreleased]` + becomes empty until the next merged change adds to it. + + `cut-release.yml` calls this so a test can exercise the real implementation + instead of a copy that would silently drift. + + Raises: + ValueError: when the `## [Unreleased]` heading is missing. + """ + + marker = "## [Unreleased]" + if marker not in changelog: + raise ValueError("CHANGELOG.md is missing the ## [Unreleased] section.") + return changelog.replace(marker, f"{marker}\n\n## [{version}] - {date}", 1) + + def added_line_numbers(diff: str) -> set[int]: """Return new-file line numbers of added lines in a unified diff.""" added: set[int] = set() @@ -351,6 +372,27 @@ def _cmd_check_unreleased(args: argparse.Namespace) -> int: ) +def _cmd_cut(args: argparse.Namespace) -> int: + path = Path(args.path) if args.path else REPO_ROOT / "CHANGELOG.md" + if not path.exists(): + return _fail(f"{path} does not exist.") + + changelog = path.read_text(encoding="utf-8") + if f"## [{args.version}]" in changelog: + print(f"CHANGELOG already has [{args.version}], skipping insertion") + return 0 + + try: + path.write_text( + apply_release_cut(changelog, args.version, args.date), encoding="utf-8" + ) + except ValueError as exc: + return _fail(str(exc)) + + print(f"CHANGELOG updated with [{args.version}] - {args.date}") + return 0 + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="command", required=True) @@ -373,6 +415,15 @@ def main(argv: list[str] | None = None) -> int: unreleased.add_argument("--path", default=None, help="Path to CHANGELOG.md.") unreleased.set_defaults(func=_cmd_check_unreleased) + cut = sub.add_parser( + "cut", + help="Open a versioned heading below [Unreleased] (used by cut-release.yml).", + ) + cut.add_argument("--version", required=True, help="Release version, e.g. 0.9.0.") + cut.add_argument("--date", required=True, help="Release date, e.g. 2026-01-31.") + cut.add_argument("--path", default=None, help="Path to CHANGELOG.md.") + cut.set_defaults(func=_cmd_cut) + args = parser.parse_args(argv) return int(args.func(args)) diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py index 6783011..7be891c 100644 --- a/src/agentops/core/agentops_config.py +++ b/src/agentops/core/agentops_config.py @@ -26,10 +26,11 @@ from __future__ import annotations +import os import re from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Mapping, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -1327,6 +1328,47 @@ def apply_agent_version_override(agent: str, override: str) -> str: ) +def is_unexpanded_ci_token(value: str) -> bool: + """Return True when *value* is an unexpanded CI variable reference. + + Azure DevOps leaves ``$(NAME)`` verbatim when a variable is undefined, and + GitHub expressions can leak through as ``${{ ... }}`` the same way. Callers + treat that as "no override" instead of trying to evaluate it as an agent + target. + """ + + candidate = value.strip() + return candidate.startswith("$(") or candidate.startswith("${{") + + +def resolve_agent_override( + agent: str, + *, + explicit: str | None = None, + env: Mapping[str, str] | None = None, +) -> str | None: + """Return the effective ``agent`` expression when an override applies. + + *explicit* is a command-line value such as ``--agent``. When it is ``None`` + the value of :data:`AGENT_OVERRIDE_ENV` in *env* (defaulting to the process + environment) is used instead. Unexpanded CI tokens and blank values are + ignored. + + Returns ``None`` when no override applies, so callers can distinguish "keep + the configured agent" from "the override happened to match the config". + """ + + source = os.environ if env is None else env + requested = explicit if explicit is not None else source.get(AGENT_OVERRIDE_ENV) + if requested is None: + return None + if is_unexpanded_ci_token(requested): + return None + if not requested.strip(): + return None + return apply_agent_version_override(agent, requested) + + def classify_agent( agent: str, protocol: Optional[Protocol] = None, diff --git a/src/agentops/pipeline/official_eval.py b/src/agentops/pipeline/official_eval.py index 6264c21..418b15f 100644 --- a/src/agentops/pipeline/official_eval.py +++ b/src/agentops/pipeline/official_eval.py @@ -9,7 +9,12 @@ from pathlib import Path from typing import Any, Iterable, Mapping, Sequence -from agentops.core.agentops_config import AgentOpsConfig, classify_agent +from agentops.core.agentops_config import ( + AGENT_OVERRIDE_ENV, + AgentOpsConfig, + classify_agent, + resolve_agent_override, +) from agentops.core.config_loader import load_agentops_config from agentops.core.evaluators import EvaluatorPreset, detect_dataset_shape, select_evaluators from agentops.utils.yaml import load_yaml @@ -108,11 +113,15 @@ class _EvalPlan: warnings: tuple[str, ...] -def analyze_official_eval_support(config_path: Path) -> OfficialEvalSupport: +def analyze_official_eval_support( + config_path: Path, + *, + agent: str | None = None, +) -> OfficialEvalSupport: """Report whether ``config_path`` can use the Microsoft Foundry eval runner.""" try: - plan = _build_plan(config_path) + plan = _build_plan(config_path, agent=agent) except OfficialEvalUnsupported as exc: return OfficialEvalSupport( eligible=False, @@ -182,10 +191,17 @@ def prepare_official_eval( output_path: Path, *, deployment_name: str | None = None, + agent: str | None = None, ) -> OfficialEvalPreparation: - """Convert AgentOps JSONL config into Microsoft Foundry AI Agent Evaluation JSON.""" + """Convert AgentOps JSONL config into Microsoft Foundry AI Agent Evaluation JSON. + + *agent* overrides the ``agent:`` target for this run. When it is ``None`` the + ``AGENTOPS_AGENT`` environment variable is honored, so a CI step that just + produced a new agent version scores that version instead of the one pinned in + ``agentops.yaml``. + """ - plan = _build_plan(config_path) + plan = _build_plan(config_path, agent=agent) deployment = _resolve_deployment_name(deployment_name) payload = _build_payload(plan) @@ -224,7 +240,7 @@ def prepare_official_eval( ) -def _build_plan(config_path: Path) -> _EvalPlan: +def _build_plan(config_path: Path, *, agent: str | None = None) -> _EvalPlan: config_path = config_path.resolve() if not config_path.exists(): raise OfficialEvalUnsupported( @@ -233,7 +249,12 @@ def _build_plan(config_path: Path) -> _EvalPlan: ) config = load_agentops_config(config_path) - target = classify_agent(config.agent, config.protocol) + try: + override = resolve_agent_override(config.agent, explicit=agent) + except ValueError as exc: + raise OfficialEvalUnsupported(str(exc)) from exc + agent_expression = override or config.agent + target = classify_agent(agent_expression, config.protocol) if target.kind != "foundry_prompt": raise OfficialEvalUnsupported( "Microsoft Foundry AI Agent Evaluation only evaluates Foundry prompt agents " @@ -262,7 +283,7 @@ def _build_plan(config_path: Path) -> _EvalPlan: config=config, config_path=config_path, dataset_path=dataset_path, - agent_ids=config.agent, + agent_ids=agent_expression, official_evaluators=tuple(official_evaluators), skipped_agentops_evaluators=tuple(skipped), warnings=tuple(warnings), @@ -410,6 +431,7 @@ def _command_prepare(args: argparse.Namespace) -> int: Path(args.config), Path(args.out), deployment_name=args.deployment_name, + agent=args.agent, ) outputs = { "data_path": str(prepared.data_path), @@ -436,6 +458,13 @@ def main(argv: Sequence[str] | None = None) -> int: prepare_parser.add_argument("--config", default="agentops.yaml") prepare_parser.add_argument("--out", required=True) prepare_parser.add_argument("--deployment-name") + prepare_parser.add_argument( + "--agent", + help=( + "Override the agent target for this run. Accepts a bare version or a " + f"full agent expression. Falls back to ${AGENT_OVERRIDE_ENV}." + ), + ) prepare_parser.add_argument("--github-output") prepare_parser.add_argument("--ado-output", action="store_true") prepare_parser.add_argument("--print-json", action="store_true") diff --git a/src/agentops/services/cicd.py b/src/agentops/services/cicd.py index 88c2ce7..babade1 100644 --- a/src/agentops/services/cicd.py +++ b/src/agentops/services/cicd.py @@ -543,6 +543,9 @@ def _github_eval_substitutions( AZURE_OPENAI_DEPLOYMENT: ${{{{ vars.AZURE_OPENAI_DEPLOYMENT }}}} AZURE_OPENAI_MODEL_NAME: ${{{{ vars.AZURE_OPENAI_MODEL_NAME }}}} {OFFICIAL_EVAL_ACTION_ENV}: {official_action} + # The prepare step reads this and scores the overridden agent version. + # An unset variable expands to an empty string, which is ignored. + {AGENT_OVERRIDE_ENV}: ${{{{ env.{AGENT_OVERRIDE_ENV} || vars.{AGENT_OVERRIDE_ENV} }}}} run: | python -m agentops.pipeline.official_eval prepare \\ --config \"{config_path}\" \\ @@ -745,6 +748,9 @@ def _ado_eval_substitutions( AZURE_OPENAI_DEPLOYMENT: $(AZURE_OPENAI_DEPLOYMENT) AZURE_OPENAI_MODEL_NAME: $(AZURE_OPENAI_MODEL_NAME) {OFFICIAL_EVAL_ADO_TASK_ENV}: {official_task} + # The prepare step reads this and scores the overridden agent version. An + # undefined variable stays literal as $(NAME) and is ignored. + {AGENT_OVERRIDE_ENV}: $({AGENT_OVERRIDE_ENV}) - task: {official_task} displayName: Run official AI Agent Evaluation diff --git a/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml b/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml index a55ff7b..2410a5d 100644 --- a/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml +++ b/src/agentops/templates/workflows/agentops-deploy-prod-azd.yml @@ -83,6 +83,10 @@ __AILZ_PREFLIGHT_COMMAND__ echo "Skipping azd provision for PROD. Use workflow_dispatch with provision=true for reviewed infra changes." fi + # No `environment:` here on purpose. This job is the gate that must run + # *before* the protected environment approval, so the reviewer sees the + # safety result when they approve. Adding an `environment:` would make the + # gate itself wait for the approval it is supposed to inform. safety-eval: name: Safety eval (gate) needs: provision diff --git a/src/agentops/templates/workflows/agentops-deploy-prod.yml b/src/agentops/templates/workflows/agentops-deploy-prod.yml index 4f92603..01a8be9 100644 --- a/src/agentops/templates/workflows/agentops-deploy-prod.yml +++ b/src/agentops/templates/workflows/agentops-deploy-prod.yml @@ -43,6 +43,10 @@ concurrency: cancel-in-progress: false jobs: + # No `environment:` here on purpose. This job is the gate that must run + # *before* the protected environment approval, so the reviewer sees the + # safety result when they approve. Adding an `environment:` would make the + # gate itself wait for the approval it is supposed to inform. safety-eval: name: Safety eval (gate) runs-on: ubuntu-latest diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py index c243846..4e0b30d 100644 --- a/tests/unit/test_agentops_config.py +++ b/tests/unit/test_agentops_config.py @@ -18,6 +18,8 @@ Threshold, apply_agent_version_override, classify_agent, + is_unexpanded_ci_token, + resolve_agent_override, ) @@ -165,6 +167,83 @@ def test_bare_version_against_model_target_is_rejected(self) -> None: apply_agent_version_override("model:gpt-4o-mini", "12") +# --------------------------------------------------------------------------- +# resolve_agent_override +# --------------------------------------------------------------------------- + + +class TestResolveAgentOverride: + """The single entry point every runner uses to pick up an override.""" + + def test_explicit_value_wins_over_the_environment(self) -> None: + assert ( + resolve_agent_override( + "helpdeskbot:11", + explicit="12", + env={AGENT_OVERRIDE_ENV: "99"}, + ) + == "helpdeskbot:12" + ) + + def test_environment_is_used_when_no_explicit_value(self) -> None: + assert ( + resolve_agent_override("helpdeskbot:11", env={AGENT_OVERRIDE_ENV: "12"}) + == "helpdeskbot:12" + ) + + def test_absent_override_returns_none(self) -> None: + assert resolve_agent_override("helpdeskbot:11", env={}) is None + + def test_blank_override_returns_none(self) -> None: + assert resolve_agent_override("helpdeskbot:11", explicit=" ") is None + + @pytest.mark.parametrize( + "token", + [ + # Azure DevOps leaves an undefined variable verbatim. + "$(AGENTOPS_AGENT)", + # GitHub can leak an expression the same way. + "${{ env.AGENTOPS_AGENT }}", + " $(AGENTOPS_AGENT) ", + ], + ) + def test_unexpanded_ci_tokens_are_ignored(self, token: str) -> None: + """A generated pipeline must stay valid when the variable is undefined.""" + + assert resolve_agent_override("helpdeskbot:11", explicit=token) is None + + def test_full_reference_override_is_returned_verbatim(self) -> None: + assert ( + resolve_agent_override("helpdeskbot:11", explicit="other:3") == "other:3" + ) + + def test_process_environment_is_the_default_source( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv(AGENT_OVERRIDE_ENV, "12") + assert resolve_agent_override("helpdeskbot:11") == "helpdeskbot:12" + + def test_bare_version_without_version_slot_still_raises(self) -> None: + with pytest.raises(ValueError, match="no version segment"): + resolve_agent_override("model:gpt-4o-mini", explicit="12") + + +class TestIsUnexpandedCiToken: + @pytest.mark.parametrize( + "value, expected", + [ + ("$(AGENTOPS_AGENT)", True), + ("${{ env.AGENTOPS_AGENT }}", True), + (" $(X)", True), + ("helpdeskbot:12", False), + ("12", False), + ("", False), + ], + ) + def test_detection(self, value: str, expected: bool) -> None: + assert is_unexpanded_ci_token(value) is expected + + # --------------------------------------------------------------------------- # Threshold parser # --------------------------------------------------------------------------- diff --git a/tests/unit/test_check_changelog.py b/tests/unit/test_check_changelog.py index 21f2e8a..387ea91 100644 --- a/tests/unit/test_check_changelog.py +++ b/tests/unit/test_check_changelog.py @@ -314,3 +314,88 @@ def test_unreleased_section_emptied_by_a_release_cut_is_still_parseable(guard): start, end = guard.unreleased_line_range(text) assert start < end assert guard.unreleased_has_content(text) is False + + +def test_real_changelog_survives_a_release_cut(guard): + """Run the real cut against the real CHANGELOG and re-parse the result. + + Release 0.8.6 failed CI because no test ever constructed the repository's + post-cut state. `apply_release_cut` is the same function `cut-release.yml` + invokes, so this exercises the shipped code path rather than a copy. + """ + before = (REPO_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + # Deliberately no precondition on `[Unreleased]` having content: on a + # release branch it is empty, and that is the state this test exists for. + + after = guard.apply_release_cut(before, "99.0.0", "2099-12-31") + + start, end = guard.unreleased_line_range(after) + assert start < end + # Everything moved under the new heading, so the section is now empty. The + # guard must report that as a fact instead of raising. + assert guard.unreleased_has_content(after) is False + assert "## [99.0.0] - 2099-12-31" in after + # No content is lost, only re-attributed. + assert after.replace("\n\n## [99.0.0] - 2099-12-31", "", 1) == before + + +def test_release_cut_is_idempotent_on_content(guard): + """Cutting twice must not duplicate the bullets under the first heading.""" + text = "# Changelog\n\n## [Unreleased]\n\n- One entry.\n" + + once = guard.apply_release_cut(text, "1.0.0", "2026-01-01") + twice = guard.apply_release_cut(once, "1.0.1", "2026-01-02") + + assert twice.count("- One entry.") == 1 + assert twice.index("## [1.0.1]") < twice.index("## [1.0.0]") + + +def test_release_cut_rejects_a_changelog_without_the_marker(guard): + with pytest.raises(ValueError, match=r"\[Unreleased\]"): + guard.apply_release_cut("# Changelog\n\n## [1.0.0] - 2026-01-01\n", "2.0.0", "x") + + +@pytest.mark.parametrize( + ("title", "labels", "changed_files"), + [ + pytest.param( + "chore(deps): bump cryptography from 48.0.1 to 50.0.0", + ["dependencies", "python:uv"], + ["uv.lock"], + id="pr-362-lockfile", + ), + pytest.param( + "chore(deps): bump actions/setup-python from 6 to 7", + ["dependencies"], + [ + ".github/workflows/agentops-watchdog.yml", + ".github/workflows/ci.yml", + ".github/workflows/e2e.yml", + ".github/workflows/release.yml", + ".github/workflows/staging.yml", + ], + id="pr-358-actions", + ), + pytest.param( + "chore(deps-dev): update mcp requirement from <2,>=1.0 to >=1.0,<3", + ["dependencies"], + ["pyproject.toml"], + id="pr-359-shipping-manifest", + ), + ], +) +def test_real_dependabot_pull_requests_never_require_an_entry( + guard, title, labels, changed_files +): + """Payloads copied verbatim from merged Dependabot PRs. + + `pyproject.toml` is shipping code, so that case only passes because the + author exemption is checked before the file classification. A regression + that reorders those checks would block every dependency bump. + """ + required, reason = guard.entry_required( + title, labels, "dependabot[bot]", changed_files + ) + + assert required is False + assert "automated author" in reason diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index 9a30397..81dc166 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -2,10 +2,17 @@ from pathlib import Path +import pytest import yaml from typer.testing import CliRunner from agentops.cli.app import app +from agentops.core.agentops_config import AGENT_OVERRIDE_ENV +from agentops.pipeline.official_eval import ( + AGENTOPS_CLOUD_RUNNER, + AGENTOPS_LOCAL_RUNNER, + AZD_EVAL_RUNNER, +) from agentops.services.cicd import ( ALL_KINDS, DEFAULT_KINDS, @@ -766,6 +773,46 @@ def _write_azd_eval_project(directory: Path) -> None: ) +def _write_cloud_eval_project(directory: Path) -> None: + """Lay out a project whose recommended eval runner is Foundry cloud eval.""" + + (directory / "azure.yaml").write_text("name: sample\n", encoding="utf-8") + (directory / "data.jsonl").write_text( + '{"input": "Hello", "expected": "Hello!"}\n', + encoding="utf-8", + ) + (directory / "agentops.yaml").write_text( + "version: 1\nagent: quickstart-agent:2\ndataset: data.jsonl\n", + encoding="utf-8", + ) + + +def _write_local_eval_project(directory: Path) -> None: + """Lay out a project whose recommended eval runner is AgentOps local eval.""" + + (directory / "azure.yaml").write_text("name: sample\n", encoding="utf-8") + (directory / "data.jsonl").write_text( + '{"input": "Hello", "expected": "Hello!"}\n', + encoding="utf-8", + ) + (directory / "agentops.yaml").write_text( + "version: 1\n" + "agent: https://example.com/api/agents/demo\n" + "protocol: http-json\n" + "dataset: data.jsonl\n", + encoding="utf-8", + ) + + +# Each writer produces a project that `recommended_eval_runner` resolves to the +# paired runner, so the generated pipelines exercise a different eval block. +_EVAL_RUNNER_PROJECTS = ( + (AZD_EVAL_RUNNER, _write_azd_eval_project), + (AGENTOPS_CLOUD_RUNNER, _write_cloud_eval_project), + (AGENTOPS_LOCAL_RUNNER, _write_local_eval_project), +) + + def _ado_stage(path: Path, stage: str) -> str: """Return the raw YAML text of a single Azure DevOps stage.""" @@ -807,23 +854,36 @@ def test_azure_devops_azd_eval_stage_configures_azd_auth(tmp_path: Path) -> None assert "${{" not in section +@pytest.mark.parametrize( + ("expected_runner", "write_project"), + _EVAL_RUNNER_PROJECTS, + ids=[name for name, _ in _EVAL_RUNNER_PROJECTS], +) def test_azure_devops_pipelines_stay_valid_yaml_for_every_eval_runner( tmp_path: Path, + expected_runner: str, + write_project, ) -> None: """Guard against block-scalar indentation regressions in the ADO templates. A line at column 0 inside a `bash: |` block terminates the scalar and makes - Azure DevOps reject the whole pipeline, so parse every generated file. + Azure DevOps reject the whole pipeline, so parse every generated file. Each + eval runner emits a different set of tasks, so run the check once per + runner instead of only for the azd backend. """ - _write_azd_eval_project(tmp_path) - generate_cicd_workflows( + write_project(tmp_path) + result = generate_cicd_workflows( directory=tmp_path, platform="azure-devops", kinds=list(ALL_KINDS), force=True, ) + # Fail loudly if a fixture stops selecting the runner it is named after, + # otherwise the parametrization would silently retest the same block. + assert result.eval_runner == expected_runner + for rel in _ADO_PATHS: content = (tmp_path / rel).read_text(encoding="utf-8") assert isinstance(_read_yaml(tmp_path / rel), dict), rel @@ -833,6 +893,67 @@ def test_azure_devops_pipelines_stay_valid_yaml_for_every_eval_runner( assert line.split(":", 1)[0].isidentifier(), f"{rel}:{number}: {line}" +def test_official_eval_prepare_step_forwards_the_agent_override(tmp_path: Path) -> None: + """The official runner must honor `--agent` / `AGENTOPS_AGENT` like the others. + + Without this, a pipeline that just published a new agent version scores the + version pinned in `agentops.yaml` instead of the candidate it built. + """ + + from agentops.pipeline.official_eval import OFFICIAL_EVAL_RUNNER + from agentops.services.cicd import _eval_substitutions + + # The two platforms name the rendered block differently. + for platform, key in (("github", "__EVAL_STEPS__"), ("azure-devops", "__EVAL_TASKS__")): + tasks = _eval_substitutions( + platform, + OFFICIAL_EVAL_RUNNER, + "agentops.yaml", + kind="pr", + )[key] + + assert AGENT_OVERRIDE_ENV in tasks, platform + + +def test_prod_workflow_documents_why_safety_eval_has_no_environment( + tmp_path: Path, +) -> None: + """The missing `environment:` on the safety gate is deliberate, not an oversight. + + The gate has to run before the protected environment approval so reviewers + can see the safety result while approving. + """ + + _write_azd_eval_project(tmp_path) + generate_cicd_workflows(directory=tmp_path, kinds=["prod"], force=True) + content = (tmp_path / _PROD_PATH).read_text(encoding="utf-8") + + marker = "No `environment:` here on purpose." + assert marker in content + assert content.index(marker) < content.index("safety-eval:") + + prod = _read_yaml(tmp_path / _PROD_PATH) + assert "environment" not in prod["jobs"]["safety-eval"] + + +def test_prod_workflow_documents_safety_eval_gate_in_placeholder_mode( + tmp_path: Path, +) -> None: + """The same rationale must survive in the non-azd prod template.""" + + generate_cicd_workflows( + directory=tmp_path, + kinds=["prod"], + deploy_mode="placeholder", + force=True, + ) + content = (tmp_path / _PROD_PATH).read_text(encoding="utf-8") + + assert "No `environment:` here on purpose." in content + prod = _read_yaml(tmp_path / _PROD_PATH) + assert "environment" not in prod["jobs"]["safety-eval"] + + def test_azure_devops_azd_mode_runs_ailz_preflight_when_script_exists(tmp_path: Path) -> None: (tmp_path / "azure.yaml").write_text("name: azure-ai-lz\n", encoding="utf-8") scripts = tmp_path / "scripts" diff --git a/tests/unit/test_official_eval.py b/tests/unit/test_official_eval.py index 464f165..410372a 100644 --- a/tests/unit/test_official_eval.py +++ b/tests/unit/test_official_eval.py @@ -3,6 +3,7 @@ import json from pathlib import Path +from agentops.core.agentops_config import AGENT_OVERRIDE_ENV from agentops.pipeline.official_eval import ( AGENTOPS_CLOUD_RUNNER, AGENTOPS_LOCAL_RUNNER, @@ -158,3 +159,117 @@ def test_prepare_cli_writes_github_outputs(tmp_path: Path) -> None: output = github_output.read_text(encoding="utf-8") assert "agent_ids=support-agent:4" in output assert "deployment_name=gpt-4o-mini" in output + + +# --------------------------------------------------------------------------- +# Agent override (#398) +# --------------------------------------------------------------------------- + + +def test_prepare_honors_explicit_agent_override(tmp_path: Path) -> None: + """`--agent 5` must retarget the version without touching agentops.yaml.""" + + _write_prompt_config(tmp_path) + _write_dataset(tmp_path) + + prepared = prepare_official_eval( + tmp_path / "agentops.yaml", + tmp_path / "input.json", + deployment_name="gpt-4o-mini", + agent="5", + ) + + metadata = json.loads(prepared.metadata_path.read_text(encoding="utf-8")) + assert metadata["agent_ids"] == "support-agent:5" + + +def test_prepare_honors_agent_override_from_the_environment( + tmp_path: Path, + monkeypatch, +) -> None: + """CI exports the version it just published instead of passing a flag.""" + + _write_prompt_config(tmp_path) + _write_dataset(tmp_path) + monkeypatch.setenv(AGENT_OVERRIDE_ENV, "7") + + prepared = prepare_official_eval( + tmp_path / "agentops.yaml", + tmp_path / "input.json", + deployment_name="gpt-4o-mini", + ) + + metadata = json.loads(prepared.metadata_path.read_text(encoding="utf-8")) + assert metadata["agent_ids"] == "support-agent:7" + + +def test_prepare_ignores_an_unexpanded_ci_token( + tmp_path: Path, + monkeypatch, +) -> None: + """The generated step stays valid when the CI variable is never defined.""" + + _write_prompt_config(tmp_path) + _write_dataset(tmp_path) + monkeypatch.setenv(AGENT_OVERRIDE_ENV, "$(AGENTOPS_AGENT)") + + prepared = prepare_official_eval( + tmp_path / "agentops.yaml", + tmp_path / "input.json", + deployment_name="gpt-4o-mini", + ) + + metadata = json.loads(prepared.metadata_path.read_text(encoding="utf-8")) + assert metadata["agent_ids"] == "support-agent:4" + + +def test_support_analysis_applies_the_override(tmp_path: Path) -> None: + _write_prompt_config(tmp_path) + _write_dataset(tmp_path) + + support = analyze_official_eval_support(tmp_path / "agentops.yaml", agent="9") + + assert support.eligible is True + assert support.agent_ids == "support-agent:9" + + +def test_override_without_a_version_slot_is_reported_as_unsupported( + tmp_path: Path, +) -> None: + """A bare version against a model target must not crash the eval gate.""" + + (tmp_path / "agentops.yaml").write_text( + "version: 1\nagent: model:gpt-4o-mini\ndataset: data.jsonl\n", + encoding="utf-8", + ) + _write_dataset(tmp_path) + + support = analyze_official_eval_support(tmp_path / "agentops.yaml", agent="9") + + assert support.eligible is False + assert "no version segment" in support.reasons[0] + + +def test_prepare_cli_accepts_the_agent_flag(tmp_path: Path) -> None: + _write_prompt_config(tmp_path) + _write_dataset(tmp_path) + github_output = tmp_path / "github-output.txt" + + code = main( + [ + "prepare", + "--config", + str(tmp_path / "agentops.yaml"), + "--out", + str(tmp_path / "input.json"), + "--deployment-name", + "gpt-4o-mini", + "--agent", + "5", + "--github-output", + str(github_output), + ] + ) + + assert code == 0 + assert "agent_ids=support-agent:5" in github_output.read_text(encoding="utf-8")