From 6576897790c404423794ffa60613e4c76f792681 Mon Sep 17 00:00:00 2001 From: Darren Wang Date: Wed, 5 Aug 2026 17:08:27 +0000 Subject: [PATCH 1/2] fix: DeepEvalAdapter to extract multi-turn conversations from service-normalized SESSION format --- .../third_party/span_mappers/registry.py | 54 +++++- .../autoevals/test_error_handling.py | 2 +- .../third_party/deepeval/test_adapter.py | 127 +++++++++++++++ .../deepeval/test_error_handling.py | 2 +- .../span_mappers/test_span_mappers.py | 154 ++++++++++++++++++ 5 files changed, 334 insertions(+), 5 deletions(-) diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py index 7db36009..1ec10571 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py @@ -125,15 +125,63 @@ def map_spans( return result +def _extract_message_text(messages: List[Dict[str, Any]]) -> Optional[str]: + """Extract text content from service message format. + + Handles the nested structure: [{role: ..., content: {content: [{text: ...}]}}] + as well as the variant: [{role: ..., content: {message: [{text: ...}]}}] + """ + for msg in messages: + content = msg.get("content", msg.get("message", {})) + if isinstance(content, dict): + # Unwrap nested content/message key + content = content.get("content", content.get("message", [])) + if isinstance(content, list): + text = " ".join(c.get("text", "") for c in content if isinstance(c, dict)).strip() + if text: + return text + elif isinstance(content, str) and content.strip(): + return content.strip() + return None + + def _extract_from_service_format(session_spans: List[Dict[str, Any]]) -> Optional[SpanMapResult]: """Extract fields from service-normalized span format. - The AgentCore evaluation service sends spans with gen_ai semantic convention - events (gen_ai.user.message, gen_ai.choice) instead of body with input/output. - This handles that format as a fallback when strands-evals mappers can't parse it. + Handles two service formats: + 1. SESSION format with span_events[*].body (multi-turn conversations where the + service collapses all ADOT spans into one span with multiple span_events) + 2. gen_ai semantic convention events (single-turn Strands spans) """ import json as _json + # --- Multi-turn: extract from span_events[*].body --- + for span in session_spans: + span_events = span.get("span_events", []) + if len(span_events) >= 1: + turns: List[Dict[str, Any]] = [] + last_input = None + last_output = None + for se in span_events: + body = se.get("body", {}) + inp_msgs = (body.get("input") or {}).get("messages", []) + out_msgs = (body.get("output") or {}).get("messages", []) + user_text = _extract_message_text(inp_msgs) if inp_msgs else None + asst_text = _extract_message_text(out_msgs) if out_msgs else None + if user_text: + turns.append({"role": "user", "content": user_text}) + last_input = user_text + if asst_text: + turns.append({"role": "assistant", "content": asst_text}) + last_output = asst_text + if turns and last_input and last_output: + return SpanMapResult( + input=last_input, + actual_output=last_output, + turns=turns if len(turns) > 2 else None, + ) + + # --- Single-turn: extract from gen_ai semantic convention events --- for span in session_spans: scope = span.get("scope", {}).get("name", "") events = span.get("events", []) diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py index 731aae67..1e7ea2ad 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py @@ -107,7 +107,7 @@ def test_02_unrecognized_scope(self): ] adapter = AutoEvalsAdapter(metric=_mock_scorer()) result = adapter(_make_evaluator_input(spans=spans)) - _assert_error_response(result, "FIELD_EXTRACTION_ERROR") + _assert_error_response(result, "MISSING_REQUIRED_FIELD") def test_03_spans_missing_body_input(self): spans = [ diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py index ede7fb7a..8eae8d1c 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py @@ -396,3 +396,130 @@ def test_conversational_metric_single_turn_returns_error(self): assert result.errorCode == "FIELD_EXTRACTION_ERROR" assert "multi-turn" in result.errorMessage.lower() or "Multiple" in result.errorMessage + + +class TestDeepEvalAdapterServiceNormalizedMultiTurn: + """Tests for conversational metrics with service-normalized SESSION format. + + The AgentCore service collapses multi-turn ADOT docs into one span with + span_events[*].body. These tests verify the adapter correctly extracts + all turns and passes a ConversationalTestCase to the metric. + """ + + def _make_session_evaluator_input(self, num_turns=3): + """Build EvaluatorInput in service-normalized SESSION format.""" + span_events = [] + for i in range(num_turns): + span_events.append({ + "body": { + "input": { + "messages": [ + {"role": "user", "content": {"content": [{"text": f"User turn {i+1}"}]}} + ] + }, + "output": { + "messages": [ + {"role": "assistant", "content": {"message": [{"text": f"Bot turn {i+1}"}]}} + ] + }, + } + }) + spans = [ + { + "traceId": "t-session", + "spanId": "s-session", + "source": "adot_cw", + "attributes": {"session.id": "multi-turn-session"}, + "span_events": span_events, + } + ] + return EvaluatorInput( + evaluation_level="SESSION", + session_spans=spans, + ) + + def test_conversational_metric_receives_all_turns(self): + """Multi-turn metric gets ConversationalTestCase with correct turn count.""" + from deepeval.metrics import BaseConversationalMetric + from deepeval.test_case import ConversationalTestCase + + metric = MagicMock(spec=BaseConversationalMetric) + type(metric).__name__ = "GoalAccuracyMetric" + metric.threshold = 0.5 + metric.score = 0.9 + metric.reason = "Goal achieved" + del metric.success + + captured_test_case = {} + + def measure_side_effect(test_case): + captured_test_case["tc"] = test_case + metric.score = 0.9 + metric.reason = "Goal achieved" + + metric.measure = MagicMock(side_effect=measure_side_effect) + adapter = DeepEvalAdapter(metric=metric) + + result = adapter(self._make_session_evaluator_input(num_turns=4)) + + assert result.value == 0.9 + assert result.label == "Pass" + tc = captured_test_case["tc"] + assert isinstance(tc, ConversationalTestCase) + assert len(tc.turns) == 8 # 4 user + 4 assistant turns + + def test_conversational_metric_turn_content_correct(self): + """Verify turn content is correctly extracted from nested message format.""" + from deepeval.metrics import BaseConversationalMetric + from deepeval.test_case import ConversationalTestCase + + metric = MagicMock(spec=BaseConversationalMetric) + type(metric).__name__ = "RoleAdherenceMetric" + metric.threshold = 0.5 + metric.score = 1.0 + metric.reason = "No violations" + del metric.success + + captured_test_case = {} + + def measure_side_effect(test_case): + captured_test_case["tc"] = test_case + metric.score = 1.0 + + metric.measure = MagicMock(side_effect=measure_side_effect) + adapter = DeepEvalAdapter(metric=metric) + + result = adapter(self._make_session_evaluator_input(num_turns=2)) + + assert result.value == 1.0 + tc = captured_test_case["tc"] + assert tc.turns[0].role == "user" + assert tc.turns[0].content == "User turn 1" + assert tc.turns[1].role == "assistant" + assert tc.turns[1].content == "Bot turn 1" + assert tc.turns[2].role == "user" + assert tc.turns[2].content == "User turn 2" + assert tc.turns[3].role == "assistant" + assert tc.turns[3].content == "Bot turn 2" + + def test_five_turn_session_evaluation(self): + """Realistic 5-turn session evaluation (matches typical MACE migration).""" + from deepeval.metrics import BaseConversationalMetric + + metric = MagicMock(spec=BaseConversationalMetric) + type(metric).__name__ = "ConversationCompletenessMetric" + metric.threshold = 0.5 + metric.score = 0.75 + metric.reason = "Mostly complete" + del metric.success + + metric.measure = MagicMock(side_effect=lambda tc: None) + adapter = DeepEvalAdapter(metric=metric) + + result = adapter(self._make_session_evaluator_input(num_turns=5)) + + assert result.value == 0.75 + assert result.label == "Pass" + metric.measure.assert_called_once() + tc = metric.measure.call_args[0][0] + assert len(tc.turns) == 10 # 5 user + 5 assistant diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py index 572c1a16..919a25f0 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py @@ -185,7 +185,7 @@ def test_15_unrecognized_scope_deepeval(self): ] adapter = DeepEvalAdapter(metric=_mock_metric()) result = adapter(_make_evaluator_input(spans=spans)) - _assert_error_response(result, "FIELD_EXTRACTION_ERROR") + _assert_error_response(result, "MISSING_REQUIRED_FIELD") def test_16_spans_missing_body_input(self): spans = [ diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py index 5a025cf0..c60ce5f8 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py @@ -112,3 +112,157 @@ def test_span_map_result_fields(self): assert result.tools_called == [{"name": "tool1", "input_parameters": {"a": 1}, "output": "result"}] assert result.expected_output is None assert result.system_prompt is None + + +def _make_service_normalized_session_spans(num_turns=3): + """Build service-normalized SESSION format spans (span_events[*].body). + + This is the format the AgentCore service sends to Lambda for SESSION-level + evaluators: one span with multiple span_events, each representing a turn. + """ + span_events = [] + for i in range(num_turns): + span_events.append({ + "body": { + "input": { + "messages": [ + {"role": "user", "content": {"content": [{"text": f"User message {i+1}"}]}} + ] + }, + "output": { + "messages": [ + {"role": "assistant", "content": {"message": [{"text": f"Assistant response {i+1}"}]}} + ] + }, + } + }) + return [ + { + "traceId": "trace-multi", + "spanId": "span-multi", + "source": "adot_cw", + "attributes": {"session.id": "session-1"}, + "span_events": span_events, + } + ] + + +class TestServiceNormalizedMultiTurn: + """Tests for multi-turn extraction from service-normalized SESSION format.""" + + def test_extracts_all_turns_from_span_events(self): + spans = _make_service_normalized_session_spans(num_turns=3) + result = map_spans(spans) + + assert result.turns is not None + assert len(result.turns) == 6 # 3 user + 3 assistant + assert result.turns[0] == {"role": "user", "content": "User message 1"} + assert result.turns[1] == {"role": "assistant", "content": "Assistant response 1"} + assert result.turns[4] == {"role": "user", "content": "User message 3"} + assert result.turns[5] == {"role": "assistant", "content": "Assistant response 3"} + + def test_input_and_output_are_last_turn(self): + spans = _make_service_normalized_session_spans(num_turns=3) + result = map_spans(spans) + + assert result.input == "User message 3" + assert result.actual_output == "Assistant response 3" + + def test_single_span_event_returns_none_turns(self): + """Single span_event should NOT populate turns (not multi-turn).""" + spans = _make_service_normalized_session_spans(num_turns=1) + result = map_spans(spans) + + # With only 1 turn (2 entries: user+assistant), turns should be None + assert result.turns is None + # But input/output should still be extracted + assert result.input == "User message 1" + assert result.actual_output == "Assistant response 1" + + def test_handles_string_content_variant(self): + """Test spans where content is a plain string instead of list of dicts.""" + spans = [ + { + "traceId": "t1", + "spanId": "s1", + "attributes": {"session.id": "sess"}, + "span_events": [ + { + "body": { + "input": {"messages": [{"role": "user", "content": "Hello plain"}]}, + "output": {"messages": [{"role": "assistant", "content": "Hi plain"}]}, + } + }, + { + "body": { + "input": {"messages": [{"role": "user", "content": "Follow up"}]}, + "output": {"messages": [{"role": "assistant", "content": "Got it"}]}, + } + }, + ], + } + ] + result = map_spans(spans) + + assert result.turns is not None + assert len(result.turns) == 4 + assert result.turns[0] == {"role": "user", "content": "Hello plain"} + assert result.turns[3] == {"role": "assistant", "content": "Got it"} + + def test_handles_nested_content_dict_variant(self): + """Test the {content: {content: [{text: ...}]}} nesting.""" + spans = [ + { + "traceId": "t1", + "spanId": "s1", + "attributes": {"session.id": "sess"}, + "span_events": [ + { + "body": { + "input": { + "messages": [ + {"role": "user", "content": {"content": [{"text": "Turn 1 input"}]}} + ] + }, + "output": { + "messages": [ + {"role": "assistant", "content": {"message": [{"text": "Turn 1 output"}]}} + ] + }, + } + }, + { + "body": { + "input": { + "messages": [ + {"role": "user", "content": {"content": [{"text": "Turn 2 input"}]}} + ] + }, + "output": { + "messages": [ + {"role": "assistant", "content": {"message": [{"text": "Turn 2 output"}]}} + ] + }, + } + }, + ], + } + ] + result = map_spans(spans) + + assert result.turns is not None + assert len(result.turns) == 4 + assert result.turns[0]["content"] == "Turn 1 input" + assert result.turns[1]["content"] == "Turn 1 output" + assert result.turns[2]["content"] == "Turn 2 input" + assert result.turns[3]["content"] == "Turn 2 output" + + def test_five_turns_for_session_evaluation(self): + """Realistic test: 5-turn conversation as sent by the service.""" + spans = _make_service_normalized_session_spans(num_turns=5) + result = map_spans(spans) + + assert result.turns is not None + assert len(result.turns) == 10 + assert result.input == "User message 5" + assert result.actual_output == "Assistant response 5" From a5494c860818b0bafc8f332e5f98c81a7f422288 Mon Sep 17 00:00:00 2001 From: Darren Wang Date: Mon, 24 Aug 2026 21:56:29 +0000 Subject: [PATCH 2/2] correctly triage multi-turn, reuse existing CloudWatch message parsing --- .../third_party/span_mappers/registry.py | 129 +++++++++++++++--- .../autoevals/test_error_handling.py | 2 +- .../third_party/deepeval/test_adapter.py | 25 +++- .../deepeval/test_error_handling.py | 2 +- .../span_mappers/test_span_mappers.py | 49 ++++++- 5 files changed, 176 insertions(+), 31 deletions(-) diff --git a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py index 1ec10571..dafcfe5f 100644 --- a/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py +++ b/src/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/registry.py @@ -1,5 +1,6 @@ """Span mapping orchestration — uses strands-evals mappers with auto-detection.""" +import json import logging import warnings from typing import Any, Dict, List, Optional @@ -83,7 +84,28 @@ def map_spans( # Mapper couldn't find AgentInvocationSpan — try service format fallback result = None - # Fallback: extract from service-normalized format (gen_ai events) + # Multi-turn reachability: when the service collapses a session into a single + # span with multiple span_events (one per turn), the CloudWatch mapper still + # produces a valid single-turn input/actual_output from the first/last event, + # so the `not result.input` guard below would never fire and the multi-turn + # turns would be lost. Detect the collapsed multi-event shape up front and run + # the service-format extractor so `turns` is populated even when the mapper + # already found a valid single-turn pair. + if _has_multi_event_span(session_spans): + service_result = _extract_from_service_format(session_spans) + if service_result is not None and service_result.turns: + if result is None: + result = service_result + else: + # Keep the mapper's richer fields (tools, retrieval_context, etc.) + # but adopt the multi-turn conversation extracted from span_events, + # including its input/actual_output (the latest turn) so single-turn + # fields stay consistent with the extracted conversation. + result.turns = service_result.turns + result.input = service_result.input + result.actual_output = service_result.actual_output + + # Fallback: extract from service-normalized format (span_events / gen_ai events) if result is None or not result.input or not result.actual_output: service_result = _extract_from_service_format(session_spans) if service_result: @@ -125,23 +147,86 @@ def map_spans( return result -def _extract_message_text(messages: List[Dict[str, Any]]) -> Optional[str]: - """Extract text content from service message format. +def _has_multi_event_span(session_spans: List[Dict[str, Any]]) -> bool: + """Return True if any span carries more than one span_event. - Handles the nested structure: [{role: ..., content: {content: [{text: ...}]}}] - as well as the variant: [{role: ..., content: {message: [{text: ...}]}}] + This is the signature of the service-normalized SESSION format, where the + evaluation service collapses all ADOT spans sharing a session.id into a + single span with one span_event per conversation turn. """ - for msg in messages: - content = msg.get("content", msg.get("message", {})) + for span in session_spans: + if isinstance(span, dict) and len(span.get("span_events", []) or []) > 1: + return True + return False + + +def _extract_message_text(messages: List[Dict[str, Any]], role: Optional[str] = None) -> Optional[str]: + """Extract plain text from a service-format message list. + + Decodes the double-encoded ``content`` that real Strands ``invoke_agent`` + bodies emit (a JSON string like ``'[{"text": ...}]'``) into plain text, + mirroring the single-turn decoding behavior (PR #454) rather than returning + the serialized JSON literally. Plain strings and already-decoded + ``[{"text": ...}]`` lists are also handled. + + Args: + messages: The ``body.input.messages`` or ``body.output.messages`` list. + role: If given, prefer the last message whose ``role`` matches (latest + user / latest assistant, in chronological order). Falls back to any + message with text. + + Returns: + The extracted plain text, or None if no text could be parsed. + """ + + def _text_from_list(items: List[Any]) -> Optional[str]: + text = " ".join(c.get("text", "") for c in items if isinstance(c, dict)).strip() + return text or None + + def _text_from_raw(raw: Any) -> Optional[str]: + # Decode the double-encoded JSON-string content: a string that parses to a + # list of {"text": ...} blocks. Falls back to the plain string on failure. + if isinstance(raw, str): + stripped = raw.strip() + if not stripped: + return None + try: + parsed = json.loads(stripped) + except (ValueError, TypeError): + return stripped + if isinstance(parsed, list): + return _text_from_list(parsed) + return stripped + # Already-decoded list variant: [{"text": ...}, ...] + if isinstance(raw, list): + return _text_from_list(raw) + return None + + def _text(msg: Dict[str, Any]) -> Optional[str]: + content = msg.get("content", msg.get("message")) + # Service shape: content/message is a dict wrapping the raw value under a + # nested "content"/"message" key (double-encoded JSON string, plain string, + # or already-decoded list). if isinstance(content, dict): - # Unwrap nested content/message key - content = content.get("content", content.get("message", [])) - if isinstance(content, list): - text = " ".join(c.get("text", "") for c in content if isinstance(c, dict)).strip() + inner = content.get("content", content.get("message")) + return _text_from_raw(inner) + # Top-level raw value: JSON string, plain string, or list. + return _text_from_raw(content) + + if role is not None: + # Prefer the latest message matching the requested role (chronological). + for msg in reversed(messages): + if isinstance(msg, dict) and msg.get("role") == role: + text = _text(msg) + if text: + return text + + # Fallback: first message that yields any text. + for msg in messages: + if isinstance(msg, dict): + text = _text(msg) if text: return text - elif isinstance(content, str) and content.strip(): - return content.strip() return None @@ -153,12 +238,14 @@ def _extract_from_service_format(session_spans: List[Dict[str, Any]]) -> Optiona service collapses all ADOT spans into one span with multiple span_events) 2. gen_ai semantic convention events (single-turn Strands spans) """ - import json as _json - # --- Multi-turn: extract from span_events[*].body --- + # Only treat a span as a collapsed multi-turn conversation when it carries + # more than one span_event. A single span_event is a single-turn span the + # CloudWatch mapper already handles, and hijacking it here would mislabel + # single-turn sessions. for span in session_spans: span_events = span.get("span_events", []) - if len(span_events) >= 1: + if len(span_events) > 1: turns: List[Dict[str, Any]] = [] last_input = None last_output = None @@ -166,8 +253,8 @@ def _extract_from_service_format(session_spans: List[Dict[str, Any]]) -> Optiona body = se.get("body", {}) inp_msgs = (body.get("input") or {}).get("messages", []) out_msgs = (body.get("output") or {}).get("messages", []) - user_text = _extract_message_text(inp_msgs) if inp_msgs else None - asst_text = _extract_message_text(out_msgs) if out_msgs else None + user_text = _extract_message_text(inp_msgs, role="user") if inp_msgs else None + asst_text = _extract_message_text(out_msgs, role="assistant") if out_msgs else None if user_text: turns.append({"role": "user", "content": user_text}) last_input = user_text @@ -199,19 +286,19 @@ def _extract_from_service_format(session_spans: List[Dict[str, Any]]) -> Optiona if event_name == "gen_ai.user.message" and content: try: - parts = _json.loads(content) + parts = json.loads(content) user_input = " ".join(p.get("text", "") for p in parts if isinstance(p, dict)).strip() except (ValueError, TypeError): user_input = content elif event_name == "gen_ai.choice" and attrs.get("message"): try: - parts = _json.loads(attrs["message"]) + parts = json.loads(attrs["message"]) assistant_output = " ".join(p.get("text", "") for p in parts if isinstance(p, dict)).strip() except (ValueError, TypeError): assistant_output = attrs["message"] elif event_name == "gen_ai.system.message" and content: try: - parts = _json.loads(content) + parts = json.loads(content) system_prompt = " ".join(p.get("text", "") for p in parts if isinstance(p, dict)).strip() except (ValueError, TypeError): system_prompt = content diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py index 1e7ea2ad..731aae67 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/autoevals/test_error_handling.py @@ -107,7 +107,7 @@ def test_02_unrecognized_scope(self): ] adapter = AutoEvalsAdapter(metric=_mock_scorer()) result = adapter(_make_evaluator_input(spans=spans)) - _assert_error_response(result, "MISSING_REQUIRED_FIELD") + _assert_error_response(result, "FIELD_EXTRACTION_ERROR") def test_03_spans_missing_body_input(self): spans = [ diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py index 8eae8d1c..7c20f6ef 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_adapter.py @@ -407,28 +407,44 @@ class TestDeepEvalAdapterServiceNormalizedMultiTurn: """ def _make_session_evaluator_input(self, num_turns=3): - """Build EvaluatorInput in service-normalized SESSION format.""" + """Build EvaluatorInput in service-normalized SESSION format. + + Uses the REAL Strands body shape: message content is double-encoded JSON + under ``content`` / ``message`` (a JSON string, matching what the service + actually sends), so this exercises the CloudWatch-consistent decoding path + rather than a pre-parsed convenience shape. + """ + import json + span_events = [] for i in range(num_turns): span_events.append({ + "event_name": "strands.telemetry.tracer", "body": { "input": { "messages": [ - {"role": "user", "content": {"content": [{"text": f"User turn {i+1}"}]}} + { + "role": "user", + "content": {"content": json.dumps([{"text": f"User turn {i + 1}"}])}, + } ] }, "output": { "messages": [ - {"role": "assistant", "content": {"message": [{"text": f"Bot turn {i+1}"}]}} + { + "role": "assistant", + "content": {"message": json.dumps([{"text": f"Bot turn {i + 1}"}])}, + } ] }, - } + }, }) spans = [ { "traceId": "t-session", "spanId": "s-session", "source": "adot_cw", + "scope": {"name": "strands.telemetry.tracer"}, "attributes": {"session.id": "multi-turn-session"}, "span_events": span_events, } @@ -471,7 +487,6 @@ def measure_side_effect(test_case): def test_conversational_metric_turn_content_correct(self): """Verify turn content is correctly extracted from nested message format.""" from deepeval.metrics import BaseConversationalMetric - from deepeval.test_case import ConversationalTestCase metric = MagicMock(spec=BaseConversationalMetric) type(metric).__name__ = "RoleAdherenceMetric" diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py index 919a25f0..572c1a16 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/deepeval/test_error_handling.py @@ -185,7 +185,7 @@ def test_15_unrecognized_scope_deepeval(self): ] adapter = DeepEvalAdapter(metric=_mock_metric()) result = adapter(_make_evaluator_input(spans=spans)) - _assert_error_response(result, "MISSING_REQUIRED_FIELD") + _assert_error_response(result, "FIELD_EXTRACTION_ERROR") def test_16_spans_missing_body_input(self): spans = [ diff --git a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py index c60ce5f8..804ca165 100644 --- a/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py +++ b/tests/bedrock_agentcore/evaluation/custom_code_based_evaluators/third_party/span_mappers/test_span_mappers.py @@ -119,28 +119,45 @@ def _make_service_normalized_session_spans(num_turns=3): This is the format the AgentCore service sends to Lambda for SESSION-level evaluators: one span with multiple span_events, each representing a turn. + + Uses the REAL Strands body shape as emitted by ``invoke_agent`` and normalized + by the service: message content is DOUBLE-ENCODED JSON under ``content`` / + ``message`` (a JSON string, not an already-parsed list). This is the same + shape the single-turn ``_valid_strands_spans`` fixtures use. Tests that assert + on turn content therefore also guard against the "raw JSON leaks into the turn + text" regression. """ + import json + span_events = [] for i in range(num_turns): span_events.append({ + "event_name": "strands.telemetry.tracer", "body": { "input": { "messages": [ - {"role": "user", "content": {"content": [{"text": f"User message {i+1}"}]}} + { + "role": "user", + "content": {"content": json.dumps([{"text": f"User message {i + 1}"}])}, + } ] }, "output": { "messages": [ - {"role": "assistant", "content": {"message": [{"text": f"Assistant response {i+1}"}]}} + { + "role": "assistant", + "content": {"message": json.dumps([{"text": f"Assistant response {i + 1}"}])}, + } ] }, - } + }, }) return [ { "traceId": "trace-multi", "spanId": "span-multi", "source": "adot_cw", + "scope": {"name": "strands.telemetry.tracer"}, "attributes": {"session.id": "session-1"}, "span_events": span_events, } @@ -161,6 +178,32 @@ def test_extracts_all_turns_from_span_events(self): assert result.turns[4] == {"role": "user", "content": "User message 3"} assert result.turns[5] == {"role": "assistant", "content": "Assistant response 3"} + def test_turn_content_has_no_json_syntax(self): + """Regression guard: turn content must be decoded plain text, not raw JSON. + + Real service bodies double-encode content as a JSON string + ('[{"text": ...}]'). If the extractor returns that string verbatim, the + turn content would contain JSON punctuation. This asserts the double + encoding is decoded to plain text. + """ + spans = _make_service_normalized_session_spans(num_turns=3) + result = map_spans(spans) + + assert result.turns is not None + for turn in result.turns: + content = turn["content"] + assert "text" not in content or content in ( + "User message 1", + "User message 2", + "User message 3", + "Assistant response 1", + "Assistant response 2", + "Assistant response 3", + ) + assert not any(ch in content for ch in ("{", "}", "[", "]", '"')), ( + f"turn content still contains raw JSON syntax: {content!r}" + ) + def test_input_and_output_are_last_turn(self): spans = _make_service_normalized_session_spans(num_turns=3) result = map_spans(spans)