feat: GenAI Client(evals) - Add agent eval data converter evals

PiperOrigin-RevId: 819989463

Author: vertex-sdk-bot

tests/unit/vertexai/genai/test_evals.py

@@ -1885,6 +1885,56 @@ def test_convert_with_additional_columns(self):
         eval_case = result_dataset.eval_cases[0]
         assert eval_case.custom_column == "custom_value"
 
+    def test_convert_with_agent_eval_fields(self):
+        """Tests that agent eval data is converted correctly from a flattened format."""
+        raw_data_df = pd.DataFrame(
+            {
+                "prompt": ["Hello"],
+                "response": ["Hi"],
+                "intermediate_events": [
+                    [
+                        {
+                            "event_id": "event1",
+                            "content": {"parts": [{"text": "intermediate event"}]},
+                        }
+                    ]
+                ],
+            }
+        )
+        raw_data = raw_data_df.to_dict(orient="records")
+        result_dataset = self.converter.convert(raw_data)
+        assert len(result_dataset.eval_cases) == 1
+        eval_case = result_dataset.eval_cases[0]
+        assert eval_case.intermediate_events[0].event_id == "event1"
+
+    def test_convert_with_intermediate_events_as_event_objects(self):
+        """Tests that agent eval data is converted correctly when intermediate_events are Event objects."""
+        raw_data_df = pd.DataFrame(
+            {
+                "prompt": ["Hello"],
+                "response": ["Hi"],
+                "intermediate_events": [
+                    [
+                        vertexai_genai_types.Event(
+                            event_id="event1",
+                            content=genai_types.Content(
+                                parts=[genai_types.Part(text="intermediate event")]
+                            ),
+                        )
+                    ]
+                ],
+            }
+        )
+        raw_data = raw_data_df.to_dict(orient="records")
+        result_dataset = self.converter.convert(raw_data)
+        assert len(result_dataset.eval_cases) == 1
+        eval_case = result_dataset.eval_cases[0]
+        assert eval_case.intermediate_events[0].event_id == "event1"
+        assert (
+            eval_case.intermediate_events[0].content.parts[0].text
+            == "intermediate event"
+        )
+
 
 class TestOpenAIDataConverter:
     """Unit tests for the _OpenAIDataConverter class."""
@@ -2765,7 +2815,10 @@ def test_merge_flatten_and_gemini_datasets(self):
         )
 
     def test_merge_empty_input_list(self):
-        with pytest.raises(ValueError, match="Input 'raw_datasets' cannot be empty."):
+        with pytest.raises(
+            ValueError,
+            match="Input 'raw_datasets' cannot be empty and must be a list of lists.",
+        ):
             _evals_data_converters.merge_response_datasets_into_canonical_format(
                 raw_datasets=[], schemas=[]
             )
@@ -2810,7 +2863,10 @@ def test_merge_mismatched_schemas_list_length(self):
         ]
         with pytest.raises(
             ValueError,
-            match="A list of schemas must be provided, one for each raw dataset.",
+            match=(
+                "A list of schemas must be provided, one for each raw dataset. Got 2"
+                " schemas for 3 datasets."
+            ),
         ):
             _evals_data_converters.merge_response_datasets_into_canonical_format(
                 [raw_dataset_1, raw_dataset_2, raw_dataset_3],
@@ -2824,7 +2880,10 @@ def test_merge_empty_schemas_list(self):
         ]
         with pytest.raises(
             ValueError,
-            match="A list of schemas must be provided, one for each raw dataset.",
+            match=(
+                "A list of schemas must be provided, one for each raw dataset. Got 0"
+                " schemas for 1 datasets."
+            ),
         ):
             _evals_data_converters.merge_response_datasets_into_canonical_format(
                 [raw_dataset_1], schemas=[]
@@ -2918,6 +2977,46 @@ def test_merge_with_different_custom_columns(self):
         assert merged_dataset.eval_cases[1].custom_col_2 == "value_2_2"
         assert merged_dataset.eval_cases[1].custom_col_3 == "value_2_3"
 
+    def test_merge_with_intermediate_events(self):
+        raw_dataset_1 = [
+            {
+                "prompt": "Prompt 1",
+                "response": "Response 1a",
+                "intermediate_events": [
+                    {
+                        "event_id": "event1",
+                        "content": {"parts": [{"text": "intermediate event"}]},
+                    }
+                ],
+            }
+        ]
+        raw_dataset_2 = [
+            {
+                "prompt": "Prompt 1",
+                "response": "Response 1b",
+                "intermediate_events": [
+                    {
+                        "event_id": "event2",
+                        "content": {"parts": [{"text": "intermediate event 2"}]},
+                    }
+                ],
+            }
+        ]
+        schemas = [
+            _evals_data_converters.EvalDatasetSchema.FLATTEN,
+            _evals_data_converters.EvalDatasetSchema.FLATTEN,
+        ]
+
+        merged_dataset = (
+            _evals_data_converters.merge_response_datasets_into_canonical_format(
+                [raw_dataset_1, raw_dataset_2], schemas=schemas
+            )
+        )
+
+        assert len(merged_dataset.eval_cases) == 1
+        assert len(merged_dataset.eval_cases[0].intermediate_events) == 1
+        assert merged_dataset.eval_cases[0].intermediate_events[0].event_id == "event1"
+
     def test_merge_with_metadata(self):
         raw_dataset_1 = [
             {

vertexai/_genai/_evals_data_converters.py

@@ -14,6 +14,7 @@
 #
 """Dataset converters for evals."""
 
+import copy
 import json
 import logging
 from typing import Any, Optional, Union
@@ -189,7 +190,7 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
                     f"Expected a dictionary for item at index {i}, but got"
                     f" {type(item_dict).__name__}: {item_dict}"
                 )
-            item = item_dict.copy()
+            item = copy.deepcopy(item_dict)
             eval_case_id = f"eval_case_{i}"
             prompt_data = item.pop("prompt", None)
             if not prompt_data:
@@ -200,6 +201,7 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
             reference_data = item.pop("reference", None)
             system_instruction_data = item.pop("instruction", None)
             rubric_groups_data = item.pop("rubric_groups", None)
+            intermediate_events_data = item.pop("intermediate_events", None)
 
             if not response_data:
                 raise ValueError(
@@ -362,6 +364,38 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
                         f"Invalid type for rubric_groups in case {i}. Expected dict."
                     )
 
+            intermediate_events: Optional[list[types.Event]] = None
+            if intermediate_events_data:
+                logger.warning(
+                    "intermediate_events attribute is experimental and may change in "
+                    "future versions."
+                )
+                if isinstance(intermediate_events_data, list):
+                    intermediate_events = []
+                    for event in intermediate_events_data:
+                        if isinstance(event, dict):
+                            try:
+                                validated_event = types.Event.model_validate(event)
+                                intermediate_events.append(validated_event)
+                            except Exception as e:
+                                logger.warning(
+                                    "Failed to validate intermediate event dict for"
+                                    f" case {i}: {e}"
+                                )
+                        elif isinstance(event, types.Event):
+                            intermediate_events.append(event)
+                        else:
+                            logger.warning(
+                                "Invalid type for intermediate_event in case"
+                                f" {i}. Expected list of dicts or list of"
+                                " types.Event objects."
+                            )
+                else:
+                    logger.warning(
+                        f"Invalid type for intermediate_events in case {i}. Expected"
+                        " list of types.Event objects."
+                    )
+
             eval_case = types.EvalCase(
                 eval_case_id=eval_case_id,
                 prompt=prompt,
@@ -370,6 +404,7 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
                 conversation_history=conversation_history,
                 system_instruction=system_instruction,
                 rubric_groups=rubric_groups,
+                intermediate_events=intermediate_events,
                 **item,  # Pass remaining columns as extra fields to EvalCase.
                 # They can be used for custom metric prompt templates.
             )
@@ -726,6 +761,7 @@ def merge_response_datasets_into_canonical_format(
                 "reference",
                 "system_instruction",
                 "conversation_history",
+                "intermediate_events",
             },
             exclude_none=True,
         )
@@ -750,6 +786,7 @@ def merge_response_datasets_into_canonical_format(
                     "reference",
                     "system_instruction",
                     "conversation_history",
+                    "intermediate_events",
                 },
                 exclude_none=True,
             )
@@ -777,6 +814,7 @@ def merge_response_datasets_into_canonical_format(
             reference=base_eval_case.reference,
             system_instruction=base_eval_case.system_instruction,
             conversation_history=base_eval_case.conversation_history,
+            intermediate_events=base_eval_case.intermediate_events,
             **eval_case_custom_columns,
         )
         merged_eval_cases.append(merged_case)

vertexai/_genai/_evals_visualization.py

@@ -56,15 +56,15 @@ def _preprocess_df_for_json(df: Optional[pd.DataFrame]) -> Optional[pd.DataFrame
         ):
 
             def stringify_cell(cell: Any) -> Optional[str]:
-                if pd.isna(cell):
-                    return None
                 if isinstance(cell, (dict, list)):
                     try:
                         return json.dumps(
                             cell, ensure_ascii=False, default=_pydantic_serializer
                         )
                     except TypeError:
                         return str(cell)
+                elif pd.isna(cell):
+                    return None
                 elif not isinstance(cell, (str, int, float, bool)):
                     if hasattr(cell, "model_dump"):
                         return json.dumps(