feat: GenAI Client(evals) - Add evaluation_df input support to create_evaluation_run method in Vertex AI GenAI SDK evals

PiperOrigin-RevId: 821845677

Author: vertex-sdk-bot

tests/unit/vertexai/genai/replays/test_create_evaluation_run.py

@@ -38,15 +38,15 @@ def test_create_eval_run_data_source_evaluation_set(client):
     evaluation_run = client.evals.create_evaluation_run(
         name="test4",
         display_name="test4",
-        data_source=types.EvaluationRunDataSource(
+        dataset=types.EvaluationRunDataSource(
             evaluation_set="projects/503583131166/locations/us-central1/evaluationSets/6619939608513740800"
         ),
         agent_info=types.AgentInfo(
             name="agent-1",
             instruction="agent-1 instruction",
             tool_declarations=[tool],
         ),
-        dest="gs://lakeyk-test-limited/eval_run_output",
+        dest="gs://lakeyk-limited-bucket/eval_run_output",
     )
     assert isinstance(evaluation_run, types.EvaluationRun)
     assert evaluation_run.display_name == "test4"
@@ -73,7 +73,7 @@ def test_create_eval_run_data_source_bigquery_request_set(client):
     evaluation_run = client.evals.create_evaluation_run(
         name="test5",
         display_name="test5",
-        data_source=types.EvaluationRunDataSource(
+        dataset=types.EvaluationRunDataSource(
             bigquery_request_set=types.BigQueryRequestSet(
                 uri="bq://lakeyk-test-limited.inference_batch_prediction_input.1317387725199900672_1b",
                 prompt_column="request",
@@ -84,7 +84,7 @@ def test_create_eval_run_data_source_bigquery_request_set(client):
                 },
             )
         ),
-        dest="gs://lakeyk-test-limited/eval_run_output",
+        dest="gs://lakeyk-limited-bucket/eval_run_output",
     )
     assert isinstance(evaluation_run, types.EvaluationRun)
     assert evaluation_run.display_name == "test5"
@@ -105,6 +105,101 @@ def test_create_eval_run_data_source_bigquery_request_set(client):
     assert evaluation_run.error is None
 
 
+# Test fails in replay mode because of the timestamp issue
+# def test_create_eval_run_data_source_evaluation_dataset(client):
+#     """Tests that create_evaluation_run() creates a correctly structured EvaluationRun with EvaluationDataset."""
+#     input_df = pd.DataFrame(
+#         {
+#             "prompt": ["prompt1", "prompt2"],
+#             "reference": ["reference1", "reference2"],
+#             "response": ["response1", "response2"],
+#             "intermediate_events": [
+#                 [
+#                     {
+#                         "content": {
+#                             "parts": [
+#                                 {"text": "first user input"},
+#                             ],
+#                             "role": "user",
+#                         },
+#                     },
+#                     {
+#                         "content": {
+#                             "parts": [
+#                                 {"text": "first model response"},
+#                             ],
+#                             "role": "model",
+#                         },
+#                     },
+#                 ],
+#                 [
+#                     {
+#                         "content": {
+#                             "parts": [
+#                                 {"text": "second user input"},
+#                             ],
+#                             "role": "user",
+#                         },
+#                     },
+#                     {
+#                         "content": {
+#                             "parts": [
+#                                 {"text": "second model response"},
+#                             ],
+#                             "role": "model",
+#                         },
+#                     },
+#                 ],
+#             ],
+#         }
+#     )
+#     evaluation_run = client.evals.create_evaluation_run(
+#         name="test6",
+#         display_name="test6",
+#         dataset=types.EvaluationDataset(
+#             candidate_name="candidate_1",
+#             eval_dataset_df=input_df,
+#         ),
+#         dest="gs://lakeyk-limited-bucket/eval_run_output",
+#     )
+#     assert isinstance(evaluation_run, types.EvaluationRun)
+#     assert evaluation_run.display_name == "test6"
+#     assert evaluation_run.state == types.EvaluationRunState.PENDING
+#     assert isinstance(evaluation_run.data_source, types.EvaluationRunDataSource)
+#     # Check evaluation set
+#     assert evaluation_run.data_source.evaluation_set
+#     eval_set = client.evals.get_evaluation_set(
+#         name=evaluation_run.data_source.evaluation_set
+#     )
+#     assert len(eval_set.evaluation_items) == 2
+#     # Check evaluation items
+#     for i, eval_item_name in enumerate(eval_set.evaluation_items):
+#         eval_item = client.evals.get_evaluation_item(name=eval_item_name)
+#         assert eval_item.evaluation_item_type == types.EvaluationItemType.REQUEST
+#         assert eval_item.evaluation_request.prompt.text == input_df.iloc[i]["prompt"]
+#         assert (
+#             eval_item.evaluation_request.candidate_responses[0].text
+#             == input_df.iloc[i]["response"]
+#         )
+#         assert (
+#             eval_item.evaluation_request.candidate_responses[0].events[0].parts[0].text
+#             == input_df.iloc[i]["intermediate_events"][0]["content"]["parts"][0]["text"]
+#         )
+#         assert (
+#             eval_item.evaluation_request.candidate_responses[0].events[0].role
+#             == input_df.iloc[i]["intermediate_events"][0]["content"]["role"]
+#         )
+#         assert (
+#             eval_item.evaluation_request.candidate_responses[0].events[1].parts[0].text
+#             == input_df.iloc[i]["intermediate_events"][1]["content"]["parts"][0]["text"]
+#         )
+#         assert (
+#             eval_item.evaluation_request.candidate_responses[0].events[1].role
+#             == input_df.iloc[i]["intermediate_events"][1]["content"]["role"]
+#         )
+#     assert evaluation_run.error is None
+
+
 pytest_plugins = ("pytest_asyncio",)
 
 
@@ -114,7 +209,7 @@ async def test_create_eval_run_async(client):
     evaluation_run = await client.aio.evals.create_evaluation_run(
         name="test8",
         display_name="test8",
-        data_source=types.EvaluationRunDataSource(
+        dataset=types.EvaluationRunDataSource(
             bigquery_request_set=types.BigQueryRequestSet(
                 uri="bq://lakeyk-test-limited.inference_batch_prediction_input.1317387725199900672_1b",
                 prompt_column="request",
@@ -125,7 +220,7 @@ async def test_create_eval_run_async(client):
                 },
             )
         ),
-        dest="gs://lakeyk-test-limited/eval_run_output",
+        dest="gs://lakeyk-limited-bucket/eval_run_output",
     )
     assert isinstance(evaluation_run, types.EvaluationRun)
     assert evaluation_run.display_name == "test8"

vertexai/_genai/_evals_common.py

@@ -1505,3 +1505,89 @@ async def _convert_evaluation_run_results_async(
         ]
         eval_items = await asyncio.gather(*tasks)
     return _get_eval_result_from_eval_items(evaluation_run_results, eval_items)
+
+
+def _object_to_dict(obj) -> dict[str, Any]:
+    """Converts an object to a dictionary."""
+    if not hasattr(obj, "__dict__"):
+        return obj  # Not an object with attributes, return as is (e.g., int, str)
+
+    result: dict[str, Any] = {}
+    for key, value in obj.__dict__.items():
+        if value is None:
+            continue
+        if isinstance(value, (int, float, str, bool)):
+            result[key] = value
+        elif isinstance(value, (list, tuple)):
+            result[key] = [_object_to_dict(item) for item in value]
+        elif hasattr(value, "__dict__"):  # Nested object
+            result[key] = _object_to_dict(value)
+        else:
+            result[key] = value  # Handle other types like sets, etc.
+    return result
+
+
+def _create_evaluation_set_from_dataframe(
+    api_client: BaseApiClient,
+    gcs_dest_prefix: str,
+    eval_df: pd.DataFrame,
+    candidate_name: Optional[str] = None,
+) -> types.EvaluationSet:
+    """Converts a dataframe to an EvaluationSet."""
+    eval_item_requests = []
+    for _, row in eval_df.iterrows():
+        intermediate_events = []
+        if "intermediate_events" in row:
+            for event in row["intermediate_events"]:
+                intermediate_events.append(
+                    genai_types.Content(
+                        parts=event["content"]["parts"], role=event["content"]["role"]
+                    )
+                )
+        eval_item_requests.append(
+            types.EvaluationItemRequest(
+                prompt=(
+                    types.EvaluationPrompt(text=row["prompt"])
+                    if "prompt" in row
+                    else None
+                ),
+                golden_response=(
+                    types.CandidateResponse(text=row["reference"])
+                    if "reference" in row
+                    else None
+                ),
+                candidate_responses=[
+                    types.CandidateResponse(
+                        candidate=candidate_name or "Candidate 1",
+                        text=row.get("response", None),
+                        events=(
+                            intermediate_events
+                            if len(intermediate_events) > 0
+                            else None
+                        ),
+                    )
+                ],
+            )
+        )
+    logger.info("Writing evaluation item requests to GCS.")
+    gcs_utils = _evals_utils.GcsUtils(api_client=api_client)
+    evals_module = evals.Evals(api_client_=api_client)
+    eval_items = []
+    for eval_item_request in eval_item_requests:
+        gcs_uri = gcs_utils.upload_json_to_prefix(
+            data=_object_to_dict(eval_item_request),
+            gcs_dest_prefix=gcs_dest_prefix,
+            filename_prefix="request",
+        )
+        eval_item = evals_module.create_evaluation_item(
+            evaluation_item_type=types.EvaluationItemType.REQUEST,
+            gcs_uri=gcs_uri,
+            display_name="sdk-generated-eval-item",
+        )
+        eval_items.append(eval_item.name)
+    logger.info("Creating evaluation set from GCS URIs")
+    evaluation_set = evals_module.create_evaluation_set(
+        evaluation_items=eval_items,
+    )
+
+    return evaluation_set

vertexai/_genai/evals.py

@@ -1307,12 +1307,24 @@ def create_evaluation_run(
         *,
         name: str,
         display_name: Optional[str] = None,
-        data_source: types.EvaluationRunDataSource,
+        dataset: Union[types.EvaluationRunDataSource, types.EvaluationDataset],
         dest: str,
         agent_info: Optional[types.AgentInfo] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
     ) -> types.EvaluationRun:
         """Creates an EvaluationRun."""
+        if type(dataset).__name__ == "EvaluationDataset":
+            logger.warning(
+                "EvaluationDataset input is experimental and may change in future versions."
+            )
+            if dataset.eval_dataset_df is None:
+                raise ValueError(
+                    "EvaluationDataset must have eval_dataset_df populated."
+                )
+            eval_set = _evals_common._create_evaluation_set_from_dataframe(
+                self._api_client, dest, dataset.eval_dataset_df, dataset.candidate_name
+            )
+            dataset = types.EvaluationRunDataSource(evaluation_set=eval_set.name)
         output_config = genai_types.OutputConfig(
             gcs_destination=genai_types.GcsDestination(output_uri_prefix=dest)
         )
@@ -1334,7 +1346,7 @@ def create_evaluation_run(
         return self._create_evaluation_run(  # type: ignore[no-any-return]
             name=name,
             display_name=display_name,
-            data_source=data_source,
+            data_source=dataset,
             evaluation_config=evaluation_config,
             inference_configs=inference_configs,
             config=config,
@@ -2092,12 +2104,24 @@ async def create_evaluation_run(
         *,
         name: str,
         display_name: Optional[str] = None,
-        data_source: types.EvaluationRunDataSource,
+        dataset: Union[types.EvaluationRunDataSource, types.EvaluationDataset],
         dest: str,
         agent_info: Optional[types.AgentInfo] = None,
         config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
     ) -> types.EvaluationRun:
         """Creates an EvaluationRun."""
+        if type(dataset).__name__ == "EvaluationDataset":
+            logger.warning(
+                "EvaluationDataset input is experimental and may change in future versions."
+            )
+            if dataset.eval_dataset_df is None:
+                raise ValueError(
+                    "EvaluationDataset must have eval_dataset_df populated."
+                )
+            eval_set = _evals_common._create_evaluation_set_from_dataframe(
+                self._api_client, dest, dataset.eval_dataset_df, dataset.candidate_name
+            )
+            dataset = types.EvaluationRunDataSource(evaluation_set=eval_set.name)
         output_config = genai_types.OutputConfig(
             gcs_destination=genai_types.GcsDestination(output_uri_prefix=dest)
         )
@@ -2119,7 +2143,7 @@ async def create_evaluation_run(
         result = await self._create_evaluation_run(  # type: ignore[no-any-return]
             name=name,
             display_name=display_name,
-            data_source=data_source,
+            data_source=dataset,
             evaluation_config=evaluation_config,
             inference_configs=inference_configs,
             config=config,