feat: GenAI Client(evals) - Add EvaluationResults to get_evaluation_run method response in Vertex AI GenAI SDK evals

PiperOrigin-RevId: 813450051

Author: vertex-sdk-bot

tests/unit/vertexai/genai/replays/test_get_evaluation_run.py

@@ -26,30 +26,7 @@ def test_get_eval_run(client):
         "projects/503583131166/locations/us-central1/evaluationRuns/1957799200510967808"
     )
     evaluation_run = client.evals.get_evaluation_run(name=evaluation_run_name)
-    assert isinstance(evaluation_run, types.EvaluationRun)
-    assert evaluation_run.name == evaluation_run_name
-    assert evaluation_run.display_name == "test2"
-    assert evaluation_run.metadata == {"pipeline_id": "4460531348888616960"}
-    assert evaluation_run.create_time == datetime.datetime(
-        2025, 9, 8, 20, 55, 41, 833176, tzinfo=datetime.timezone.utc
-    )
-    assert evaluation_run.completion_time == datetime.datetime(
-        2025, 9, 8, 20, 56, 13, 492971, tzinfo=datetime.timezone.utc
-    )
-    assert evaluation_run.state == types.EvaluationRunState.SUCCEEDED
-    assert evaluation_run.evaluation_set_snapshot == (
-        "projects/503583131166/locations/us-central1/evaluationSets/8069535738573619200"
-    )
-    assert evaluation_run.data_source.bigquery_request_set == types.BigQueryRequestSet(
-        uri="bq://lakeyk-test-limited.inference_batch_prediction_input.1317387725199900672_1b",
-        prompt_column="request",
-        candidate_response_columns={
-            "baseline_model_response": "baseline_model_response",
-            "checkpoint_1": "checkpoint_1",
-            "checkpoint_2": "checkpoint_2",
-        },
-    )
-    assert evaluation_run.error is None
+    check_run_1957799200510967808(evaluation_run, evaluation_run_name)
 
 
 def test_get_eval_run_bq_source(client):
@@ -104,13 +81,30 @@ def test_get_eval_run_eval_set_source(client):
 async def test_get_eval_run_async(client):
     """Tests that get_evaluation_run() returns a correctly structured EvaluationRun."""
     eval_run_id = "1957799200510967808"
-    eval_run_name = (
+    evaluation_run_name = (
         f"projects/503583131166/locations/us-central1/evaluationRuns/{eval_run_id}"
     )
     evaluation_run = await client.aio.evals.get_evaluation_run(name=eval_run_id)
+    check_run_1957799200510967808(evaluation_run, evaluation_run_name)
+
+
+def check_run_1957799200510967808(
+    evaluation_run: types.EvaluationRun, evaluation_run_name: str
+):
     assert isinstance(evaluation_run, types.EvaluationRun)
-    assert evaluation_run.name == eval_run_name
+    assert evaluation_run.name == evaluation_run_name
     assert evaluation_run.display_name == "test2"
+    assert evaluation_run.metadata == {"pipeline_id": "4460531348888616960"}
+    assert evaluation_run.create_time == datetime.datetime(
+        2025, 9, 8, 20, 55, 41, 833176, tzinfo=datetime.timezone.utc
+    )
+    assert evaluation_run.completion_time == datetime.datetime(
+        2025, 9, 8, 20, 56, 13, 492971, tzinfo=datetime.timezone.utc
+    )
+    assert evaluation_run.state == types.EvaluationRunState.SUCCEEDED
+    assert evaluation_run.evaluation_set_snapshot == (
+        "projects/503583131166/locations/us-central1/evaluationSets/8069535738573619200"
+    )
     assert evaluation_run.data_source.bigquery_request_set == types.BigQueryRequestSet(
         uri="bq://lakeyk-test-limited.inference_batch_prediction_input.1317387725199900672_1b",
         prompt_column="request",
@@ -120,6 +114,80 @@ async def test_get_eval_run_async(client):
             "checkpoint_2": "checkpoint_2",
         },
     )
+    assert evaluation_run.evaluation_results.evaluation_set == (
+        "projects/503583131166/locations/us-central1/evaluationSets/102386522778501120"
+    )
+    assert evaluation_run.evaluation_results.summary_metrics == (
+        types.SummaryMetric(
+            metrics={
+                "checkpoint_1/user_defined/MODE": 5,
+                "checkpoint_2/universal/P90": 1,
+                "gemini-2.0-flash-001@default/universal/AVERAGE": 0.6943817985685249,
+                "gemini-2.0-flash-001@default/user_defined/P90": 5,
+                "gemini-2.0-flash-001@default/universal/VARIANCE": 0.03146487552180889,
+                "gemini-2.0-flash-001@default/user_defined/P95": 5,
+                "checkpoint_1/universal/MINIMUM": 0.8571428656578064,
+                "checkpoint_1/universal/VARIANCE": 0.0015452162403157982,
+                "gemini-2.0-flash-001@default/universal/STANDARD_DEVIATION": 0.17738341388587855,
+                "checkpoint_2/user_defined/P95": 5,
+                "checkpoint_2/universal/MODE": 1,
+                "checkpoint_2/user_defined/P90": 5,
+                "checkpoint_2/universal/P99": 1,
+                "gemini-2.0-flash-001@default/universal/MAXIMUM": 1,
+                "checkpoint_2/universal/P95": 1,
+                "checkpoint_2/user_defined/P99": 5,
+                "checkpoint_2/universal/MINIMUM": 0.7777777910232544,
+                "gemini-2.0-flash-001@default/universal/P90": 0.8777777791023255,
+                "checkpoint_1/universal/AVERAGE": 0.986633250587865,
+                "checkpoint_1/universal/MAXIMUM": 1,
+                "checkpoint_1/universal/STANDARD_DEVIATION": 0.0393092386127714,
+                "gemini-2.0-flash-001@default/universal/P95": 0.9000000059604645,
+                "gemini-2.0-flash-001@default/user_defined/MAXIMUM": 5,
+                "gemini-2.0-flash-001@default/user_defined/MINIMUM": 3,
+                "gemini-2.0-flash-001@default/user_defined/VARIANCE": 0.4044321329639886,
+                "checkpoint_2/user_defined/MAXIMUM": 5,
+                "checkpoint_1/universal/MEDIAN": 1,
+                "gemini-2.0-flash-001@default/universal/MEDIAN": 0.7142857313156128,
+                "gemini-2.0-flash-001@default/user_defined/AVERAGE": 4.736842105263158,
+                "gemini-2.0-flash-001@default/user_defined/MEDIAN": 5,
+                "checkpoint_2/user_defined/AVERAGE": 5,
+                "checkpoint_2/user_defined/MEDIAN": 5,
+                "checkpoint_2/user_defined/STANDARD_DEVIATION": 0,
+                "checkpoint_2/universal/MAXIMUM": 1,
+                "checkpoint_1/universal/MODE": 1,
+                "checkpoint_2/user_defined/MINIMUM": 5,
+                "checkpoint_1/user_defined/VARIANCE": 0,
+                "checkpoint_2/universal/VARIANCE": 0.005771725970062436,
+                "checkpoint_2/universal/AVERAGE": 0.9438178790243048,
+                "checkpoint_1/user_defined/MINIMUM": 5,
+                "gemini-2.0-flash-001@default/universal/P99": 0.9800000011920929,
+                "gemini-2.0-flash-001@default/universal/MINIMUM": 0.2857142984867096,
+                "checkpoint_2/user_defined/VARIANCE": 0,
+                "checkpoint_1/user_defined/MEDIAN": 5,
+                "checkpoint_2/universal/STANDARD_DEVIATION": 0.07597187617837561,
+                "checkpoint_1/user_defined/AVERAGE": 5,
+                "checkpoint_1/user_defined/MAXIMUM": 5,
+                "gemini-2.0-flash-001@default/user_defined/MODE": 5,
+                "checkpoint_1/user_defined/P95": 5,
+                "checkpoint_1/universal/P99": 1,
+                "checkpoint_1/user_defined/P90": 5,
+                "checkpoint_2/universal/MEDIAN": 1,
+                "checkpoint_1/universal/P95": 1,
+                "checkpoint_1/user_defined/STANDARD_DEVIATION": 0,
+                "gemini-2.0-flash-001@default/user_defined/STANDARD_DEVIATION": 0.6359497880839245,
+                "checkpoint_1/user_defined/P99": 5,
+                "gemini-2.0-flash-001@default/universal/MODE": [
+                    0.75,
+                    0.8571428656578064,
+                ],
+                "checkpoint_2/user_defined/MODE": 5,
+                "checkpoint_1/universal/P90": 1,
+                "gemini-2.0-flash-001@default/user_defined/P99": 5,
+            },
+            total_items=19,
+        )
+    )
+    assert evaluation_run.error is None
 
 
 pytestmark = pytest_helper.setup(

vertexai/_genai/evals.py

@@ -438,6 +438,26 @@ def _EvaluationRunDataSource_to_vertex(
     return to_object
 
 
+def _EvaluationRunResults_from_vertex(
+    from_object: Union[dict[str, Any], object],
+    parent_object: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    to_object: dict[str, Any] = {}
+    if getv(from_object, ["evaluationSet"]) is not None:
+        setv(to_object, ["evaluation_set"], getv(from_object, ["evaluationSet"]))
+
+    if getv(from_object, ["summaryMetrics"]) is not None:
+        setv(
+            to_object,
+            ["summary_metrics"],
+            _SummaryMetric_from_vertex(
+                getv(from_object, ["summaryMetrics"]), to_object
+            ),
+        )
+
+    return to_object
+
+
 def _EvaluationRun_from_vertex(
     from_object: Union[dict[str, Any], object],
     parent_object: Optional[dict[str, Any]] = None,
@@ -480,6 +500,15 @@ def _EvaluationRun_from_vertex(
             ),
         )
 
+    if getv(from_object, ["evaluationResults"]) is not None:
+        setv(
+            to_object,
+            ["evaluation_results"],
+            _EvaluationRunResults_from_vertex(
+                getv(from_object, ["evaluationResults"]), to_object
+            ),
+        )
+
     return to_object
 
 
@@ -890,6 +919,23 @@ def _SamplingConfig_to_vertex(
     return to_object
 
 
+def _SummaryMetric_from_vertex(
+    from_object: Union[dict[str, Any], object],
+    parent_object: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    to_object: dict[str, Any] = {}
+    if getv(from_object, ["metrics"]) is not None:
+        setv(to_object, ["metrics"], getv(from_object, ["metrics"]))
+
+    if getv(from_object, ["totalItems"]) is not None:
+        setv(to_object, ["total_items"], getv(from_object, ["totalItems"]))
+
+    if getv(from_object, ["failedItems"]) is not None:
+        setv(to_object, ["failed_items"], getv(from_object, ["failedItems"]))
+
+    return to_object
+
+
 def _ToolCallValidInput_to_vertex(
     from_object: Union[dict[str, Any], object],
     parent_object: Optional[dict[str, Any]] = None,

vertexai/_genai/types.py

@@ -488,6 +488,61 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False):
 ]
 
 
+class SummaryMetric(_common.BaseModel):
+    """Represents a summary metric for an evaluation run."""
+
+    metrics: Optional[dict[str, Any]] = Field(
+        default=None, description="""Map of metric name to metric value."""
+    )
+    total_items: Optional[int] = Field(
+        default=None, description="""The total number of items that were evaluated."""
+    )
+    failed_items: Optional[int] = Field(
+        default=None, description="""The number of items that failed to be evaluated."""
+    )
+
+
+class SummaryMetricDict(TypedDict, total=False):
+    """Represents a summary metric for an evaluation run."""
+
+    metrics: Optional[dict[str, Any]]
+    """Map of metric name to metric value."""
+
+    total_items: Optional[int]
+    """The total number of items that were evaluated."""
+
+    failed_items: Optional[int]
+    """The number of items that failed to be evaluated."""
+
+
+SummaryMetricOrDict = Union[SummaryMetric, SummaryMetricDict]
+
+
+class EvaluationRunResults(_common.BaseModel):
+    """Represents the results of an evaluation run."""
+
+    evaluation_set: Optional[str] = Field(
+        default=None,
+        description="""The evaluation set where item level results are stored.""",
+    )
+    summary_metrics: Optional[SummaryMetric] = Field(
+        default=None, description="""The summary metrics for the evaluation run."""
+    )
+
+
+class EvaluationRunResultsDict(TypedDict, total=False):
+    """Represents the results of an evaluation run."""
+
+    evaluation_set: Optional[str]
+    """The evaluation set where item level results are stored."""
+
+    summary_metrics: Optional[SummaryMetricDict]
+    """The summary metrics for the evaluation run."""
+
+
+EvaluationRunResultsOrDict = Union[EvaluationRunResults, EvaluationRunResultsDict]
+
+
 class EvaluationRun(_common.BaseModel):
     """Represents an evaluation run."""
 
@@ -506,6 +561,9 @@ class EvaluationRun(_common.BaseModel):
     data_source: Optional[EvaluationRunDataSource] = Field(
         default=None, description=""""""
     )
+    evaluation_results: Optional[EvaluationRunResults] = Field(
+        default=None, description="""The results for the evaluation run."""
+    )
 
 
 class EvaluationRunDict(TypedDict, total=False):
@@ -538,6 +596,9 @@ class EvaluationRunDict(TypedDict, total=False):
     data_source: Optional[EvaluationRunDataSourceDict]
     """"""
 
+    evaluation_results: Optional[EvaluationRunResultsDict]
+    """The results for the evaluation run."""
+
 
 EvaluationRunOrDict = Union[EvaluationRun, EvaluationRunDict]