feat: GenAI Client(evals) - Register hallucination and tool_usage_quality metrics for agent eval

PiperOrigin-RevId: 821797229

Author: vertex-sdk-bot

tests/unit/vertexai/genai/test_evals.py

@@ -4266,6 +4266,28 @@ def test_execute_evaluation_llm_metric(
         call_args = mock_eval_dependencies["mock_evaluate_instances"].call_args
         assert "pointwise_metric_input" in call_args[1]["metric_config"]
 
+    def test_execute_evaluation_hallucination_metric(self, mock_api_client_fixture):
+        dataset_df = pd.DataFrame(
+            [{"prompt": "Test prompt", "response": "Test response"}]
+        )
+        input_dataset = vertexai_genai_types.EvaluationDataset(
+            eval_dataset_df=dataset_df
+        )
+
+        result = _evals_common._execute_evaluation(
+            api_client=mock_api_client_fixture,
+            dataset=input_dataset,
+            metrics=[
+                vertexai_genai_types.RubricMetric.HALLUCINATION,
+                vertexai_genai_types.RubricMetric.TOOL_USE_QUALITY,
+            ],
+        )
+        assert isinstance(result, vertexai_genai_types.EvaluationResult)
+        assert result.evaluation_dataset == [input_dataset]
+        assert len(result.summary_metrics) == 2
+        assert result.summary_metrics[0].metric_name == "hallucination_v1"
+        assert result.summary_metrics[1].metric_name == "tool_use_quality_v1"
+
     @mock.patch.object(_evals_data_converters, "get_dataset_converter")
     def test_execute_evaluation_with_openai_schema(
         self,

vertexai/_genai/_evals_constant.py

@@ -26,6 +26,8 @@
         "final_response_match_v2",
         "final_response_reference_free_v1",
         "final_response_quality_v1",
+        "hallucination_v1",
+        "tool_use_quality_v1",
     }
 )
 

vertexai/_genai/_evals_utils.py

@@ -595,6 +595,14 @@ def MULTI_TURN_SAFETY(self) -> LazyLoadedPrebuiltMetric:
     def FINAL_RESPONSE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
         return self.__getattr__("FINAL_RESPONSE_QUALITY")
 
+    @property
+    def HALLUCINATION(self) -> LazyLoadedPrebuiltMetric:
+        return self.__getattr__("HALLUCINATION")
+
+    @property
+    def TOOL_USE_QUALITY(self) -> LazyLoadedPrebuiltMetric:
+        return self.__getattr__("TOOL_USE_QUALITY")
+
 
 PrebuiltMetric = PrebuiltMetricLoader()
 RubricMetric = PrebuiltMetric

vertexai/_genai/_evals_visualization.py

@@ -450,22 +450,74 @@ def _get_evaluation_html(eval_result_json: str) -> str:
                 const candidateMetrics = (caseResult.response_candidate_results && caseResult.response_candidate_results[0] && caseResult.response_candidate_results[0].metric_results) || {{}};
                 Object.entries(candidateMetrics).forEach(([name, val]) => {{
                     let metricNameCell = name;
-                    if (val.rubric_verdicts && val.rubric_verdicts.length > 0) {{
-                        metricNameCell += '<div class="rubric-bubble-container" style="margin-top: 8px;">';
+                    let explanationHandled = false;
+                    let bubbles = '';
+
+                    if (name.startsWith('hallucination') && val.explanation) {{
+                        try {{
+                            const explanationData = JSON.parse(val.explanation);
+                            if (Array.isArray(explanationData) && explanationData.length > 0 && explanationData[0].sentence) {{
+                                bubbles += '<div class="rubric-bubble-container" style="margin-top: 8px;">';
+                                explanationData.forEach(item => {{
+                                    const sentence = item.sentence || 'N/A';
+                                    const label = item.label ? item.label.toLowerCase() : '';
+                                    const verdictText = label === 'no_rad' ? '<span class="pass">Pass</span>' : '<span class="fail">Fail</span>';
+                                    const rationale = item.rationale || 'N/A';
+                                    const itemJson = JSON.stringify(item, null, 2);
+                                    bubbles += `
+                                        <details class="rubric-details">
+                                            <summary class="rubric-bubble">${{verdictText}}: ${{DOMPurify.sanitize(sentence)}}</summary>
+                                            <div class="explanation" style="padding: 10px 0 0 20px;">${{DOMPurify.sanitize(rationale)}}</div>
+                                            <pre class="raw-json-container">${{DOMPurify.sanitize(itemJson)}}</pre>
+                                        </details>`;
+                                }});
+                                bubbles += '</div>';
+                                explanationHandled = true;
+                            }}
+                        }} catch (e) {{
+                            console.error("Failed to parse hallucination explanation:", e);
+                        }}
+                    }} else if (name.startsWith('safety') && val.score != null) {{
+                        try {{
+                            bubbles += '<div class="rubric-bubble-container" style="margin-top: 8px;">';
+                            const verdictText = val.score >= 1.0 ? '<span class="pass">Pass</span>' : '<span class="fail">Fail</span>';
+                            const explanation = val.explanation || (val.score >= 1.0 ? 'Safety check passed' : 'Safety check failed');
+                            const itemJson = JSON.stringify(val, null, 2);
+                            bubbles += `
+                                <details class="rubric-details">
+                                    <summary class="rubric-bubble">${{verdictText}}: ${{DOMPurify.sanitize(explanation)}}</summary>
+                                    <pre class="raw-json-container">${{DOMPurify.sanitize(itemJson)}}</pre>
+                                </details>`;
+                            bubbles += '</div>';
+                            explanationHandled = true;
+                        }} catch (e) {{
+                            console.error("Failed to process safety metric:", e);
+                        }}
+                    }}
+
+                    if (!bubbles && val.rubric_verdicts && val.rubric_verdicts.length > 0) {{
+                        bubbles += '<div class="rubric-bubble-container" style="margin-top: 8px;">';
                         val.rubric_verdicts.forEach(verdict => {{
                             const rubricDescription = verdict.evaluated_rubric && verdict.evaluated_rubric.content && verdict.evaluated_rubric.content.property ? verdict.evaluated_rubric.content.property.description : 'N/A';
                             const verdictText = verdict.verdict ? '<span class="pass">Pass</span>' : '<span class="fail">Fail</span>';
                             const verdictJson = JSON.stringify(verdict, null, 2);
-                            metricNameCell += `
+                            bubbles += `
                                 <details class="rubric-details">
                                     <summary class="rubric-bubble">${{verdictText}}: ${{DOMPurify.sanitize(rubricDescription)}}</summary>
                                     <pre class="raw-json-container">${{DOMPurify.sanitize(verdictJson)}}</pre>
                                 </details>`;
                         }});
-                        metricNameCell += '</div>';
+                        bubbles += '</div>';
+                    }}
+
+                    if(bubbles) {{
+                        metricNameCell += bubbles;
                     }}
+
                     metricTable += `<tr><td>${{metricNameCell}}</td><td><b>${{val.score != null ? val.score.toFixed(2) : 'N/A'}}</b></td></tr>`;
-                    if (val.explanation) {{ metricTable += `<tr><td colspan="2"><div class="explanation">${{DOMPurify.sanitize(marked.parse(String(val.explanation)))}}</div></td></tr>`; }}
+                    if (val.explanation && !explanationHandled) {{
+                        metricTable += `<tr><td colspan="2"><div class="explanation">${{DOMPurify.sanitize(marked.parse(String(val.explanation)))}}</div></td></tr>`;
+                    }}
                 }});
                 card += metricTable + '</tbody></table>';
                 container.innerHTML += card + '</details>';