Merge pull request #189 from salesforcecli/wr/addVerboseFlag

fix: add --verbose to agent test results/resume/run for generatedData @W-19345316@

Author: WillieRuemmele

command-snapshot.json

@@ -96,7 +96,7 @@
     "command": "agent:test:results",
     "flagAliases": [],
     "flagChars": ["d", "i", "o"],
-    "flags": ["api-version", "flags-dir", "job-id", "json", "output-dir", "result-format", "target-org"],
+    "flags": ["api-version", "flags-dir", "job-id", "json", "output-dir", "result-format", "target-org", "verbose"],
     "plugin": "@salesforce/plugin-agent"
   },
   {
@@ -113,6 +113,7 @@
       "result-format",
       "target-org",
       "use-most-recent",
+      "verbose",
       "wait"
     ],
     "plugin": "@salesforce/plugin-agent"
@@ -122,7 +123,17 @@
     "command": "agent:test:run",
     "flagAliases": [],
     "flagChars": ["d", "n", "o", "w"],
-    "flags": ["api-name", "api-version", "flags-dir", "json", "output-dir", "result-format", "target-org", "wait"],
+    "flags": [
+      "api-name",
+      "api-version",
+      "flags-dir",
+      "json",
+      "output-dir",
+      "result-format",
+      "target-org",
+      "verbose",
+      "wait"
+    ],
     "plugin": "@salesforce/plugin-agent"
   }
 ]

messages/shared.md

@@ -10,6 +10,16 @@ Directory to write the agent test results into.
 
 If the agent test run completes, write the results to the specified directory. If the test is still running, the test results aren't written.
 
+# flags.verbose.summary
+
+Show generated data in the test results output.
+
+# flags.verbose.description
+
+When enabled, includes detailed generated data (such as invoked actions) in the human-readable test results output. This is useful for debugging test failures and understanding what actions were actually invoked during the test run.
+
+The generated data is in JSON format and includes the Apex classes or Flows that were invoked, the Salesforce objects that were touched, and so on. Use the JSON structure of this information to build the test case JSONPath expression when using custom evaluations.
+
 # error.invalidAgentType
 
 agentType must be either "customer" or "internal". Found: [%s]

src/commands/agent/test/results.ts

@@ -8,7 +8,7 @@
 import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
 import { Messages } from '@salesforce/core';
 import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents';
-import { resultFormatFlag, testOutputDirFlag } from '../../../flags.js';
+import { resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js';
 import { handleTestResults } from '../../../handleTestResults.js';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
@@ -31,6 +31,7 @@ export default class AgentTestResults extends SfCommand<AgentTestResultsResult>
     }),
     'result-format': resultFormatFlag(),
     'output-dir': testOutputDirFlag(),
+    verbose: verboseFlag,
   };
 
   public async run(): Promise<AgentTestResultsResult> {
@@ -44,6 +45,7 @@ export default class AgentTestResults extends SfCommand<AgentTestResultsResult>
       results: response,
       jsonEnabled: this.jsonEnabled(),
       outputDir: flags['output-dir'],
+      verbose: flags.verbose,
     });
     return response;
   }

src/commands/agent/test/resume.ts

@@ -10,7 +10,7 @@ import { Messages } from '@salesforce/core';
 import { AgentTester } from '@salesforce/agents';
 import { AgentTestCache } from '../../../agentTestCache.js';
 import { TestStages } from '../../../testStages.js';
-import { AgentTestRunResult, resultFormatFlag, testOutputDirFlag } from '../../../flags.js';
+import { AgentTestRunResult, resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js';
 import { handleTestResults } from '../../../handleTestResults.js';
 
 Messages.importMessagesDirectoryFromMetaUrl(import.meta.url);
@@ -43,6 +43,7 @@ export default class AgentTestResume extends SfCommand<AgentTestRunResult> {
     }),
     'result-format': resultFormatFlag(),
     'output-dir': testOutputDirFlag(),
+    verbose: verboseFlag,
   };
 
   public async run(): Promise<AgentTestRunResult> {
@@ -72,6 +73,7 @@ export default class AgentTestResume extends SfCommand<AgentTestRunResult> {
       results: response,
       jsonEnabled: this.jsonEnabled(),
       outputDir: outputDir ?? flags['output-dir'],
+      verbose: flags.verbose,
     });
 
     return { ...response!, runId, status: 'COMPLETED' };

src/commands/agent/test/run.ts

@@ -17,6 +17,7 @@ import {
   promptForAiEvaluationDefinitionApiName,
   resultFormatFlag,
   testOutputDirFlag,
+  verboseFlag,
 } from '../../../flags.js';
 import { AgentTestCache } from '../../../agentTestCache.js';
 import { TestStages } from '../../../testStages.js';
@@ -65,6 +66,7 @@ export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
     }),
     'result-format': resultFormatFlag(),
     'output-dir': testOutputDirFlag(),
+    verbose: verboseFlag,
   };
 
   private mso: TestStages | undefined;
@@ -109,6 +111,7 @@ export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
         results: detailsResponse,
         jsonEnabled: this.jsonEnabled(),
         outputDir: flags['output-dir'],
+        verbose: flags.verbose,
       });
 
       return { ...detailsResponse!, status: 'COMPLETED', runId: response.runId };

src/flags.ts

@@ -53,6 +53,11 @@ export const testOutputDirFlag = Flags.custom<string>({
   summary: messages.getMessage('flags.output-dir.summary'),
 });
 
+export const verboseFlag = Flags.boolean({
+  summary: messages.getMessage('flags.verbose.summary'),
+  description: messages.getMessage('flags.verbose.description'),
+});
+
 function validateInput(input: string, validate: (input: string) => boolean | string): never | string {
   const result = validate(input);
   if (typeof result === 'string') throw new Error(result);

src/handleTestResults.ts

@@ -69,7 +69,7 @@ export function readableTime(time: number, decimalPlaces = 2): string {
   return `${hours}h ${minutes}m`;
 }
 
-export function humanFormat(results: AgentTestResultsResponse): string {
+export function humanFormat(results: AgentTestResultsResponse, verbose = false): string {
   const ux = new Ux();
 
   const tables: string[] = [];
@@ -128,7 +128,7 @@ export function humanFormat(results: AgentTestResultsResponse): string {
       metricResults.push(...metrics);
     }
     // it's not a real string[], more like just a string  "['IdentifyRecordByName']", so >2 means more than "[]"
-    if (testCase.generatedData?.actionsSequence?.length > 2) {
+    if (verbose && testCase.generatedData?.actionsSequence?.length > 2) {
       tables.push(
         ux.makeTable({
           title: 'Generated Data',
@@ -214,12 +214,14 @@ export async function handleTestResults({
   results,
   jsonEnabled,
   outputDir,
+  verbose = false,
 }: {
   id: string;
   format: 'human' | 'json' | 'junit' | 'tap';
   results: AgentTestResultsResponse | undefined;
   jsonEnabled: boolean;
   outputDir?: string;
+  verbose?: boolean;
 }): Promise<void> {
   if (!results) {
     // do nothing since there are no results to handle
@@ -229,7 +231,7 @@ export async function handleTestResults({
   const ux = new Ux({ jsonEnabled });
 
   if (format === 'human') {
-    const formatted = humanFormat(results);
+    const formatted = humanFormat(results, verbose);
     if (outputDir) {
       const file = `test-result-${id}.txt`;
       await writeFileToDir(outputDir, file, stripVTControlCharacters(formatted));