Merge pull request #67 from salesforcecli/mdonnalley/gen-test-spec

feat(W-17671338): use spec file for tests

Author: WillieRuemmele

command-snapshot.json

@@ -74,15 +74,7 @@
   },
   {
     "alias": [],
-    "command": "agent:generate:test-cases",
-    "flagAliases": [],
-    "flagChars": [],
-    "flags": ["flags-dir"],
-    "plugin": "@salesforce/plugin-agent"
-  },
-  {
-    "alias": [],
-    "command": "agent:generate:test-definition",
+    "command": "agent:generate:test-spec",
     "flagAliases": [],
     "flagChars": [],
     "flags": ["flags-dir"],
@@ -104,6 +96,14 @@
     "flags": ["api-version", "flags-dir", "job-id", "json", "target-org", "use-most-recent"],
     "plugin": "@salesforce/plugin-agent"
   },
+  {
+    "alias": [],
+    "command": "agent:test:create",
+    "flagAliases": [],
+    "flagChars": ["o", "p", "s"],
+    "flags": ["api-version", "flags-dir", "json", "no-prompt", "preview", "spec", "target-org"],
+    "plugin": "@salesforce/plugin-agent"
+  },
   {
     "alias": [],
     "command": "agent:test:list",

messages/agent.generate.test-cases.md

@@ -0,0 +1,11 @@
+# summary
+
+Interactively generate a specification file for a AI evaluation test.
+
+# description
+
+This command will prompt you for the necessary information to create a new spec file (in yaml format). You can then create a new AI evaluation using "sf agent test create --spec <spec-file>".
+
+# examples
+
+- <%= config.bin %> <%= command.id %>

messages/agent.generate.test-definition.md

@@ -0,0 +1,27 @@
+# summary
+
+Summary of a command.
+
+# description
+
+More information about a command. Don't repeat the summary.
+
+# flags.spec.summary
+
+Description of a flag.
+
+# flags.spec.description
+
+More information about a flag. Don't repeat the summary.
+
+# flags.preview.summary
+
+Preview the test metadata without deploying to your org.
+
+# flags.no-prompt.summary
+
+Don't prompt for confirmation when overwriting an existing test.
+
+# examples
+
+- <%= config.bin %> <%= command.id %>

messages/agent.generate.test-spec.md

@@ -9,10 +9,11 @@
     "@inquirer/prompts": "^7.2.0",
     "@oclif/core": "^4",
     "@oclif/multi-stage-output": "^0.7.12",
-    "@salesforce/agents": "^0.8.0",
+    "@salesforce/agents": "^0.9.0",
     "@salesforce/core": "^8.8.0",
     "@salesforce/kit": "^3.2.1",
     "@salesforce/sf-plugins-core": "^12.1.0",
+    "@salesforce/source-deploy-retrieve": "^12.14.0",
     "ansis": "^3.3.2",
     "fast-xml-parser": "^4.5.1",
     "ink": "^5.0.1",
@@ -77,6 +78,9 @@
           },
           "generate": {
             "external": true
+          },
+          "create": {
+            "external": true
           }
         }
       }

messages/agent.test.create.md

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentTestCreateResult",
+  "definitions": {
+    "AgentTestCreateResult": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "contents": {
+          "type": "string"
+        }
+      },
+      "required": ["path", "contents"],
+      "additionalProperties": false
+    }
+  }
+}

package.json

@@ -45,30 +45,33 @@
     },
     "TestStatus": {
       "type": "string",
-      "enum": ["New", "InProgress", "Completed", "Error"]
+      "enum": ["NEW", "IN_PROGRESS", "COMPLETED", "ERROR", "TERMINATED"]
     },
     "TestCaseResult": {
       "type": "object",
       "properties": {
         "status": {
           "$ref": "#/definitions/TestStatus"
         },
-        "utterance": {
-          "type": "string"
-        },
         "startTime": {
           "type": "string"
         },
         "endTime": {
           "type": "string"
         },
+        "inputs": {
+          "type": "object",
+          "properties": {
+            "utterance": {
+              "type": "string"
+            }
+          },
+          "required": ["utterance"],
+          "additionalProperties": false
+        },
         "generatedData": {
           "type": "object",
           "properties": {
-            "type": {
-              "type": "string",
-              "const": "AGENT"
-            },
             "actionsSequence": {
               "type": "array",
               "items": {
@@ -80,18 +83,12 @@
             },
             "topic": {
               "type": "string"
-            },
-            "inputTokensCount": {
-              "type": "string"
-            },
-            "outputTokensCount": {
-              "type": "string"
             }
           },
-          "required": ["type", "actionsSequence", "outcome", "topic", "inputTokensCount", "outputTokensCount"],
+          "required": ["actionsSequence", "outcome", "topic"],
           "additionalProperties": false
         },
-        "expectationResults": {
+        "testResults": {
           "type": "array",
           "items": {
             "type": "object",
@@ -150,7 +147,7 @@
           }
         }
       },
-      "required": ["status", "utterance", "startTime", "generatedData", "expectationResults"],
+      "required": ["status", "startTime", "inputs", "generatedData", "testResults"],
       "additionalProperties": false
     }
   }