feat: use spec for tests

Author: mdonnalley

command-snapshot.json

@@ -74,15 +74,7 @@
   },
   {
     "alias": [],
-    "command": "agent:generate:test-cases",
-    "flagAliases": [],
-    "flagChars": [],
-    "flags": ["flags-dir"],
-    "plugin": "@salesforce/plugin-agent"
-  },
-  {
-    "alias": [],
-    "command": "agent:generate:test-definition",
+    "command": "agent:generate:test-spec",
     "flagAliases": [],
     "flagChars": [],
     "flags": ["flags-dir"],
@@ -104,6 +96,14 @@
     "flags": ["api-version", "flags-dir", "job-id", "json", "target-org", "use-most-recent"],
     "plugin": "@salesforce/plugin-agent"
   },
+  {
+    "alias": [],
+    "command": "agent:test:create",
+    "flagAliases": [],
+    "flagChars": ["o", "p", "s"],
+    "flags": ["api-version", "flags-dir", "json", "no-prompt", "preview", "spec", "target-org"],
+    "plugin": "@salesforce/plugin-agent"
+  },
   {
     "alias": [],
     "command": "agent:test:list",

messages/agent.generate.test-cases.md

@@ -0,0 +1,11 @@
+# summary
+
+Interactively generate a specification file for a AI evaluation test.
+
+# description
+
+This command will prompt you for the necessary information to create a new spec file (in yaml format). You can then create a new AI evaluation using "sf agent test create --spec <spec-file>".
+
+# examples
+
+- <%= config.bin %> <%= command.id %>

messages/agent.generate.test-definition.md

@@ -0,0 +1,27 @@
+# summary
+
+Summary of a command.
+
+# description
+
+More information about a command. Don't repeat the summary.
+
+# flags.spec.summary
+
+Description of a flag.
+
+# flags.spec.description
+
+More information about a flag. Don't repeat the summary.
+
+# flags.preview.summary
+
+Preview the test metadata without deploying to your org.
+
+# flags.no-prompt.summary
+
+Don't prompt for confirmation when overwriting an existing test.
+
+# examples
+
+- <%= config.bin %> <%= command.id %>

messages/agent.generate.test-spec.md

@@ -13,6 +13,7 @@
     "@salesforce/core": "^8.8.0",
     "@salesforce/kit": "^3.2.1",
     "@salesforce/sf-plugins-core": "^12.1.0",
+    "@salesforce/source-deploy-retrieve": "^12.14.0",
     "ansis": "^3.3.2",
     "fast-xml-parser": "^4.5.1",
     "ink": "^5.0.1",
@@ -77,6 +78,9 @@
           },
           "generate": {
             "external": true
+          },
+          "create": {
+            "external": true
           }
         }
       }

messages/agent.test.create.md

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentTestCreateResult",
+  "definitions": {
+    "AgentTestCreateResult": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "contents": {
+          "type": "string"
+        }
+      },
+      "required": ["path", "contents"],
+      "additionalProperties": false
+    }
+  }
+}