Skip to content

Commit 3d5261b

Browse files
shetzelshetzel
committed
feat: add generate spec v2 command and tests
1 parent 1eaad00 commit 3d5261b

File tree

14 files changed

+635
-5
lines changed

14 files changed

+635
-5
lines changed

‎LICENSE.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (c) 2024, Salesforce.com, Inc.
1+
Copyright (c) 2025, Salesforce.com, Inc.
22
All rights reserved.
33

44
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

‎command-snapshot.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,29 @@
3535
],
3636
"plugin": "@salesforce/plugin-agent"
3737
},
38+
{
39+
"alias": [],
40+
"command": "agent:generate:spec-v2",
41+
"flagAliases": [],
42+
"flagChars": ["d", "f", "o", "t"],
43+
"flags": [
44+
"api-version",
45+
"company-description",
46+
"company-name",
47+
"company-website",
48+
"file-name",
49+
"flags-dir",
50+
"grounding-context",
51+
"json",
52+
"max-topics",
53+
"output-dir",
54+
"prompt-template",
55+
"role",
56+
"target-org",
57+
"type"
58+
],
59+
"plugin": "@salesforce/plugin-agent"
60+
},
3861
{
3962
"alias": [],
4063
"command": "agent:generate:testset",

‎messages/agent.generate.spec-v2.md‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# summary
2+
3+
Generate an agent spec, which is the list of jobs that the agent performs.
4+
5+
# description
6+
7+
When using Salesforce CLI to create an agent in your org, you can choose to generate a YAML-formatted agent spec file with this command as a first step.
8+
9+
An agent spec is a list of jobs and descriptions that capture what the agent can do. Use flags such as --role and --company-description to provide details about your company and the role that the agent plays in your company; you can also enter the information interactively if you prefer. When you then execute this command, the large language model (LLM) associated with your org uses the information to generate the list of jobs that the agent most likely performs. We recommend that you provide good details for --role, --company-description, etc, so that the LLM can generate the best and most relevant list of jobs and descriptions. Once generated, you can edit the spec file; for example, you can remove jobs that don't apply to your agent.
10+
11+
When your agent spec is ready, you then create the agent in your org by specifying the agent spec file to the --job-spec flag of the "agent create" CLI command.
12+
13+
# flags.type.summary
14+
15+
Type of agent to create.
16+
17+
# flags.role.summary
18+
19+
Role of the agent.
20+
21+
# flags.company-name.summary
22+
23+
Name of your company.
24+
25+
# flags.company-description.summary
26+
27+
Description of your company.
28+
29+
# flags.company-website.summary
30+
31+
Website URL of your company.
32+
33+
# flags.output-dir.summary
34+
35+
Directory where the agent spec file is written; can be an absolute or relative path.
36+
37+
# flags.file-name.summary
38+
39+
Name of the generated agent spec file (yaml).
40+
41+
# flags.max-topics.summary
42+
43+
Maximum number of agent job topics to generate in the spec; default is 10.
44+
45+
# flags.prompt-template.summary
46+
47+
Developer name of a customized prompt template to use instead of the default.
48+
49+
# flags.grounding-context.summary
50+
51+
Context information to be used with the customized prompt template.
52+
53+
# examples
54+
55+
- Create an agent spec for your default org in the default location and use flags to specify the agent's role and your company details:
56+
57+
<%= config.bin %> <%= command.id %> --type customer --role "Assist users in navigating and managing bookings" --company-name "Coral Cloud" --company-description "Resort that manages guests and their reservations and experiences"
58+
59+
- Create an agent spec by being prompted for role and company details interactively; write the generated file to the "specs" directory and use the org with alias "my-org":
60+
61+
<%= config.bin %> <%= command.id %> --output-dir specs --target-org my-org

‎mocks/einstein_ai-evaluations_runs.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"aiEvaluationId": "4KBSM000000003F4AQ",
3+
"status": "NEW"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"status": "IN_PROGRESS",
3+
"startTime": "2024-11-13T15:00:00.000Z"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"status": "IN_PROGRESS",
3+
"startTime": "2024-11-13T15:00:00.000Z"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"status": "COMPLETED",
3+
"startTime": "2024-11-13T15:00:00.000Z"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"status": "COMPLETED",
3+
"startTime": "2024-11-28T12:00:00Z",
4+
"endTime": "2024-11-28T12:05:00Z",
5+
"errorMessage": null,
6+
"testCases": [
7+
{
8+
"status": "COMPLETED",
9+
"number": 1,
10+
"startTime": "2024-11-28T12:00:10Z",
11+
"endTime": "2024-11-28T12:00:20Z",
12+
"generatedData": {
13+
"type": "AGENT",
14+
"actionsSequence": ["Action1", "Action2"],
15+
"outcome": "Success",
16+
"topic": "Mathematics",
17+
"inputTokensCount": 50,
18+
"outputTokensCount": 55
19+
},
20+
"expectationResults": [
21+
{
22+
"name": "topic_sequence_match",
23+
"actualValue": "Result A",
24+
"expectedValue": "Result A",
25+
"score": 1.0,
26+
"result": "Passed",
27+
"metricLabel": "Accuracy",
28+
"metricExplainability": "Measures the correctness of the result.",
29+
"status": "Completed",
30+
"startTime": "2024-11-28T12:00:12Z",
31+
"endTime": "2024-11-28T12:00:13Z",
32+
"errorCode": null,
33+
"errorMessage": null
34+
},
35+
{
36+
"name": "action_sequence_match",
37+
"actualValue": "Result B",
38+
"expectedValue": "Result B",
39+
"score": 0.9,
40+
"result": "Passed",
41+
"metricLabel": "Precision",
42+
"metricExplainability": "Measures the precision of the result.",
43+
"status": "Completed",
44+
"startTime": "2024-11-28T12:00:14Z",
45+
"endTime": "2024-11-28T12:00:15Z",
46+
"errorCode": null,
47+
"errorMessage": null
48+
}
49+
]
50+
},
51+
{
52+
"status": "ERROR",
53+
"number": 2,
54+
"startTime": "2024-11-28T12:00:30Z",
55+
"endTime": "2024-11-28T12:00:40Z",
56+
"generatedData": {
57+
"type": "AGENT",
58+
"actionsSequence": ["Action3", "Action4"],
59+
"outcome": "Failure",
60+
"topic": "Physics",
61+
"inputTokensCount": 60,
62+
"outputTokensCount": 50
63+
},
64+
"expectationResults": [
65+
{
66+
"name": "topic_sequence_match",
67+
"actualValue": "Result C",
68+
"expectedValue": "Result D",
69+
"score": 0.5,
70+
"result": "Failed",
71+
"metricLabel": "Accuracy",
72+
"metricExplainability": "Measures the correctness of the result.",
73+
"status": "Completed",
74+
"startTime": "2024-11-28T12:00:32Z",
75+
"endTime": "2024-11-28T12:00:33Z",
76+
"errorCode": null,
77+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
78+
},
79+
{
80+
"name": "topic_sequence_match",
81+
"actualValue": "Result C",
82+
"expectedValue": "Result D",
83+
"score": 0.5,
84+
"result": "Failed",
85+
"metricLabel": "Accuracy",
86+
"metricExplainability": "Measures the correctness of the result.",
87+
"status": "Completed",
88+
"startTime": "2024-11-28T12:00:32Z",
89+
"endTime": "2024-11-28T12:00:33Z",
90+
"errorCode": null,
91+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
92+
}
93+
]
94+
}
95+
]
96+
}

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@
1818
"ansis": "^3.3.2",
1919
"ink": "^5.0.1",
2020
"ink-text-input": "^6.0.0",
21-
"react": "^18.3.1"
21+
"react": "^18.3.1",
22+
"yaml": "^2.7.0"
2223
},
2324
"devDependencies": {
2425
"@oclif/plugin-command-snapshot": "^5.2.19",

‎schemas/agent-generate-spec__v2.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"$ref": "#/definitions/AgentCreateSpecResult",
4+
"definitions": {
5+
"AgentCreateSpecResult": {
6+
"type": "object",
7+
"additionalProperties": false,
8+
"properties": {
9+
"topics": {
10+
"$ref": "#/definitions/DraftAgentTopics"
11+
},
12+
"agentType": {
13+
"type": "string",
14+
"enum": ["customer", "internal"],
15+
"description": "Internal type is copilots; used by customers' employees. Customer type is agents; used by customers' customers."
16+
},
17+
"role": {
18+
"type": "string"
19+
},
20+
"companyName": {
21+
"type": "string"
22+
},
23+
"companyDescription": {
24+
"type": "string"
25+
},
26+
"companyWebsite": {
27+
"type": "string"
28+
},
29+
"maxNumOfTopics": {
30+
"type": "number",
31+
"description": "The maximum number of topics to create in the spec. Default is 10."
32+
},
33+
"promptTemplateName": {
34+
"type": "string",
35+
"description": "Developer name of the prompt template."
36+
},
37+
"groundingContext": {
38+
"type": "string",
39+
"description": "Context info to be used in customized prompt template"
40+
},
41+
"isSuccess": {
42+
"type": "boolean"
43+
},
44+
"errorMessage": {
45+
"type": "string"
46+
},
47+
"specPath": {
48+
"type": "string"
49+
}
50+
},
51+
"required": ["agentType", "companyDescription", "companyName", "isSuccess", "role", "topics"]
52+
},
53+
"DraftAgentTopics": {
54+
"type": "array",
55+
"items": {
56+
"type": "object",
57+
"properties": {
58+
"name": {
59+
"type": "string"
60+
},
61+
"description": {
62+
"type": "string"
63+
}
64+
},
65+
"required": ["name", "description"],
66+
"additionalProperties": false
67+
},
68+
"minItems": 1,
69+
"maxItems": 1
70+
}
71+
}
72+
}

0 commit comments

Comments
 (0)