Skip to content

Commit f742205

Browse files
WillieRuemmeleWillieRuemmele
authored
Merge pull request #55 from salesforcecli/sh/agent-generate-spec-v2
2 parents 1eaad00 + 7f65066 commit f742205

File tree

14 files changed

+757
-76
lines changed

14 files changed

+757
-76
lines changed

‎LICENSE.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (c) 2024, Salesforce.com, Inc.
1+
Copyright (c) 2025, Salesforce.com, Inc.
22
All rights reserved.
33

44
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:

‎command-snapshot.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,29 @@
3535
],
3636
"plugin": "@salesforce/plugin-agent"
3737
},
38+
{
39+
"alias": [],
40+
"command": "agent:generate:spec-v2",
41+
"flagAliases": [],
42+
"flagChars": ["o", "t"],
43+
"flags": [
44+
"api-version",
45+
"company-description",
46+
"company-name",
47+
"company-website",
48+
"flags-dir",
49+
"grounding-context",
50+
"json",
51+
"max-topics",
52+
"output-file",
53+
"prompt-template",
54+
"role",
55+
"spec",
56+
"target-org",
57+
"type"
58+
],
59+
"plugin": "@salesforce/plugin-agent"
60+
},
3861
{
3962
"alias": [],
4063
"command": "agent:generate:testset",

‎messages/agent.generate.spec-v2.md‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# summary
2+
3+
Generate an agent spec, which is the list of jobs that the agent performs.
4+
5+
# description
6+
7+
When using Salesforce CLI to create an agent in your org, you can choose to generate a YAML-formatted agent spec file with this command as a first step.
8+
9+
An agent spec is a list of jobs and descriptions that capture what the agent can do. Use flags such as --role and --company-description to provide details about your company and the role that the agent plays in your company; you can also enter the information interactively if you prefer. When you then execute this command, the large language model (LLM) associated with your org uses the information to generate the list of jobs that the agent most likely performs. We recommend that you provide good details for --role, --company-description, etc, so that the LLM can generate the best and most relevant list of jobs and descriptions. Once generated, you can edit the spec file; for example, you can remove jobs that don't apply to your agent.
10+
11+
When your agent spec is ready, you then create the agent in your org by specifying the agent spec file to the --job-spec flag of the "agent create" CLI command.
12+
13+
# flags.type.summary
14+
15+
Type of agent to create.
16+
17+
# flags.role.summary
18+
19+
Role of the agent.
20+
21+
# flags.company-name.summary
22+
23+
Name of your company.
24+
25+
# flags.company-description.summary
26+
27+
Description of your company.
28+
29+
# flags.company-website.summary
30+
31+
Website URL of your company.
32+
33+
# flags.output-file.summary
34+
35+
Path for the generated agent spec file (yaml); can be an absolute or relative path.
36+
37+
# flags.max-topics.summary
38+
39+
Maximum number of agent job topics to generate in the spec; default is 10.
40+
41+
# flags.prompt-template.summary
42+
43+
Developer name of a customized prompt template to use instead of the default.
44+
45+
# flags.grounding-context.summary
46+
47+
Context information to be used with the customized prompt template.
48+
49+
# flags.spec.summary
50+
51+
Spec file (yaml) to use as input to the command.
52+
53+
# examples
54+
55+
- Create an agent spec for your default org in the default location and use flags to specify the agent's role and your company details:
56+
57+
<%= config.bin %> <%= command.id %> --type customer --role "Assist users in navigating and managing bookings" --company-name "Coral Cloud" --company-description "Resort that manages guests and their reservations and experiences"
58+
59+
- Create an agent spec by being prompted for role and company details interactively; write the generated file to the "specs" directory and use the org with alias "my-org":
60+
61+
<%= config.bin %> <%= command.id %> --output-dir specs --target-org my-org
62+
63+
# error.invalidAgentType
64+
65+
agentType must be either "customer" or "internal". Found: [%s]
66+
67+
# error.invalidMaxTopics
68+
69+
maxNumOfTopics must be a number greater than 0. Found: [%s]
70+
71+
# error.missingRequiredFlags
72+
73+
Missing required flags: %s

‎mocks/einstein_ai-evaluations_runs.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"aiEvaluationId": "4KBSM000000003F4AQ",
3+
"status": "NEW"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/1.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"status": "IN_PROGRESS",
3+
"startTime": "2024-11-13T15:00:00.000Z"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/2.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"status": "IN_PROGRESS",
3+
"startTime": "2024-11-13T15:00:00.000Z"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ/3.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"status": "COMPLETED",
3+
"startTime": "2024-11-13T15:00:00.000Z"
4+
}

‎mocks/einstein_ai-evaluations_runs_4KBSM000000003F4AQ_details.json‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"status": "COMPLETED",
3+
"startTime": "2024-11-28T12:00:00Z",
4+
"endTime": "2024-11-28T12:05:00Z",
5+
"errorMessage": null,
6+
"testCases": [
7+
{
8+
"status": "COMPLETED",
9+
"number": 1,
10+
"startTime": "2024-11-28T12:00:10Z",
11+
"endTime": "2024-11-28T12:00:20Z",
12+
"generatedData": {
13+
"type": "AGENT",
14+
"actionsSequence": ["Action1", "Action2"],
15+
"outcome": "Success",
16+
"topic": "Mathematics",
17+
"inputTokensCount": 50,
18+
"outputTokensCount": 55
19+
},
20+
"expectationResults": [
21+
{
22+
"name": "topic_sequence_match",
23+
"actualValue": "Result A",
24+
"expectedValue": "Result A",
25+
"score": 1.0,
26+
"result": "Passed",
27+
"metricLabel": "Accuracy",
28+
"metricExplainability": "Measures the correctness of the result.",
29+
"status": "Completed",
30+
"startTime": "2024-11-28T12:00:12Z",
31+
"endTime": "2024-11-28T12:00:13Z",
32+
"errorCode": null,
33+
"errorMessage": null
34+
},
35+
{
36+
"name": "action_sequence_match",
37+
"actualValue": "Result B",
38+
"expectedValue": "Result B",
39+
"score": 0.9,
40+
"result": "Passed",
41+
"metricLabel": "Precision",
42+
"metricExplainability": "Measures the precision of the result.",
43+
"status": "Completed",
44+
"startTime": "2024-11-28T12:00:14Z",
45+
"endTime": "2024-11-28T12:00:15Z",
46+
"errorCode": null,
47+
"errorMessage": null
48+
}
49+
]
50+
},
51+
{
52+
"status": "ERROR",
53+
"number": 2,
54+
"startTime": "2024-11-28T12:00:30Z",
55+
"endTime": "2024-11-28T12:00:40Z",
56+
"generatedData": {
57+
"type": "AGENT",
58+
"actionsSequence": ["Action3", "Action4"],
59+
"outcome": "Failure",
60+
"topic": "Physics",
61+
"inputTokensCount": 60,
62+
"outputTokensCount": 50
63+
},
64+
"expectationResults": [
65+
{
66+
"name": "topic_sequence_match",
67+
"actualValue": "Result C",
68+
"expectedValue": "Result D",
69+
"score": 0.5,
70+
"result": "Failed",
71+
"metricLabel": "Accuracy",
72+
"metricExplainability": "Measures the correctness of the result.",
73+
"status": "Completed",
74+
"startTime": "2024-11-28T12:00:32Z",
75+
"endTime": "2024-11-28T12:00:33Z",
76+
"errorCode": null,
77+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
78+
},
79+
{
80+
"name": "topic_sequence_match",
81+
"actualValue": "Result C",
82+
"expectedValue": "Result D",
83+
"score": 0.5,
84+
"result": "Failed",
85+
"metricLabel": "Accuracy",
86+
"metricExplainability": "Measures the correctness of the result.",
87+
"status": "Completed",
88+
"startTime": "2024-11-28T12:00:32Z",
89+
"endTime": "2024-11-28T12:00:33Z",
90+
"errorCode": null,
91+
"errorMessage": "Expected \"Result D\" but got \"Result C\"."
92+
}
93+
]
94+
}
95+
]
96+
}

‎package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,15 @@
1111
"@inquirer/select": "^4.0.1",
1212
"@oclif/core": "^4",
1313
"@oclif/multi-stage-output": "^0.7.12",
14-
"@salesforce/agents": "^0.5.2",
14+
"@salesforce/agents": "^0.6.0",
1515
"@salesforce/core": "^8.8.0",
1616
"@salesforce/kit": "^3.2.1",
1717
"@salesforce/sf-plugins-core": "^12.1.0",
1818
"ansis": "^3.3.2",
1919
"ink": "^5.0.1",
2020
"ink-text-input": "^6.0.0",
21-
"react": "^18.3.1"
21+
"react": "^18.3.1",
22+
"yaml": "^2.7.0"
2223
},
2324
"devDependencies": {
2425
"@oclif/plugin-command-snapshot": "^5.2.19",

‎schemas/agent-generate-spec__v2.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
"$schema": "http://json-schema.org/draft-07/schema#",
3+
"$ref": "#/definitions/AgentCreateSpecResult",
4+
"definitions": {
5+
"AgentCreateSpecResult": {
6+
"type": "object",
7+
"additionalProperties": false,
8+
"properties": {
9+
"topics": {
10+
"$ref": "#/definitions/DraftAgentTopics"
11+
},
12+
"agentType": {
13+
"type": "string",
14+
"enum": ["customer", "internal"],
15+
"description": "Internal type is copilots; used by customers' employees. Customer type is agents; used by customers' customers."
16+
},
17+
"role": {
18+
"type": "string"
19+
},
20+
"companyName": {
21+
"type": "string"
22+
},
23+
"companyDescription": {
24+
"type": "string"
25+
},
26+
"companyWebsite": {
27+
"type": "string"
28+
},
29+
"maxNumOfTopics": {
30+
"type": "number",
31+
"description": "The maximum number of topics to create in the spec. Default is 10."
32+
},
33+
"promptTemplateName": {
34+
"type": "string",
35+
"description": "Developer name of the prompt template."
36+
},
37+
"groundingContext": {
38+
"type": "string",
39+
"description": "Context info to be used in customized prompt template"
40+
},
41+
"isSuccess": {
42+
"type": "boolean"
43+
},
44+
"errorMessage": {
45+
"type": "string"
46+
},
47+
"specPath": {
48+
"type": "string"
49+
}
50+
},
51+
"required": ["agentType", "companyDescription", "companyName", "isSuccess", "role", "topics"]
52+
},
53+
"DraftAgentTopics": {
54+
"type": "array",
55+
"items": {
56+
"type": "object",
57+
"properties": {
58+
"name": {
59+
"type": "string"
60+
},
61+
"description": {
62+
"type": "string"
63+
}
64+
},
65+
"required": ["name", "description"],
66+
"additionalProperties": false
67+
},
68+
"minItems": 1,
69+
"maxItems": 1
70+
}
71+
}
72+
}

0 commit comments

Comments
 (0)