Skip to content

Commit 658bc53

Browse files
stainless-app[bot]stainless-app[bot]
committed
feat(api): api update
1 parent 90c79c2 commit 658bc53

File tree

13 files changed

+182
-24
lines changed

13 files changed

+182
-24
lines changed

‎.stats.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
configured_endpoints: 24
2-
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/lemma%2Flemma-b2821b31483edc05c92932e74b3bc0c2bac91c377136645b9df94adc6aa8a9d1.yml
3-
openapi_spec_hash: 14e609ef6b1e22905d6586fb257a83c0
2+
openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/lemma%2Flemma-8cd151afe45f9f40ddf5285bd840af3451896e7b9a53b06093dfbefcdc968bc0.yml
3+
openapi_spec_hash: 21ff168e4da2129dc4cdc2f4e4e91ec1
44
config_hash: 017a31b2ac0856f4d4152a5173e6fa18

‎api.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,12 +91,14 @@ Types:
9191
- <code><a href="./src/resources/evaluators.ts">Evaluator</a></code>
9292
- <code><a href="./src/resources/evaluators.ts">RubricCriterion</a></code>
9393
- <code><a href="./src/resources/evaluators.ts">RubricExample</a></code>
94+
- <code><a href="./src/resources/evaluators.ts">EvaluatorRetrieveResponse</a></code>
95+
- <code><a href="./src/resources/evaluators.ts">EvaluatorUpdateResponse</a></code>
9496
- <code><a href="./src/resources/evaluators.ts">EvaluatorListResponse</a></code>
9597
- <code><a href="./src/resources/evaluators.ts">EvaluatorCreateRubricResponse</a></code>
9698

9799
Methods:
98100

99-
- <code title="get /evaluators/{evaluator_id}">client.evaluators.<a href="./src/resources/evaluators.ts">retrieve</a>(evaluatorID) -> Evaluator</code>
100-
- <code title="put /evaluators/{evaluator_id}">client.evaluators.<a href="./src/resources/evaluators.ts">update</a>(evaluatorID, { ...params }) -> Evaluator</code>
101+
- <code title="get /evaluators/{evaluator_id}">client.evaluators.<a href="./src/resources/evaluators.ts">retrieve</a>(evaluatorID) -> EvaluatorRetrieveResponse</code>
102+
- <code title="put /evaluators/{evaluator_id}">client.evaluators.<a href="./src/resources/evaluators.ts">update</a>(evaluatorID, { ...params }) -> EvaluatorUpdateResponse</code>
101103
- <code title="get /evaluators">client.evaluators.<a href="./src/resources/evaluators.ts">list</a>() -> EvaluatorListResponse</code>
102104
- <code title="post /evaluators/rubric">client.evaluators.<a href="./src/resources/evaluators.ts">createRubric</a>({ ...params }) -> EvaluatorCreateRubricResponse</code>

‎packages/mcp-server/src/tools/evaluators/create-rubric-evaluators.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export const metadata: Metadata = {
1818
export const tool: Tool = {
1919
name: 'create_rubric_evaluators',
2020
description:
21-
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new rubric for evaluation.\n\nCreates a new rubric that defines how to evaluate AI model outputs.\nRubrics provide structured criteria for assessment.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator_create_rubric_response',\n $defs: {\n evaluator_create_rubric_response: {\n type: 'object',\n title: 'Rubric',\n properties: {\n criteria: {\n type: 'array',\n title: 'Criteria',\n items: {\n $ref: '#/$defs/rubric_criterion'\n }\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n examples: {\n type: 'array',\n title: 'Examples',\n items: {\n $ref: '#/$defs/rubric_example'\n }\n },\n name: {\n type: 'string',\n title: 'Name'\n }\n },\n required: [ 'criteria',\n 'description',\n 'examples',\n 'name'\n ]\n },\n rubric_criterion: {\n type: 'object',\n title: 'RubricCriterion',\n properties: {\n id: {\n type: 'string',\n title: 'Id'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n scoreLevels: {\n type: 'array',\n title: 'Scorelevels',\n items: {\n type: 'object',\n title: 'RubricScoreLevel',\n properties: {\n description: {\n type: 'string',\n title: 'Description'\n },\n label: {\n type: 'string',\n title: 'Label'\n },\n score: {\n type: 'integer',\n title: 'Score'\n }\n },\n required: [ 'description',\n 'label',\n 'score'\n ]\n }\n },\n weight: {\n type: 'number',\n title: 'Weight'\n }\n },\n required: [ 'id',\n 'description',\n 'name',\n 'scoreLevels',\n 'weight'\n ]\n },\n rubric_example: {\n type: 'object',\n title: 'RubricExample',\n properties: {\n id: {\n type: 'string',\n title: 'Id'\n },\n criterionId: {\n type: 'string',\n title: 'Criterionid'\n },\n expectedScore: {\n type: 'integer',\n title: 'Expectedscore'\n },\n input: {\n type: 'string',\n title: 'Input'\n },\n reasoning: {\n type: 'string',\n title: 'Reasoning'\n }\n },\n required: [ 'id',\n 'criterionId',\n 'expectedScore',\n 'input',\n 'reasoning'\n ]\n }\n }\n}\n```",
21+
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nCreate a new rubric for evaluation.\n\nCreates a new rubric that defines how to evaluate AI model outputs.\nRubrics provide structured criteria for assessment.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator_create_rubric_response',\n $defs: {\n evaluator_create_rubric_response: {\n type: 'object',\n title: 'EvaluatorEntry',\n description: 'Evaluator entry database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n tenantId: {\n type: 'string',\n title: 'Tenantid'\n },\n id: {\n type: 'string',\n title: 'Id'\n },\n createdAt: {\n type: 'string',\n title: 'Createdat',\n description: 'Timestamp when the record was created',\n format: 'date-time'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n },\n updatedAt: {\n type: 'string',\n title: 'Updatedat',\n description: 'Timestamp when the record was last updated',\n format: 'date-time'\n }\n },\n required: [ 'config',\n 'name',\n 'tenantId'\n ]\n }\n }\n}\n```",
2222
inputSchema: {
2323
type: 'object',
2424
properties: {
@@ -111,20 +111,24 @@ export const tool: Tool = {
111111
type: 'string',
112112
title: 'Criterionid',
113113
},
114-
expectedScore: {
115-
type: 'integer',
116-
title: 'Expectedscore',
117-
},
118114
input: {
119115
type: 'string',
120116
title: 'Input',
121117
},
118+
output: {
119+
type: 'string',
120+
title: 'Output',
121+
},
122122
reasoning: {
123123
type: 'string',
124124
title: 'Reasoning',
125125
},
126+
score: {
127+
type: 'integer',
128+
title: 'Score',
129+
},
126130
},
127-
required: ['id', 'criterionId', 'expectedScore', 'input', 'reasoning'],
131+
required: ['id', 'criterionId', 'input', 'output', 'reasoning', 'score'],
128132
},
129133
},
130134
},

‎packages/mcp-server/src/tools/evaluators/list-evaluators.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export const metadata: Metadata = {
1818
export const tool: Tool = {
1919
name: 'list_evaluators',
2020
description:
21-
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nList all evaluators for the current tenant.\n\nRetrieves all evaluators that belong to the current user's tenant.\nEvaluators are used to assess the quality of AI model outputs.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator_list_response',\n $defs: {\n evaluator_list_response: {\n type: 'array',\n title: 'Response List Evaluators Evaluators Get',\n items: {\n $ref: '#/$defs/evaluator'\n }\n },\n evaluator: {\n type: 'object',\n title: 'Evaluator',\n description: 'Evaluator database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n }\n },\n required: [ 'config',\n 'name'\n ]\n }\n }\n}\n```",
21+
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nList all evaluators for the current tenant.\n\nRetrieves all evaluators that belong to the current user's tenant.\nEvaluators are used to assess the quality of AI model outputs.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator_list_response',\n $defs: {\n evaluator_list_response: {\n type: 'array',\n title: 'Response List Evaluators Evaluators Get',\n items: {\n type: 'object',\n title: 'EvaluatorEntry',\n description: 'Evaluator entry database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n tenantId: {\n type: 'string',\n title: 'Tenantid'\n },\n id: {\n type: 'string',\n title: 'Id'\n },\n createdAt: {\n type: 'string',\n title: 'Createdat',\n description: 'Timestamp when the record was created',\n format: 'date-time'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n },\n updatedAt: {\n type: 'string',\n title: 'Updatedat',\n description: 'Timestamp when the record was last updated',\n format: 'date-time'\n }\n },\n required: [ 'config',\n 'name',\n 'tenantId'\n ]\n }\n }\n }\n}\n```",
2222
inputSchema: {
2323
type: 'object',
2424
properties: {

‎packages/mcp-server/src/tools/evaluators/retrieve-evaluators.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export const metadata: Metadata = {
1818
export const tool: Tool = {
1919
name: 'retrieve_evaluators',
2020
description:
21-
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nGet a specific evaluator by ID.\n\nRetrieves detailed information about a specific evaluator.\nThe evaluator must belong to the current user's tenant.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator',\n $defs: {\n evaluator: {\n type: 'object',\n title: 'Evaluator',\n description: 'Evaluator database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n }\n },\n required: [ 'config',\n 'name'\n ]\n }\n }\n}\n```",
21+
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nGet a specific evaluator by ID.\n\nRetrieves detailed information about a specific evaluator.\nThe evaluator must belong to the current user's tenant.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator_retrieve_response',\n $defs: {\n evaluator_retrieve_response: {\n type: 'object',\n title: 'EvaluatorEntry',\n description: 'Evaluator entry database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n tenantId: {\n type: 'string',\n title: 'Tenantid'\n },\n id: {\n type: 'string',\n title: 'Id'\n },\n createdAt: {\n type: 'string',\n title: 'Createdat',\n description: 'Timestamp when the record was created',\n format: 'date-time'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n },\n updatedAt: {\n type: 'string',\n title: 'Updatedat',\n description: 'Timestamp when the record was last updated',\n format: 'date-time'\n }\n },\n required: [ 'config',\n 'name',\n 'tenantId'\n ]\n }\n }\n}\n```",
2222
inputSchema: {
2323
type: 'object',
2424
properties: {

‎packages/mcp-server/src/tools/evaluators/update-evaluators.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export const metadata: Metadata = {
1818
export const tool: Tool = {
1919
name: 'update_evaluators',
2020
description:
21-
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nUpdate an existing evaluator.\n\nUpdates the configuration and settings of an existing evaluator.\nThe evaluator must belong to the current user's tenant.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator',\n $defs: {\n evaluator: {\n type: 'object',\n title: 'Evaluator',\n description: 'Evaluator database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n }\n },\n required: [ 'config',\n 'name'\n ]\n }\n }\n}\n```",
21+
"When using this tool, always use the `jq_filter` parameter to reduce the response size and improve performance.\n\nOnly omit if you're sure you don't need the data.\n\nUpdate an existing evaluator.\n\nUpdates the configuration and settings of an existing evaluator.\nThe evaluator must belong to the current user's tenant.\n\n# Response Schema\n```json\n{\n $ref: '#/$defs/evaluator_update_response',\n $defs: {\n evaluator_update_response: {\n type: 'object',\n title: 'EvaluatorEntry',\n description: 'Evaluator entry database model.',\n properties: {\n config: {\n type: 'object',\n title: 'Config',\n description: 'Variables used in the prompt template',\n additionalProperties: true\n },\n name: {\n type: 'string',\n title: 'Name'\n },\n tenantId: {\n type: 'string',\n title: 'Tenantid'\n },\n id: {\n type: 'string',\n title: 'Id'\n },\n createdAt: {\n type: 'string',\n title: 'Createdat',\n description: 'Timestamp when the record was created',\n format: 'date-time'\n },\n description: {\n type: 'string',\n title: 'Description'\n },\n type: {\n type: 'string',\n title: 'EvaluatorType',\n enum: [ 'RUBRIC',\n 'WEBHOOK',\n 'CUSTOM'\n ]\n },\n updatedAt: {\n type: 'string',\n title: 'Updatedat',\n description: 'Timestamp when the record was last updated',\n format: 'date-time'\n }\n },\n required: [ 'config',\n 'name',\n 'tenantId'\n ]\n }\n }\n}\n```",
2222
inputSchema: {
2323
type: 'object',
2424
properties: {

‎packages/mcp-server/src/tools/playground/run-playground.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ export const tool: Tool = {
3535
title: 'Evaluator Id',
3636
description: 'Optional ID of the evaluator to use',
3737
},
38+
input_variables: {
39+
type: 'object',
40+
title: 'Input Variables',
41+
description: 'Input variables for Jinja template substitution',
42+
additionalProperties: true,
43+
},
3844
},
3945
required: ['model', 'prompt_id'],
4046
},

‎src/client.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,9 @@ import {
2121
EvaluatorCreateRubricParams,
2222
EvaluatorCreateRubricResponse,
2323
EvaluatorListResponse,
24+
EvaluatorRetrieveResponse,
2425
EvaluatorUpdateParams,
26+
EvaluatorUpdateResponse,
2527
Evaluators,
2628
RubricCriterion,
2729
RubricExample,
@@ -812,6 +814,8 @@ export declare namespace Lemma {
812814
type Evaluator as Evaluator,
813815
type RubricCriterion as RubricCriterion,
814816
type RubricExample as RubricExample,
817+
type EvaluatorRetrieveResponse as EvaluatorRetrieveResponse,
818+
type EvaluatorUpdateResponse as EvaluatorUpdateResponse,
815819
type EvaluatorListResponse as EvaluatorListResponse,
816820
type EvaluatorCreateRubricResponse as EvaluatorCreateRubricResponse,
817821
type EvaluatorUpdateParams as EvaluatorUpdateParams,

0 commit comments

Comments
 (0)