feat(api): Add evalapi to sdk

.stats.yml

@@ -1,4 +1,4 @@
-configured_endpoints: 82
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-4bce8217a697c729ac98046d4caf2c9e826b54c427fb0ab4f98e549a2e0ce31c.yml
-openapi_spec_hash: 7996d2c34cc44fe2ce9ffe93c0ab774e
-config_hash: bcd2cacdcb9fae9938f273cd167f613c
+configured_endpoints: 97
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-472fe3036ea745365257fe870c0330917fb3153705c2826f49873cd631319b0a.yml
+openapi_spec_hash: ea86343b5e9858a74e85da8ab2c532f6
+config_hash: ef19d36c307306f14f2e1cd5c834a151

api.md

@@ -234,6 +234,22 @@ Methods:
 
 - <code title="get /fine_tuning/jobs/{fine_tuning_job_id}/checkpoints">client.fineTuning.jobs.checkpoints.<a href="./src/resources/fine-tuning/jobs/checkpoints.ts">list</a>(fineTuningJobId, { ...params }) -> FineTuningJobCheckpointsPage</code>
 
+## Checkpoints
+
+### Permissions
+
+Types:
+
+- <code><a href="./src/resources/fine-tuning/checkpoints/permissions.ts">PermissionCreateResponse</a></code>
+- <code><a href="./src/resources/fine-tuning/checkpoints/permissions.ts">PermissionRetrieveResponse</a></code>
+- <code><a href="./src/resources/fine-tuning/checkpoints/permissions.ts">PermissionDeleteResponse</a></code>
+
+Methods:
+
+- <code title="post /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions">client.fineTuning.checkpoints.permissions.<a href="./src/resources/fine-tuning/checkpoints/permissions.ts">create</a>(fineTunedModelCheckpoint, { ...params }) -> PermissionCreateResponsesPage</code>
+- <code title="get /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions">client.fineTuning.checkpoints.permissions.<a href="./src/resources/fine-tuning/checkpoints/permissions.ts">retrieve</a>(fineTunedModelCheckpoint, { ...params }) -> PermissionRetrieveResponse</code>
+- <code title="delete /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions">client.fineTuning.checkpoints.permissions.<a href="./src/resources/fine-tuning/checkpoints/permissions.ts">del</a>(fineTunedModelCheckpoint) -> PermissionDeleteResponse</code>
+
 # VectorStores
 
 Types:
@@ -627,3 +643,59 @@ Types:
 Methods:
 
 - <code title="get /responses/{response_id}/input_items">client.responses.inputItems.<a href="./src/resources/responses/input-items.ts">list</a>(responseId, { ...params }) -> ResponseItemsPage</code>
+
+# Evals
+
+Types:
+
+- <code><a href="./src/resources/evals/evals.ts">EvalCustomDataSourceConfig</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalLabelModelGrader</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalStoredCompletionsDataSourceConfig</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalStringCheckGrader</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalTextSimilarityGrader</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalCreateResponse</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalRetrieveResponse</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalUpdateResponse</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalListResponse</a></code>
+- <code><a href="./src/resources/evals/evals.ts">EvalDeleteResponse</a></code>
+
+Methods:
+
+- <code title="post /evals">client.evals.<a href="./src/resources/evals/evals.ts">create</a>({ ...params }) -> EvalCreateResponse</code>
+- <code title="get /evals/{eval_id}">client.evals.<a href="./src/resources/evals/evals.ts">retrieve</a>(evalId) -> EvalRetrieveResponse</code>
+- <code title="post /evals/{eval_id}">client.evals.<a href="./src/resources/evals/evals.ts">update</a>(evalId, { ...params }) -> EvalUpdateResponse</code>
+- <code title="get /evals">client.evals.<a href="./src/resources/evals/evals.ts">list</a>({ ...params }) -> EvalListResponsesPage</code>
+- <code title="delete /evals/{eval_id}">client.evals.<a href="./src/resources/evals/evals.ts">del</a>(evalId) -> EvalDeleteResponse</code>
+
+## Runs
+
+Types:
+
+- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalCompletionsRunDataSource</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">CreateEvalJSONLRunDataSource</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">EvalAPIError</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">RunCreateResponse</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">RunRetrieveResponse</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">RunListResponse</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">RunDeleteResponse</a></code>
+- <code><a href="./src/resources/evals/runs/runs.ts">RunCancelResponse</a></code>
+
+Methods:
+
+- <code title="post /evals/{eval_id}/runs">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">create</a>(evalId, { ...params }) -> RunCreateResponse</code>
+- <code title="get /evals/{eval_id}/runs/{run_id}">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">retrieve</a>(evalId, runId) -> RunRetrieveResponse</code>
+- <code title="get /evals/{eval_id}/runs">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">list</a>(evalId, { ...params }) -> RunListResponsesPage</code>
+- <code title="delete /evals/{eval_id}/runs/{run_id}">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">del</a>(evalId, runId) -> RunDeleteResponse</code>
+- <code title="post /evals/{eval_id}/runs/{run_id}">client.evals.runs.<a href="./src/resources/evals/runs/runs.ts">cancel</a>(evalId, runId) -> RunCancelResponse</code>
+
+### OutputItems
+
+Types:
+
+- <code><a href="./src/resources/evals/runs/output-items.ts">OutputItemRetrieveResponse</a></code>
+- <code><a href="./src/resources/evals/runs/output-items.ts">OutputItemListResponse</a></code>
+
+Methods:
+
+- <code title="get /evals/{eval_id}/runs/{run_id}/output_items/{output_item_id}">client.evals.runs.outputItems.<a href="./src/resources/evals/runs/output-items.ts">retrieve</a>(evalId, runId, outputItemId) -> OutputItemRetrieveResponse</code>
+- <code title="get /evals/{eval_id}/runs/{run_id}/output_items">client.evals.runs.outputItems.<a href="./src/resources/evals/runs/output-items.ts">list</a>(evalId, runId, { ...params }) -> OutputItemListResponsesPage</code>

src/index.ts

@@ -66,6 +66,23 @@ import {
 import { Audio, AudioModel, AudioResponseFormat } from './resources/audio/audio';
 import { Beta } from './resources/beta/beta';
 import { Chat } from './resources/chat/chat';
+import {
+  EvalCreateParams,
+  EvalCreateResponse,
+  EvalCustomDataSourceConfig,
+  EvalDeleteResponse,
+  EvalLabelModelGrader,
+  EvalListParams,
+  EvalListResponse,
+  EvalListResponsesPage,
+  EvalRetrieveResponse,
+  EvalStoredCompletionsDataSourceConfig,
+  EvalStringCheckGrader,
+  EvalTextSimilarityGrader,
+  EvalUpdateParams,
+  EvalUpdateResponse,
+  Evals,
+} from './resources/evals/evals';
 import { FineTuning } from './resources/fine-tuning/fine-tuning';
 import { Responses } from './resources/responses/responses';
 import {
@@ -293,6 +310,7 @@ export class OpenAI extends Core.APIClient {
   batches: API.Batches = new API.Batches(this);
   uploads: API.Uploads = new API.Uploads(this);
   responses: API.Responses = new API.Responses(this);
+  evals: API.Evals = new API.Evals(this);
 
   protected override defaultQuery(): Core.DefaultQuery | undefined {
     return this._options.defaultQuery;
@@ -356,6 +374,8 @@ OpenAI.Batches = Batches;
 OpenAI.BatchesPage = BatchesPage;
 OpenAI.Uploads = UploadsAPIUploads;
 OpenAI.Responses = Responses;
+OpenAI.Evals = Evals;
+OpenAI.EvalListResponsesPage = EvalListResponsesPage;
 export declare namespace OpenAI {
   export type RequestOptions = Core.RequestOptions;
 
@@ -508,6 +528,24 @@ export declare namespace OpenAI {
 
   export { Responses as Responses };
 
+  export {
+    Evals as Evals,
+    type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig,
+    type EvalLabelModelGrader as EvalLabelModelGrader,
+    type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig,
+    type EvalStringCheckGrader as EvalStringCheckGrader,
+    type EvalTextSimilarityGrader as EvalTextSimilarityGrader,
+    type EvalCreateResponse as EvalCreateResponse,
+    type EvalRetrieveResponse as EvalRetrieveResponse,
+    type EvalUpdateResponse as EvalUpdateResponse,
+    type EvalListResponse as EvalListResponse,
+    type EvalDeleteResponse as EvalDeleteResponse,
+    EvalListResponsesPage as EvalListResponsesPage,
+    type EvalCreateParams as EvalCreateParams,
+    type EvalUpdateParams as EvalUpdateParams,
+    type EvalListParams as EvalListParams,
+  };
+
   export type AllModels = API.AllModels;
   export type ChatModel = API.ChatModel;
   export type ComparisonFilter = API.ComparisonFilter;

src/resources/evals.ts

@@ -0,0 +1,3 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+export * from './evals/index';