feat(api): api update

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 24
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/lemma%2Flemma-b2821b31483edc05c92932e74b3bc0c2bac91c377136645b9df94adc6aa8a9d1.yml
-openapi_spec_hash: 14e609ef6b1e22905d6586fb257a83c0
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/lemma%2Flemma-8cd151afe45f9f40ddf5285bd840af3451896e7b9a53b06093dfbefcdc968bc0.yml
+openapi_spec_hash: 21ff168e4da2129dc4cdc2f4e4e91ec1
 config_hash: 017a31b2ac0856f4d4152a5173e6fa18

api.md

@@ -95,14 +95,16 @@ from lemma.types import (
     Evaluator,
     RubricCriterion,
     RubricExample,
+    EvaluatorRetrieveResponse,
+    EvaluatorUpdateResponse,
     EvaluatorListResponse,
     EvaluatorCreateRubricResponse,
 )
 ```
 
 Methods:
 
-- <code title="get /evaluators/{evaluator_id}">client.evaluators.<a href="./src/lemma/resources/evaluators.py">retrieve</a>(evaluator_id) -> <a href="./src/lemma/types/evaluator.py">Evaluator</a></code>
-- <code title="put /evaluators/{evaluator_id}">client.evaluators.<a href="./src/lemma/resources/evaluators.py">update</a>(evaluator_id, \*\*<a href="src/lemma/types/evaluator_update_params.py">params</a>) -> <a href="./src/lemma/types/evaluator.py">Evaluator</a></code>
+- <code title="get /evaluators/{evaluator_id}">client.evaluators.<a href="./src/lemma/resources/evaluators.py">retrieve</a>(evaluator_id) -> <a href="./src/lemma/types/evaluator_retrieve_response.py">EvaluatorRetrieveResponse</a></code>
+- <code title="put /evaluators/{evaluator_id}">client.evaluators.<a href="./src/lemma/resources/evaluators.py">update</a>(evaluator_id, \*\*<a href="src/lemma/types/evaluator_update_params.py">params</a>) -> <a href="./src/lemma/types/evaluator_update_response.py">EvaluatorUpdateResponse</a></code>
 - <code title="get /evaluators">client.evaluators.<a href="./src/lemma/resources/evaluators.py">list</a>() -> <a href="./src/lemma/types/evaluator_list_response.py">EvaluatorListResponse</a></code>
 - <code title="post /evaluators/rubric">client.evaluators.<a href="./src/lemma/resources/evaluators.py">create_rubric</a>(\*\*<a href="src/lemma/types/evaluator_create_rubric_params.py">params</a>) -> <a href="./src/lemma/types/evaluator_create_rubric_response.py">EvaluatorCreateRubricResponse</a></code>

src/lemma/resources/evaluators.py

@@ -19,10 +19,11 @@
     async_to_streamed_response_wrapper,
 )
 from .._base_client import make_request_options
-from ..types.evaluator import Evaluator
 from ..types.rubric_example_param import RubricExampleParam
 from ..types.rubric_criterion_param import RubricCriterionParam
 from ..types.evaluator_list_response import EvaluatorListResponse
+from ..types.evaluator_update_response import EvaluatorUpdateResponse
+from ..types.evaluator_retrieve_response import EvaluatorRetrieveResponse
 from ..types.evaluator_create_rubric_response import EvaluatorCreateRubricResponse
 
 __all__ = ["EvaluatorsResource", "AsyncEvaluatorsResource"]
@@ -58,7 +59,7 @@ def retrieve(
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluator:
+    ) -> EvaluatorRetrieveResponse:
         """
         Get a specific evaluator by ID.
 
@@ -81,7 +82,7 @@ def retrieve(
             options=make_request_options(
                 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
             ),
-            cast_to=Evaluator,
+            cast_to=EvaluatorRetrieveResponse,
         )
 
     def update(
@@ -98,7 +99,7 @@ def update(
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluator:
+    ) -> EvaluatorUpdateResponse:
         """
         Update an existing evaluator.
 
@@ -132,7 +133,7 @@ def update(
             options=make_request_options(
                 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
             ),
-            cast_to=Evaluator,
+            cast_to=EvaluatorUpdateResponse,
         )
 
     def list(
@@ -236,7 +237,7 @@ async def retrieve(
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluator:
+    ) -> EvaluatorRetrieveResponse:
         """
         Get a specific evaluator by ID.
 
@@ -259,7 +260,7 @@ async def retrieve(
             options=make_request_options(
                 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
             ),
-            cast_to=Evaluator,
+            cast_to=EvaluatorRetrieveResponse,
         )
 
     async def update(
@@ -276,7 +277,7 @@ async def update(
         extra_query: Query | None = None,
         extra_body: Body | None = None,
         timeout: float | httpx.Timeout | None | NotGiven = not_given,
-    ) -> Evaluator:
+    ) -> EvaluatorUpdateResponse:
         """
         Update an existing evaluator.
 
@@ -310,7 +311,7 @@ async def update(
             options=make_request_options(
                 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
             ),
-            cast_to=Evaluator,
+            cast_to=EvaluatorUpdateResponse,
         )
 
     async def list(

src/lemma/resources/playground.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from typing import Optional
+from typing import Dict, Optional
 
 import httpx
 
@@ -48,6 +48,7 @@ def run(
         model: str,
         prompt_id: str,
         evaluator_id: Optional[str] | Omit = omit,
+        input_variables: Dict[str, object] | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -65,6 +66,8 @@ def run(
 
           evaluator_id: Optional ID of the evaluator to use
 
+          input_variables: Input variables for Jinja template substitution
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -80,6 +83,7 @@ def run(
                     "model": model,
                     "prompt_id": prompt_id,
                     "evaluator_id": evaluator_id,
+                    "input_variables": input_variables,
                 },
                 playground_run_params.PlaygroundRunParams,
             ),
@@ -116,6 +120,7 @@ async def run(
         model: str,
         prompt_id: str,
         evaluator_id: Optional[str] | Omit = omit,
+        input_variables: Dict[str, object] | Omit = omit,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
         extra_headers: Headers | None = None,
@@ -133,6 +138,8 @@ async def run(
 
           evaluator_id: Optional ID of the evaluator to use
 
+          input_variables: Input variables for Jinja template substitution
+
           extra_headers: Send extra headers
 
           extra_query: Add additional query parameters to the request
@@ -148,6 +155,7 @@ async def run(
                     "model": model,
                     "prompt_id": prompt_id,
                     "evaluator_id": evaluator_id,
+                    "input_variables": input_variables,
                 },
                 playground_run_params.PlaygroundRunParams,
             ),

src/lemma/types/__init__.py

@@ -5,10 +5,7 @@
 from .log import Log as Log
 from .prompt import Prompt as Prompt
 from .dataset import Dataset as Dataset
-from .evaluator import Evaluator as Evaluator
-from .rubric_example import RubricExample as RubricExample
 from .log_list_params import LogListParams as LogListParams
-from .rubric_criterion import RubricCriterion as RubricCriterion
 from .log_create_params import LogCreateParams as LogCreateParams
 from .log_list_response import LogListResponse as LogListResponse
 from .prompt_create_params import PromptCreateParams as PromptCreateParams
@@ -23,6 +20,8 @@
 from .dataset_delete_response import DatasetDeleteResponse as DatasetDeleteResponse
 from .evaluator_list_response import EvaluatorListResponse as EvaluatorListResponse
 from .evaluator_update_params import EvaluatorUpdateParams as EvaluatorUpdateParams
+from .evaluator_update_response import EvaluatorUpdateResponse as EvaluatorUpdateResponse
+from .evaluator_retrieve_response import EvaluatorRetrieveResponse as EvaluatorRetrieveResponse
 from .dataset_generate_schema_params import DatasetGenerateSchemaParams as DatasetGenerateSchemaParams
 from .evaluator_create_rubric_params import EvaluatorCreateRubricParams as EvaluatorCreateRubricParams
 from .dataset_generate_dataset_params import DatasetGenerateDatasetParams as DatasetGenerateDatasetParams

src/lemma/types/evaluator.py

@@ -1,19 +1,32 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import List
+from typing import Dict, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from pydantic import Field as FieldInfo
 
 from .._models import BaseModel
-from .rubric_example import RubricExample
-from .rubric_criterion import RubricCriterion
 
 __all__ = ["EvaluatorCreateRubricResponse"]
 
 
 class EvaluatorCreateRubricResponse(BaseModel):
-    criteria: List[RubricCriterion]
+    config: Dict[str, object]
+    """Variables used in the prompt template"""
+
+    name: str
 
-    description: str
+    tenant_id: str = FieldInfo(alias="tenantId")
 
-    examples: List[RubricExample]
+    id: Optional[str] = None
 
-    name: str
+    created_at: Optional[datetime] = FieldInfo(alias="createdAt", default=None)
+    """Timestamp when the record was created"""
+
+    description: Optional[str] = None
+
+    type: Optional[Literal["RUBRIC", "WEBHOOK", "CUSTOM"]] = None
+
+    updated_at: Optional[datetime] = FieldInfo(alias="updatedAt", default=None)
+    """Timestamp when the record was last updated"""

src/lemma/types/evaluator_create_rubric_response.py

@@ -1,10 +1,35 @@
 # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
 
-from typing import List
-from typing_extensions import TypeAlias
+from typing import Dict, List, Optional
+from datetime import datetime
+from typing_extensions import Literal, TypeAlias
 
-from .evaluator import Evaluator
+from pydantic import Field as FieldInfo
 
-__all__ = ["EvaluatorListResponse"]
+from .._models import BaseModel
 
-EvaluatorListResponse: TypeAlias = List[Evaluator]
+__all__ = ["EvaluatorListResponse", "EvaluatorListResponseItem"]
+
+
+class EvaluatorListResponseItem(BaseModel):
+    config: Dict[str, object]
+    """Variables used in the prompt template"""
+
+    name: str
+
+    tenant_id: str = FieldInfo(alias="tenantId")
+
+    id: Optional[str] = None
+
+    created_at: Optional[datetime] = FieldInfo(alias="createdAt", default=None)
+    """Timestamp when the record was created"""
+
+    description: Optional[str] = None
+
+    type: Optional[Literal["RUBRIC", "WEBHOOK", "CUSTOM"]] = None
+
+    updated_at: Optional[datetime] = FieldInfo(alias="updatedAt", default=None)
+    """Timestamp when the record was last updated"""
+
+
+EvaluatorListResponse: TypeAlias = List[EvaluatorListResponseItem]

src/lemma/types/evaluator_list_response.py

@@ -0,0 +1,32 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from pydantic import Field as FieldInfo
+
+from .._models import BaseModel
+
+__all__ = ["EvaluatorRetrieveResponse"]
+
+
+class EvaluatorRetrieveResponse(BaseModel):
+    config: Dict[str, object]
+    """Variables used in the prompt template"""
+
+    name: str
+
+    tenant_id: str = FieldInfo(alias="tenantId")
+
+    id: Optional[str] = None
+
+    created_at: Optional[datetime] = FieldInfo(alias="createdAt", default=None)
+    """Timestamp when the record was created"""
+
+    description: Optional[str] = None
+
+    type: Optional[Literal["RUBRIC", "WEBHOOK", "CUSTOM"]] = None
+
+    updated_at: Optional[datetime] = FieldInfo(alias="updatedAt", default=None)
+    """Timestamp when the record was last updated"""

src/lemma/types/evaluator_retrieve_response.py

@@ -0,0 +1,32 @@
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from pydantic import Field as FieldInfo
+
+from .._models import BaseModel
+
+__all__ = ["EvaluatorUpdateResponse"]
+
+
+class EvaluatorUpdateResponse(BaseModel):
+    config: Dict[str, object]
+    """Variables used in the prompt template"""
+
+    name: str
+
+    tenant_id: str = FieldInfo(alias="tenantId")
+
+    id: Optional[str] = None
+
+    created_at: Optional[datetime] = FieldInfo(alias="createdAt", default=None)
+    """Timestamp when the record was created"""
+
+    description: Optional[str] = None
+
+    type: Optional[Literal["RUBRIC", "WEBHOOK", "CUSTOM"]] = None
+
+    updated_at: Optional[datetime] = FieldInfo(alias="updatedAt", default=None)
+    """Timestamp when the record was last updated"""