feat(api): api update

Author: stainless-app[bot]

.stats.yml

@@ -1,4 +1,4 @@
 configured_endpoints: 18
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/lemma%2Flemma-584f07ea035720c85ecac56802077189e3ef8e2970d3a7f9ffb75a605df61fef.yml
-openapi_spec_hash: 941d7b54bafb4c217116dc8fea329900
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/lemma%2Flemma-a7a66ada4907eaf7c12f8686ea23314984295b2c28a8662c2c7bb5f74deeea08.yml
+openapi_spec_hash: be8496356007ef4d20617cc2fea77f47
 config_hash: 3911423d0a218a9e0813f5fe6370ae95

api.md

@@ -63,13 +63,7 @@ Methods:
 Types:
 
 ```python
-from lemma.types import (
-    Criterion,
-    Evaluator,
-    Example,
-    EvaluatorListResponse,
-    EvaluatorCreateRubricResponse,
-)
+from lemma.types import Evaluator, EvaluatorListResponse, EvaluatorCreateRubricResponse
 ```
 
 Methods:

src/lemma/resources/evaluators.py

@@ -20,8 +20,6 @@
 )
 from .._base_client import make_request_options
 from ..types.evaluator import Evaluator
-from ..types.example_param import ExampleParam
-from ..types.criterion_param import CriterionParam
 from ..types.evaluator_list_response import EvaluatorListResponse
 from ..types.evaluator_create_rubric_response import EvaluatorCreateRubricResponse
 
@@ -162,9 +160,9 @@ def list(
     def create_rubric(
         self,
         *,
-        criteria: Iterable[CriterionParam],
+        criteria: Iterable[evaluator_create_rubric_params.Criterion],
         description: str,
-        examples: Iterable[ExampleParam],
+        examples: Iterable[evaluator_create_rubric_params.Example],
         name: str,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.
@@ -340,9 +338,9 @@ async def list(
     async def create_rubric(
         self,
         *,
-        criteria: Iterable[CriterionParam],
+        criteria: Iterable[evaluator_create_rubric_params.Criterion],
         description: str,
-        examples: Iterable[ExampleParam],
+        examples: Iterable[evaluator_create_rubric_params.Example],
         name: str,
         # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
         # The extra values given here take precedence over values defined on the client or passed to this method.

src/lemma/types/__init__.py

@@ -4,11 +4,7 @@
 
 from .log import Log as Log
 from .dataset import Dataset as Dataset
-from .example import Example as Example
-from .criterion import Criterion as Criterion
 from .evaluator import Evaluator as Evaluator
-from .example_param import ExampleParam as ExampleParam
-from .criterion_param import CriterionParam as CriterionParam
 from .log_list_params import LogListParams as LogListParams
 from .log_create_params import LogCreateParams as LogCreateParams
 from .log_list_response import LogListResponse as LogListResponse

src/lemma/types/criterion.py

@@ -3,19 +3,50 @@
 from __future__ import annotations
 
 from typing import Iterable
-from typing_extensions import Required, TypedDict
+from typing_extensions import Required, Annotated, TypedDict
 
-from .example_param import ExampleParam
-from .criterion_param import CriterionParam
+from .._utils import PropertyInfo
 
-__all__ = ["EvaluatorCreateRubricParams"]
+__all__ = ["EvaluatorCreateRubricParams", "Criterion", "CriterionScoreLevel", "Example"]
 
 
 class EvaluatorCreateRubricParams(TypedDict, total=False):
-    criteria: Required[Iterable[CriterionParam]]
+    criteria: Required[Iterable[Criterion]]
 
     description: Required[str]
 
-    examples: Required[Iterable[ExampleParam]]
+    examples: Required[Iterable[Example]]
 
     name: Required[str]
+
+
+class CriterionScoreLevel(TypedDict, total=False):
+    description: Required[str]
+
+    label: Required[str]
+
+    score: Required[int]
+
+
+class Criterion(TypedDict, total=False):
+    id: Required[str]
+
+    description: Required[str]
+
+    name: Required[str]
+
+    score_levels: Required[Annotated[Iterable[CriterionScoreLevel], PropertyInfo(alias="scoreLevels")]]
+
+    weight: Required[float]
+
+
+class Example(TypedDict, total=False):
+    id: Required[str]
+
+    criterion_id: Required[Annotated[str, PropertyInfo(alias="criterionId")]]
+
+    expected_score: Required[Annotated[int, PropertyInfo(alias="expectedScore")]]
+
+    input: Required[str]
+
+    reasoning: Required[str]

src/lemma/types/criterion_param.py

@@ -2,11 +2,43 @@
 
 from typing import List
 
-from .example import Example
+from pydantic import Field as FieldInfo
+
 from .._models import BaseModel
-from .criterion import Criterion
 
-__all__ = ["EvaluatorCreateRubricResponse"]
+__all__ = ["EvaluatorCreateRubricResponse", "Criterion", "CriterionScoreLevel", "Example"]
+
+
+class CriterionScoreLevel(BaseModel):
+    description: str
+
+    label: str
+
+    score: int
+
+
+class Criterion(BaseModel):
+    id: str
+
+    description: str
+
+    name: str
+
+    score_levels: List[CriterionScoreLevel] = FieldInfo(alias="scoreLevels")
+
+    weight: float
+
+
+class Example(BaseModel):
+    id: str
+
+    criterion_id: str = FieldInfo(alias="criterionId")
+
+    expected_score: int = FieldInfo(alias="expectedScore")
+
+    input: str
+
+    reasoning: str
 
 
 class EvaluatorCreateRubricResponse(BaseModel):