feat(benchmark): expose vote aggregation on leaderboards

docs/mri_advanced.md

@@ -134,6 +134,34 @@ leaderboard = benchmark.create_leaderboard(
 )
 ```
 
+### Vote Aggregation
+
+Each matchup — one comparison of two models on one prompt — collects several
+responses. By default the matchup's winner is the side the majority of those
+responses picked, with ties split 0.5/0.5.
+
+Pass `vote_aggregation=VoteAggregation.ALL_VOTES` to count every individual
+response as its own matchup instead:
+
+```python
+from rapidata import VoteAggregation
+
+leaderboard = benchmark.create_leaderboard(
+    name="Realism",
+    instruction="Which image is more realistic?",
+    vote_aggregation=VoteAggregation.ALL_VOTES,
+)
+```
+
+Standings are derived from the raw responses on every read, so switching the
+aggregation on an existing leaderboard also changes how its already-collected
+responses are counted — no re-evaluation needed.
+
+```python
+print(leaderboard.vote_aggregation)  # VoteAggregation.ALL_VOTES
+leaderboard.update(vote_aggregation=VoteAggregation.MAJORITY_VOTE)
+```
+
 ### Level of Detail (response budget)
 
 `level_of_detail` sets the leaderboard's **response budget** — the total number of
@@ -177,26 +205,41 @@ leaderboard_precise = benchmark.create_leaderboard(
 )
 ```
 
-You can also read or change the budget on an existing leaderboard through the
-`level_of_detail` property, which likewise accepts a named level or a custom
-integer. Changes apply to future evaluations; standings that have already been
-computed are not recomputed.
+You can read the budget on an existing leaderboard through the `level_of_detail`
+property and change it with `update()`, which likewise accepts a named level or a
+custom integer. Changes apply to future evaluations; standings that have already
+been computed are not recomputed.
 
 ```python
-print(leaderboard.level_of_detail)   # e.g. "low"
-leaderboard.level_of_detail = "high" # named level
-leaderboard.level_of_detail = 5000   # custom budget
+print(leaderboard.level_of_detail)              # e.g. "low"
+leaderboard.update(level_of_detail="high")      # named level
+leaderboard.update(level_of_detail=5000)        # custom budget
 ```
 
 A custom budget reads back as `"custom"`; use the `response_budget` property to
 get the exact number.
 
 ```python
-leaderboard.level_of_detail = 5000
+leaderboard.update(level_of_detail=5000)
 print(leaderboard.level_of_detail)   # "custom"
 print(leaderboard.response_budget)   # 5000
 ```
 
+### Changing a leaderboard's configuration
+
+`update()` is the single entry point for every mutable leaderboard setting. Only
+the arguments you pass are changed; anything you omit keeps its stored value, and
+all of them go out in one request.
+
+```python
+leaderboard.update(
+    name="Realism v2",
+    level_of_detail="high",
+    min_responses_per_matchup=5,
+    vote_aggregation=VoteAggregation.MAJORITY_VOTE,
+)
+```
+
 ### Restricting which prompts a leaderboard uses
 
 A leaderboard normally builds matchups from every prompt in its benchmark. Pass

src/rapidata/__init__.py

@@ -66,6 +66,7 @@
     DeviceType,
     Tag,
     Origin,
+    VoteAggregation,
     Datapoint,
     ContextManager,
     FailedUploadException,

src/rapidata/rapidata_client/__init__.py

@@ -28,6 +28,7 @@
     EffortSelection,
 )
 from .benchmark.prompt_metadata import Origin, Tag
+from .benchmark.leaderboard.vote_aggregation import VoteAggregation
 from .benchmark.participant.sample_upload import SampleUpload
 from .datapoints import Datapoint
 from .context import ContextManager

src/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard.py

@@ -10,6 +10,9 @@
     LevelOfDetail,
     ResolvedLevelOfDetail,
 )
+from rapidata.rapidata_client.benchmark.leaderboard.vote_aggregation import (
+    VoteAggregation,
+)
 from rapidata.api_client.models.audience_audience_id_jobs_get_job_id_parameter import (
     AudienceAudienceIdJobsGetJobIdParameter,
 )
@@ -37,6 +40,7 @@ class RapidataLeaderboard:
         openapi_service: The OpenAPIService instance for API interaction.
         included_tags: The prompt tag values the leaderboard restricts its matchups to.
         excluded_tags: The prompt tag values the leaderboard skips when building matchups.
+        vote_aggregation: How the responses on a single matchup are aggregated into that matchup's result. None resolves it on first read.
     """
 
     def __init__(
@@ -53,6 +57,7 @@ def __init__(
         openapi_service: OpenAPIService,
         included_tags: list[str] | None = None,
         excluded_tags: list[str] | None = None,
+        vote_aggregation: VoteAggregation | None = None,
     ):
         self.__openapi_service = openapi_service
         self.__name = name
@@ -65,6 +70,7 @@ def __init__(
         self.__benchmark_id = benchmark_id
         self.__included_tags = list(included_tags) if included_tags else []
         self.__excluded_tags = list(excluded_tags) if excluded_tags else []
+        self.__vote_aggregation = vote_aggregation
         self.id = id
         self.__leaderboard_page = f"https://app.{self.__openapi_service.environment}/mri/benchmarks/{self.__benchmark_id}/leaderboard/{self.id}"
 
@@ -95,46 +101,117 @@ def level_of_detail(self) -> ResolvedLevelOfDetail:
         """
         return DetailMapper.get_level_of_detail(self.__response_budget)
 
-    @level_of_detail.setter
-    def level_of_detail(self, level_of_detail: LevelOfDetail | int):
-        """
-        Sets the level of detail (response budget) of the leaderboard.
-
-        Accepts one of the named levels or a positive integer for a custom response
-        budget. Takes effect for future evaluations; already-computed standings are
-        not recomputed.
-        """
-        with tracer.start_as_current_span("RapidataLeaderboard.level_of_detail.setter"):
-            logger.debug(f"Setting level of detail to {level_of_detail}")
-            self.__response_budget = DetailMapper.resolve_budget(level_of_detail)
-            self._update_config()
-
     @property
     def min_responses_per_matchup(self) -> int:
         """
         Returns the minimum number of responses required to be considered for the leaderboard.
         """
         return self.__min_responses_per_matchup
 
-    @min_responses_per_matchup.setter
-    def min_responses_per_matchup(self, min_responses: int):
-        """
-        Sets the minimum number of responses required to be considered for the leaderboard.
+    @property
+    def vote_aggregation(self) -> VoteAggregation:
+        """
+        How the responses on a single matchup are aggregated into that matchup's result.
+
+        :attr:`VoteAggregation.MAJORITY_VOTE` collapses each matchup to one win for the
+        side the majority of responses picked (ties split 0.5/0.5), so every matchup
+        carries the same weight regardless of how many responses it collected.
+        :attr:`VoteAggregation.ALL_VOTES` instead counts every individual response as
+        its own matchup.
+        """
+        # The benchmark's leaderboard listing does not carry the aggregation, so
+        # leaderboards read from it fetch it once, on demand.
+        if self.__vote_aggregation is None:
+            with tracer.start_as_current_span("RapidataLeaderboard.vote_aggregation"):
+                result = self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_get(
+                    leaderboard_id=self.id
+                )
+                self.__vote_aggregation = VoteAggregation._from_backend_model(
+                    result.vote_aggregation
+                )
+
+        return self.__vote_aggregation
+
+    def update(
+        self,
+        name: str | None = None,
+        level_of_detail: LevelOfDetail | int | None = None,
+        min_responses_per_matchup: int | None = None,
+        vote_aggregation: VoteAggregation | None = None,
+    ) -> None:
         """
-        with tracer.start_as_current_span(
-            "RapidataLeaderboard.min_responses_per_matchup.setter"
-        ):
-            if not isinstance(min_responses, int):
-                raise ValueError("Min responses per matchup must be an integer")
+        Updates the leaderboard's configuration.
+
+        Only the arguments you pass are changed; anything omitted keeps its stored
+        value.
+
+        Args:
+            name: The new name of the leaderboard. (not shown to the users)
+            level_of_detail: The new response budget — either one of the named levels ('debug', 'low', 'medium', 'high', 'very high') or a positive integer for a custom budget. Takes effect for future evaluations; already-computed standings are not recomputed.
+            min_responses_per_matchup: The new minimum number of responses collected per matchup. Must be at least 3.
+            vote_aggregation: How the responses on a single matchup are aggregated into that matchup's result. Standings are derived from the raw responses on every read, so this also changes how already-collected responses are counted — no re-evaluation needed.
+        """
+        with tracer.start_as_current_span("RapidataLeaderboard.update"):
+            if name is not None and (not isinstance(name, str) or len(name) < 1):
+                raise ValueError("Name must be a string of at least 1 character")
+
+            response_budget = (
+                DetailMapper.resolve_budget(level_of_detail)
+                if level_of_detail is not None
+                else None
+            )
+
+            if min_responses_per_matchup is not None:
+                # bool is an int subclass — reject it so `True` isn't read as 1.
+                if isinstance(min_responses_per_matchup, bool) or not isinstance(
+                    min_responses_per_matchup, int
+                ):
+                    raise ValueError("Min responses per matchup must be an integer")
+
+                if min_responses_per_matchup < 3:
+                    raise ValueError("Min responses per matchup must be at least 3")
+
+            if vote_aggregation is not None and not isinstance(
+                vote_aggregation, VoteAggregation
+            ):
+                raise ValueError(
+                    "Vote aggregation must be one of: "
+                    + ", ".join(
+                        f"VoteAggregation.{member.name}" for member in VoteAggregation
+                    )
+                )
 
-            if min_responses < 3:
-                raise ValueError("Min responses per matchup must be at least 3")
+            logger.info(
+                "Updating leaderboard %s with name %s, response_budget %s, min_responses_per_matchup %s, vote_aggregation %s",
+                self.id,
+                name,
+                response_budget,
+                min_responses_per_matchup,
+                vote_aggregation.name if vote_aggregation is not None else None,
+            )
 
-            logger.debug(
-                f"Setting min responses per matchup to {min_responses} for leaderboard {self.name}"
+            self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_patch(
+                leaderboard_id=self.id,
+                update_leaderboard_endpoint_input=UpdateLeaderboardEndpointInput(
+                    name=name,
+                    responseBudget=response_budget,
+                    minResponses=min_responses_per_matchup,
+                    voteAggregation=(
+                        vote_aggregation._to_backend_model()
+                        if vote_aggregation is not None
+                        else None
+                    ),
+                ),
             )
-            self.__min_responses_per_matchup = min_responses
-            self._update_config()
+
+            if name is not None:
+                self.__name = name
+            if response_budget is not None:
+                self.__response_budget = response_budget
+            if min_responses_per_matchup is not None:
+                self.__min_responses_per_matchup = min_responses_per_matchup
+            if vote_aggregation is not None:
+                self.__vote_aggregation = vote_aggregation
 
     @property
     def show_prompt_asset(self) -> bool:
@@ -196,20 +273,6 @@ def name(self) -> str:
         """
         return self.__name
 
-    @name.setter
-    def name(self, name: str):
-        """
-        Sets the name of the leaderboard.
-        """
-        with tracer.start_as_current_span("RapidataLeaderboard.name.setter"):
-            if not isinstance(name, str):
-                raise ValueError("Name must be a string")
-            if len(name) < 1:
-                raise ValueError("Name must be at least 1 character long")
-
-            self.__name = name
-            self._update_config()
-
     @property
     def jobs(self) -> list[RapidataJob]:
         """
@@ -355,22 +418,6 @@ def view(self) -> None:
                 + Fore.RESET
             )
 
-    def _custom_config(self, response_budget: int, min_responses_per_matchup: int):
-        self.__response_budget = response_budget
-        self.__min_responses_per_matchup = min_responses_per_matchup
-        self._update_config()
-
-    def _update_config(self):
-        with tracer.start_as_current_span("RapidataLeaderboard._update_config"):
-            self.__openapi_service.leaderboard.leaderboard_api.leaderboard_leaderboard_id_patch(
-                leaderboard_id=self.id,
-                update_leaderboard_endpoint_input=UpdateLeaderboardEndpointInput(
-                    name=self.__name,
-                    responseBudget=self.__response_budget,
-                    minResponses=self.__min_responses_per_matchup,
-                ),
-            )
-
     def __str__(self) -> str:
         return f"RapidataLeaderboard(name={self.name}, instruction={self.instruction}, show_prompt={self.show_prompt}, leaderboard_id={self.id})"
 

src/rapidata/rapidata_client/benchmark/leaderboard/vote_aggregation.py

@@ -0,0 +1,29 @@
+from enum import Enum
+
+from rapidata.api_client.models.vote_aggregation import (
+    VoteAggregation as VoteAggregationModel,
+)
+
+
+class VoteAggregation(Enum):
+    """VoteAggregation Enum
+
+    How the individual annotator responses on a single matchup (one comparison of
+    two models on one prompt) are aggregated into that matchup's result.
+
+    Attributes:
+        MAJORITY_VOTE (VoteAggregation): Collapses each matchup to a single win for
+            the side the majority of responses picked, splitting ties 0.5/0.5.
+        ALL_VOTES (VoteAggregation): Counts every individual response as its own
+            matchup.
+    """
+
+    MAJORITY_VOTE = VoteAggregationModel.MAJORITYVOTE
+    ALL_VOTES = VoteAggregationModel.ALLVOTES
+
+    def _to_backend_model(self) -> VoteAggregationModel:
+        return VoteAggregationModel(self.value)
+
+    @classmethod
+    def _from_backend_model(cls, model: VoteAggregationModel) -> "VoteAggregation":
+        return cls(model)