Add question_id functionality to mcq object (#185)

src/aviary/utils.py

@@ -269,7 +269,7 @@ def shuffle(
 class MultipleChoiceQuestion(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
-    OPEN_ANSWER_PROMPT_TEMPLATE: ClassVar[str] = "Q: {question}"
+    OPEN_ANSWER_PROMPT_TEMPLATE: ClassVar[str] = "{question_id}: {question}"
     MC_QUESTION_PROMPT_TEMPLATE: ClassVar[str] = "\n\n".join((
         OPEN_ANSWER_PROMPT_TEMPLATE,
         "Options:\n{options}",
@@ -284,6 +284,11 @@ class MultipleChoiceQuestion(BaseModel):
     question: str = Field(
         description="Question to answer (without multiple choice options)."
     )
+
+    question_id: str = Field(
+        default="Q", description="Question identifier used in the prompt."
+    )
+
     prompt_without_options: bool = Field(
         default=False,
         description=(
@@ -347,18 +352,32 @@ def add_answers_and_shuffle(self) -> Self:
     def ideal_answer_index(self) -> int:
         return self.options.index(self.ideal_answer)
 
+    @property
+    def ideal_answer_letter(self) -> str:
+        return chr(_CAPITAL_A_INDEX + self.ideal_answer_index)
+
     @property
     def unsure_answer_index(self) -> int | None:
         if self.unsure_answer is None:
             return None
         return self.options.index(self.unsure_answer)
 
+    @property
+    def unsure_answer_letter(self) -> str | None:
+        if self.unsure_answer_index is None:
+            return None
+        return chr(_CAPITAL_A_INDEX + self.unsure_answer_index)
+
     @property
     def question_prompt(self) -> str:
         if self.prompt_without_options:
-            return self.OPEN_ANSWER_PROMPT_TEMPLATE.format(question=self.question)
+            return self.OPEN_ANSWER_PROMPT_TEMPLATE.format(
+                question=self.question,
+                question_id=self.question_id,
+            )
         return self.MC_QUESTION_PROMPT_TEMPLATE.format(
             question=self.question,
+            question_id=self.question_id,
             options="\n".join([
                 f"{_CAPITAL_A_INDEX + i:c}) {o}" for i, o in enumerate(self.options)
             ]),

tests/test_utils.py

@@ -369,6 +369,64 @@ def test_consistent_open_answer(self) -> None:
             "Serialization then deserialization should lead to same prompts"
         )
 
+    @pytest.mark.parametrize(
+        (
+            "options",
+            "ideal_answer",
+            "unsure_answer",
+            "seed",
+            "expected_ideal_letter",
+            "expected_unsure_letter",
+        ),
+        [
+            # Test cases for ideal and unsure answer letters
+            (["A", "B"], "C", "Not sure", 42, "D", "B"),  # With seed 42
+            (["X", "Y"], "Z", "Unsure", 0, "D", "A"),  # With seed 0
+            (["A", "B", "C"], "B", None, 42, "C", None),  # Ideal answer in options
+            (
+                ["D", "E", "F"],
+                "E",
+                MultipleChoiceQuestion.DEFAULT_UNSURE_OPTION,
+                0,
+                "B",
+                "A",
+            ),
+            (
+                ["A", "B", "Not sure"],
+                "C",
+                "Not sure",
+                0,
+                "A",
+                "D",
+            ),  # Unsure answer in options
+        ],
+    )
+    def test_answer_letters(
+        self,
+        options: list[str],
+        ideal_answer: str,
+        unsure_answer: str | None,
+        seed: int,
+        expected_ideal_letter: str,
+        expected_unsure_letter: str | None,
+    ) -> None:
+        """Test that ideal_answer_letter and unsure_answer_letter return correct letters after shuffling."""
+        mc_question = MultipleChoiceQuestion(
+            question="test question",
+            options=options,
+            ideal_answer=ideal_answer,
+            unsure_answer=unsure_answer,
+            shuffle_seed=seed,  # Use specific seeds for predictable shuffling
+        )
+        # Check ideal answer letter
+        assert mc_question.ideal_answer_letter == expected_ideal_letter
+        assert ideal_answer in mc_question.options
+
+        # Check unsure answer letter
+        assert mc_question.unsure_answer_letter == expected_unsure_letter
+        if unsure_answer is not None:
+            assert unsure_answer in mc_question.options
+
 
 class TestMultipleChoiceEvaluation:
     @pytest.mark.parametrize(