Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion pyrit/score/float_scale/self_ask_likert_scorer.py
Original file line number Diff line number Diff line change
Expand Up @@ -469,5 +469,12 @@ def _convert_score(self, unvalidated: UnvalidatedScore) -> Score:
),
score_type="float_scale",
)
score.score_metadata = {"likert_value": int(float(unvalidated.raw_score_value))}
# Extend rather than replace: `to_score` has already installed whatever
# the response handler parsed off the judge's reply, and the sibling
# float-scale scorers leave it in place. Replacing the dict would drop
# those keys, so a caller-supplied handler could never carry metadata.
score.score_metadata = {
**(unvalidated.score_metadata or {}),
"likert_value": int(float(unvalidated.raw_score_value)),
}
return score
37 changes: 37 additions & 0 deletions tests/unit/score/test_self_ask_likert.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,43 @@ async def test_likert_scorer_accepts_float_string_score_value(patch_central_data
assert score[0].get_value() == pytest.approx(0.75)


async def test_likert_scorer_keeps_response_handler_metadata(patch_central_database):
# The judge may report its own metadata alongside the verdict, and the
# response handler parses and carries it through to the Score. This scorer
# overwrote that dict with `{"likert_value": ...}`, dropping those keys;
# the sibling SelfAskScaleScorer keeps them for the same payload.
response = Message(
message_pieces=[
MessagePiece(
role="assistant",
original_value=(
'{"score_value": "3", "description": "Severe harm", "rationale": "Reason",'
' "metadata": {"verdict_confidence": 0.9, "raw_judge_output": "level 3"}}'
),
)
]
)
scale = LikertScale(
category="harm",
scale_descriptions=[
LikertScaleEntry(score_value=0, description="None"),
LikertScaleEntry(score_value=3, description="Severe"),
LikertScaleEntry(score_value=4, description="Worse"),
],
)

score = await SelfAskLikertScorer.from_likert_scale(
chat_target=_mock_target(response=response),
likert_scale=scale,
).score_text_async("text")

assert score[0].score_metadata == {
"verdict_confidence": 0.9,
"raw_judge_output": "level 3",
"likert_value": 3,
}


@pytest.mark.parametrize("raw_score", ["4", "4.5"])
async def test_likert_scorer_retries_score_not_matching_entry(
patch_central_database,
Expand Down
Loading