Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pyrit/score/response_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,10 @@ def parse(
objective=objective,
)

normalized_value = score.raw_score_value.lower()
# Strip surrounding whitespace before comparing: a judge that returns
# "true\n" or " false" is giving a valid verdict, and should not be
# rejected as out-of-domain over incidental whitespace.
normalized_value = score.raw_score_value.strip().lower()
if normalized_value not in {"true", "false"}:
raise InvalidJsonException(
message=f"True/false score_value must be 'true' or 'false', not {score.raw_score_value!r}."
Expand Down
23 changes: 23 additions & 0 deletions tests/unit/score/test_response_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,29 @@ def test_true_false_response_handler_accepts_boolean_values(json_value: str, exp
assert score.raw_score_value == expected


@pytest.mark.parametrize(
("json_value", "expected"),
[
('"true "', "true"),
('" false"', "false"),
('"True\\n"', "true"),
('" FALSE "', "false"),
],
)
def test_true_false_response_handler_strips_whitespace_around_verdict(json_value: str, expected: str) -> None:
# A judge returning a valid verdict with incidental surrounding whitespace
# (e.g. a trailing newline) must not be rejected as out-of-domain.
handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler())

score = handler.parse(
response_text=f'{{"score_value": {json_value}, "rationale": "test"}}',
scorer_identifier=SCORER_IDENTIFIER,
scored_prompt_id="test-id",
)

assert score.raw_score_value == expected


def test_true_false_response_handler_rejects_value_outside_domain() -> None:
handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler())

Expand Down
Loading