diff --git a/pyrit/score/response_handler.py b/pyrit/score/response_handler.py index 6dea0a9d0a..0c27efd120 100644 --- a/pyrit/score/response_handler.py +++ b/pyrit/score/response_handler.py @@ -303,7 +303,10 @@ def parse( objective=objective, ) - normalized_value = score.raw_score_value.lower() + # Strip surrounding whitespace before comparing: a judge that returns + # "true\n" or " false" is giving a valid verdict, and should not be + # rejected as out-of-domain over incidental whitespace. + normalized_value = score.raw_score_value.strip().lower() if normalized_value not in {"true", "false"}: raise InvalidJsonException( message=f"True/false score_value must be 'true' or 'false', not {score.raw_score_value!r}." diff --git a/tests/unit/score/test_response_handler.py b/tests/unit/score/test_response_handler.py index 16084a5cf6..fac1b55ac0 100644 --- a/tests/unit/score/test_response_handler.py +++ b/tests/unit/score/test_response_handler.py @@ -43,6 +43,29 @@ def test_true_false_response_handler_accepts_boolean_values(json_value: str, exp assert score.raw_score_value == expected +@pytest.mark.parametrize( + ("json_value", "expected"), + [ + ('"true "', "true"), + ('" false"', "false"), + ('"True\\n"', "true"), + ('" FALSE "', "false"), + ], +) +def test_true_false_response_handler_strips_whitespace_around_verdict(json_value: str, expected: str) -> None: + # A judge returning a valid verdict with incidental surrounding whitespace + # (e.g. a trailing newline) must not be rejected as out-of-domain. + handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler()) + + score = handler.parse( + response_text=f'{{"score_value": {json_value}, "rationale": "test"}}', + scorer_identifier=SCORER_IDENTIFIER, + scored_prompt_id="test-id", + ) + + assert score.raw_score_value == expected + + def test_true_false_response_handler_rejects_value_outside_domain() -> None: handler = TrueFalseResponseHandler(response_handler=JsonSchemaResponseHandler())