fix(framework): pass eval case fields to scorers

py/src/braintrust/framework.py

@@ -211,13 +211,16 @@ def parameters(self) -> ValidatedParameters | None:
 
 class EvalScorerArgs(SerializableDataClass, Generic[Input, Output, Expected]):
     """
-    Arguments passed to an evaluator scorer. This includes the input, expected output, actual output, and metadata.
+    Arguments passed to an evaluator scorer. This includes the input, expected output, actual output, metadata,
+    tags, and evaluation case ID.
     """
 
     input: Input
     output: Output
     expected: Expected | None = None
     metadata: Metadata | None = None
+    id: str | None = None
+    tags: Sequence[str] | None = None
 
 
 OneOrMoreScores = float | int | bool | None | ScoreLike | Sequence[ScoreLike]
@@ -1748,6 +1751,8 @@ async def ensure_spans_flushed():
                     "metadata": metadata,
                     "output": output,
                     "trace": trace,
+                    "id": datum.id,
+                    "tags": tags,
                 }
                 score_promises = [
                     asyncio.create_task(await_or_run_scorer(root_span, score, name, **scorer_kwargs))

py/src/braintrust/test_framework.py

@@ -226,6 +226,30 @@ def exact_match(input_value, output, expected):
     assert result.summary.scores["exact_match"].score == 1.0
 
 
+@pytest.mark.asyncio
+async def test_eval_case_id_and_tags_are_passed_to_scorers():
+    scorer_args = None
+
+    def scorer(input_value, output, expected, *, id=None, tags=None):
+        nonlocal scorer_args
+        scorer_args = {"id": id, "tags": tags}
+        return 1
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-evaluator",
+        data=[EvalCase(id="dataset-row-id", input=1, expected=2, tags=["dataset-tag"])],
+        task=lambda input_value: input_value * 2,
+        scores=[scorer],
+        experiment_name=None,
+        metadata=None,
+    )
+
+    await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
+
+    assert scorer_args == {"id": "dataset-row-id", "tags": ["dataset-tag"]}
+
+
 @pytest.mark.asyncio
 async def test_run_evaluator_forwards_base_experiment_id_to_summary(with_memory_logger, with_simulate_login):
     def exact_match(input_value, output, expected):