fix(eval): match JUnit names ignoring optional eval. prefix

src/programbench/eval/eval.py

@@ -122,6 +122,17 @@ def count_testcases(raw_xml: str) -> int:
     return sum(1 for _ in root.iter("testcase"))
 
 
+def _canonical_test_name(name: str) -> str:
+    """Strip an optional leading ``eval.`` package root from a test name.
+
+    ``tests.json`` is inconsistent about the package root: some tasks store
+    ``eval.tests.foo`` (pytest's module path under /workspace/eval) while
+    others store ``tests.foo``. JUnit classnames are always rooted at
+    ``eval.tests.foo``, so both sides are compared in this canonical form.
+    """
+    return name[5:] if name.startswith("eval.") else name
+
+
 def _process_branch_xml(
     raw_xml: str,
     branch: str,
@@ -152,10 +163,12 @@ def _process_branch_xml(
         warnings.append(f"{tag}: no expected test list, cannot verify completeness")
         return results, warnings
 
-    ignored_names = {n.split("/", 1)[1] for n in (ignored_tests or set()) if n.startswith(f"{branch}/")}
-    expected_active = [n for n in expected if n not in ignored_names]
-    got = {t.name for t in parsed}
-    missing = [name for name in expected_active if name not in got]
+    ignored_names = {
+        _canonical_test_name(n.split("/", 1)[1]) for n in (ignored_tests or set()) if n.startswith(f"{branch}/")
+    }
+    expected_active = [n for n in expected if _canonical_test_name(n) not in ignored_names]
+    got = {_canonical_test_name(t.name) for t in parsed}
+    missing = [name for name in expected_active if _canonical_test_name(name) not in got]
     if missing:
         log.warning(
             "%s: %d/%d expected tests missing from JUnit XML",
@@ -172,7 +185,7 @@ def _process_branch_xml(
             )
             for name in missing
         )
-    unexpected = got - set(expected) - ignored_names
+    unexpected = got - {_canonical_test_name(n) for n in expected} - ignored_names
     if unexpected:
         log.warning(
             "%s: %d test(s) in JUnit XML not in tests.json",

src/programbench/eval/eval_batch.py

@@ -32,7 +32,7 @@
 from tqdm.contrib.logging import logging_redirect_tqdm
 
 from programbench.constants import DOCKER_CPUS
-from programbench.eval.eval import EvaluationResult, Evaluator
+from programbench.eval.eval import EvaluationResult, Evaluator, _canonical_test_name
 from programbench.utils.instance_filters import filter_instances
 
 log = logging.getLogger(__name__)
@@ -179,8 +179,10 @@ def get_branches_to_eval(
             continue
         expected = tests_by_branch.get(branch, [])
         active_expected = [t for t in expected if f"{branch}/{t}" not in ignored_tests]
-        present = {t.name for t in existing.test_results if t.branch == branch and t.status != "not_run"}
-        if any(t not in present for t in active_expected):
+        present = {
+            _canonical_test_name(t.name) for t in existing.test_results if t.branch == branch and t.status != "not_run"
+        }
+        if any(_canonical_test_name(t) not in present for t in active_expected):
             needs_eval.append(branch)
     return needs_eval
 

tests/test_eval.py

@@ -39,6 +39,16 @@
 </testsuites>
 """
 
+JUNIT_XML_EVAL_PREFIX = """\
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+  <testsuite name="pytest" errors="0" failures="0" skipped="0" tests="2">
+    <testcase classname="eval.tests.test_calculator" name="test_addition" time="0.01"/>
+    <testcase classname="eval.tests.test_calculator" name="test_subtraction" time="0.02"/>
+  </testsuite>
+</testsuites>
+"""
+
 JUNIT_XML_MIXED = """\
 <?xml version="1.0" encoding="utf-8"?>
 <testsuites>
@@ -138,6 +148,31 @@ def test_unexpected_tests_warn(self):
         results, warnings = _process_branch_xml(JUNIT_XML_ALL_PASS, "b1", tests_by_branch)
         assert any("not in tests.json" in w for w in warnings)
 
+    def test_eval_prefix_namespace_mismatch_matches(self):
+        tests_by_branch = {
+            "b1": [
+                "tests.test_calculator.test_addition",
+                "tests.test_calculator.test_subtraction",
+            ]
+        }
+        results, warnings = _process_branch_xml(JUNIT_XML_EVAL_PREFIX, "b1", tests_by_branch)
+        assert len(results) == 2
+        assert all(r.status == "passed" for r in results)
+        assert not any(r.status == "not_run" for r in results)
+        assert not any("not in tests.json" in w for w in warnings)
+
+    def test_eval_prefix_missing_still_not_run(self):
+        tests_by_branch = {
+            "b1": [
+                "tests.test_calculator.test_addition",
+                "tests.test_calculator.test_missing",
+            ]
+        }
+        results, warnings = _process_branch_xml(JUNIT_XML_EVAL_PREFIX, "b1", tests_by_branch)
+        by_name = {r.name: r for r in results}
+        assert by_name["tests.test_calculator.test_missing"].status == "not_run"
+        assert by_name["eval.tests.test_calculator.test_addition"].status == "passed"
+
 
 class TestCountWorkerCrashes:
     XDIST_CRASH_XML = """\
@@ -269,6 +304,25 @@ def test_branch_with_error_needs_reeval(self, tmp_path):
             ignored_tests=set(),
         ) == ["b1"]
 
+    def test_namespace_mismatch_considered_evaluated(self, tmp_path):
+        eval_json = tmp_path / "eval.json"
+        result = EvaluationResult(
+            test_results=[
+                TestResult(name="eval.tests.t1.test_a", branch="b1", status="passed", extra={}),
+            ],
+            test_branches=["b1"],
+        )
+        eval_json.write_text(result.model_dump_json())
+        assert (
+            get_branches_to_eval(
+                eval_json=eval_json,
+                all_test_branches=["b1"],
+                tests_by_branch={"b1": ["tests.t1.test_a"]},
+                ignored_tests=set(),
+            )
+            == []
+        )
+
 
 class TestInstanceEvalSummary:
     def test_from_eval_result(self):