diff --git a/src/programbench/eval/eval.py b/src/programbench/eval/eval.py
index 2f17b8cd..7a3f0463 100644
--- a/src/programbench/eval/eval.py
+++ b/src/programbench/eval/eval.py
@@ -122,6 +122,17 @@ def count_testcases(raw_xml: str) -> int:
return sum(1 for _ in root.iter("testcase"))
+def _canonical_test_name(name: str) -> str:
+ """Strip an optional leading ``eval.`` package root from a test name.
+
+ ``tests.json`` is inconsistent about the package root: some tasks store
+ ``eval.tests.foo`` (pytest's module path under /workspace/eval) while
+ others store ``tests.foo``. JUnit classnames are always rooted at
+ ``eval.tests.foo``, so both sides are compared in this canonical form.
+ """
+ return name[5:] if name.startswith("eval.") else name
+
+
def _process_branch_xml(
raw_xml: str,
branch: str,
@@ -152,10 +163,12 @@ def _process_branch_xml(
warnings.append(f"{tag}: no expected test list, cannot verify completeness")
return results, warnings
- ignored_names = {n.split("/", 1)[1] for n in (ignored_tests or set()) if n.startswith(f"{branch}/")}
- expected_active = [n for n in expected if n not in ignored_names]
- got = {t.name for t in parsed}
- missing = [name for name in expected_active if name not in got]
+ ignored_names = {
+ _canonical_test_name(n.split("/", 1)[1]) for n in (ignored_tests or set()) if n.startswith(f"{branch}/")
+ }
+ expected_active = [n for n in expected if _canonical_test_name(n) not in ignored_names]
+ got = {_canonical_test_name(t.name) for t in parsed}
+ missing = [name for name in expected_active if _canonical_test_name(name) not in got]
if missing:
log.warning(
"%s: %d/%d expected tests missing from JUnit XML",
@@ -172,7 +185,7 @@ def _process_branch_xml(
)
for name in missing
)
- unexpected = got - set(expected) - ignored_names
+ unexpected = got - {_canonical_test_name(n) for n in expected} - ignored_names
if unexpected:
log.warning(
"%s: %d test(s) in JUnit XML not in tests.json",
diff --git a/src/programbench/eval/eval_batch.py b/src/programbench/eval/eval_batch.py
index 5f412ceb..baf1165b 100644
--- a/src/programbench/eval/eval_batch.py
+++ b/src/programbench/eval/eval_batch.py
@@ -32,7 +32,7 @@
from tqdm.contrib.logging import logging_redirect_tqdm
from programbench.constants import DOCKER_CPUS
-from programbench.eval.eval import EvaluationResult, Evaluator
+from programbench.eval.eval import EvaluationResult, Evaluator, _canonical_test_name
from programbench.utils.instance_filters import filter_instances
log = logging.getLogger(__name__)
@@ -179,8 +179,10 @@ def get_branches_to_eval(
continue
expected = tests_by_branch.get(branch, [])
active_expected = [t for t in expected if f"{branch}/{t}" not in ignored_tests]
- present = {t.name for t in existing.test_results if t.branch == branch and t.status != "not_run"}
- if any(t not in present for t in active_expected):
+ present = {
+ _canonical_test_name(t.name) for t in existing.test_results if t.branch == branch and t.status != "not_run"
+ }
+ if any(_canonical_test_name(t) not in present for t in active_expected):
needs_eval.append(branch)
return needs_eval
diff --git a/tests/test_eval.py b/tests/test_eval.py
index dcb95f4e..4dbd61db 100644
--- a/tests/test_eval.py
+++ b/tests/test_eval.py
@@ -39,6 +39,16 @@
"""
+JUNIT_XML_EVAL_PREFIX = """\
+
+
+
+
+
+
+
+"""
+
JUNIT_XML_MIXED = """\
@@ -138,6 +148,31 @@ def test_unexpected_tests_warn(self):
results, warnings = _process_branch_xml(JUNIT_XML_ALL_PASS, "b1", tests_by_branch)
assert any("not in tests.json" in w for w in warnings)
+ def test_eval_prefix_namespace_mismatch_matches(self):
+ tests_by_branch = {
+ "b1": [
+ "tests.test_calculator.test_addition",
+ "tests.test_calculator.test_subtraction",
+ ]
+ }
+ results, warnings = _process_branch_xml(JUNIT_XML_EVAL_PREFIX, "b1", tests_by_branch)
+ assert len(results) == 2
+ assert all(r.status == "passed" for r in results)
+ assert not any(r.status == "not_run" for r in results)
+ assert not any("not in tests.json" in w for w in warnings)
+
+ def test_eval_prefix_missing_still_not_run(self):
+ tests_by_branch = {
+ "b1": [
+ "tests.test_calculator.test_addition",
+ "tests.test_calculator.test_missing",
+ ]
+ }
+ results, warnings = _process_branch_xml(JUNIT_XML_EVAL_PREFIX, "b1", tests_by_branch)
+ by_name = {r.name: r for r in results}
+ assert by_name["tests.test_calculator.test_missing"].status == "not_run"
+ assert by_name["eval.tests.test_calculator.test_addition"].status == "passed"
+
class TestCountWorkerCrashes:
XDIST_CRASH_XML = """\
@@ -269,6 +304,25 @@ def test_branch_with_error_needs_reeval(self, tmp_path):
ignored_tests=set(),
) == ["b1"]
+ def test_namespace_mismatch_considered_evaluated(self, tmp_path):
+ eval_json = tmp_path / "eval.json"
+ result = EvaluationResult(
+ test_results=[
+ TestResult(name="eval.tests.t1.test_a", branch="b1", status="passed", extra={}),
+ ],
+ test_branches=["b1"],
+ )
+ eval_json.write_text(result.model_dump_json())
+ assert (
+ get_branches_to_eval(
+ eval_json=eval_json,
+ all_test_branches=["b1"],
+ tests_by_branch={"b1": ["tests.t1.test_a"]},
+ ignored_tests=set(),
+ )
+ == []
+ )
+
class TestInstanceEvalSummary:
def test_from_eval_result(self):