@W-23692110: Reconcile Node TCK exclusions

native-lib/node/tests/tck/accounting.ts

@@ -1,4 +1,4 @@
-export type TckOutcome = "passed" | "failed" | "skipped";
+export type TckOutcome = "passed" | "failed" | "skipped" | "xfailed";
 
 export interface TckResult {
   identifier: string;
@@ -10,6 +10,7 @@ export interface TckSummary {
   passed: number;
   failed: number;
   skipped: number;
+  xfailed: number;
   accounted: number;
   unaccounted: number;
 }
@@ -21,12 +22,14 @@ export function accountTckResults(
   const passed = results.filter((result) => result.outcome === "passed").length;
   const failed = results.filter((result) => result.outcome === "failed").length;
   const skipped = results.filter((result) => result.outcome === "skipped").length;
-  const accounted = passed + failed + skipped;
+  const xfailed = results.filter((result) => result.outcome === "xfailed").length;
+  const accounted = passed + failed + skipped + xfailed;
   return {
     selected: selected.size,
     passed,
     failed,
     skipped,
+    xfailed,
     accounted,
     unaccounted: selected.size - accounted,
   };

native-lib/node/tests/tck/ignore-list.ts

@@ -37,6 +37,8 @@ export interface IgnoreEntry {
   reason: string;
 }
 
+export type ExpectedFailurePolicy = Readonly<Record<string, string>>;
+
 const EXPECTED_RUNNABLE_CASES = 729;
 const EXPECTED_STRUCTURAL_SKIPS = 193;
 
@@ -145,7 +147,7 @@ function categoryFor(reason: string): IgnoreCategory {
   return "runtime-baseline-mismatch";
 }
 
-export const IGNORED_CASES: Readonly<Record<string, IgnoreEntry>> = Object.fromEntries(
+const LEGACY_POLICY: Readonly<Record<string, IgnoreEntry>> = Object.fromEntries(
   Object.entries(LEGACY_IGNORED_CASES).map(([caseName, entry]) => {
     const suite = CORE_MODULE_CASES.has(caseName) ? "core-modules" : "runtime";
     const caseIdentifier = `${suite}/${caseName}`;
@@ -157,6 +159,49 @@ export const IGNORED_CASES: Readonly<Record<string, IgnoreEntry>> = Object.fromE
   })
 );
 
+export const ACCEPTED_BASELINE_MISMATCHES: ExpectedFailurePolicy = {
+  "core-modules/csv-invalid-utf8-out.csv:out.csv": "runtime emits a replacement character where the fixture expects an empty CSV value",
+  "core-modules/multipart-binary-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture",
+  "core-modules/multipart-class-cast-issue-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture",
+  "core-modules/multipart-empty-part-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture",
+  "core-modules/multipart-mixed-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture",
+  "core-modules/multipart-write-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture",
+  "core-modules/multipart-write-subtype-override-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture",
+  "core-modules/properties-passthrough-out.properties:out.properties": "properties writer output differs from the baseline fixture",
+  "core-modules/xml-escaped-data-out.xml:out.xml": "XML character escaping differs from the baseline fixture",
+  "core-modules/xml-streaming-selectors-out.xml:out.xml": "streaming XML serialization differs from the baseline fixture",
+  "core-modules/xml-value-selector-out.xml:out.xml": "XML namespace scoping differs from the baseline fixture",
+  "core-modules/xml_empty_namespace-out.xml:out.xml": "empty XML namespace serialization differs from the baseline fixture",
+  "runtime/access_raw_value-out.json:out.json": "runtime coercion output differs from the baseline fixture",
+  "runtime/coerciones_toString-out.json:out.json": "locale-sensitive runtime output differs from the baseline fixture",
+  "runtime/properties-writer-out.properties:out.properties": "properties writer output differs from the baseline fixture",
+  "runtime/read-concat-out.json:out.json": "runtime coercion output differs from the baseline fixture",
+  "runtime/runtime_dataFormatsDescriptors-out.json:out.json": "dw::Runtime output differs from the baseline fixture",
+  "runtime/runtime_orElseTry-out.json:out.json": "source-location runtime output differs from the baseline fixture",
+  "runtime/runtime_run-out.json:out.json": "dw::Runtime output differs from the baseline fixture",
+  "runtime/try-recursive-call-out.json:out.json": "source-location runtime output differs from the baseline fixture",
+  "runtime/update-op-out.dwl:out.dwl": "runtime coercion output differs from the baseline fixture",
+};
+
+export const REENABLED_CASES = [
+  "runtime/big_intersection-out.json",
+  "runtime/dates_atBeginningOfDay-out.json",
+  "runtime/dates_atBeginningOfMonth-out.json",
+  "runtime/dates_atBeginningOfWeek-out.json",
+  "runtime/dates_atBeginningOfYear-out.json",
+  "runtime/multi_attribute_selector_after_empty_filter_slot-out.json",
+  "runtime/repeated_attribute_selector_map_slot_permutations-out.json",
+] as const;
+
+export const CAPABILITY_EXCLUSIONS = Object.fromEntries(
+  Object.entries(LEGACY_POLICY).filter(([identifier]) =>
+    !Object.keys(ACCEPTED_BASELINE_MISMATCHES).some((scenario) => scenario.startsWith(`${identifier}:`))
+      && !REENABLED_CASES.includes(identifier as typeof REENABLED_CASES[number])
+  )
+);
+
+export const IGNORED_CASES = CAPABILITY_EXCLUSIONS;
+
 export const STRUCTURAL_MODULE_CASES = new Set([
   "runtime/implicit_type_parameters-out.json",
   "runtime/import_mapping-out.json",
@@ -207,6 +252,47 @@ export function validateInventoryPolicy(runnableCases: number, structuralSkips:
   return errors;
 }
 
+export function validateReconciledPolicy(
+  exclusions: Readonly<Record<string, IgnoreEntry>>,
+  expectedFailures: ExpectedFailurePolicy,
+  reenabledCases: readonly string[] = [],
+  runnableScenarios?: ReadonlySet<string>,
+): string[] {
+  const errors: string[] = [];
+  const reenabledCounts = new Map<string, number>();
+  for (const identifier of reenabledCases) {
+    reenabledCounts.set(identifier, (reenabledCounts.get(identifier) ?? 0) + 1);
+  }
+  for (const identifier of reenabledCounts.keys()) {
+    if (Object.prototype.hasOwnProperty.call(exclusions, identifier)) {
+      errors.push(`${identifier}: case is both skipped and re-enabled`);
+    }
+    if (runnableScenarios && ![...runnableScenarios].some((scenario) => scenario.startsWith(`${identifier}:`))) {
+      errors.push(`${identifier}: not a discovered runnable case`);
+    }
+  }
+  errors.push(...[...reenabledCounts]
+    .filter(([, count]) => count > 1)
+    .map(([identifier]) => `${identifier}: duplicate re-enabled case`));
+  const expectedFailureCases = new Set(
+    Object.keys(expectedFailures).map((identifier) => identifier.slice(0, identifier.lastIndexOf(":")))
+  );
+  errors.push(...[...reenabledCounts.keys()]
+    .filter((identifier) => expectedFailureCases.has(identifier))
+    .map((identifier) => `${identifier}: case is both re-enabled and expected to fail`));
+  for (const [identifier, reason] of Object.entries(expectedFailures)) {
+    if (!reason.trim()) errors.push(`${identifier}: missing expected-failure reason`);
+    if (runnableScenarios && !runnableScenarios.has(identifier)) {
+      errors.push(`${identifier}: not a discovered runnable scenario`);
+    }
+    const caseIdentifier = identifier.slice(0, identifier.lastIndexOf(":"));
+    if (Object.prototype.hasOwnProperty.call(exclusions, caseIdentifier)) {
+      errors.push(`${identifier}: case is both skipped and expected to fail`);
+    }
+  }
+  return errors.sort();
+}
+
 export function validateStructuralModulePolicy(
   entries: ReadonlySet<string>,
   structuralModuleCases: ReadonlySet<string>

native-lib/node/tests/tck/reporter.ts

@@ -8,6 +8,10 @@ function isTckCase(testCase: TestCase): boolean {
   return testCase.module.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE);
 }
 
+function scenarioIdentifier(name: string): string {
+  return name.split(/ \[(?:skip|xfail):/, 1)[0];
+}
+
 export function matchesTestNamePattern(name: string, pattern: RegExp | undefined): boolean {
   if (!pattern) return true;
   pattern.lastIndex = 0;
@@ -23,10 +27,15 @@ export class TckAccountingReporter implements Reporter {
     this.testNamePattern = vitest.config.testNamePattern;
   }
 
+  onTestRunStart(): void {
+    this.selected.clear();
+    this.results.length = 0;
+  }
+
   onTestModuleCollected(testModule: TestModule): void {
     if (!testModule.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE)) return;
     for (const testCase of testModule.children.allTests()) {
-      const identifier = testCase.name.split(" [skip:", 1)[0];
+      const identifier = scenarioIdentifier(testCase.name);
       const isPolicySkip = testCase.options.mode === "skip" && testCase.name.includes(" [skip:");
       if (isPolicySkip && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) {
         this.selected.add(identifier);
@@ -36,19 +45,19 @@ export class TckAccountingReporter implements Reporter {
 
   onTestCaseReady(testCase: TestCase): void {
     if (isTckCase(testCase) && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) {
-      this.selected.add(testCase.name.split(" [skip:", 1)[0]);
+      this.selected.add(scenarioIdentifier(testCase.name));
     }
   }
 
   onTestCaseResult(testCase: TestCase): void {
     if (!isTckCase(testCase)) return;
-    const identifier = testCase.name.split(" [skip:", 1)[0];
+    const identifier = scenarioIdentifier(testCase.name);
     if (!this.selected.has(identifier) && testCase.result().state === "skipped") return;
     const state = testCase.result().state;
     if (state === "pending") return;
     this.results.push({
       identifier,
-      outcome: state,
+      outcome: state === "passed" && testCase.name.includes(" [xfail:") ? "xfailed" : state,
     });
   }
 
@@ -57,7 +66,8 @@ export class TckAccountingReporter implements Reporter {
     const summary = accountTckResults(this.selected, this.results);
     console.log(
       `TCK totals: selected=${summary.selected}, passed=${summary.passed}, failed=${summary.failed}, `
-      + `skipped=${summary.skipped}, accounted=${summary.accounted}, unaccounted=${summary.unaccounted}`
+      + `skipped=${summary.skipped}, xfailed=${summary.xfailed}, `
+      + `accounted=${summary.accounted}, unaccounted=${summary.unaccounted}`
     );
     const errors = validateTckAccounting(this.selected, this.results);
     if (errors.length > 0) {

native-lib/node/tests/tck/tck.test.ts

@@ -13,12 +13,15 @@ import { DataWeave, modulesFromDirectory } from "../../src/index";
 import { hasAdjacentDwlModule, parseCase, MAIN_TRANSFORM, type TckScenario } from "./case-loader";
 import { compareOutput } from "./compare";
 import {
+  ACCEPTED_BASELINE_MISMATCHES,
   IGNORED_CASES,
+  REENABLED_CASES,
   STRUCTURAL_MODULE_CASES,
   isIgnored,
   ignoreReason,
   validateIgnorePolicy,
   validateInventoryPolicy,
+  validateReconciledPolicy,
   validateStructuralModulePolicy,
 } from "./ignore-list";
 
@@ -78,9 +81,11 @@ if (!existsSync(SUITES_DIR)) {
 } else {
   const { cases, skipped, structuralModuleCases } = discoverCases();
   const runnableCases = new Set(cases.map((item) => item.caseIdentifier));
+  const runnableScenarios = new Set(cases.flatMap((item) => item.scenarios.map((scenario) => scenario.name)));
   const policyErrors = [
     ...validateInventoryPolicy(cases.length, skipped),
     ...validateIgnorePolicy(IGNORED_CASES, runnableCases),
+    ...validateReconciledPolicy(IGNORED_CASES, ACCEPTED_BASELINE_MISMATCHES, REENABLED_CASES, runnableScenarios),
     ...validateStructuralModulePolicy(STRUCTURAL_MODULE_CASES, structuralModuleCases),
   ];
   if (policyErrors.length > 0) {
@@ -97,15 +102,21 @@ if (!existsSync(SUITES_DIR)) {
     // eslint-disable-next-line no-console
     console.log(
       `TCK: ${cases.length} runnable cases, ${skipped} structurally skipped, `
-      + `${structuralModuleCases.size} structural module cases, ${Object.keys(IGNORED_CASES).length} exclusions`
+      + `${structuralModuleCases.size} structural module cases, ${Object.keys(IGNORED_CASES).length} exclusions, `
+      + `${Object.keys(ACCEPTED_BASELINE_MISMATCHES).length} expected failures`
     );
     dw.initialize();
 
     for (const c of cases) {
       const ignored = isIgnored(c.caseIdentifier);
       for (const scenario of c.scenarios) {
-        const testFn = ignored ? it.skip : it;
-        const label = ignored ? `${scenario.name} [skip: ${ignoreReason(c.caseIdentifier)}]` : scenario.name;
+        const expectedFailure = ACCEPTED_BASELINE_MISMATCHES[scenario.name];
+        const testFn = ignored ? it.skip : expectedFailure ? it.fails : it;
+        const label = ignored
+          ? `${scenario.name} [skip: ${ignoreReason(c.caseIdentifier)}]`
+          : expectedFailure
+            ? `${scenario.name} [xfail: ${expectedFailure}]`
+            : scenario.name;
         testFn(label, () => {
           const script = readFileSync(join(c.dir, MAIN_TRANSFORM), "utf-8");
 

native-lib/node/tests/unit/tck-accounting.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from "vitest";
 import { accountTckResults, validateTckAccounting } from "../../tests/tck/accounting";
-import { matchesTestNamePattern } from "../../tests/tck/reporter";
+import { matchesTestNamePattern, TckAccountingReporter } from "../../tests/tck/reporter";
 
 describe("TCK result accounting", () => {
   it("reports each selected outcome exactly once", () => {
@@ -18,11 +18,27 @@ describe("TCK result accounting", () => {
       passed: 1,
       failed: 1,
       skipped: 1,
+      xfailed: 0,
       accounted: 3,
       unaccounted: 0,
     });
   });
 
+  it("reports expected failures separately from ordinary failures", () => {
+    expect(accountTckResults(
+      new Set(["runtime/xfail:out.json"]),
+      [{ identifier: "runtime/xfail:out.json", outcome: "xfailed" }],
+    )).toEqual({
+      selected: 1,
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      xfailed: 1,
+      accounted: 1,
+      unaccounted: 0,
+    });
+  });
+
   it("rejects missing, duplicate, and unexpected results", () => {
     expect(validateTckAccounting(
       new Set(["runtime/a:out.json", "runtime/b:out.json"]),
@@ -52,4 +68,19 @@ describe("TCK result accounting", () => {
     expect(matchesTestNamePattern("runtime/other:out.json [skip: unsupported]", pattern)).toBe(false);
     expect(matchesTestNamePattern("runtime/other:out.json [skip: unsupported]", undefined)).toBe(true);
   });
+
+  it("clears accumulated accounting state before a watch rerun", () => {
+    const reporter = new TckAccountingReporter();
+    const state = reporter as unknown as {
+      selected: Set<string>;
+      results: Array<{ identifier: string; outcome: "passed" }>;
+    };
+    state.selected.add("runtime/first:out.json");
+    state.results.push({ identifier: "runtime/first:out.json", outcome: "passed" });
+
+    reporter.onTestRunStart?.([]);
+
+    expect(state.selected.size).toBe(0);
+    expect(state.results).toEqual([]);
+  });
 });

native-lib/node/tests/unit/tck-policy.test.ts

@@ -1,9 +1,13 @@
 import { describe, expect, it } from "vitest";
 import {
+  ACCEPTED_BASELINE_MISMATCHES,
+  CAPABILITY_EXCLUSIONS,
   IGNORED_CASES,
+  REENABLED_CASES,
   STRUCTURAL_MODULE_CASES,
   validateIgnorePolicy,
   validateInventoryPolicy,
+  validateReconciledPolicy,
   validateStructuralModulePolicy,
 } from "../../tests/tck/ignore-list";
 
@@ -39,8 +43,45 @@ describe("TCK ignore policy", () => {
     ]);
   });
 
-  it("preserves the current 59 exclusions", () => {
-    expect(Object.keys(IGNORED_CASES)).toHaveLength(59);
+  it("reconciles exclusions into capability skips and strict xfails", () => {
+    expect(Object.keys(CAPABILITY_EXCLUSIONS)).toHaveLength(31);
+    expect(Object.keys(ACCEPTED_BASELINE_MISMATCHES)).toHaveLength(21);
+    expect(REENABLED_CASES).toHaveLength(7);
+    expect(IGNORED_CASES).toBe(CAPABILITY_EXCLUSIONS);
+    expect(validateReconciledPolicy(
+      CAPABILITY_EXCLUSIONS,
+      ACCEPTED_BASELINE_MISMATCHES,
+      REENABLED_CASES,
+    )).toEqual([]);
+  });
+
+  it("rejects overlapping or incomplete reconciled policy entries", () => {
+    expect(validateReconciledPolicy(
+      {
+        "runtime/overlap": {
+          caseIdentifier: "runtime/overlap",
+          category: "unavailable-java-module",
+          reason: "missing Java module",
+        },
+      },
+      {
+        "runtime/overlap:out.json": "known mismatch",
+        "runtime/blank:out.json": " ",
+        "runtime/xfail:out.json": "known mismatch",
+        "runtime/stale:out.json": "stale mismatch",
+      },
+      ["runtime/overlap", "runtime/reenabled", "runtime/reenabled", "runtime/xfail"],
+      new Set(["runtime/overlap:out.json", "runtime/blank:out.json", "runtime/xfail:out.json"]),
+    )).toEqual([
+      "runtime/blank:out.json: missing expected-failure reason",
+      "runtime/overlap: case is both re-enabled and expected to fail",
+      "runtime/overlap: case is both skipped and re-enabled",
+      "runtime/overlap:out.json: case is both skipped and expected to fail",
+      "runtime/reenabled: duplicate re-enabled case",
+      "runtime/reenabled: not a discovered runnable case",
+      "runtime/stale:out.json: not a discovered runnable scenario",
+      "runtime/xfail: case is both re-enabled and expected to fail",
+    ]);
   });
 
   it("rejects drift in the staged suite inventory", () => {