diff --git a/native-lib/node/tests/tck/accounting.ts b/native-lib/node/tests/tck/accounting.ts index 073386f6..e73942f1 100644 --- a/native-lib/node/tests/tck/accounting.ts +++ b/native-lib/node/tests/tck/accounting.ts @@ -1,4 +1,4 @@ -export type TckOutcome = "passed" | "failed" | "skipped"; +export type TckOutcome = "passed" | "failed" | "skipped" | "xfailed"; export interface TckResult { identifier: string; @@ -10,6 +10,7 @@ export interface TckSummary { passed: number; failed: number; skipped: number; + xfailed: number; accounted: number; unaccounted: number; } @@ -21,12 +22,14 @@ export function accountTckResults( const passed = results.filter((result) => result.outcome === "passed").length; const failed = results.filter((result) => result.outcome === "failed").length; const skipped = results.filter((result) => result.outcome === "skipped").length; - const accounted = passed + failed + skipped; + const xfailed = results.filter((result) => result.outcome === "xfailed").length; + const accounted = passed + failed + skipped + xfailed; return { selected: selected.size, passed, failed, skipped, + xfailed, accounted, unaccounted: selected.size - accounted, }; diff --git a/native-lib/node/tests/tck/ignore-list.ts b/native-lib/node/tests/tck/ignore-list.ts index 6bf200a5..5afd03c0 100644 --- a/native-lib/node/tests/tck/ignore-list.ts +++ b/native-lib/node/tests/tck/ignore-list.ts @@ -37,6 +37,8 @@ export interface IgnoreEntry { reason: string; } +export type ExpectedFailurePolicy = Readonly>; + const EXPECTED_RUNNABLE_CASES = 729; const EXPECTED_STRUCTURAL_SKIPS = 193; @@ -145,7 +147,7 @@ function categoryFor(reason: string): IgnoreCategory { return "runtime-baseline-mismatch"; } -export const IGNORED_CASES: Readonly> = Object.fromEntries( +const LEGACY_POLICY: Readonly> = Object.fromEntries( Object.entries(LEGACY_IGNORED_CASES).map(([caseName, entry]) => { const suite = CORE_MODULE_CASES.has(caseName) ? "core-modules" : "runtime"; const caseIdentifier = `${suite}/${caseName}`; @@ -157,6 +159,49 @@ export const IGNORED_CASES: Readonly> = Object.fromE }) ); +export const ACCEPTED_BASELINE_MISMATCHES: ExpectedFailurePolicy = { + "core-modules/csv-invalid-utf8-out.csv:out.csv": "runtime emits a replacement character where the fixture expects an empty CSV value", + "core-modules/multipart-binary-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-class-cast-issue-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-empty-part-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-mixed-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-write-message-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/multipart-write-subtype-override-out.multipart:out.multipart": "multipart writer output differs from the baseline fixture", + "core-modules/properties-passthrough-out.properties:out.properties": "properties writer output differs from the baseline fixture", + "core-modules/xml-escaped-data-out.xml:out.xml": "XML character escaping differs from the baseline fixture", + "core-modules/xml-streaming-selectors-out.xml:out.xml": "streaming XML serialization differs from the baseline fixture", + "core-modules/xml-value-selector-out.xml:out.xml": "XML namespace scoping differs from the baseline fixture", + "core-modules/xml_empty_namespace-out.xml:out.xml": "empty XML namespace serialization differs from the baseline fixture", + "runtime/access_raw_value-out.json:out.json": "runtime coercion output differs from the baseline fixture", + "runtime/coerciones_toString-out.json:out.json": "locale-sensitive runtime output differs from the baseline fixture", + "runtime/properties-writer-out.properties:out.properties": "properties writer output differs from the baseline fixture", + "runtime/read-concat-out.json:out.json": "runtime coercion output differs from the baseline fixture", + "runtime/runtime_dataFormatsDescriptors-out.json:out.json": "dw::Runtime output differs from the baseline fixture", + "runtime/runtime_orElseTry-out.json:out.json": "source-location runtime output differs from the baseline fixture", + "runtime/runtime_run-out.json:out.json": "dw::Runtime output differs from the baseline fixture", + "runtime/try-recursive-call-out.json:out.json": "source-location runtime output differs from the baseline fixture", + "runtime/update-op-out.dwl:out.dwl": "runtime coercion output differs from the baseline fixture", +}; + +export const REENABLED_CASES = [ + "runtime/big_intersection-out.json", + "runtime/dates_atBeginningOfDay-out.json", + "runtime/dates_atBeginningOfMonth-out.json", + "runtime/dates_atBeginningOfWeek-out.json", + "runtime/dates_atBeginningOfYear-out.json", + "runtime/multi_attribute_selector_after_empty_filter_slot-out.json", + "runtime/repeated_attribute_selector_map_slot_permutations-out.json", +] as const; + +export const CAPABILITY_EXCLUSIONS = Object.fromEntries( + Object.entries(LEGACY_POLICY).filter(([identifier]) => + !Object.keys(ACCEPTED_BASELINE_MISMATCHES).some((scenario) => scenario.startsWith(`${identifier}:`)) + && !REENABLED_CASES.includes(identifier as typeof REENABLED_CASES[number]) + ) +); + +export const IGNORED_CASES = CAPABILITY_EXCLUSIONS; + export const STRUCTURAL_MODULE_CASES = new Set([ "runtime/implicit_type_parameters-out.json", "runtime/import_mapping-out.json", @@ -207,6 +252,47 @@ export function validateInventoryPolicy(runnableCases: number, structuralSkips: return errors; } +export function validateReconciledPolicy( + exclusions: Readonly>, + expectedFailures: ExpectedFailurePolicy, + reenabledCases: readonly string[] = [], + runnableScenarios?: ReadonlySet, +): string[] { + const errors: string[] = []; + const reenabledCounts = new Map(); + for (const identifier of reenabledCases) { + reenabledCounts.set(identifier, (reenabledCounts.get(identifier) ?? 0) + 1); + } + for (const identifier of reenabledCounts.keys()) { + if (Object.prototype.hasOwnProperty.call(exclusions, identifier)) { + errors.push(`${identifier}: case is both skipped and re-enabled`); + } + if (runnableScenarios && ![...runnableScenarios].some((scenario) => scenario.startsWith(`${identifier}:`))) { + errors.push(`${identifier}: not a discovered runnable case`); + } + } + errors.push(...[...reenabledCounts] + .filter(([, count]) => count > 1) + .map(([identifier]) => `${identifier}: duplicate re-enabled case`)); + const expectedFailureCases = new Set( + Object.keys(expectedFailures).map((identifier) => identifier.slice(0, identifier.lastIndexOf(":"))) + ); + errors.push(...[...reenabledCounts.keys()] + .filter((identifier) => expectedFailureCases.has(identifier)) + .map((identifier) => `${identifier}: case is both re-enabled and expected to fail`)); + for (const [identifier, reason] of Object.entries(expectedFailures)) { + if (!reason.trim()) errors.push(`${identifier}: missing expected-failure reason`); + if (runnableScenarios && !runnableScenarios.has(identifier)) { + errors.push(`${identifier}: not a discovered runnable scenario`); + } + const caseIdentifier = identifier.slice(0, identifier.lastIndexOf(":")); + if (Object.prototype.hasOwnProperty.call(exclusions, caseIdentifier)) { + errors.push(`${identifier}: case is both skipped and expected to fail`); + } + } + return errors.sort(); +} + export function validateStructuralModulePolicy( entries: ReadonlySet, structuralModuleCases: ReadonlySet diff --git a/native-lib/node/tests/tck/reporter.ts b/native-lib/node/tests/tck/reporter.ts index 4af0e58c..e7f44309 100644 --- a/native-lib/node/tests/tck/reporter.ts +++ b/native-lib/node/tests/tck/reporter.ts @@ -8,6 +8,10 @@ function isTckCase(testCase: TestCase): boolean { return testCase.module.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE); } +function scenarioIdentifier(name: string): string { + return name.split(/ \[(?:skip|xfail):/, 1)[0]; +} + export function matchesTestNamePattern(name: string, pattern: RegExp | undefined): boolean { if (!pattern) return true; pattern.lastIndex = 0; @@ -23,10 +27,15 @@ export class TckAccountingReporter implements Reporter { this.testNamePattern = vitest.config.testNamePattern; } + onTestRunStart(): void { + this.selected.clear(); + this.results.length = 0; + } + onTestModuleCollected(testModule: TestModule): void { if (!testModule.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE)) return; for (const testCase of testModule.children.allTests()) { - const identifier = testCase.name.split(" [skip:", 1)[0]; + const identifier = scenarioIdentifier(testCase.name); const isPolicySkip = testCase.options.mode === "skip" && testCase.name.includes(" [skip:"); if (isPolicySkip && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) { this.selected.add(identifier); @@ -36,19 +45,19 @@ export class TckAccountingReporter implements Reporter { onTestCaseReady(testCase: TestCase): void { if (isTckCase(testCase) && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) { - this.selected.add(testCase.name.split(" [skip:", 1)[0]); + this.selected.add(scenarioIdentifier(testCase.name)); } } onTestCaseResult(testCase: TestCase): void { if (!isTckCase(testCase)) return; - const identifier = testCase.name.split(" [skip:", 1)[0]; + const identifier = scenarioIdentifier(testCase.name); if (!this.selected.has(identifier) && testCase.result().state === "skipped") return; const state = testCase.result().state; if (state === "pending") return; this.results.push({ identifier, - outcome: state, + outcome: state === "passed" && testCase.name.includes(" [xfail:") ? "xfailed" : state, }); } @@ -57,7 +66,8 @@ export class TckAccountingReporter implements Reporter { const summary = accountTckResults(this.selected, this.results); console.log( `TCK totals: selected=${summary.selected}, passed=${summary.passed}, failed=${summary.failed}, ` - + `skipped=${summary.skipped}, accounted=${summary.accounted}, unaccounted=${summary.unaccounted}` + + `skipped=${summary.skipped}, xfailed=${summary.xfailed}, ` + + `accounted=${summary.accounted}, unaccounted=${summary.unaccounted}` ); const errors = validateTckAccounting(this.selected, this.results); if (errors.length > 0) { diff --git a/native-lib/node/tests/tck/tck.test.ts b/native-lib/node/tests/tck/tck.test.ts index d8a16334..9b4bee56 100644 --- a/native-lib/node/tests/tck/tck.test.ts +++ b/native-lib/node/tests/tck/tck.test.ts @@ -13,12 +13,15 @@ import { DataWeave, modulesFromDirectory } from "../../src/index"; import { hasAdjacentDwlModule, parseCase, MAIN_TRANSFORM, type TckScenario } from "./case-loader"; import { compareOutput } from "./compare"; import { + ACCEPTED_BASELINE_MISMATCHES, IGNORED_CASES, + REENABLED_CASES, STRUCTURAL_MODULE_CASES, isIgnored, ignoreReason, validateIgnorePolicy, validateInventoryPolicy, + validateReconciledPolicy, validateStructuralModulePolicy, } from "./ignore-list"; @@ -78,9 +81,11 @@ if (!existsSync(SUITES_DIR)) { } else { const { cases, skipped, structuralModuleCases } = discoverCases(); const runnableCases = new Set(cases.map((item) => item.caseIdentifier)); + const runnableScenarios = new Set(cases.flatMap((item) => item.scenarios.map((scenario) => scenario.name))); const policyErrors = [ ...validateInventoryPolicy(cases.length, skipped), ...validateIgnorePolicy(IGNORED_CASES, runnableCases), + ...validateReconciledPolicy(IGNORED_CASES, ACCEPTED_BASELINE_MISMATCHES, REENABLED_CASES, runnableScenarios), ...validateStructuralModulePolicy(STRUCTURAL_MODULE_CASES, structuralModuleCases), ]; if (policyErrors.length > 0) { @@ -97,15 +102,21 @@ if (!existsSync(SUITES_DIR)) { // eslint-disable-next-line no-console console.log( `TCK: ${cases.length} runnable cases, ${skipped} structurally skipped, ` - + `${structuralModuleCases.size} structural module cases, ${Object.keys(IGNORED_CASES).length} exclusions` + + `${structuralModuleCases.size} structural module cases, ${Object.keys(IGNORED_CASES).length} exclusions, ` + + `${Object.keys(ACCEPTED_BASELINE_MISMATCHES).length} expected failures` ); dw.initialize(); for (const c of cases) { const ignored = isIgnored(c.caseIdentifier); for (const scenario of c.scenarios) { - const testFn = ignored ? it.skip : it; - const label = ignored ? `${scenario.name} [skip: ${ignoreReason(c.caseIdentifier)}]` : scenario.name; + const expectedFailure = ACCEPTED_BASELINE_MISMATCHES[scenario.name]; + const testFn = ignored ? it.skip : expectedFailure ? it.fails : it; + const label = ignored + ? `${scenario.name} [skip: ${ignoreReason(c.caseIdentifier)}]` + : expectedFailure + ? `${scenario.name} [xfail: ${expectedFailure}]` + : scenario.name; testFn(label, () => { const script = readFileSync(join(c.dir, MAIN_TRANSFORM), "utf-8"); diff --git a/native-lib/node/tests/unit/tck-accounting.test.ts b/native-lib/node/tests/unit/tck-accounting.test.ts index ad07f5f8..6836ede4 100644 --- a/native-lib/node/tests/unit/tck-accounting.test.ts +++ b/native-lib/node/tests/unit/tck-accounting.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { accountTckResults, validateTckAccounting } from "../../tests/tck/accounting"; -import { matchesTestNamePattern } from "../../tests/tck/reporter"; +import { matchesTestNamePattern, TckAccountingReporter } from "../../tests/tck/reporter"; describe("TCK result accounting", () => { it("reports each selected outcome exactly once", () => { @@ -18,11 +18,27 @@ describe("TCK result accounting", () => { passed: 1, failed: 1, skipped: 1, + xfailed: 0, accounted: 3, unaccounted: 0, }); }); + it("reports expected failures separately from ordinary failures", () => { + expect(accountTckResults( + new Set(["runtime/xfail:out.json"]), + [{ identifier: "runtime/xfail:out.json", outcome: "xfailed" }], + )).toEqual({ + selected: 1, + passed: 0, + failed: 0, + skipped: 0, + xfailed: 1, + accounted: 1, + unaccounted: 0, + }); + }); + it("rejects missing, duplicate, and unexpected results", () => { expect(validateTckAccounting( new Set(["runtime/a:out.json", "runtime/b:out.json"]), @@ -52,4 +68,19 @@ describe("TCK result accounting", () => { expect(matchesTestNamePattern("runtime/other:out.json [skip: unsupported]", pattern)).toBe(false); expect(matchesTestNamePattern("runtime/other:out.json [skip: unsupported]", undefined)).toBe(true); }); + + it("clears accumulated accounting state before a watch rerun", () => { + const reporter = new TckAccountingReporter(); + const state = reporter as unknown as { + selected: Set; + results: Array<{ identifier: string; outcome: "passed" }>; + }; + state.selected.add("runtime/first:out.json"); + state.results.push({ identifier: "runtime/first:out.json", outcome: "passed" }); + + reporter.onTestRunStart?.([]); + + expect(state.selected.size).toBe(0); + expect(state.results).toEqual([]); + }); }); diff --git a/native-lib/node/tests/unit/tck-policy.test.ts b/native-lib/node/tests/unit/tck-policy.test.ts index 6a5a34af..95e27eb9 100644 --- a/native-lib/node/tests/unit/tck-policy.test.ts +++ b/native-lib/node/tests/unit/tck-policy.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from "vitest"; import { + ACCEPTED_BASELINE_MISMATCHES, + CAPABILITY_EXCLUSIONS, IGNORED_CASES, + REENABLED_CASES, STRUCTURAL_MODULE_CASES, validateIgnorePolicy, validateInventoryPolicy, + validateReconciledPolicy, validateStructuralModulePolicy, } from "../../tests/tck/ignore-list"; @@ -39,8 +43,45 @@ describe("TCK ignore policy", () => { ]); }); - it("preserves the current 59 exclusions", () => { - expect(Object.keys(IGNORED_CASES)).toHaveLength(59); + it("reconciles exclusions into capability skips and strict xfails", () => { + expect(Object.keys(CAPABILITY_EXCLUSIONS)).toHaveLength(31); + expect(Object.keys(ACCEPTED_BASELINE_MISMATCHES)).toHaveLength(21); + expect(REENABLED_CASES).toHaveLength(7); + expect(IGNORED_CASES).toBe(CAPABILITY_EXCLUSIONS); + expect(validateReconciledPolicy( + CAPABILITY_EXCLUSIONS, + ACCEPTED_BASELINE_MISMATCHES, + REENABLED_CASES, + )).toEqual([]); + }); + + it("rejects overlapping or incomplete reconciled policy entries", () => { + expect(validateReconciledPolicy( + { + "runtime/overlap": { + caseIdentifier: "runtime/overlap", + category: "unavailable-java-module", + reason: "missing Java module", + }, + }, + { + "runtime/overlap:out.json": "known mismatch", + "runtime/blank:out.json": " ", + "runtime/xfail:out.json": "known mismatch", + "runtime/stale:out.json": "stale mismatch", + }, + ["runtime/overlap", "runtime/reenabled", "runtime/reenabled", "runtime/xfail"], + new Set(["runtime/overlap:out.json", "runtime/blank:out.json", "runtime/xfail:out.json"]), + )).toEqual([ + "runtime/blank:out.json: missing expected-failure reason", + "runtime/overlap: case is both re-enabled and expected to fail", + "runtime/overlap: case is both skipped and re-enabled", + "runtime/overlap:out.json: case is both skipped and expected to fail", + "runtime/reenabled: duplicate re-enabled case", + "runtime/reenabled: not a discovered runnable case", + "runtime/stale:out.json: not a discovered runnable scenario", + "runtime/xfail: case is both re-enabled and expected to fail", + ]); }); it("rejects drift in the staged suite inventory", () => {