From 680f34f71069d605ef0051ab760e0ad45a421e7c Mon Sep 17 00:00:00 2001 From: mlischetti Date: Fri, 21 Aug 2026 16:10:00 -0300 Subject: [PATCH] Harden Node TCK coverage accounting --- native-lib/node/tests/tck/accounting.ts | 54 ++++++++ native-lib/node/tests/tck/case-loader.ts | 25 +++- native-lib/node/tests/tck/ignore-list.ts | 129 +++++++++++++++++- native-lib/node/tests/tck/reporter.ts | 69 ++++++++++ native-lib/node/tests/tck/tck.test.ts | 52 +++++-- .../node/tests/unit/tck-accounting.test.ts | 55 ++++++++ .../node/tests/unit/tck-case-loader.test.ts | 26 ++-- native-lib/node/tests/unit/tck-policy.test.ts | 68 +++++++++ native-lib/node/vitest.config.ts | 3 +- 9 files changed, 447 insertions(+), 34 deletions(-) create mode 100644 native-lib/node/tests/tck/accounting.ts create mode 100644 native-lib/node/tests/tck/reporter.ts create mode 100644 native-lib/node/tests/unit/tck-accounting.test.ts create mode 100644 native-lib/node/tests/unit/tck-policy.test.ts diff --git a/native-lib/node/tests/tck/accounting.ts b/native-lib/node/tests/tck/accounting.ts new file mode 100644 index 00000000..073386f6 --- /dev/null +++ b/native-lib/node/tests/tck/accounting.ts @@ -0,0 +1,54 @@ +export type TckOutcome = "passed" | "failed" | "skipped"; + +export interface TckResult { + identifier: string; + outcome: TckOutcome; +} + +export interface TckSummary { + selected: number; + passed: number; + failed: number; + skipped: number; + accounted: number; + unaccounted: number; +} + +export function accountTckResults( + selected: ReadonlySet, + results: readonly TckResult[] +): TckSummary { + const passed = results.filter((result) => result.outcome === "passed").length; + const failed = results.filter((result) => result.outcome === "failed").length; + const skipped = results.filter((result) => result.outcome === "skipped").length; + const accounted = passed + failed + skipped; + return { + selected: selected.size, + passed, + failed, + skipped, + accounted, + unaccounted: selected.size - accounted, + }; +} + +export function validateTckAccounting( + selected: ReadonlySet, + results: readonly TckResult[] +): string[] { + const counts = new Map(); + for (const result of results) { + counts.set(result.identifier, (counts.get(result.identifier) ?? 0) + 1); + } + + const errors = [...counts] + .filter(([, count]) => count > 1) + .map(([identifier]) => `${identifier}: duplicate result`); + errors.push(...[...selected] + .filter((identifier) => !counts.has(identifier)) + .map((identifier) => `${identifier}: missing result`)); + errors.push(...[...counts.keys()] + .filter((identifier) => !selected.has(identifier)) + .map((identifier) => `${identifier}: result was not selected`)); + return errors.sort(); +} diff --git a/native-lib/node/tests/tck/case-loader.ts b/native-lib/node/tests/tck/case-loader.ts index 3dc6ac93..3ca3045a 100644 --- a/native-lib/node/tests/tck/case-loader.ts +++ b/native-lib/node/tests/tck/case-loader.ts @@ -21,6 +21,16 @@ const OUTPUT_PATTERN = /^out\.[a-zA-Z]+$/; const INPUT_CONFIG_PATTERN = /^in[0-9]+-config\.properties$/; const OUTPUT_CONFIG_PATTERN = /^out[0-9]*-config\.properties$/; +/** Whether a structurally skipped case carries a DWL module beside transform.dwl. */ +export function hasAdjacentDwlModule(fileNames: string[]): boolean { + return fileNames.some( + (name) => extensionOf(name) === "dwl" + && name !== MAIN_TRANSFORM + && !INPUT_PATTERN.test(name) + && !OUTPUT_PATTERN.test(name) + ); +} + /** Returns the extension (without dot, lowercased) of a file name, or "" if none. */ export function extensionOf(name: string): string { const i = name.lastIndexOf("."); @@ -39,8 +49,10 @@ export interface TckInput { /** One runnable scenario: the transform plus its inputs and a single expected output. */ export interface TckScenario { - /** Scenario id: the case directory name (`-out.`). */ + /** Scenario id: `/:`. */ name: string; + /** Case id shared by all output scenarios: `/`. */ + caseIdentifier: string; inputs: TckInput[]; /** Expected-output file name (e.g. `out.json`). */ outputFileName: string; @@ -63,11 +75,12 @@ export type CaseParseResult = * extension maps to an unsupported format (e.g. yaml) are dropped; if none * remain the case is skipped. * - * @param caseName - The directory (case) name, used as the scenario prefix. + * @param suiteName - The staged suite name, such as `runtime`. + * @param caseName - The directory (case) name. * @param fileNames - The file names directly inside the case directory. * @returns Either the runnable scenarios or a reason the case was skipped. */ -export function parseCase(caseName: string, fileNames: string[]): CaseParseResult { +export function parseCase(suiteName: string, caseName: string, fileNames: string[]): CaseParseResult { if (caseName.endsWith("_wip") || caseName.endsWith("wip")) { return { kind: "skipped", reason: "work-in-progress (_wip)" }; } @@ -115,8 +128,10 @@ export function parseCase(caseName: string, fileNames: string[]): CaseParseResul .filter((out) => isSupportedExtension(extensionOf(out))) .map((outputFileName) => { const outputExtension = extensionOf(outputFileName); + const caseIdentifier = `${suiteName}/${caseName}`; return { - name: caseName, + name: `${caseIdentifier}:${outputFileName}`, + caseIdentifier, inputs, outputFileName, outputExtension, @@ -135,4 +150,4 @@ function mimeOrThrow(fileName: string): string { const mime = mimeForExtension(extensionOf(fileName)); if (!mime) throw new Error(`no MIME mapping for ${fileName}`); return mime; -} \ No newline at end of file +} diff --git a/native-lib/node/tests/tck/ignore-list.ts b/native-lib/node/tests/tck/ignore-list.ts index ea3b5c92..6bf200a5 100644 --- a/native-lib/node/tests/tck/ignore-list.ts +++ b/native-lib/node/tests/tck/ignore-list.ts @@ -21,11 +21,26 @@ // coercion/runtime — runtime coercion/streaming behavior, also CLI-ignored. // xml — attribute selector runtime behavior or namespace differences. +export const SUPPORTED_CATEGORIES = [ + "unavailable-module-or-resource", + "unavailable-java-module", + "runtime-baseline-mismatch", + "environment-sensitive", + "slow", +] as const; + +export type IgnoreCategory = typeof SUPPORTED_CATEGORIES[number]; + export interface IgnoreEntry { + caseIdentifier: string; + category: IgnoreCategory | string; reason: string; } -export const IGNORED_CASES: Readonly> = { +const EXPECTED_RUNNABLE_CASES = 729; +const EXPECTED_STRUCTURAL_SKIPS = 193; + +const LEGACY_IGNORED_CASES: Readonly> = { // unresolved-module — library/resource not present in dwlib "dw-binary-out.dwl": { reason: "unresolved-module: readUrl/classpath resource" }, "is-empty-using-empty-stream-out.json": { reason: "unresolved-module: dw::Client streaming" }, @@ -105,12 +120,114 @@ export const IGNORED_CASES: Readonly> = { "xml_empty_namespace-out.xml": { reason: "xml: empty namespace serialization" }, }; +const CORE_MODULE_CASES = new Set([ + "read-binary-files-out.bin", + "multipart-binary-out.multipart", + "multipart-class-cast-issue-out.multipart", + "multipart-empty-part-out.multipart", + "multipart-mixed-message-out.multipart", + "multipart-write-binary-out.json", + "multipart-write-message-out.multipart", + "multipart-write-subtype-override-out.multipart", + "properties-passthrough-out.properties", + "csv-invalid-utf8-out.csv", + "xml-escaped-data-out.xml", + "xml-streaming-selectors-out.xml", + "xml-value-selector-out.xml", + "xml_empty_namespace-out.xml", +]); + +function categoryFor(reason: string): IgnoreCategory { + if (reason.startsWith("unresolved-module:")) return "unavailable-module-or-resource"; + if (reason.startsWith("java:")) return "unavailable-java-module"; + if (reason.startsWith("nondeterministic:")) return "environment-sensitive"; + if (reason.startsWith("slow:")) return "slow"; + return "runtime-baseline-mismatch"; +} + +export const IGNORED_CASES: Readonly> = Object.fromEntries( + Object.entries(LEGACY_IGNORED_CASES).map(([caseName, entry]) => { + const suite = CORE_MODULE_CASES.has(caseName) ? "core-modules" : "runtime"; + const caseIdentifier = `${suite}/${caseName}`; + return [caseIdentifier, { + caseIdentifier, + category: categoryFor(entry.reason), + reason: entry.reason, + }]; + }) +); + +export const STRUCTURAL_MODULE_CASES = new Set([ + "runtime/implicit_type_parameters-out.json", + "runtime/import_mapping-out.json", + "runtime/import_mapping_with_functions-out.json", + "runtime/import_mapping_with_implicit_input-out.json", + "runtime/import_namespace-out.xml", + "runtime/infinit_list-out.json", + "runtime/interceptor_functions-out.json", + "runtime/lazy_metadata_definition-out.json", + "runtime/location-out.json", + "runtime/locationString-out.json", + "runtime/logwith_function-out.json", + "runtime/read-function-by-id-out.json", + "runtime/read-function-out.json", + "runtime/runtime_evalUrl-out.json", + "runtime/runtime_runUrl-out.json", + "runtime/type_selector_materialize-out.json", + "runtime/weave_multiple_namespace-out.dwl", +]); + +export function validateIgnorePolicy( + entries: Readonly>, + runnableCases?: ReadonlySet +): string[] { + const errors: string[] = []; + for (const [identifier, entry] of Object.entries(entries)) { + if (!entry.caseIdentifier) errors.push(`${identifier}: missing case identity`); + else if (entry.caseIdentifier !== identifier) errors.push(`${identifier}: case identity must match registry key`); + if (!SUPPORTED_CATEGORIES.includes(entry.category as IgnoreCategory)) { + errors.push(`${identifier}: unsupported category ${entry.category}`); + } + if (!entry.reason.trim()) errors.push(`${identifier}: missing reason`); + if (runnableCases && !runnableCases.has(identifier)) { + errors.push(`${identifier}: not a discovered runnable case`); + } + } + return errors; +} + +export function validateInventoryPolicy(runnableCases: number, structuralSkips: number): string[] { + const errors: string[] = []; + if (runnableCases !== EXPECTED_RUNNABLE_CASES) { + errors.push(`expected ${EXPECTED_RUNNABLE_CASES} runnable cases, discovered ${runnableCases}`); + } + if (structuralSkips !== EXPECTED_STRUCTURAL_SKIPS) { + errors.push(`expected ${EXPECTED_STRUCTURAL_SKIPS} structurally skipped cases, discovered ${structuralSkips}`); + } + return errors; +} + +export function validateStructuralModulePolicy( + entries: ReadonlySet, + structuralModuleCases: ReadonlySet +): string[] { + const errors = [...entries] + .filter((identifier) => !structuralModuleCases.has(identifier)) + .sort() + .map((identifier) => `${identifier}: not a structural module case`); + errors.push(...[...structuralModuleCases] + .filter((identifier) => !entries.has(identifier)) + .sort() + .map((identifier) => `${identifier}: structural module case is not registered`)); + return errors; +} + /** Whether a case is on the ignore list. */ -export function isIgnored(caseName: string): boolean { - return Object.prototype.hasOwnProperty.call(IGNORED_CASES, caseName); +export function isIgnored(caseIdentifier: string): boolean { + return Object.prototype.hasOwnProperty.call(IGNORED_CASES, caseIdentifier); } /** The documented skip reason for a case, or undefined if not ignored. */ -export function ignoreReason(caseName: string): string | undefined { - return IGNORED_CASES[caseName]?.reason; -} \ No newline at end of file +export function ignoreReason(caseIdentifier: string): string | undefined { + return IGNORED_CASES[caseIdentifier]?.reason; +} diff --git a/native-lib/node/tests/tck/reporter.ts b/native-lib/node/tests/tck/reporter.ts new file mode 100644 index 00000000..4af0e58c --- /dev/null +++ b/native-lib/node/tests/tck/reporter.ts @@ -0,0 +1,69 @@ +import type { Reporter } from "vitest/reporters"; +import type { TestCase, TestModule, Vitest } from "vitest/node"; +import { accountTckResults, validateTckAccounting, type TckResult } from "./accounting"; + +const TCK_FILE = "/tests/tck/tck.test.ts"; + +function isTckCase(testCase: TestCase): boolean { + return testCase.module.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE); +} + +export function matchesTestNamePattern(name: string, pattern: RegExp | undefined): boolean { + if (!pattern) return true; + pattern.lastIndex = 0; + return pattern.test(name); +} + +export class TckAccountingReporter implements Reporter { + private readonly selected = new Set(); + private readonly results: TckResult[] = []; + private testNamePattern?: RegExp; + + onInit(vitest: Vitest): void { + this.testNamePattern = vitest.config.testNamePattern; + } + + onTestModuleCollected(testModule: TestModule): void { + if (!testModule.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE)) return; + for (const testCase of testModule.children.allTests()) { + const identifier = testCase.name.split(" [skip:", 1)[0]; + const isPolicySkip = testCase.options.mode === "skip" && testCase.name.includes(" [skip:"); + if (isPolicySkip && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) { + this.selected.add(identifier); + } + } + } + + onTestCaseReady(testCase: TestCase): void { + if (isTckCase(testCase) && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) { + this.selected.add(testCase.name.split(" [skip:", 1)[0]); + } + } + + onTestCaseResult(testCase: TestCase): void { + if (!isTckCase(testCase)) return; + const identifier = testCase.name.split(" [skip:", 1)[0]; + if (!this.selected.has(identifier) && testCase.result().state === "skipped") return; + const state = testCase.result().state; + if (state === "pending") return; + this.results.push({ + identifier, + outcome: state, + }); + } + + onTestRunEnd(): void { + if (this.selected.size === 0) return; + const summary = accountTckResults(this.selected, this.results); + console.log( + `TCK totals: selected=${summary.selected}, passed=${summary.passed}, failed=${summary.failed}, ` + + `skipped=${summary.skipped}, accounted=${summary.accounted}, unaccounted=${summary.unaccounted}` + ); + const errors = validateTckAccounting(this.selected, this.results); + if (errors.length > 0) { + throw new Error(`Invalid TCK result accounting:\n${errors.join("\n")}`); + } + } +} + +export default TckAccountingReporter; diff --git a/native-lib/node/tests/tck/tck.test.ts b/native-lib/node/tests/tck/tck.test.ts index aecfec2f..d8a16334 100644 --- a/native-lib/node/tests/tck/tck.test.ts +++ b/native-lib/node/tests/tck/tck.test.ts @@ -10,16 +10,24 @@ import { describe, it, expect } from "vitest"; import { readFileSync, readdirSync, existsSync, statSync } from "node:fs"; import { join } from "node:path"; import { DataWeave, modulesFromDirectory } from "../../src/index"; -import { parseCase, MAIN_TRANSFORM, type TckScenario } from "./case-loader"; +import { hasAdjacentDwlModule, parseCase, MAIN_TRANSFORM, type TckScenario } from "./case-loader"; import { compareOutput } from "./compare"; -import { isIgnored, ignoreReason } from "./ignore-list"; +import { + IGNORED_CASES, + STRUCTURAL_MODULE_CASES, + isIgnored, + ignoreReason, + validateIgnorePolicy, + validateInventoryPolicy, + validateStructuralModulePolicy, +} from "./ignore-list"; const SUITES_DIR = join(__dirname, "suites"); const FIXTURES_DIR = join(__dirname, "fixtures"); /** A discovered case: its directory and the scenarios parsed from it. */ interface DiscoveredCase { - caseName: string; + caseIdentifier: string; dir: string; scenarios: TckScenario[]; } @@ -35,22 +43,30 @@ function listDirs(parent: string): string[] { } /** Walks every staged suite and returns the runnable cases (skips logged by the caller). */ -function discoverCases(): { cases: DiscoveredCase[]; skipped: number } { +function discoverCases(): { + cases: DiscoveredCase[]; + skipped: number; + structuralModuleCases: Set; +} { const cases: DiscoveredCase[] = []; + const structuralModuleCases = new Set(); let skipped = 0; for (const suite of listDirs(SUITES_DIR)) { const suiteDir = join(SUITES_DIR, suite); for (const caseName of listDirs(suiteDir)) { const dir = join(suiteDir, caseName); - const parsed = parseCase(caseName, readdirSync(dir)); + const fileNames = readdirSync(dir); + const caseIdentifier = `${suite}/${caseName}`; + const parsed = parseCase(suite, caseName, fileNames); if (parsed.kind === "skipped") { skipped++; + if (hasAdjacentDwlModule(fileNames)) structuralModuleCases.add(caseIdentifier); continue; } - cases.push({ caseName, dir, scenarios: parsed.scenarios }); + cases.push({ caseIdentifier, dir, scenarios: parsed.scenarios }); } } - return { cases, skipped }; + return { cases, skipped, structuralModuleCases }; } if (!existsSync(SUITES_DIR)) { @@ -60,7 +76,16 @@ if (!existsSync(SUITES_DIR)) { it("skipped", () => {}); }); } else { - const { cases, skipped } = discoverCases(); + const { cases, skipped, structuralModuleCases } = discoverCases(); + const runnableCases = new Set(cases.map((item) => item.caseIdentifier)); + const policyErrors = [ + ...validateInventoryPolicy(cases.length, skipped), + ...validateIgnorePolicy(IGNORED_CASES, runnableCases), + ...validateStructuralModulePolicy(STRUCTURAL_MODULE_CASES, structuralModuleCases), + ]; + if (policyErrors.length > 0) { + throw new Error(`Invalid TCK policy:\n${policyErrors.join("\n")}`); + } // One shared runtime for the whole lane. Modules imported by a handful of // TCK cases (org::mule::weave::v2::libs::lib) live only in the private @@ -70,14 +95,17 @@ if (!existsSync(SUITES_DIR)) { describe("TCK conformance", () => { // eslint-disable-next-line no-console - console.log(`TCK: ${cases.length} runnable cases, ${skipped} structurally skipped`); + console.log( + `TCK: ${cases.length} runnable cases, ${skipped} structurally skipped, ` + + `${structuralModuleCases.size} structural module cases, ${Object.keys(IGNORED_CASES).length} exclusions` + ); dw.initialize(); for (const c of cases) { - const ignored = isIgnored(c.caseName); + const ignored = isIgnored(c.caseIdentifier); for (const scenario of c.scenarios) { const testFn = ignored ? it.skip : it; - const label = ignored ? `${scenario.name} [skip: ${ignoreReason(c.caseName)}]` : scenario.name; + const label = ignored ? `${scenario.name} [skip: ${ignoreReason(c.caseIdentifier)}]` : scenario.name; testFn(label, () => { const script = readFileSync(join(c.dir, MAIN_TRANSFORM), "utf-8"); @@ -103,4 +131,4 @@ if (!existsSync(SUITES_DIR)) { } } }); -} \ No newline at end of file +} diff --git a/native-lib/node/tests/unit/tck-accounting.test.ts b/native-lib/node/tests/unit/tck-accounting.test.ts new file mode 100644 index 00000000..ad07f5f8 --- /dev/null +++ b/native-lib/node/tests/unit/tck-accounting.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { accountTckResults, validateTckAccounting } from "../../tests/tck/accounting"; +import { matchesTestNamePattern } from "../../tests/tck/reporter"; + +describe("TCK result accounting", () => { + it("reports each selected outcome exactly once", () => { + const summary = accountTckResults( + new Set(["runtime/pass:out.json", "runtime/fail:out.json", "runtime/skip:out.json"]), + [ + { identifier: "runtime/pass:out.json", outcome: "passed" }, + { identifier: "runtime/fail:out.json", outcome: "failed" }, + { identifier: "runtime/skip:out.json", outcome: "skipped" }, + ], + ); + + expect(summary).toEqual({ + selected: 3, + passed: 1, + failed: 1, + skipped: 1, + accounted: 3, + unaccounted: 0, + }); + }); + + it("rejects missing, duplicate, and unexpected results", () => { + expect(validateTckAccounting( + new Set(["runtime/a:out.json", "runtime/b:out.json"]), + [ + { identifier: "runtime/a:out.json", outcome: "passed" }, + { identifier: "runtime/a:out.json", outcome: "passed" }, + { identifier: "runtime/extra:out.json", outcome: "passed" }, + ], + )).toEqual([ + "runtime/a:out.json: duplicate result", + "runtime/b:out.json: missing result", + "runtime/extra:out.json: result was not selected", + ]); + }); + + it("allows intentionally filtered selections", () => { + expect(validateTckAccounting( + new Set(["runtime/a:out.json"]), + [{ identifier: "runtime/a:out.json", outcome: "passed" }], + )).toEqual([]); + }); + + it("selects policy skips only when they match the active test-name filter", () => { + const pattern = /runtime\/selected:out\.json/; + + expect(matchesTestNamePattern("runtime/selected:out.json [skip: unsupported]", pattern)).toBe(true); + expect(matchesTestNamePattern("runtime/other:out.json [skip: unsupported]", pattern)).toBe(false); + expect(matchesTestNamePattern("runtime/other:out.json [skip: unsupported]", undefined)).toBe(true); + }); +}); diff --git a/native-lib/node/tests/unit/tck-case-loader.test.ts b/native-lib/node/tests/unit/tck-case-loader.test.ts index 8de2415e..1dbe1963 100644 --- a/native-lib/node/tests/unit/tck-case-loader.test.ts +++ b/native-lib/node/tests/unit/tck-case-loader.test.ts @@ -1,9 +1,9 @@ import { describe, it, expect } from "vitest"; -import { parseCase, extensionOf, MAIN_TRANSFORM } from "../../tests/tck/case-loader"; +import { parseCase, extensionOf, hasAdjacentDwlModule, MAIN_TRANSFORM } from "../../tests/tck/case-loader"; /** Asserts parseCase skipped the case, returning the reason for further checks. */ function expectSkipped(caseName: string, files: string[]): string { - const r = parseCase(caseName, files); + const r = parseCase("runtime", caseName, files); expect(r.kind).toBe("skipped"); return r.kind === "skipped" ? r.reason : ""; } @@ -20,34 +20,40 @@ describe("extensionOf", () => { describe("parseCase — happy paths", () => { it("parses a single-input single-output case", () => { - const r = parseCase("as-operator-out.json", [MAIN_TRANSFORM, "in0.json", "out.json"]); + const r = parseCase("runtime", "as-operator-out.json", [MAIN_TRANSFORM, "in0.json", "out.json"]); expect(r.kind).toBe("scenarios"); if (r.kind !== "scenarios") return; expect(r.scenarios).toHaveLength(1); const s = r.scenarios[0]; - expect(s.name).toBe("as-operator-out.json"); + expect(s.name).toBe("runtime/as-operator-out.json:out.json"); + expect(s.caseIdentifier).toBe("runtime/as-operator-out.json"); expect(s.inputs).toEqual([{ name: "in0", fileName: "in0.json", mimeType: "application/json" }]); expect(s.outputMime).toBe("application/json"); expect(s.outputExtension).toBe("json"); }); it("binds multiple inputs by base name, sorted", () => { - const r = parseCase("multi-out.json", [MAIN_TRANSFORM, "in1.xml", "in0.json", "out.json"]); + const r = parseCase("runtime", "multi-out.json", [MAIN_TRANSFORM, "in1.xml", "in0.json", "out.json"]); if (r.kind !== "scenarios") throw new Error("expected scenarios"); - expect(r.scenarios[0].name).toBe("multi-out.json"); + expect(r.scenarios[0].name).toBe("runtime/multi-out.json:out.json"); expect(r.scenarios[0].inputs.map((i) => i.name)).toEqual(["in0", "in1"]); expect(r.scenarios[0].inputs.map((i) => i.mimeType)).toEqual(["application/json", "application/xml"]); }); it("names the scenario after the case directory", () => { - const r = parseCase("literal-out.json", [MAIN_TRANSFORM, "out.json"]); + const r = parseCase("runtime", "literal-out.json", [MAIN_TRANSFORM, "out.json"]); if (r.kind !== "scenarios") throw new Error("expected scenarios"); - expect(r.scenarios[0].name).toBe("literal-out.json"); + expect(r.scenarios[0].name).toBe("runtime/literal-out.json:out.json"); expect(r.scenarios[0].inputs).toEqual([]); }); }); describe("parseCase — structural skips", () => { + it("identifies adjacent DWL modules without treating inputs as modules", () => { + expect(hasAdjacentDwlModule([MAIN_TRANSFORM, "include.dwl", "in0.dwl", "out.dwl"])).toBe(true); + expect(hasAdjacentDwlModule([MAIN_TRANSFORM, "in0.dwl", "out.dwl"])).toBe(false); + }); + it("skips _wip cases", () => { expect(expectSkipped("feature_wip", [MAIN_TRANSFORM, "out.json"])).toMatch(/wip/i); }); @@ -92,7 +98,7 @@ describe("parseCase — unsupported formats", () => { }); it("drops unsupported output scenarios, keeping supported ones", () => { - const r = parseCase("mix", [MAIN_TRANSFORM, "in0.json", "out.json", "out.yaml"]); + const r = parseCase("runtime", "mix", [MAIN_TRANSFORM, "in0.json", "out.json", "out.yaml"]); if (r.kind !== "scenarios") throw new Error("expected scenarios"); expect(r.scenarios.map((s) => s.outputExtension)).toEqual(["json"]); }); @@ -101,4 +107,4 @@ describe("parseCase — unsupported formats", () => { expect(expectSkipped("y", [MAIN_TRANSFORM, "in0.json", "out.yaml"])) .toMatch(/no scenarios with a supported output/); }); -}); \ No newline at end of file +}); diff --git a/native-lib/node/tests/unit/tck-policy.test.ts b/native-lib/node/tests/unit/tck-policy.test.ts new file mode 100644 index 00000000..6a5a34af --- /dev/null +++ b/native-lib/node/tests/unit/tck-policy.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + IGNORED_CASES, + STRUCTURAL_MODULE_CASES, + validateIgnorePolicy, + validateInventoryPolicy, + validateStructuralModulePolicy, +} from "../../tests/tck/ignore-list"; + +describe("TCK ignore policy", () => { + it("requires identity, supported category, and reason", () => { + expect(validateIgnorePolicy({ + "runtime/missing-identity": { caseIdentifier: "", category: "unavailable-java-module", reason: "missing" }, + "runtime/mismatched": { caseIdentifier: "runtime/other", category: "unavailable-java-module", reason: "missing" }, + "runtime/bad-category": { caseIdentifier: "runtime/bad-category", category: "broad-runtime", reason: "missing" }, + "runtime/blank-reason": { caseIdentifier: "runtime/blank-reason", category: "unavailable-java-module", reason: " " }, + })).toEqual([ + "runtime/missing-identity: missing case identity", + "runtime/mismatched: case identity must match registry key", + "runtime/bad-category: unsupported category broad-runtime", + "runtime/blank-reason: missing reason", + ]); + }); + + it("rejects policy entries that are not runnable cases", () => { + expect(validateIgnorePolicy({ + "runtime/runnable": { + caseIdentifier: "runtime/runnable", + category: "unavailable-java-module", + reason: "missing Java type", + }, + "runtime/stale": { + caseIdentifier: "runtime/stale", + category: "unavailable-java-module", + reason: "stale", + }, + }, new Set(["runtime/runnable"]))).toEqual([ + "runtime/stale: not a discovered runnable case", + ]); + }); + + it("preserves the current 59 exclusions", () => { + expect(Object.keys(IGNORED_CASES)).toHaveLength(59); + }); + + it("rejects drift in the staged suite inventory", () => { + expect(validateInventoryPolicy(728, 194)).toEqual([ + "expected 729 runnable cases, discovered 728", + "expected 193 structurally skipped cases, discovered 194", + ]); + }); +}); + +describe("TCK structural-module policy", () => { + it("validates the inventory in both directions", () => { + expect(validateStructuralModulePolicy( + new Set(["runtime/registered", "runtime/stale"]), + new Set(["runtime/registered", "runtime/missing"]), + )).toEqual([ + "runtime/stale: not a structural module case", + "runtime/missing: structural module case is not registered", + ]); + }); + + it("catalogues 17 adjacent-DWL cases", () => { + expect(STRUCTURAL_MODULE_CASES.size).toBe(17); + }); +}); diff --git a/native-lib/node/vitest.config.ts b/native-lib/node/vitest.config.ts index 5cf67d12..2a7b558d 100644 --- a/native-lib/node/vitest.config.ts +++ b/native-lib/node/vitest.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ // Groups are built out incrementally, so a lane with no test files yet // (e.g. tck) must not fail the run. passWithNoTests: true, + reporters: ["default", "./tests/tck/reporter.ts"], // Three test groups. Run all with `vitest run`, or one lane with // `vitest run --project unit` (etc.). Coverage merges across whichever ran. // - unit: pure TS logic, no native library (dwlib) required. @@ -41,4 +42,4 @@ export default defineConfig({ reporter: ["text", "lcov", "html"], }, }, -}); \ No newline at end of file +});