@W-23692110: Harden Node TCK coverage accounting

native-lib/node/tests/tck/accounting.ts

@@ -0,0 +1,54 @@
+export type TckOutcome = "passed" | "failed" | "skipped";
+
+export interface TckResult {
+  identifier: string;
+  outcome: TckOutcome;
+}
+
+export interface TckSummary {
+  selected: number;
+  passed: number;
+  failed: number;
+  skipped: number;
+  accounted: number;
+  unaccounted: number;
+}
+
+export function accountTckResults(
+  selected: ReadonlySet<string>,
+  results: readonly TckResult[]
+): TckSummary {
+  const passed = results.filter((result) => result.outcome === "passed").length;
+  const failed = results.filter((result) => result.outcome === "failed").length;
+  const skipped = results.filter((result) => result.outcome === "skipped").length;
+  const accounted = passed + failed + skipped;
+  return {
+    selected: selected.size,
+    passed,
+    failed,
+    skipped,
+    accounted,
+    unaccounted: selected.size - accounted,
+  };
+}
+
+export function validateTckAccounting(
+  selected: ReadonlySet<string>,
+  results: readonly TckResult[]
+): string[] {
+  const counts = new Map<string, number>();
+  for (const result of results) {
+    counts.set(result.identifier, (counts.get(result.identifier) ?? 0) + 1);
+  }
+
+  const errors = [...counts]
+    .filter(([, count]) => count > 1)
+    .map(([identifier]) => `${identifier}: duplicate result`);
+  errors.push(...[...selected]
+    .filter((identifier) => !counts.has(identifier))
+    .map((identifier) => `${identifier}: missing result`));
+  errors.push(...[...counts.keys()]
+    .filter((identifier) => !selected.has(identifier))
+    .map((identifier) => `${identifier}: result was not selected`));
+  return errors.sort();
+}

native-lib/node/tests/tck/case-loader.ts

@@ -21,6 +21,16 @@ const OUTPUT_PATTERN = /^out\.[a-zA-Z]+$/;
 const INPUT_CONFIG_PATTERN = /^in[0-9]+-config\.properties$/;
 const OUTPUT_CONFIG_PATTERN = /^out[0-9]*-config\.properties$/;
 
+/** Whether a structurally skipped case carries a DWL module beside transform.dwl. */
+export function hasAdjacentDwlModule(fileNames: string[]): boolean {
+  return fileNames.some(
+    (name) => extensionOf(name) === "dwl"
+      && name !== MAIN_TRANSFORM
+      && !INPUT_PATTERN.test(name)
+      && !OUTPUT_PATTERN.test(name)
+  );
+}
+
 /** Returns the extension (without dot, lowercased) of a file name, or "" if none. */
 export function extensionOf(name: string): string {
   const i = name.lastIndexOf(".");
@@ -39,8 +49,10 @@ export interface TckInput {
 
 /** One runnable scenario: the transform plus its inputs and a single expected output. */
 export interface TckScenario {
-  /** Scenario id: the case directory name (`<scenario>-out.<ext>`). */
+  /** Scenario id: `<suite>/<case>:<output-file>`. */
   name: string;
+  /** Case id shared by all output scenarios: `<suite>/<case>`. */
+  caseIdentifier: string;
   inputs: TckInput[];
   /** Expected-output file name (e.g. `out.json`). */
   outputFileName: string;
@@ -63,11 +75,12 @@ export type CaseParseResult =
  * extension maps to an unsupported format (e.g. yaml) are dropped; if none
  * remain the case is skipped.
  *
- * @param caseName - The directory (case) name, used as the scenario prefix.
+ * @param suiteName - The staged suite name, such as `runtime`.
+ * @param caseName - The directory (case) name.
  * @param fileNames - The file names directly inside the case directory.
  * @returns Either the runnable scenarios or a reason the case was skipped.
  */
-export function parseCase(caseName: string, fileNames: string[]): CaseParseResult {
+export function parseCase(suiteName: string, caseName: string, fileNames: string[]): CaseParseResult {
   if (caseName.endsWith("_wip") || caseName.endsWith("wip")) {
     return { kind: "skipped", reason: "work-in-progress (_wip)" };
   }
@@ -115,8 +128,10 @@ export function parseCase(caseName: string, fileNames: string[]): CaseParseResul
     .filter((out) => isSupportedExtension(extensionOf(out)))
     .map((outputFileName) => {
       const outputExtension = extensionOf(outputFileName);
+      const caseIdentifier = `${suiteName}/${caseName}`;
       return {
-        name: caseName,
+        name: `${caseIdentifier}:${outputFileName}`,
+        caseIdentifier,
         inputs,
         outputFileName,
         outputExtension,
@@ -135,4 +150,4 @@ function mimeOrThrow(fileName: string): string {
   const mime = mimeForExtension(extensionOf(fileName));
   if (!mime) throw new Error(`no MIME mapping for ${fileName}`);
   return mime;
-}
+}

native-lib/node/tests/tck/ignore-list.ts

@@ -21,11 +21,26 @@
 //   coercion/runtime  — runtime coercion/streaming behavior, also CLI-ignored.
 //   xml               — attribute selector runtime behavior or namespace differences.
 
+export const SUPPORTED_CATEGORIES = [
+  "unavailable-module-or-resource",
+  "unavailable-java-module",
+  "runtime-baseline-mismatch",
+  "environment-sensitive",
+  "slow",
+] as const;
+
+export type IgnoreCategory = typeof SUPPORTED_CATEGORIES[number];
+
 export interface IgnoreEntry {
+  caseIdentifier: string;
+  category: IgnoreCategory | string;
   reason: string;
 }
 
-export const IGNORED_CASES: Readonly<Record<string, IgnoreEntry>> = {
+const EXPECTED_RUNNABLE_CASES = 729;
+const EXPECTED_STRUCTURAL_SKIPS = 193;
+
+const LEGACY_IGNORED_CASES: Readonly<Record<string, { reason: string }>> = {
   // unresolved-module — library/resource not present in dwlib
   "dw-binary-out.dwl": { reason: "unresolved-module: readUrl/classpath resource" },
   "is-empty-using-empty-stream-out.json": { reason: "unresolved-module: dw::Client streaming" },
@@ -105,12 +120,114 @@ export const IGNORED_CASES: Readonly<Record<string, IgnoreEntry>> = {
   "xml_empty_namespace-out.xml": { reason: "xml: empty namespace serialization" },
 };
 
+const CORE_MODULE_CASES = new Set([
+  "read-binary-files-out.bin",
+  "multipart-binary-out.multipart",
+  "multipart-class-cast-issue-out.multipart",
+  "multipart-empty-part-out.multipart",
+  "multipart-mixed-message-out.multipart",
+  "multipart-write-binary-out.json",
+  "multipart-write-message-out.multipart",
+  "multipart-write-subtype-override-out.multipart",
+  "properties-passthrough-out.properties",
+  "csv-invalid-utf8-out.csv",
+  "xml-escaped-data-out.xml",
+  "xml-streaming-selectors-out.xml",
+  "xml-value-selector-out.xml",
+  "xml_empty_namespace-out.xml",
+]);
+
+function categoryFor(reason: string): IgnoreCategory {
+  if (reason.startsWith("unresolved-module:")) return "unavailable-module-or-resource";
+  if (reason.startsWith("java:")) return "unavailable-java-module";
+  if (reason.startsWith("nondeterministic:")) return "environment-sensitive";
+  if (reason.startsWith("slow:")) return "slow";
+  return "runtime-baseline-mismatch";
+}
+
+export const IGNORED_CASES: Readonly<Record<string, IgnoreEntry>> = Object.fromEntries(
+  Object.entries(LEGACY_IGNORED_CASES).map(([caseName, entry]) => {
+    const suite = CORE_MODULE_CASES.has(caseName) ? "core-modules" : "runtime";
+    const caseIdentifier = `${suite}/${caseName}`;
+    return [caseIdentifier, {
+      caseIdentifier,
+      category: categoryFor(entry.reason),
+      reason: entry.reason,
+    }];
+  })
+);
+
+export const STRUCTURAL_MODULE_CASES = new Set([
+  "runtime/implicit_type_parameters-out.json",
+  "runtime/import_mapping-out.json",
+  "runtime/import_mapping_with_functions-out.json",
+  "runtime/import_mapping_with_implicit_input-out.json",
+  "runtime/import_namespace-out.xml",
+  "runtime/infinit_list-out.json",
+  "runtime/interceptor_functions-out.json",
+  "runtime/lazy_metadata_definition-out.json",
+  "runtime/location-out.json",
+  "runtime/locationString-out.json",
+  "runtime/logwith_function-out.json",
+  "runtime/read-function-by-id-out.json",
+  "runtime/read-function-out.json",
+  "runtime/runtime_evalUrl-out.json",
+  "runtime/runtime_runUrl-out.json",
+  "runtime/type_selector_materialize-out.json",
+  "runtime/weave_multiple_namespace-out.dwl",
+]);
+
+export function validateIgnorePolicy(
+  entries: Readonly<Record<string, IgnoreEntry>>,
+  runnableCases?: ReadonlySet<string>
+): string[] {
+  const errors: string[] = [];
+  for (const [identifier, entry] of Object.entries(entries)) {
+    if (!entry.caseIdentifier) errors.push(`${identifier}: missing case identity`);
+    else if (entry.caseIdentifier !== identifier) errors.push(`${identifier}: case identity must match registry key`);
+    if (!SUPPORTED_CATEGORIES.includes(entry.category as IgnoreCategory)) {
+      errors.push(`${identifier}: unsupported category ${entry.category}`);
+    }
+    if (!entry.reason.trim()) errors.push(`${identifier}: missing reason`);
+    if (runnableCases && !runnableCases.has(identifier)) {
+      errors.push(`${identifier}: not a discovered runnable case`);
+    }
+  }
+  return errors;
+}
+
+export function validateInventoryPolicy(runnableCases: number, structuralSkips: number): string[] {
+  const errors: string[] = [];
+  if (runnableCases !== EXPECTED_RUNNABLE_CASES) {
+    errors.push(`expected ${EXPECTED_RUNNABLE_CASES} runnable cases, discovered ${runnableCases}`);
+  }
+  if (structuralSkips !== EXPECTED_STRUCTURAL_SKIPS) {
+    errors.push(`expected ${EXPECTED_STRUCTURAL_SKIPS} structurally skipped cases, discovered ${structuralSkips}`);
+  }
+  return errors;
+}
+
+export function validateStructuralModulePolicy(
+  entries: ReadonlySet<string>,
+  structuralModuleCases: ReadonlySet<string>
+): string[] {
+  const errors = [...entries]
+    .filter((identifier) => !structuralModuleCases.has(identifier))
+    .sort()
+    .map((identifier) => `${identifier}: not a structural module case`);
+  errors.push(...[...structuralModuleCases]
+    .filter((identifier) => !entries.has(identifier))
+    .sort()
+    .map((identifier) => `${identifier}: structural module case is not registered`));
+  return errors;
+}
+
 /** Whether a case is on the ignore list. */
-export function isIgnored(caseName: string): boolean {
-  return Object.prototype.hasOwnProperty.call(IGNORED_CASES, caseName);
+export function isIgnored(caseIdentifier: string): boolean {
+  return Object.prototype.hasOwnProperty.call(IGNORED_CASES, caseIdentifier);
 }
 
 /** The documented skip reason for a case, or undefined if not ignored. */
-export function ignoreReason(caseName: string): string | undefined {
-  return IGNORED_CASES[caseName]?.reason;
-}
+export function ignoreReason(caseIdentifier: string): string | undefined {
+  return IGNORED_CASES[caseIdentifier]?.reason;
+}

native-lib/node/tests/tck/reporter.ts

@@ -0,0 +1,69 @@
+import type { Reporter } from "vitest/reporters";
+import type { TestCase, TestModule, Vitest } from "vitest/node";
+import { accountTckResults, validateTckAccounting, type TckResult } from "./accounting";
+
+const TCK_FILE = "/tests/tck/tck.test.ts";
+
+function isTckCase(testCase: TestCase): boolean {
+  return testCase.module.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE);
+}
+
+export function matchesTestNamePattern(name: string, pattern: RegExp | undefined): boolean {
+  if (!pattern) return true;
+  pattern.lastIndex = 0;
+  return pattern.test(name);
+}
+
+export class TckAccountingReporter implements Reporter {
+  private readonly selected = new Set<string>();
+  private readonly results: TckResult[] = [];
+  private testNamePattern?: RegExp;
+
+  onInit(vitest: Vitest): void {
+    this.testNamePattern = vitest.config.testNamePattern;
+  }
+
+  onTestModuleCollected(testModule: TestModule): void {
+    if (!testModule.moduleId.replaceAll("\\", "/").endsWith(TCK_FILE)) return;
+    for (const testCase of testModule.children.allTests()) {
+      const identifier = testCase.name.split(" [skip:", 1)[0];
+      const isPolicySkip = testCase.options.mode === "skip" && testCase.name.includes(" [skip:");
+      if (isPolicySkip && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) {
+        this.selected.add(identifier);
+      }
+    }
+  }
+
+  onTestCaseReady(testCase: TestCase): void {
+    if (isTckCase(testCase) && matchesTestNamePattern(testCase.fullName, this.testNamePattern)) {
+      this.selected.add(testCase.name.split(" [skip:", 1)[0]);
+    }
+  }
+
+  onTestCaseResult(testCase: TestCase): void {
+    if (!isTckCase(testCase)) return;
+    const identifier = testCase.name.split(" [skip:", 1)[0];
+    if (!this.selected.has(identifier) && testCase.result().state === "skipped") return;
+    const state = testCase.result().state;
+    if (state === "pending") return;
+    this.results.push({
+      identifier,
+      outcome: state,
+    });
+  }
+
+  onTestRunEnd(): void {
+    if (this.selected.size === 0) return;
+    const summary = accountTckResults(this.selected, this.results);
+    console.log(
+      `TCK totals: selected=${summary.selected}, passed=${summary.passed}, failed=${summary.failed}, `
+      + `skipped=${summary.skipped}, accounted=${summary.accounted}, unaccounted=${summary.unaccounted}`
+    );
+    const errors = validateTckAccounting(this.selected, this.results);
+    if (errors.length > 0) {
+      throw new Error(`Invalid TCK result accounting:\n${errors.join("\n")}`);
+    }
+  }
+}
+
+export default TckAccountingReporter;