diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts index 71a055cea676..bfec753f6b2d 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner-worker.ts @@ -6,9 +6,10 @@ * found in the LICENSE file at https://angular.dev/license */ -import remapping, { type EncodedSourceMap, type SourceMapInput } from '@ampproject/remapping'; +import remapping, { type DecodedSourceMap, type SourceMapInput } from '@ampproject/remapping'; import { MagicString } from 'magic-string'; import assert from 'node:assert'; +import { deserialize } from 'node:v8'; import { workerData } from 'node:worker_threads'; import { Visitor, parseSync } from 'oxc-parser'; @@ -28,9 +29,11 @@ interface InlineFileRequest { locale: string; /** - * The translation messages for the locale that should be used during the inlining process of the file. + * The serialized translation messages for the locale that should be used during the inlining + * process of the file. A Blob is used so that the messages are shared with the Worker by + * reference instead of being copied into it for every request. */ - translation?: Record; + translation?: Blob; } /** @@ -53,19 +56,55 @@ interface InlineCodeRequest { locale: string; /** - * The translation messages for the locale that should be used during the inlining process of the file. + * The serialized translation messages for the locale that should be used during the inlining + * process of the file. A Blob is used so that the messages are shared with the Worker by + * reference instead of being copied into it for every request. */ - translation?: Record; + translation?: Blob; } // Extract the application files and common options used for inline requests from the Worker context -// TODO: Evaluate overall performance difference of passing translations here as well const { files, missingTranslation, shouldOptimize } = (workerData || {}) as { files: ReadonlyMap; missingTranslation: 'error' | 'warning' | 'ignore'; shouldOptimize: boolean; }; +/** + * The translation messages deserialized for the locale most recently requested of this Worker. + * Locales are inlined one at a time, so retaining only the active locale is enough to avoid + * deserializing the messages once per file while holding at most one set of messages in memory. + */ +let activeTranslation: { locale: string; messages: Promise> } | undefined; + +/** + * Deserializes the translation messages for an inline request, reusing the result for any + * subsequent request that targets the same locale. + * @param request An inline request containing the locale and its serialized messages. + * @returns The translation messages, or undefined if the locale has no translations. + */ +function loadTranslation( + request: InlineFileRequest | InlineCodeRequest, +): Promise> | undefined { + const { locale, translation } = request; + if (!translation) { + return undefined; + } + + if (activeTranslation?.locale !== locale) { + activeTranslation = { + locale, + // Deserializing within the stored promise ensures that concurrent requests for a locale + // share the one deserialization instead of each performing their own. + messages: translation + .arrayBuffer() + .then((buffer) => deserialize(new Uint8Array(buffer)) as Record), + }; + } + + return activeTranslation.messages; +} + /** * Inlines the provided locale and translation into a JavaScript file that contains `$localize` usage. * This function is the main entry for the Worker's action that is called by the worker pool. @@ -80,7 +119,12 @@ export default async function inlineFile(request: InlineFileRequest) { const code = await data.text(); const map = await files.get(request.filename + '.map')?.text(); - const result = await transformWithOxc(code, map && (JSON.parse(map) as SourceMapInput), request); + const result = await transformWithOxc( + code, + map && (JSON.parse(map) as SourceMapInput), + request, + await loadTranslation(request), + ); return { file: request.filename, @@ -98,7 +142,12 @@ export default async function inlineFile(request: InlineFileRequest) { * @returns An object containing the inlined code. */ export async function inlineCode(request: InlineCodeRequest) { - const result = await transformWithOxc(request.code, undefined, request); + const result = await transformWithOxc( + request.code, + undefined, + request, + await loadTranslation(request), + ); return { output: result.code, @@ -136,12 +185,14 @@ async function loadLocalizeTools(): Promise { * @param code A string containing the JavaScript code to transform. * @param map A sourcemap object for the provided JavaScript code. * @param options The inline request options to use. + * @param translation The translation messages to inline, or undefined for an untranslated locale. * @returns An object containing the code, map, and diagnostics from the transformation. */ async function transformWithOxc( code: string, map: SourceMapInput | undefined, options: InlineFileRequest, + translation: Record | undefined, ) { const { program } = parseSync(options.filename, code, { sourceType: 'unambiguous', @@ -169,10 +220,10 @@ async function transformWithOxc( const [translatedParts, translatedSubstitutions] = translate( diagnostics, - options.translation || {}, + translation || {}, messageParts, node.quasi.expressions.map((_, index) => index), - options.translation === undefined ? 'ignore' : missingTranslation, + translation === undefined ? 'ignore' : missingTranslation, ); // Reconstruct the new template/string literal replacement @@ -209,12 +260,15 @@ async function transformWithOxc( const outputCode = magicString.toString(); let outputMap; if (map && magicString.hasChanged()) { - const rawMap = magicString.generateMap({ + // A decoded map is generated here rather than an encoded one because remapping decodes its + // inputs. Encoding the mappings only for remapping to immediately decode them again doubles + // the peak memory of the largest structure involved in inlining a file. + const rawMap = magicString.generateDecodedMap({ source: options.filename, includeContent: true, hires: 'boundary', }); - outputMap = remapping([rawMap as EncodedSourceMap, map], () => null); + outputMap = remapping([{ ...rawMap, version: 3 } satisfies DecodedSourceMap, map], () => null); } return { diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner.ts index a6d4b5aeccbb..071097315e5b 100644 --- a/packages/angular/build/src/tools/esbuild/i18n-inliner.ts +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner.ts @@ -9,6 +9,7 @@ import assert from 'node:assert'; import { createHash } from 'node:crypto'; import { extname, join } from 'node:path'; +import { serialize } from 'node:v8'; import { WorkerPool } from '../../utils/worker-pool'; import { type BuildOutputFile, BuildOutputFileType, createOutputFile } from './bundler-files'; import { type PersistentCacheStore, createPersistentCacheStore } from './cache'; @@ -19,6 +20,20 @@ import { type PersistentCacheStore, createPersistentCacheStore } from './cache'; */ const LOCALIZE_KEYWORD = '$localize'; +/** + * Serializes the translation messages for a locale for transfer to an inliner Worker. + * + * A Blob is used because cloning one shares its data by reference, whereas sending the messages + * themselves copies them into a Worker for every request that carries them. A locale can contain + * tens of thousands of messages, which makes that copy the dominant cost of inlining a locale. + * + * @param translation The translation messages for a locale, if the locale has any. + * @returns A Blob containing the serialized messages, or undefined if the locale has none. + */ +function serializeTranslation(translation: Record | undefined): Blob | undefined { + return translation && new Blob([serialize(translation)]); +} + /** * Inlining options that should apply to all transformed code. */ @@ -125,6 +140,10 @@ export class I18nInliner { await this.initCache(); const { shouldOptimize, missingTranslation } = this.options; + + // Serialized once here and then shared by the request for every file of this locale + const translationBlob = serializeTranslation(translation); + // Request inlining for each file that contains localize calls const requests = []; @@ -138,10 +157,12 @@ export class I18nInliner { let cacheResultPromise = Promise.resolve(null); if (this.#cache) { - fileCacheKeyBase ??= Buffer.from( - JSON.stringify({ locale, translation, missingTranslation, shouldOptimize }), - 'utf-8', - ); + // The options are digested here so that each file's key is derived from a fixed number + // of bytes. Hashing the options directly would re-hash the full set of messages, which + // can be several megabytes, once for every file. + fileCacheKeyBase ??= createHash('sha256') + .update(JSON.stringify({ locale, translation, missingTranslation, shouldOptimize })) + .digest(); // NOTE: If additional options are added, this may need to be updated. // TODO: Consider xxhash or similar instead of SHA256 @@ -160,7 +181,11 @@ export class I18nInliner { return cachedResult; } - const result = await this.#workerPool.run({ filename, locale, translation }); + const result = await this.#workerPool.run({ + filename, + locale, + translation: translationBlob, + }); if (this.#cache && cacheKey) { try { // Failure to set the value should not fail the transform @@ -227,7 +252,12 @@ export class I18nInliner { } const { output, messages } = await this.#workerPool.run( - { code: templateCode, filename: templateId, locale, translation }, + { + code: templateCode, + filename: templateId, + locale, + translation: serializeTranslation(translation), + }, { name: 'inlineCode' }, ); diff --git a/packages/angular/build/src/tools/esbuild/i18n-inliner_spec.ts b/packages/angular/build/src/tools/esbuild/i18n-inliner_spec.ts new file mode 100644 index 000000000000..e3e7320e391d --- /dev/null +++ b/packages/angular/build/src/tools/esbuild/i18n-inliner_spec.ts @@ -0,0 +1,202 @@ +/** + * @license + * Copyright Google LLC All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.dev/license + */ + +import { transform } from 'esbuild'; +import { type BuildOutputFile, BuildOutputFileType, createOutputFile } from './bundler-files'; +import { I18nInliner } from './i18n-inliner'; + +/** + * A module that uses a `$localize` message with an explicit message identifier so that the + * translations for a test can be keyed by a known name. + */ +const GREETING_SOURCE = 'export const greeting = $localize`:@@greeting:Hello`;\n'; + +/** + * Creates the parsed translation form that `@angular/localize` expects for a message without + * placeholders. + */ +function translationFor(message: string): Record { + return { messageParts: [message], placeholderNames: [], text: message }; +} + +function browserFile(path: string, contents: string): BuildOutputFile { + return createOutputFile(path, contents, BuildOutputFileType.Browser); +} + +function findFile(outputFiles: BuildOutputFile[], path: string): BuildOutputFile { + const file = outputFiles.find((output) => output.path === path); + if (!file) { + throw new Error(`Expected output files to contain '${path}'.`); + } + + return file; +} + +describe('I18nInliner', () => { + let inliner: I18nInliner | undefined; + + // A single thread is used throughout so that every file of every locale is inlined by the same + // Worker. Any translation state that a Worker retains between requests is then observable. + function createInliner(outputFiles: BuildOutputFile[]): I18nInliner { + inliner = new I18nInliner({ missingTranslation: 'warning', outputFiles }, 1); + + return inliner; + } + + afterEach(async () => { + await inliner?.close(); + inliner = undefined; + }); + + it('inlines the translations of a locale', async () => { + const { outputFiles, errors, warnings } = await createInliner([ + browserFile('main.js', GREETING_SOURCE), + ]).inlineForLocale('fr', { greeting: translationFor('Bonjour') }); + + expect(errors).toEqual([]); + expect(warnings).toEqual([]); + expect(findFile(outputFiles, 'main.js').text).toContain('"Bonjour"'); + expect(findFile(outputFiles, 'main.js').text).not.toContain('$localize'); + }); + + it('inlines the translations of each locale when several are inlined in sequence', async () => { + const localeInliner = createInliner([browserFile('main.js', GREETING_SOURCE)]); + + const french = await localeInliner.inlineForLocale('fr', { + greeting: translationFor('Bonjour'), + }); + const german = await localeInliner.inlineForLocale('de', { greeting: translationFor('Hallo') }); + // Repeats the first locale to cover a locale being inlined again after another has been. + const frenchAgain = await localeInliner.inlineForLocale('fr', { + greeting: translationFor('Bonjour'), + }); + + expect(findFile(french.outputFiles, 'main.js').text).toContain('"Bonjour"'); + expect(findFile(german.outputFiles, 'main.js').text).toContain('"Hallo"'); + expect(findFile(frenchAgain.outputFiles, 'main.js').text).toContain('"Bonjour"'); + }); + + it('inlines the translations of a locale into every file that uses them', async () => { + const { outputFiles } = await createInliner([ + browserFile('main.js', GREETING_SOURCE), + browserFile('chunk.js', GREETING_SOURCE), + ]).inlineForLocale('fr', { greeting: translationFor('Bonjour') }); + + expect(findFile(outputFiles, 'main.js').text).toContain('"Bonjour"'); + expect(findFile(outputFiles, 'chunk.js').text).toContain('"Bonjour"'); + }); + + it('retains the original messages for a locale without translations', async () => { + const { outputFiles, errors, warnings } = await createInliner([ + browserFile('main.js', GREETING_SOURCE), + ]).inlineForLocale('en-US', undefined); + + // A locale without translations is the source locale, so its messages are not missing. + expect(errors).toEqual([]); + expect(warnings).toEqual([]); + expect(findFile(outputFiles, 'main.js').text).toContain('"Hello"'); + }); + + it('warns and retains the original message when a locale is missing a translation', async () => { + const { outputFiles, errors, warnings } = await createInliner([ + browserFile('main.js', GREETING_SOURCE), + ]).inlineForLocale('fr', { unrelated: translationFor('Sans rapport') }); + + expect(errors).toEqual([]); + expect(warnings.length).toBe(1); + expect(warnings[0]).toContain('greeting'); + expect(findFile(outputFiles, 'main.js').text).toContain('"Hello"'); + }); + + it('replaces the locale placeholder with the locale being inlined', async () => { + // The placeholder is only inlined for files that use `$localize`, which is where the build + // inserts it, so the message is present alongside it here. + const { outputFiles } = await createInliner([ + browserFile('main.js', `export const locale = "___NG_LOCALE_INSERT___";\n${GREETING_SOURCE}`), + ]).inlineForLocale('fr', { greeting: translationFor('Bonjour') }); + + expect(findFile(outputFiles, 'main.js').text).toContain('"fr"'); + expect(findFile(outputFiles, 'main.js').text).not.toContain('___NG_LOCALE_INSERT___'); + }); + + it('remaps the source map of a file it modifies', async () => { + // esbuild provides a map from the emitted code back to an original file, matching what the + // inliner receives during a build. + const { code, map } = await transform(GREETING_SOURCE, { + sourcefile: 'greeting.ts', + loader: 'ts', + sourcemap: 'external', + }); + + const { outputFiles } = await createInliner([ + browserFile('main.js', code), + browserFile('main.js.map', map), + ]).inlineForLocale('fr', { greeting: translationFor('Bonjour') }); + + const outputMap = JSON.parse(findFile(outputFiles, 'main.js.map').text) as { + version: number; + sources: string[]; + mappings: string; + }; + + expect(outputMap.version).toBe(3); + // The map must still resolve to the original file rather than to the inliner's input. + expect(outputMap.sources).toContain('greeting.ts'); + expect(outputMap.mappings.length).toBeGreaterThan(0); + }); + + describe('inlineTemplateUpdate', () => { + it('inlines the translations of a locale into a template update', async () => { + const { code, errors, warnings } = await createInliner([]).inlineTemplateUpdate( + 'fr', + { greeting: translationFor('Bonjour') }, + GREETING_SOURCE, + 'template-id', + ); + + expect(errors).toEqual([]); + expect(warnings).toEqual([]); + expect(code).toContain('"Bonjour"'); + expect(code).not.toContain('$localize'); + }); + + it('retains the original messages for a locale without translations', async () => { + const { code, errors, warnings } = await createInliner([]).inlineTemplateUpdate( + 'en-US', + undefined, + GREETING_SOURCE, + 'template-id', + ); + + expect(errors).toEqual([]); + expect(warnings).toEqual([]); + expect(code).toContain('"Hello"'); + }); + + it('returns the code untouched when it has no localize calls', async () => { + const source = 'export const answer = 42;\n'; + const { code } = await createInliner([]).inlineTemplateUpdate( + 'fr', + { greeting: translationFor('Bonjour') }, + source, + 'template-id', + ); + + expect(code).toBe(source); + }); + }); + + it('leaves files without localize calls unmodified', async () => { + const { outputFiles } = await createInliner([ + browserFile('main.js', GREETING_SOURCE), + browserFile('other.js', 'export const answer = 42;\n'), + ]).inlineForLocale('fr', { greeting: translationFor('Bonjour') }); + + expect(findFile(outputFiles, 'other.js').text).toBe('export const answer = 42;\n'); + }); +});