From 19e7095a4f3bbd6891710311d58bef4fb21da05d Mon Sep 17 00:00:00 2001 From: James Brunton Date: Mon, 3 Aug 2026 15:34:21 +0100 Subject: [PATCH] Convert unused translations test to use a trie to more efficiently scan (#7263) # Description of Changes Unused translations test takes ~8 seconds to run on my computer with no contention, but when my CPU is under heavy contention, it often takes >20 seconds and occasionally goes over the 30 second timeout. This PR changes the test to use [a trie](https://en.wikipedia.org/wiki/Trie) to more efficiently search for the strings, taking the test down to ~1.8 seconds. I've also increased the timeout for the missing & unused translations tests for belt-and-braces. --- .../editor/src/core/i18n/translationAudit.ts | 78 ++++++++++++++++--- .../core/tests/missingTranslations.test.ts | 3 +- .../src/core/tests/unusedTranslations.test.ts | 3 +- 3 files changed, 71 insertions(+), 13 deletions(-) diff --git a/frontend/editor/src/core/i18n/translationAudit.ts b/frontend/editor/src/core/i18n/translationAudit.ts index 9e4e57a8a5..2a9bd33d6d 100644 --- a/frontend/editor/src/core/i18n/translationAudit.ts +++ b/frontend/editor/src/core/i18n/translationAudit.ts @@ -218,14 +218,15 @@ const extractStaticKeys = (file: string): MissingKey[] => { file, code, ts.ScriptTarget.Latest, - true, + false, getScriptKind(file), ); const found: MissingKey[] = []; const record = (node: ts.Node, key: string, fallback = "") => { + // Pass sourceFile explicitly: the tree is parsed without parent links. const { line, character } = sourceFile.getLineAndCharacterOfPosition( - node.getStart(), + node.getStart(sourceFile), ); found.push({ key, fallback, file, line: line + 1, column: character + 1 }); }; @@ -287,8 +288,11 @@ const extractStaticKeys = (file: string): MissingKey[] => { * (not just t() sites; keys are often assembled in helpers/constants), but * only kept if a shape carries an identifier-like static fragment. */ -const extractTemplateShapes = (file: string, acc: Set): void => { - const code = fs.readFileSync(file, "utf8"); +const extractTemplateShapes = ( + file: string, + code: string, + acc: Set, +): void => { if (!code.includes("${")) return; const sourceFile = ts.createSourceFile( @@ -319,6 +323,53 @@ const extractTemplateShapes = (file: string, acc: Set): void => { ts.forEachChild(sourceFile, visit); }; +interface TrieNode { + children: Map; + needle?: string; +} + +/** + * Which of `needles` appear as a substring of `text`. + * + * Same answer as `needles.filter((n) => text.includes(n))`, but walks the text + * once against a trie of every needle rather than once per needle: the locale + * carries thousands of keys and the joined source is tens of MB, so per-key + * scanning dominated the suite's runtime. + */ +const findSubstrings = ( + text: string, + needles: Iterable, +): Set => { + const root: TrieNode = { children: new Map() }; + for (const needle of needles) { + if (!needle) continue; + let node = root; + for (let i = 0; i < needle.length; i++) { + const code = needle.charCodeAt(i); + let next = node.children.get(code); + if (!next) { + next = { children: new Map() }; + node.children.set(code, next); + } + node = next; + } + node.needle = needle; + } + + const found = new Set(); + for (let start = 0; start < text.length; start++) { + let node = root.children.get(text.charCodeAt(start)); + let at = start; + while (node) { + if (node.needle !== undefined) found.add(node.needle); + at += 1; + if (at >= text.length) break; + node = node.children.get(text.charCodeAt(at)); + } + } + return found; +}; + const shapeToMatcher = (shape: string): RegExp => { // Each * is one runtime path segment, so match `[^.]+` (not `.+`) to avoid // spanning key levels. Multi-segment interpolations use ignoredKeyPatterns. @@ -363,19 +414,24 @@ export function findUnusedKeys(project: TranslationProject): { } { const localeKeys = Array.from(collectLocaleKeys(project.localeFile)); const files = listSourceFiles([project.srcRoot]); - const source = files.map((file) => fs.readFileSync(file, "utf8")).join("\n"); + const contents = files.map((file) => fs.readFileSync(file, "utf8")); + const source = contents.join("\n"); const shapes = new Set(); - for (const file of files) extractTemplateShapes(file, shapes); + files.forEach((file, i) => extractTemplateShapes(file, contents[i], shapes)); const matchers = [...shapes].map(shapeToMatcher); const patterns = project.ignoredKeyPatterns ?? []; - const unused = localeKeys.filter((key) => { - if (patterns.some((re) => re.test(key))) return false; + const candidates = localeKeys.filter( + (key) => !patterns.some((re) => re.test(key)), + ); + // Direct: the literal appears anywhere in source (static t(), i18nKey, + // constants, comments). Plural variants count when their base is referenced. + const referenced = findSubstrings(source, candidates.flatMap(getLookupKeys)); + + const unused = candidates.filter((key) => { const lookups = getLookupKeys(key); - // Direct: the literal appears anywhere in source (static t(), i18nKey, - // constants, comments). Plural variants count when their base is referenced. - if (lookups.some((k) => source.includes(k))) return false; + if (lookups.some((k) => referenced.has(k))) return false; // Dynamic: the key matches a template-literal shape from source. return !lookups.some((k) => matchers.some((re) => re.test(k))); }); diff --git a/frontend/editor/src/core/tests/missingTranslations.test.ts b/frontend/editor/src/core/tests/missingTranslations.test.ts index 617699e43a..64b6392dc1 100644 --- a/frontend/editor/src/core/tests/missingTranslations.test.ts +++ b/frontend/editor/src/core/tests/missingTranslations.test.ts @@ -14,7 +14,8 @@ describe.each(I18N_PROJECTS)( (project) => { test( "fails if any en-US key used in source is missing from the locale", - { timeout: 10000 }, + // Scans/parses the whole source tree: generous headroom for a loaded CPU. + { timeout: 60_000 }, () => { expect(fs.existsSync(project.localeFile)).toBe(true); diff --git a/frontend/editor/src/core/tests/unusedTranslations.test.ts b/frontend/editor/src/core/tests/unusedTranslations.test.ts index f0d0fbce93..e91c2f4485 100644 --- a/frontend/editor/src/core/tests/unusedTranslations.test.ts +++ b/frontend/editor/src/core/tests/unusedTranslations.test.ts @@ -15,7 +15,8 @@ describe.each(I18N_PROJECTS)( (project) => { test( "fails if any en-US key has no source references", - { timeout: 30_000 }, + // Scans/parses the whole source tree: generous headroom for a loaded CPU. + { timeout: 60_000 }, () => { expect(fs.existsSync(project.localeFile)).toBe(true);