From bbd4d2c3ac106321edcbf9c8df8ef9f45e2ae70f Mon Sep 17 00:00:00 2001 From: James Brunton Date: Wed, 29 Jul 2026 15:33:34 +0100 Subject: [PATCH] Redesign toml sorting to speed up from ~40s to ~2s (#7192) # Description of Changes The `pre-commit` tool to sort the translations is really slow. It took ~40 seconds to run because it's using a parser which attempts to save all of the formatting data from the Toml. Our translations toml is pretty much entirely formatted anyway, so there's no point in trying to preserve any of that data. The only thing we lose is 5 comments, none of which are needed anyway and only appear in the US translation file. By switching to Python stdlib `tomllib` reading and `tomli-w` for writing, we can make the Toml formatting job take 2.11 seconds, where it used to take 39.78s. The whole pre-commit job now takes 4.58 seconds. --- .taskfiles/pre-commit.yml | 4 +- .../public/locales/en-US/translation.toml | 5 - scripts/pre-commit/pyproject.toml | 2 +- scripts/pre-commit/sort_locale_toml.py | 96 +++++++++++++++++++ scripts/pre-commit/uv.lock | 24 ++--- 5 files changed, 105 insertions(+), 26 deletions(-) create mode 100644 scripts/pre-commit/sort_locale_toml.py diff --git a/.taskfiles/pre-commit.yml b/.taskfiles/pre-commit.yml index 136f852a5b..f2e488e815 100644 --- a/.taskfiles/pre-commit.yml +++ b/.taskfiles/pre-commit.yml @@ -73,7 +73,7 @@ tasks: - task: gitleaks install: - desc: "Install the pinned pre-commit Python tools (ruff, codespell, toml-sort)" + desc: "Install the pinned pre-commit Python tools" run: once cmds: - uv sync --project scripts/pre-commit --locked @@ -112,7 +112,7 @@ tasks: toml-sort: deps: [install] cmds: - - uv run --project scripts/pre-commit --no-sync toml-sort --all --ignore-case {{if .FIX}}--in-place{{else}}--check{{end}} {{.LOCALE_TOML}} + - uv run --project scripts/pre-commit --no-sync python scripts/pre-commit/sort_locale_toml.py {{if .FIX}}--fix {{end}}{{.LOCALE_TOML}} whitespace: cmds: diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index 913cdf20a4..abbe1446ff 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -692,7 +692,6 @@ manualLinks = "Manual downloads: click the links and place the files into the te noLanguages = "No tessdata languages found in the configured directory." permissionNotice = "The tessdata path is not writable. Downloads will be opened in the browser; please save the .traineddata files manually into the tessdata folder." -# AI engine admin settings (AI nav group) [admin.settings.ai.documents] description = "Configure the embedding model and retrieval settings used to answer questions over documents. Applied to the AI engine when saved." title = "Documents & RAG" @@ -7320,7 +7319,6 @@ sectionsAriaLabel = "Infrastructure sections" subtitle = "Deployments, credentials, security posture, storage, and the audit trail for your Stirling workspace." title = "Infrastructure" -# Fixed-enum label maps rendered via t(MAP[value]) in the infrastructure tabs. [portal.infrastructure.apiKeys] createKey = "Create key" heading = "API keys" @@ -8415,13 +8413,10 @@ region = "State / region" regionPlaceholder = "California" running = "{{annual}} / yr ยท {{years}}-yr {{tcv}}" s1Sub = "Your team, and the PDFs you expect to run each year." -# Step 1 โ€” volume s1Title = "How much will you process?" s2Sub = "Longer terms discount the rate; your service level sets support." -# Step 2 โ€” commitment & service s2Title = "Commitment and service" s3Sub = "For the quote and the agreement it generates." -# Step 3 โ€” details s3Title = "Your details" serviceLevel = "Service level" size_compact = "Compact" diff --git a/scripts/pre-commit/pyproject.toml b/scripts/pre-commit/pyproject.toml index cd13b56f5e..9a730bdcb5 100644 --- a/scripts/pre-commit/pyproject.toml +++ b/scripts/pre-commit/pyproject.toml @@ -9,7 +9,7 @@ requires-python = ">=3.11" dependencies = [ "ruff==0.15.14", "codespell==2.4.2", - "toml-sort==0.24.4", + "tomli-w==1.2.0", ] [tool.uv] diff --git a/scripts/pre-commit/sort_locale_toml.py b/scripts/pre-commit/sort_locale_toml.py new file mode 100644 index 0000000000..1515fe2cfe --- /dev/null +++ b/scripts/pre-commit/sort_locale_toml.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Key-sort the locale translation.toml files. + +python sort_locale_toml.py ... # check: report, exit 1 if unsorted +python sort_locale_toml.py --fix ... # fix: rewrite in place +""" + +from __future__ import annotations + +import subprocess +import sys +import tomllib +from pathlib import Path + +import tomli_w + + +class SortError(Exception): + """A file could not be sorted without risking its contents.""" + + +def ordered(table: dict[str, object]) -> dict[str, object]: + """Rebuild a table with its keys sorted, and sub-tables after its own keys.""" + keys = {key: value for key, value in table.items() if not isinstance(value, dict)} + subtables = {key: value for key, value in table.items() if isinstance(value, dict)} + result: dict[str, object] = {key: keys[key] for key in sorted(keys, key=str.lower)} + for key in sorted(subtables, key=str.lower): + result[key] = ordered(subtables[key]) + return result + + +def tracked_files(path_specs: list[str]) -> list[str]: + result = subprocess.run( + ["git", "ls-files", "-z", *path_specs], + check=True, + capture_output=True, + text=True, + ) + return [path for path in result.stdout.split("\0") if path] + + +def sort_file(path: str, fix: bool) -> bool: + """Rewrite one file if `fix`; return whether it was not already sorted.""" + text = Path(path).read_text(encoding="utf-8") + try: + original = tomllib.loads(text) + except tomllib.TOMLDecodeError as exc: + raise SortError(f"{path}: invalid TOML: {exc}") from exc + + expected = tomli_w.dumps(ordered(original)) + if expected == text: + return False + + try: + reordered = tomllib.loads(expected) + except tomllib.TOMLDecodeError as exc: + raise SortError( + f"{path}: refusing to sort, the sorted output is not valid TOML: {exc}" + ) from exc + if reordered != original: + raise SortError( + f"{path}: refusing to sort, sorting would change the file's contents" + ) + + if fix: + Path(path).write_text(expected, encoding="utf-8") + return True + + +def main() -> int: + args = sys.argv[1:] + fix = "--fix" in args + pathspecs = [a for a in args if a != "--fix"] + + offenders: list[str] = [] + errors: list[str] = [] + for path in tracked_files(pathspecs): + try: + if sort_file(path, fix): + offenders.append(path) + except SortError as exc: + errors.append(str(exc)) + + for error in errors: + print(error, file=sys.stderr) + if offenders and not fix: + print(f"{len(offenders)} file(s) need TOML sorting:") + for path in offenders: + print(f" {path}") + if offenders and fix: + print(f"Sorted TOML in {len(offenders)} file(s).") + return 1 if errors or (offenders and not fix) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pre-commit/uv.lock b/scripts/pre-commit/uv.lock index b90593a6e8..2a3c8ee11d 100644 --- a/scripts/pre-commit/uv.lock +++ b/scripts/pre-commit/uv.lock @@ -43,33 +43,21 @@ source = { virtual = "." } dependencies = [ { name = "codespell" }, { name = "ruff" }, - { name = "toml-sort" }, + { name = "tomli-w" }, ] [package.metadata] requires-dist = [ { name = "codespell", specifier = "==2.4.2" }, { name = "ruff", specifier = "==0.15.14" }, - { name = "toml-sort", specifier = "==0.24.4" }, + { name = "tomli-w", specifier = "==1.2.0" }, ] [[package]] -name = "toml-sort" -version = "0.24.4" +name = "tomli-w" +version = "1.2.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tomlkit" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/47/c5/d6f650fdcf8e1f83096815fa67fb13a9a345b99da6015c60c4b7e4a8ea2b/toml_sort-0.24.4.tar.gz", hash = "sha256:429b69f5b98b7047a11380c80ecf0838556bdea1a8902d0be564961c48841423", size = 17793, upload-time = "2026-03-24T14:05:53.637Z" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/5a/1f0e54df4eacf0f4d8f94ba50cf72be33d2a3f04babdfb1931bead48a0ab/toml_sort-0.24.4-py3-none-any.whl", hash = "sha256:125aa5fb94f33c542c6901040456145dd38f79bbb310b56b436a93057d30a739", size = 16577, upload-time = "2026-03-24T14:05:54.757Z" }, -] - -[[package]] -name = "tomlkit" -version = "0.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, ]