Redesign toml sorting to speed up from ~40s to ~2s (#7192)

# Description of Changes
The `pre-commit` tool to sort the translations is really slow. It took
~40 seconds to run because it's using a parser which attempts to save
all of the formatting data from the Toml. Our translations toml is
pretty much entirely formatted anyway, so there's no point in trying to
preserve any of that data. The only thing we lose is 5 comments, none of
which are needed anyway and only appear in the US translation file. By
switching to Python stdlib `tomllib` reading and `tomli-w` for writing,
we can make the Toml formatting job take 2.11 seconds, where it used to
take 39.78s. The whole pre-commit job now takes 4.58 seconds.
This commit is contained in:
James Brunton
2026-07-29 14:33:34 +00:00
committed by GitHub
parent 4d207f0c3f
commit bbd4d2c3ac
5 changed files with 105 additions and 26 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ requires-python = ">=3.11"
dependencies = [
"ruff==0.15.14",
"codespell==2.4.2",
"toml-sort==0.24.4",
"tomli-w==1.2.0",
]
[tool.uv]
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Key-sort the locale translation.toml files.
python sort_locale_toml.py <pathspec>... # check: report, exit 1 if unsorted
python sort_locale_toml.py --fix <pathspec>... # fix: rewrite in place
"""
from __future__ import annotations
import subprocess
import sys
import tomllib
from pathlib import Path
import tomli_w
class SortError(Exception):
"""A file could not be sorted without risking its contents."""
def ordered(table: dict[str, object]) -> dict[str, object]:
"""Rebuild a table with its keys sorted, and sub-tables after its own keys."""
keys = {key: value for key, value in table.items() if not isinstance(value, dict)}
subtables = {key: value for key, value in table.items() if isinstance(value, dict)}
result: dict[str, object] = {key: keys[key] for key in sorted(keys, key=str.lower)}
for key in sorted(subtables, key=str.lower):
result[key] = ordered(subtables[key])
return result
def tracked_files(path_specs: list[str]) -> list[str]:
result = subprocess.run(
["git", "ls-files", "-z", *path_specs],
check=True,
capture_output=True,
text=True,
)
return [path for path in result.stdout.split("\0") if path]
def sort_file(path: str, fix: bool) -> bool:
"""Rewrite one file if `fix`; return whether it was not already sorted."""
text = Path(path).read_text(encoding="utf-8")
try:
original = tomllib.loads(text)
except tomllib.TOMLDecodeError as exc:
raise SortError(f"{path}: invalid TOML: {exc}") from exc
expected = tomli_w.dumps(ordered(original))
if expected == text:
return False
try:
reordered = tomllib.loads(expected)
except tomllib.TOMLDecodeError as exc:
raise SortError(
f"{path}: refusing to sort, the sorted output is not valid TOML: {exc}"
) from exc
if reordered != original:
raise SortError(
f"{path}: refusing to sort, sorting would change the file's contents"
)
if fix:
Path(path).write_text(expected, encoding="utf-8")
return True
def main() -> int:
args = sys.argv[1:]
fix = "--fix" in args
pathspecs = [a for a in args if a != "--fix"]
offenders: list[str] = []
errors: list[str] = []
for path in tracked_files(pathspecs):
try:
if sort_file(path, fix):
offenders.append(path)
except SortError as exc:
errors.append(str(exc))
for error in errors:
print(error, file=sys.stderr)
if offenders and not fix:
print(f"{len(offenders)} file(s) need TOML sorting:")
for path in offenders:
print(f" {path}")
if offenders and fix:
print(f"Sorted TOML in {len(offenders)} file(s).")
return 1 if errors or (offenders and not fix) else 0
if __name__ == "__main__":
sys.exit(main())
+6 -18
View File
@@ -43,33 +43,21 @@ source = { virtual = "." }
dependencies = [
{ name = "codespell" },
{ name = "ruff" },
{ name = "toml-sort" },
{ name = "tomli-w" },
]
[package.metadata]
requires-dist = [
{ name = "codespell", specifier = "==2.4.2" },
{ name = "ruff", specifier = "==0.15.14" },
{ name = "toml-sort", specifier = "==0.24.4" },
{ name = "tomli-w", specifier = "==1.2.0" },
]
[[package]]
name = "toml-sort"
version = "0.24.4"
name = "tomli-w"
version = "1.2.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomlkit" },
]
sdist = { url = "https://files.pythonhosted.org/packages/47/c5/d6f650fdcf8e1f83096815fa67fb13a9a345b99da6015c60c4b7e4a8ea2b/toml_sort-0.24.4.tar.gz", hash = "sha256:429b69f5b98b7047a11380c80ecf0838556bdea1a8902d0be564961c48841423", size = 17793, upload-time = "2026-03-24T14:05:53.637Z" }
sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/0f/5a/1f0e54df4eacf0f4d8f94ba50cf72be33d2a3f04babdfb1931bead48a0ab/toml_sort-0.24.4-py3-none-any.whl", hash = "sha256:125aa5fb94f33c542c6901040456145dd38f79bbb310b56b436a93057d30a739", size = 16577, upload-time = "2026-03-24T14:05:54.757Z" },
]
[[package]]
name = "tomlkit"
version = "0.15.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/51/db/03eaf4331631ef6b27d6e3c9b68c54dc6f0d63d87201fed600cc409307fd/tomlkit-0.15.0.tar.gz", hash = "sha256:7d1a9ecba3086638211b13814ea79c90dd54dd11993564376f3aa92271f5c7a3", size = 161875, upload-time = "2026-05-10T07:38:22.245Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/6a/43/8bd850ee71a191bf072e31302c73a66be413fecdd98fdcd111ecbcce13ca/tomlkit-0.15.0-py3-none-any.whl", hash = "sha256:4dbc8f0fc024412b57ced8757ac7461305126a648ff8c2c807fcb8e133a78738", size = 41328, upload-time = "2026-05-10T07:38:23.517Z" },
{ url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" },
]