Stirling-Tools/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Stirling-Tools/Stirling-PDF.git synced 2026-09-03 05:10:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/auto-form-detection

Browse Source
This commit is contained in:
Anthony Stirling
2026-08-11 11:00:23 +01:00
parent f6c4982a7c 05eb74022a
commit ab14889f6b
254 changed files with 17441 additions and 5122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/analyze_pdf_json.py
+12 -22
View File
@@ -15,9 +15,10 @@ from __future__ import annotations
import argparse
import json
import math
from pathlib import Path
from collections.abc import Iterable
from dataclasses import dataclass
from typing import Any, Dict, Iterable, List, Tuple
from pathlib import Path
from typing import Any
def human_bytes(value: float) -> str:
@@ -49,7 +50,7 @@ class FontBreakdown:
web_program_bytes: int = 0
pdf_program_bytes: int = 0
metadata_bytes: int = 0
sample_cos_ids: List[Tuple[str | None, str | None]] = None
sample_cos_ids: list[tuple[str | None, str | None]] = None
@dataclass
@@ -82,7 +83,7 @@ def approx_struct_size(obj: Any) -> int:
return len(json.dumps(obj, separators=(",", ":")))
def analyze_fonts(fonts: Iterable[Dict[str, Any]]) -> FontBreakdown:
def analyze_fonts(fonts: Iterable[dict[str, Any]]) -> FontBreakdown:
total = 0
with_cos = 0
with_prog = 0
@@ -92,7 +93,7 @@ def analyze_fonts(fonts: Iterable[Dict[str, Any]]) -> FontBreakdown:
web_program_bytes = 0
pdf_program_bytes = 0
metadata_bytes = 0
sample_cos_ids: List[Tuple[str | None, str | None]] = []
sample_cos_ids: list[tuple[str | None, str | None]] = []
for font in fonts:
total += 1
@@ -105,11 +106,7 @@ def analyze_fonts(fonts: Iterable[Dict[str, Any]]) -> FontBreakdown:
sample_cos_ids.append((font_id, uid))
metadata_bytes += approx_struct_size(
{
k: v
for k, v in font.items()
if k not in {"program", "webProgram", "pdfProgram"}
}
{k: v for k, v in font.items() if k not in {"program", "webProgram", "pdfProgram"}}
)
program = font.get("program")
@@ -140,7 +137,7 @@ def analyze_fonts(fonts: Iterable[Dict[str, Any]]) -> FontBreakdown:
)
def analyze_pages(pages: Iterable[Dict[str, Any]]) -> PageBreakdown:
def analyze_pages(pages: Iterable[dict[str, Any]]) -> PageBreakdown:
page_count = 0
total_text = 0
total_images = 0
@@ -185,7 +182,7 @@ def analyze_pages(pages: Iterable[Dict[str, Any]]) -> PageBreakdown:
)
def analyze_document(document: Dict[str, Any], total_size: int) -> DocumentBreakdown:
def analyze_document(document: dict[str, Any], total_size: int) -> DocumentBreakdown:
fonts = document.get("fonts") or []
pages = document.get("pages") or []
metadata = document.get("metadata") or {}
@@ -259,17 +256,10 @@ def main() -> None:
print(f" XMP metadata bytes: {human_bytes(summary.xmp_bytes)}")
print(f" Form fields bytes: {human_bytes(summary.form_fields_bytes)}")
print(f" Lazy flag bytes: {summary.lazy_flag_bytes}")
print(
f" Text payload characters (not counting JSON overhead): "
f"{page_stats.text_payload_chars:,}"
)
print(f" Text payload characters (not counting JSON overhead): {page_stats.text_payload_chars:,}")
print(f" Approx text structure bytes: {human_bytes(page_stats.text_struct_bytes)}")
print(
f" Approx image structure bytes: {human_bytes(page_stats.image_struct_bytes)}"
)
print(
f" Approx content stream bytes: {human_bytes(page_stats.content_stream_bytes)}"
)
print(f" Approx image structure bytes: {human_bytes(page_stats.image_struct_bytes)}")
print(f" Approx content stream bytes: {human_bytes(page_stats.content_stream_bytes)}")
print(f" Approx annotations bytes: {human_bytes(page_stats.annotations_bytes)}")
scripts/convert_cff_to_ttf.py
+11 -24
View File
@@ -4,12 +4,13 @@ Wrap raw CFF/Type1C data (extracted from PDFs) as OpenType-CFF for web compatibi
Builds proper Unicode cmap from PDF ToUnicode data.
"""
import sys
import re
from pathlib import Path
import sys
from io import BytesIO
from fontTools.ttLib import TTFont, newTable
from pathlib import Path
from fontTools.cffLib import CFFFontSet
from fontTools.ttLib import TTFont, newTable
from fontTools.ttLib.tables._c_m_a_p import cmap_format_4, cmap_format_12
from fontTools.ttLib.tables._n_a_m_e import NameRecord
from fontTools.ttLib.tables.O_S_2f_2 import Panose
@@ -117,15 +118,11 @@ def wrap_cff_as_otf(input_path, output_path, tounicode_path=None):
# Get glyph names
if hasattr(cff_font, "charset") and cff_font.charset is not None:
glyph_order = [".notdef"] + [
name for name in cff_font.charset if name != ".notdef"
]
glyph_order = [".notdef"] + [name for name in cff_font.charset if name != ".notdef"]
else:
# Fallback to CharStrings keys
charstrings = cff_font.CharStrings
glyph_order = [".notdef"] + [
name for name in charstrings.keys() if name != ".notdef"
]
glyph_order = [".notdef"] + [name for name in charstrings.keys() if name != ".notdef"]
otf.setGlyphOrder(glyph_order)
@@ -139,9 +136,7 @@ def wrap_cff_as_otf(input_path, output_path, tounicode_path=None):
# Get defaults from CFF Private dict
private_dict = getattr(cff_font, "Private", None)
default_width = (
getattr(private_dict, "defaultWidthX", 500) if private_dict else 500
)
default_width = getattr(private_dict, "defaultWidthX", 500) if private_dict else 500
# Calculate bounding box, widths, and LSBs
x_min = 0
@@ -280,9 +275,7 @@ def wrap_cff_as_otf(input_path, output_path, tounicode_path=None):
# For CID fonts: glyph names are "cid00123" (5-digit zero-padded)
# For non-CID fonts: glyph names vary but GID == array index
is_cid_font = any(
gn.startswith("cid") for gn in glyph_order[1:6]
) # Check first few non-.notdef glyphs
is_cid_font = any(gn.startswith("cid") for gn in glyph_order[1:6]) # Check first few non-.notdef glyphs
for gid, unicode_val in gid_to_unicode.items():
if unicode_val > 0:
@@ -355,14 +348,10 @@ def wrap_cff_as_otf(input_path, output_path, tounicode_path=None):
cmap4_mac.cmap = {cp: gn for cp, gn in unicode_to_glyph.items() if cp <= 0xFFFF}
cmap_tables.append(cmap4_mac)
cmap.tables = [t for t in cmap_tables if t.cmap] or [
cmap4_win
] # Ensure at least one
cmap.tables = [t for t in cmap_tables if t.cmap] or [cmap4_win] # Ensure at least one
otf["cmap"] = cmap
print(
f"Built cmap with {len(unicode_to_glyph)} Unicode mappings", file=sys.stderr
)
print(f"Built cmap with {len(unicode_to_glyph)} Unicode mappings", file=sys.stderr)
# === Create OS/2 table with correct metrics ===
os2 = newTable("OS/2")
@@ -515,9 +504,7 @@ Examples:
# Add named arguments
parser.add_argument("--input", dest="input_file", help="Input CFF file path")
parser.add_argument("--output", dest="output_file", help="Output OTF file path")
parser.add_argument(
"--to-unicode", dest="tounicode_file", help="ToUnicode mapping file path"
)
parser.add_argument("--to-unicode", dest="tounicode_file", help="ToUnicode mapping file path")
# Add positional arguments for backward compatibility
parser.add_argument("input_pos", nargs="?", help="Input CFF file (positional)")
scripts/counter_translation_v3.py
+6 -18
View File
@@ -50,16 +50,13 @@ import glob
import os
import re
import sys
from collections.abc import Mapping
from typing import Iterable
from collections.abc import Iterable, Mapping
# Ensure tomlkit is installed before importing
try:
import tomlkit
except ImportError:
raise ImportError(
"The 'tomlkit' library is not installed. Please install it using 'pip install tomlkit'."
)
raise ImportError("The 'tomlkit' library is not installed. Please install it using 'pip install tomlkit'.")
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
@@ -242,15 +239,11 @@ def compare_files(
"ignore" not in sort_ignore_translation[language]
or len(sort_ignore_translation[language].get("ignore", [])) < 1
):
sort_ignore_translation[language]["ignore"] = tomlkit.array(
["language.direction"]
)
sort_ignore_translation[language]["ignore"] = tomlkit.array(["language.direction"])
# Clean up ignore list to only include keys present in reference
sort_ignore_translation[language]["ignore"] = [
key
for key in sort_ignore_translation[language]["ignore"]
if key in ref_keys or key == "language.direction"
key for key in sort_ignore_translation[language]["ignore"] if key in ref_keys or key == "language.direction"
]
translation_entries = load_translation_entries(file_path)
@@ -264,10 +257,7 @@ def compare_files(
continue
file_value = translation_entries[default_key]
if (
default_value == file_value
and default_key not in sort_ignore_translation[language]["ignore"]
):
if default_value == file_value and default_key not in sort_ignore_translation[language]["ignore"]:
# Missing translation (same as default and not ignored)
fails += 1
missing_str_keys.append(default_key)
@@ -357,9 +347,7 @@ def main() -> None:
lang_file = lang_input
else:
candidate = os.path.join(locales_dir, lang_input)
candidate_with_file = os.path.join(
locales_dir, lang_input, "translation.toml"
)
candidate_with_file = os.path.join(locales_dir, lang_input, "translation.toml")
if os.path.exists(candidate):
if os.path.isdir(candidate):
lang_file = candidate_with_file
scripts/coverage-matrix.py
+9 -17
View File
@@ -46,14 +46,14 @@ import json
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional
from defusedxml.ElementTree import ParseError as _XMLParseError
# defusedxml hardens the parser against XXE / billion-laughs / entity-
# expansion attacks. JaCoCo XML on a CI runner is trusted input today,
# but using the hardened parser is a one-line change and silences
# scanners that pattern-match on `xml.etree.ElementTree.parse`.
from defusedxml.ElementTree import parse as _xml_parse
from defusedxml.ElementTree import ParseError as _XMLParseError
AREAS = ("core", "proprietary", "saas", "desktop")
@@ -69,7 +69,7 @@ class Bucket:
def pct(self) -> float:
return 100.0 * self.covered / self.total if self.total else 0.0
def add(self, other: "Bucket") -> None:
def add(self, other: Bucket) -> None:
self.covered += other.covered
self.total += other.total
@@ -78,9 +78,7 @@ class Bucket:
class RowBuckets:
"""Per-area buckets for one row of the matrix."""
by_area: dict[str, Bucket] = field(
default_factory=lambda: {a: Bucket() for a in AREAS}
)
by_area: dict[str, Bucket] = field(default_factory=lambda: {a: Bucket() for a in AREAS})
# Some inputs (Playwright V8) don't have source-path info, so they
# only contribute to ALL without an area attribution. Track those
# separately so per-area cells stay honest.
@@ -94,7 +92,7 @@ class RowBuckets:
agg.add(self.unattributed)
return agg
def merge(self, other: "RowBuckets") -> None:
def merge(self, other: RowBuckets) -> None:
for area in AREAS:
self.by_area[area].add(other.by_area[area])
self.unattributed.add(other.unattributed)
@@ -103,7 +101,7 @@ class RowBuckets:
# --------------------------------------------------------------------- jacoco
def _classify_backend(package_name: str) -> Optional[str]:
def _classify_backend(package_name: str) -> str | None:
"""Map a JaCoCo package name to an area, or None to skip."""
if not package_name:
return None
@@ -152,7 +150,7 @@ def parse_jacoco_methods(path: Path) -> RowBuckets:
# --------------------------------------------------------------- vitest (frontend)
def _classify_frontend(file_path: str) -> Optional[str]:
def _classify_frontend(file_path: str) -> str | None:
"""Map a vitest per-file path (anything containing src/<area>/) to an area."""
if not file_path:
return None
@@ -170,9 +168,7 @@ def parse_vitest_per_file(path: Path) -> RowBuckets:
try:
data = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as exc:
print(
f"::warning::Failed to parse vitest summary {path}: {exc}", file=sys.stderr
)
print(f"::warning::Failed to parse vitest summary {path}: {exc}", file=sys.stderr)
return row
for file_path, metrics in data.items():
if file_path == "total":
@@ -267,11 +263,7 @@ def main(argv: list[str] | None = None) -> int:
args = parser.parse_args(argv)
# Build each row from its source(s).
fe_e2e = (
parse_playwright_frontend_total(args.playwright_frontend)
if args.playwright_frontend
else RowBuckets()
)
fe_e2e = parse_playwright_frontend_total(args.playwright_frontend) if args.playwright_frontend else RowBuckets()
fe_unit = parse_vitest_per_file(args.vitest) if args.vitest else RowBuckets()
fe_all = RowBuckets()
fe_all.merge(fe_unit)
scripts/coverage-summary.py
+5 -6
View File
@@ -21,16 +21,17 @@ import argparse
import json
import os
import sys
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable
from defusedxml.ElementTree import ParseError as _XMLParseError
# `defusedxml` swaps out the stdlib expat parser for one that rejects the
# usual XML attack vectors (XXE / billion laughs / entity expansion). Even
# though JaCoCo XML on a CI runner is trusted input, swapping the parser is
# a one-line change that silences security scanners and costs nothing.
from defusedxml.ElementTree import parse as _xml_parse
from defusedxml.ElementTree import ParseError as _XMLParseError
JACOCO_COUNTERS = ("LINE", "BRANCH", "METHOD", "CLASS", "INSTRUCTION", "COMPLEXITY")
@@ -48,7 +49,7 @@ class CounterTotals:
def pct(self) -> float:
return 100.0 * self.covered / self.total if self.total else 0.0
def add(self, other: "CounterTotals") -> None:
def add(self, other: CounterTotals) -> None:
self.covered += other.covered
self.missed += other.missed
@@ -108,9 +109,7 @@ def render_jacoco(reports: Iterable[tuple[str, Path]]) -> str:
return body
lines: list[str] = []
lines.append(
"| Metric | " + " | ".join(label for label, _ in rows) + " | **Aggregate** |"
)
lines.append("| Metric | " + " | ".join(label for label, _ in rows) + " | **Aggregate** |")
lines.append("|---" * (len(rows) + 2) + "|")
for t in ("LINE", "BRANCH", "METHOD", "CLASS"):
scripts/download_pdf_samples.py
+8 -16
View File
@@ -27,7 +27,6 @@ import os
import re
import sys
from pathlib import Path
from typing import List, Optional, Set, Tuple
from urllib.parse import unquote, urlparse
import requests
@@ -71,9 +70,9 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
def load_urls(args: argparse.Namespace) -> List[str]:
urls: List[str] = []
seen: Set[str] = set()
def load_urls(args: argparse.Namespace) -> list[str]:
urls: list[str] = []
seen: set[str] = set()
def add(url: str) -> None:
clean = url.strip()
@@ -125,7 +124,7 @@ def download_pdf(
output_dir: Path,
timeout: int,
overwrite: bool,
) -> Tuple[str, Optional[Path], Optional[str]]:
) -> tuple[str, Path | None, str | None]:
try:
dest = build_filename(url, output_dir)
if dest.exists() and not overwrite:
@@ -161,20 +160,15 @@ def main() -> None:
output_dir = Path(args.output_dir).resolve()
output_dir.mkdir(parents=True, exist_ok=True)
print(
f"Downloading {len(urls)} PDFs to {output_dir} using {args.workers} workers..."
)
print(f"Downloading {len(urls)} PDFs to {output_dir} using {args.workers} workers...")
successes = 0
skipped = 0
failures: List[Tuple[str, str]] = []
failures: list[tuple[str, str]] = []
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
future_to_url = {
executor.submit(
download_pdf, url, output_dir, args.timeout, args.overwrite
): url
for url in urls
executor.submit(download_pdf, url, output_dir, args.timeout, args.overwrite): url for url in urls
}
for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
@@ -190,9 +184,7 @@ def main() -> None:
print(f"[OK] {url} -> {path}")
print()
print(
f"Completed. Success: {successes}, Skipped: {skipped}, Failures: {len(failures)}"
)
print(f"Completed. Success: {successes}, Skipped: {skipped}, Failures: {len(failures)}")
if failures:
print("Failures:")
for url, error in failures:
scripts/generate_requirements.bat
-48
View File
@@ -1,48 +0,0 @@
@echo off
REM --------------------------------------------------
REM Batch script to (re-)generate all requirements
REM with check for pip-compile and user confirmation
REM --------------------------------------------------
REM Check if pip-compile is available
pip-compile --version >nul 2>&1
if %ERRORLEVEL% neq 0 (
echo ERROR: pip-compile was not found.
echo Please install pip-tools:
echo pip install pip-tools
echo and ensure that pip-compile is in your PATH.
pause
exit /b 1
)
echo pip-compile detected.
REM Prompt user for confirmation (default = Yes on ENTER)
set /p confirm="Do you want to generate all requirements? [Y/n] "
if /I "%confirm%"=="" set confirm=Y
if /I not "%confirm%"=="Y" (
echo Generation cancelled by user.
pause
exit /b 0
)
echo Starting generation...
echo Generating .github\scripts\requirements_dev.txt
pip-compile --allow-unsafe --generate-hashes --upgrade --strip-extras ^
--output-file=".github\scripts\requirements_dev.txt" ^
".github\scripts\requirements_dev.in"
echo Generating .github\scripts\requirements_sync_readme.txt
pip-compile --generate-hashes --upgrade --strip-extras ^
--output-file=".github\scripts\requirements_sync_readme.txt" ^
".github\scripts\requirements_sync_readme.in"
echo Generating testing\cucumber\requirements.txt
pip-compile --generate-hashes --upgrade --strip-extras ^
--output-file="testing\cucumber\requirements.txt" ^
"testing\cucumber\requirements.in"
echo All done!
pause
scripts/harvest_type3_fonts.py
+13 -23
View File
@@ -27,16 +27,14 @@ import re
import shlex
import subprocess
import sys
from collections.abc import Sequence
from pathlib import Path
from typing import Dict, List, Sequence, Tuple
REPO_ROOT = Path(__file__).resolve().parents[1]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Bulk collect Type3 font signatures from PDFs."
)
parser = argparse.ArgumentParser(description="Bulk collect Type3 font signatures from PDFs.")
parser.add_argument(
"--input",
nargs="+",
@@ -72,8 +70,8 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
def discover_pdfs(paths: Sequence[str]) -> List[Path]:
pdfs: List[Path] = []
def discover_pdfs(paths: Sequence[str]) -> list[Path]:
pdfs: list[Path] = []
for raw in paths:
path = Path(raw).resolve()
if path.is_file():
@@ -113,8 +111,8 @@ def load_signature_file(path: Path) -> dict:
return json.load(handle)
def collect_known_signatures(signatures_dir: Path) -> Dict[str, dict]:
known: Dict[str, dict] = {}
def collect_known_signatures(signatures_dir: Path) -> dict[str, dict]:
known: dict[str, dict] = {}
if not signatures_dir.exists():
return known
for json_file in signatures_dir.rglob("*.json"):
@@ -139,9 +137,7 @@ def collect_known_signatures(signatures_dir: Path) -> Dict[str, dict]:
return known
def run_signature_tool(
gradle_cmd: str, pdf: Path, output_path: Path, pretty: bool, cwd: Path
) -> None:
def run_signature_tool(gradle_cmd: str, pdf: Path, output_path: Path, pretty: bool, cwd: Path) -> None:
output_path.parent.mkdir(parents=True, exist_ok=True)
args = f"--pdf {shlex.quote(str(pdf))} --output {shlex.quote(str(output_path))}"
if pretty:
@@ -157,12 +153,10 @@ def run_signature_tool(
text=True,
)
if completed.returncode != 0:
raise RuntimeError(
f"Gradle Type3SignatureTool failed for {pdf}:\n{completed.stderr.strip()}"
)
raise RuntimeError(f"Gradle Type3SignatureTool failed for {pdf}:\n{completed.stderr.strip()}")
def extract_fonts_from_payload(payload: dict) -> List[dict]:
def extract_fonts_from_payload(payload: dict) -> list[dict]:
pdf = payload.get("pdf")
fonts = []
for font in payload.get("fonts", []):
@@ -182,7 +176,7 @@ def extract_fonts_from_payload(payload: dict) -> List[dict]:
return fonts
def write_report(report_path: Path, fonts_by_signature: Dict[str, dict]) -> None:
def write_report(report_path: Path, fonts_by_signature: dict[str, dict]) -> None:
ordered = sorted(fonts_by_signature.values(), key=lambda entry: entry["signature"])
report = {
"generatedAt": dt.datetime.utcnow().isoformat(timespec="seconds") + "Z",
@@ -201,7 +195,7 @@ def main() -> None:
pdfs = discover_pdfs(args.input)
known = collect_known_signatures(signatures_dir)
newly_added: List[Tuple[str, str]] = []
newly_added: list[tuple[str, str]] = []
for pdf in pdfs:
signature_path = derive_signature_path(pdf, signatures_dir)
@@ -209,15 +203,11 @@ def main() -> None:
try:
payload = load_signature_file(signature_path)
except Exception as exc:
print(
f"[WARN] Failed to parse cached signature {signature_path}: {exc}"
)
print(f"[WARN] Failed to parse cached signature {signature_path}: {exc}")
payload = None
else:
try:
run_signature_tool(
args.gradle_cmd, pdf, signature_path, args.pretty, REPO_ROOT
)
run_signature_tool(args.gradle_cmd, pdf, signature_path, args.pretty, REPO_ROOT)
except Exception as exc:
print(f"[ERROR] Harvest failed for {pdf}: {exc}", file=sys.stderr)
continue
scripts/playwright-coverage-summary.py
+1 -5
View File
@@ -157,11 +157,7 @@ def main(argv: list[str] | None = None) -> int:
stats = aggregate(args.dump_dir)
write_vitest_summary(stats, args.out)
pct = (
100.0 * stats["functions_covered"] / stats["functions_total"]
if stats["functions_total"]
else 0.0
)
pct = 100.0 * stats["functions_covered"] / stats["functions_total"] if stats["functions_total"] else 0.0
print(
f"Aggregated {stats['tests']} tests / {stats['scripts']} scripts: "
f"{stats['functions_covered']}/{stats['functions_total']} functions "
scripts/pre-commit/install_gitleaks.py
+5 -13
View File
@@ -2,7 +2,7 @@
"""Download the pinned gitleaks binary into .task/bin, verifying its checksum.
gitleaks is a Go binary with no PyPI package, so it can't be locked like the
other tools (ruff/codespell/toml-sort live in scripts/pre-commit/pyproject.toml).
other tools (ruff/codespell/toml-sort live in engine/pyproject.toml).
This script is the single source of truth for the gitleaks version and the
SHA-256 of each release asset. It is cross-platform (stdlib only) and idempotent:
if the cached binary already reports the pinned version it does nothing, so
@@ -40,9 +40,7 @@ BIN = REPO_ROOT / ".task" / "bin" / ("gitleaks.exe" if IS_WINDOWS else "gitleaks
def platform_key() -> str:
os_name = {"Linux": "linux", "Darwin": "darwin", "Windows": "windows"}.get(
platform.system()
)
os_name = {"Linux": "linux", "Darwin": "darwin", "Windows": "windows"}.get(platform.system())
arch = {
"x86_64": "x64",
"amd64": "x64",
@@ -55,9 +53,7 @@ def platform_key() -> str:
"armv6l": "armv6",
}.get(platform.machine().lower())
if not os_name or not arch:
raise SystemExit(
f"Unsupported platform for gitleaks: {platform.system()}/{platform.machine()}"
)
raise SystemExit(f"Unsupported platform for gitleaks: {platform.system()}/{platform.machine()}")
return f"{os_name}_{arch}"
@@ -65,9 +61,7 @@ def cached_version() -> str | None:
if not BIN.exists():
return None
try:
return subprocess.run(
[str(BIN), "version"], capture_output=True, text=True
).stdout.strip()
return subprocess.run([str(BIN), "version"], capture_output=True, text=True).stdout.strip()
except OSError:
return None
@@ -90,9 +84,7 @@ def main() -> int:
archive, _ = urllib.request.urlretrieve(url)
digest = hashlib.sha256(Path(archive).read_bytes()).hexdigest()
if digest != expected:
raise SystemExit(
f"gitleaks checksum mismatch: expected {expected}, got {digest}"
)
raise SystemExit(f"gitleaks checksum mismatch: expected {expected}, got {digest}")
member = "gitleaks.exe" if IS_WINDOWS else "gitleaks"
if suffix == "zip":
scripts/pre-commit/pyproject.toml
-16
View File
@@ -1,16 +0,0 @@
# Pinned Python lint/format tools for `task pre-commit`. uv.lock locks these
# plus their transitive dependencies by hash, so `uv run --project
# scripts/pre-commit --locked <tool>` is reproducible and integrity-checked.
# This is not a packaged project - it only exists to lock the tooling.
[project]
name = "stirling-precommit-tools"
version = "0"
requires-python = ">=3.11"
dependencies = [
"ruff==0.15.14",
"codespell==2.4.2",
"tomli-w==1.2.0",
]
[tool.uv]
package = false
scripts/pre-commit/sort_locale_toml.py
+2 -6
View File
@@ -54,13 +54,9 @@ def sort_file(path: str, fix: bool) -> bool:
try:
reordered = tomllib.loads(expected)
except tomllib.TOMLDecodeError as exc:
raise SortError(
f"{path}: refusing to sort, the sorted output is not valid TOML: {exc}"
) from exc
raise SortError(f"{path}: refusing to sort, the sorted output is not valid TOML: {exc}") from exc
if reordered != original:
raise SortError(
f"{path}: refusing to sort, sorting would change the file's contents"
)
raise SortError(f"{path}: refusing to sort, sorting would change the file's contents")
if fix:
Path(path).write_text(expected, encoding="utf-8")
scripts/pre-commit/uv.lock
Generated
-63
View File
@@ -1,63 +0,0 @@
version = 1
revision = 3
requires-python = ">=3.11"
[[package]]
name = "codespell"
version = "2.4.2"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/2d/9d/1d0903dff693160f893ca6abcabad545088e7a2ee0a6deae7c24e958be69/codespell-2.4.2.tar.gz", hash = "sha256:3c33be9ae34543807f088aeb4832dfad8cb2dae38da61cac0a7045dd376cfdf3", size = 352058, upload-time = "2026-03-05T18:10:42.936Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/42/a1/52fa05533e95fe45bcc09bcf8a503874b1c08f221a4e35608017e0938f55/codespell-2.4.2-py3-none-any.whl", hash = "sha256:97e0c1060cf46bd1d5db89a936c98db8c2b804e1fdd4b5c645e82a1ec6b1f886", size = 353715, upload-time = "2026-03-05T18:10:41.398Z" },
]
[[package]]
name = "ruff"
version = "0.15.14"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/dc/8a/8bce2894573e9dae6ff4d77fe34ad727d79b9e6238ad288c5638990d90f6/ruff-0.15.14.tar.gz", hash = "sha256:48e866b165be4a9bdbf310f7d3c9a07edef2fe8cd63ffeb4e00bb590506ebf9f", size = 4700910, upload-time = "2026-05-21T14:34:55.177Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b9/c8/74a92c6ff9fcfb4f1f947126d3ebee8389276e161ecc85de5bda7cda51bd/ruff-0.15.14-py3-none-linux_armv6l.whl", hash = "sha256:8dd2db9416e487c8d4b01fa7056bb02c4d05969d4f8d17a08c229c2f4ff3c108", size = 10739177, upload-time = "2026-05-21T14:34:37.332Z" },
{ url = "https://files.pythonhosted.org/packages/45/91/254a35c20acc38a7223c9d2d594af12e794432464f2cdeb52af1dc4a892d/ruff-0.15.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be4ff55af755bd71a00ab3dc6bd7ffc467bd76e0df6881e286c2e3d23e8fb43b", size = 11144969, upload-time = "2026-05-21T14:34:43.978Z" },
{ url = "https://files.pythonhosted.org/packages/56/9e/d13e40f83b8d0a94430e6778ce1d94a43b38cf2efe63278bdd2b4c65abbf/ruff-0.15.14-py3-none-macosx_11_0_arm64.whl", hash = "sha256:48d5909d7d06276ce7dde6d32bfa4b0d4cb2651145cd8ee4b440722cbc77832f", size = 10478207, upload-time = "2026-05-21T14:34:48.378Z" },
{ url = "https://files.pythonhosted.org/packages/8d/f1/b15a7839fa4f332f8acec78e20564f26bb2d866e3d21710b877fd0263000/ruff-0.15.14-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca8cbfa94c4f90984a67561978602746d4cd27103568f745fa90eee3f0d4107d", size = 10818459, upload-time = "2026-05-21T14:34:22.318Z" },
{ url = "https://files.pythonhosted.org/packages/45/33/53d651177f84f94b400a0e27f8824eeada3dddc9d5ee8aeb048f4352a520/ruff-0.15.14-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9a6bbc0333f1ab053423bcbf6226477d266ca7cec7738c4c8e3f55647803f3c4", size = 10541800, upload-time = "2026-05-21T14:34:20.209Z" },
{ url = "https://files.pythonhosted.org/packages/b8/a6/868f87e0bf9786ed24b5d0d0ad8676b8a94fd1912f42cddf9cfc7857818a/ruff-0.15.14-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a24a4f7605d7003a6674d4387651effd939dead3fddd0f36561eb77a9a2e542", size = 11342149, upload-time = "2026-05-21T14:34:46.365Z" },
{ url = "https://files.pythonhosted.org/packages/a7/8b/38cd5c19faffdcc05a408d2b78edccc69492ab9720eadb49ea15ef80d768/ruff-0.15.14-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:049b5326e53ed80978f2fc041a280603f69dd6b0c95464342a2bb4572d9d9e2f", size = 12212563, upload-time = "2026-05-21T14:34:28.579Z" },
{ url = "https://files.pythonhosted.org/packages/3e/4d/a3c5b874a556d5731e3e657aaf04311bb76f0a5c3ec220ed43051be6b64b/ruff-0.15.14-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4ed42e6696c8dfa5f06728e6441993901f548eb92d73bc472cb5a38d1395fbf", size = 11493299, upload-time = "2026-05-21T14:34:41.836Z" },
{ url = "https://files.pythonhosted.org/packages/1e/c0/56472c251d09858a53e51efbd485b09e1995d8731668b76d52e5dd6ee0f1/ruff-0.15.14-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:715c543cf450c4888251f91c52f1942a800541d9bddd7ac060aa4e6b77ae7cba", size = 11455931, upload-time = "2026-05-21T14:34:57.276Z" },
{ url = "https://files.pythonhosted.org/packages/2c/4a/e2e7b4d8dbf233d4eace59c75bc3435fa6d8bd3bae82d351d4e4300c0fd1/ruff-0.15.14-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:72ebab6013ec887d439d8b7593737a0a4ffb06d45d209d4e4bf2e92813082d3f", size = 11400794, upload-time = "2026-05-21T14:34:39.773Z" },
{ url = "https://files.pythonhosted.org/packages/97/c7/83c0539fe34c3e09136204d1e75d6052492364e0b3cb05e9465423f567d7/ruff-0.15.14-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:49072d36abdbe97a8dd7f480afe9c675699c0c495d4c84076e2c1203c4550581", size = 10804759, upload-time = "2026-05-21T14:34:31.045Z" },
{ url = "https://files.pythonhosted.org/packages/86/a6/18f2bfc095a2ab4a78745644e428205532ce6653a5d0fa8501572891534d/ruff-0.15.14-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:958522aee105068640c2c2ceae08f413ae44d922f52a1374ac13d6a96032fc93", size = 10539517, upload-time = "2026-05-21T14:34:53.064Z" },
{ url = "https://files.pythonhosted.org/packages/54/3a/5a8b3b69c654d4e4bf1d246ac5b49cbcdac6eaab6905925f8915f31e3b80/ruff-0.15.14-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f3707da619a143a2e8830e2abab8224478d69ace2d28cb6c20543ae97c36bf61", size = 11065169, upload-time = "2026-05-21T14:34:24.484Z" },
{ url = "https://files.pythonhosted.org/packages/ed/c5/8864e4e7925b836ea354b31d57641ec03830564e281a8b6f061f8c3e0ec1/ruff-0.15.14-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bb01d645694e3ec0102105d07ef2d53703970407d59c04e59d3ba0b7a1d53553", size = 11560214, upload-time = "2026-05-21T14:34:50.975Z" },
{ url = "https://files.pythonhosted.org/packages/36/38/012bf76752e1f89ed50b77b99532d90f3a3e287bc7918e1fc0948ac866ac/ruff-0.15.14-py3-none-win32.whl", hash = "sha256:6d0c1ad2a0ab718d39b6d8fd2217981ce4d625cd96a720095f798fb47d8b13e6", size = 10805548, upload-time = "2026-05-21T14:34:33.453Z" },
{ url = "https://files.pythonhosted.org/packages/d1/b7/4ea2c170f10ad760fff2a5250beb18897719dc8b52b53a24cddbb9dd3f19/ruff-0.15.14-py3-none-win_amd64.whl", hash = "sha256:802342981e056db3851a7836e5b070f8f15f67d4a685ae2a6160939d364b2902", size = 11939523, upload-time = "2026-05-21T14:34:18.077Z" },
{ url = "https://files.pythonhosted.org/packages/62/d5/bc97ff895ec35cf3925d4bd60f3b39d822f377a446906ec9bcc87405e59b/ruff-0.15.14-py3-none-win_arm64.whl", hash = "sha256:ff47b90a9ef6a40c9e2f3b479c1fb78531adf055b94c1eba0a7ba04b31951826", size = 11208607, upload-time = "2026-05-21T14:34:26.525Z" },
]
[[package]]
name = "stirling-precommit-tools"
version = "0"
source = { virtual = "." }
dependencies = [
{ name = "codespell" },
{ name = "ruff" },
{ name = "tomli-w" },
]
[package.metadata]
requires-dist = [
{ name = "codespell", specifier = "==2.4.2" },
{ name = "ruff", specifier = "==0.15.14" },
{ name = "tomli-w", specifier = "==1.2.0" },
]
[[package]]
name = "tomli-w"
version = "1.2.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" },
]
scripts/pre-commit/whitespace.py
+4 -1
View File
@@ -40,7 +40,10 @@ def normalise(data: bytes) -> bytes:
body = b"\n".join(lines)
# Ensure a non-empty file ends with exactly one newline.
stripped = body.rstrip(b"\r\n")
return stripped + b"\n" if stripped else body
if not stripped:
return body
newline = b"\r\n" if data.endswith(b"\r\n") else b"\n"
return stripped + newline
def main() -> int:
scripts/summarize_type3_signatures.py
+7 -15
View File
@@ -14,13 +14,10 @@ import argparse
import json
from collections import defaultdict
from pathlib import Path
from typing import Dict, List
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Summarize Type3 signature JSON dumps."
)
parser = argparse.ArgumentParser(description="Summarize Type3 signature JSON dumps.")
parser.add_argument(
"--input",
default="docs/type3/signatures",
@@ -34,8 +31,8 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
def load_signatures(directory: Path) -> Dict[str, List[dict]]:
inventory: Dict[str, List[dict]] = defaultdict(list)
def load_signatures(directory: Path) -> dict[str, list[dict]]:
inventory: dict[str, list[dict]] = defaultdict(list)
for path in sorted(directory.glob("*.json")):
with path.open("r", encoding="utf-8") as handle:
payload = json.load(handle)
@@ -55,15 +52,12 @@ def load_signatures(directory: Path) -> Dict[str, List[dict]]:
return inventory
def write_markdown(
inventory: Dict[str, List[dict]], output: Path, input_dir: Path
) -> None:
lines: List[str] = []
def write_markdown(inventory: dict[str, list[dict]], output: Path, input_dir: Path) -> None:
lines: list[str] = []
lines.append("# Type3 Signature Inventory")
lines.append("")
lines.append(
f"_Generated from `{input_dir}`. "
"Run `scripts/summarize_type3_signatures.py` after capturing new samples._"
f"_Generated from `{input_dir}`. Run `scripts/summarize_type3_signatures.py` after capturing new samples._"
)
lines.append("")
@@ -76,9 +70,7 @@ def write_markdown(
for entry in entries:
signature = entry.get("signature") or "—"
sample = Path(entry["source"]).name
glyph_count = (
entry.get("glyphCount") if entry.get("glyphCount") is not None else "—"
)
glyph_count = entry.get("glyphCount") if entry.get("glyphCount") is not None else "—"
coverage = entry.get("glyphCoverage") or []
preview = ", ".join(str(code) for code in coverage[:10])
lines.append(f"| `{signature}` | `{sample}` | {glyph_count} | {preview} |")
scripts/sync_en_us_spelling.py
+4 -12
View File
@@ -382,9 +382,7 @@ def make_converter(mapping: dict[str, str]):
return lambda text: (text, [])
# Longest-first so multi-word/longer forms win; \b ensures whole words.
pattern = re.compile(
r"\b("
+ "|".join(re.escape(w) for w in sorted(mapping, key=len, reverse=True))
+ r")\b",
r"\b(" + "|".join(re.escape(w) for w in sorted(mapping, key=len, reverse=True)) + r")\b",
re.IGNORECASE,
)
@@ -509,9 +507,7 @@ def parse_structured(
continue
kv = KV_RE.match(s)
if kv:
(top if section == "" else sections[section]).append(
(kv.group(1), kv.group(2))
)
(top if section == "" else sections[section]).append((kv.group(1), kv.group(2)))
return top, order, sections
@@ -563,9 +559,7 @@ def sync_en_us(dry_run: bool) -> int:
# en-US-only keys that belong to this (shared) section
for k, v in us_sections.get(name, []):
if k not in gb_section_keys:
_insert_ci(
merged, (k, uk_to_us_convert(v)[0]), lambda kv: kv[0].lower()
)
_insert_ci(merged, (k, uk_to_us_convert(v)[0]), lambda kv: kv[0].lower())
out_sections.append((name, merged))
# en-US-only sections (absent from en-GB): insert by ci header order
@@ -595,9 +589,7 @@ def sync_en_us(dry_run: bool) -> int:
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument(
"--dry-run", action="store_true", help="report changes without writing"
)
ap.add_argument("--dry-run", action="store_true", help="report changes without writing")
args = ap.parse_args()
if not EN_US.exists() or not EN_GB.exists():
scripts/translations/ai_translation_helper.py
+39 -92
View File
@@ -6,14 +6,15 @@ batch processing, quality checks, and integration helpers.
TOML format only.
"""
import json
from pathlib import Path
from typing import Dict, List, Any
import argparse
import re
from datetime import datetime
import csv
import json
import re
import tomllib
from datetime import datetime
from pathlib import Path
from typing import Any
import tomli_w
@@ -22,7 +23,7 @@ class AITranslationHelper:
self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
def _load_translation_file(self, file_path: Path) -> Dict:
def _load_translation_file(self, file_path: Path) -> dict:
"""Load TOML translation file."""
try:
with open(file_path, "rb") as f:
@@ -31,14 +32,14 @@ class AITranslationHelper:
print(f"Error loading {file_path}: {e}")
return {}
def _save_translation_file(self, data: Dict, file_path: Path) -> None:
def _save_translation_file(self, data: dict, file_path: Path) -> None:
"""Save TOML translation file."""
with open(file_path, "wb") as f:
tomli_w.dump(data, f)
def create_ai_batch_file(
self,
languages: List[str],
languages: list[str],
output_file: Path,
max_entries_per_language: int = 50,
) -> None:
@@ -74,14 +75,9 @@ class AITranslationHelper:
untranslated = self._find_untranslated_entries(golden_truth, lang_data)
# Limit entries if specified
if (
max_entries_per_language
and len(untranslated) > max_entries_per_language
):
if max_entries_per_language and len(untranslated) > max_entries_per_language:
# Prioritize by key importance
untranslated = self._prioritize_translation_keys(
untranslated, max_entries_per_language
)
untranslated = self._prioritize_translation_keys(untranslated, max_entries_per_language)
batch_data["translations"][lang] = {}
for key, value in untranslated.items():
@@ -94,15 +90,11 @@ class AITranslationHelper:
# Always save batch files as JSON for compatibility
with open(output_file, "w", encoding="utf-8") as f:
json.dump(batch_data, f, indent=2, ensure_ascii=False)
total_entries = sum(
len(lang_data) for lang_data in batch_data["translations"].values()
)
total_entries = sum(len(lang_data) for lang_data in batch_data["translations"].values())
print(f"Created AI batch file: {output_file}")
print(f"Total entries to translate: {total_entries}")
def _find_untranslated_entries(
self, golden_truth: Dict, lang_data: Dict
) -> Dict[str, str]:
def _find_untranslated_entries(self, golden_truth: dict, lang_data: dict) -> dict[str, str]:
"""Find entries that need translation."""
golden_flat = self._flatten_dict(golden_truth)
lang_flat = self._flatten_dict(lang_data)
@@ -112,19 +104,14 @@ class AITranslationHelper:
if (
key not in lang_flat
or lang_flat[key] == value
or (
isinstance(lang_flat[key], str)
and lang_flat[key].startswith("[UNTRANSLATED]")
)
or (isinstance(lang_flat[key], str) and lang_flat[key].startswith("[UNTRANSLATED]"))
):
if not self._is_expected_identical(key, value):
untranslated[key] = value
return untranslated
def _flatten_dict(
self, d: Dict, parent_key: str = "", separator: str = "."
) -> Dict[str, Any]:
def _flatten_dict(self, d: dict, parent_key: str = "", separator: str = ".") -> dict[str, Any]:
"""Flatten nested dictionary."""
items = []
for k, v in d.items():
@@ -141,9 +128,7 @@ class AITranslationHelper:
return True
return "language.direction" in key.lower()
def _prioritize_translation_keys(
self, untranslated: Dict[str, str], max_count: int
) -> Dict[str, str]:
def _prioritize_translation_keys(self, untranslated: dict[str, str], max_count: int) -> dict[str, str]:
"""Prioritize which keys to translate first based on importance."""
# Define priority order (higher score = higher priority)
priority_patterns = [
@@ -191,19 +176,17 @@ class AITranslationHelper:
if len(parts) > 0:
main_section = parts[0]
context = contexts.get(
main_section, f"Part of {main_section} functionality"
)
context = contexts.get(main_section, f"Part of {main_section} functionality")
if len(parts) > 1:
context += f", specifically for {parts[-1]}"
return context
return "General application text"
def validate_ai_translations(self, batch_file: Path) -> Dict[str, List[str]]:
def validate_ai_translations(self, batch_file: Path) -> dict[str, list[str]]:
"""Validate AI translations for common issues."""
# Batch files are always JSON
with open(batch_file, "r", encoding="utf-8") as f:
with open(batch_file, encoding="utf-8") as f:
batch_data = json.load(f)
issues = {"errors": [], "warnings": []}
@@ -227,29 +210,21 @@ class AITranslationHelper:
)
# Check if translation is identical to original (might be untranslated)
if translated == original and not self._is_expected_identical(
key, original
):
issues["warnings"].append(
f"{lang}.{key}: Translation identical to original"
)
if translated == original and not self._is_expected_identical(key, original):
issues["warnings"].append(f"{lang}.{key}: Translation identical to original")
# Check for common AI translation artifacts
artifacts = ["[TRANSLATE]", "[TODO]", "UNTRANSLATED", "{{", "}}"]
for artifact in artifacts:
if artifact in translated:
issues["errors"].append(
f"{lang}.{key}: Contains translation artifact: {artifact}"
)
issues["errors"].append(f"{lang}.{key}: Contains translation artifact: {artifact}")
return issues
def apply_ai_batch_translations(
self, batch_file: Path, validate: bool = True
) -> Dict[str, Any]:
def apply_ai_batch_translations(self, batch_file: Path, validate: bool = True) -> dict[str, Any]:
"""Apply translations from AI batch file to individual language files."""
# Batch files are always JSON
with open(batch_file, "r", encoding="utf-8") as f:
with open(batch_file, encoding="utf-8") as f:
batch_data = json.load(f)
results = {"applied": {}, "errors": [], "warnings": []}
@@ -291,7 +266,7 @@ class AITranslationHelper:
return results
def _set_nested_value(self, data: Dict, key_path: str, value: Any) -> None:
def _set_nested_value(self, data: dict, key_path: str, value: Any) -> None:
"""Set value in nested dict using dot notation."""
keys = key_path.split(".")
current = data
@@ -300,24 +275,18 @@ class AITranslationHelper:
current[key] = {}
elif not isinstance(current[key], dict):
# If the current value is not a dict, we can't nest into it
print(
f"Warning: Converting non-dict value at '{key}' to dict to allow nesting"
)
print(f"Warning: Converting non-dict value at '{key}' to dict to allow nesting")
current[key] = {}
current = current[key]
current[keys[-1]] = value
def export_for_external_translation(
self, languages: List[str], output_format: str = "csv"
) -> None:
def export_for_external_translation(self, languages: list[str], output_format: str = "csv") -> None:
"""Export translations for external translation services."""
golden_truth = self._load_translation_file(self.golden_truth_file)
golden_flat = self._flatten_dict(golden_truth)
if output_format == "csv":
output_file = Path(
f"translations_export_{datetime.now().strftime('%Y%m%d')}.csv"
)
output_file = Path(f"translations_export_{datetime.now().strftime('%Y%m%d')}.csv")
with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
fieldnames = ["key", "context", "en_US"] + languages
@@ -353,9 +322,7 @@ class AITranslationHelper:
print(f"Exported to {output_file}")
elif output_format == "json":
output_file = Path(
f"translations_export_{datetime.now().strftime('%Y%m%d')}.json"
)
output_file = Path(f"translations_export_{datetime.now().strftime('%Y%m%d')}.json")
export_data = {"languages": languages, "translations": {}}
for key, en_value in golden_flat.items():
@@ -386,9 +353,7 @@ class AITranslationHelper:
def main():
parser = argparse.ArgumentParser(
description="AI Translation Helper", epilog="Works with TOML translation files."
)
parser = argparse.ArgumentParser(description="AI Translation Helper", epilog="Works with TOML translation files.")
parser.add_argument(
"--locales-dir",
default="frontend/editor/public/locales",
@@ -398,40 +363,24 @@ def main():
subparsers = parser.add_subparsers(dest="command", help="Available commands")
# Create batch command
batch_parser = subparsers.add_parser(
"create-batch", help="Create AI translation batch file"
)
batch_parser.add_argument(
"--languages", nargs="+", required=True, help="Language codes to include"
)
batch_parser = subparsers.add_parser("create-batch", help="Create AI translation batch file")
batch_parser.add_argument("--languages", nargs="+", required=True, help="Language codes to include")
batch_parser.add_argument("--output", required=True, help="Output batch file")
batch_parser.add_argument(
"--max-entries", type=int, default=100, help="Max entries per language"
)
batch_parser.add_argument("--max-entries", type=int, default=100, help="Max entries per language")
# Validate command
validate_parser = subparsers.add_parser("validate", help="Validate AI translations")
validate_parser.add_argument("batch_file", help="Batch file to validate")
# Apply command
apply_parser = subparsers.add_parser(
"apply-batch", help="Apply AI batch translations"
)
apply_parser = subparsers.add_parser("apply-batch", help="Apply AI batch translations")
apply_parser.add_argument("batch_file", help="Batch file with translations")
apply_parser.add_argument(
"--skip-validation", action="store_true", help="Skip validation before applying"
)
apply_parser.add_argument("--skip-validation", action="store_true", help="Skip validation before applying")
# Export command
export_parser = subparsers.add_parser(
"export", help="Export for external translation"
)
export_parser.add_argument(
"--languages", nargs="+", required=True, help="Language codes to export"
)
export_parser.add_argument(
"--format", choices=["csv", "json"], default="csv", help="Export format"
)
export_parser = subparsers.add_parser("export", help="Export for external translation")
export_parser.add_argument("--languages", nargs="+", required=True, help="Language codes to export")
export_parser.add_argument("--format", choices=["csv", "json"], default="csv", help="Export format")
args = parser.parse_args()
@@ -464,9 +413,7 @@ def main():
elif args.command == "apply-batch":
batch_file = Path(args.batch_file)
results = helper.apply_ai_batch_translations(
batch_file, validate=not args.skip_validation
)
results = helper.apply_ai_batch_translations(batch_file, validate=not args.skip_validation)
total_applied = sum(results["applied"].values())
print(f"Total translations applied: {total_applied}")
scripts/translations/auto_translate.py
+17 -45
View File
@@ -5,16 +5,15 @@ Extracts, translates, merges, and beautifies translations for a language.
TOML format only.
"""
import json
import sys
import argparse
import json
import os
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import time
import tomllib
from pathlib import Path
def run_command(cmd, description=""):
@@ -50,27 +49,19 @@ def load_translation_file(file_path):
def extract_untranslated(language_code, batch_size=500, include_existing=False):
"""Extract untranslated entries and split into batches."""
mode = (
"all untranslated (including existing)" if include_existing else "new (missing)"
)
mode = "all untranslated (including existing)" if include_existing else "new (missing)"
print(f"\n🔍 Extracting {mode} entries for {language_code}...")
# Load files
golden_path = find_translation_file(Path("frontend/editor/public/locales/en-US"))
lang_path = find_translation_file(
Path(f"frontend/editor/public/locales/{language_code}")
)
lang_path = find_translation_file(Path(f"frontend/editor/public/locales/{language_code}"))
if not golden_path:
print(
"Error: Golden truth file not found in frontend/editor/public/locales/en-US"
)
print("Error: Golden truth file not found in frontend/editor/public/locales/en-US")
return None
if not lang_path:
print(
f"Error: Language file not found in frontend/editor/public/locales/{language_code}"
)
print(f"Error: Language file not found in frontend/editor/public/locales/{language_code}")
return None
def flatten_dict(d, parent_key="", separator="."):
@@ -101,10 +92,7 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
if (
key not in lang_flat
or lang_flat.get(key) == value
or (
isinstance(lang_flat.get(key), str)
and lang_flat.get(key).startswith("[UNTRANSLATED]")
)
or (isinstance(lang_flat.get(key), str) and lang_flat.get(key).startswith("[UNTRANSLATED]"))
):
untranslated[key] = value
else:
@@ -141,9 +129,7 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
return batch_files
def translate_batches(
batch_files, language_code, api_key, timeout=600, model="gpt-5.5", parallel=1
):
def translate_batches(batch_files, language_code, api_key, timeout=600, model="gpt-5.5", parallel=1):
"""Translate all batch files using the given OpenAI model."""
if not batch_files:
return []
@@ -169,9 +155,7 @@ def translate_batches(
cmd = f'python3 scripts/translations/batch_translator.py "{batch_file}" --language {language_code} --api-key "{api_key}" --model {model}'
try:
result = subprocess.run(
cmd, shell=True, capture_output=True, text=True, timeout=timeout
)
result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
except subprocess.TimeoutExpired:
print(f"✗ Timed out after {timeout}s: {batch_file}", file=sys.stderr)
return None
@@ -222,7 +206,7 @@ def merge_translations(translated_files, language_code):
print(f"Error: Translated file not found: {filename}")
return None
with open(filename, "r", encoding="utf-8") as f:
with open(filename, encoding="utf-8") as f:
merged.update(json.load(f))
lang_code_safe = language_code.replace("-", "_")
@@ -313,18 +297,10 @@ Examples:
)
parser.add_argument("language", help="Language code (e.g., es-ES, de-DE, zh-CN)")
parser.add_argument(
"--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)"
)
parser.add_argument(
"--batch-size", type=int, default=500, help="Entries per batch (default: 500)"
)
parser.add_argument(
"--no-cleanup", action="store_true", help="Keep temporary batch files"
)
parser.add_argument(
"--skip-verification", action="store_true", help="Skip final completion check"
)
parser.add_argument("--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)")
parser.add_argument("--batch-size", type=int, default=500, help="Entries per batch (default: 500)")
parser.add_argument("--no-cleanup", action="store_true", help="Keep temporary batch files")
parser.add_argument("--skip-verification", action="store_true", help="Skip final completion check")
parser.add_argument(
"--timeout",
type=int,
@@ -353,9 +329,7 @@ Examples:
# Verify API key
api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
if not api_key:
print(
"Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable"
)
print("Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable")
sys.exit(1)
print("=" * 60)
@@ -369,9 +343,7 @@ Examples:
try:
# Step 1: Extract and split
batch_files = extract_untranslated(
args.language, args.batch_size, args.include_existing
)
batch_files = extract_untranslated(args.language, args.batch_size, args.include_existing)
if batch_files is None:
sys.exit(1)
scripts/translations/batch_translator.py
+12 -31
View File
@@ -9,11 +9,11 @@ Automatically translates JSON batch files to target language while preserving:
Note: Works with JSON batch files. Translation files can be TOML or JSON format.
"""
import argparse
import json
import sys
import argparse
from pathlib import Path
import time
from pathlib import Path
try:
from openai import OpenAI
@@ -117,9 +117,7 @@ Return ONLY the translated JSON. No markdown, no explanations, just the JSON obj
cost_note = f", ~${cost:.4f}" if cost else ""
print(f" Tokens: {prompt_tokens:,} in / {completion_tokens:,} out{cost_note}")
def translate_batch(
self, batch_data: dict, target_language: str, language_code: str
) -> dict:
def translate_batch(self, batch_data: dict, target_language: str, language_code: str) -> dict:
"""Translate a batch file using OpenAI API."""
# Convert batch to compact JSON for API
input_json = json.dumps(batch_data, ensure_ascii=False, separators=(",", ":"))
@@ -134,9 +132,7 @@ Return ONLY the translated JSON. No markdown, no explanations, just the JSON obj
messages=[
{
"role": "system",
"content": self.get_translation_prompt(
target_language, language_code
),
"content": self.get_translation_prompt(target_language, language_code),
},
{
"role": "user",
@@ -198,9 +194,7 @@ Return ONLY the translated JSON. No markdown, no explanations, just the JSON obj
trans_placeholders = set(re.findall(placeholder_pattern, trans_value))
if orig_placeholders != trans_placeholders:
issues.append(
f"Placeholder mismatch in '{key}': {orig_placeholders} vs {trans_placeholders}"
)
issues.append(f"Placeholder mismatch in '{key}': {orig_placeholders} vs {trans_placeholders}")
if issues:
print("\n⚠ Validation warnings:")
@@ -276,12 +270,8 @@ Examples:
""",
)
parser.add_argument(
"input_files", nargs="+", help="Input batch JSON file(s) or pattern"
)
parser.add_argument(
"--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)"
)
parser.add_argument("input_files", nargs="+", help="Input batch JSON file(s) or pattern")
parser.add_argument("--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)")
parser.add_argument(
"--language",
"-l",
@@ -298,9 +288,7 @@ Examples:
default="_translated",
help="Suffix for output files (default: _translated)",
)
parser.add_argument(
"--skip-validation", action="store_true", help="Skip validation checks"
)
parser.add_argument("--skip-validation", action="store_true", help="Skip validation checks")
parser.add_argument(
"--delay",
type=float,
@@ -315,9 +303,7 @@ Examples:
api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
if not api_key:
print(
"Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable"
)
print("Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable")
sys.exit(1)
# Get language info
@@ -356,13 +342,11 @@ Examples:
try:
# Load input file
with open(input_file, "r", encoding="utf-8") as f:
with open(input_file, encoding="utf-8") as f:
batch_data = json.load(f)
# Translate
translated_data = translator.translate_batch(
batch_data, language_name, language_code
)
translated_data = translator.translate_batch(batch_data, language_name, language_code)
# Validate
if not args.skip_validation:
@@ -396,10 +380,7 @@ Examples:
# Cost summary
print("-" * 60)
print(
f"Total tokens: {translator.total_prompt_tokens:,} in / "
f"{translator.total_completion_tokens:,} out"
)
print(f"Total tokens: {translator.total_prompt_tokens:,} in / {translator.total_completion_tokens:,} out")
if translator.total_cost:
print(f"Estimated cost ({args.model}): ${translator.total_cost:.4f}")
scripts/translations/bulk_auto_translate.py
+10 -21
View File
@@ -7,16 +7,13 @@ Supports concurrent translation with configurable thread pool.
import argparse
import os
import sys
import time
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor, as_completed
import subprocess
from typing import List, Tuple, Optional
import sys
import threading
import time
import tomllib
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
# Thread-safe print lock
print_lock = threading.Lock()
@@ -28,7 +25,7 @@ def safe_print(*args, **kwargs):
print(*args, **kwargs)
def get_all_languages(locales_dir: Path) -> List[str]:
def get_all_languages(locales_dir: Path) -> list[str]:
"""Get all language codes from locales directory."""
languages = []
@@ -45,7 +42,7 @@ def get_all_languages(locales_dir: Path) -> List[str]:
return languages
def get_language_completion(locales_dir: Path, language: str) -> Optional[float]:
def get_language_completion(locales_dir: Path, language: str) -> float | None:
"""Get completion percentage for a language."""
lang_dir = locales_dir / language
toml_file = lang_dir / "translation.toml"
@@ -77,11 +74,7 @@ def get_language_completion(locales_dir: Path, language: str) -> Optional[float]
target_flat = flatten(target_data)
# Count translated (not equal to en-US)
translated = sum(
1
for k in en_us_flat
if k in target_flat and target_flat[k] != en_us_flat[k]
)
translated = sum(1 for k in en_us_flat if k in target_flat and target_flat[k] != en_us_flat[k])
total = len(en_us_flat)
return (translated / total * 100) if total > 0 else 0.0
@@ -99,7 +92,7 @@ def translate_language(
skip_verification: bool,
include_existing: bool,
model: str,
) -> Tuple[str, bool, str]:
) -> tuple[str, bool, str]:
"""
Translate a single language.
Returns: (language_code, success, message)
@@ -178,9 +171,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
""",
)
parser.add_argument(
"--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)"
)
parser.add_argument("--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)")
parser.add_argument(
"--model",
default="gpt-5.5",
@@ -241,9 +232,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
# Verify API key (unless dry run)
api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
if not args.dry_run and not api_key:
print(
"Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable"
)
print("Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable")
sys.exit(1)
locales_dir = Path(args.locales_dir)
scripts/translations/compact_translator.py
+7 -17
View File
@@ -5,11 +5,11 @@ Outputs untranslated entries in minimal JSON format with whitespace stripped.
TOML format only.
"""
import argparse
import json
import sys
from pathlib import Path
import argparse
import tomllib # Python 3.11+ (stdlib)
from pathlib import Path
class CompactTranslationExtractor:
@@ -50,9 +50,7 @@ class CompactTranslationExtractor:
try:
with open(self.ignore_file, "rb") as f:
ignore_data = tomllib.load(f)
return {
lang: set(data.get("ignore", [])) for lang, data in ignore_data.items()
}
return {lang: set(data.get("ignore", [])) for lang, data in ignore_data.items()}
except Exception as e:
print(
f"Warning: Could not load ignore file {self.ignore_file}: {e}",
@@ -60,9 +58,7 @@ class CompactTranslationExtractor:
)
return {}
def _flatten_dict(
self, d: dict, parent_key: str = "", separator: str = "."
) -> dict:
def _flatten_dict(self, d: dict, parent_key: str = "", separator: str = ".") -> dict:
"""Flatten nested dictionary into dot-notation keys."""
items = []
for k, v in d.items():
@@ -102,12 +98,8 @@ class CompactTranslationExtractor:
target_value = target_flat[key]
golden_value = golden_flat[key]
if (
isinstance(target_value, str)
and target_value.startswith("[UNTRANSLATED]")
) or (
golden_value == target_value
and not self._is_expected_identical(key, golden_value)
if (isinstance(target_value, str) and target_value.startswith("[UNTRANSLATED]")) or (
golden_value == target_value and not self._is_expected_identical(key, golden_value)
):
untranslated_keys.add(key)
@@ -151,9 +143,7 @@ def main():
default="scripts/ignore_translation.toml",
help="Path to ignore patterns file",
)
parser.add_argument(
"--max-entries", type=int, help="Maximum number of entries to output"
)
parser.add_argument("--max-entries", type=int, help="Maximum number of entries to output")
parser.add_argument("--output", help="Output file (default: stdout)")
args = parser.parse_args()
scripts/translations/toml_beautifier.py
+28 -59
View File
@@ -4,13 +4,13 @@ TOML Beautifier and Structure Fixer for Stirling PDF Frontend
Restructures translation TOML files to match en-US structure and key order exactly.
"""
import sys
from pathlib import Path
from typing import Dict, Any, List
import argparse
from collections import OrderedDict
import sys
import tomllib
from collections import OrderedDict
from pathlib import Path
from typing import Any
import tomli_w
@@ -20,7 +20,7 @@ class TOMLBeautifier:
self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
self.golden_structure = self._load_toml(self.golden_truth_file)
def _load_toml(self, file_path: Path) -> Dict:
def _load_toml(self, file_path: Path) -> dict:
"""Load TOML file with error handling."""
try:
with open(file_path, "rb") as f:
@@ -32,7 +32,7 @@ class TOMLBeautifier:
print(f"Error: Invalid TOML in {file_path}: {e}")
sys.exit(1)
def _save_toml(self, data: Dict, file_path: Path, backup: bool = False) -> None:
def _save_toml(self, data: dict, file_path: Path, backup: bool = False) -> None:
"""Save TOML file with proper formatting."""
if backup and file_path.exists():
backup_path = file_path.with_suffix(".backup.restructured.toml")
@@ -44,9 +44,7 @@ class TOMLBeautifier:
with open(file_path, "wb") as f:
tomli_w.dump(data, f)
def _flatten_dict(
self, d: Dict, parent_key: str = "", separator: str = "."
) -> Dict[str, Any]:
def _flatten_dict(self, d: dict, parent_key: str = "", separator: str = ".") -> dict[str, Any]:
"""Flatten nested dictionary into dot-notation keys."""
items = []
for k, v in d.items():
@@ -57,9 +55,7 @@ class TOMLBeautifier:
items.append((new_key, v))
return dict(items)
def _rebuild_structure(
self, flat_dict: Dict[str, Any], reference_structure: Dict
) -> Dict:
def _rebuild_structure(self, flat_dict: dict[str, Any], reference_structure: dict) -> dict:
"""Rebuild nested structure based on reference structure and available translations."""
def build_recursive(ref_obj: Any, current_path: str = "") -> Any:
@@ -94,7 +90,7 @@ class TOMLBeautifier:
return build_recursive(reference_structure) or OrderedDict()
def restructure_translation_file(self, target_file: Path) -> Dict[str, Any]:
def restructure_translation_file(self, target_file: Path) -> dict[str, Any]:
"""Restructure a translation file to match en-US structure exactly."""
if not target_file.exists():
print(f"Error: Target file does not exist: {target_file}")
@@ -111,9 +107,7 @@ class TOMLBeautifier:
return restructured
def beautify_and_restructure(
self, target_file: Path, backup: bool = False
) -> Dict[str, Any]:
def beautify_and_restructure(self, target_file: Path, backup: bool = False) -> dict[str, Any]:
"""Main function to beautify and restructure a translation file."""
lang_code = target_file.parent.name
print(f"Restructuring {lang_code} translation file...")
@@ -135,18 +129,16 @@ class TOMLBeautifier:
"language": lang_code,
"total_reference_keys": total_keys,
"preserved_keys": preserved_keys,
"structure_match": self._compare_structures(
self.golden_structure, restructured_data
),
"structure_match": self._compare_structures(self.golden_structure, restructured_data),
}
print(f"Restructured {lang_code}: {preserved_keys}/{total_keys} keys preserved")
return result
def _compare_structures(self, ref: Dict, target: Dict) -> Dict[str, bool]:
def _compare_structures(self, ref: dict, target: dict) -> dict[str, bool]:
"""Compare structures between reference and target."""
def compare_recursive(r: Any, t: Any, path: str = "") -> List[str]:
def compare_recursive(r: Any, t: Any, path: str = "") -> list[str]:
issues = []
if isinstance(r, dict) and isinstance(t, dict):
@@ -157,9 +149,7 @@ class TOMLBeautifier:
missing_sections = ref_keys - target_keys
if missing_sections:
for section in missing_sections:
issues.append(
f"Missing section: {path}.{section}" if path else section
)
issues.append(f"Missing section: {path}.{section}" if path else section)
# Recurse into common sections
for key in ref_keys & target_keys:
@@ -176,11 +166,11 @@ class TOMLBeautifier:
"total_issues": len(issues),
}
def validate_key_order(self, target_file: Path) -> Dict[str, Any]:
def validate_key_order(self, target_file: Path) -> dict[str, Any]:
"""Validate that keys appear in the same order as en-US."""
target_data = self._load_toml(target_file)
def get_key_order(obj: Dict, path: str = "") -> List[str]:
def get_key_order(obj: dict, path: str = "") -> list[str]:
keys = []
for key in obj.keys():
new_path = f"{path}.{key}" if path else key
@@ -195,19 +185,14 @@ class TOMLBeautifier:
# Find common keys and check their relative order
common_keys = set(golden_order) & set(target_order)
golden_indices = {
key: idx for idx, key in enumerate(golden_order) if key in common_keys
}
target_indices = {
key: idx for idx, key in enumerate(target_order) if key in common_keys
}
golden_indices = {key: idx for idx, key in enumerate(golden_order) if key in common_keys}
target_indices = {key: idx for idx, key in enumerate(target_order) if key in common_keys}
order_preserved = all(
golden_indices[key1] < golden_indices[key2]
for key1 in common_keys
for key2 in common_keys
if golden_indices[key1] < golden_indices[key2]
and target_indices[key1] < target_indices[key2]
if golden_indices[key1] < golden_indices[key2] and target_indices[key1] < target_indices[key2]
)
return {
@@ -229,12 +214,8 @@ def main():
help="Path to locales directory",
)
parser.add_argument("--language", help="Restructure specific language only")
parser.add_argument(
"--all-languages", action="store_true", help="Restructure all language files"
)
parser.add_argument(
"--backup", action="store_true", help="Create backup files before modifying"
)
parser.add_argument("--all-languages", action="store_true", help="Restructure all language files")
parser.add_argument("--backup", action="store_true", help="Create backup files before modifying")
parser.add_argument(
"--validate-only",
action="store_true",
@@ -255,21 +236,13 @@ def main():
order_result = beautifier.validate_key_order(target_file)
print(f"Key order validation for {args.language}:")
print(f" Order preserved: {order_result['order_preserved']}")
print(
f" Common keys: {order_result['common_keys_count']}/{order_result['golden_keys_count']}"
)
print(f" Common keys: {order_result['common_keys_count']}/{order_result['golden_keys_count']}")
else:
result = beautifier.beautify_and_restructure(
target_file, backup=args.backup
)
result = beautifier.beautify_and_restructure(target_file, backup=args.backup)
print(f"\nResults for {result['language']}:")
print(
f" Keys preserved: {result['preserved_keys']}/{result['total_reference_keys']}"
)
print(f" Keys preserved: {result['preserved_keys']}/{result['total_reference_keys']}")
if result["structure_match"]["total_issues"] > 0:
print(
f" Structure issues: {result['structure_match']['total_issues']}"
)
print(f" Structure issues: {result['structure_match']['total_issues']}")
for issue in result["structure_match"]["issues"]:
print(f" - {issue}")
@@ -281,13 +254,9 @@ def main():
if translation_file.exists():
if args.validate_only:
order_result = beautifier.validate_key_order(translation_file)
print(
f"{lang_dir.name}: Order preserved = {order_result['order_preserved']}"
)
print(f"{lang_dir.name}: Order preserved = {order_result['order_preserved']}")
else:
result = beautifier.beautify_and_restructure(
translation_file, backup=args.backup
)
result = beautifier.beautify_and_restructure(translation_file, backup=args.backup)
results.append(result)
if not args.validate_only and results:
scripts/translations/toml_validator.py
+5 -10
View File
@@ -12,17 +12,16 @@ Usage:
python3 toml_validator.py --all-batches ar_AR
"""
import sys
import argparse
import glob
import sys
import tomllib
def get_line_context(file_path, line_num, context_lines=3):
"""Get lines around the error for context"""
try:
with open(file_path, "r", encoding="utf-8") as f:
with open(file_path, encoding="utf-8") as f:
lines = f.readlines()
start = max(0, line_num - context_lines - 1)
@@ -41,7 +40,7 @@ def get_line_context(file_path, line_num, context_lines=3):
def get_character_context(file_path, char_pos, context_chars=100):
"""Get characters around the error position"""
try:
with open(file_path, "r", encoding="utf-8") as f:
with open(file_path, encoding="utf-8") as f:
content = f.read()
start = max(0, char_pos - context_chars)
@@ -144,12 +143,8 @@ def main():
metavar="LANG",
help="Validate all batch files for a language (e.g., ar_AR)",
)
parser.add_argument(
"--brief", action="store_true", help="Show brief output without context"
)
parser.add_argument(
"--quiet", action="store_true", help="Only show files with errors"
)
parser.add_argument("--brief", action="store_true", help="Show brief output without context")
parser.add_argument("--quiet", action="store_true", help="Only show files with errors")
args = parser.parse_args()
scripts/translations/translation_analyzer.py
+20 -42
View File
@@ -4,12 +4,11 @@ Translation Analyzer for Stirling PDF Frontend
Compares language files against en-US golden truth file.
"""
import argparse
import json
import sys
from pathlib import Path
from typing import Dict, List, Set
import argparse
import tomllib
from pathlib import Path
class TranslationAnalyzer:
@@ -24,7 +23,7 @@ class TranslationAnalyzer:
self.ignore_file = Path(ignore_file)
self.ignore_patterns = self._load_ignore_patterns()
def _load_translation_file(self, file_path: Path) -> Dict:
def _load_translation_file(self, file_path: Path) -> dict:
"""Load TOML translation file with error handling."""
try:
with open(file_path, "rb") as f:
@@ -36,7 +35,7 @@ class TranslationAnalyzer:
print(f"Error: Invalid file {file_path}: {e}")
sys.exit(1)
def _load_ignore_patterns(self) -> Dict[str, Set[str]]:
def _load_ignore_patterns(self) -> dict[str, set[str]]:
"""Load ignore patterns from TOML file."""
if not self.ignore_file.exists():
return {}
@@ -56,9 +55,7 @@ class TranslationAnalyzer:
print(f"Warning: Could not load ignore file {self.ignore_file}: {e}")
return {}
def _flatten_dict(
self, d: Dict, parent_key: str = "", separator: str = "."
) -> Dict[str, str]:
def _flatten_dict(self, d: dict, parent_key: str = "", separator: str = ".") -> dict[str, str]:
"""Flatten nested dictionary into dot-notation keys."""
items = []
for k, v in d.items():
@@ -69,7 +66,7 @@ class TranslationAnalyzer:
items.append((new_key, str(v)))
return dict(items)
def get_all_language_files(self) -> List[Path]:
def get_all_language_files(self) -> list[Path]:
"""Get all translation files except en-US."""
files = []
for lang_dir in self.locales_dir.iterdir():
@@ -79,7 +76,7 @@ class TranslationAnalyzer:
files.append(toml_file)
return sorted(files)
def find_missing_translations(self, target_file: Path) -> Set[str]:
def find_missing_translations(self, target_file: Path) -> set[str]:
"""Find keys that exist in en-US but missing in target file."""
target_data = self._load_translation_file(target_file)
@@ -93,7 +90,7 @@ class TranslationAnalyzer:
ignore_set = self.ignore_patterns.get(lang_code, set())
return missing - ignore_set
def find_untranslated_entries(self, target_file: Path) -> Set[str]:
def find_untranslated_entries(self, target_file: Path) -> set[str]:
"""Find entries that appear to be untranslated (identical to en-US)."""
target_data = self._load_translation_file(target_file)
@@ -110,10 +107,7 @@ class TranslationAnalyzer:
golden_value = golden_flat[key]
# Check if marked as [UNTRANSLATED] or identical to en-US
if (
isinstance(target_value, str)
and target_value.startswith("[UNTRANSLATED]")
) or (
if (isinstance(target_value, str) and target_value.startswith("[UNTRANSLATED]")) or (
golden_value == target_value
and key not in ignore_set
and not self._is_expected_identical(key, golden_value)
@@ -138,7 +132,7 @@ class TranslationAnalyzer:
return False
def find_extra_translations(self, target_file: Path) -> Set[str]:
def find_extra_translations(self, target_file: Path) -> set[str]:
"""Find keys that exist in target file but not in en-US."""
target_data = self._load_translation_file(target_file)
@@ -147,7 +141,7 @@ class TranslationAnalyzer:
return set(target_flat.keys()) - set(golden_flat.keys())
def analyze_file(self, target_file: Path) -> Dict:
def analyze_file(self, target_file: Path) -> dict:
"""Complete analysis of a single translation file."""
lang_code = target_file.parent.name
@@ -172,14 +166,10 @@ class TranslationAnalyzer:
if key in target_flat:
value = target_flat[key]
if not (isinstance(value, str) and value.startswith("[UNTRANSLATED]")):
if (
key not in untranslated
): # Not identical to en-US (unless expected)
if key not in untranslated: # Not identical to en-US (unless expected)
properly_translated += 1
completion_rate = (
(properly_translated / total_keys) * 100 if total_keys > 0 else 0
)
completion_rate = (properly_translated / total_keys) * 100 if total_keys > 0 else 0
return {
"language": lang_code,
@@ -194,7 +184,7 @@ class TranslationAnalyzer:
"completion_rate": completion_rate,
}
def analyze_all_files(self) -> List[Dict]:
def analyze_all_files(self) -> list[dict]:
"""Analyze all translation files."""
results = []
for file_path in self.get_all_language_files():
@@ -203,9 +193,7 @@ class TranslationAnalyzer:
def main():
parser = argparse.ArgumentParser(
description="Analyze translation files against en-US golden truth"
)
parser = argparse.ArgumentParser(description="Analyze translation files against en-US golden truth")
parser.add_argument(
"--locales-dir",
default="frontend/editor/public/locales",
@@ -217,20 +205,14 @@ def main():
help="Path to ignore patterns TOML file",
)
parser.add_argument("--language", help="Analyze specific language only")
parser.add_argument(
"--missing-only", action="store_true", help="Show only missing translations"
)
parser.add_argument("--missing-only", action="store_true", help="Show only missing translations")
parser.add_argument(
"--untranslated-only",
action="store_true",
help="Show only untranslated entries",
)
parser.add_argument(
"--summary", action="store_true", help="Show summary statistics only"
)
parser.add_argument(
"--format", choices=["text", "json"], default="text", help="Output format"
)
parser.add_argument("--summary", action="store_true", help="Show summary statistics only")
parser.add_argument("--format", choices=["text", "json"], default="text", help="Output format")
args = parser.parse_args()
@@ -287,16 +269,12 @@ def main():
print(f"\n{'=' * 60}")
print("SUMMARY")
print(f"{'=' * 60}")
avg_completion = (
sum(r["completion_rate"] for r in results) / len(results) if results else 0
)
avg_completion = sum(r["completion_rate"] for r in results) / len(results) if results else 0
print(f"Average Completion Rate: {avg_completion:.1f}%")
print(f"Languages Analyzed: {len(results)}")
# Top languages by completion
sorted_by_completion = sorted(
results, key=lambda x: x["completion_rate"], reverse=True
)
sorted_by_completion = sorted(results, key=lambda x: x["completion_rate"], reverse=True)
print("\nTop 5 Most Complete Languages:")
for result in sorted_by_completion[:5]:
print(f" {result['language']}: {result['completion_rate']:.1f}%")
scripts/translations/translation_merger.py
+29 -83
View File
@@ -6,28 +6,24 @@ Useful for AI-assisted translation workflows.
TOML format only.
"""
import os
import argparse
import json
import os
import shutil
import sys
import tomllib
from datetime import datetime
from pathlib import Path
from typing import Any
import tomllib
import tomli_w
class TranslationMerger:
def __init__(
self,
locales_dir: str = os.path.join(
os.getcwd(), "frontend", "editor", "public", "locales"
),
ignore_file: str = os.path.join(
os.getcwd(), "scripts", "ignore_translation.toml"
),
locales_dir: str = os.path.join(os.getcwd(), "frontend", "editor", "public", "locales"),
ignore_file: str = os.path.join(os.getcwd(), "scripts", "ignore_translation.toml"),
):
self.locales_dir = Path(locales_dir)
self.golden_truth_file = self.locales_dir / "en-US" / "translation.toml"
@@ -47,14 +43,10 @@ class TranslationMerger:
print(f"Error: Invalid file {file_path}: {e}")
sys.exit(1)
def _save_translation_file(
self, data: dict[str, Any], file_path: Path, backup: bool = False
) -> None:
def _save_translation_file(self, data: dict[str, Any], file_path: Path, backup: bool = False) -> None:
"""Save TOML translation file with backup option."""
if backup and file_path.exists():
backup_path = file_path.with_suffix(
f".backup.{datetime.now().strftime('%Y%m%d_%H%M%S')}.toml"
)
backup_path = file_path.with_suffix(f".backup.{datetime.now().strftime('%Y%m%d_%H%M%S')}.toml")
shutil.copy2(file_path, backup_path)
print(f"Backup created: {backup_path}")
@@ -71,9 +63,7 @@ class TranslationMerger:
ignore_data = tomllib.load(f)
# Convert to sets for faster lookup
return {
lang: set(data.get("ignore", [])) for lang, data in ignore_data.items()
}
return {lang: set(data.get("ignore", [])) for lang, data in ignore_data.items()}
except Exception as e:
print(f"Warning: Could not load ignore file {self.ignore_file}: {e}")
return {}
@@ -89,9 +79,7 @@ class TranslationMerger:
return None
return current
def _set_nested_value(
self, data: dict[str, Any], key_path: str, value: Any
) -> None:
def _set_nested_value(self, data: dict[str, Any], key_path: str, value: Any) -> None:
"""Set value in nested dict using dot notation."""
keys = key_path.split(".")
current = data
@@ -101,16 +89,12 @@ class TranslationMerger:
elif not isinstance(current[key], dict):
# If the current value is not a dict, we can't nest into it
# This handles cases where a key exists as a string but we need to make it a dict
print(
f"Warning: Converting non-dict value at '{key}' to dict to allow nesting"
)
print(f"Warning: Converting non-dict value at '{key}' to dict to allow nesting")
current[key] = {}
current = current[key]
current[keys[-1]] = value
def _flatten_dict(
self, d: dict[str, Any], parent_key: str = "", separator: str = "."
) -> dict[str, Any]:
def _flatten_dict(self, d: dict[str, Any], parent_key: str = "", separator: str = ".") -> dict[str, Any]:
"""Flatten nested dictionary into dot-notation keys."""
items = []
for k, v in d.items():
@@ -207,9 +191,7 @@ class TranslationMerger:
"data": target_data,
}
def extract_untranslated_entries(
self, target_file: Path, output_file: Path | None = None
) -> dict[str, Any]:
def extract_untranslated_entries(self, target_file: Path, output_file: Path | None = None) -> dict[str, Any]:
"""Extract entries marked as untranslated or identical to en-US for AI translation."""
if not target_file.exists():
print(f"Error: Target file does not exist: {target_file}")
@@ -233,9 +215,7 @@ class TranslationMerger:
"reason": "marked_untranslated",
}
# Check if identical to golden (and should be translated)
elif value == golden_value and not self._is_expected_identical(
key, value
):
elif value == golden_value and not self._is_expected_identical(key, value):
untranslated_entries[key] = {
"original": golden_value,
"current": value,
@@ -279,9 +259,7 @@ class TranslationMerger:
for key, translation in translations.items():
try:
# Remove [UNTRANSLATED] marker if present
if isinstance(translation, str) and translation.startswith(
"[UNTRANSLATED]"
):
if isinstance(translation, str) and translation.startswith("[UNTRANSLATED]"):
translation = translation.replace("[UNTRANSLATED]", "").strip()
self._set_nested_value(target_data, key, translation)
@@ -390,45 +368,25 @@ def main():
subparsers = parser.add_subparsers(dest="command", help="Available commands")
# Add missing command
add_parser = subparsers.add_parser(
"add-missing", help="Add missing translations from en-US"
)
add_parser.add_argument(
"--backup", action="store_true", help="Create backup before modifying files"
)
add_parser = subparsers.add_parser("add-missing", help="Add missing translations from en-US")
add_parser.add_argument("--backup", action="store_true", help="Create backup before modifying files")
# Extract untranslated command
extract_parser = subparsers.add_parser(
"extract-untranslated", help="Extract untranslated entries"
)
extract_parser = subparsers.add_parser("extract-untranslated", help="Extract untranslated entries")
extract_parser.add_argument("--output", help="Output file path")
# Create template command
template_parser = subparsers.add_parser(
"create-template", help="Create AI translation template"
)
template_parser.add_argument(
"--output", required=True, help="Output template file path"
)
template_parser = subparsers.add_parser("create-template", help="Create AI translation template")
template_parser.add_argument("--output", required=True, help="Output template file path")
# Apply translations command
apply_parser = subparsers.add_parser(
"apply-translations", help="Apply translations from JSON file"
)
apply_parser.add_argument(
"--translations-file", required=True, help="JSON file with translations"
)
apply_parser.add_argument(
"--backup", action="store_true", help="Create backup before modifying files"
)
apply_parser = subparsers.add_parser("apply-translations", help="Apply translations from JSON file")
apply_parser.add_argument("--translations-file", required=True, help="JSON file with translations")
apply_parser.add_argument("--backup", action="store_true", help="Create backup before modifying files")
# Remove unused translations command
remove_parser = subparsers.add_parser(
"remove-unused", help="Remove unused translations not present in en-US"
)
remove_parser.add_argument(
"--backup", action="store_true", help="Create backup before modifying files"
)
remove_parser = subparsers.add_parser("remove-unused", help="Remove unused translations not present in en-US")
remove_parser.add_argument("--backup", action="store_true", help="Create backup before modifying files")
args = parser.parse_args()
@@ -453,9 +411,7 @@ def main():
continue
target_file = lang_dir / "translation.toml"
print(f"Processing {lang_dir.name}...")
result = merger.add_missing_translations(
target_file, backup=args.backup
)
result = merger.add_missing_translations(target_file, backup=args.backup)
added = result["added_count"]
total_added += added
print(f"Added {added} missing translations")
@@ -475,9 +431,7 @@ def main():
continue
target_file = lang_dir / "translation.toml"
print(f"Processing {lang_dir.name}...")
result = merger.remove_unused_translations(
target_file, backup=args.backup
)
result = merger.remove_unused_translations(target_file, backup=args.backup)
removed = result["removed_count"]
total_removed += removed
print(f"Removed {removed} unused translations")
@@ -489,11 +443,7 @@ def main():
sys.exit(1)
lang_dir = Path(args.locales_dir) / args.language
target_file = lang_dir / "translation.toml"
output_file = (
Path(args.output)
if args.output
else target_file.with_suffix(".untranslated.json")
)
output_file = Path(args.output) if args.output else target_file.with_suffix(".untranslated.json")
untranslated = merger.extract_untranslated_entries(target_file, output_file)
print(f"Extracted {len(untranslated)} untranslated entries to {output_file}")
@@ -512,22 +462,18 @@ def main():
lang_dir = Path(args.locales_dir) / args.language
target_file = lang_dir / "translation.toml"
with open(args.translations_file, "r", encoding="utf-8") as f:
with open(args.translations_file, encoding="utf-8") as f:
translations_data = json.load(f)
# Extract translations from template format or simple dict
if "translations" in translations_data:
translations = {
k: v["translated"]
for k, v in translations_data["translations"].items()
if v.get("translated")
k: v["translated"] for k, v in translations_data["translations"].items() if v.get("translated")
}
else:
translations = translations_data
result = merger.apply_translations(
target_file, translations, backup=args.backup
)
result = merger.apply_translations(target_file, translations, backup=args.backup)
if result["success"]:
print(f"Applied {result['applied_count']} translations")
scripts/translations/validate_json_structure.py
+9 -20
View File
@@ -13,15 +13,14 @@ Usage:
python scripts/translations/validate_json_structure.py [--language LANG]
"""
import argparse
import json
import sys
from pathlib import Path
from typing import Dict, Set
import argparse
import tomllib # Python 3.11+ (stdlib)
from pathlib import Path
def get_all_keys(d: dict, parent_key: str = "", sep: str = ".") -> Set[str]:
def get_all_keys(d: dict, parent_key: str = "", sep: str = ".") -> set[str]:
"""Get all keys from nested dict as dot-notation paths."""
keys = set()
for k, v in d.items():
@@ -42,9 +41,7 @@ def validate_translation_file(file_path: Path) -> tuple[bool, str]:
return False, f"Error reading file: {str(e)}"
def validate_structure(
en_us_keys: Set[str], lang_keys: Set[str], lang_code: str
) -> Dict:
def validate_structure(en_us_keys: set[str], lang_keys: set[str], lang_code: str) -> dict:
"""Compare structure between en-US and target language."""
missing_keys = en_us_keys - lang_keys
extra_keys = lang_keys - en_us_keys
@@ -60,7 +57,7 @@ def validate_structure(
}
def print_validation_result(result: Dict, verbose: bool = False):
def print_validation_result(result: dict, verbose: bool = False):
"""Print validation results in readable format."""
lang = result["language"]
@@ -111,9 +108,7 @@ def main():
help="Specific language code to validate (e.g., es-ES)",
default=None,
)
parser.add_argument(
"--verbose", "-v", action="store_true", help="Show all missing/extra keys"
)
parser.add_argument("--verbose", "-v", action="store_true", help="Show all missing/extra keys")
parser.add_argument("--json", action="store_true", help="Output results as JSON")
args = parser.parse_args()
@@ -162,9 +157,7 @@ def main():
# First check if file is valid
is_valid, message = validate_translation_file(lang_path)
if not is_valid:
json_errors.append(
{"language": lang_code, "file": str(lang_path), "error": message}
)
json_errors.append({"language": lang_code, "file": str(lang_path), "error": message})
continue
# Load and compare structure
@@ -194,9 +187,7 @@ def main():
print("\n📊 Structure Validation Summary:")
print(f" Languages validated: {len(results)}")
perfect = sum(
1 for r in results if r["missing_count"] == 0 and r["extra_count"] == 0
)
perfect = sum(1 for r in results if r["missing_count"] == 0 and r["extra_count"] == 0)
print(f" Perfect matches: {perfect}/{len(results)}")
total_missing = sum(r["missing_count"] for r in results)
@@ -211,9 +202,7 @@ def main():
print("\n✅ All translations have perfect structure!")
# Exit with error code if issues found
has_issues = len(json_errors) > 0 or any(
r["missing_count"] > 0 or r["extra_count"] > 0 for r in results
)
has_issues = len(json_errors) > 0 or any(r["missing_count"] > 0 or r["extra_count"] > 0 for r in results)
sys.exit(1 if has_issues else 0)
scripts/translations/validate_placeholders.py
+7 -12
View File
@@ -9,23 +9,22 @@ Usage:
--fix: Automatically remove extra placeholders (use with caution)
"""
import argparse
import json
import re
import sys
from pathlib import Path
from typing import Dict, List, Set
import argparse
import tomllib # Python 3.11+ (stdlib)
from pathlib import Path
def find_placeholders(text: str) -> Set[str]:
def find_placeholders(text: str) -> set[str]:
"""Find all placeholders in text like {n}, {{var}}, {0}, etc."""
if not isinstance(text, str):
return set()
return set(re.findall(r"\{\{?[^}]+\}\}?", text))
def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> Dict[str, str]:
def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> dict[str, str]:
"""Flatten nested dict to dot-notation keys."""
items = []
for k, v in d.items():
@@ -37,9 +36,7 @@ def flatten_dict(d: dict, parent_key: str = "", sep: str = ".") -> Dict[str, str
return dict(items)
def validate_language(
en_us_flat: Dict[str, str], lang_flat: Dict[str, str], lang_code: str
) -> List[Dict]:
def validate_language(en_us_flat: dict[str, str], lang_flat: dict[str, str], lang_code: str) -> list[dict]:
"""Validate placeholders for a language against en-US."""
issues = []
@@ -67,7 +64,7 @@ def validate_language(
return issues
def print_issues(issues: List[Dict], verbose: bool = False):
def print_issues(issues: list[dict], verbose: bool = False):
"""Print validation issues in a readable format."""
if not issues:
print("✅ No placeholder validation issues found!")
@@ -93,9 +90,7 @@ def print_issues(issues: List[Dict], verbose: bool = False):
def main():
parser = argparse.ArgumentParser(
description="Validate translation placeholder consistency"
)
parser = argparse.ArgumentParser(description="Validate translation placeholder consistency")
parser.add_argument(
"--language",
help="Specific language code to validate (e.g., es-ES)",
scripts/type3_to_cff.py
+36 -53
View File
@@ -19,9 +19,9 @@ import argparse
import json
import math
import sys
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
from fontTools.fontBuilder import FontBuilder
from fontTools.misc.fixedTools import otRound
@@ -29,17 +29,16 @@ from fontTools.pens.cu2quPen import Cu2QuPen
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.pens.ttGlyphPen import TTGlyphPen
Command = Dict[str, object]
Matrix = Tuple[float, float, float, float, float, float]
Command = dict[str, object]
Matrix = tuple[float, float, float, float, float, float]
@dataclass
class GlyphSource:
name: str
width: float
unicode: Optional[int]
char_code: Optional[int]
unicode: int | None
char_code: int | None
outline: Sequence[Command]
@@ -48,34 +47,20 @@ class GlyphBuildResult:
name: str
width: int
charstring: object
ttf_glyph: Optional[object]
unicode: Optional[int]
char_code: Optional[int]
bounds: Optional[Tuple[float, float, float, float]]
ttf_glyph: object | None
unicode: int | None
char_code: int | None
bounds: tuple[float, float, float, float] | None
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Synthesize fonts from Type3 glyph JSON."
)
parser.add_argument(
"--input", required=True, help="Path to glyph JSON emitted by the backend"
)
parser.add_argument(
"--otf-output", required=True, help="Destination path for the CFF/OTF font"
)
parser.add_argument(
"--ttf-output", help="Optional destination path for a TrueType font"
)
parser.add_argument(
"--family-name", default="Type3 Synth", help="Family name for the output"
)
parser.add_argument(
"--style-name", default="Regular", help="Style name for the output"
)
parser.add_argument(
"--units-per-em", type=int, default=1000, help="Units per EM value"
)
parser = argparse.ArgumentParser(description="Synthesize fonts from Type3 glyph JSON.")
parser.add_argument("--input", required=True, help="Path to glyph JSON emitted by the backend")
parser.add_argument("--otf-output", required=True, help="Destination path for the CFF/OTF font")
parser.add_argument("--ttf-output", help="Optional destination path for a TrueType font")
parser.add_argument("--family-name", default="Type3 Synth", help="Family name for the output")
parser.add_argument("--style-name", default="Regular", help="Style name for the output")
parser.add_argument("--units-per-em", type=int, default=1000, help="Units per EM value")
parser.add_argument(
"--cu2qu-error",
type=float,
@@ -85,7 +70,7 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
def load_json(path: Path) -> Dict[str, object]:
def load_json(path: Path) -> dict[str, object]:
try:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
@@ -94,7 +79,7 @@ def load_json(path: Path) -> Dict[str, object]:
sys.exit(2)
def parse_font_matrix(rows: Optional[Iterable[Iterable[float]]]) -> Matrix:
def parse_font_matrix(rows: Iterable[Iterable[float]] | None) -> Matrix:
"""
Retrieve the raw 2×3 FontMatrix entries for diagnostics. Type3 glyph
outlines in our extractor are emitted in their native coordinate system, so
@@ -102,7 +87,7 @@ def parse_font_matrix(rows: Optional[Iterable[Iterable[float]]]) -> Matrix:
"""
if not rows:
return (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
values: List[List[float]] = []
values: list[list[float]] = []
for row in rows:
try:
values.append([float(col) for col in row])
@@ -132,10 +117,10 @@ def resolve_width(raw_width: float, default: int) -> int:
def quadratic_to_cubic(
current: Tuple[float, float],
ctrl: Tuple[float, float],
end: Tuple[float, float],
) -> Tuple[Tuple[float, float], Tuple[float, float], Tuple[float, float]]:
current: tuple[float, float],
ctrl: tuple[float, float],
end: tuple[float, float],
) -> tuple[tuple[float, float], tuple[float, float], tuple[float, float]]:
"""
Convert a quadratic Bézier segment to cubic control points.
"""
@@ -150,9 +135,9 @@ def quadratic_to_cubic(
return c1, c2, end
def iterate_glyphs(data: Dict[str, object]) -> List[GlyphSource]:
def iterate_glyphs(data: dict[str, object]) -> list[GlyphSource]:
glyph_records = data.get("glyphs") or []
sources: List[GlyphSource] = []
sources: list[GlyphSource] = []
for index, record in enumerate(glyph_records, start=1):
if not isinstance(record, dict):
continue
@@ -170,9 +155,7 @@ def iterate_glyphs(data: Dict[str, object]) -> List[GlyphSource]:
char_code_value = record.get("code")
if not isinstance(char_code_value, int):
char_code_value = record.get("charCodeRaw")
if not isinstance(char_code_value, int) or not (
0 <= char_code_value <= 0x10FFFF
):
if not isinstance(char_code_value, int) or not (0 <= char_code_value <= 0x10FFFF):
char_code_value = None
outline = record.get("outline")
if not isinstance(outline, list):
@@ -192,19 +175,19 @@ def iterate_glyphs(data: Dict[str, object]) -> List[GlyphSource]:
def build_cff_charstring(
glyph: GlyphSource,
width: int,
) -> Tuple[object, Optional[Tuple[float, float, float, float]]]:
) -> tuple[object, tuple[float, float, float, float] | None]:
pen = T2CharStringPen(width=width, glyphSet=None)
bounds = [math.inf, math.inf, -math.inf, -math.inf]
def update_bounds(point: Tuple[float, float]) -> None:
def update_bounds(point: tuple[float, float]) -> None:
x, y = point
bounds[0] = min(bounds[0], x)
bounds[1] = min(bounds[1], y)
bounds[2] = max(bounds[2], x)
bounds[3] = max(bounds[3], y)
current: Optional[Tuple[float, float]] = None
start_point: Optional[Tuple[float, float]] = None
current: tuple[float, float] | None = None
start_point: tuple[float, float] | None = None
open_path = False
for command in glyph.outline:
@@ -278,7 +261,7 @@ def build_cff_charstring(
return charstring, bbox
def build_ttf_glyph(glyph: GlyphSource, max_error: float) -> Optional[object]:
def build_ttf_glyph(glyph: GlyphSource, max_error: float) -> object | None:
pen = TTGlyphPen(glyphSet=None)
draw_pen = Cu2QuPen(pen, max_error, reverse_direction=False)
@@ -321,9 +304,9 @@ def build_ttf_glyph(glyph: GlyphSource, max_error: float) -> Optional[object]:
def synthesise_fonts(
data: Dict[str, object],
data: dict[str, object],
otf_output: Path,
ttf_output: Optional[Path],
ttf_output: Path | None,
family_name: str,
style_name: str,
units_per_em: int,
@@ -332,7 +315,7 @@ def synthesise_fonts(
_font_matrix = parse_font_matrix(data.get("fontMatrix"))
glyphs = iterate_glyphs(data)
results: List[GlyphBuildResult] = []
results: list[GlyphBuildResult] = []
global_y_min = math.inf
global_y_max = -math.inf
@@ -377,7 +360,7 @@ def synthesise_fonts(
horizontal_metrics = {result.name: (result.width, 0) for result in results}
horizontal_metrics[".notdef"] = (default_width, 0)
cmap: Dict[int, str] = {}
cmap: dict[int, str] = {}
next_private = 0xF000
for result in results:
code_point = result.unicode
@@ -433,7 +416,7 @@ def synthesise_fonts(
if ttf_output is None:
return
glyph_objects: Dict[str, object] = {}
glyph_objects: dict[str, object] = {}
empty_pen = TTGlyphPen(None)
empty_pen.moveTo((0, 0))
empty_pen.lineTo((0, 0))
scripts/update_type3_library.py
+14 -32
View File
@@ -17,25 +17,15 @@ from __future__ import annotations
import argparse
import json
import sys
from collections.abc import Iterable
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_SIGNATURES = REPO_ROOT / "docs" / "type3" / "signatures"
DEFAULT_INDEX = (
REPO_ROOT
/ "app"
/ "core"
/ "src"
/ "main"
/ "resources"
/ "type3"
/ "library"
/ "index.json"
)
DEFAULT_INDEX = REPO_ROOT / "app" / "core" / "src" / "main" / "resources" / "type3" / "library" / "index.json"
def normalize_alias(value: Optional[str]) -> Optional[str]:
def normalize_alias(value: str | None) -> str | None:
if not value:
return None
trimmed = value.strip()
@@ -75,9 +65,9 @@ def iter_signature_fonts(signature_file: Path):
}
def make_alias_index(entries: List[Dict]) -> Tuple[Dict[str, Dict], Dict[str, Dict]]:
alias_index: Dict[str, Dict] = {}
signature_index: Dict[str, Dict] = {}
def make_alias_index(entries: list[dict]) -> tuple[dict[str, dict], dict[str, dict]]:
alias_index: dict[str, dict] = {}
signature_index: dict[str, dict] = {}
for entry in entries:
for alias in entry.get("aliases", []) or []:
normalized = normalize_alias(alias)
@@ -91,7 +81,7 @@ def make_alias_index(entries: List[Dict]) -> Tuple[Dict[str, Dict], Dict[str, Di
return alias_index, signature_index
def ensure_list(container: Dict, key: str) -> List:
def ensure_list(container: dict, key: str) -> list:
value = container.get(key)
if isinstance(value, list):
return value
@@ -100,11 +90,11 @@ def ensure_list(container: Dict, key: str) -> List:
return value
def merge_sorted_unique(values: Iterable[int]) -> List[int]:
def merge_sorted_unique(values: Iterable[int]) -> list[int]:
return sorted({int(v) for v in values if isinstance(v, int)})
def normalize_source_path(pdf_path: Optional[str]) -> Optional[str]:
def normalize_source_path(pdf_path: str | None) -> str | None:
if not pdf_path:
return None
try:
@@ -117,13 +107,13 @@ def normalize_source_path(pdf_path: Optional[str]) -> Optional[str]:
def update_library(
signatures_dir: Path, index_path: Path, apply_changes: bool
) -> Tuple[int, int, List[Tuple[str, Path]]]:
) -> tuple[int, int, list[tuple[str, Path]]]:
entries = load_json(index_path)
alias_index, signature_index = make_alias_index(entries)
modifications = 0
updated_entries = set()
unmatched: List[Tuple[str, Path]] = []
unmatched: list[tuple[str, Path]] = []
signature_files = sorted(signatures_dir.glob("*.json"))
if not signature_files:
@@ -198,9 +188,7 @@ def update_library(
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Update Type3 library index using signature dumps."
)
parser = argparse.ArgumentParser(description="Update Type3 library index using signature dumps.")
parser.add_argument(
"--signatures-dir",
type=Path,
@@ -223,11 +211,7 @@ def parse_args() -> argparse.Namespace:
def main() -> None:
args = parse_args()
signatures_dir = (
args.signatures_dir
if args.signatures_dir.is_absolute()
else (REPO_ROOT / args.signatures_dir)
)
signatures_dir = args.signatures_dir if args.signatures_dir.is_absolute() else (REPO_ROOT / args.signatures_dir)
index_path = args.index if args.index.is_absolute() else (REPO_ROOT / args.index)
if not signatures_dir.exists():
@@ -237,9 +221,7 @@ def main() -> None:
print(f"Index file not found: {index_path}", file=sys.stderr)
sys.exit(2)
modifications, updated_entries, unmatched = update_library(
signatures_dir, index_path, apply_changes=args.apply
)
modifications, updated_entries, unmatched = update_library(signatures_dir, index_path, apply_changes=args.apply)
mode = "APPLIED" if args.apply else "DRY-RUN"
print(