mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-02 21:03:34 +03:00
Update translations for 40 languages via GPT-5.5 (#6954)
# Description of Changes - Adds and updates translations across **40 languages** (~1,400–2,200 keys each) using GPT-5.5, filling previously-missing UI strings. - Switches the translation scripts' default model from the year-old `gpt-5` (5.0) to `gpt-5.5`, adding a `--model` flag and token/cost reporting. - Purely additive and validated: no existing translations changed, all 40 files match the en-US key structure, and no new placeholder issues introduced. --- ## Checklist ### General - [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md) - [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable) - [x] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable) - [x] I have performed a self-review of my own code - [x] My changes generate no new warnings ### Documentation - [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed) - [x] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only) ### Translations (if applicable) - [x] I ran [`scripts/counter_translation.py`](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/docs/counter_translation.md) ### UI Changes (if applicable) - [ ] Screenshots or videos demonstrating the UI changes are attached (e.g., as comments or direct attachments in the PR) ### Testing (if applicable) - [ ] I have run `task check` to verify linters, typechecks, and tests pass - [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#7-testing) for more details.
This commit is contained in:
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -175,8 +175,6 @@ ignore = [
|
||||
'certSign.collab.participant.certInvalid',
|
||||
'certSign.collab.signRequest.steps.invisible',
|
||||
'certSign.collab.signRequest.steps.visible',
|
||||
'certSign.collab.signRequest.text.colorLabel',
|
||||
'certSign.collab.signRequest.text.fontSizePlaceholder',
|
||||
'common.error',
|
||||
'compare.pixel.base',
|
||||
'config.account.profilePicture.cropper.zoom',
|
||||
@@ -205,7 +203,6 @@ ignore = [
|
||||
'getPdfInfo.summary.tech.javaScript',
|
||||
'getPdfInfo.summary.tech.multimedia',
|
||||
'home.devApi.title',
|
||||
'landing.heroTitle',
|
||||
'language.direction',
|
||||
'mobileScanner.flash',
|
||||
'no',
|
||||
@@ -482,7 +479,6 @@ ignore = [
|
||||
'admin.settings.connections.saml2.label',
|
||||
'admin.settings.security.xFrameOptions.label',
|
||||
'certSign.collab.participant.certInvalid',
|
||||
'certSign.collab.signRequest.text.fontSizePlaceholder',
|
||||
'convert.cbrToPdf',
|
||||
'convert.cbzToPdf',
|
||||
'convert.dpi',
|
||||
|
||||
@@ -29,7 +29,7 @@ python3 scripts/translations/auto_translate.py es-ES --no-cleanup
|
||||
**What it does:**
|
||||
1. Extracts untranslated entries from the language file
|
||||
2. Splits into batches (default 500 entries each)
|
||||
3. Translates each batch using GPT-5 with specialized prompts
|
||||
3. Translates each batch using GPT-5.6 with specialized prompts
|
||||
4. Validates placeholders are preserved
|
||||
5. Merges translated batches
|
||||
6. Applies translations to language file
|
||||
@@ -39,7 +39,7 @@ python3 scripts/translations/auto_translate.py es-ES --no-cleanup
|
||||
|
||||
**Time:** ~8-10 minutes per language with 1200+ untranslated entries
|
||||
|
||||
**Cost:** ~$2-4 per language using GPT-5 (or use `gpt-5-mini` for lower cost)
|
||||
**Cost:** ~$8-15 per language using GPT-5.5 (or use `gpt-5.6-luna` for ~$2-4 if your org has 5.6 access)
|
||||
|
||||
See [`auto_translate.py`](#auto_translatepy-automated-translation-pipeline) for full details.
|
||||
|
||||
@@ -248,7 +248,7 @@ python scripts/translations/compact_translator.py it-IT --output to_translate.js
|
||||
|
||||
### 5. `auto_translate.py` - Automated Translation Pipeline
|
||||
|
||||
**NEW: Fully automated translation workflow using GPT-5.**
|
||||
**NEW: Fully automated translation workflow using GPT-5.6.**
|
||||
|
||||
Combines all translation steps into a single command that handles everything from extraction to verification.
|
||||
|
||||
@@ -276,7 +276,7 @@ python3 scripts/translations/auto_translate.py es-ES --skip-verification
|
||||
|
||||
**Features:**
|
||||
- Fully automated end-to-end translation pipeline
|
||||
- Uses GPT-5 with specialized prompts for Stirling PDF
|
||||
- Uses GPT-5.6 with specialized prompts for Stirling PDF
|
||||
- Preserves all placeholders ({n}, {{variable}}, etc.)
|
||||
- Maintains consistent terminology
|
||||
- Validates translations automatically
|
||||
@@ -286,7 +286,7 @@ python3 scripts/translations/auto_translate.py es-ES --skip-verification
|
||||
**Pipeline Steps:**
|
||||
1. **Extract**: Finds all untranslated entries
|
||||
2. **Split**: Divides into manageable batches (default: 500 entries)
|
||||
3. **Translate**: Uses GPT-5 to translate each batch with specialized prompts
|
||||
3. **Translate**: Uses GPT-5.6 to translate each batch with specialized prompts
|
||||
4. **Validate**: Ensures placeholders are preserved
|
||||
5. **Merge**: Combines all translated batches
|
||||
6. **Apply**: Updates the language file
|
||||
@@ -304,7 +304,7 @@ python3 scripts/translations/auto_translate.py es-ES --skip-verification
|
||||
**Supported Languages:**
|
||||
All language codes from `frontend/editor/public/locales/` (e.g., es-ES, de-DE, fr-FR, zh-CN, ar-AR, etc.)
|
||||
|
||||
### 6. `batch_translator.py` - GPT-5 Translation Engine
|
||||
### 6. `batch_translator.py` - GPT-5.6 Translation Engine
|
||||
|
||||
Low-level translation script used by `auto_translate.py`. Can be used standalone for manual batch translation.
|
||||
|
||||
@@ -316,25 +316,27 @@ python3 scripts/translations/batch_translator.py my_batch.json --language es-ES
|
||||
# Translate multiple batches
|
||||
python3 scripts/translations/batch_translator.py batch_*.json --language de-DE --api-key YOUR_KEY
|
||||
|
||||
# Use different GPT model
|
||||
python3 scripts/translations/batch_translator.py batch.json --language fr-FR --model gpt-5-mini
|
||||
# Use a cheaper model
|
||||
python3 scripts/translations/batch_translator.py batch.json --language fr-FR --model gpt-5.6-luna
|
||||
|
||||
# Skip validation
|
||||
python3 scripts/translations/batch_translator.py batch.json --language it-IT --skip-validation
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Translates JSON batch files using OpenAI GPT-5
|
||||
- Translates JSON batch files using OpenAI GPT-5.6
|
||||
- Specialized system prompts for Stirling PDF translations
|
||||
- Automatic placeholder validation
|
||||
- Supports pattern matching for multiple files
|
||||
- Configurable model selection (gpt-5, gpt-5-mini, gpt-5-nano)
|
||||
- Configurable model selection (gpt-5.6-sol, gpt-5.6-terra, gpt-5.6-luna)
|
||||
- Per-batch and per-run token + cost reporting
|
||||
- Rate limiting with configurable delays
|
||||
|
||||
**Models:**
|
||||
- `gpt-5` (default): Best quality, $1.25/1M input, $10/1M output
|
||||
- `gpt-5-mini`: Balanced quality/cost
|
||||
- `gpt-5-nano`: Fastest, most economical
|
||||
- `gpt-5.5` (default): Previous flagship, strong translation quality, $5/1M input, $30/1M output
|
||||
- `gpt-5.6-sol`: Newest flagship reasoning, $5/1M input, $30/1M output (requires org 5.6 access)
|
||||
- `gpt-5.6-terra`: Balanced quality/cost, $2.50/1M input, $15/1M output (requires org 5.6 access)
|
||||
- `gpt-5.6-luna`: Fastest/cheapest, $1/1M input, $6/1M output (requires org 5.6 access)
|
||||
|
||||
### 7. `json_beautifier.py`
|
||||
Restructures and beautifies translation JSON files to match en-US structure exactly.
|
||||
|
||||
@@ -140,12 +140,14 @@ def extract_untranslated(language_code, batch_size=500, include_existing=False):
|
||||
return batch_files
|
||||
|
||||
|
||||
def translate_batches(batch_files, language_code, api_key, timeout=600):
|
||||
"""Translate all batch files using GPT-5."""
|
||||
def translate_batches(
|
||||
batch_files, language_code, api_key, timeout=600, model="gpt-5.5"
|
||||
):
|
||||
"""Translate all batch files using the given OpenAI model."""
|
||||
if not batch_files:
|
||||
return []
|
||||
|
||||
print(f"\n🤖 Translating {len(batch_files)} batches using GPT-5...")
|
||||
print(f"\n🤖 Translating {len(batch_files)} batches using {model}...")
|
||||
print(f"Timeout: {timeout}s ({timeout // 60} minutes) per batch")
|
||||
|
||||
translated_files = []
|
||||
@@ -154,7 +156,7 @@ def translate_batches(batch_files, language_code, api_key, timeout=600):
|
||||
print(f"\n[{i}/{len(batch_files)}] Translating {batch_file}...")
|
||||
|
||||
# Always pass API key since it's required
|
||||
cmd = f'python3 scripts/translations/batch_translator.py "{batch_file}" --language {language_code} --api-key "{api_key}"'
|
||||
cmd = f'python3 scripts/translations/batch_translator.py "{batch_file}" --language {language_code} --api-key "{api_key}" --model {model}'
|
||||
|
||||
# Run with timeout
|
||||
result = subprocess.run(
|
||||
@@ -308,6 +310,11 @@ Examples:
|
||||
action="store_true",
|
||||
help="Also retranslate existing keys that match English (default: only translate missing keys)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="gpt-5.5",
|
||||
help="OpenAI model (default: gpt-5.5; gpt-5.6-sol/terra/luna if your org has 5.6 access)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -322,6 +329,7 @@ Examples:
|
||||
print("=" * 60)
|
||||
print("Automated Translation Pipeline")
|
||||
print(f"Language: {args.language}")
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Batch Size: {args.batch_size} entries")
|
||||
print("=" * 60)
|
||||
|
||||
@@ -341,7 +349,7 @@ Examples:
|
||||
|
||||
# Step 2: Translate all batches
|
||||
translated_files = translate_batches(
|
||||
batch_files, args.language, api_key, args.timeout
|
||||
batch_files, args.language, api_key, args.timeout, args.model
|
||||
)
|
||||
if translated_files is None:
|
||||
sys.exit(1)
|
||||
|
||||
@@ -22,11 +22,32 @@ except ImportError:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# USD per 1M tokens (input, output)
|
||||
MODEL_PRICING = {
|
||||
"gpt-5.5": (5.0, 30.0),
|
||||
"gpt-5.6-sol": (5.0, 30.0),
|
||||
"gpt-5.6-terra": (2.5, 15.0),
|
||||
"gpt-5.6-luna": (1.0, 6.0),
|
||||
"gpt-5": (1.25, 10.0),
|
||||
}
|
||||
|
||||
|
||||
def estimate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float:
|
||||
"""Estimate USD cost for a call; 0.0 if model pricing is unknown."""
|
||||
if model not in MODEL_PRICING:
|
||||
return 0.0
|
||||
in_price, out_price = MODEL_PRICING[model]
|
||||
return (prompt_tokens * in_price + completion_tokens * out_price) / 1_000_000
|
||||
|
||||
|
||||
class BatchTranslator:
|
||||
def __init__(self, api_key: str, model: str = "gpt-5"):
|
||||
def __init__(self, api_key: str, model: str = "gpt-5.5"):
|
||||
"""Initialize translator with OpenAI API key."""
|
||||
self.client = OpenAI(api_key=api_key)
|
||||
self.model = model
|
||||
self.total_prompt_tokens = 0
|
||||
self.total_completion_tokens = 0
|
||||
self.total_cost = 0.0
|
||||
|
||||
def get_translation_prompt(self, language_name: str, language_code: str) -> str:
|
||||
"""Generate the system prompt for translation."""
|
||||
@@ -79,6 +100,23 @@ CRITICAL RULES - MUST FOLLOW EXACTLY:
|
||||
|
||||
Return ONLY the translated JSON. No markdown, no explanations, just the JSON object."""
|
||||
|
||||
def _record_usage(self, response) -> None:
|
||||
"""Accumulate token usage/cost and print a per-batch line."""
|
||||
usage = getattr(response, "usage", None)
|
||||
if usage is None:
|
||||
return
|
||||
|
||||
prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
|
||||
completion_tokens = getattr(usage, "completion_tokens", 0) or 0
|
||||
cost = estimate_cost(self.model, prompt_tokens, completion_tokens)
|
||||
|
||||
self.total_prompt_tokens += prompt_tokens
|
||||
self.total_completion_tokens += completion_tokens
|
||||
self.total_cost += cost
|
||||
|
||||
cost_note = f", ~${cost:.4f}" if cost else ""
|
||||
print(f" Tokens: {prompt_tokens:,} in / {completion_tokens:,} out{cost_note}")
|
||||
|
||||
def translate_batch(
|
||||
self, batch_data: dict, target_language: str, language_code: str
|
||||
) -> dict:
|
||||
@@ -90,7 +128,7 @@ Return ONLY the translated JSON. No markdown, no explanations, just the JSON obj
|
||||
print(f"Input size: {len(input_json)} characters")
|
||||
|
||||
try:
|
||||
# GPT-5 only supports temperature=1, so we don't include it
|
||||
# GPT-5.x models only support the default temperature, so we omit it
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
@@ -107,6 +145,8 @@ Return ONLY the translated JSON. No markdown, no explanations, just the JSON obj
|
||||
],
|
||||
)
|
||||
|
||||
self._record_usage(response)
|
||||
|
||||
translated_text = response.choices[0].message.content.strip()
|
||||
|
||||
# Remove markdown code blocks if present
|
||||
@@ -250,8 +290,8 @@ Examples:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="gpt-5",
|
||||
help="OpenAI model to use (default: gpt-5, options: gpt-5-mini, gpt-5-nano)",
|
||||
default="gpt-5.5",
|
||||
help="OpenAI model (default: gpt-5.5; gpt-5.6-sol/terra/luna if your org has 5.6 access)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-suffix",
|
||||
@@ -354,6 +394,15 @@ Examples:
|
||||
if failed > 0:
|
||||
print(f"Failed: {failed}/{len(input_files)}")
|
||||
|
||||
# Cost summary
|
||||
print("-" * 60)
|
||||
print(
|
||||
f"Total tokens: {translator.total_prompt_tokens:,} in / "
|
||||
f"{translator.total_completion_tokens:,} out"
|
||||
)
|
||||
if translator.total_cost:
|
||||
print(f"Estimated cost ({args.model}): ${translator.total_cost:.4f}")
|
||||
|
||||
sys.exit(0 if failed == 0 else 1)
|
||||
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ def translate_language(
|
||||
timeout: int,
|
||||
skip_verification: bool,
|
||||
include_existing: bool,
|
||||
model: str,
|
||||
) -> Tuple[str, bool, str]:
|
||||
"""
|
||||
Translate a single language.
|
||||
@@ -115,6 +116,8 @@ def translate_language(
|
||||
str(batch_size),
|
||||
"--timeout",
|
||||
str(timeout),
|
||||
"--model",
|
||||
model,
|
||||
]
|
||||
|
||||
if skip_verification:
|
||||
@@ -139,11 +142,11 @@ def translate_language(
|
||||
safe_print(f"[{language}] ✓ Success")
|
||||
return (language, True, "Success")
|
||||
else:
|
||||
error_msg = (
|
||||
result.stderr.strip() or result.stdout.strip() or "Unknown error"
|
||||
)
|
||||
safe_print(f"[{language}] ✗ Failed: {error_msg[:100]}")
|
||||
return (language, False, error_msg[:200]) # Truncate long errors
|
||||
# Show the actual error (last line of output), not the header
|
||||
output = result.stderr.strip() or result.stdout.strip() or "Unknown error"
|
||||
error_msg = output.splitlines()[-1][:200]
|
||||
safe_print(f"[{language}] ✗ Failed: {error_msg}")
|
||||
return (language, False, error_msg)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
safe_print(f"[{language}] ✗ Timeout exceeded")
|
||||
@@ -178,6 +181,11 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
|
||||
parser.add_argument(
|
||||
"--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="gpt-5.5",
|
||||
help="OpenAI model (default: gpt-5.5; gpt-5.6-sol/terra/luna if your org has 5.6 access)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--parallel",
|
||||
type=int,
|
||||
@@ -277,6 +285,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
|
||||
print("Bulk Translation Configuration")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"Languages to translate: {len(languages)}")
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Parallel threads: {args.parallel}")
|
||||
print(f"Batch size: {args.batch_size}")
|
||||
print(f"Timeout per batch: {args.timeout}s")
|
||||
@@ -308,6 +317,7 @@ Note: Requires OPENAI_API_KEY environment variable or --api-key argument.
|
||||
args.timeout,
|
||||
args.skip_verification,
|
||||
args.include_existing,
|
||||
args.model,
|
||||
): lang
|
||||
for lang in languages
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user