mirror of
https://github.com/Stirling-Tools/Stirling-PDF.git
synced 2026-09-03 05:10:16 +03:00
Ludy87
c835f2da06
Bump Python tooling to Ruff 0.16.2 and datamodel-code-generator 0.72.3, add the py313 target in pre-commit, and refresh the engine lockfile. This also narrows broad exception handling and applies small Python cleanups across the engine and helper scripts to satisfy newer lint rules without changing runtime behavior.
392 lines
14 KiB
Python
392 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Batch Translation Script using OpenAI API
|
|
Automatically translates JSON batch files to target language while preserving:
|
|
- Placeholders: {n}, {total}, {filename}, {{variable}}
|
|
- HTML tags: <strong>, </strong>, etc.
|
|
- Technical terms: PDF, API, OAuth2, SAML2, JWT, etc.
|
|
|
|
Note: Works with JSON batch files. Translation files can be TOML or JSON format.
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from openai import OpenAI
|
|
except ImportError:
|
|
print("Error: openai package not installed. Install with: pip install openai")
|
|
sys.exit(1)
|
|
|
|
|
|
# USD per 1M tokens (input, output)
|
|
MODEL_PRICING = {
|
|
"gpt-5.5": (5.0, 30.0),
|
|
"gpt-5.6-sol": (5.0, 30.0),
|
|
"gpt-5.6-terra": (2.5, 15.0),
|
|
"gpt-5.6-luna": (1.0, 6.0),
|
|
"gpt-5": (1.25, 10.0),
|
|
}
|
|
|
|
|
|
def estimate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float:
|
|
"""Estimate USD cost for a call; 0.0 if model pricing is unknown."""
|
|
if model not in MODEL_PRICING:
|
|
return 0.0
|
|
in_price, out_price = MODEL_PRICING[model]
|
|
return (prompt_tokens * in_price + completion_tokens * out_price) / 1_000_000
|
|
|
|
|
|
class BatchTranslator:
|
|
def __init__(self, api_key: str, model: str = "gpt-5.5"):
|
|
"""Initialize translator with OpenAI API key."""
|
|
self.client = OpenAI(api_key=api_key)
|
|
self.model = model
|
|
self.total_prompt_tokens = 0
|
|
self.total_completion_tokens = 0
|
|
self.total_cost = 0.0
|
|
|
|
def get_translation_prompt(self, language_name: str, language_code: str) -> str:
|
|
"""Generate the system prompt for translation."""
|
|
return f"""You are a professional translator for Stirling PDF, an open-source PDF manipulation tool.
|
|
|
|
Translate the following JSON from English to {language_name} ({language_code}) for the Stirling PDF user interface.
|
|
|
|
CRITICAL RULES - MUST FOLLOW EXACTLY:
|
|
|
|
1. PRESERVE ALL PLACEHOLDERS EXACTLY AS-IS:
|
|
- Single braces: {{{{n}}}}, {{{{total}}}}, {{{{filename}}}}, {{{{count}}}}, {{{{date}}}}, {{{{planName}}}}, {{{{toolName}}}}, {{{{variable}}}}
|
|
- Double braces: {{{{{{{{variable}}}}}}}}
|
|
- Never translate, modify, or remove these - they are template variables
|
|
|
|
2. KEEP ALL HTML TAGS INTACT:
|
|
- <strong>, </strong>, <br>, <code>, </code>, etc.
|
|
- Do not translate tag names, only text between tags
|
|
|
|
3. DO NOT TRANSLATE TECHNICAL TERMS:
|
|
- File formats: PDF, JSON, CSV, XML, HTML, ZIP, DOCX, XLSX, PNG, JPG
|
|
- Protocols: API, OAuth2, SAML2, JWT, SMTP, HTTP, HTTPS, SSL, TLS
|
|
- Technologies: Git, GitHub, Google, PostHog, Scarf, LibreOffice, Ghostscript, Tesseract, OCR
|
|
- Technical keywords: URL, URI, DPI, RGB, CMYK, QR
|
|
- "Stirling PDF" - always keep as-is
|
|
|
|
4. MAINTAIN CONSISTENT TERMINOLOGY:
|
|
- Use the SAME translation for repeated terms throughout
|
|
- Do not introduce new terminology or synonyms
|
|
- Keep UI action words consistent (e.g., "upload", "download", "compress")
|
|
|
|
5. PRESERVE SPECIAL KEYWORDS IN CONTEXT:
|
|
- Mathematical expressions: "2n", "2n-1", "3n" (in page selection)
|
|
- Special keywords: "all", "odd", "even" (in page contexts)
|
|
- Code examples and technical patterns
|
|
|
|
6. JSON STRUCTURE:
|
|
- Translate ONLY the values (text after :), NEVER the keys
|
|
- Return ONLY valid JSON with exact same structure
|
|
- Maintain all quotes, commas, and braces
|
|
|
|
7. TONE & STYLE:
|
|
- Use appropriate formal/informal tone for {language_name} UI
|
|
- Keep translations concise and user-friendly
|
|
- Maintain the professional but accessible tone of the original
|
|
|
|
8. DO NOT ADD OR REMOVE TEXT:
|
|
- Do not add explanations, comments, or extra text
|
|
- Do not remove any part of the original meaning
|
|
- Keep the same level of detail
|
|
|
|
Return ONLY the translated JSON. No markdown, no explanations, just the JSON object."""
|
|
|
|
def _record_usage(self, response) -> None:
|
|
"""Accumulate token usage/cost and print a per-batch line."""
|
|
usage = getattr(response, "usage", None)
|
|
if usage is None:
|
|
return
|
|
|
|
prompt_tokens = getattr(usage, "prompt_tokens", 0) or 0
|
|
completion_tokens = getattr(usage, "completion_tokens", 0) or 0
|
|
cost = estimate_cost(self.model, prompt_tokens, completion_tokens)
|
|
|
|
self.total_prompt_tokens += prompt_tokens
|
|
self.total_completion_tokens += completion_tokens
|
|
self.total_cost += cost
|
|
|
|
cost_note = f", ~${cost:.4f}" if cost else ""
|
|
print(f" Tokens: {prompt_tokens:,} in / {completion_tokens:,} out{cost_note}")
|
|
|
|
def translate_batch(self, batch_data: dict, target_language: str, language_code: str) -> dict:
|
|
"""Translate a batch file using OpenAI API."""
|
|
# Convert batch to compact JSON for API
|
|
input_json = json.dumps(batch_data, ensure_ascii=False, separators=(",", ":"))
|
|
|
|
print(f"Translating {len(batch_data)} entries to {target_language}...")
|
|
print(f"Input size: {len(input_json)} characters")
|
|
|
|
try:
|
|
# GPT-5.x models only support the default temperature, so we omit it
|
|
response = self.client.chat.completions.create(
|
|
model=self.model,
|
|
messages=[
|
|
{
|
|
"role": "system",
|
|
"content": self.get_translation_prompt(target_language, language_code),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": f"Translate this JSON:\n\n{input_json}",
|
|
},
|
|
],
|
|
)
|
|
|
|
self._record_usage(response)
|
|
|
|
translated_text = response.choices[0].message.content.strip()
|
|
|
|
# Remove markdown code blocks if present
|
|
if translated_text.startswith("```"):
|
|
lines = translated_text.split("\n")
|
|
translated_text = "\n".join(lines[1:-1])
|
|
|
|
# Parse the translated JSON
|
|
translated_data = json.loads(translated_text)
|
|
|
|
print("✓ Translation complete")
|
|
return translated_data
|
|
|
|
except json.JSONDecodeError as e:
|
|
print(f"Error: AI returned invalid JSON: {e}")
|
|
print(f"Response: {translated_text[:500]}...")
|
|
raise
|
|
except (OSError, ValueError, KeyError, RuntimeError) as e:
|
|
print(f"Error during translation: {e}")
|
|
raise
|
|
|
|
def validate_translation(self, original: dict, translated: dict) -> bool:
|
|
"""Validate that translation preserved all placeholders and structure."""
|
|
issues = []
|
|
|
|
# Check that all keys are present
|
|
if set(original.keys()) != set(translated.keys()):
|
|
missing = set(original.keys()) - set(translated.keys())
|
|
extra = set(translated.keys()) - set(original.keys())
|
|
if missing:
|
|
issues.append(f"Missing keys: {missing}")
|
|
if extra:
|
|
issues.append(f"Extra keys: {extra}")
|
|
|
|
# Check placeholders in each value
|
|
import re
|
|
|
|
placeholder_pattern = r"\{[^}]+\}|\{\{[^}]+\}\}"
|
|
|
|
for key, original_value in original.items():
|
|
if key not in translated:
|
|
continue
|
|
|
|
orig_value = str(original_value)
|
|
trans_value = str(translated[key])
|
|
|
|
# Find all placeholders in original
|
|
orig_placeholders = set(re.findall(placeholder_pattern, orig_value))
|
|
trans_placeholders = set(re.findall(placeholder_pattern, trans_value))
|
|
|
|
if orig_placeholders != trans_placeholders:
|
|
issues.append(f"Placeholder mismatch in '{key}': {orig_placeholders} vs {trans_placeholders}")
|
|
|
|
if issues:
|
|
print("\n⚠ Validation warnings:")
|
|
for issue in issues[:10]: # Show first 10 issues
|
|
print(f" - {issue}")
|
|
if len(issues) > 10:
|
|
print(f" ... and {len(issues) - 10} more issues")
|
|
return False
|
|
|
|
print("✓ Validation passed")
|
|
return True
|
|
|
|
|
|
def get_language_info(language_code: str) -> tuple:
|
|
"""Get full language name from code."""
|
|
languages = {
|
|
"zh-CN": ("Simplified Chinese", "zh-CN"),
|
|
"es-ES": ("Spanish", "es-ES"),
|
|
"it-IT": ("Italian", "it-IT"),
|
|
"de-DE": ("German", "de-DE"),
|
|
"ar-AR": ("Arabic", "ar-AR"),
|
|
"pt-BR": ("Brazilian Portuguese", "pt-BR"),
|
|
"ru-RU": ("Russian", "ru-RU"),
|
|
"fr-FR": ("French", "fr-FR"),
|
|
"ja-JP": ("Japanese", "ja-JP"),
|
|
"ko-KR": ("Korean", "ko-KR"),
|
|
"nl-NL": ("Dutch", "nl-NL"),
|
|
"pl-PL": ("Polish", "pl-PL"),
|
|
"sv-SE": ("Swedish", "sv-SE"),
|
|
"da-DK": ("Danish", "da-DK"),
|
|
"no-NB": ("Norwegian", "no-NB"),
|
|
"fi-FI": ("Finnish", "fi-FI"),
|
|
"tr-TR": ("Turkish", "tr-TR"),
|
|
"vi-VN": ("Vietnamese", "vi-VN"),
|
|
"th-TH": ("Thai", "th-TH"),
|
|
"id-ID": ("Indonesian", "id-ID"),
|
|
"hi-IN": ("Hindi", "hi-IN"),
|
|
"cs-CZ": ("Czech", "cs-CZ"),
|
|
"hu-HU": ("Hungarian", "hu-HU"),
|
|
"ro-RO": ("Romanian", "ro-RO"),
|
|
"uk-UA": ("Ukrainian", "uk-UA"),
|
|
"el-GR": ("Greek", "el-GR"),
|
|
"bg-BG": ("Bulgarian", "bg-BG"),
|
|
"hr-HR": ("Croatian", "hr-HR"),
|
|
"sk-SK": ("Slovak", "sk-SK"),
|
|
"sl-SI": ("Slovenian", "sl-SI"),
|
|
"ca-CA": ("Catalan", "ca-CA"),
|
|
}
|
|
|
|
return languages.get(language_code, (language_code, language_code))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Translate JSON batch files using OpenAI API (output supports TOML and JSON)",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Note: This script works with JSON batch files. The translation files it updates can be TOML or JSON.
|
|
|
|
Examples:
|
|
# Translate single batch file
|
|
python batch_translator.py zh_CN_batch_1_of_4.json --api-key YOUR_KEY --language zh-CN
|
|
|
|
# Translate all batches for a language (with pattern)
|
|
python batch_translator.py "zh_CN_batch_*_of_*.json" --api-key YOUR_KEY --language zh-CN
|
|
|
|
# Use environment variable for API key
|
|
export OPENAI_API_KEY=your_key_here
|
|
python batch_translator.py zh_CN_batch_1_of_4.json --language zh-CN
|
|
|
|
# Use different model
|
|
python batch_translator.py file.json --api-key KEY --language es-ES --model gpt-4-turbo
|
|
""",
|
|
)
|
|
|
|
parser.add_argument("input_files", nargs="+", help="Input batch JSON file(s) or pattern")
|
|
parser.add_argument("--api-key", help="OpenAI API key (or set OPENAI_API_KEY env var)")
|
|
parser.add_argument(
|
|
"--language",
|
|
"-l",
|
|
required=True,
|
|
help="Target language code (e.g., zh-CN, es-ES)",
|
|
)
|
|
parser.add_argument(
|
|
"--model",
|
|
default="gpt-5.5",
|
|
help="OpenAI model (default: gpt-5.5; gpt-5.6-sol/terra/luna if your org has 5.6 access)",
|
|
)
|
|
parser.add_argument(
|
|
"--output-suffix",
|
|
default="_translated",
|
|
help="Suffix for output files (default: _translated)",
|
|
)
|
|
parser.add_argument("--skip-validation", action="store_true", help="Skip validation checks")
|
|
parser.add_argument(
|
|
"--delay",
|
|
type=float,
|
|
default=1.0,
|
|
help="Delay between API calls in seconds (default: 1.0)",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Get API key from args or environment
|
|
import os
|
|
|
|
api_key = args.api_key or os.environ.get("OPENAI_API_KEY")
|
|
if not api_key:
|
|
print("Error: OpenAI API key required. Provide via --api-key or OPENAI_API_KEY environment variable")
|
|
sys.exit(1)
|
|
|
|
# Get language info
|
|
language_name, language_code = get_language_info(args.language)
|
|
|
|
# Expand file patterns
|
|
import glob
|
|
|
|
input_files = []
|
|
for pattern in args.input_files:
|
|
matched = glob.glob(pattern)
|
|
if matched:
|
|
input_files.extend(matched)
|
|
else:
|
|
input_files.append(pattern) # Use as literal filename
|
|
|
|
if not input_files:
|
|
print("Error: No input files found")
|
|
sys.exit(1)
|
|
|
|
print("Batch Translator")
|
|
print(f"Target Language: {language_name} ({language_code})")
|
|
print(f"Model: {args.model}")
|
|
print(f"Files to translate: {len(input_files)}")
|
|
print("=" * 60)
|
|
|
|
# Initialize translator
|
|
translator = BatchTranslator(api_key, args.model)
|
|
|
|
# Process each file
|
|
successful = 0
|
|
failed = 0
|
|
|
|
for i, input_file in enumerate(input_files, 1):
|
|
print(f"\n[{i}/{len(input_files)}] Processing: {input_file}")
|
|
|
|
try:
|
|
# Load input file
|
|
with open(input_file, encoding="utf-8") as f:
|
|
batch_data = json.load(f)
|
|
|
|
# Translate
|
|
translated_data = translator.translate_batch(batch_data, language_name, language_code)
|
|
|
|
# Validate
|
|
if not args.skip_validation:
|
|
translator.validate_translation(batch_data, translated_data)
|
|
|
|
# Save output
|
|
input_path = Path(input_file)
|
|
output_file = input_path.stem + args.output_suffix + input_path.suffix
|
|
|
|
with open(output_file, "w", encoding="utf-8") as f:
|
|
json.dump(translated_data, f, ensure_ascii=False, separators=(",", ":"))
|
|
|
|
print(f"✓ Saved to: {output_file}")
|
|
successful += 1
|
|
|
|
# Delay between API calls to avoid rate limits
|
|
if i < len(input_files):
|
|
time.sleep(args.delay)
|
|
|
|
except (OSError, ValueError, KeyError, RuntimeError) as e:
|
|
print(f"✗ Failed: {e}")
|
|
failed += 1
|
|
continue
|
|
|
|
# Summary
|
|
print("\n" + "=" * 60)
|
|
print("Translation complete!")
|
|
print(f"Successful: {successful}/{len(input_files)}")
|
|
if failed > 0:
|
|
print(f"Failed: {failed}/{len(input_files)}")
|
|
|
|
# Cost summary
|
|
print("-" * 60)
|
|
print(f"Total tokens: {translator.total_prompt_tokens:,} in / {translator.total_completion_tokens:,} out")
|
|
if translator.total_cost:
|
|
print(f"Estimated cost ({args.model}): ${translator.total_cost:.4f}")
|
|
|
|
sys.exit(0 if failed == 0 else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|