import argparse import glob import json import math import os import re import sys from dataclasses import dataclass from pathlib import Path from typing import Dict, Iterable, List, Optional, Sequence, Tuple import requests # ---------------------------- # Defaults / configuration # ---------------------------- DEFAULT_OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434/api/generate") DEFAULT_OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "translategemma:12b") DEFAULT_DOCS_DIR = os.getenv("DOCS_DIR", "docs/tech-specs") # With chunked translation, a smaller ctx is usually faster and more reliable. DEFAULT_NUM_CTX = int(os.getenv("OLLAMA_NUM_CTX", "4096")) # Critical: many incomplete translations come from low/default num_predict. DEFAULT_NUM_PREDICT = int(os.getenv("OLLAMA_NUM_PREDICT", "4096")) DEFAULT_TEMPERATURE = float(os.getenv("OLLAMA_TEMPERATURE", "0.1")) DEFAULT_TOP_P = float(os.getenv("OLLAMA_TOP_P", "0.9")) DEFAULT_REPEAT_PENALTY = float(os.getenv("OLLAMA_REPEAT_PENALTY", "1.1")) DEFAULT_KEEP_ALIVE = os.getenv("OLLAMA_KEEP_ALIVE", "15m") DEFAULT_CONNECT_TIMEOUT = int(os.getenv("OLLAMA_CONNECT_TIMEOUT", "30")) # If Ollama stops streaming mid-response, we want to recover quickly. # This is a *per-socket-read* timeout (it resets whenever a chunk arrives). DEFAULT_READ_TIMEOUT = int(os.getenv("OLLAMA_READ_TIMEOUT", "180")) DEFAULT_MAX_CHUNK_LINES = int(os.getenv("TG_TRANSLATE_MAX_CHUNK_LINES", "200")) DEFAULT_MAX_CHUNK_CHARS = int(os.getenv("TG_TRANSLATE_MAX_CHUNK_CHARS", "12000")) DEFAULT_RETRIES = int(os.getenv("TG_TRANSLATE_RETRIES", "2")) LANGUAGES: Dict[str, str] = { "Spanish": "es", "Swahili": "sw", "Portuguese": "pt", "Turkish": "tr", "Hindi": "hi", "Hebrew": "he", "Arabic": "ar", "Chinese (simplified)": "zh-cn", "Russian": "ru", } # ---------------------------- # Markdown splitting helpers # ---------------------------- @dataclass(frozen=True) class Block: kind: str # "text" | "code" lines: List[str] # each element includes its original newline (if present) FENCE_RE = re.compile(r"^[ \t]*(```+|~~~+)") def _split_line_ending(line: str) -> Tuple[str, str]: if line.endswith("\r\n"): return line[:-2], "\r\n" if line.endswith("\n"): return line[:-1], "\n" return line, "" def split_markdown_blocks(text: str) -> List[Block]: lines = text.splitlines(keepends=True) if not lines: return [Block(kind="text", lines=[])] blocks: List[Block] = [] # YAML front-matter: treat as non-translatable. idx = 0 if lines and lines[0].strip() == "---": for j in range(1, len(lines)): if lines[j].strip() in {"---", "..."}: blocks.append(Block(kind="code", lines=lines[: j + 1])) idx = j + 1 break current: List[str] = [] in_fence = False fence_marker: Optional[str] = None fence_len = 0 def flush(kind: str) -> None: nonlocal current if current: blocks.append(Block(kind=kind, lines=current)) current = [] for line in lines[idx:]: m = FENCE_RE.match(line) if m: marker = m.group(1) if not in_fence: flush("text") in_fence = True fence_marker = marker[0:3] fence_len = len(marker) current.append(line) continue # Close if same fence char and len >= opener len if fence_marker and marker.startswith(fence_marker) and len(marker) >= fence_len: current.append(line) flush("code") in_fence = False fence_marker = None fence_len = 0 continue current.append(line) flush("code" if in_fence else "text") return blocks def chunk_lines(lines: Sequence[str], *, max_lines: int, max_chars: int) -> List[List[str]]: """Split lines into reasonably-sized chunks. This prefers breaking at safe Markdown boundaries (blank lines, headings, rules) so the model stays focused and is less likely to drift. """ def is_good_break(line: str) -> bool: s = line.strip() if s == "": return True if s.startswith("#"): return True if s in {"---", "***", "___"}: return True return False chunks: List[List[str]] = [] n = len(lines) start = 0 while start < n: end = start cur_chars = 0 last_good_end: Optional[int] = None while end < n: line_len = len(lines[end]) # Ensure progress even if a single line exceeds max_chars. if end == start and line_len > max_chars: end += 1 last_good_end = end break if (end - start) >= max_lines: break if (cur_chars + line_len) > max_chars: break cur_chars += line_len end += 1 if is_good_break(lines[end - 1]): last_good_end = end if end >= n: chunks.append(list(lines[start:end])) break cut = last_good_end if (last_good_end is not None and last_good_end > start) else end if cut <= start: cut = min(start + 1, n) chunks.append(list(lines[start:cut])) start = cut return chunks # ---------------------------- # Ollama client # ---------------------------- class OllamaClient: def __init__( self, *, url: str, model: str, num_ctx: int, num_predict: int, temperature: float, top_p: float, repeat_penalty: float, keep_alive: str, timeout: Tuple[int, int] = (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT), ) -> None: self.url = url self.model = model self.num_ctx = num_ctx self.num_predict = num_predict self.temperature = temperature self.top_p = top_p self.repeat_penalty = repeat_penalty self.keep_alive = keep_alive self.timeout = timeout self.session = requests.Session() def reset_session(self) -> None: try: self.session.close() except Exception: pass self.session = requests.Session() def generate( self, *, prompt: str, system: Optional[str] = None, stop: Optional[List[str]] = None, progress_dots: bool = False, ) -> str: payload: Dict[str, object] = { "model": self.model, "prompt": prompt, "stream": True, "keep_alive": self.keep_alive, "options": { "num_ctx": self.num_ctx, "num_predict": self.num_predict, "temperature": self.temperature, "top_p": self.top_p, "repeat_penalty": self.repeat_penalty, }, } if system: payload["system"] = system if stop: payload["options"]["stop"] = stop out: List[str] = [] stream_error: Optional[str] = None dot_counter = 0 # Ensure we always release the connection back to the pool. with self.session.post(self.url, json=payload, stream=True, timeout=self.timeout) as response: response.raise_for_status() for raw in response.iter_lines(): if not raw: continue try: chunk = json.loads(raw) except json.JSONDecodeError: continue if "error" in chunk and chunk["error"]: stream_error = str(chunk["error"]) break piece = chunk.get("response") if piece: out.append(piece) if progress_dots: dot_counter += 1 if dot_counter % 50 == 0: sys.stdout.write(".") sys.stdout.flush() if chunk.get("done") is True: break if stream_error: raise requests.exceptions.RequestException(f"Ollama error: {stream_error}") result = "".join(out) if result == "": raise requests.exceptions.RequestException("Empty response from Ollama") return result # ---------------------------- # Translation (validated, line-indexed) # ---------------------------- INLINE_CODE_RE = re.compile(r"`[^`]*`") URL_RE = re.compile(r"https?://[^\s)>]+") # Some models occasionally insert leading whitespace or a colon after the prefix. PREFIX_RE = re.compile(r"^\s*\[\[(\d+)\]\]\s*[:\-]?\s*(.*)$") def _protect_spans(text: str) -> Tuple[str, Dict[str, str]]: """Protect inline code spans and URLs with placeholders that must be kept verbatim.""" placeholders: Dict[str, str] = {} def repl_code(m: re.Match) -> str: key = f"⟦CODE_{len(placeholders)}⟧" placeholders[key] = m.group(0) return key def repl_url(m: re.Match) -> str: key = f"⟦URL_{len(placeholders)}⟧" placeholders[key] = m.group(0) return key text = INLINE_CODE_RE.sub(repl_code, text) text = URL_RE.sub(repl_url, text) return text, placeholders def _restore_spans(text: str, placeholders: Dict[str, str]) -> str: for key, original in placeholders.items(): text = text.replace(key, original) return text def _build_numbered_input(lines: Sequence[str]) -> Tuple[str, List[Tuple[str, str, str, Dict[str, str]]]]: """Return (prompt_text, meta) where meta items are (leading_ws, newline, original_empty_marker, placeholders).""" meta: List[Tuple[str, str, str, Dict[str, str]]] = [] width = max(4, len(str(len(lines)))) numbered: List[str] = [] for i, line in enumerate(lines, start=1): content, newline = _split_line_ending(line) leading_ws = re.match(r"^[\t ]*", content).group(0) rest = content[len(leading_ws) :] protected, placeholders = _protect_spans(rest) prefix = f"[[{i:0{width}d}]]" # NOTE: we do NOT send leading whitespace to the model; we reattach it to keep indentation exact. numbered.append(f"{prefix} {protected}") meta.append((leading_ws, newline, "", placeholders)) return "\n".join(numbered), meta def _parse_numbered_output(output: str) -> Dict[int, str]: parsed: Dict[int, str] = {} for raw_line in output.splitlines(): m = PREFIX_RE.match(raw_line.strip("\r")) if not m: continue idx = int(m.group(1)) parsed[idx] = m.group(2) return parsed def _find_missing_indices(parsed: Dict[int, str], expected: int) -> List[int]: return [i for i in range(1, expected + 1) if i not in parsed] def _repair_missing_numbered_lines( client: OllamaClient, *, missing_numbered_lines: Sequence[str], target_language_name: str, target_language_code: str, sentinel: str, ) -> Dict[int, str]: """Ask the model to translate only the missing numbered lines.""" system = ( "You are a meticulous translation engine. " "Never summarize, never omit content, and never add commentary. " "Follow the output format exactly." ) prompt = ( f"Translate ONLY the following missing Markdown lines into {target_language_name} ({target_language_code}).\n" "\n" "OUTPUT CONTRACT (must follow exactly):\n" "- Output ONLY the translated lines for the provided [[NNNN]] prefixes.\n" "- Preserve Markdown syntax exactly.\n" "- Do NOT translate placeholders like ⟦CODE_0⟧ or ⟦URL_0⟧; keep them exactly unchanged.\n" "- Do NOT add any additional lines or prefixes beyond those provided.\n" "- Do NOT wrap the output in code fences.\n" f"- After the last translated line, output a final line containing exactly: {sentinel}\n" "\n" "MISSING INPUT LINES:\n" + "\n".join(missing_numbered_lines) + "\n" ) # Keep CLI output clean: the main generation already shows progress dots. raw = client.generate(prompt=prompt, system=system, stop=["\n" + sentinel], progress_dots=False) return _parse_numbered_output(raw) def translate_lines_strict( client: OllamaClient, *, lines: Sequence[str], target_language_name: str, target_language_code: str, retries: int, ) -> Optional[str]: """Translate the given lines (including newlines) and return the translated text for this chunk.""" numbered_input, meta = _build_numbered_input(lines) expected = len(lines) sentinel = "[[__END_OF_TRANSLATION__]]" system = ( "You are a meticulous translation engine. " "Never summarize, never omit content, and never add commentary. " "Follow the output format exactly." ) prompt = ( f"Translate the following Markdown lines into {target_language_name} ({target_language_code}).\n" "\n" "OUTPUT CONTRACT (must follow exactly):\n" "- Translate EVERY line; do not omit, reorder, merge, or split lines.\n" "- Preserve Markdown syntax exactly (headings, bullets, tables, links, HTML tags).\n" "- Do NOT translate placeholders like ⟦CODE_0⟧ or ⟦URL_0⟧; keep them exactly unchanged.\n" "- Return ONLY translated lines, each starting with its original [[NNNN]] prefix.\n" "- Do NOT wrap the output in code fences.\n" "- Do NOT add blank lines between numbered lines.\n" f"- After the last line, output a final line containing exactly: {sentinel}\n" "\n" "EXAMPLE FORMAT (illustrative):\n" "INPUT:\n" "[[0001]] Example line\n" "[[0002]] Another line\n" "OUTPUT:\n" "[[0001]] \n" "[[0002]] \n" f"{sentinel}\n" "\n" "INPUT LINES:\n" f"{numbered_input}\n" ) last_error: Optional[str] = None for attempt in range(retries + 1): try: sys.stdout.write(" [Generating]") sys.stdout.flush() raw = client.generate(prompt=prompt, system=system, stop=["\n" + sentinel], progress_dots=True) sys.stdout.write(" Done!\n") except requests.exceptions.Timeout: sys.stdout.write(" Timeout!\n") last_error = "timeout" client.reset_session() continue except requests.exceptions.RequestException as e: sys.stdout.write(" Failed!\n") last_error = str(e) client.reset_session() continue parsed = _parse_numbered_output(raw) # Ignore harmless extra indices if the model repeats itself. parsed = {i: v for i, v in parsed.items() if 1 <= i <= expected} missing = _find_missing_indices(parsed, expected) if missing: # Fast path: repair only a small number of missing lines. # This avoids re-translating large chunks when we only missed a couple of prefixes. if len(missing) <= min(12, max(3, expected // 10)): numbered_lines = numbered_input.splitlines() if len(numbered_lines) == expected: missing_input = [numbered_lines[i - 1] for i in missing] try: repaired = _repair_missing_numbered_lines( client, missing_numbered_lines=missing_input, target_language_name=target_language_name, target_language_code=target_language_code, sentinel=sentinel, ) repaired = {i: v for i, v in repaired.items() if i in missing} parsed.update(repaired) missing = _find_missing_indices(parsed, expected) except requests.exceptions.Timeout: client.reset_session() except requests.exceptions.RequestException: client.reset_session() if missing: last_error = f"missing lines ({expected - len(missing)}/{expected})" # Let the caller split smaller rather than retrying the same large prompt. break # Validate that protected placeholders were preserved. placeholder_missing = 0 for i in range(1, expected + 1): _leading_ws, _newline, _unused, placeholders = meta[i - 1] if not placeholders: continue out_line = parsed.get(i, "") for key in placeholders.keys(): if key not in out_line: placeholder_missing += 1 if placeholder_missing: last_error = f"placeholder lost ({placeholder_missing})" break # Reconstruct chunk, preserving indentation + original newlines. out_lines: List[str] = [] for i in range(1, expected + 1): leading_ws, newline, _unused, placeholders = meta[i - 1] translated_rest = parsed[i] translated_rest = _restore_spans(translated_rest, placeholders) out_lines.append(leading_ws + translated_rest + newline) return "".join(out_lines) print(f" [!] Chunk translation failed after retries: {last_error}") return None def _is_good_split_boundary(line: str) -> bool: s = line.strip() if s == "": return True if s.startswith("#"): return True if s in {"---", "***", "___"}: return True return False def _find_best_split_index(lines: Sequence[str]) -> int: """Pick a split point near the middle, preferably after a safe Markdown boundary.""" n = len(lines) if n <= 1: return 1 mid = n // 2 candidates: List[int] = [] for i in range(1, n): if _is_good_split_boundary(lines[i - 1]): candidates.append(i) if not candidates: return mid return min(candidates, key=lambda i: abs(i - mid)) def translate_single_line_fallback( client: OllamaClient, *, line: str, target_language_name: str, target_language_code: str, retries: int, ) -> Optional[str]: """Last-resort translation for one line when the numbered contract keeps failing.""" content, newline = _split_line_ending(line) leading_ws = re.match(r"^[\t ]*", content).group(0) rest = content[len(leading_ws) :] # Empty/whitespace lines: keep as-is. if rest.strip() == "": return line protected, placeholders = _protect_spans(rest) system = ( "You are a meticulous translation engine. " "Never summarize, never omit content, and never add commentary." ) prompt = ( f"Translate this SINGLE Markdown line into {target_language_name} ({target_language_code}).\n" "Rules:\n" "- Output EXACTLY one line (no surrounding quotes, no code fences, no prefixes).\n" "- Preserve Markdown punctuation exactly (e.g., `#`, `-`, `*`, `|`, links).\n" "- Do NOT translate placeholders like ⟦CODE_0⟧ or ⟦URL_0⟧; keep them unchanged.\n" "- Do NOT add explanations.\n" "\n" f"LINE: {protected}\n" ) last_error: Optional[str] = None for _attempt in range(retries + 1): try: raw = client.generate(prompt=prompt, system=system) except requests.exceptions.Timeout: last_error = "timeout" client.reset_session() continue except requests.exceptions.RequestException as e: last_error = str(e) client.reset_session() continue candidate = raw.strip("\r\n") if "\n" in candidate: candidate = candidate.splitlines()[0] candidate = candidate.strip() if not candidate: last_error = "empty response" continue # Ensure placeholders survive for inline code/URLs. for key in placeholders.keys(): if key not in candidate: last_error = "placeholder lost" candidate = "" break if candidate == "": continue candidate = _restore_spans(candidate, placeholders) return leading_ws + candidate + newline print(f" [!] Single-line fallback failed: {last_error}") return None def translate_lines_resilient( client: OllamaClient, *, lines: Sequence[str], target_language_name: str, target_language_code: str, retries: int, split_budget: Optional[int] = None, ) -> Optional[str]: """Translate lines, automatically splitting smaller if the strict contract fails. This prevents a single bad chunk from failing an entire language run. """ if not lines: return "" # Adaptive budget: ensure we can always reach single-line fallback when needed. if split_budget is None: # Depth to reach 1 line by halving: ceil(log2(n)). Add a small buffer. split_budget = max(6, int(math.ceil(math.log2(len(lines)))) + 2) translated = translate_lines_strict( client, lines=lines, target_language_name=target_language_name, target_language_code=target_language_code, retries=retries, ) if translated is not None: return translated if len(lines) == 1: return translate_single_line_fallback( client, line=lines[0], target_language_name=target_language_name, target_language_code=target_language_code, retries=retries, ) if split_budget <= 0: return None split_at = _find_best_split_index(lines) left = translate_lines_resilient( client, lines=lines[:split_at], target_language_name=target_language_name, target_language_code=target_language_code, retries=retries, split_budget=split_budget - 1, ) if left is None: return None right = translate_lines_resilient( client, lines=lines[split_at:], target_language_name=target_language_name, target_language_code=target_language_code, retries=retries, split_budget=split_budget - 1, ) if right is None: return None return left + right def translate_markdown_document( client: OllamaClient, *, content: str, target_language_name: str, target_language_code: str, max_chunk_lines: int, max_chunk_chars: int, retries: int, ) -> Optional[str]: blocks = split_markdown_blocks(content) out_parts: List[str] = [] for block in blocks: if block.kind == "code": out_parts.append("".join(block.lines)) continue # chunk and translate for chunk in chunk_lines(block.lines, max_lines=max_chunk_lines, max_chars=max_chunk_chars): translated = translate_lines_resilient( client, lines=chunk, target_language_name=target_language_name, target_language_code=target_language_code, retries=retries, ) if translated is None: return None out_parts.append(translated) return "".join(out_parts) def looks_incomplete(existing_translation: str, source: str) -> bool: """Conservative heuristic to detect summaries/truncation in already-generated files.""" if not existing_translation.strip(): return True src_len = len(source.strip()) out_len = len(existing_translation.strip()) if src_len >= 2000 and out_len < int(src_len * 0.35): return True src_lines = len([l for l in source.splitlines() if l.strip()]) out_lines = len([l for l in existing_translation.splitlines() if l.strip()]) if src_lines >= 80 and out_lines < int(src_lines * 0.45): return True # Obvious “assistant” preambles. lowered = existing_translation.lstrip().lower() if lowered.startswith("sure") or lowered.startswith("here is") or "summary" in lowered[:200]: return True return False # ---------------------------- # CLI / main # ---------------------------- def _parse_langs(arg: Optional[str]) -> Dict[str, str]: if not arg: return dict(LANGUAGES) wanted = {a.strip().lower() for a in arg.split(",") if a.strip()} resolved: Dict[str, str] = {} for name, code in LANGUAGES.items(): if name.lower() in wanted or code.lower() in wanted: resolved[name] = code unknown = wanted - {n.lower() for n in LANGUAGES.keys()} - {c.lower() for c in LANGUAGES.values()} if unknown: raise SystemExit(f"Unknown languages: {', '.join(sorted(unknown))}") return resolved def main() -> None: parser = argparse.ArgumentParser(description="Translate docs/tech-specs markdown files using a local Ollama model.") parser.add_argument("--docs-dir", default=DEFAULT_DOCS_DIR, help="Directory containing source .md files") parser.add_argument("--model", default=DEFAULT_OLLAMA_MODEL, help="Ollama model name") parser.add_argument("--ollama-url", default=DEFAULT_OLLAMA_URL, help="Ollama /api/generate URL") parser.add_argument("--langs", default=None, help="Comma-separated language names or codes (e.g. 'es,ru')") parser.add_argument("--files", default="*.md", help="Glob pattern within docs-dir to translate (default: *.md)") parser.add_argument("--force", action="store_true", help="Overwrite existing translations") parser.add_argument("--max-chunk-lines", type=int, default=DEFAULT_MAX_CHUNK_LINES) parser.add_argument("--max-chunk-chars", type=int, default=DEFAULT_MAX_CHUNK_CHARS) parser.add_argument("--retries", type=int, default=DEFAULT_RETRIES) parser.add_argument("--num-ctx", type=int, default=DEFAULT_NUM_CTX) parser.add_argument("--num-predict", type=int, default=DEFAULT_NUM_PREDICT) parser.add_argument("--temperature", type=float, default=DEFAULT_TEMPERATURE) args = parser.parse_args() langs = _parse_langs(args.langs) client = OllamaClient( url=args.ollama_url, model=args.model, num_ctx=args.num_ctx, num_predict=args.num_predict, temperature=args.temperature, top_p=DEFAULT_TOP_P, repeat_penalty=DEFAULT_REPEAT_PENALTY, keep_alive=DEFAULT_KEEP_ALIVE, ) docs_dir = args.docs_dir patterns = [args.files] files: List[str] = [] for pattern in patterns: for path in glob.glob(os.path.join(docs_dir, pattern)): # Skip files that are already translated (contain e.g. .es.md) if any(f".{code}." in path for code in LANGUAGES.values()): continue files.append(path) files = sorted(set(files)) if not files: print(f"No source files found in {docs_dir}") return print(f"Found {len(files)} files to translate in '{docs_dir}'.") print(f"Using Ollama model: {client.model}") print(f"Chunking: <= {args.max_chunk_lines} lines, <= {args.max_chunk_chars} chars") print("-" * 40) for filepath in files: path_obj = Path(filepath) filename_without_ext = path_obj.stem extension = path_obj.suffix print(f"Processing: {filepath}") with open(filepath, "r", encoding="utf-8") as f: content = f.read() if not content.strip(): print(" Skipping empty file.") continue for lang_name, lang_code in langs.items(): out_filename = f"{filename_without_ext}.{lang_code}{extension}" out_filepath = os.path.join(docs_dir, out_filename) if os.path.exists(out_filepath) and os.path.getsize(out_filepath) > 0 and not args.force: with open(out_filepath, "r", encoding="utf-8") as existing_f: existing = existing_f.read() if not looks_incomplete(existing, content): print(f" [-] Skipping {lang_name} ({out_filename}) - already exists.") continue print(f" [!] Existing {lang_name} looks incomplete; re-translating...") print(f" [+] Translating to {lang_name}...") translated = translate_markdown_document( client, content=content, target_language_name=lang_name, target_language_code=lang_code, max_chunk_lines=args.max_chunk_lines, max_chunk_chars=args.max_chunk_chars, retries=args.retries, ) if translated: with open(out_filepath, "w", encoding="utf-8") as out_f: out_f.write(translated) print(f" Saved -> {out_filepath}") else: print(f" Failed to translate {filepath} to {lang_name}.") if __name__ == "__main__": main()