From 37beb1eba5d952e6fd048071403946122b231b41 Mon Sep 17 00:00:00 2001 From: "Jenkins, Kenneth Alexander" Date: Mon, 13 Apr 2026 19:35:37 -0400 Subject: [PATCH] add mark fixes Signed-off-by: Jenkins, Kenneth Alexander --- .coverage | Bin 53248 -> 0 bytes generate_i18n_packs.py | 233 ----- translate_docs.py | 906 ------------------ trustgraph-base/trustgraph/i18n/__init__.py | 3 + .../trustgraph/gateway/endpoint/i18n.py | 6 + trustgraph/trustgraph/trustgraph_version.py | 1 - updates.md | 21 - 7 files changed, 9 insertions(+), 1161 deletions(-) delete mode 100644 .coverage delete mode 100644 generate_i18n_packs.py delete mode 100755 translate_docs.py delete mode 100644 trustgraph/trustgraph/trustgraph_version.py delete mode 100644 updates.md diff --git a/.coverage b/.coverage deleted file mode 100644 index 7ec3fd07cd1ebbef8a53d57204a39adf454cfd50..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53248 zcmeI)%WoS+90%~-y7k76BL_v16>`X2kYm+xWmSmR0Rlv#q7oDW;)FZ4$M%AE*V#uN z2ZUTIl`D7tIR1i495`{|_nX=E+DTmXR%!XJVrL&Sv-A7R%kH-Q_Q_*6R$>%{o{Ysq zYumDI>k}a?%i5sV7QM2~p`E+g0sXd@_SfxhSe-xq+NytV)yh9u^&hw1t{>ILwZAw2 zti50PZqs$zl@49N1_1~_00M;(m_MmGjlDhl*)Or|PgNYsfeOw0)@R?G9G{+u)8kJc zp9phK999KwM@QmVgu#WFsZfmEsS>U~bO$na{jrEADodS2YRGFAougHY6Lxfd-s`zT z$`z|I6)_84Pli|GXLZ%q3DUd8>M}Mbs1W6jeNG_`)9ft~s*wtnKTwgW#i2WFRa?LP zT6G#bJN65uW0K*3el}O4p%Zy1k499`AP9$C`+g|>!9+!EAr}!20;=a_Y@!VV??D`h zsq5=7{m6~oz!&OL4U$+5Zw;``XxT=1%h6C-PdT8cCTDuDC7nF;EjiQVY331C&JQxy zJR{$lGV40~iPtCcpAv7zjYB!ijpJZl&UJa?APx4l+vb+G%;Q~Y&WxN})Z|Kh_bgG1 z#&3%}XIqEi2W`PYP!D)#v)uS#$F`_fs>|qXN*|qiGKm9y+@r?tb?L44{H+bAad2S2 zIMW@4d3t>}jutzWl}@uby{7B%_4WORN6Bg@;ZZc5ieRLN4CfZ6ai}*hkC_Jc&J1i!yq_Up&YADdeuw|Z@ME< z*XdTujgy0QU1!k+dfmltb-q`28vFb9vx@FUX=|mwoRuz=wd5_+Xm-g{GD^a1B6-y$ zndQi5$!v_IS16n#r`$N+Ul&fAuGdX>Hs;P;B7!x_k24lh@Z2kRNCEl&YnZ)A0T}h$5GkRbIBD z53{qPIgt_1LQzxngJ7zpZ`KCN@hGBux{=UzxVb8QoR<+@aT(+Drmj=AxbQT8N`o#N zU$oJ2NujQaw2|!<`#t zsl1~2jXsyLid|0$PNFNLJJyAa#2{33Fu}=^Z(+vF(QNj1Rwwb2tWV= N5P$##AOHaf{0C9+AV>fJ diff --git a/generate_i18n_packs.py b/generate_i18n_packs.py deleted file mode 100644 index a379efb2..00000000 --- a/generate_i18n_packs.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 - -"""Generate TrustGraph pre-built language packs using the local Ollama translate model. - -This script translates a curated set of UI/CLI strings (keys) from English into -all supported languages and writes JSON packs under: - trustgraph-base/trustgraph/i18n/packs/ - -Design goal: no runtime translation. Packs are generated ahead of time and -loaded via `trustgraph.i18n`. - -Requires: -- Ollama running locally (default: http://localhost:11434) -- Model available (default: translategemma:12b) - -Usage: - python generate_i18n_packs.py - -You can override: - OLLAMA_URL, OLLAMA_MODEL -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -from pathlib import Path -from typing import Dict, List, Tuple - -# Reuse the hardened translation engine we already use for docs. -import translate_docs - - -DEFAULT_OUTPUT_DIR = Path("trustgraph-base/trustgraph/i18n/packs") - -# 10 languages (including English) -LANGS: Dict[str, Tuple[str, str]] = { - "en": ("English", "en"), - "es": ("Spanish", "es"), - "sw": ("Swahili", "sw"), - "pt": ("Portuguese", "pt"), - "tr": ("Turkish", "tr"), - "hi": ("Hindi", "hi"), - "he": ("Hebrew", "he"), - "ar": ("Arabic", "ar"), - "zh-cn": ("Chinese (simplified)", "zh-cn"), - "ru": ("Russian", "ru"), -} - - -# Curated strings currently used by tg-verify-system-status. -# Keep keys stable. -EN_STRINGS: Dict[str, str] = { - "cli.verify_system_status.title": "TrustGraph System Status Verification", - "cli.verify_system_status.phase_1": "Phase 1: Infrastructure", - "cli.verify_system_status.phase_2": "Phase 2: Core Services", - "cli.verify_system_status.phase_3": "Phase 3: Data Services", - "cli.verify_system_status.phase_4": "Phase 4: User Interface", - "cli.verify_system_status.summary": "Summary", - "cli.verify_system_status.checking": "Checking {name}...", - "cli.verify_system_status.checking_attempt": "Checking {name}... (attempt {attempt})", - "cli.verify_system_status.failed_timeout": "{name}: Failed (timeout after {attempt} attempts)", - "cli.verify_system_status.pulsar_not_responding": "Pulsar is not responding - other checks may fail", - "cli.verify_system_status.checks_passed": "Checks passed: {passed}/{total}", - "cli.verify_system_status.checks_failed": "Checks failed: {failed}/{total}", - "cli.verify_system_status.total_time": "Total time: {elapsed}", - "cli.verify_system_status.system_healthy": "System is healthy!", - "cli.verify_system_status.system_failing": "System has {failed} failing check(s)", - - "cli.verify_system_status.check_name.pulsar": "Pulsar", - "cli.verify_system_status.check_name.api_gateway": "API Gateway", - "cli.verify_system_status.check_name.processors": "Processors", - "cli.verify_system_status.check_name.flow_blueprints": "Flow Blueprints", - "cli.verify_system_status.check_name.flows": "Flows", - "cli.verify_system_status.check_name.prompts": "Prompts", - "cli.verify_system_status.check_name.library": "Library", - "cli.verify_system_status.check_name.workbench_ui": "Workbench UI", - - "cli.verify_system_status.pulsar.healthy": "Pulsar healthy ({clusters} cluster(s))", - "cli.verify_system_status.pulsar.status": "Pulsar returned status {status_code}", - "cli.verify_system_status.pulsar.timeout": "Pulsar connection timeout", - "cli.verify_system_status.pulsar.cannot_connect": "Cannot connect to Pulsar", - "cli.verify_system_status.pulsar.error": "Pulsar error: {error}", - - "cli.verify_system_status.api_gateway.responding": "API Gateway is responding", - "cli.verify_system_status.api_gateway.status": "API Gateway returned status {status_code}", - "cli.verify_system_status.api_gateway.timeout": "API Gateway connection timeout", - "cli.verify_system_status.api_gateway.cannot_connect": "Cannot connect to API Gateway", - "cli.verify_system_status.api_gateway.error": "API Gateway error: {error}", - - "cli.verify_system_status.processors.found": "Found {count} processors (≥ {min})", - "cli.verify_system_status.processors.only": "Only {count} processors running (need {min})", - "cli.verify_system_status.processors.metrics_status": "Metrics returned status {status_code}", - "cli.verify_system_status.processors.error": "Processor check error: {error}", - - "cli.verify_system_status.flow_blueprints.found": "Found {count} flow blueprint(s)", - "cli.verify_system_status.flow_blueprints.none": "No flow blueprints found", - "cli.verify_system_status.flow_blueprints.error": "Flow blueprints check error: {error}", - - "cli.verify_system_status.flows.responding": "Flow manager responding ({count} flow(s))", - "cli.verify_system_status.flows.error": "Flow manager check error: {error}", - - "cli.verify_system_status.prompts.found": "Found {count} prompt(s)", - "cli.verify_system_status.prompts.none": "No prompts found", - "cli.verify_system_status.prompts.error": "Prompts check error: {error}", - - "cli.verify_system_status.library.responding": "Library responding ({count} document(s))", - "cli.verify_system_status.library.error": "Library check error: {error}", - - "cli.verify_system_status.ui.responding": "Workbench UI is responding", - "cli.verify_system_status.ui.status": "UI returned status {status_code}", - "cli.verify_system_status.ui.timeout": "UI connection timeout", - "cli.verify_system_status.ui.cannot_connect": "Cannot connect to UI", - "cli.verify_system_status.ui.error": "UI error: {error}", -} - - -_FMT_RE = re.compile(r"\{[a-zA-Z0-9_]+\}") - - -def _protect_format_placeholders(text: str) -> Tuple[str, Dict[str, str]]: - """Replace `{name}` placeholders with CODE-like placeholders that the doc translator preserves.""" - - mapping: Dict[str, str] = {} - - def repl(m: re.Match[str]) -> str: - idx = len(mapping) - token = f"⟦CODE_{idx}⟧" - mapping[token] = m.group(0) - return token - - protected = _FMT_RE.sub(repl, text) - return protected, mapping - - -def _restore_format_placeholders(text: str, mapping: Dict[str, str]) -> str: - out = text - for token, original in mapping.items(): - out = out.replace(token, original) - return out - - -def _validate_placeholders(restored: str, originals: List[str]) -> bool: - return all(ph in restored for ph in originals) - - -def translate_pack(client: translate_docs.OllamaClient, *, lang_name: str, lang_code: str) -> Dict[str, str]: - keys = list(EN_STRINGS.keys()) - - protected_lines: List[str] = [] - placeholder_maps: List[Dict[str, str]] = [] - placeholder_originals: List[List[str]] = [] - - for k in keys: - text = EN_STRINGS[k] - protected, mapping = _protect_format_placeholders(text) - protected_lines.append(protected + "\n") - placeholder_maps.append(mapping) - placeholder_originals.append(list(mapping.values())) - - translated = translate_docs.translate_lines_resilient( - client, - lines=protected_lines, - target_language_name=lang_name, - target_language_code=lang_code, - retries=2, - ) - - if translated is None: - raise RuntimeError(f"Translation failed for {lang_code}") - - out_lines = translated.splitlines() - if len(out_lines) != len(keys): - raise RuntimeError(f"Unexpected line count for {lang_code}: {len(out_lines)} != {len(keys)}") - - pack: Dict[str, str] = {} - for i, k in enumerate(keys): - restored = _restore_format_placeholders(out_lines[i], placeholder_maps[i]) - - # Ensure python-format placeholders survived - if not _validate_placeholders(restored, placeholder_originals[i]): - raise RuntimeError(f"Placeholder lost for key {k} in {lang_code}") - - pack[k] = restored - - return pack - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--output-dir", - default=str(DEFAULT_OUTPUT_DIR), - help=f"Output directory for packs (default: {DEFAULT_OUTPUT_DIR})", - ) - - args = parser.parse_args() - - output_dir = Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - client = translate_docs.OllamaClient( - url=os.getenv("OLLAMA_URL", translate_docs.DEFAULT_OLLAMA_URL), - model=os.getenv("OLLAMA_MODEL", translate_docs.DEFAULT_OLLAMA_MODEL), - num_ctx=translate_docs.DEFAULT_NUM_CTX, - num_predict=translate_docs.DEFAULT_NUM_PREDICT, - temperature=translate_docs.DEFAULT_TEMPERATURE, - top_p=translate_docs.DEFAULT_TOP_P, - repeat_penalty=translate_docs.DEFAULT_REPEAT_PENALTY, - keep_alive=translate_docs.DEFAULT_KEEP_ALIVE, - ) - - # Always write English as the source pack - (output_dir / "en.json").write_text(json.dumps(EN_STRINGS, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - - for lang, (name, code) in LANGS.items(): - if lang == "en": - continue - - print(f"\n=== Generating pack: {lang} ({name}) ===") - pack = translate_pack(client, lang_name=name, lang_code=code) - (output_dir / f"{lang}.json").write_text( - json.dumps(pack, ensure_ascii=False, indent=2) + "\n", - encoding="utf-8", - ) - - print("\nDone.") - - -if __name__ == "__main__": - main() diff --git a/translate_docs.py b/translate_docs.py deleted file mode 100755 index 6a41de96..00000000 --- a/translate_docs.py +++ /dev/null @@ -1,906 +0,0 @@ -import argparse -import glob -import json -import math -import os -import re -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Tuple - -import requests - - -# ---------------------------- -# Defaults / configuration -# ---------------------------- - -DEFAULT_OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434/api/generate") -DEFAULT_OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "translategemma:12b") -DEFAULT_DOCS_DIR = os.getenv("DOCS_DIR", "docs") - -# With chunked translation, a smaller ctx is usually faster and more reliable. -DEFAULT_NUM_CTX = int(os.getenv("OLLAMA_NUM_CTX", "4096")) - -# Critical: many incomplete translations come from low/default num_predict. -DEFAULT_NUM_PREDICT = int(os.getenv("OLLAMA_NUM_PREDICT", "4096")) - -DEFAULT_TEMPERATURE = float(os.getenv("OLLAMA_TEMPERATURE", "0.1")) -DEFAULT_TOP_P = float(os.getenv("OLLAMA_TOP_P", "0.9")) -DEFAULT_REPEAT_PENALTY = float(os.getenv("OLLAMA_REPEAT_PENALTY", "1.1")) - -DEFAULT_KEEP_ALIVE = os.getenv("OLLAMA_KEEP_ALIVE", "15m") - -DEFAULT_CONNECT_TIMEOUT = int(os.getenv("OLLAMA_CONNECT_TIMEOUT", "30")) -# If Ollama stops streaming mid-response, we want to recover quickly. -# This is a *per-socket-read* timeout (it resets whenever a chunk arrives). -DEFAULT_READ_TIMEOUT = int(os.getenv("OLLAMA_READ_TIMEOUT", "180")) - -DEFAULT_MAX_CHUNK_LINES = int(os.getenv("TG_TRANSLATE_MAX_CHUNK_LINES", "200")) -DEFAULT_MAX_CHUNK_CHARS = int(os.getenv("TG_TRANSLATE_MAX_CHUNK_CHARS", "12000")) -DEFAULT_RETRIES = int(os.getenv("TG_TRANSLATE_RETRIES", "2")) - -LANGUAGES: Dict[str, str] = { - "Spanish": "es", - "Swahili": "sw", - "Portuguese": "pt", - "Turkish": "tr", - "Hindi": "hi", - "Hebrew": "he", - "Arabic": "ar", - "Chinese (simplified)": "zh-cn", - "Russian": "ru", -} - - -# ---------------------------- -# Markdown splitting helpers -# ---------------------------- - - -@dataclass(frozen=True) -class Block: - kind: str # "text" | "code" - lines: List[str] # each element includes its original newline (if present) - - -FENCE_RE = re.compile(r"^[ \t]*(```+|~~~+)") - - -def _split_line_ending(line: str) -> Tuple[str, str]: - if line.endswith("\r\n"): - return line[:-2], "\r\n" - if line.endswith("\n"): - return line[:-1], "\n" - return line, "" - - -def split_markdown_blocks(text: str) -> List[Block]: - lines = text.splitlines(keepends=True) - if not lines: - return [Block(kind="text", lines=[])] - - blocks: List[Block] = [] - - # YAML front-matter: treat as non-translatable. - idx = 0 - if lines and lines[0].strip() == "---": - for j in range(1, len(lines)): - if lines[j].strip() in {"---", "..."}: - blocks.append(Block(kind="code", lines=lines[: j + 1])) - idx = j + 1 - break - - current: List[str] = [] - in_fence = False - fence_marker: Optional[str] = None - fence_len = 0 - - def flush(kind: str) -> None: - nonlocal current - if current: - blocks.append(Block(kind=kind, lines=current)) - current = [] - - for line in lines[idx:]: - m = FENCE_RE.match(line) - if m: - marker = m.group(1) - if not in_fence: - flush("text") - in_fence = True - fence_marker = marker[0:3] - fence_len = len(marker) - current.append(line) - continue - - # Close if same fence char and len >= opener len - if fence_marker and marker.startswith(fence_marker) and len(marker) >= fence_len: - current.append(line) - flush("code") - in_fence = False - fence_marker = None - fence_len = 0 - continue - - current.append(line) - - flush("code" if in_fence else "text") - return blocks - - -def chunk_lines(lines: Sequence[str], *, max_lines: int, max_chars: int) -> List[List[str]]: - """Split lines into reasonably-sized chunks. - - This prefers breaking at safe Markdown boundaries (blank lines, headings, rules) - so the model stays focused and is less likely to drift. - """ - - def is_good_break(line: str) -> bool: - s = line.strip() - if s == "": - return True - if s.startswith("#"): - return True - if s in {"---", "***", "___"}: - return True - return False - - chunks: List[List[str]] = [] - n = len(lines) - start = 0 - - while start < n: - end = start - cur_chars = 0 - last_good_end: Optional[int] = None - - while end < n: - line_len = len(lines[end]) - - # Ensure progress even if a single line exceeds max_chars. - if end == start and line_len > max_chars: - end += 1 - last_good_end = end - break - - if (end - start) >= max_lines: - break - if (cur_chars + line_len) > max_chars: - break - - cur_chars += line_len - end += 1 - if is_good_break(lines[end - 1]): - last_good_end = end - - if end >= n: - chunks.append(list(lines[start:end])) - break - - cut = last_good_end if (last_good_end is not None and last_good_end > start) else end - if cut <= start: - cut = min(start + 1, n) - - chunks.append(list(lines[start:cut])) - start = cut - - return chunks - - -# ---------------------------- -# Ollama client -# ---------------------------- - - -class OllamaClient: - def __init__( - self, - *, - url: str, - model: str, - num_ctx: int, - num_predict: int, - temperature: float, - top_p: float, - repeat_penalty: float, - keep_alive: str, - timeout: Tuple[int, int] = (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT), - ) -> None: - self.url = url - self.model = model - self.num_ctx = num_ctx - self.num_predict = num_predict - self.temperature = temperature - self.top_p = top_p - self.repeat_penalty = repeat_penalty - self.keep_alive = keep_alive - self.timeout = timeout - self.session = requests.Session() - - def reset_session(self) -> None: - try: - self.session.close() - except Exception: - pass - self.session = requests.Session() - - def generate( - self, - *, - prompt: str, - system: Optional[str] = None, - stop: Optional[List[str]] = None, - progress_dots: bool = False, - ) -> str: - payload: Dict[str, object] = { - "model": self.model, - "prompt": prompt, - "stream": True, - "keep_alive": self.keep_alive, - "options": { - "num_ctx": self.num_ctx, - "num_predict": self.num_predict, - "temperature": self.temperature, - "top_p": self.top_p, - "repeat_penalty": self.repeat_penalty, - }, - } - if system: - payload["system"] = system - if stop: - payload["options"]["stop"] = stop - - out: List[str] = [] - stream_error: Optional[str] = None - dot_counter = 0 - - # Ensure we always release the connection back to the pool. - with self.session.post(self.url, json=payload, stream=True, timeout=self.timeout) as response: - response.raise_for_status() - - for raw in response.iter_lines(): - if not raw: - continue - try: - chunk = json.loads(raw) - except json.JSONDecodeError: - continue - - if "error" in chunk and chunk["error"]: - stream_error = str(chunk["error"]) - break - - piece = chunk.get("response") - if piece: - out.append(piece) - if progress_dots: - dot_counter += 1 - if dot_counter % 50 == 0: - sys.stdout.write(".") - sys.stdout.flush() - if chunk.get("done") is True: - break - - if stream_error: - raise requests.exceptions.RequestException(f"Ollama error: {stream_error}") - - result = "".join(out) - if result == "": - raise requests.exceptions.RequestException("Empty response from Ollama") - return result - - -# ---------------------------- -# Translation (validated, line-indexed) -# ---------------------------- - - -INLINE_CODE_RE = re.compile(r"`[^`]*`") -URL_RE = re.compile(r"https?://[^\s)>]+") -# Some models occasionally insert leading whitespace or a colon after the prefix. -PREFIX_RE = re.compile(r"^\s*\[\[(\d+)\]\]\s*[:\-]?\s*(.*)$") - - -def _protect_spans(text: str) -> Tuple[str, Dict[str, str]]: - """Protect inline code spans and URLs with placeholders that must be kept verbatim.""" - placeholders: Dict[str, str] = {} - - def repl_code(m: re.Match) -> str: - key = f"⟦CODE_{len(placeholders)}⟧" - placeholders[key] = m.group(0) - return key - - def repl_url(m: re.Match) -> str: - key = f"⟦URL_{len(placeholders)}⟧" - placeholders[key] = m.group(0) - return key - - text = INLINE_CODE_RE.sub(repl_code, text) - text = URL_RE.sub(repl_url, text) - return text, placeholders - - -def _restore_spans(text: str, placeholders: Dict[str, str]) -> str: - for key, original in placeholders.items(): - text = text.replace(key, original) - return text - - -def _build_numbered_input(lines: Sequence[str]) -> Tuple[str, List[Tuple[str, str, str, Dict[str, str]]]]: - """Return (prompt_text, meta) where meta items are (leading_ws, newline, original_empty_marker, placeholders).""" - meta: List[Tuple[str, str, str, Dict[str, str]]] = [] - width = max(4, len(str(len(lines)))) - numbered: List[str] = [] - - for i, line in enumerate(lines, start=1): - content, newline = _split_line_ending(line) - leading_ws = re.match(r"^[\t ]*", content).group(0) - rest = content[len(leading_ws) :] - - protected, placeholders = _protect_spans(rest) - - prefix = f"[[{i:0{width}d}]]" - # NOTE: we do NOT send leading whitespace to the model; we reattach it to keep indentation exact. - numbered.append(f"{prefix} {protected}") - meta.append((leading_ws, newline, "", placeholders)) - - return "\n".join(numbered), meta - - -def _parse_numbered_output(output: str) -> Dict[int, str]: - parsed: Dict[int, str] = {} - for raw_line in output.splitlines(): - m = PREFIX_RE.match(raw_line.strip("\r")) - if not m: - continue - idx = int(m.group(1)) - parsed[idx] = m.group(2) - return parsed - - -def _find_missing_indices(parsed: Dict[int, str], expected: int) -> List[int]: - return [i for i in range(1, expected + 1) if i not in parsed] - - -def _repair_missing_numbered_lines( - client: OllamaClient, - *, - missing_numbered_lines: Sequence[str], - target_language_name: str, - target_language_code: str, - sentinel: str, -) -> Dict[int, str]: - """Ask the model to translate only the missing numbered lines.""" - system = ( - "You are a meticulous translation engine. " - "Never summarize, never omit content, and never add commentary. " - "Follow the output format exactly." - ) - - prompt = ( - f"Translate ONLY the following missing Markdown lines into {target_language_name} ({target_language_code}).\n" - "\n" - "OUTPUT CONTRACT (must follow exactly):\n" - "- Output ONLY the translated lines for the provided [[NNNN]] prefixes.\n" - "- Preserve Markdown syntax exactly.\n" - "- Do NOT translate placeholders like ⟦CODE_0⟧ or ⟦URL_0⟧; keep them exactly unchanged.\n" - "- Do NOT add any additional lines or prefixes beyond those provided.\n" - "- Do NOT wrap the output in code fences.\n" - f"- After the last translated line, output a final line containing exactly: {sentinel}\n" - "\n" - "MISSING INPUT LINES:\n" - + "\n".join(missing_numbered_lines) - + "\n" - ) - - # Keep CLI output clean: the main generation already shows progress dots. - raw = client.generate(prompt=prompt, system=system, stop=["\n" + sentinel], progress_dots=False) - return _parse_numbered_output(raw) - - -def translate_lines_strict( - client: OllamaClient, - *, - lines: Sequence[str], - target_language_name: str, - target_language_code: str, - retries: int, -) -> Optional[str]: - """Translate the given lines (including newlines) and return the translated text for this chunk.""" - numbered_input, meta = _build_numbered_input(lines) - expected = len(lines) - sentinel = "[[__END_OF_TRANSLATION__]]" - - system = ( - "You are a meticulous translation engine. " - "Never summarize, never omit content, and never add commentary. " - "Follow the output format exactly." - ) - - prompt = ( - f"Translate the following Markdown lines into {target_language_name} ({target_language_code}).\n" - "\n" - "OUTPUT CONTRACT (must follow exactly):\n" - "- Translate EVERY line; do not omit, reorder, merge, or split lines.\n" - "- Preserve Markdown syntax exactly (headings, bullets, tables, links, HTML tags).\n" - "- Do NOT translate placeholders like ⟦CODE_0⟧ or ⟦URL_0⟧; keep them exactly unchanged.\n" - "- Return ONLY translated lines, each starting with its original [[NNNN]] prefix.\n" - "- Do NOT wrap the output in code fences.\n" - "- Do NOT add blank lines between numbered lines.\n" - f"- After the last line, output a final line containing exactly: {sentinel}\n" - "\n" - "EXAMPLE FORMAT (illustrative):\n" - "INPUT:\n" - "[[0001]] Example line\n" - "[[0002]] Another line\n" - "OUTPUT:\n" - "[[0001]] \n" - "[[0002]] \n" - f"{sentinel}\n" - "\n" - "INPUT LINES:\n" - f"{numbered_input}\n" - ) - - last_error: Optional[str] = None - for attempt in range(retries + 1): - try: - sys.stdout.write(" [Generating]") - sys.stdout.flush() - raw = client.generate(prompt=prompt, system=system, stop=["\n" + sentinel], progress_dots=True) - sys.stdout.write(" Done!\n") - except requests.exceptions.Timeout: - sys.stdout.write(" Timeout!\n") - last_error = "timeout" - client.reset_session() - continue - except requests.exceptions.RequestException as e: - sys.stdout.write(" Failed!\n") - last_error = str(e) - client.reset_session() - continue - - parsed = _parse_numbered_output(raw) - # Ignore harmless extra indices if the model repeats itself. - parsed = {i: v for i, v in parsed.items() if 1 <= i <= expected} - - missing = _find_missing_indices(parsed, expected) - if missing: - # Fast path: repair only a small number of missing lines. - # This avoids re-translating large chunks when we only missed a couple of prefixes. - if len(missing) <= min(12, max(3, expected // 10)): - numbered_lines = numbered_input.splitlines() - if len(numbered_lines) == expected: - missing_input = [numbered_lines[i - 1] for i in missing] - try: - repaired = _repair_missing_numbered_lines( - client, - missing_numbered_lines=missing_input, - target_language_name=target_language_name, - target_language_code=target_language_code, - sentinel=sentinel, - ) - repaired = {i: v for i, v in repaired.items() if i in missing} - parsed.update(repaired) - missing = _find_missing_indices(parsed, expected) - except requests.exceptions.Timeout: - client.reset_session() - except requests.exceptions.RequestException: - client.reset_session() - - if missing: - last_error = f"missing lines ({expected - len(missing)}/{expected})" - # Let the caller split smaller rather than retrying the same large prompt. - break - - # Validate that protected placeholders were preserved. - placeholder_missing = 0 - for i in range(1, expected + 1): - _leading_ws, _newline, _unused, placeholders = meta[i - 1] - if not placeholders: - continue - out_line = parsed.get(i, "") - for key in placeholders.keys(): - if key not in out_line: - placeholder_missing += 1 - if placeholder_missing: - last_error = f"placeholder lost ({placeholder_missing})" - break - - # Reconstruct chunk, preserving indentation + original newlines. - out_lines: List[str] = [] - for i in range(1, expected + 1): - leading_ws, newline, _unused, placeholders = meta[i - 1] - translated_rest = parsed[i] - translated_rest = _restore_spans(translated_rest, placeholders) - out_lines.append(leading_ws + translated_rest + newline) - - return "".join(out_lines) - - print(f" [!] Chunk translation failed after retries: {last_error}") - return None - - -def _is_good_split_boundary(line: str) -> bool: - s = line.strip() - if s == "": - return True - if s.startswith("#"): - return True - if s in {"---", "***", "___"}: - return True - return False - - -def _find_best_split_index(lines: Sequence[str]) -> int: - """Pick a split point near the middle, preferably after a safe Markdown boundary.""" - n = len(lines) - if n <= 1: - return 1 - - mid = n // 2 - candidates: List[int] = [] - for i in range(1, n): - if _is_good_split_boundary(lines[i - 1]): - candidates.append(i) - if not candidates: - return mid - - return min(candidates, key=lambda i: abs(i - mid)) - - -def translate_single_line_fallback( - client: OllamaClient, - *, - line: str, - target_language_name: str, - target_language_code: str, - retries: int, -) -> Optional[str]: - """Last-resort translation for one line when the numbered contract keeps failing.""" - content, newline = _split_line_ending(line) - leading_ws = re.match(r"^[\t ]*", content).group(0) - rest = content[len(leading_ws) :] - - # Empty/whitespace lines: keep as-is. - if rest.strip() == "": - return line - - protected, placeholders = _protect_spans(rest) - - system = ( - "You are a meticulous translation engine. " - "Never summarize, never omit content, and never add commentary." - ) - - prompt = ( - f"Translate this SINGLE Markdown line into {target_language_name} ({target_language_code}).\n" - "Rules:\n" - "- Output EXACTLY one line (no surrounding quotes, no code fences, no prefixes).\n" - "- Preserve Markdown punctuation exactly (e.g., `#`, `-`, `*`, `|`, links).\n" - "- Do NOT translate placeholders like ⟦CODE_0⟧ or ⟦URL_0⟧; keep them unchanged.\n" - "- Do NOT add explanations.\n" - "\n" - f"LINE: {protected}\n" - ) - - last_error: Optional[str] = None - for _attempt in range(retries + 1): - try: - raw = client.generate(prompt=prompt, system=system) - except requests.exceptions.Timeout: - last_error = "timeout" - client.reset_session() - continue - except requests.exceptions.RequestException as e: - last_error = str(e) - client.reset_session() - continue - - candidate = raw.strip("\r\n") - if "\n" in candidate: - candidate = candidate.splitlines()[0] - candidate = candidate.strip() - if not candidate: - last_error = "empty response" - continue - - # Ensure placeholders survive for inline code/URLs. - for key in placeholders.keys(): - if key not in candidate: - last_error = "placeholder lost" - candidate = "" - break - if candidate == "": - continue - - candidate = _restore_spans(candidate, placeholders) - return leading_ws + candidate + newline - - print(f" [!] Single-line fallback failed: {last_error}") - return None - - -def translate_lines_resilient( - client: OllamaClient, - *, - lines: Sequence[str], - target_language_name: str, - target_language_code: str, - retries: int, - split_budget: Optional[int] = None, -) -> Optional[str]: - """Translate lines, automatically splitting smaller if the strict contract fails. - - This prevents a single bad chunk from failing an entire language run. - """ - if not lines: - return "" - - # Adaptive budget: ensure we can always reach single-line fallback when needed. - if split_budget is None: - # Depth to reach 1 line by halving: ceil(log2(n)). Add a small buffer. - split_budget = max(6, int(math.ceil(math.log2(len(lines)))) + 2) - - translated = translate_lines_strict( - client, - lines=lines, - target_language_name=target_language_name, - target_language_code=target_language_code, - retries=retries, - ) - if translated is not None: - return translated - - if len(lines) == 1: - return translate_single_line_fallback( - client, - line=lines[0], - target_language_name=target_language_name, - target_language_code=target_language_code, - retries=retries, - ) - - if split_budget <= 0: - return None - - split_at = _find_best_split_index(lines) - left = translate_lines_resilient( - client, - lines=lines[:split_at], - target_language_name=target_language_name, - target_language_code=target_language_code, - retries=retries, - split_budget=split_budget - 1, - ) - if left is None: - return None - - right = translate_lines_resilient( - client, - lines=lines[split_at:], - target_language_name=target_language_name, - target_language_code=target_language_code, - retries=retries, - split_budget=split_budget - 1, - ) - if right is None: - return None - - return left + right - - -def translate_markdown_document( - client: OllamaClient, - *, - content: str, - target_language_name: str, - target_language_code: str, - max_chunk_lines: int, - max_chunk_chars: int, - retries: int, -) -> Optional[str]: - blocks = split_markdown_blocks(content) - out_parts: List[str] = [] - - for block in blocks: - if block.kind == "code": - out_parts.append("".join(block.lines)) - continue - - # chunk and translate - for chunk in chunk_lines(block.lines, max_lines=max_chunk_lines, max_chars=max_chunk_chars): - translated = translate_lines_resilient( - client, - lines=chunk, - target_language_name=target_language_name, - target_language_code=target_language_code, - retries=retries, - ) - if translated is None: - return None - out_parts.append(translated) - - return "".join(out_parts) - - -def looks_incomplete(existing_translation: str, source: str) -> bool: - """Conservative heuristic to detect summaries/truncation in already-generated files.""" - if not existing_translation.strip(): - return True - - src_len = len(source.strip()) - out_len = len(existing_translation.strip()) - if src_len >= 2000 and out_len < int(src_len * 0.35): - return True - - src_lines = len([l for l in source.splitlines() if l.strip()]) - out_lines = len([l for l in existing_translation.splitlines() if l.strip()]) - if src_lines >= 80 and out_lines < int(src_lines * 0.45): - return True - - # Obvious “assistant” preambles. - lowered = existing_translation.lstrip().lower() - if lowered.startswith("sure") or lowered.startswith("here is") or "summary" in lowered[:200]: - return True - - return False - - -# ---------------------------- -# CLI / main -# ---------------------------- - - -def get_git_mtime(filepath: str) -> int: - """Return the modification time of a file based on git history. - If the file has local uncommitted changes, returns its filesystem mtime. - If git is unavailable, falls back to filesystem mtime. - """ - try: - # Check for uncommitted changes first - status = subprocess.check_output(["git", "status", "--porcelain", filepath], text=True).strip() - if status: - return int(os.path.getmtime(filepath)) - - # Get the timestamp of the last commit that modified this file - out = subprocess.check_output(["git", "log", "-1", "--format=%ct", filepath], text=True).strip() - if out: - return int(out) - except Exception: - pass - - # Fallback to local filesystem mtime - try: - return int(os.path.getmtime(filepath)) - except OSError: - return 0 - - -def _parse_langs(arg: Optional[str]) -> Dict[str, str]: - if not arg: - return dict(LANGUAGES) - - wanted = {a.strip().lower() for a in arg.split(",") if a.strip()} - resolved: Dict[str, str] = {} - - for name, code in LANGUAGES.items(): - if name.lower() in wanted or code.lower() in wanted: - resolved[name] = code - - unknown = wanted - {n.lower() for n in LANGUAGES.keys()} - {c.lower() for c in LANGUAGES.values()} - if unknown: - raise SystemExit(f"Unknown languages: {', '.join(sorted(unknown))}") - return resolved - - -def main() -> None: - parser = argparse.ArgumentParser(description="Translate docs/tech-specs markdown files using a local Ollama model.") - parser.add_argument("--docs-dir", default=DEFAULT_DOCS_DIR, help="Directory containing source .md files") - parser.add_argument("--model", default=DEFAULT_OLLAMA_MODEL, help="Ollama model name") - parser.add_argument("--ollama-url", default=DEFAULT_OLLAMA_URL, help="Ollama /api/generate URL") - parser.add_argument("--langs", default=None, help="Comma-separated language names or codes (e.g. 'es,ru')") - parser.add_argument("--files", default="*.md", help="Glob pattern within docs-dir to translate (default: *.md)") - parser.add_argument("--force", action="store_true", help="Overwrite existing translations") - parser.add_argument("--max-chunk-lines", type=int, default=DEFAULT_MAX_CHUNK_LINES) - parser.add_argument("--max-chunk-chars", type=int, default=DEFAULT_MAX_CHUNK_CHARS) - parser.add_argument("--retries", type=int, default=DEFAULT_RETRIES) - parser.add_argument("--num-ctx", type=int, default=DEFAULT_NUM_CTX) - parser.add_argument("--num-predict", type=int, default=DEFAULT_NUM_PREDICT) - parser.add_argument("--temperature", type=float, default=DEFAULT_TEMPERATURE) - args = parser.parse_args() - - langs = _parse_langs(args.langs) - - client = OllamaClient( - url=args.ollama_url, - model=args.model, - num_ctx=args.num_ctx, - num_predict=args.num_predict, - temperature=args.temperature, - top_p=DEFAULT_TOP_P, - repeat_penalty=DEFAULT_REPEAT_PENALTY, - keep_alive=DEFAULT_KEEP_ALIVE, - ) - - docs_dir = args.docs_dir - patterns = [args.files] - - files: List[str] = [] - for pattern in patterns: - for path in glob.glob(os.path.join(docs_dir, pattern)): - # Skip files that are already translated (contain e.g. .es.md) - if any(f".{code}." in path for code in LANGUAGES.values()): - continue - files.append(path) - - files = sorted(set(files)) - if not files: - print(f"No source files found in {docs_dir}") - return - - print(f"Found {len(files)} files to translate in '{docs_dir}'.") - print(f"Using Ollama model: {client.model}") - print(f"Chunking: <= {args.max_chunk_lines} lines, <= {args.max_chunk_chars} chars") - print("-" * 40) - - for filepath in files: - path_obj = Path(filepath) - filename_without_ext = path_obj.stem - extension = path_obj.suffix - - print(f"Processing: {filepath}") - with open(filepath, "r", encoding="utf-8") as f: - content = f.read() - - if not content.strip(): - print(" Skipping empty file.") - continue - - for lang_name, lang_code in langs.items(): - out_filename = f"{filename_without_ext}.{lang_code}{extension}" - out_filepath = os.path.join(docs_dir, out_filename) - - if os.path.exists(out_filepath) and os.path.getsize(out_filepath) > 0 and not args.force: - with open(out_filepath, "r", encoding="utf-8") as existing_f: - existing = existing_f.read() - - source_mtime = get_git_mtime(filepath) - dest_mtime = get_git_mtime(out_filepath) - - is_outdated = source_mtime > dest_mtime - is_inc = looks_incomplete(existing, content) - - if not is_outdated and not is_inc: - print(f" [-] Skipping {lang_name} ({out_filename}) - already exists and up to date.") - continue - - if is_outdated: - print(f" [!] Source file is newer than existing {lang_name} translation; re-translating...") - elif is_inc: - print(f" [!] Existing {lang_name} looks incomplete; re-translating...") - - print(f" [+] Translating to {lang_name}...") - translated = translate_markdown_document( - client, - content=content, - target_language_name=lang_name, - target_language_code=lang_code, - max_chunk_lines=args.max_chunk_lines, - max_chunk_chars=args.max_chunk_chars, - retries=args.retries, - ) - - if translated: - with open(out_filepath, "w", encoding="utf-8") as out_f: - out_f.write(translated) - print(f" Saved -> {out_filepath}") - else: - print(f" Failed to translate {filepath} to {lang_name}.") - - -if __name__ == "__main__": - main() diff --git a/trustgraph-base/trustgraph/i18n/__init__.py b/trustgraph-base/trustgraph/i18n/__init__.py index 77674bf8..756738e8 100644 --- a/trustgraph-base/trustgraph/i18n/__init__.py +++ b/trustgraph-base/trustgraph/i18n/__init__.py @@ -98,11 +98,14 @@ def normalize_language(value: Optional[str]) -> str: return "en" +# Returns a mutable object - caller must not mutate! @lru_cache(maxsize=32) def get_language_pack(language: str) -> Dict[str, str]: """Load the language pack for `language` from package resources.""" lang = normalize_language(language) + if lang not in SUPPORTED_LANGUAGES: + lang = "en" try: with importlib_resources.open_text( diff --git a/trustgraph-flow/trustgraph/gateway/endpoint/i18n.py b/trustgraph-flow/trustgraph/gateway/endpoint/i18n.py index 67cd3c13..b949a499 100644 --- a/trustgraph-flow/trustgraph/gateway/endpoint/i18n.py +++ b/trustgraph-flow/trustgraph/gateway/endpoint/i18n.py @@ -40,6 +40,12 @@ class I18nPackEndpoint: return web.HTTPUnauthorized() lang = request.match_info.get("lang") or "en" + + # This is a path traversal defense, and is a critical sec defense. + # Do not remove! + if "/" in lang or ".." in lang: + return web.HTTPBadRequest(reason="Invalid language code") + pack = get_language_pack(lang) return web.json_response(pack) diff --git a/trustgraph/trustgraph/trustgraph_version.py b/trustgraph/trustgraph/trustgraph_version.py deleted file mode 100644 index b08ba8d4..00000000 --- a/trustgraph/trustgraph/trustgraph_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "2.2.999" diff --git a/updates.md b/updates.md deleted file mode 100644 index 451f323c..00000000 --- a/updates.md +++ /dev/null @@ -1,21 +0,0 @@ -# TrustGraph i18n & Documentation Translation Updates - -The TrustGraph project has recently introduced comprehensive internationalization (i18n) support alongside powerful automated documentation translation capabilities. This update brings native language localization directly into the TrustGraph Command Line Interface (CLI) and includes an intelligent script designed to keep multi-language repository documentation fully synchronized with upstream changes. - -## How the Translation & i18n Works - -* **Native CLI i18n**: The TrustGraph CLI has built-in translation support that dynamically loads language strings. You can test and use different languages by simply passing the `--lang` flag (e.g., `--lang es` for Spanish, `--lang ru` for Russian) or by configuring your environment's `LANG` variable. -* **Automated LLM Translation**: The repository includes `translate_docs.py`, a script that leverages local Ollama models (such as `translategemma:12b`) to autonomously translate Markdown documentation into several target languages, including Spanish, Swahili, Portuguese, Turkish, Hindi, Hebrew, Arabic, Simplified Chinese, and Russian. -* **Intelligent Git-Aware Syncing**: The doc translator uses `git status` and `git log` modification timestamps to detect out-of-sync files. If a source file (like a core English `.md` file) receives newer commits or local edits, the script automatically flags the outdated translations and re-translates them to match the latest base version. -* **Safe Markdown Parsing**: To prevent the LLM from destroying technical formatting, the translation script chunks the text at safe boundaries, protects code fences, and uses placeholder tokens for inline code and URLs to ensure precision. - -## Updated Documents - -Because of the Git-aware translation capabilities, several core documents in the `docs/` folder have been successfully updated to parity across all supported languages, including: -* `python-api.md` -* API Gateway changelogs -* CLI changelogs -* Contributor License Agreements (CLAs) -* Assorted technical READMEs - -*Note: The `translate_docs.py` translation engine will be moved out of the core TrustGraph repository into its own dedicated standalone repository in the near future.* \ No newline at end of file