i18n_layers.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. """Catalog layers above the bundled ``locales/`` tree: plugin language packs and the user overlay.
  2. A language pack is per-language and multi-surface: ``<lang>.yaml`` (core, read by ``agent.i18n.t``),
  3. ``<lang>.tui.yaml`` and ``<lang>.desktop.yaml`` (opaque to Python: parsed, flattened, served to the
  4. renderers over ``i18n.catalog``). Every layer is a flat ``{dotted.key: text}`` mapping and may be
  5. PARTIAL — it only carries the keys it overrides.
  6. Two layers live here:
  7. - **packs** — registered by plugins through ``PluginContext.register_locale`` (last registered wins);
  8. - **overlay** — ``$HERMES_HOME/locales/<lang>[.<surface>].yaml`` of the *current* profile home.
  9. Merged views are cached per ``(lang, surface)`` / ``(home, lang, surface)``; :func:`clear_cache` (called
  10. by ``agent.i18n.reset_language_cache``) drops them, and every pack mutation resets the facade's caches
  11. so the next ``t()`` sees the new layer.
  12. """
  13. from __future__ import annotations
  14. import logging
  15. import re
  16. import threading
  17. from dataclasses import dataclass, field
  18. from itertools import count
  19. from pathlib import Path
  20. from typing import Any, Mapping
  21. logger = logging.getLogger(__name__)
  22. CORE_SURFACE = "core"
  23. SURFACES: tuple[str, ...] = (CORE_SURFACE, "tui", "desktop")
  24. # ``en``, ``zh-hant``, ``pt-br``, ``sr-latn-rs``: lowercase language subtag + optional region/script parts.
  25. _LANGUAGE_ID_RE = re.compile(r"^[a-z]{2,3}(-[a-z0-9]{2,8})*$")
  26. # ``pl.yaml`` -> (pl, core); ``pl.tui.yaml`` -> (pl, tui). Anything else in a locales dir is ignored.
  27. _LOCALE_FILE_RE = re.compile(r"^(?P<lang>[a-z]{2,3}(?:-[a-z0-9]{2,8})*)(?:\.(?P<surface>[a-z]+))?\.ya?ml$")
  28. def normalize_language_id(value: Any) -> str:
  29. """Canonical spelling of a language id: lowercase, ``_`` -> ``-``, surrounding whitespace dropped."""
  30. return str(value).strip().lower().replace("_", "-") if isinstance(value, str) else ""
  31. def is_language_id(value: Any) -> bool:
  32. return bool(_LANGUAGE_ID_RE.match(normalize_language_id(value)))
  33. # ── parsing ───────────────────────────────────────────────────────────────────────────────────
  34. def flatten(node: Any, prefix: str = "", out: dict[str, str] | None = None) -> dict[str, str]:
  35. """Nested mapping -> ``{dotted.key: text}``. Non-string, non-mapping leaves are dropped (catalogs are
  36. text-only); :func:`non_text_leaves` reports them for the validator."""
  37. flat: dict[str, str] = {} if out is None else out
  38. if isinstance(node, Mapping):
  39. for key, value in node.items():
  40. flatten(value, f"{prefix}.{key}" if prefix else str(key), flat)
  41. elif isinstance(node, str):
  42. flat[prefix] = node
  43. return flat
  44. def non_text_leaves(node: Any, prefix: str = "") -> list[str]:
  45. """Dotted paths of leaves that are neither text nor a mapping (numbers, lists, booleans, nulls)."""
  46. if isinstance(node, Mapping):
  47. found: list[str] = []
  48. for key, value in node.items():
  49. found.extend(non_text_leaves(value, f"{prefix}.{key}" if prefix else str(key)))
  50. return found
  51. return [] if isinstance(node, str) else [prefix or "<root>"]
  52. def parse_locale_file(path: Path) -> dict[str, str]:
  53. """Parse one locale YAML into a flat catalog. Raises on unreadable/unparseable input or a non-mapping
  54. document — the strict form for registration and validation."""
  55. import hermes_yaml as yaml
  56. with Path(path).open("r", encoding="utf-8-sig") as handle:
  57. data = yaml.safe_load(handle)
  58. if data is None:
  59. return {}
  60. if not isinstance(data, Mapping):
  61. raise ValueError(f"{path}: top level must be a mapping, got {type(data).__name__}")
  62. return flatten(data)
  63. def load_locale_source(source: Any) -> dict[str, str]:
  64. """``register_locale`` input -> flat catalog: a mapping (nested or already flat) is flattened; a path
  65. is parsed with :func:`parse_locale_file`."""
  66. if isinstance(source, Mapping):
  67. return flatten(source)
  68. if isinstance(source, (str, Path)):
  69. return parse_locale_file(Path(source))
  70. raise TypeError(f"locale source must be a mapping or a path to a YAML file, got {type(source).__name__}")
  71. def scan_locale_dir(directory: Path) -> list[tuple[str, str, Path]]:
  72. """``(lang, surface, path)`` for every ``<lang>[.<surface>].yaml`` under *directory* (sorted; empty when
  73. the directory is missing). Unknown surfaces are reported so a typo like ``pl.tiu.yaml`` is visible."""
  74. directory = Path(directory)
  75. if not directory.is_dir():
  76. return []
  77. found: list[tuple[str, str, Path]] = []
  78. for path in sorted(directory.iterdir()):
  79. match = _LOCALE_FILE_RE.match(path.name)
  80. if match is None or not path.is_file():
  81. continue
  82. surface = match.group("surface") or CORE_SURFACE
  83. if surface not in SURFACES:
  84. logger.warning("Ignoring locale file %s: unknown surface %r (expected one of %s)",
  85. path, surface, ", ".join(SURFACES))
  86. continue
  87. found.append((match.group("lang"), surface, path))
  88. return found
  89. # ── plugin packs ──────────────────────────────────────────────────────────────────────────────
  90. @dataclass(frozen=True)
  91. class PackEntry:
  92. """One registered pack layer. ``token`` orders registrations; the last one registered wins."""
  93. lang: str
  94. surface: str
  95. messages: Mapping[str, str]
  96. source: str
  97. endonym: str | None = None
  98. rtl: bool = False
  99. token: int = field(default=0, compare=False)
  100. _lock = threading.RLock()
  101. _tokens = count(1)
  102. _packs: list[PackEntry] = []
  103. _pack_cache: dict[tuple[str, str], dict[str, str]] = {}
  104. _overlay_cache: dict[tuple[str, str, str], dict[str, str]] = {}
  105. _overlay_files_cache: dict[str, frozenset[tuple[str, str]]] = {}
  106. def _reset_facade() -> None:
  107. from agent.i18n import reset_language_cache # lazy: the facade imports this module
  108. reset_language_cache()
  109. def register_pack(lang: str, surface: str, messages: Mapping[str, str], *, source: str,
  110. endonym: str | None = None, rtl: bool = False) -> PackEntry:
  111. """Add a pack layer for ``(lang, surface)``; later registrations shadow earlier ones key by key."""
  112. lang_id = normalize_language_id(lang)
  113. if not is_language_id(lang_id):
  114. raise ValueError(f"invalid language id {lang!r} (expected e.g. 'pl', 'pt-br', 'zh-hant')")
  115. if surface not in SURFACES:
  116. raise ValueError(f"unknown locale surface {surface!r} (expected one of {', '.join(SURFACES)})")
  117. entry = PackEntry(lang_id, surface, dict(messages), source, endonym or None, bool(rtl), next(_tokens))
  118. with _lock:
  119. _packs.append(entry)
  120. _reset_facade()
  121. logger.debug("i18n pack registered: %s/%s from %s (%d keys)", lang_id, surface, source, len(entry.messages))
  122. return entry
  123. def unregister_pack(entry: PackEntry) -> bool:
  124. """Drop one registration (``PluginRegistration`` release). ``False`` when it was already gone."""
  125. with _lock:
  126. try:
  127. _packs.remove(entry)
  128. except ValueError:
  129. return False
  130. _reset_facade()
  131. return True
  132. def pack_layer(lang: str, surface: str = CORE_SURFACE) -> dict[str, str]:
  133. """Merged pack messages for ``(lang, surface)`` in registration order (last wins); cached."""
  134. key = (lang, surface)
  135. with _lock:
  136. cached = _pack_cache.get(key)
  137. if cached is not None:
  138. return cached
  139. merged: dict[str, str] = {}
  140. for entry in _packs:
  141. if entry.lang == lang and entry.surface == surface:
  142. merged.update(entry.messages)
  143. _pack_cache[key] = merged
  144. return merged
  145. def pack_surfaces() -> frozenset[tuple[str, str]]:
  146. """``(lang, surface)`` for every registered pack layer."""
  147. with _lock:
  148. return frozenset((entry.lang, entry.surface) for entry in _packs)
  149. def pack_languages() -> frozenset[str]:
  150. return frozenset(lang for lang, _surface in pack_surfaces())
  151. def pack_info(lang: str) -> dict[str, Any] | None:
  152. """``{endonym, rtl, source}`` of the newest pack registration for *lang* (``None`` when no pack has
  153. it). ``endonym``/``rtl`` come from the newest registration that declared them."""
  154. with _lock:
  155. entries = [entry for entry in _packs if entry.lang == lang]
  156. if not entries:
  157. return None
  158. endonym = next((entry.endonym for entry in reversed(entries) if entry.endonym), None)
  159. rtl = next((entry.rtl for entry in reversed(entries) if entry.endonym is not None), entries[-1].rtl)
  160. return {"endonym": endonym, "rtl": rtl, "source": entries[-1].source}
  161. # ── user overlay ──────────────────────────────────────────────────────────────────────────────
  162. def overlay_dir(home: Path | str) -> Path:
  163. return Path(home) / "locales"
  164. def overlay_layer(home: Path | str, lang: str, surface: str = CORE_SURFACE) -> dict[str, str]:
  165. """``<home>/locales/<lang>[.<surface>].yaml`` flattened; ``{}`` (logged) when missing or broken. Cached
  166. per home so a multiplexed process serving profile A then B then A never mixes overlays."""
  167. key = (str(home), lang, surface)
  168. with _lock:
  169. cached = _overlay_cache.get(key)
  170. if cached is not None:
  171. return cached
  172. name = f"{lang}.yaml" if surface == CORE_SURFACE else f"{lang}.{surface}.yaml"
  173. path = overlay_dir(home) / name
  174. flat: dict[str, str] = {}
  175. if path.is_file():
  176. try:
  177. flat = parse_locale_file(path)
  178. except Exception as exc:
  179. logger.warning("Failed to load i18n overlay %s: %s", path, exc)
  180. flat = {}
  181. with _lock:
  182. _overlay_cache[key] = flat
  183. return flat
  184. def overlay_surfaces(home: Path | str) -> frozenset[tuple[str, str]]:
  185. """``(lang, surface)`` for every overlay file under *home*."""
  186. key = str(home)
  187. with _lock:
  188. cached = _overlay_files_cache.get(key)
  189. if cached is not None:
  190. return cached
  191. found = frozenset((lang, surface) for lang, surface, _path in scan_locale_dir(overlay_dir(home)))
  192. with _lock:
  193. _overlay_files_cache[key] = found
  194. return found
  195. def overlay_languages(home: Path | str) -> frozenset[str]:
  196. """Language ids that have at least one overlay file (any surface) under *home*."""
  197. return frozenset(lang for lang, _surface in overlay_surfaces(home))
  198. def surface_catalog(home: Path | str, lang: str, surface: str = CORE_SURFACE) -> dict[str, str]:
  199. """Overlay + packs (packs win) for one surface — the layer ``i18n.catalog`` ships to a renderer, which
  200. merges it over its own bundled catalog. Bundled Python locales are NOT included."""
  201. merged = dict(overlay_layer(home, lang, surface))
  202. merged.update(pack_layer(lang, surface))
  203. return merged
  204. def layered_languages(home: Path | str) -> frozenset[str]:
  205. """Languages supplied by any layer above bundled (overlay of *home* ∪ packs)."""
  206. return overlay_languages(home) | pack_languages()
  207. def clear_cache() -> None:
  208. """Drop merged views (not the registrations); the facade's ``reset_language_cache`` calls this."""
  209. with _lock:
  210. _pack_cache.clear()
  211. _overlay_cache.clear()
  212. _overlay_files_cache.clear()
  213. def _reset_registry_for_tests() -> None:
  214. with _lock:
  215. _packs.clear()
  216. clear_cache()
  217. def registered_packs() -> tuple[PackEntry, ...]:
  218. with _lock:
  219. return tuple(_packs)
  220. __all__ = [
  221. "CORE_SURFACE",
  222. "SURFACES",
  223. "PackEntry",
  224. "clear_cache",
  225. "flatten",
  226. "is_language_id",
  227. "layered_languages",
  228. "load_locale_source",
  229. "non_text_leaves",
  230. "normalize_language_id",
  231. "overlay_dir",
  232. "overlay_languages",
  233. "overlay_layer",
  234. "overlay_surfaces",
  235. "pack_info",
  236. "pack_languages",
  237. "pack_layer",
  238. "pack_surfaces",
  239. "parse_locale_file",
  240. "register_pack",
  241. "registered_packs",
  242. "scan_locale_dir",
  243. "surface_catalog",
  244. "unregister_pack",
  245. ]