i18n.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281
  1. """Lightweight i18n for Hermes' static user-facing strings (approval prompts, gateway replies, CLI, tips).
  2. Catalogs are flat dotted-key mappings resolved through layers, top first:
  3. 1. plugin language packs (``PluginContext.register_locale``; last registered wins) — ``agent.i18n_layers``
  4. 2. the user overlay ``$HERMES_HOME/locales/<lang>.yaml`` of the current profile home
  5. 3. the bundled ``locales/<lang>.yaml``
  6. 4. the same chain for ``en``
  7. 5. the bare key
  8. Every layer may be partial. Language resolution: explicit ``lang=`` > ``HERMES_LANGUAGE`` >
  9. ``display.language`` > ``en``; any id that some layer supplies is accepted, so a pack-only language
  10. (``pl``) works the moment its plugin loads. ``t()`` is a hot path: one cached merged dict per
  11. ``(home, lang)``, invalidated by :func:`reset_language_cache` (which every pack registration calls).
  12. """
  13. from __future__ import annotations
  14. import logging
  15. import os
  16. import threading
  17. from functools import lru_cache
  18. from pathlib import Path
  19. from typing import Any
  20. from agent import i18n_layers
  21. from agent.i18n_languages import language_options
  22. logger = logging.getLogger(__name__)
  23. # Bundled catalogs (compat: tests and the parity check iterate this). ``supported_languages()`` is the
  24. # live set including overlay and pack languages.
  25. SUPPORTED_LANGUAGES: tuple[str, ...] = (
  26. "en", "zh", "zh-hant", "ja", "de", "es", "fr", "tr", "uk",
  27. "af", "ko", "it", "ga", "pt", "ru", "hu", "ar",
  28. )
  29. DEFAULT_LANGUAGE = "en"
  30. # Natural aliases so "chinese" / "zh-CN" / "jp" hit the right catalog instead of
  31. # silently falling back to English. Bare "chinese" defaults to Simplified;
  32. # Taiwan/HK/Macau tags route to the distinct Traditional catalog. pt-br shares
  33. # the pt catalog unless a pack supplies a real pt-br one (a supplied id always wins over an alias).
  34. _LANGUAGE_ALIASES: dict[str, str] = {
  35. "english": "en", "en-us": "en", "en-gb": "en",
  36. "chinese": "zh", "mandarin": "zh", "zh-cn": "zh", "zh-hans": "zh", "zh-sg": "zh",
  37. "traditional-chinese": "zh-hant", "traditional_chinese": "zh-hant",
  38. "zh-tw": "zh-hant", "zh-hk": "zh-hant", "zh-mo": "zh-hant",
  39. "japanese": "ja", "jp": "ja", "ja-jp": "ja",
  40. "german": "de", "deutsch": "de", "de-de": "de", "de-at": "de", "de-ch": "de",
  41. "spanish": "es", "español": "es", "espanol": "es", "es-es": "es", "es-mx": "es", "es-ar": "es",
  42. "french": "fr", "français": "fr", "france": "fr", "fr-fr": "fr", "fr-be": "fr", "fr-ca": "fr", "fr-ch": "fr",
  43. "ukrainian": "uk", "ukrainisch": "uk", "українська": "uk", "uk-ua": "uk", "ua": "uk",
  44. "turkish": "tr", "türkçe": "tr", "tr-tr": "tr",
  45. "afrikaans": "af", "af-za": "af",
  46. "korean": "ko", "한국어": "ko", "ko-kr": "ko",
  47. "italian": "it", "italiano": "it", "it-it": "it", "it-ch": "it",
  48. "irish": "ga", "gaeilge": "ga", "ga-ie": "ga",
  49. "portuguese": "pt", "português": "pt", "portugues": "pt",
  50. "pt-pt": "pt", "pt-br": "pt", "brazilian": "pt", "brasileiro": "pt",
  51. "russian": "ru", "русский": "ru", "ru-ru": "ru",
  52. "hungarian": "hu", "magyar": "hu", "hu-hu": "hu",
  53. "arabic": "ar", "العربية": "ar",
  54. "ar-sa": "ar", "ar-eg": "ar", "ar-ae": "ar", "ar-ma": "ar", "ar-dz": "ar",
  55. }
  56. # (home, lang) -> merged catalog (packs over overlay over bundled). home -> supported tuple.
  57. _catalog_cache: dict[tuple[str, str], dict[str, str]] = {}
  58. _supported_cache: dict[str, tuple[str, ...]] = {}
  59. _catalog_lock = threading.Lock()
  60. def _locales_dir() -> Path:
  61. """Locale dir: ``HERMES_BUNDLED_LOCALES`` (sealed packaging, e.g. Nix) if it exists, else ``<repo-root>/locales``.
  62. The source path is returned even when missing so ``_load_bundled`` can log
  63. the path it looked at rather than raise.
  64. """
  65. override = os.getenv("HERMES_BUNDLED_LOCALES", "").strip()
  66. if override and Path(override).is_dir():
  67. return Path(override)
  68. if override:
  69. logger.warning(
  70. "HERMES_BUNDLED_LOCALES points to a non-directory path (%s); "
  71. "falling back to bundled/source locale resolution", override,
  72. )
  73. return Path(__file__).resolve().parent.parent / "locales"
  74. def _current_home() -> str:
  75. from hermes_constants import get_hermes_home
  76. return str(get_hermes_home())
  77. def supported_languages(home: str | None = None) -> tuple[str, ...]:
  78. """Every language some layer supplies for the current profile home: bundled ∪ user overlay ∪ plugin
  79. packs, ``en`` first then sorted. Cached until :func:`reset_language_cache`."""
  80. home = home or _current_home()
  81. with _catalog_lock:
  82. cached = _supported_cache.get(home)
  83. if cached is not None:
  84. return cached
  85. langs = set(SUPPORTED_LANGUAGES) | i18n_layers.layered_languages(home)
  86. result = (DEFAULT_LANGUAGE, *sorted(langs - {DEFAULT_LANGUAGE}))
  87. with _catalog_lock:
  88. _supported_cache[home] = result
  89. return result
  90. @lru_cache(maxsize=4)
  91. def _bundled_surfaces(locales_dir: str) -> frozenset[tuple[str, str]]:
  92. """``(lang, surface)`` for every bundled ``locales/<lang>[.<surface>].yaml``. Keyed by directory so a
  93. ``HERMES_BUNDLED_LOCALES`` switch re-scans; cleared with the other layer views by
  94. :func:`reset_language_cache`."""
  95. return frozenset((lang, surface) for lang, surface, _path in i18n_layers.scan_locale_dir(Path(locales_dir)))
  96. def language_surfaces(home: str | None = None) -> dict[str, tuple[str, ...]]:
  97. """Per language, the surfaces some layer has strings for (bundled ∪ user overlay ∪ plugin packs), in
  98. ``SURFACES`` order. An overlay file or pack that loaded no strings (empty, comment-only, unparseable)
  99. does not count. A renderer that bundles its own catalog for a language does not need this; for any
  100. other language a missing surface means that renderer would show English."""
  101. home = home or _current_home()
  102. layered = {(lang, surface) for lang, surface in i18n_layers.overlay_surfaces(home) | i18n_layers.pack_surfaces()
  103. if i18n_layers.overlay_layer(home, lang, surface) or i18n_layers.pack_layer(lang, surface)}
  104. pairs = _bundled_surfaces(str(_locales_dir())) | layered
  105. found: dict[str, set[str]] = {}
  106. for lang, surface in pairs:
  107. found.setdefault(lang, set()).add(surface)
  108. return {lang: tuple(s for s in i18n_layers.SURFACES if s in surfaces) for lang, surfaces in found.items()}
  109. def resolve_language_id(value: Any, home: str | None = None) -> str | None:
  110. """Canonical supported id for a user-supplied value (code, alias, regional tag), or ``None`` when no
  111. layer supplies it — the validation ``hermes config set display.language`` runs."""
  112. key = i18n_layers.normalize_language_id(value)
  113. if not key:
  114. return None
  115. supported = supported_languages(home)
  116. if key in supported:
  117. return key
  118. alias = _LANGUAGE_ALIASES.get(key)
  119. if alias in supported:
  120. return alias
  121. base = key.split("-", 1)[0] # strip region suffix
  122. return base if base in supported else None
  123. def _normalize_lang(value: Any, home: str | None = None) -> str:
  124. """Map a user-supplied value to a supported code (any layer), else the default."""
  125. return resolve_language_id(value, home) or DEFAULT_LANGUAGE
  126. def _load_bundled(lang: str) -> dict[str, str]:
  127. """One bundled locale YAML flattened to dotted keys (empty dict on any failure — never crashes)."""
  128. path = _locales_dir() / f"{lang}.yaml"
  129. if not path.is_file():
  130. logger.debug("i18n catalog missing for %s at %s", lang, path)
  131. return {}
  132. try:
  133. return i18n_layers.parse_locale_file(path)
  134. except Exception as exc:
  135. logger.warning("Failed to load i18n catalog %s: %s", path, exc)
  136. return {}
  137. def _flatten_into(node: Any, prefix: str, out: dict[str, str]) -> None:
  138. """Flatten a nested mapping into ``out`` (text leaves only)."""
  139. i18n_layers.flatten(node, prefix, out)
  140. def _load_catalog(lang: str, home: str | None = None) -> dict[str, str]:
  141. """Merged catalog for ``lang`` in ``home`` (packs > overlay > bundled); cached per (home, lang)."""
  142. home = home or _current_home()
  143. key = (home, lang)
  144. with _catalog_lock:
  145. cached = _catalog_cache.get(key)
  146. if cached is not None:
  147. return cached
  148. merged = _load_bundled(lang)
  149. merged.update(i18n_layers.overlay_layer(home, lang))
  150. merged.update(i18n_layers.pack_layer(lang))
  151. with _catalog_lock:
  152. _catalog_cache[key] = merged
  153. return merged
  154. def surface_catalog(lang: str, surface: str = i18n_layers.CORE_SURFACE) -> dict[str, str]:
  155. """What ``i18n.catalog`` serves for one surface: packs > overlay > bundled ``locales/<lang>.<surface>.yaml``.
  156. The core surface omits the bundled layer (the Python renderer already has it); the TUI ships only an
  157. English catalog in-tree, so its bundled translations live in ``locales/<lang>.tui.yaml`` and ride
  158. along here."""
  159. lang = _normalize_lang(lang)
  160. merged: dict[str, str] = {}
  161. if surface != i18n_layers.CORE_SURFACE:
  162. merged.update(_load_bundled(f"{lang}.{surface}"))
  163. merged.update(i18n_layers.surface_catalog(_current_home(), lang, surface))
  164. return merged
  165. @lru_cache(maxsize=8)
  166. def _config_language_cached(hermes_home: str) -> str | None:
  167. """``display.language`` from config.yaml, read once per profile home (``t()`` is a hot path).
  168. Keyed by home so a multiplexed gateway serving several profiles doesn't freeze the first
  169. profile's language for every other profile."""
  170. try:
  171. from hermes_cli.config import load_config_readonly
  172. lang = (load_config_readonly().get("display") or {}).get("language")
  173. return _normalize_lang(lang, hermes_home) if lang else None
  174. except Exception as exc:
  175. logger.debug("Could not read display.language from config: %s", exc)
  176. return None
  177. def _config_language() -> str | None:
  178. return _config_language_cached(_current_home())
  179. def reset_language_cache() -> None:
  180. """Invalidate cached language resolution, merged catalogs and every layer view (call after
  181. ``save_config`` changes ``display.language``, after a pack registers/unregisters, or after editing
  182. an overlay file)."""
  183. _config_language_cached.cache_clear()
  184. _bundled_surfaces.cache_clear()
  185. with _catalog_lock:
  186. _catalog_cache.clear()
  187. _supported_cache.clear()
  188. i18n_layers.clear_cache()
  189. def _resolve_language(home: str) -> str:
  190. from agent.secret_scope import UnscopedSecretError, get_secret
  191. try:
  192. env_lang = get_secret("HERMES_LANGUAGE")
  193. except UnscopedSecretError:
  194. env_lang = os.environ.get("HERMES_LANGUAGE") # unscoped default-profile path: environ IS its own value
  195. return _normalize_lang(env_lang, home) if env_lang else _config_language_cached(home) or DEFAULT_LANGUAGE
  196. def get_language() -> str:
  197. """Resolve the active language using env > config > default order. ``HERMES_LANGUAGE`` is a
  198. per-profile ``.env`` value, so it is read through the secret scope: under multiplexing a raw
  199. environ read would impose the default profile's language on every other profile."""
  200. return _resolve_language(_current_home())
  201. def t(key: str, lang: str | None = None, **format_kwargs: Any) -> str:
  202. """Translate a dotted catalog key to the active (or explicit ``lang``) language.
  203. ``format_kwargs`` are applied with ``str.format``. Falls back to English,
  204. then to the bare key; a format failure returns the unformatted string.
  205. """
  206. home = _current_home()
  207. target = _normalize_lang(lang, home) if lang else _resolve_language(home)
  208. value = _load_catalog(target, home).get(key)
  209. if value is None and target != DEFAULT_LANGUAGE:
  210. value = _load_catalog(DEFAULT_LANGUAGE, home).get(key)
  211. if value is None:
  212. logger.debug("i18n miss: key=%r lang=%r", key, target)
  213. value = key
  214. if not format_kwargs:
  215. return value
  216. try:
  217. return value.format(**format_kwargs)
  218. except (KeyError, IndexError, ValueError) as exc:
  219. logger.warning("i18n format failed for key=%r lang=%r kwargs=%r: %s", key, target, format_kwargs, exc)
  220. return value
  221. __all__ = [
  222. "DEFAULT_LANGUAGE",
  223. "SUPPORTED_LANGUAGES",
  224. "get_language",
  225. "language_options",
  226. "language_surfaces",
  227. "reset_language_cache",
  228. "resolve_language_id",
  229. "supported_languages",
  230. "surface_catalog",
  231. "t",
  232. ]