utils.py 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874
  1. """Shared utility functions for hermes-agent."""
  2. import errno
  3. import hashlib
  4. import json
  5. import logging
  6. import os
  7. import re
  8. import shutil
  9. import stat
  10. import sys
  11. import tempfile
  12. import threading
  13. import time
  14. from contextlib import suppress
  15. from pathlib import Path
  16. from typing import Any, Union
  17. from urllib.parse import urlparse
  18. import hermes_yaml as yaml
  19. logger = logging.getLogger(__name__)
  20. TRUTHY_STRINGS = frozenset({"1", "true", "yes", "on"})
  21. def is_truthy_value(value: Any, default: bool = False) -> bool:
  22. """Coerce bool-ish values using the project's shared truthy string set."""
  23. if value is None:
  24. return default
  25. if isinstance(value, str):
  26. return value.strip().lower() in TRUTHY_STRINGS
  27. return bool(value)
  28. def env_var_enabled(name: str, default: str = "") -> bool:
  29. """Return True when an environment variable is set to a truthy value."""
  30. return is_truthy_value(os.getenv(name, default), default=False)
  31. def file_signature(st: os.stat_result) -> tuple[int, int, int, int]:
  32. """Change-detection key for a stat result: ``(st_mtime_ns, st_size, st_ino, st_ctime_ns)``.
  33. mtime + size alone miss a replacement that preserves both (``cp -p``, ``rsync -t``, a tar
  34. restore, a script pinning the timestamp with ``os.utime``). The inode changes on an atomic
  35. replace and ctime cannot be backdated from user space, so the pair catches those writers.
  36. On Windows ``st_ino`` may be 0 and ``st_ctime_ns`` is the creation time — both stable across
  37. an in-place rewrite, so the key degrades to mtime + size there rather than misfiring.
  38. """
  39. return (st.st_mtime_ns, st.st_size, st.st_ino, st.st_ctime_ns)
  40. # Filesystem timestamps tick coarsely (~4 ms; whole seconds on some filesystems), so a write in the
  41. # same tick as a stat leaves the signature unchanged. A signature this fresh is "racily clean" (as in
  42. # git's index) and must not be trusted as "nothing changed since".
  43. RACY_MTIME_WINDOW_NS = 2_000_000_000
  44. def mtime_is_racy(mtime_ns: int) -> bool:
  45. """True while ``mtime_ns`` is too recent to rule out a same-tick write after the stat."""
  46. return time.time_ns() - mtime_ns < RACY_MTIME_WINDOW_NS
  47. def content_digest(data: bytes) -> str:
  48. """Short digest of ``data`` for cache keys (not a security primitive)."""
  49. return hashlib.blake2b(data, digest_size=16).hexdigest()
  50. def _frozen_config_worker(path: str | Path) -> bool:
  51. """True for a ``config.yaml`` read inside a worker whose config is a frozen snapshot: such a
  52. worker never opens a profile config file, so a cache key there stays stat-only."""
  53. if Path(path).name != "config.yaml":
  54. return False
  55. policy = sys.modules.get("agent.safe_worker_policy") # binding a snapshot imports it first
  56. return policy is not None and policy.worker_config_snapshot() is not None
  57. def _file_digest(path: str | Path) -> str:
  58. """:func:`content_digest` of the file's bytes. A file that stats but cannot be read here
  59. (permission, removed since the stat, a directory) gets ``<ErrorName>``, which no hex digest
  60. matches: the key differs from any cached one, so the caller's own read runs and meets the error
  61. in its normal recovery, as it did when the key was stat-only and never opened the file."""
  62. try:
  63. with open(path, "rb") as f:
  64. return content_digest(f.read())
  65. except OSError as exc:
  66. return f"<{type(exc).__name__}>"
  67. def racy_content_digest(path: str | Path, sig: tuple[int, int, int, int]) -> str:
  68. """``""`` once the :func:`file_signature` ``sig`` has settled; while it is racy, a digest of the
  69. file's bytes (:func:`_file_digest`, so it never raises). A cache keyed on a stat signature appends
  70. it, because two same-size in-place writes in one timestamp tick share mtime, ctime, size and inode
  71. (kernels without multigrain timestamps). Racy is judged on the later of mtime and ctime: userland
  72. can restore an older mtime after a write (``cp -p``, ``rsync --inplace -t``), never ctime. Read
  73. before the parse it guards: a write in between then costs one re-parse instead of pinning the
  74. older content."""
  75. if not mtime_is_racy(max(sig[0], sig[3])) or _frozen_config_worker(path):
  76. return ""
  77. return _file_digest(path)
  78. def file_cache_version(path: str | Path, st: os.stat_result | None = None, *,
  79. always_digest: bool = False) -> tuple[Any, ...]:
  80. """Key for anything derived from a file's content: :func:`file_signature` plus a content digest.
  81. The digest is :func:`racy_content_digest` (empty once the file is 2 s old), which suits a cache
  82. re-validated on every read: a racily-stored entry costs one re-parse when the file settles.
  83. ``always_digest`` hashes on every call, for a signature stored and compared much later (a
  84. change detector whose false "changed" restarts something, or a "still the broken file" record),
  85. where the racy digest turning empty would read as a change. A worker with a frozen config
  86. snapshot never hashes a ``config.yaml`` (the key is stat-only there, as before). Raises
  87. ``OSError`` only from the stat (none given ``st``): a file that cannot be read gets the error's
  88. name as its digest (:func:`_file_digest`)."""
  89. sig = file_signature(Path(path).stat() if st is None else st)
  90. if always_digest and not _frozen_config_worker(path):
  91. return (*sig, _file_digest(path))
  92. return (*sig, racy_content_digest(path, sig))
  93. def _preserve_file_mode(path: Path) -> int | None:
  94. """Permission bits of *path* if it exists, else ``None``."""
  95. try:
  96. return stat.S_IMODE(path.stat().st_mode) if path.exists() else None
  97. except OSError:
  98. return None
  99. def _preserve_file_owner(path: Path) -> tuple[int, int] | None:
  100. """Owning ``(uid, gid)`` of *path* on POSIX, else ``None``."""
  101. try:
  102. st = path.stat() if os.name == "posix" else None
  103. except OSError:
  104. return None
  105. return (st.st_uid, st.st_gid) if st else None
  106. def _restore_file_metadata(path: Path, owner: tuple[int, int] | None, mode: int | None) -> None:
  107. """Best-effort re-apply of uid/gid and permission bits after an atomic replace.
  108. Docker/NAS installs often run some commands as root on a volume owned by the runtime user;
  109. ``os.replace`` swaps in the temp file's owner, so privileged callers chown it back. ``mkstemp``
  110. creates 0o600 files; without re-applying *mode* the target would inherit that and break
  111. volume mounts relying on broader permissions.
  112. """
  113. if owner is not None and hasattr(os, "chown"):
  114. with suppress(OSError):
  115. os.chown(path, owner[0], owner[1])
  116. if mode is not None:
  117. with suppress(OSError):
  118. os.chmod(path, mode)
  119. def default_new_file_mode() -> int | None:
  120. """The mode ``open(path, "w")`` gives a file it has to create (``0o666 & ~umask``); ``None``
  121. when the umask cannot be read or on non-POSIX hosts (Windows mode bits are synthesized).
  122. ``mkstemp`` always creates at 0600, so publishing a *new* non-secret file through a temp
  123. file would tighten it to owner-only — the Docker/NAS volume-mount hazard
  124. :func:`_restore_file_metadata` documents. The transient mask is 0o077: a thread that opens
  125. a file in the read window gets a tighter file, never a looser one.
  126. """
  127. if os.name != "posix":
  128. return None
  129. try:
  130. current = os.umask(0o077)
  131. os.umask(current)
  132. except OSError:
  133. return None
  134. return 0o666 & ~current
  135. def _restore_file_owner(path: Path, owner: tuple[int, int] | None) -> None:
  136. _restore_file_metadata(path, owner, None)
  137. def _restore_file_mode(path: Path, mode: int | None) -> None:
  138. _restore_file_metadata(path, None, mode)
  139. _IS_WINDOWS = os.name == "nt"
  140. # Windows rename failures possibly caused by another handle on the target. CPython opens files
  141. # without FILE_SHARE_DELETE, so ``os.replace`` onto an open file is denied with 5 ERROR_ACCESS_DENIED
  142. # (what a held *target* handle actually reports — measured: a plain reader yields 5, NOT 32),
  143. # 32 ERROR_SHARING_VIOLATION (the *source* temp file is held) or 33 ERROR_LOCK_VIOLATION (byte-range
  144. # lock on the target). Ambiguous (a real ACL denial is also 5), so recovery is bounded and a
  145. # still-failing write is re-raised unchanged rather than classified up front.
  146. _WINDOWS_CONTENDED_REPLACE_ERRORS = frozenset({5, 32, 33})
  147. # Retry budget for the atomic rename. A rename that wins here keeps the write fully atomic, so the
  148. # budget covers a realistic hold (desktop auth-init holds auth.json >100 ms): ~200 ms recovered
  149. # atomically, ~310 ms worst case. The cap matters as much as the count — gateway_state.json is
  150. # rewritten every turn, so a permanently-held target pays the full budget per write. Jittered so
  151. # concurrent writers don't retry in lockstep.
  152. _REPLACE_RETRY_ATTEMPTS = 4
  153. _REPLACE_RETRY_BASE_DELAY_S = 0.02
  154. _REPLACE_RETRY_MAX_DELAY_S = 0.1
  155. _CROSS_DEVICE_ERRNOS = (errno.EXDEV, errno.EBUSY)
  156. def _is_contended_windows_replace_error(exc: OSError) -> bool:
  157. """Candidate-only: winerror 5 also covers a genuine ACL denial."""
  158. return _IS_WINDOWS and getattr(exc, "winerror", None) in _WINDOWS_CONTENDED_REPLACE_ERRORS
  159. def _rewrite_in_place(tmp_str: str, real_path: str) -> None:
  160. """Overwrite *real_path* through the existing file — last resort for a still-held target.
  161. Not atomic (a smaller window than a copy, not none), so it runs only after the rename has
  162. genuinely failed. Writing through the target also preserves its ACL, which ``os.replace``
  163. does not (the temp file's inherited ACL wins there).
  164. """
  165. with open(tmp_str, "rb") as src:
  166. data = src.read()
  167. fd = os.open(real_path, os.O_WRONLY | getattr(os, "O_BINARY", 0))
  168. try:
  169. written = 0
  170. while written < len(data):
  171. written += os.write(fd, data[written:])
  172. os.ftruncate(fd, len(data))
  173. with suppress(OSError):
  174. os.fsync(fd)
  175. finally:
  176. os.close(fd)
  177. os.unlink(tmp_str)
  178. def _copy_fallback(tmp_str: str, real_path: str) -> None:
  179. """Copy/fsync/unlink fallback for cross-device and bind-mount renames."""
  180. shutil.copyfile(tmp_str, real_path)
  181. with suppress(OSError):
  182. shutil.copystat(tmp_str, real_path)
  183. with suppress(OSError), open(real_path, "rb") as f:
  184. os.fsync(f.fileno())
  185. os.unlink(tmp_str)
  186. def atomic_replace(tmp_path: str | Path, target: str | Path) -> str:
  187. """Atomically move *tmp_path* onto *target*, preserving symlinks.
  188. Resolves a symlink first so ``os.replace`` writes the real file in place and the symlink
  189. survives. Otherwise identical to ``os.replace`` unless the rename fails with EXDEV/EBUSY
  190. (cross-device, bind-mount, busy file: copy/fsync/unlink immediately — these never clear on
  191. retry) or a Windows rename contended by another open handle (winerror 5/32/33: bounded retry,
  192. then in-place rewrite).
  193. """
  194. target_str = str(target)
  195. real_path = _publish_path(target_str)
  196. tmp_str = str(tmp_path)
  197. try:
  198. os.replace(tmp_str, real_path)
  199. return real_path
  200. except OSError as exc:
  201. contended = _is_contended_windows_replace_error(exc)
  202. if exc.errno not in _CROSS_DEVICE_ERRNOS and not contended:
  203. raise
  204. if contended:
  205. # Lazy: keeps ``utils`` free of a package-level dependency on ``agent``.
  206. from agent.retry_utils import jittered_backoff
  207. for attempt in range(1, _REPLACE_RETRY_ATTEMPTS + 1):
  208. time.sleep(jittered_backoff(attempt, base_delay=_REPLACE_RETRY_BASE_DELAY_S, max_delay=_REPLACE_RETRY_MAX_DELAY_S))
  209. try:
  210. os.replace(tmp_str, real_path)
  211. return real_path
  212. except OSError as retry_exc:
  213. exc = retry_exc
  214. if retry_exc.errno in _CROSS_DEVICE_ERRNOS:
  215. contended = False # not contention after all — stop burning the budget
  216. break
  217. if not _is_contended_windows_replace_error(retry_exc):
  218. raise
  219. logger.debug("atomic_replace: %s -> %s failed with %s; falling back to %s", tmp_str, real_path,
  220. getattr(exc, "winerror", None) or errno.errorcode.get(exc.errno or 0, exc.errno),
  221. "in-place rewrite" if contended else "copy")
  222. # The rewrite re-raises its own error, so an ACL denial is reported as such, not as contention.
  223. (_rewrite_in_place if contended else _copy_fallback)(tmp_str, real_path)
  224. return real_path
  225. def _publish_path(target_str: str) -> str:
  226. """The path :func:`atomic_replace` renames onto: a symlink's real file, else the target itself."""
  227. return os.path.realpath(target_str) if os.path.islink(target_str) else target_str
  228. def mkstemp_beside(target: str | Path, **kw: Any) -> tuple[int, str]:
  229. """``tempfile.mkstemp`` in the directory :func:`atomic_replace` will rename into.
  230. A temp staged next to a symlink whose target lives on another filesystem turns the publish
  231. rename into EXDEV, and atomic_replace's copy fallback then rewrites the file in place (torn on
  232. a crash). Staging beside the resolved target keeps the rename atomic. If that directory is not
  233. writable to us (the file itself may still be), stage beside the link instead: the save keeps
  234. working through the non-atomic copy fallback, exactly as before.
  235. """
  236. target_str = str(target)
  237. link_dir = str(Path(target_str).parent)
  238. stage_dir = os.path.dirname(_publish_path(target_str)) or link_dir
  239. try:
  240. return tempfile.mkstemp(dir=stage_dir, **kw)
  241. except PermissionError:
  242. if stage_dir == link_dir:
  243. raise
  244. return tempfile.mkstemp(dir=link_dir, **kw)
  245. def fsync_directory(path: str | Path) -> None:
  246. """Best-effort fsync of a directory entry so a just-renamed file survives power loss.
  247. No-op on Windows (directories can't be opened with ``os.open``; the file fsync still applies)
  248. and on any OSError — durability of the directory entry is never worth failing a write that
  249. has already been replaced into place.
  250. """
  251. if os.name == "nt":
  252. return
  253. try:
  254. fd = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
  255. except OSError:
  256. return
  257. try:
  258. with suppress(OSError):
  259. os.fsync(fd)
  260. finally:
  261. os.close(fd)
  262. def unlink_files_older_than(directory: str | Path, pattern: str, max_age_seconds: float, *,
  263. now: float | None = None) -> int:
  264. """Delete regular files in *directory* matching *pattern* whose mtime is more than
  265. *max_age_seconds* before *now* (default: the current time); return how many were removed.
  266. The one age-based cache sweep. Never raises: a missing directory, or an entry that cannot be
  267. stat'ed or unlinked, is skipped (the next sweep retries it).
  268. """
  269. cutoff = (time.time() if now is None else now) - max_age_seconds
  270. removed = 0
  271. with suppress(OSError):
  272. for path in Path(directory).glob(pattern):
  273. with suppress(OSError):
  274. if path.is_file() and path.stat().st_mtime < cutoff:
  275. path.unlink()
  276. removed += 1
  277. return removed
  278. def rmtree_readonly(path: str | Path, *, ignore_errors: bool = False) -> None:
  279. """``shutil.rmtree`` that can also delete read-only trees.
  280. ``shutil.rmtree`` stops at the first entry it cannot unlink. Git marks
  281. loose object files read-only on Windows (``WinError 5``), and package
  282. installs (Nix store, deb/rpm) are copied ``r--r--r--`` into ``0555``
  283. directories on POSIX, where unlinking needs a writable *parent*. Clear the
  284. write bit on the failing path and on its parent, then retry the exact
  285. operation that failed. Only ``PermissionError`` is retried: every other
  286. failure keeps ``shutil.rmtree``'s semantics (and ``ignore_errors``).
  287. """
  288. def _on_error(func, fpath, exc_info):
  289. # ``onerror`` (3.11) passes ``exc_info``, ``onexc`` (3.12+) the exception.
  290. exc = exc_info[1] if isinstance(exc_info, tuple) else exc_info
  291. if not isinstance(exc, PermissionError):
  292. raise exc
  293. for candidate in (os.path.dirname(fpath), fpath):
  294. if candidate:
  295. with suppress(OSError):
  296. os.chmod(candidate, os.stat(candidate).st_mode | stat.S_IWUSR | stat.S_IXUSR)
  297. func(fpath)
  298. try:
  299. try:
  300. shutil.rmtree(path, onexc=_on_error)
  301. except TypeError: # ``onexc`` is 3.12+; 3.11 only knows ``onerror``
  302. shutil.rmtree(path, onerror=_on_error)
  303. except OSError:
  304. # ``ignore_errors`` still gets the read-only recovery; it only swallows
  305. # whatever is left after the retry.
  306. if not ignore_errors:
  307. raise
  308. def _atomic_write(path: Path, write, *, prefix: str, encoding: str = "utf-8", mode: int | None = None,
  309. preserve_owner: bool = True, binary: bool = False, fsync_dir: bool = False) -> None:
  310. """Temp file + fsync + :func:`atomic_replace`, then re-apply owner/mode.
  311. *write(f)* emits the payload into the open handle (text, or bytes when *binary*). The temp file
  312. is created by ``mkstemp`` — ``O_CREAT|O_EXCL`` at 0600 regardless of umask — so a secret is
  313. never readable at process umask, not even between create and chmod. *mode* is fchmod'd onto
  314. the temp fd BEFORE the replace so the target never transits through mkstemp's 0600 (fchmod is
  315. Unix-only; the post-replace chmod is the sole path on Windows). With no *mode* a NEW target
  316. gets what ``open(path, "w")`` would have given it (process umask) — the callers this replaced
  317. wrote at umask, and silently tightening every fresh cache/state file to 0600 breaks shared
  318. volume mounts; an existing target with no *mode* keeps mkstemp's bits, as before. *fsync_dir*
  319. also fsyncs the resolved target's parent so the rename itself is durable. The temp file is
  320. removed on any failure — ``BaseException`` on purpose, so KeyboardInterrupt / SystemExit still
  321. clean up.
  322. """
  323. # A profile delete leaves a tombstone beside its removed home. Background
  324. # writers may retain that home in a context variable, so a plain mkdir here
  325. # would resurrect the profile before the write can fail.
  326. from hermes_constants import mkdir_under_hermes_home
  327. mkdir_under_hermes_home(path.parent)
  328. if mode is None and not path.exists():
  329. mode = default_new_file_mode()
  330. original_owner = _preserve_file_owner(path) if preserve_owner else None
  331. fd, tmp_path = mkstemp_beside(path, prefix=prefix, suffix=".tmp")
  332. try:
  333. with os.fdopen(fd, "wb" if binary else "w", encoding=None if binary else encoding) as f:
  334. if mode is not None and hasattr(os, "fchmod"):
  335. os.fchmod(f.fileno(), mode)
  336. write(f)
  337. f.flush()
  338. os.fsync(f.fileno())
  339. replaced = Path(atomic_replace(tmp_path, path)) # symlink-preserving actual destination
  340. _restore_file_metadata(replaced, original_owner, mode)
  341. if fsync_dir:
  342. fsync_directory(replaced.parent)
  343. except BaseException:
  344. with suppress(OSError):
  345. os.unlink(tmp_path)
  346. raise
  347. def _mode_for_write(path: Path, create_mode: int | None, preserve: bool = True) -> int | None:
  348. """Existing permission bits of *path* (when *preserve*), else *create_mode* for a new file."""
  349. mode = _preserve_file_mode(path) if preserve else None
  350. return mode if mode is not None or path.exists() else create_mode
  351. def atomic_write_text(path: str | Path, content: str, *, encoding: str = "utf-8", tmp_prefix: str = ".tmp_",
  352. preserve_mode: bool = False, create_mode: int | None = None, mode: int | None = None,
  353. fsync_dir: bool = False) -> None:
  354. """Write *content* to *path* via temp file + fsync + atomic rename.
  355. The target is never left partially written on crash/interrupt. Shared by every destructive
  356. file rewrite (memory store, skill manager, agent importer, ...). *mode* forces the final
  357. permission bits (secret files: ``0o600``) regardless of what exists; *create_mode* applies only
  358. when the target is new and *preserve_mode* carries an existing file's bits and owner across.
  359. """
  360. path = Path(path)
  361. _atomic_write(path, lambda f: f.write(content), prefix=tmp_prefix, encoding=encoding,
  362. mode=mode if mode is not None else _mode_for_write(path, create_mode, preserve=preserve_mode),
  363. preserve_owner=preserve_mode, fsync_dir=fsync_dir)
  364. def atomic_write_bytes(path: str | Path, content: bytes, *, tmp_prefix: str = ".tmp_",
  365. mode: int | None = None, fsync_dir: bool = False) -> None:
  366. """Bytes variant of :func:`atomic_write_text` (encrypted blobs, key material)."""
  367. path = Path(path)
  368. _atomic_write(path, lambda f: f.write(content), prefix=tmp_prefix, binary=True, preserve_owner=False,
  369. mode=mode if mode is not None else _preserve_file_mode(path), fsync_dir=fsync_dir)
  370. def _dump_json(data: Any, f, *, indent: int | None, ensure_ascii: bool, dump_kwargs: dict) -> None:
  371. """``json.dump`` that survives surrogate-escaped strings.
  372. ``os.fsdecode`` of a non-UTF-8 filename/argv yields lone surrogates (``'\\udcff'``); a utf-8
  373. text handle rejects them with ``UnicodeEncodeError`` — a ValueError, which callers guarding
  374. ``except OSError`` never see. ``ensure_ascii=True`` escapes them as ``\\udcff`` and
  375. ``json.loads`` restores the identical str, so the retry round-trips; ``surrogateescape``
  376. would emit a raw 0xFF byte that the reader's utf-8 decode rejects. Serializing to a str first
  377. keeps the failure before any byte reaches the file, so no partial payload is left behind.
  378. """
  379. text = json.dumps(data, indent=indent, ensure_ascii=ensure_ascii, **dump_kwargs)
  380. try:
  381. f.write(text)
  382. except UnicodeEncodeError:
  383. f.write(json.dumps(data, indent=indent, ensure_ascii=True, **dump_kwargs))
  384. def atomic_json_write(
  385. path: str | Path, data: Any, *, indent: int = 2, mode: int | None = None,
  386. ensure_ascii: bool = False, fsync_dir: bool = False, **dump_kwargs: Any,
  387. ) -> None:
  388. """Write JSON to *path* atomically (temp file + fsync + replace).
  389. Surrogate-escaped strings (non-UTF-8 argv/paths) are always persisted: the write falls back
  390. to ``ensure_ascii=True`` escapes for that payload only, so normal content keeps its raw UTF-8
  391. bytes. ``mode=0o600`` is the private-credential form: the temp file is 0600 from creation
  392. (mkstemp), so the payload is never umask-readable.
  393. """
  394. path = Path(path)
  395. _atomic_write(path, lambda f: _dump_json(data, f, indent=indent, ensure_ascii=ensure_ascii, dump_kwargs=dump_kwargs),
  396. prefix=f".{path.stem}_", mode=mode if mode is not None else _preserve_file_mode(path),
  397. fsync_dir=fsync_dir)
  398. def read_json_or_empty(path: str | Path) -> dict:
  399. """The JSON object at *path*, or ``{}`` when the file is missing, unreadable, malformed or
  400. not an object. The read half of every ``read → merge → atomic_json_write`` config store
  401. (memory-provider ``save_config``), so a corrupt sidecar degrades to defaults instead of
  402. taking the provider down."""
  403. try:
  404. data = json.loads(Path(path).read_text(encoding="utf-8-sig")) # utf-8-sig: a Windows-editor BOM must not wipe the config
  405. except (OSError, ValueError):
  406. return {}
  407. return data if isinstance(data, dict) else {}
  408. def warn_if_credential_file_broadly_readable(path: str | Path, *, label: str = "", log: logging.Logger | None = None) -> bool:
  409. """Warn when a credential file is group/world-readable; True when a warning was emitted.
  410. Hand-made secret files (or ones older Hermes wrote without an explicit mode) commonly end up
  411. 0o644 under the default umask; call this before loading any token/credential file. No-op on
  412. non-POSIX (Windows ACLs don't map onto group/other bits; st_mode there is synthesized), when
  413. the file is missing, or when permissions are already tight.
  414. """
  415. p = Path(path)
  416. try:
  417. file_mode = p.stat().st_mode
  418. except OSError:
  419. return False
  420. if os.name != "posix" or not (file_mode & (stat.S_IRGRP | stat.S_IROTH)):
  421. return False
  422. (log or logger).warning("%s%s is group/world-readable (mode 0%o) and contains secrets. Run: chmod 600 %s",
  423. f"{label} " if label else "", p.name, stat.S_IMODE(file_mode), p)
  424. return True
  425. def atomic_yaml_write(path: str | Path, data: Any, *, default_flow_style: bool = False, sort_keys: bool = False,
  426. extra_content: str | None = None, create_mode: int | None = None) -> None:
  427. """Write YAML to *path* atomically (temp file + fsync + replace)."""
  428. path = Path(path)
  429. def _write(f) -> None:
  430. yaml.safe_dump(data, f, default_flow_style=default_flow_style, sort_keys=sort_keys)
  431. if extra_content:
  432. f.write(extra_content)
  433. _atomic_write(path, _write, prefix=f".{path.stem}_", mode=_mode_for_write(path, create_mode))
  434. def _roundtrip_load(path: Path):
  435. """``(yaml_rt, CommentedMap)``: a ruamel round-trip loader keeping quotes/Unicode with 2-space
  436. indents, plus *path* loaded through it (empty map when missing/blank)."""
  437. from ruamel.yaml.comments import CommentedMap
  438. yaml_rt = yaml.roundtrip_yaml()
  439. data = yaml_rt.load(path.read_text(encoding="utf-8")) if path.exists() else None
  440. return yaml_rt, data if isinstance(data, CommentedMap) else CommentedMap(data or {})
  441. def _roundtrip_dump(path: Path, yaml_rt, config, *, extra_content: str | None = None) -> None:
  442. def _write(f) -> None:
  443. yaml_rt.dump(config, f)
  444. if extra_content:
  445. f.write(extra_content)
  446. _atomic_write(path, _write, prefix=f".{path.stem}_", mode=_preserve_file_mode(path))
  447. def atomic_roundtrip_yaml_update(path: str | Path, key_path: str, value: Any) -> None:
  448. """Update one dotted YAML key while preserving comments, ordering, quoting and Unicode.
  449. Narrower than :func:`atomic_yaml_write` on purpose: for user-edited config files where a
  450. single setting mutation must not disturb the rest. Still writes via temp file + atomic replace.
  451. ``value=None`` removes the key (a ``key: null`` leftover reads as absent everywhere but
  452. litters the file and diverges from whole-document writers that drop the key).
  453. """
  454. from ruamel.yaml.comments import CommentedMap
  455. # Honor escaped dots and prefer existing literal dotted keys (model IDs like ``glm-5.3``) over
  456. # blind splitting — same navigation as ``hermes config set``'s ``_set_nested``; otherwise
  457. # /model + TUI persistence wrote ``glm-5: {'3': ...}`` phantom siblings.
  458. # See #91607.
  459. from hermes_cli.config import _greedy_literal_match, _split_key_path
  460. path = Path(path)
  461. from hermes_constants import mkdir_under_hermes_home
  462. mkdir_under_hermes_home(path.parent)
  463. yaml_rt, config = _roundtrip_load(path)
  464. current = config
  465. keys = _split_key_path(key_path)
  466. i = 0
  467. while True:
  468. remaining = keys[i:]
  469. seg, consumed = _greedy_literal_match(dict(current), remaining) or (remaining[0], 1)
  470. if i + consumed == len(keys):
  471. if value is None:
  472. current.pop(seg, None)
  473. else:
  474. current[seg] = value
  475. break
  476. next_value = current.get(seg)
  477. if not isinstance(next_value, CommentedMap):
  478. if value is None:
  479. return # nothing to remove under a missing/scalar parent
  480. next_value = CommentedMap()
  481. current[seg] = next_value
  482. current = next_value
  483. i += consumed
  484. _roundtrip_dump(path, yaml_rt, config)
  485. # ruamel's round-trip dumper resolves plain scalars under YAML 1.2, where only true/false/null are
  486. # reserved — so a str like "off" or "yes" is emitted unquoted. Every other config reader here
  487. # (PyYAML, yaml.safe_load sites) parses under YAML 1.1, where on/off/yes/no are booleans: an
  488. # unquoted ``approvals.mode: off`` would silently round-trip back as ``False``.
  489. _YAML11_AMBIGUOUS_WORDS = frozenset({"y", "n", "yes", "no", "true", "false", "on", "off", "null", "~"})
  490. def _rt_value(value: Any) -> Any:
  491. """Plain Python value → ruamel node: YAML 1.1-ambiguous strings force-quoted at every depth
  492. (a bare ``off`` inside a list reads back as ``False`` just like one at the top level)."""
  493. from ruamel.yaml.comments import CommentedMap, CommentedSeq
  494. from ruamel.yaml.scalarstring import DoubleQuotedScalarString
  495. if isinstance(value, dict):
  496. node = CommentedMap()
  497. for k, v in value.items():
  498. node[k] = _rt_value(v)
  499. return node
  500. if isinstance(value, (list, tuple)):
  501. return CommentedSeq(_rt_value(v) for v in value)
  502. if isinstance(value, str) and value.lower() in _YAML11_AMBIGUOUS_WORDS:
  503. return DoubleQuotedScalarString(value)
  504. return value
  505. def atomic_roundtrip_yaml_save(path: str | Path, new_state: dict, *,
  506. extra_content_on_create: str | None = None) -> None:
  507. """Persist a full config-state dict while preserving comments and ordering.
  508. THE on-disk primitive for ``config.yaml`` (production callers reach it through
  509. ``hermes_cli.config.atomic_config_write`` or ``atomic_config_replace``): the document is
  510. loaded through ruamel round-trip mode and *new_state* is merged onto it, so comments, key order, quotes, blank
  511. lines and readable Unicode survive. Only nodes whose value actually changed are reassigned;
  512. an untouched scalar or list keeps its inline comments and formatting. Keys absent from
  513. *new_state* are deleted ("explicit absence": ``cfg.pop(k)`` + save removes ``k`` from disk).
  514. ``extra_content_on_create`` (commented example blocks) is appended only when the file is
  515. being created — re-appending it on every rewrite is how the stock boilerplate replaced
  516. users' own comments (#92554).
  517. """
  518. from ruamel.yaml.comments import CommentedMap, CommentedSeq
  519. from hermes_cli.config import require_readable_config_before_write
  520. path = Path(path)
  521. from hermes_constants import mkdir_under_hermes_home
  522. mkdir_under_hermes_home(path.parent)
  523. require_readable_config_before_write(path)
  524. creating = not path.exists() or not path.read_text(encoding="utf-8").strip()
  525. yaml_rt, existing = _roundtrip_load(path)
  526. def _unchanged(current: Any, value: Any) -> bool:
  527. # ``True == 1`` in Python; a bool↔int flip is a real change for YAML readers.
  528. return current == value and isinstance(current, bool) is isinstance(value, bool)
  529. def _merge_seq(dst: CommentedSeq, src: list) -> None:
  530. # Element-wise so appending/editing one entry keeps the comments on its siblings.
  531. for i, value in enumerate(src):
  532. if i < len(dst):
  533. _merge_item(dst, i, value)
  534. else:
  535. dst.append(_rt_value(value))
  536. del dst[len(src):]
  537. def _merge_item(dst, key, value) -> None:
  538. current = dst[key]
  539. if isinstance(value, dict) and isinstance(current, CommentedMap):
  540. _merge(current, value)
  541. elif isinstance(value, list) and isinstance(current, CommentedSeq):
  542. _merge_seq(current, value)
  543. elif not _unchanged(current, value):
  544. dst[key] = _rt_value(value)
  545. # else: unchanged — keep the existing node and the comments/quoting attached to it
  546. def _merge(dst: CommentedMap, src: dict) -> None:
  547. for key, value in src.items():
  548. if key in dst:
  549. _merge_item(dst, key, value)
  550. else:
  551. dst[key] = _rt_value(value)
  552. for key in [k for k in dst if k not in src]:
  553. del dst[key]
  554. _merge(existing, new_state)
  555. _roundtrip_dump(path, yaml_rt, existing, extra_content=extra_content_on_create if creating else None)
  556. def safe_json_loads(text: str, default: Any = None) -> Any:
  557. """Parse JSON, returning *default* on any parse error (a too-deeply nested body included)."""
  558. try:
  559. return json.loads(text)
  560. except (json.JSONDecodeError, TypeError, ValueError, RecursionError):
  561. return default
  562. def fast_safe_load(stream: Any) -> Any:
  563. """Use the shared safe reader (which selects ruamel's C parser when available)."""
  564. return yaml.safe_load(stream)
  565. _YAML_FILE_CACHE: dict = {}
  566. _YAML_FILE_CACHE_LOCK = threading.Lock()
  567. def load_yaml_file_readonly(path: str | Path) -> Any:
  568. """``fast_safe_load`` of a file, re-parsed only when its :func:`file_cache_version` changes.
  569. Returns the cached object itself — callers must never mutate it. Parse errors propagate and
  570. are not cached; a missing file raises ``FileNotFoundError`` like ``open`` does."""
  571. path = Path(path)
  572. sig = file_cache_version(path)
  573. key = str(path)
  574. with _YAML_FILE_CACHE_LOCK:
  575. cached = _YAML_FILE_CACHE.get(key)
  576. if cached is not None and cached[0] == sig:
  577. return cached[1]
  578. with open(path, encoding="utf-8") as f:
  579. data = fast_safe_load(f)
  580. with _YAML_FILE_CACHE_LOCK:
  581. _YAML_FILE_CACHE[key] = (sig, data)
  582. return data
  583. def _env_number(key: str, default, cast):
  584. raw = os.getenv(key, "").strip()
  585. try:
  586. return cast(raw) if raw else default
  587. except (ValueError, TypeError):
  588. return default
  589. def env_int(key: str, default: int = 0) -> int:
  590. """Read an environment variable as an integer, with fallback."""
  591. return _env_number(key, default, int)
  592. def env_float(key: str, default: float = 0.0) -> float:
  593. """Read an environment variable as a float, with fallback."""
  594. return _env_number(key, default, float)
  595. def env_bool(key: str, default: bool = False) -> bool:
  596. """Read an environment variable as a boolean."""
  597. return is_truthy_value(os.getenv(key, ""), default=default)
  598. _PROXY_ENV_KEYS = ("HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "https_proxy", "http_proxy", "all_proxy")
  599. _NO_PROXY_ENV_KEYS = ("NO_PROXY", "no_proxy")
  600. def normalize_proxy_url(proxy_url: str | None) -> str | None:
  601. """Normalize proxy URLs for httpx/aiohttp: WSL/Clash export ``socks://``, httpx needs ``socks5://``."""
  602. candidate = str(proxy_url or "").strip()
  603. if candidate.lower().startswith("socks://"):
  604. return f"socks5://{candidate[len('socks://'):]}"
  605. return candidate or None
  606. def _bare_ipv6_literal(entry: str) -> str | None:
  607. """The bare IPv6 literal a NO_PROXY entry should become, or None when the entry is not an
  608. IPv6 form httpx cannot compile (``[::1]``, ``[::1]:8080``, ``[2001:db8::1]/64``, ``::1/128``).
  609. httpx 0.28.1's ``get_environment_proxies`` routes bracketed entries through its wildcard
  610. branch (``all://*[::1]`` → ``InvalidURL: Invalid port`` at ``Client.__init__``) and cannot
  611. compile IPv6 CIDR bypasses at all, so both degrade to the bare literal — which httpx compiles
  612. into a working ``all://[<v6>]`` bypass and urllib, requests and ``agent.proxy_bypass`` all
  613. understand. A ``[<v6>]:port`` entry loses its port: httpx has no port-scoped IPv6 bypass form,
  614. and the portless literal still bypasses every port (the operator's intent superset). Bracketed
  615. or CIDR IPv4 is left alone (httpx compiles those forms fine).
  616. """
  617. import ipaddress
  618. candidate = str(entry or "").strip()
  619. m = (re.fullmatch(r"\[([0-9A-Fa-f:.]+)\](?:[/:].*)?", candidate)
  620. or re.fullmatch(r"([0-9A-Fa-f:.]+)/\d+", candidate))
  621. if not m:
  622. return None
  623. try:
  624. ip = ipaddress.ip_address(m.group(1))
  625. except ValueError:
  626. return None # bracketed junk: httpx's wildcard branch compiles it; not ours to touch
  627. return m.group(1) if ip.version == 6 else None
  628. def sanitize_no_proxy_entries(no_proxy_value: str | None) -> str:
  629. """Rewrite NO_PROXY entries httpx 0.28.1 cannot compile into the equivalent bare-IPv6 forms.
  630. Every other entry passes through verbatim, and the original string is returned byte-stable
  631. when nothing needed rewriting (an env without bracketed/CIDR IPv6 entries is untouched).
  632. """
  633. raw = str(no_proxy_value or "")
  634. entries = [part for part in re.split(r"[\s,]+", raw.strip()) if part]
  635. if not entries or "*" in entries: # a wildcard bypasses everything; httpx returns no mounts
  636. return raw
  637. rewritten: list[str] = []
  638. for entry in entries:
  639. bare = _bare_ipv6_literal(entry)
  640. if bare is None:
  641. rewritten.append(entry) # not ours to touch: IPv4, IPv4 CIDR, domains, junk
  642. elif bare not in rewritten:
  643. rewritten.append(bare)
  644. if rewritten == entries:
  645. return raw
  646. return ",".join(rewritten)
  647. def normalize_proxy_env_vars() -> None:
  648. """Rewrite supported proxy env vars to canonical URL forms in-place."""
  649. for key in _PROXY_ENV_KEYS:
  650. value = os.getenv(key, "")
  651. normalized = normalize_proxy_url(value)
  652. if normalized and normalized != value:
  653. os.environ[key] = normalized
  654. for key in _NO_PROXY_ENV_KEYS:
  655. value = os.getenv(key)
  656. if value:
  657. sanitized = sanitize_no_proxy_entries(value)
  658. if sanitized != value:
  659. os.environ[key] = sanitized
  660. def _parse_base_url(base_url: str):
  661. """``urlparse`` that tolerates a bare ``host[:port][/path]`` (no scheme)."""
  662. raw = (base_url or "").strip()
  663. return urlparse(raw if "://" in raw else f"//{raw}") if raw else None
  664. def _hostname_of(parsed) -> str:
  665. return (parsed.hostname or "").lower().rstrip(".") if parsed else ""
  666. def base_url_hostname(base_url: str) -> str:
  667. """Lowercased hostname for a base URL, or ``""`` if absent.
  668. Compare exact hostnames against provider hosts instead of substring-matching the raw URL:
  669. ``https://api.openai.com.example/v1`` or ``https://proxy.test/api.openai.com/v1`` would
  670. otherwise pass as native endpoints and mis-route api_mode and auth.
  671. """
  672. return _hostname_of(_parse_base_url(base_url))
  673. def base_url_path(base_url: str) -> str:
  674. """Lowercased URL path without the trailing slash (``""`` for a bare host); same scheme
  675. tolerance as :func:`base_url_hostname` so host and path checks agree on one URL."""
  676. parsed = _parse_base_url(base_url)
  677. return (parsed.path if parsed else "").lower().rstrip("/")
  678. def model_forces_max_completion_tokens(model: str) -> bool:
  679. """True for OpenAI families that reject ``max_tokens`` (HTTP 400 ``unsupported_parameter``)."""
  680. m = (model or "").strip().lower().rsplit("/", 1)[-1]
  681. return m.startswith(("gpt-4o", "gpt-4.1", "gpt-5", "o1", "o3", "o4"))
  682. def base_url_origin(base_url: str) -> tuple[str, str, int]:
  683. """``(scheme, hostname, effective_port)`` for a base URL; ``("", "", 0)`` on no host/bad port.
  684. Origin, not just host: ``https://h`` vs ``http://h`` and two ports on one host are different
  685. trust boundaries, so handing a bearer secret to a new URL must compare all three — hostname
  686. alone would authorise an HTTPS→HTTP downgrade. Port defaults to 443/80 so ``https://h``
  687. equals ``https://h:443``.
  688. """
  689. parsed = _parse_base_url(base_url)
  690. hostname = _hostname_of(parsed)
  691. if not hostname:
  692. return ("", "", 0)
  693. scheme = (parsed.scheme or "").lower()
  694. try:
  695. port = parsed.port
  696. except ValueError: # out-of-range or non-numeric port — not a usable origin
  697. return ("", "", 0)
  698. return (scheme, hostname, {"https": 443, "http": 80}.get(scheme, 0) if port is None else port)
  699. def base_url_host_matches(base_url: str, domain: str) -> bool:
  700. """True when the base URL's hostname is ``domain`` or a subdomain.
  701. Safer than ``domain in base_url`` (``evil.com/moonshot.ai`` / ``moonshot.ai.evil`` must not
  702. match). Accepts bare hosts, full URLs, and URLs with paths.
  703. """
  704. hostname = base_url_hostname(base_url)
  705. domain = (domain or "").strip().lower().rstrip(".")
  706. return bool(hostname and domain) and (hostname == domain or hostname.endswith("." + domain))