hermes_logging.py 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047
  1. """Centralized logging setup for Hermes Agent.
  2. Log files: agent.log (INFO+, everything), errors.log (WARNING+), gateway.log (INFO+,
  3. gateway components; ``mode="gateway"``), gui.log (INFO+, dashboard/TUI-gateway;
  4. ``mode="gui"``). All are rotating files driven through one async queue and formatted
  5. with ``RedactingFormatter`` so secrets never reach disk.
  6. """
  7. import atexit
  8. import contextlib
  9. import copy
  10. import io
  11. import logging
  12. import os
  13. import queue
  14. import sys
  15. import threading
  16. from logging.handlers import QueueHandler, QueueListener
  17. from pathlib import Path
  18. from time import monotonic as _monotonic
  19. from typing import Optional, Sequence
  20. from hermes_constants import (
  21. get_config_path, get_hermes_home, mkdir_under_hermes_home, named_profile_is_deleted,
  22. )
  23. # setup_logging() is idempotent: a second call is a no-op unless ``force=True``.
  24. _logging_initialized = False
  25. # True only when CLH was rejected at import because portalocker cannot take a
  26. # lock on this Windows box; file handlers then use stdlib rotation (rollover
  27. # disabled — see the module-header comment) and setup_logging() warns once.
  28. _WINDOWS_CLH_FALLBACK = False
  29. _WINDOWS_CLH_FALLBACK_REASON = ""
  30. _fallback_warned = False
  31. def _portalocker_probe() -> bool:
  32. """Return True when portalocker can actually take a lock on this box.
  33. concurrent-log-handler locks every write through portalocker, which on
  34. Windows instantiates Win32Locker and imports pywintypes. Bundled payloads
  35. have shipped with that import broken (the venv's .pth files were never
  36. processed), and portalocker 3.x gives no msvcrt fallback — every emit then
  37. dies with the ImportError, CLH retries 20x, and the suppressed "Cannot
  38. acquire lock" RuntimeError below hides it completely. Probe a scratch file
  39. once at import so we can fall back to stdlib rotation instead of silently
  40. dropping every record. No-op (True) off Windows, where stdlib is in use.
  41. """
  42. global _WINDOWS_CLH_FALLBACK_REASON
  43. if sys.platform != "win32":
  44. return True
  45. try:
  46. import portalocker
  47. import tempfile
  48. except Exception as exc:
  49. _WINDOWS_CLH_FALLBACK_REASON = repr(exc)
  50. return False
  51. fd, path = tempfile.mkstemp(prefix="hermes-portalocker-")
  52. try:
  53. with os.fdopen(fd, "r+b") as stream:
  54. portalocker.lock(stream, portalocker.LOCK_EX)
  55. portalocker.unlock(stream)
  56. except Exception as exc:
  57. _WINDOWS_CLH_FALLBACK_REASON = repr(exc)
  58. return False
  59. finally:
  60. try:
  61. os.unlink(path)
  62. except OSError:
  63. pass
  64. return True
  65. # Windows-ONLY swap (#44873): stdlib ``RotatingFileHandler.doRollover()`` calls
  66. # ``os.rename()``, which fails with ``PermissionError [WinError 32]`` whenever
  67. # another process holds an append handle on ``agent.log`` — essentially always
  68. # in Hermes (TUI, gateway, hy_memory, MCP servers, CLI commands all log) —
  69. # pinning the file at the size threshold and spamming stderr on every emit.
  70. # ``concurrent-log-handler`` serializes rollover with a cross-process lock.
  71. # POSIX keeps stdlib: renames of open files work, and managed mode (NixOS)
  72. # relies on stdlib's exact ``_open()``/``doRollover()`` lifecycle for the
  73. # 0660 chmod and eager file creation; CLH opens lazily and rotates differently.
  74. if sys.platform == "win32":
  75. if _portalocker_probe():
  76. from concurrent_log_handler import (
  77. ConcurrentRotatingFileHandler as RotatingFileHandler,
  78. )
  79. else:
  80. # portalocker cannot take a lock on this box (typical cause: a sealed
  81. # bundle whose venv never processed pywin32.pth, so `import pywintypes`
  82. # fails and portalocker's Win32Locker has no msvcrt fallback). CLH
  83. # would silently drop every record through the suppressed lock-timeout
  84. # below; fall back to stdlib rotation instead. Rollover is disabled in
  85. # the fallback: multi-process appends make Windows renames fail with
  86. # WinError 32, the exact #44873 trap CLH exists to avoid.
  87. from logging.handlers import RotatingFileHandler
  88. _WINDOWS_CLH_FALLBACK = True
  89. else:
  90. from logging.handlers import RotatingFileHandler
  91. # Thread-local per-conversation session context.
  92. _session_context = threading.local()
  93. # ``%(session_tag)s`` exists on every LogRecord via _install_session_record_factory().
  94. _LOG_FORMAT = "%(asctime)s %(levelname)s%(session_tag)s %(name)s: %(message)s"
  95. _LOG_FORMAT_VERBOSE = "%(asctime)s - %(name)s - %(levelname)s%(session_tag)s - %(message)s"
  96. # The stdout stream _line_buffer_piped_stdout() already reconfigured: setup_logging runs on every
  97. # AIAgent build (per message in the gateway), so later calls skip the flush + reconfigure.
  98. _line_buffered_stdout = None
  99. def _line_buffer_piped_stdout() -> None:
  100. """Best-effort line buffering for a piped stdout (#92281).
  101. Python block-buffers stdout when it isn't a TTY, so an agent loop
  102. driving ``print()`` into a supervisor's pipe delivers its output in
  103. large delayed bursts — a headless run looks stalled for minutes while
  104. the work is actually progressing (only stderr, which we already give
  105. ``line_buffering=True`` in ``_safe_stderr()``, shows up on time).
  106. Reconfigure the interpreter's own stdout for line buffering when it is
  107. piped; interactive TTY stdout is already line-buffered. ACP reaches this
  108. too (its AIAgent calls ``setup_logging``) with stdout as its protocol
  109. channel — harmless, since line buffering only adds flushes at newlines.
  110. """
  111. global _line_buffered_stdout
  112. stream = sys.stdout
  113. if stream is None or stream is _line_buffered_stdout:
  114. return
  115. if getattr(stream, "line_buffering", False) is True:
  116. _line_buffered_stdout = stream
  117. return
  118. reconfigure = getattr(stream, "reconfigure", None)
  119. if reconfigure is None:
  120. return
  121. # A closed/detached stream raises ValueError / io.UnsupportedOperation (an OSError);
  122. # buffering is observability, not correctness — never crash setup_logging over it.
  123. with contextlib.suppress(OSError, ValueError):
  124. if not stream.isatty():
  125. reconfigure(line_buffering=True)
  126. _line_buffered_stdout = stream
  127. def _safe_stderr(): # type: ignore[return]
  128. """Return a stderr stream that tolerates Unicode on all platforms.
  129. Wraps ``sys.stderr`` with ``errors='replace'`` so un-encodable characters become
  130. ``?`` instead of crashing the process.
  131. """
  132. stream = sys.stderr
  133. encoding = getattr(stream, "encoding", None) or "utf-8"
  134. if encoding.lower().replace("-", "") in ("utf8", "utf8surrogateescape"):
  135. return stream
  136. try:
  137. wrapped = io.TextIOWrapper(stream.buffer, encoding="utf-8", errors="replace", line_buffering=True)
  138. # Prevent the wrapper from closing the underlying buffer when garbage-collected.
  139. wrapped.close = lambda: None # type: ignore[assignment]
  140. return wrapped
  141. except Exception:
  142. return stream # best-effort: no buffer / wrapping failed -> original stream
  143. class _RedactingStdio:
  144. """Text stream proxy that redacts each write: a detached gateway's stdout/stderr IS a
  145. persisted log (``logs/gateway-stdio.log``), and agent status lines (provider error echoes)
  146. reach it through ``print``, never through ``RedactingFormatter``."""
  147. def __init__(self, stream):
  148. self._stream = stream
  149. def write(self, text):
  150. from agent.redact import redact_sensitive_text
  151. return self._stream.write(redact_sensitive_text(text) if isinstance(text, str) else text)
  152. def __getattr__(self, name):
  153. return getattr(self._stream, name)
  154. def redact_detached_stdio() -> None:
  155. """Route a detached process's stdout/stderr through secret redaction (idempotent)."""
  156. for name in ("stdout", "stderr"):
  157. stream = getattr(sys, name)
  158. if stream is not None and not isinstance(stream, _RedactingStdio):
  159. setattr(sys, name, _RedactingStdio(stream))
  160. def _is_windows_concurrent_log_lock_timeout(exc: BaseException | None) -> bool:
  161. """True for concurrent-log-handler's Windows lock timeout.
  162. Slash-command workers and the gateway share rotating files on Windows Desktop;
  163. when another process holds the rollover lock too long CLH raises this
  164. RuntimeError, which must not escape into Desktop chat output.
  165. """
  166. return (
  167. sys.platform == "win32"
  168. and isinstance(exc, RuntimeError)
  169. and "Cannot acquire lock after 20 attempts" in str(exc)
  170. )
  171. _windows_lock_timeout_warned = False
  172. _windows_lock_timeout_warn_lock = threading.Lock()
  173. def _warn_windows_lock_timeout_once() -> None:
  174. """Report a suppressed CLH lock timeout exactly once per process.
  175. Every emit after the first failure raises the same RuntimeError, so the
  176. warning must be one-shot or it would spam errors.log as badly as the
  177. stderr noise it replaces. CLH does not chain the underlying cause (the
  178. RuntimeError is raised outside the except, from the retry loop's else
  179. clause), so the message cannot include it; a constantly repeating timeout
  180. means file logging is degraded — the startup portalocker probe in this
  181. module should have caught a dead portalocker and fallen back already.
  182. """
  183. global _windows_lock_timeout_warned
  184. with _windows_lock_timeout_warn_lock:
  185. if _windows_lock_timeout_warned:
  186. return
  187. _windows_lock_timeout_warned = True
  188. logging.getLogger("hermes_logging").warning(
  189. "concurrent-log-handler timed out acquiring the cross-process log "
  190. "lock; this and later records were dropped (the Desktop slash-worker "
  191. "surface stays clean, but file logging is degraded)."
  192. )
  193. def _is_unavailable_log_stream(exc: BaseException | None) -> bool:
  194. """True when a file handler lost its backing stream during teardown or I/O."""
  195. return (
  196. (isinstance(exc, OSError) and exc.errno == 5)
  197. or (isinstance(exc, ValueError) and "closed file" in str(exc).lower())
  198. )
  199. # Third-party loggers that are noisy at DEBUG/INFO level.
  200. _NOISY_LOGGERS = (
  201. "openai", "openai._base_client", "httpx", "httpcore", "asyncio", "hpack", "hpack.hpack",
  202. "grpc", "modal", "urllib3", "urllib3.connectionpool", "websockets", "charset_normalizer",
  203. "markdown_it",
  204. )
  205. def _quiet_noisy_loggers() -> None:
  206. """Pin noisy third-party loggers at WARNING."""
  207. for name in _NOISY_LOGGERS:
  208. logging.getLogger(name).setLevel(logging.WARNING)
  209. def set_session_context(session_id: str) -> None:
  210. """Set the session ID for the current thread."""
  211. _session_context.session_id = session_id
  212. def clear_session_context() -> None:
  213. """Clear the session ID for the current thread."""
  214. _session_context.session_id = None
  215. def _install_session_record_factory() -> None:
  216. """Replace the global LogRecord factory with one that adds ``session_tag``.
  217. Unlike a Filter, the record factory runs for EVERY record in the process (propagated
  218. and third-party-handled ones included), so ``%(session_tag)s`` never KeyErrors.
  219. Idempotent via a marker attribute.
  220. """
  221. current_factory = logging.getLogRecordFactory()
  222. if getattr(current_factory, "_hermes_session_injector", False):
  223. return
  224. def _session_record_factory(*args, **kwargs):
  225. record = current_factory(*args, **kwargs)
  226. sid = getattr(_session_context, "session_id", None)
  227. record.session_tag = f" [{sid}]" if sid else "" # type: ignore[attr-defined]
  228. # QueueListener formats on its own thread, after the profile-scoped
  229. # ContextVar is gone; keep the resolved home on the record so a
  230. # multiplex desktop ticker can route to the job owner's files (#97489).
  231. try:
  232. record.hermes_home = str(get_hermes_home().resolve()) # type: ignore[attr-defined]
  233. except Exception:
  234. record.hermes_home = "" # type: ignore[attr-defined]
  235. return record
  236. _session_record_factory._hermes_session_injector = True # type: ignore[attr-defined]
  237. logging.setLogRecordFactory(_session_record_factory)
  238. # Install on import so session_tag exists on all records even before setup_logging().
  239. _install_session_record_factory()
  240. class _ComponentFilter(logging.Filter):
  241. """Only pass records whose logger name starts with one of *prefixes*."""
  242. def __init__(self, prefixes: Sequence[str]) -> None:
  243. super().__init__()
  244. self._prefixes = tuple(prefixes)
  245. def filter(self, record: logging.LogRecord) -> bool:
  246. return record.name.startswith(self._prefixes)
  247. # Logger name prefixes per component; used by _ComponentFilter and ``hermes logs --component``.
  248. COMPONENT_PREFIXES = {
  249. # ``plugins.platforms``: messaging adapters that migrated out of
  250. # ``gateway/platforms/`` into bundled plugins (#41112) are still gateway
  251. # components and belong in gateway.log. ``agent.memory_caller_scope``: a multiplexed gateway's
  252. # memory-provider profile refusals (#140058), which the operator reads in gateway.log.
  253. "gateway": ("gateway", "hermes_plugins", "plugins.platforms", "agent.memory_caller_scope"),
  254. "agent": ("agent", "run_agent", "model_tools", "batch_runner"),
  255. "tools": ("tools",),
  256. "cli": ("hermes_cli", "cli"),
  257. "cron": ("cron",),
  258. "gui": ("hermes_cli.web_server", "hermes_cli.pty_bridge", "hermes_cli.desktop", "tui_gateway", "uvicorn"),
  259. }
  260. def _known_log_homes() -> set[Path]:
  261. """Homes the queued file handlers already serve: static handlers by their file, routers by
  262. their default home plus every profile home they route. Caller holds ``_queue_state_lock``."""
  263. homes: set[Path] = set()
  264. for handler in _queued_file_handlers:
  265. if isinstance(handler, _ProfileRoutingFileHandler):
  266. homes.add(handler._default_home)
  267. homes.update(handler._profile_homes)
  268. elif isinstance(handler, RotatingFileHandler):
  269. try:
  270. homes.add(Path(handler.baseFilename).resolve().parent.parent)
  271. except (TypeError, ValueError, OSError):
  272. continue
  273. return homes
  274. def _adopt_secondary_home(home: Path) -> bool:
  275. """Route *home*'s records to its own files when this process already logs for another home.
  276. Enables profile routing for the union of homes (or widens the live routers); False when
  277. *home* is the first home seen or is already served."""
  278. try:
  279. resolved = Path(home).expanduser().resolve()
  280. except (TypeError, ValueError, OSError):
  281. return False
  282. with _queue_state_lock:
  283. known = _known_log_homes()
  284. if not known or resolved in known:
  285. return False
  286. return enable_profile_log_routing([*sorted(known), resolved])
  287. def setup_logging(
  288. *,
  289. hermes_home: Optional[Path] = None,
  290. log_level: Optional[str] = None,
  291. max_size_mb: Optional[int] = None,
  292. backup_count: Optional[int] = None,
  293. mode: Optional[str] = None,
  294. force: bool = False,
  295. ) -> Path:
  296. """Configure the Hermes logging subsystem; returns the ``logs/`` directory.
  297. Safe to call multiple times; the second call is a no-op unless *force*. Level and
  298. rotation defaults come from config.yaml ``logging.*``. ``mode="gateway"`` adds
  299. ``gateway.log`` and ``mode="gui"`` adds ``gui.log``.
  300. """
  301. global _logging_initialized
  302. global _fallback_warned
  303. home = hermes_home or get_hermes_home()
  304. log_dir = mkdir_under_hermes_home(home / "logs")
  305. # Stdout is block-buffered when piped (no TTY); line-buffer it so a
  306. # headless supervisor's log stream tracks the agent loop incrementally
  307. # (#92281). Runs before the initialized check so every entry mode that
  308. # reaches this function gets it; a no-op once this stdout is line-buffered.
  309. _line_buffer_piped_stdout()
  310. # A second Hermes home in a process that already logs for another one — a dashboard or
  311. # ``hermes serve`` backend building agents for several profiles, a multiplexed gateway —
  312. # gets routed by record home. Stacking another file handler here would hand it EVERY
  313. # profile's records (the handlers carry no home filter), and a duplicate writer on top of
  314. # an existing router.
  315. if _adopt_secondary_home(home):
  316. return log_dir
  317. cfg_level, cfg_max_size, cfg_backup = _read_logging_config()
  318. level_name = (log_level or cfg_level or "INFO").upper()
  319. level = getattr(logging, level_name, logging.INFO)
  320. max_bytes = (max_size_mb or cfg_max_size or 5) * 1024 * 1024
  321. backups = backup_count or cfg_backup or 3
  322. from agent.redact import RedactingFormatter # lazy: circular at module load
  323. root = logging.getLogger()
  324. # (filename, level, max_bytes, backup_count, component) — a component gates
  325. # the file on ``mode`` and restricts it to that component's logger prefixes.
  326. handler_specs = (
  327. ("agent.log", level, max_bytes, backups, None),
  328. ("errors.log", logging.WARNING, 2 * 1024 * 1024, 2, None),
  329. ("gateway.log", logging.INFO, 5 * 1024 * 1024, 3, "gateway"),
  330. ("gui.log", logging.INFO, 10 * 1024 * 1024, 5, "gui"),
  331. )
  332. for filename, lvl, size, count, component in handler_specs:
  333. if component is not None and mode != component:
  334. continue
  335. _add_rotating_handler(
  336. log_dir / filename, level=lvl, max_bytes=size, backup_count=count,
  337. formatter=RedactingFormatter(_LOG_FORMAT),
  338. log_filter=_ComponentFilter(COMPONENT_PREFIXES[component]) if component else None,
  339. )
  340. if _WINDOWS_CLH_FALLBACK and not _fallback_warned:
  341. # One-shot, and the file handlers above are already live, so this lands
  342. # in errors.log/agent.log — the fallback must never be invisible again.
  343. _fallback_warned = True
  344. logging.getLogger("hermes_logging").warning(
  345. "concurrent-log-handler unavailable on this Windows install (%s); "
  346. "file logging fell back to stdlib rotation without rollover.",
  347. _WINDOWS_CLH_FALLBACK_REASON or "portalocker probe failed",
  348. )
  349. if _logging_initialized and not force:
  350. return log_dir
  351. # Root level must be low enough for the handlers to fire.
  352. if root.level == logging.NOTSET or root.level > level:
  353. root.setLevel(level)
  354. _quiet_noisy_loggers()
  355. _logging_initialized = True
  356. return log_dir
  357. def setup_verbose_logging() -> None:
  358. """Enable DEBUG-level console logging for ``--verbose`` / ``-v`` mode."""
  359. from agent.redact import RedactingFormatter
  360. root = logging.getLogger()
  361. if any(getattr(h, "_hermes_verbose", False) for h in root.handlers):
  362. return
  363. handler = logging.StreamHandler(_safe_stderr())
  364. handler.setLevel(logging.DEBUG)
  365. handler.setFormatter(RedactingFormatter(_LOG_FORMAT_VERBOSE, datefmt="%H:%M:%S"))
  366. handler._hermes_verbose = True # type: ignore[attr-defined]
  367. root.addHandler(handler)
  368. if root.level > logging.DEBUG:
  369. root.setLevel(logging.DEBUG)
  370. _quiet_noisy_loggers()
  371. # rex-deploy at INFO for sandbox status.
  372. logging.getLogger("rex-deploy").setLevel(logging.INFO)
  373. def _quietly(fn) -> None:
  374. """Call *fn* (a ``close``/``stop`` bound method) swallowing errors — teardown must never raise."""
  375. try:
  376. fn()
  377. except Exception:
  378. pass
  379. class _ManagedRotatingFileHandler(RotatingFileHandler):
  380. """RotatingFileHandler with managed-mode perms and external-rotation detection.
  381. In managed mode (NixOS) the setgid stateDir needs group-readable files, but
  382. ``_open()``/``doRollover()`` honor the umask (0644), so ``chmod 0660`` follows both.
  383. A rotating handler also holds an fd: if the file is rotated externally (logrotate,
  384. ``mv``) writes silently go to the old inode, so each emit compares the path's inode
  385. to the open stream's and reopens on mismatch (the ``WatchedFileHandler`` pattern).
  386. """
  387. def __init__(self, *args, **kwargs):
  388. from hermes_cli.config import is_managed
  389. self._managed = is_managed()
  390. self._unavailable_reported = False
  391. # Set by _ProfileRoutingFileHandler: a vanished logs/ dir means the profile was deleted,
  392. # so skip the write (the router re-routes the record) instead of a FileNotFoundError traceback.
  393. self._skip_write_when_dir_missing = False
  394. self.skipped_missing_dir = False
  395. # Set by _ProfileRoutingFileHandler: once closed (router close, logging.shutdown, a released
  396. # profile) never reopen. The stdlib reopens an append-mode file on emit after close.
  397. self._never_reopen_after_close = False
  398. super().__init__(*args, **kwargs)
  399. self._record_stream_stat()
  400. def _chmod_if_managed(self):
  401. if self._managed:
  402. try:
  403. os.chmod(self.baseFilename, 0o660)
  404. except OSError:
  405. pass
  406. def _record_stream_stat(self, st: Optional[os.stat_result] = None) -> None:
  407. """Snapshot dev/ino of ``baseFilename`` so emit() can detect external rotation."""
  408. try:
  409. st = st or os.stat(self.baseFilename)
  410. self._stat_dev, self._stat_ino = st.st_dev, st.st_ino
  411. except OSError:
  412. self._stat_dev, self._stat_ino = None, None
  413. def _reopen_stream(self, stat_result=None) -> None:
  414. """Close and reopen ``baseFilename`` (best-effort).
  415. On failure the stream is left ``None`` so the next emit bails rather than
  416. writing to a stale inode.
  417. """
  418. if self.stream is not None:
  419. _quietly(self.stream.close)
  420. self.stream = None # type: ignore[assignment]
  421. try:
  422. self.stream = self._open()
  423. except Exception:
  424. return
  425. self._record_stream_stat(stat_result)
  426. def _reopen_if_externally_rotated(self) -> None:
  427. """Reopen when ``baseFilename`` was renamed, unlinked, or replaced by another inode.
  428. Silent + best-effort: any error falls back to the existing (possibly stale)
  429. stream so logging keeps working instead of dying on a stat failure.
  430. """
  431. try:
  432. st = os.stat(self.baseFilename)
  433. except FileNotFoundError:
  434. self._reopen_stream() # rotated/unlinked underneath us: recreate at the path
  435. return
  436. except OSError:
  437. return # transient — try again on the next emit
  438. if self._stat_dev is None or self._stat_ino is None:
  439. self._record_stream_stat(st)
  440. elif (st.st_dev, st.st_ino) != (self._stat_dev, self._stat_ino):
  441. self._reopen_stream(st)
  442. def emit(self, record: logging.LogRecord) -> None:
  443. self.skipped_missing_dir = False
  444. # Under this handler's own lock (Handler.handle), which close() also takes: a record that
  445. # picked this handler before its router retired it is dropped, never written to a reopened
  446. # file (on Windows that reopened file keeps the profile's logs/ locked until exit).
  447. if self._never_reopen_after_close and getattr(self, "_closed", False):
  448. return
  449. had_stream = self.stream is not None
  450. # The kernel caches inode metadata, so this stat is sub-microsecond on a hot file.
  451. file_exists = had_stream or os.path.exists(self.baseFilename)
  452. if file_exists:
  453. self._reopen_if_externally_rotated()
  454. # Only when the file is gone (or its reopen just failed), so a healthy emit pays no extra
  455. # stat. ``stream is None`` alone is not a signal: on Windows concurrent-log-handler closes
  456. # the stream after every write.
  457. if (self.stream is None and (had_stream or not file_exists) and self._skip_write_when_dir_missing
  458. and not os.path.isdir(os.path.dirname(self.baseFilename))):
  459. self.skipped_missing_dir = True
  460. return
  461. super().emit(record)
  462. # A record actually reached the file: only now has the destination recovered. Resetting
  463. # in _open() is wrong — open() succeeds on a device whose write/flush still raise EIO,
  464. # which re-armed the report and printed the path once per record.
  465. if self.stream is not None:
  466. self._unavailable_reported = False
  467. def handleError(self, record: logging.LogRecord) -> None:
  468. """Suppress the known Windows ``concurrent-log-handler`` lock timeout.
  469. CLH's ``emit()`` routes that RuntimeError here, so this is the single point to
  470. silence it before stdlib prints to stderr (which the Desktop slash-worker
  471. captures into chat output). Silencing is not silent: warn once through the
  472. logging system so a wedged lock is visible in the logs instead of a black hole.
  473. """
  474. exc = sys.exc_info()[1]
  475. if _is_windows_concurrent_log_lock_timeout(exc):
  476. _warn_windows_lock_timeout_once()
  477. return
  478. if _is_unavailable_log_stream(exc):
  479. # The QueueListener must not turn a failing log destination into a traceback for
  480. # every queued record. Name the path once, drop the stale stream; the next emit
  481. # reopens it if the destination has recovered.
  482. if not self._unavailable_reported:
  483. self._unavailable_reported = True
  484. _quietly(lambda: print(
  485. f"hermes_logging: {self.baseFilename} unavailable ({exc}); "
  486. "file logging paused until it recovers", file=_safe_stderr()))
  487. if self.stream is not None:
  488. _quietly(self.stream.close)
  489. self.stream = None # type: ignore[assignment]
  490. return
  491. super().handleError(record)
  492. def _open(self):
  493. stream = super()._open()
  494. self._chmod_if_managed()
  495. return stream
  496. def doRollover(self):
  497. # The stdlib rollover opens a fresh baseFilename owned by whichever process crossed
  498. # maxBytes. With one rotating handler per profile that is usually the long-lived root
  499. # gateway, and a worker on another uid can never reopen its own agent.log (#120151).
  500. # Only root can hand the file back; an unprivileged process never changed the owner.
  501. try:
  502. previous = os.stat(self.baseFilename)
  503. except OSError:
  504. previous = None
  505. super().doRollover()
  506. if previous is not None:
  507. try:
  508. if getattr(os, "geteuid", lambda: -1)() == 0:
  509. os.chown(self.baseFilename, previous.st_uid, previous.st_gid)
  510. os.chmod(self.baseFilename, previous.st_mode & 0o7777)
  511. except OSError:
  512. pass # a log that cannot be chowned is still a working log
  513. self._chmod_if_managed()
  514. # Our own rollover writes a new baseFilename; refresh the snapshot so
  515. # the next emit doesn't mistake it for external rotation.
  516. self._record_stream_stat()
  517. def _new_file_handler(
  518. path: Path, *, level: int, max_bytes: int, backup_count: int, formatter
  519. ) -> _ManagedRotatingFileHandler:
  520. """Create the ``logs/`` directory and a configured ``_ManagedRotatingFileHandler``."""
  521. mkdir_under_hermes_home(path.parent)
  522. if _WINDOWS_CLH_FALLBACK:
  523. # stdlib fallback: no rollover, or the file pins at the size threshold
  524. # and every emit re-triggers the WinError 32 rename failure (#44873).
  525. max_bytes, backup_count = 0, 0
  526. handler = _ManagedRotatingFileHandler(
  527. str(path), maxBytes=max_bytes, backupCount=backup_count, encoding="utf-8"
  528. )
  529. handler.setLevel(level)
  530. handler.setFormatter(formatter)
  531. return handler
  532. # A routed profile home is re-checked for an out-of-band delete (missing dir or tombstone) at
  533. # most this often. WHY: the check is two stats per record per router on the listener thread,
  534. # nearly doubling the syscalls of every live-profile emit; a deleted home only needs catching
  535. # within a couple of seconds. Inside that window a tombstoned (not yet removed) home still gets
  536. # its records; an rmtree is caught at once by the write that finds the directory gone.
  537. _PROFILE_LIVENESS_RECHECK_S = 2.0
  538. class _ProfileRoutingFileHandler(logging.Handler):
  539. """Route queued records to the log file for their Hermes home.
  540. Used only behind the QueueListener, so its small routing lock never blocks an agent
  541. or dashboard event loop. Per-home handlers keep rotation, redaction and managed perms.
  542. """
  543. def __init__(self, existing: RotatingFileHandler, profile_homes: Sequence[Path]) -> None:
  544. """Take over *existing*'s path, level, rotation, formatter and filters."""
  545. super().__init__(level=existing.level)
  546. resolved = Path(existing.baseFilename).resolve()
  547. self.baseFilename = str(resolved)
  548. self._hermes_routed_log_path = resolved
  549. self._default_home = resolved.parent.parent.resolve()
  550. self._profile_homes = {Path(home).expanduser().resolve() for home in profile_homes}
  551. self._filename = resolved.name
  552. self._max_bytes = getattr(existing, "maxBytes", 0)
  553. self._backup_count = getattr(existing, "backupCount", 0)
  554. self._profile_handlers: dict[Path, _ManagedRotatingFileHandler] = {}
  555. self._profile_handlers_lock = threading.RLock()
  556. self._router_closed = False # under _profile_handlers_lock; close() is final
  557. # Home -> monotonic time of its last liveness check (see _PROFILE_LIVENESS_RECHECK_S).
  558. self._liveness_checked_at: dict[Path, float] = {}
  559. self.setFormatter(existing.formatter)
  560. for log_filter in existing.filters:
  561. self.addFilter(log_filter)
  562. def _home_for_record(self, record: logging.LogRecord) -> Path:
  563. raw_home = getattr(record, "hermes_home", "")
  564. try:
  565. candidate = Path(raw_home).expanduser().resolve()
  566. except (TypeError, ValueError, OSError):
  567. candidate = self._default_home
  568. if candidate == self._default_home or candidate not in self._profile_homes:
  569. return self._default_home
  570. last = self._liveness_checked_at.get(candidate)
  571. if last is None or _monotonic() - last >= _PROFILE_LIVENESS_RECHECK_S:
  572. # First sighting always checks, so a home deleted before it ever logged is caught on
  573. # its first record; afterwards at most once per interval. An rmtree inside the
  574. # interval is caught by emit() on the record whose write finds the dir gone.
  575. if self._release_if_deleted(candidate):
  576. return self._default_home
  577. return candidate
  578. def _release_if_deleted(self, home: Path) -> bool:
  579. """Release *home* if it was deleted out of band; True when it was."""
  580. self._liveness_checked_at[home] = _monotonic()
  581. if home.is_dir() and not named_profile_is_deleted(home):
  582. return False
  583. # Deleted out of band (CLI ``hermes profile delete`` while this process runs; it
  584. # tombstones before rmtree). The startup snapshot still names it, so its handler would
  585. # retry the vanished logs/ path on every record (#103777). Release it the way an
  586. # in-process delete does: the stale fd is closed on this routed record and the home is
  587. # dropped from the routing set, so later records fall back to the default home. This
  588. # does not reach into another process: a CLI delete racing a live server's open fd
  589. # still relies on release_profile_log_handlers in-process and #130285.
  590. self.release_profile(home)
  591. return True
  592. def _handler_for_home(self, home: Path) -> Optional[_ManagedRotatingFileHandler]:
  593. """The open handler for *home*; ``None`` once the router or that home is retired."""
  594. with self._profile_handlers_lock:
  595. handler = self._profile_handlers.get(home)
  596. if handler is not None:
  597. return handler
  598. # Built with no router lock held: ``Handler.__init__`` takes ``logging._lock``, and
  599. # ``logging.shutdown`` (uvicorn's ``dictConfig``) holds that lock while it acquires each
  600. # handler's lock and closes it (``close`` takes ``_profile_handlers_lock``). Holding
  601. # either router lock here deadlocked the QueueListener against ``dictConfig`` (T23).
  602. built = _new_file_handler(
  603. home / "logs" / self._filename, level=self.level, max_bytes=self._max_bytes,
  604. backup_count=self._backup_count, formatter=self.formatter,
  605. )
  606. built._skip_write_when_dir_missing = home != self._default_home
  607. built._never_reopen_after_close = True
  608. with self._profile_handlers_lock:
  609. # Closed (logging.shutdown) or released while this was built: publishing now would leak
  610. # the new handler's fd into a router nothing will close again.
  611. retired = self._router_closed or (home != self._default_home and home not in self._profile_homes)
  612. handler = None if retired else self._profile_handlers.setdefault(home, built)
  613. if handler is not built: # retired, or another thread opened it first
  614. _quietly(built.close)
  615. return handler
  616. def handle(self, record: logging.LogRecord):
  617. """``Handler.handle`` without this router's own I/O lock: each per-home handler takes its
  618. own lock to write, and routing state has ``_profile_handlers_lock``. Holding ``self.lock``
  619. across ``emit`` (which can open a handler, taking ``logging._lock``) inverted the order
  620. ``logging.shutdown`` uses (module lock, then handler lock) (T23)."""
  621. rv = self.filter(record)
  622. if isinstance(rv, logging.LogRecord):
  623. record = rv
  624. if rv:
  625. self.emit(record)
  626. return rv
  627. def emit(self, record: logging.LogRecord) -> None:
  628. try:
  629. home = self._home_for_record(record)
  630. handler = self._handler_for_home(home)
  631. if handler is None and home != self._default_home: # released in flight
  632. home, handler = self._default_home, self._handler_for_home(self._default_home)
  633. if handler is None: # the router is closed
  634. return
  635. if home == self._default_home:
  636. handler.handle(record)
  637. return
  638. # Formatted here, on the listener thread, where the record's profile scope is gone: bind its home so
  639. # RedactingFormatter applies THAT profile's redact_secrets policy and vault values, not the launch's.
  640. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  641. token = set_hermes_home_override(str(home))
  642. try:
  643. handler.handle(record)
  644. finally:
  645. reset_hermes_home_override(token)
  646. # The write was skipped because logs/ is gone (an rmtree inside the liveness interval,
  647. # see _ManagedRotatingFileHandler.emit): re-check now, and keep the record in the
  648. # default log either way (a live profile whose logs/ was removed by hand included).
  649. if handler.skipped_missing_dir:
  650. self._release_if_deleted(home)
  651. if (fallback := self._handler_for_home(self._default_home)) is not None:
  652. fallback.handle(record)
  653. except Exception:
  654. self.handleError(record)
  655. def close(self) -> None:
  656. with self._profile_handlers_lock:
  657. self._router_closed = True
  658. handlers = list(self._profile_handlers.values())
  659. self._profile_handlers.clear()
  660. for handler in handlers:
  661. _quietly(handler.close)
  662. super().close()
  663. def release_profile(self, home: Path) -> bool:
  664. """Close and forget the routed files belonging to a deleted profile."""
  665. with self._profile_handlers_lock:
  666. handler = self._profile_handlers.pop(home, None)
  667. self._profile_homes.discard(home)
  668. self._liveness_checked_at.pop(home, None)
  669. if handler is None:
  670. return False
  671. _quietly(handler.close)
  672. return True
  673. # Asynchronous file logging: an ``emit`` can block on the cross-process
  674. # rotation lock (module header); on an asyncio thread that stalls the loop and
  675. # drops WebSocket clients. Every file handler is therefore driven by a single
  676. # QueueListener thread; loggers only do a non-blocking enqueue.
  677. _log_queue: Optional[queue.SimpleQueue] = None
  678. _queue_listener: Optional[QueueListener] = None
  679. _queued_file_handlers: list = []
  680. _queue_atexit_registered = False
  681. # Guards every read-modify-write of the four globals above. setup_logging()
  682. # holds no lock and its _logging_initialized guard runs AFTER handler
  683. # registration, so _register_queued_handler() can race a flush/reset from
  684. # another thread (gateway init vs a plugin/CLI path); without this, two
  685. # threads can interleave stop()/reassign/start() and leave two live listeners.
  686. _queue_state_lock = threading.Lock()
  687. class _NonFormattingQueueHandler(QueueHandler):
  688. """``QueueHandler`` for an in-process queue.
  689. Stdlib ``prepare()`` formats and strips ``args``/``exc_info`` for cross-process
  690. pickling; ours is in-process, so targets get the unformatted record and apply their
  691. own ``RedactingFormatter`` on the listener thread. A shallow copy is returned because
  692. the emitting thread's synchronous handlers may mutate ``record.message`` meanwhile.
  693. """
  694. def prepare(self, record: logging.LogRecord) -> logging.LogRecord:
  695. return copy.copy(record)
  696. def _stop_queue_listener() -> None:
  697. """Flush and stop the background log listener (idempotent; atexit hook, so it takes the lock)."""
  698. global _queue_listener
  699. with _queue_state_lock:
  700. listener, _queue_listener = _queue_listener, None
  701. if listener is not None:
  702. _quietly(listener.stop)
  703. def _start_queue_listener_locked() -> None:
  704. """(Re)build + start a listener over the current handler set (``_queue_state_lock`` held).
  705. A running listener is stopped first; this only happens while handlers are being
  706. added (queue empty), so ``stop()`` returns immediately.
  707. """
  708. global _queue_listener
  709. if _queue_listener is not None:
  710. _queue_listener.stop()
  711. _queue_listener = QueueListener(_log_queue, *_queued_file_handlers, respect_handler_level=True)
  712. _queue_listener.start()
  713. def _register_queued_handler(handler: logging.Handler) -> None:
  714. """Route *handler* through the shared async queue instead of attaching it to root.
  715. Emitting threads never block on file I/O or the rotation lock; the ``QueueListener``
  716. applies each handler's own level and filters on its worker thread.
  717. """
  718. global _log_queue, _queue_atexit_registered
  719. with _queue_state_lock:
  720. if _log_queue is None:
  721. _log_queue = queue.SimpleQueue()
  722. qh = _NonFormattingQueueHandler(_log_queue)
  723. qh._hermes_queue = True # type: ignore[attr-defined]
  724. # Always on the root logger so records from any logger reach the queue.
  725. logging.getLogger().addHandler(qh)
  726. _queued_file_handlers.append(handler)
  727. _start_queue_listener_locked()
  728. if not _queue_atexit_registered:
  729. # Runs before logging.shutdown (registered earlier at import time),
  730. # so the listener stops before its file handlers are closed.
  731. atexit.register(_stop_queue_listener)
  732. _queue_atexit_registered = True
  733. def flush_log_queue() -> None:
  734. """Block until all queued records have been written, then resume.
  735. Stops the listener (which processes every pending record before joining) and
  736. restarts it. ``stop()`` joins the worker thread — do NOT call this on a hard-exit
  737. path where the listener may be wedged on the rotation lock; use
  738. ``drain_log_queue()`` there, which bounds the wait.
  739. """
  740. with _queue_state_lock:
  741. listener = _queue_listener
  742. if listener is not None:
  743. listener.stop()
  744. listener.start()
  745. def drain_log_queue(timeout: float = 1.0) -> None:
  746. """Best-effort, time-bounded drain for hard-exit paths (no restart).
  747. If the listener's worker is wedged on the cross-process rotation lock — the very
  748. failure async logging exists to survive — an unbounded join would re-freeze shutdown.
  749. """
  750. listener = _queue_listener
  751. if listener is None:
  752. return
  753. t = threading.Thread(target=lambda: _quietly(listener.stop), name="hermes-log-drain", daemon=True)
  754. t.start()
  755. t.join(timeout)
  756. def release_profile_log_handlers(profile_home: str | Path) -> int:
  757. """Release this process's routed log files below a profile before it is removed.
  758. The Desktop serve process can route records for several profiles through one
  759. ``QueueListener``. On Windows, each routed concurrent log handler keeps its
  760. lock file open, so closing only external profile resources still leaves
  761. ``logs/.__agent.lock`` and ``logs/.__errors.lock`` unavailable to rmtree.
  762. """
  763. global _queue_listener
  764. try:
  765. home = Path(profile_home).expanduser().resolve()
  766. except (TypeError, ValueError, OSError):
  767. return 0
  768. with _queue_state_lock:
  769. listener = _queue_listener
  770. if listener is not None:
  771. listener.stop()
  772. _queue_listener = None
  773. released = sum(
  774. handler.release_profile(home)
  775. for handler in _queued_file_handlers
  776. if isinstance(handler, _ProfileRoutingFileHandler)
  777. )
  778. if listener is not None:
  779. _start_queue_listener_locked()
  780. return released
  781. def enable_profile_log_routing(profile_homes: Sequence[str | Path]) -> bool:
  782. """Make the queued file logs follow a desktop profile context.
  783. ``setup_logging`` binds handlers to one process home; the desktop dashboard's
  784. embedded cron ticker may run jobs for every profile, so its static file handlers
  785. are replaced with profile routers once the profile list is known. Returns ``True``
  786. when routing is (or already was) enabled; a single-profile caller is left untouched.
  787. """
  788. global _queue_listener
  789. homes: list[Path] = []
  790. for entry in profile_homes:
  791. try:
  792. resolved = Path(entry[1] if isinstance(entry, tuple) else entry).expanduser().resolve()
  793. except (TypeError, ValueError, OSError):
  794. continue
  795. if resolved not in homes:
  796. homes.append(resolved)
  797. if len(homes) < 2:
  798. return False
  799. # Each routed profile's redaction policy, read now: the formatter never reads config per record.
  800. from agent.redact import resolve_profile_policy
  801. for home in homes:
  802. resolve_profile_policy(home)
  803. with _queue_state_lock:
  804. if not _queued_file_handlers:
  805. return False
  806. routers = [h for h in _queued_file_handlers if isinstance(h, _ProfileRoutingFileHandler)]
  807. if routers:
  808. for handler in routers:
  809. with handler._profile_handlers_lock:
  810. handler._profile_homes = handler._profile_homes.union(homes)
  811. return True
  812. listener = _queue_listener
  813. if listener is not None:
  814. listener.stop()
  815. _queue_listener = None
  816. replacement = []
  817. for existing in _queued_file_handlers:
  818. if isinstance(existing, RotatingFileHandler):
  819. replacement.append(_ProfileRoutingFileHandler(existing, homes))
  820. _quietly(existing.close)
  821. else:
  822. replacement.append(existing)
  823. _queued_file_handlers[:] = replacement
  824. if listener is not None:
  825. _start_queue_listener_locked()
  826. return True
  827. def _reset_queued_handlers() -> None:
  828. """Tear down the async logging queue + listener (test-isolation helper)."""
  829. global _log_queue
  830. _stop_queue_listener()
  831. with _queue_state_lock:
  832. root = logging.getLogger()
  833. for h in list(root.handlers):
  834. if getattr(h, "_hermes_queue", False):
  835. root.removeHandler(h)
  836. for h in _queued_file_handlers:
  837. _quietly(h.close)
  838. _queued_file_handlers.clear()
  839. _log_queue = None
  840. def _add_rotating_handler(
  841. path: Path,
  842. *,
  843. level: int,
  844. max_bytes: int,
  845. backup_count: int,
  846. formatter: logging.Formatter,
  847. log_filter: Optional[logging.Filter] = None,
  848. ) -> None:
  849. """Register a queued ``RotatingFileHandler`` for *path*; idempotent per resolved path."""
  850. resolved = path.resolve()
  851. for existing in _queued_file_handlers:
  852. # Already attached directly, or already covered by the profile router — for its default
  853. # home or any profile home it routes (a bare handler beside it would take every record).
  854. if getattr(existing, "_hermes_routed_log_path", None) == resolved or (
  855. isinstance(existing, RotatingFileHandler)
  856. and Path(getattr(existing, "baseFilename", "")).resolve() == resolved
  857. ):
  858. return
  859. if isinstance(existing, _ProfileRoutingFileHandler) and existing._filename == resolved.name and (
  860. resolved.parent.parent == existing._default_home or resolved.parent.parent in existing._profile_homes
  861. ):
  862. return
  863. handler = _new_file_handler(
  864. path, level=level, max_bytes=max_bytes, backup_count=backup_count, formatter=formatter,
  865. )
  866. if log_filter is not None:
  867. handler.addFilter(log_filter)
  868. # Routing already on (a second home adopted earlier): a component log added now —
  869. # ``mode="gateway"`` after the fact — must route too, or it takes every home's records.
  870. routers = [h for h in _queued_file_handlers if isinstance(h, _ProfileRoutingFileHandler)]
  871. if routers:
  872. homes: set[Path] = set()
  873. for router in routers:
  874. homes.add(router._default_home)
  875. homes.update(router._profile_homes)
  876. routed = _ProfileRoutingFileHandler(handler, sorted(homes))
  877. _quietly(handler.close)
  878. handler = routed
  879. # Queue, not ``addHandler``: the rotation-lock wait never runs on the caller's thread.
  880. _register_queued_handler(handler)
  881. def _read_logging_config():
  882. """Best-effort read of ``logging.*`` from config.yaml."""
  883. try:
  884. # Prefer the shared effective-config cache (managed overlay included, so an administrator
  885. # can pin logging.*) so this reuses hermes_cli.main's early parse (one config.yaml parse
  886. # per process); fall back to a direct parse for bare hermes_logging consumers.
  887. try:
  888. from hermes_cli.config_effective import load_user_config_effective
  889. cfg = load_user_config_effective(get_config_path())
  890. except Exception:
  891. from hermes_cli.config_backend import config_exists, read_config_doc
  892. config_path = get_config_path()
  893. cfg = (read_config_doc(config_path) or {}) if config_exists(config_path) else {}
  894. if not cfg:
  895. return (None, None, None)
  896. log_cfg = cfg.get("logging", {})
  897. if isinstance(log_cfg, dict):
  898. return (log_cfg.get("level"), log_cfg.get("max_size_mb"), log_cfg.get("backup_count"))
  899. except Exception:
  900. pass
  901. return (None, None, None)