quiet_single_query.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. """Quiet ``hermes chat -Q`` helpers: bind this session's key and resume nested notifies.
  2. Bot Mode delivers a local DM as ``hermes -p <bot> chat -Q --query-file``. Interactive
  3. chat binds ``set_current_session_key(self.session_id)`` around the turn; the quiet
  4. path did not, so a nested ``message_agent`` notify inherited the dispatcher's
  5. ``HERMES_SESSION_KEY`` and never woke the recipient. Quiet also printed and exited
  6. after one turn, so a nested teammate reply that finished during the one-shot linger
  7. was never injected as a follow-up.
  8. """
  9. from __future__ import annotations
  10. import contextlib
  11. import json
  12. import os
  13. import subprocess
  14. import sys
  15. import threading
  16. import time
  17. from typing import Any, Callable, MutableMapping
  18. # Nested A→B→C is one extra turn; this caps a runaway message_agent chain.
  19. _MAX_QUIET_NOTIFY_ROUNDS = 8
  20. # Last line a Kanban worker leaves in its own log: ``[kanban-worker-exit] rc=<code>``. A per-tick
  21. # ``hermes kanban dispatch`` process never reaped the worker, so ``os.waitpid`` cannot tell it how
  22. # the worker exited; the trailer is the process-independent witness the dead-worker sweep reads
  23. # instead, so a clean exit without a terminal board call is booked as the same protocol violation
  24. # (and a 75 as the same rate-limit requeue) whichever process notices the death.
  25. KANBAN_WORKER_EXIT_TRAILER = "[kanban-worker-exit] rc="
  26. def exit_single_query(code: int) -> None:
  27. """``sys.exit(code)`` for a one-shot turn; a Kanban worker first writes the exit trailer to its log."""
  28. if os.environ.get("HERMES_KANBAN_TASK"):
  29. with contextlib.suppress(Exception):
  30. # stderr: stdout may be the ``--stream-json`` record stream, and the worker log
  31. # captures both streams.
  32. print(f"\n{KANBAN_WORKER_EXIT_TRAILER}{int(code)}", file=sys.stderr, flush=True)
  33. sys.exit(code)
  34. # A spawner that bounds only the TURN (the cron Bot Chat lane) hands the quiet child a report
  35. # path here. The child records the turn's outcome there the moment the turn ends, BEFORE the
  36. # one-shot exit linger, so the spawner can book the delivery and stop waiting while the linger
  37. # keeps protecting nested ``notify_on_complete`` replies. Popped before the turn runs (same
  38. # contract as HERMES_TURN_AUTHOR): nothing the turn spawns inherits it, and a nested one-shot
  39. # never writes over its host's report — the record also carries the writer's pid.
  40. TURN_REPORT_FILE_ENV = "HERMES_QUIET_TURN_REPORT_FILE"
  41. def take_turn_report_path(environ: MutableMapping[str, str] = os.environ) -> str | None:
  42. """Read and remove the spawner's turn-report path so subprocesses started during the turn do not inherit it."""
  43. return environ.pop(TURN_REPORT_FILE_ENV, None) or None
  44. def write_turn_report(path: str | None, *, exit_code: int, error: str = "", reply: str = "") -> None:
  45. """Atomically record ``{pid, exit_code, error, reply}`` at *path*; a no-op without a path. Never
  46. raises: the report is the spawner's convenience, the turn itself is already persisted. ``reply``
  47. is what the run will print — a spawner booking a lingering child from its report relays it."""
  48. if not path:
  49. return
  50. from utils import atomic_json_write
  51. record = {"pid": os.getpid(), "exit_code": int(exit_code), "error": str(error or ""), "reply": str(reply or "")}
  52. # 0600 from creation: the record now carries the turn's answer, like the 0600 query file beside it.
  53. with contextlib.suppress(Exception):
  54. atomic_json_write(path, record, indent=None, mode=0o600)
  55. def read_turn_report(path: str, pid: int) -> dict | None:
  56. """The child's turn report, or None while absent, unreadable, or written by another process."""
  57. try:
  58. with open(path, encoding="utf-8-sig") as fh:
  59. record = json.load(fh)
  60. except (OSError, ValueError):
  61. return None
  62. if not isinstance(record, dict) or record.get("pid") != pid:
  63. return None
  64. return record
  65. # After the child reports its turn, a child with nothing to linger for exits at once; a spawner
  66. # that needs only the outcome gives it that long so its real exit code and stream tails are
  67. # booked instead of the report's summary.
  68. REPORTED_TURN_EXIT_GRACE_SECONDS = 2.0
  69. def run_reported_turn(argv: list, *, env: MutableMapping[str, str], report_path: str, timeout: float,
  70. exit_grace: float | None = REPORTED_TURN_EXIT_GRACE_SECONDS, cwd: str | None = None,
  71. encoding: str | None = None) -> subprocess.CompletedProcess:
  72. """Run one ``hermes chat -Q`` delivery child; *timeout* bounds the TURN, not the process.
  73. The child records its turn at *report_path* (``write_turn_report``) the moment the turn ends,
  74. then runs the one-shot exit linger for nested ``notify_on_complete`` replies — bounded by
  75. ``terminal.oneshot_completion_wait_seconds``, whose default equals the delivery caps, so
  76. waiting for process exit booked every delivered turn that left a reply pending as a timeout
  77. and killed the linger (#113608, #114980). A child that exits is booked from its real exit
  78. code and streams. A child still lingering once its report exists is booked from the report
  79. and left running (a daemon thread drains and reaps it): after *exit_grace* seconds for a
  80. spawner that needs only the outcome, or at the cap when *exit_grace* is None, for a spawner
  81. that relays the printed answer — a teammate's reply during the linger may still become it.
  82. Only a turn that never ends is killed, as ``subprocess.TimeoutExpired``.
  83. *cwd* pins the child's directory (a spawner sitting in a reaped scratch workspace must not
  84. hand its dead cwd on — the child dies at CLI startup, #102941). The pipes decode lossily
  85. everywhere: a stray non-UTF-8 byte (a grandchild sharing the pipe interleaving a partial
  86. multi-byte write) must not raise in the drain thread and take the reply and the failure tail
  87. with it (#105582). Without an explicit *encoding* they decode as UTF-8 only on win32, where
  88. the child is guaranteed UTF-8 (hermes_bootstrap reconfigures its streams even under
  89. PYTHONIOENCODING=cp1252) while the gateway parent is not started in UTF-8 mode, so the
  90. locale default mangled or lost accented replies (#115894); on POSIX the child keeps the
  91. locale codec, so the locale default stays correct there (#66566).
  92. """
  93. from hermes_cli._subprocess_compat import windows_hide_flags
  94. if encoding is None and sys.platform == "win32":
  95. encoding = "utf-8"
  96. proc = subprocess.Popen(
  97. argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
  98. encoding=encoding, errors="replace", env={**env, TURN_REPORT_FILE_ENV: report_path},
  99. cwd=cwd, creationflags=windows_hide_flags())
  100. streams: dict = {}
  101. def _drain() -> None:
  102. streams["out"], streams["err"] = proc.communicate()
  103. drain = threading.Thread(target=_drain, name=f"quiet-turn-drain-{proc.pid}", daemon=True)
  104. drain.start()
  105. deadline = time.monotonic() + timeout
  106. report = None
  107. while True:
  108. drain.join(timeout=exit_grace if report is not None and exit_grace is not None else 0.25)
  109. if not drain.is_alive():
  110. return subprocess.CompletedProcess(argv, proc.returncode, streams.get("out", ""), streams.get("err", ""))
  111. if report is not None and exit_grace is not None:
  112. break
  113. # Re-read while waiting for the cap: a follow-up turn rewrites the report with its answer.
  114. report = read_turn_report(report_path, proc.pid) or report
  115. if time.monotonic() >= deadline:
  116. if report is not None:
  117. break
  118. proc.kill()
  119. drain.join(timeout=5.0)
  120. # A killed child cannot run further, but the turn may have ENDED (and delivered)
  121. # in the window between the last report check and the kill landing. Re-read once:
  122. # a report that appeared means the turn completed — book it instead of
  123. # misreporting a delivered turn as a timeout (and never re-notifying).
  124. report = read_turn_report(report_path, proc.pid)
  125. if report is not None:
  126. break
  127. raise subprocess.TimeoutExpired(argv, timeout)
  128. # Turn over, child still lingering for a nested reply: not this spawner's wait.
  129. return subprocess.CompletedProcess(
  130. argv, int(report["exit_code"]), report.get("reply") or "", report.get("error") or "")
  131. @contextlib.contextmanager
  132. def bind_quiet_session_key(session_id: str):
  133. """Bind the approval/session key to *this* quiet session for the enclosing ``with`` block."""
  134. from tools.approval_context import reset_current_session_key, set_current_session_key
  135. token = set_current_session_key(session_id or "default")
  136. try:
  137. yield
  138. finally:
  139. reset_current_session_key(token)
  140. def _diagnostic_only_wake_muted(events) -> bool:
  141. """True when every drained event is an automatic diagnostic AND the CLI policy suppresses them."""
  142. from agent.notification_presentation import diagnostic_process_event
  143. from gateway.warning_notifications import warning_notifications_enabled
  144. if not events or not all(diagnostic_process_event(e) for e in events if isinstance(e, dict)):
  145. return False
  146. return not warning_notifications_enabled("cli")
  147. def quiet_notify_linger_seconds() -> float:
  148. """Total linger budget for one quiet run: the shared ``terminal.oneshot_completion_wait_seconds``.
  149. One budget covers the drain loop here AND the later ``_wait_for_oneshot_background_completions``
  150. pass, so a stuck ``notify_on_complete`` child cannot stack round-after-round waits on top of the
  151. finalize re-wait (pre-fix worst case: 8 rounds x 600s + 600s).
  152. """
  153. from tools.process_registry import ProcessRegistry
  154. return ProcessRegistry._oneshot_completion_wait_seconds()
  155. def continue_quiet_notify_completions(
  156. session_id: str,
  157. run_turn: Callable[[str], Any],
  158. *,
  159. owns_event=None,
  160. max_rounds: int = _MAX_QUIET_NOTIFY_ROUNDS,
  161. linger_budget: float | None = None,
  162. ) -> Any:
  163. """Linger for ``notify_on_complete`` work, then run owned completion texts as follow-up turns.
  164. Returns the last ``run_turn`` result, or ``None`` when nothing owned completed. The whole
  165. loop shares ONE linger budget (default: ``terminal.oneshot_completion_wait_seconds``) — a
  166. process that times out is waited on no further this run: after the current round's drained
  167. texts run, the loop stops (the finalize linger still covers it once, bounded, via the
  168. budget handshake below).
  169. """
  170. from tools.process_registry import process_registry
  171. from tools.async_delegation import claim_event_delivery, complete_event_delivery
  172. last: Any = None
  173. key = session_id or ""
  174. if linger_budget is None:
  175. linger_budget = quiet_notify_linger_seconds()
  176. deadline = time.monotonic() + max(float(linger_budget), 0.0)
  177. for _ in range(max_rounds):
  178. wait = process_registry.wait_for_pending_completions(None, timeout=max(deadline - time.monotonic(), 0.0))
  179. drained = []
  180. for event, text in process_registry.drain_notifications(session_key=key, owns_event=owns_event):
  181. # Durable async_delegation events carry a delivery ledger: without the
  182. # claim/complete handshake the row stays delivery_state='pending' and
  183. # restore_undelivered_completions re-queues it on the next process start,
  184. # injecting the same result twice. Same contract as every other drain consumer.
  185. claim = claim_event_delivery(event, "cli-quiet")
  186. if claim is None:
  187. continue
  188. complete_event_delivery(event, claim)
  189. drained.append((event, text))
  190. # Every drained event type carries formatted text (completions, watch matches,
  191. # async_delegation results): drain_notifications POPS owned events off the queue,
  192. # so filtering by type here would consume-and-silently-drop owned
  193. # async_delegation results. Keep everything that rendered.
  194. texts = [text for _event, text in drained if text]
  195. if texts:
  196. follow = run_turn("\n\n".join(texts))
  197. # Same admission rule as the interactive CLI turn: a wake made ONLY of automatic
  198. # diagnostics (early failure / watch notices) still runs, but under suppression its
  199. # reply never displaces the requested one-shot answer on stdout.
  200. if not _diagnostic_only_wake_muted([event for event, text in drained if text]):
  201. last = follow
  202. if wait.get("timed_out"):
  203. break
  204. if not texts:
  205. return last
  206. return last