process_registry_wait_policy.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188
  1. """Stop repeated ``process_manage`` wait/poll calls on a silent running process.
  2. Models re-issue byte-identical ``wait``/``poll`` calls on a background process that has printed
  3. nothing new; each repeat is a full model turn. After ``_PROMOTE_AFTER`` consecutive wait/poll results
  4. with no new output, this promotes the session to ``notify_on_complete`` (wiring gateway delivery when
  5. the process was started silently) and tells the model to stop checking: the exit arrives as a message.
  6. Only where a completion can actually reach this agent: one-shot runners (``hermes -z``), cron, Kanban
  7. workers, stateless HTTP sessions, delegated subagents, profiles whose effective
  8. ``display.background_process_notifications`` is ``off``, and callers that do not own the process's
  9. completion (another session's process, a subagent's un-handed-off process) keep the old result, since
  10. nothing would start a turn for the notice there. ``terminal.process_wait_auto_notify: false`` restores the old behaviour
  11. everywhere.
  12. """
  13. from __future__ import annotations
  14. import logging
  15. import threading
  16. logger = logging.getLogger(__name__)
  17. _PROMOTE_AFTER = 2
  18. _MAX_TRACKED = 512
  19. # session id -> (total_output_chars at the last wait/poll, consecutive unchanged count)
  20. _state: dict[str, tuple[int, int]] = {}
  21. _state_lock = threading.Lock()
  22. _STOP_NOTE = (
  23. "The process is still running and has printed nothing new since your last check. Hermes has turned on "
  24. "notify_on_complete for it: when it exits, a message with its exit code and final output starts a new turn "
  25. "automatically, even after you end this turn. Do NOT call process_manage wait or poll on {sid} again, and do "
  26. "not sleep or loop to wait for it. If you have other work, do it now; otherwise end your turn with a one-line "
  27. "status and report the result when the completion message arrives."
  28. )
  29. def _effective_config() -> dict:
  30. """The profile's effective user config (managed overlay, ``${VAR}`` expansion): the same reader the
  31. completion consumers use (``gateway.run._load_gateway_config``, ``tui_gateway.server._load_cfg``), so
  32. an admin-managed or env-expanded value decides promotion exactly as it decides delivery."""
  33. from hermes_cli.config_effective import load_user_config_effective
  34. return load_user_config_effective()
  35. def _enabled() -> bool:
  36. """``terminal.process_wait_auto_notify`` (default true); true when config is unreadable."""
  37. from hermes_cli.config import cfg_get
  38. try:
  39. return cfg_get(_effective_config(), "terminal", "process_wait_auto_notify", default=True) is not False
  40. except Exception: # a broken config.yaml must not break process_manage
  41. logger.debug("process_wait_auto_notify unreadable; using default", exc_info=True)
  42. return True
  43. def _forget(session_id: str) -> None:
  44. with _state_lock:
  45. _state.pop(session_id, None)
  46. def _count_unchanged(session) -> int:
  47. """Record one still-running wait/poll observation; return the consecutive no-new-output count."""
  48. total = int(getattr(session, "total_output_chars", 0) or 0)
  49. with _state_lock:
  50. # Baseline is the spawn (0 chars observed): the first check on a silent process already counts.
  51. last_total, count = _state.get(session.id, (0, 0))
  52. count = count + 1 if total == last_total else 0
  53. _state[session.id] = (total, count)
  54. while len(_state) > _MAX_TRACKED:
  55. _state.pop(next(iter(_state)))
  56. return count
  57. def _can_deliver_completion() -> bool:
  58. """Whether a completion notice for this session can reach the agent that is waiting.
  59. Finite turns (non-interactive ``hermes chat -q`` / ``-z``, as ``delegate_task`` decides inline execution) end
  60. the run when the turn ends, so a promised notice would never arrive."""
  61. from agent.delegation_context import is_delegated_child_context
  62. from gateway.session_context import async_delivery_supported, get_session_env
  63. from gateway.session_finite import finite_turn_required
  64. finite = finite_turn_required()
  65. if finite is True or (finite is None and get_session_env("HERMES_SINGLE_QUERY_SESSION") == "1"):
  66. return False
  67. if not async_delivery_supported() or is_delegated_child_context():
  68. return False
  69. # ``off`` drains the completion but never starts the agent turn (CLI, TUI and gateway alike). The
  70. # gateway also honours the HERMES_BACKGROUND_NOTIFICATIONS override; either one saying off is off.
  71. from hermes_cli.config import cfg_get
  72. try:
  73. from gateway.platforms._shared import platform_gate_env
  74. modes = (platform_gate_env("HERMES_BACKGROUND_NOTIFICATIONS"),
  75. cfg_get(_effective_config(), "display", "background_process_notifications"))
  76. except Exception:
  77. logger.debug("background_process_notifications unreadable; assuming on", exc_info=True)
  78. return True
  79. return not any(m is False or str(m or "").strip().lower() == "off" for m in modes)
  80. def _caller_receives_completion(session) -> bool:
  81. """Whether the completion this process would publish is routed to the CALLER, judged by the
  82. consumers' own predicates: ``ProcessRegistry._owns_event`` (session-key routing; another session's
  83. process stays with its owner) and ``should_surface_notification`` (a subagent-owned process is
  84. suppressed unless handed off or ``delegation.surface_child_process_notifications`` is on). Fails
  85. closed: an unbound caller cannot prove ownership of a keyed process."""
  86. from gateway.session_context import get_session_env
  87. from tools.process_registry import ProcessRegistry
  88. from tools.process_registry_notifications import should_surface_notification
  89. caller_key = get_session_env("HERMES_SESSION_KEY", "")
  90. evt = {"type": "completion", "session_id": session.id, "session_key": session.session_key,
  91. "task_id": session.task_id, "owner_task_id": session.owner_task_id}
  92. if session.session_key and not caller_key:
  93. return False
  94. return ProcessRegistry._owns_event(evt, caller_key, None, False) and should_surface_notification(evt)
  95. def _claim_notify_if_running(registry, session) -> bool:
  96. """Set ``notify_on_complete`` unless the exit already won; False means it did.
  97. ``_move_to_finished`` claims the finish under ``registry._lock`` and reads the flag afterwards to
  98. decide whether to queue the completion, so flipping it under the same lock while the session is
  99. still in ``_running`` guarantees the exit publication sees it."""
  100. with registry._lock:
  101. if session._finish_claimed or session.id not in registry._running:
  102. return False
  103. session.notify_on_complete = True
  104. return True
  105. def _promote(registry, session) -> None:
  106. """Wire completion delivery for a process just claimed by ``_claim_notify_if_running``.
  107. CLI, TUI/Desktop and ``hermes chat -q`` drain ``completion_queue``, which the exit path fills
  108. whenever the flag is set. The messaging gateway delivers only through a per-process watcher, so a
  109. process started silently also needs the chat routing stamped and a watcher armed."""
  110. from tools.terminal_tool_background import _register_completion_watcher, _stamp_gateway_routing
  111. from gateway.session_context import get_session_env
  112. if not session.watcher_platform:
  113. _stamp_gateway_routing(session, get_session_env)
  114. # watcher_interval is set only by _register_completion_watcher: 0 means no watcher yet.
  115. if session.watcher_platform and not session.watcher_interval:
  116. _register_completion_watcher(registry, session, session.session_key)
  117. try:
  118. registry._write_checkpoint()
  119. except Exception:
  120. logger.debug("checkpoint after auto-notify promotion failed", exc_info=True)
  121. logger.info("process %s: repeated wait/poll with no new output; auto-enabled notify_on_complete", session.id)
  122. def apply_wait_poll_policy(action: str, session_id: str, result: dict) -> dict:
  123. """Post-process one ``process_manage`` session action result (see module docstring)."""
  124. if not isinstance(result, dict) or not _enabled():
  125. return result
  126. from tools.process_registry import process_registry
  127. session = process_registry.get(session_id)
  128. if session is None:
  129. return result
  130. if action == "wait":
  131. running = result.get("status") == "timeout"
  132. elif action == "poll":
  133. running = result.get("status") == "running"
  134. else:
  135. running = False
  136. if not running:
  137. # Exit, any other action on the session, or a released/interrupted wait starts over.
  138. if action not in ("wait", "poll") or result.get("status") != "interrupted":
  139. _forget(session.id)
  140. return result
  141. if session.exited or _count_unchanged(session) < _PROMOTE_AFTER or not _can_deliver_completion():
  142. return result
  143. if not _caller_receives_completion(session):
  144. return result # the completion would reach its owner, not this caller: keep plain polling
  145. if not session.notify_on_complete:
  146. if not _claim_notify_if_running(process_registry, session):
  147. # Exited between the wait/poll and this check, before the flag was set: no completion
  148. # was queued, so hand back the exit itself instead of promising a message.
  149. _forget(session.id)
  150. return process_registry.poll(session_id) if action == "poll" else process_registry.wait(session_id, timeout=5)
  151. _promote(process_registry, session)
  152. note = _STOP_NOTE.format(sid=session.id)
  153. result["notify_on_complete"] = True
  154. if action == "wait":
  155. clamp = result.get("timeout_note", "")
  156. clamp = clamp.split(". Wait window", 1)[0] + ". " if clamp.startswith("Requested wait") else ""
  157. result["timeout_note"] = clamp + note
  158. else:
  159. result["note"] = note
  160. return result