terminal_tool_background.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286
  1. """Background-process launch path: ``terminal(background=true)`` spawns a tracked
  2. process via the process registry (Popen locally, ``env.execute`` in a sandbox),
  3. stamps gateway routing metadata for completion / watch-pattern notifications and
  4. returns the JSON result. Lazy ``tools.terminal_tool`` imports keep the origin's
  5. monkeypatch points authoritative.
  6. """
  7. import json
  8. import logging
  9. import sys
  10. from typing import Any, List, Optional
  11. logger = logging.getLogger("tools.terminal_tool")
  12. # A silent background process (no notify_on_complete / watch_patterns) is right
  13. # only for servers/watchers; for bounded tasks the agent almost always wanted a
  14. # notification and forgot the flag, so nudge it (cheap false positive).
  15. _SILENT_BACKGROUND_HINT = (
  16. 'background=true without notify_on_complete=true means this process runs SILENTLY — you '
  17. 'will not be told when it exits. If this is a bounded task (test suite, build, CI poller, '
  18. 'deploy, anything with a defined end), you almost certainly wanted notify_on_complete=true '
  19. 'so the system pings you on exit. Re-launch with notify_on_complete=true, or call '
  20. "process(action='poll') / process(action='wait') yourself to learn the outcome. Only "
  21. 'ignore this hint for genuine long-lived processes that never exit (servers, watchers, '
  22. 'daemons).'
  23. )
  24. # Homebrewed CI pollers built on `gh pr view --json statusCheckRollup` or
  25. # `gh pr checks | jq` fail silently in known ways (block-buffered stdout never
  26. # reaches capture, jq null-key edge cases exit the loop, conclusion-vs-status
  27. # confusion declares all-green early, TTY-only banners never appear piped).
  28. # Detector is deliberately narrow: the canonical column-2 awk poller is fine.
  29. _HOMEBREW_CI_POLLER_HINT = (
  30. 'This looks like a homebrewed CI poller built from `gh pr view --json statusCheckRollup` '
  31. 'and/or `gh pr checks | jq`. That shape has burned us repeatedly in hermes-agent dev work '
  32. '(PRs #31329, #31448, #31695, #31709, #31745, #32264, #33131) — stdout buffering kills '
  33. 'output capture, jq null-key edge cases silently exit the loop, conclusion-vs-status field '
  34. 'confusion exits early with bogus all-green verdicts, TTY-only summary banners never '
  35. 'appear when piped. Use the canonical snippets in the green-ci-policy skill instead: the '
  36. 'exit-code-driven `gh pr checks $PR >/dev/null` (rc 0 = green, 8 = pending, else fail) for '
  37. 'exit-on-first-fail behavior, or the column-2 awk-on-tabs poller (`awk -F"\\t" '
  38. '"$2==\\"pending\\""`) for sharded matrices. Load '
  39. "skill_view(name='github/hermes-agent-dev', file_path='references/green-ci-policy.md') for "
  40. 'the verbatim snippets. If you must roll a custom loop with rich structured output, write '
  41. "each tick to a known file (`tee -a $TMPDIR/ci.log`) and rely on `process(action='log')` to "
  42. 'read THAT file — do not rely on background-process stdout capture for line-buffered shell '
  43. 'loops.'
  44. )
  45. # A foreground call whose only backgrounding was a trailing `&` is spawned as background=true.
  46. _AUTO_BACKGROUND_NOTE = (
  47. "Ran in the background (the trailing & was converted to background=true; Hermes tracks the "
  48. "process). Do NOT re-run it. Use process_manage (poll/log/wait/kill, session_id above) to check it."
  49. )
  50. _ASYNC_UNSUPPORTED_NOTE = (
  51. 'notify_on_complete / watch_patterns are not available in this session — it cannot receive '
  52. 'an async completion after the turn ends (a one-shot runner such as `hermes -z`, a /bg '
  53. 'background task, a cron job, a Kanban worker, or a stateless HTTP endpoint). The process is running in the '
  54. "background; retrieve its result with process(action='poll') or process(action='wait')."
  55. )
  56. # proc_session attribute -> HERMES_SESSION_* env var carrying it.
  57. _ROUTING_FIELDS = (
  58. ("watcher_chat_id", "HERMES_SESSION_CHAT_ID"),
  59. ("watcher_user_id", "HERMES_SESSION_USER_ID"),
  60. ("watcher_user_name", "HERMES_SESSION_USER_NAME"),
  61. ("watcher_thread_id", "HERMES_SESSION_THREAD_ID"),
  62. ("watcher_message_id", "HERMES_SESSION_MESSAGE_ID"),
  63. # The spawning conversation's session-db id lets the gateway's completion
  64. # pre-flight drop the notification if the user closed this session (/new)
  65. # before the process finished, instead of injecting it into the NEW one.
  66. ("parent_session_id", "HERMES_SESSION_ID"),
  67. )
  68. def _looks_like_homebrew_ci_poller(command: str) -> bool:
  69. has_gh = "gh pr view" in command or "gh pr checks" in command
  70. has_jq = " jq " in command or "| jq" in command or "$(jq" in command
  71. # `gh pr checks` doesn't emit JSON, so piping it to jq is confused intent.
  72. return "statusCheckRollup" in command or (has_gh and has_jq)
  73. def _stamp_gateway_routing(proc_session, get_session_env) -> None:
  74. """Copy the spawning chat's routing metadata onto the process session so
  75. completion / watch notifications reach the right chat/thread."""
  76. platform = get_session_env("HERMES_SESSION_PLATFORM", "")
  77. if not platform:
  78. return
  79. proc_session.watcher_platform = platform
  80. for attr, var in _ROUTING_FIELDS:
  81. setattr(proc_session, attr, get_session_env(var, ""))
  82. def _spawn(process_registry, *, env, env_type, command, cwd, effective_task_id, task_id,
  83. session_key, effective_pty, persist_on_release: bool = False):
  84. common = dict(command=command, cwd=cwd, task_id=effective_task_id,
  85. owner_task_id=task_id or effective_task_id, session_key=session_key,
  86. persist_on_release=persist_on_release)
  87. if env_type == "local":
  88. return process_registry.spawn_local(
  89. env_vars=env.env if hasattr(env, 'env') else None, use_pty=effective_pty,
  90. restore_login_path=True, **common)
  91. return process_registry.spawn_via_env(env=env, **common)
  92. def _apply_async_support(proc_session, result_data, notify_on_complete, watch_patterns):
  93. """Finite sessions (stateless HTTP, one-shot Kanban workers) can't route a
  94. completion back after the turn ends: drop the flags and tell the agent to
  95. poll. Otherwise stamp gateway routing. Returns (notify, watch_patterns)."""
  96. if not (notify_on_complete or watch_patterns):
  97. return notify_on_complete, watch_patterns
  98. from gateway.session_context import async_delivery_supported, get_session_env
  99. if async_delivery_supported():
  100. _stamp_gateway_routing(proc_session, get_session_env)
  101. return notify_on_complete, watch_patterns
  102. result_data["notify_on_complete"] = False
  103. result_data["notify_unsupported"] = _ASYNC_UNSUPPORTED_NOTE
  104. logger.info("background proc %s: async delivery unsupported on this "
  105. "session; notify_on_complete/watch_patterns disabled", proc_session.id)
  106. return False, None
  107. def _register_completion_watcher(process_registry, proc_session, session_key) -> None:
  108. """Gateway mode: register a fast watcher so completion triggers a new
  109. agent turn (CLI mode uses the completion_queue directly).
  110. Armed on the live gateway loop right away: the post-turn drain alone leaves a
  111. process that finishes while its launching turn is still running unwatched, and
  112. the chat mute for as long as that turn lasts (#112033). Before the gateway
  113. serves, or while it stops, the descriptor waits in ``pending_watchers`` for the
  114. startup / post-turn drain instead."""
  115. proc_session.watcher_interval = 5
  116. watcher = {
  117. "session_id": proc_session.id, "check_interval": 5, "session_key": session_key,
  118. "platform": proc_session.watcher_platform,
  119. **{attr.removeprefix("watcher_"): getattr(proc_session, attr)
  120. for attr, _ in _ROUTING_FIELDS[:-1]},
  121. "notify_on_complete": True, "parent_session_id": proc_session.parent_session_id,
  122. }
  123. runner_ref = getattr(sys.modules.get("gateway.run"), "_gateway_runner_ref", None)
  124. runner = runner_ref() if callable(runner_ref) else None
  125. if runner is not None and runner.arm_process_watcher(watcher):
  126. return
  127. process_registry.pending_watchers.append(watcher)
  128. def spawn_background_process(
  129. *, command: str, env: Any, env_type: str, effective_task_id: str, task_id: Optional[str],
  130. session_key: str, workdir: Optional[str], cwd: str, effective_pty: bool,
  131. notify_on_complete: bool, watch_patterns: Optional[list[str]], approval_note: Optional[str],
  132. completion_output_chars: int = 0,
  133. pty_disabled_reason: Optional[str],
  134. heartbeat_seconds: int = 0,
  135. persist_on_release: bool = False,
  136. mounted_host: Optional[str] = None,
  137. auto_backgrounded: bool = False,
  138. ) -> str:
  139. """Spawn *command* as a tracked background process and return the JSON result.
  140. ``auto_backgrounded`` marks a trailing-`&` conversion: the result carries the conversion note
  141. and skips the silent-background hint, whose "re-launch" advice would start a second copy.
  142. Never inline-polls ``is_interrupted()``: the spawn detaches and returns
  143. exit_code 0 immediately, so the stale-interrupt kill cannot occur here.
  144. """
  145. from tools.process_registry import process_registry
  146. from tools.terminal_tool import (
  147. _redact_terminal_error_text, _resolve_command_cwd, _resolve_notification_flag_conflict,
  148. )
  149. effective_cwd = _resolve_command_cwd(
  150. workdir=workdir, default_cwd=cwd, session_key=session_key, env_type=env_type,
  151. mounted_host=mounted_host if mounted_host is not None else getattr(env, "host_cwd", None),
  152. env=env,
  153. )
  154. try:
  155. proc_session = _spawn(
  156. process_registry, env=env, env_type=env_type, command=command, cwd=effective_cwd,
  157. effective_task_id=effective_task_id, task_id=task_id, session_key=session_key,
  158. effective_pty=effective_pty, persist_on_release=persist_on_release,
  159. )
  160. result_data = {"output": "Background process started", "session_id": proc_session.id,
  161. "pid": proc_session.pid, "exit_code": 0, "error": None}
  162. if persist_on_release:
  163. result_data["persist_on_release"] = True
  164. if approval_note:
  165. result_data["approval"] = approval_note
  166. if pty_disabled_reason:
  167. result_data["pty_note"] = pty_disabled_reason
  168. if auto_backgrounded:
  169. result_data["auto_backgrounded"] = _AUTO_BACKGROUND_NOTE
  170. elif not notify_on_complete and not watch_patterns:
  171. result_data["hint"] = _SILENT_BACKGROUND_HINT
  172. if command and _looks_like_homebrew_ci_poller(command):
  173. existing = result_data.get("hint", "")
  174. result_data["hint"] = (existing + "\n\n" + _HOMEBREW_CI_POLLER_HINT if existing
  175. else _HOMEBREW_CI_POLLER_HINT)
  176. notify_on_complete, watch_patterns = _apply_async_support(
  177. proc_session, result_data, notify_on_complete, watch_patterns)
  178. watch_patterns, conflict_note = _resolve_notification_flag_conflict(
  179. notify_on_complete=bool(notify_on_complete), watch_patterns=watch_patterns, background=True,
  180. )
  181. if conflict_note:
  182. logger.warning("background proc %s: %s", proc_session.id, conflict_note)
  183. result_data["watch_patterns_ignored"] = conflict_note
  184. if notify_on_complete:
  185. proc_session.notify_on_complete = True
  186. result_data["notify_on_complete"] = True
  187. if completion_output_chars:
  188. proc_session.completion_output_chars = int(completion_output_chars)
  189. if proc_session.watcher_platform:
  190. _register_completion_watcher(process_registry, proc_session, session_key)
  191. from agent.delegation_context import is_delegated_child_context
  192. if is_delegated_child_context():
  193. result_data["notify_on_complete"] = False
  194. result_data["subagent_note"] = _SUBAGENT_NOTIFY_NOTE
  195. elif heartbeat_seconds:
  196. # Heartbeats ride the same delivery path as the completion notice, so they are
  197. # only armed where that notice can actually reach the agent.
  198. result_data["heartbeat_seconds"] = process_registry.arm_heartbeat(proc_session, heartbeat_seconds)
  199. elif heartbeat_seconds:
  200. result_data["heartbeat_ignored"] = "heartbeat needs notify=true delivery, which this session cannot receive"
  201. if watch_patterns:
  202. proc_session.watch_patterns = list(watch_patterns)
  203. result_data["watch_patterns"] = proc_session.watch_patterns
  204. return json.dumps(result_data, ensure_ascii=False)
  205. except Exception as e:
  206. return json.dumps({
  207. "output": "", "exit_code": -1,
  208. "error": _redact_terminal_error_text(f"Failed to start background process: {e}"),
  209. }, ensure_ascii=False)
  210. _SUBAGENT_NOTIFY_NOTE = (
  211. "You are a subagent: this process's completion notice will NOT reach your parent, and the process is killed when "
  212. "you finish. Before you finish, either wait for it (process_manage wait), kill it, or hand it to your parent with "
  213. "process_manage(action='handoff', session_id=..., data='<purpose>') so the parent receives its completion. For CI "
  214. "watchers prefer returning the fact (PR number, SHA) and letting the parent watch."
  215. )
  216. _YIELDED_NOTE = (
  217. "The user sent a message while this command was running, so it was moved to the "
  218. "background WITHOUT being killed and is still running. You will be notified when it "
  219. "exits (notify_on_complete). Read the user's message and respond to it now; use "
  220. "process(action='poll'|'wait'|'log', session_id=...) to check on this command."
  221. )
  222. def yield_to_background_handler(
  223. *, command: str, env_type: str, cwd: Optional[str], effective_task_id: str,
  224. task_id: Optional[str], session_key: str,
  225. ):
  226. """Build the ``yield_handler`` a foreground ``env.execute`` calls when the tool thread is
  227. asked to yield (a user message arrived mid-command). Local backend only: the live Popen
  228. is adopted by the process registry as a notify-on-complete background session and the
  229. partial output is returned to the model right away. Other backends return None (no
  230. adoptable host process) and the foreground wait continues."""
  231. if env_type != "local":
  232. return None
  233. def _handler(proc, output_so_far: str) -> dict:
  234. from tools.process_registry import process_registry
  235. session = process_registry.adopt_local(
  236. proc, command=command, cwd=cwd, task_id=effective_task_id,
  237. owner_task_id=task_id or effective_task_id, session_key=session_key,
  238. output_so_far=output_so_far)
  239. _stamp_routing_if_gateway(process_registry, session, session_key)
  240. logger.info("foreground command yielded to background as %s (pid %s)", session.id, session.pid)
  241. return {
  242. "output": output_so_far, "returncode": None, "yielded_session_id": session.id, "pid": session.pid,
  243. }
  244. return _handler
  245. def _stamp_routing_if_gateway(process_registry, session, session_key) -> None:
  246. """Route the adopted session's completion like a normal notify_on_complete spawn."""
  247. from gateway.session_context import async_delivery_supported, get_session_env
  248. if not async_delivery_supported():
  249. session.notify_on_complete = False
  250. return
  251. _stamp_gateway_routing(session, get_session_env)
  252. if session.watcher_platform:
  253. _register_completion_watcher(process_registry, session, session_key)