turn_finalizer.py 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912
  1. """Post-loop turn finalization for ``run_conversation``.
  2. Budget summary, trajectory save, persist, diagnostics, response transforms, result
  3. assembly, steer drain, memory/skill review. Synchronous, single return. ``logger`` is
  4. imported lazily from ``agent.conversation_loop`` (no cycle, same logger name)."""
  5. from __future__ import annotations
  6. import logging
  7. import os
  8. import time
  9. from contextlib import suppress
  10. from typing import Any, Callable, List, Optional, Tuple
  11. from agent.codex_responses_adapter import _summarize_user_message_for_log
  12. from agent.delegation_context import is_dispatcher_owned_worker_context
  13. from agent.interrupt_control import interrupted_during_api_call_reason
  14. from agent.turn_failure_copy import exit_reason_failure, stamp_failure
  15. from agent.context_compressor import _DB_PERSISTED_MARKER
  16. from agent.message_content import flatten_message_text
  17. from agent.message_metadata import append_message, stamp_message_timestamp
  18. from agent.message_sanitization import _sanitize_surrogates
  19. from agent.served_model import result_model_fields
  20. # Verification-continuation nudges (verify-on-stop / pre_verify) must be stripped from
  21. # returned/live history to avoid role-alternation breaks; the assistant response is
  22. # real content and is not flagged. (#65919)
  23. _VERIFICATION_CONTINUATION_FLAGS = ("_verification_stop_synthetic", "_pre_verify_synthetic")
  24. _SENTENCE_END = {".", "!", "?", "。", "!", "?", "`", ")"}
  25. # ``result[key] = agent.session_<key>`` for the per-session usage/cost counters.
  26. _SESSION_TOKEN_KEYS = (
  27. "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens",
  28. "reasoning_tokens", "prompt_tokens", "completion_tokens", "total_tokens",
  29. )
  30. _SESSION_COST_KEYS = ("estimated_cost_usd", "cost_status", "cost_source")
  31. def _assistant_row_missing_visible_text(msg: dict) -> bool:
  32. """True when an assistant row has no visible text (blank final or tool-only)."""
  33. if not isinstance(msg, dict) or msg.get("role") != "assistant":
  34. return False
  35. return not flatten_message_text(msg.get("content")).strip()
  36. def _record_kanban_budget_exhausted(
  37. kanban_task: str, api_call_count: int, max_iterations: int, logger: logging.Logger
  38. ) -> None:
  39. """Record a terminal ``timed_out`` outcome for a kanban worker out of budget.
  40. Routed via ``_record_task_failure`` (not ``kanban_block``) so it counts toward the
  41. consecutive-failure circuit breaker. Idempotent via the ``_end_run`` CAS
  42. (``WHERE ended_at IS NULL``), so safe from multiple exit paths.
  43. This is a bounded fallback (#87096): the CAS invariant in ``_end_run`` (``WHERE ended_at IS NULL``)
  44. guarantees idempotence — if another path already closed the run this is a no-op — so it is safe to call
  45. from multiple exit paths.
  46. """
  47. try:
  48. from hermes_cli import kanban_db as _kb
  49. from hermes_cli import kanban_db_connect as _kbc
  50. from hermes_cli import kanban_db_dispatch as _kbd
  51. _conn = _kbc.connect()
  52. try:
  53. _kbd._record_task_failure(
  54. _conn,
  55. kanban_task,
  56. error=(
  57. f"Iteration budget exhausted ({api_call_count}/{max_iterations}) — "
  58. "task could not complete within the allowed iterations"
  59. ),
  60. outcome="timed_out",
  61. release_claim=True,
  62. end_run=True,
  63. event_payload_extra={"budget_used": api_call_count, "budget_max": max_iterations},
  64. )
  65. finally:
  66. with suppress(Exception):
  67. _conn.close()
  68. except Exception:
  69. logger.warning(
  70. "Failed to record budget-exhausted failure for task %s", kanban_task, exc_info=True
  71. )
  72. def _drop_verification_continuation_scaffolding(messages) -> None:
  73. """Remove verification-continuation nudges in place; only the synthetic nudges carry
  74. these flags, so the real attempted final answer persisted to state.db survives."""
  75. messages[:] = [
  76. m for m in messages
  77. if not (isinstance(m, dict) and any(m.get(f) for f in _VERIFICATION_CONTINUATION_FLAGS))
  78. ]
  79. def _clone_background_review_messages(messages):
  80. """Copy the review input without aliasing the live transcript."""
  81. # Lazy: conversation_loop imports this module (cycle).
  82. from agent.conversation_loop import _clone_message_for_send
  83. return [_clone_message_for_send(message) for message in messages]
  84. def _invoke_hook_safely(name: str, logger: logging.Logger, **kwargs) -> list:
  85. """Fire a lifecycle plugin hook; a failing hook is logged, never fatal."""
  86. try:
  87. from hermes_cli.lifecycle import invoke_hook
  88. return invoke_hook(name, **kwargs)
  89. except Exception as exc:
  90. logger.warning("%s hook failed: %s", name, exc)
  91. return []
  92. def _guarded_cleanup(label: str, fn: Callable[[], Any], errors: list[str], logger) -> None:
  93. """Post-loop cleanup must never lose the response: each step is guarded
  94. independently and errors surface via ``cleanup_errors`` (#8049)."""
  95. try:
  96. fn()
  97. except Exception as err:
  98. errors.append(f"{label}: {err}")
  99. logger.error("finalize_turn: _%s failed: %s", label, err, exc_info=True)
  100. def _resolve_budget_fallback(
  101. agent, *, final_response, api_call_count, interrupted, failed, messages, _turn_exit_reason,
  102. _pending_verification_response, _pending_verification_response_previewed, logger,
  103. ) -> tuple[Any, Any, bool, Any]:
  104. """Iteration-budget exhaustion. Returns ``(final_response, _turn_exit_reason,
  105. preserved_verification_fallback, interrupted)``."""
  106. budget_exhausted = (
  107. api_call_count >= agent.max_iterations or agent.iteration_budget.remaining <= 0
  108. )
  109. preserved_verification_fallback = False
  110. if (
  111. final_response is None and budget_exhausted and not interrupted and not failed
  112. and str(_turn_exit_reason) in {"unknown", "budget_exhausted"}
  113. ):
  114. _turn_exit_reason = f"max_iterations_reached({api_call_count}/{agent.max_iterations})"
  115. if _pending_verification_response:
  116. # A verification gate withheld a composed answer, then the budget ran out:
  117. # preserve it rather than make another fallible call. The explicit pending
  118. # value is the provenance guard; unrelated error exits never enter here.
  119. # Previewed only if the reused candidate was actually streamed as interim.
  120. final_response = _pending_verification_response
  121. if _pending_verification_response_previewed:
  122. agent._response_was_previewed = True
  123. # Reuses the candidate the user already saw sealed as an interim (#130396).
  124. agent._reused_response_text = final_response
  125. preserved_verification_fallback = True
  126. else:
  127. # _handle_max_iterations makes one extra toolless request for a summary.
  128. agent._emit_diagnostic_status(
  129. f"⚠️ Iteration budget exhausted ({api_call_count}/{agent.max_iterations}) "
  130. "— asking model to summarise"
  131. )
  132. if not agent.quiet_mode:
  133. agent._safe_print(
  134. f"\n⚠️ Iteration budget exhausted ({api_call_count}/{agent.max_iterations}) "
  135. "— requesting summary...", diagnostic=True,
  136. )
  137. _summary_start = time.time()
  138. try:
  139. final_response = agent._handle_max_iterations(messages, api_call_count)
  140. except InterruptedError:
  141. # The turn ends interrupted, so the pending interrupt message is returned
  142. # for requeue instead of being cleared behind a fallback summary. A redirect
  143. # also ends it: the budget is spent, so there is no loop to restart into.
  144. from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX
  145. interrupted = True
  146. _turn_exit_reason = interrupted_during_api_call_reason(agent)
  147. final_response = f"{INTERRUPT_WAITING_FOR_MODEL_PREFIX}{time.time() - _summary_start:.1f}s elapsed)."
  148. # A kanban worker must record a terminal outcome whether or not a fallback path
  149. # was eligible, so the dispatcher learns the worker could not complete. Only the
  150. # dispatcher-owned worker owns the task: an in-process delegate_task child or cron run
  151. # inherits ``HERMES_KANBAN_TASK`` via os.environ but exhausting ITS budget must not
  152. # close the parent's run and release its claim (#112817).
  153. _kanban_task = (
  154. os.environ.get("HERMES_KANBAN_TASK")
  155. if budget_exhausted and is_dispatcher_owned_worker_context() else None
  156. )
  157. # If running as a kanban worker, signal the dispatcher that the worker could not complete (rather than
  158. # treating it as a protocol violation). This applies whether the user-facing fallback came from the
  159. # summary call or an explicitly pending continuation; both exhausted the task budget and must advance
  160. # the failure circuit. We route through ``_record_task_failure(outcome="timed_out")`` rather than
  161. # ``kanban_block`` so this counts toward the dispatcher's consecutive-failure circuit breaker (#29747
  162. # gap 2).
  163. # Bounded fallback (#87096): budget was exhausted but none of the normal fallback paths were eligible
  164. # (interrupted / failed / anomalous exit_reason). If running as a kanban worker we must still record a
  165. # terminal outcome so the task does not remain in an ambiguous lifecycle state. The worker's run is
  166. # closed via ``_record_task_failure`` (compare-and-swap receipt path) which is a no-op if another path
  167. # closed it — the CAS invariant in ``_end_run`` (``WHERE ended_at IS NULL``) guarantees idempotence.
  168. if _kanban_task:
  169. _record_kanban_budget_exhausted(_kanban_task, api_call_count, agent.max_iterations, logger)
  170. return final_response, _turn_exit_reason, preserved_verification_fallback, interrupted
  171. def _rollback_interrupted_preflight_display(agent, interrupted) -> None:
  172. """Roll back the preflight-seeded display count only when an interrupt wins before
  173. any provider response; compaction state (incl. ``-1``) stays with the real-usage
  174. path. Type-pinned guards keep MagicMock/SimpleNamespace doubles inert."""
  175. _preflight_snapshot = getattr(agent, "_turn_preflight_display_snapshot", None)
  176. if (
  177. interrupted is True
  178. and isinstance(_preflight_snapshot, int)
  179. and not isinstance(_preflight_snapshot, bool)
  180. and getattr(agent, "_turn_received_provider_response", False) is not True
  181. and getattr(agent, "context_compressor", None) is not None
  182. ):
  183. _rollback_fn = getattr(
  184. agent.context_compressor, "rollback_interrupted_preflight_display_tokens", None
  185. )
  186. if callable(_rollback_fn):
  187. _rollback_fn(_preflight_snapshot)
  188. def _drop_transcript_scaffolding(agent, messages) -> None:
  189. """Strip private retry scaffolding first, or a later "continue" replays
  190. assistant("(empty)") / recovery nudges into the same empty-response loop. Only
  191. the synthetic verification nudges go; the assistant candidate persists (#65919)."""
  192. agent._drop_trailing_empty_response_scaffolding(messages)
  193. _drop_verification_continuation_scaffolding(messages)
  194. def _recover_final_from_stream(agent, final_response, interrupted, failed) -> tuple[Any, bool]:
  195. """An empty terminal completion is not authoritative when the stream already
  196. delivered text; recover before persist so a blank tail isn't frozen (#95514).
  197. Returns ``(final_response, recovered_from_stream)``. Called by the finalizer BEFORE
  198. the fallible tail-shaping/persist steps so the recovered text is already bound when
  199. one of them raises — a persist failure must not lose text the user already saw."""
  200. if interrupted or failed:
  201. return final_response, False
  202. _streamed = getattr(agent, "_current_streamed_assistant_text", "") or ""
  203. _streamed = _streamed.strip() if isinstance(_streamed, str) else ""
  204. if not (flatten_message_text(final_response).strip() if final_response else "") and _streamed:
  205. return _streamed, True
  206. return final_response, False
  207. def _close_transcript_tail(agent, messages, final_response, interrupted, _recovered_from_stream) -> None:
  208. """Shape the transcript tail before the durable snapshot (scaffolding already dropped
  209. and ``final_response`` already stream-recovered by the caller)."""
  210. # An interrupt can leave a tool result as the tail; close the sequence so strict
  211. # providers don't see ``tool → user`` (placeholder: final_response is usually empty).
  212. if interrupted:
  213. from agent.message_sanitization import close_interrupted_tool_sequence
  214. close_interrupted_tool_sequence(messages, final_response, stopped=True)
  215. # Recovery ``break`` sites can return a final_response with no closing assistant
  216. # row; enforce "delivered final_response ⇒ assistant row" here. Compare content,
  217. # not role, so a matching verification candidate isn't dup'd.
  218. if final_response and not interrupted:
  219. # Some recovery/fallback paths return a real final_response without adding a closing assistant
  220. # message to the transcript (e.g. the partial-stream and prior-turn-content recovery ``break`` sites
  221. # in ``conversation_loop``). If persisted as-is, the durable session can end at a tool/user message
  222. # even though the caller — and the gateway platform — already saw a completed assistant response.
  223. # The next turn then replays a user-only backlog and the model re-answers every "unanswered"
  224. # message. Close the durable turn at the source, at the single chokepoint every recovery ``break``
  225. # flows through, so the invariant "delivered final_response ⇒ assistant row in transcript" holds
  226. # regardless of which path produced it. (#43849 / #44100) Compare content (not just role) so a
  227. # verification candidate that matches the final response is not duplicated at budget exhaustion.
  228. # (#65919 §7)
  229. _tail = messages[-1] if messages else None
  230. if not isinstance(_tail, dict) or _tail.get("role") != "assistant":
  231. append_message(messages, {"role": "assistant", "content": final_response})
  232. elif (
  233. _tail.get("content") != final_response
  234. and _assistant_row_missing_visible_text(_tail)
  235. and (_tail.get("tool_calls") or _recovered_from_stream)
  236. ):
  237. # Pure tool-call turn or stream-recovered blank (#95514): fill the persisted
  238. # blank row's content rather than append a second row.
  239. _tail["content"] = final_response
  240. stamp_message_timestamp(_tail)
  241. _tail.pop(_DB_PERSISTED_MARKER, None)
  242. agent._db_flush_scan_prefix = None
  243. # Request is complete, so replace API-local voice/model/skill guidance with the
  244. # clean user input before the durable snapshot (earlier flushes still needed them).
  245. # Earlier turn-start flushes use the DB-only override because their messages are still needed for the
  246. # API request; this finalizer runs after that request is complete (#48677 / #63766).
  247. _apply_override = getattr(agent, "_apply_persist_user_message_override", None)
  248. if callable(_apply_override):
  249. _apply_override(messages)
  250. def _store_rewrite_on_tail(agent, messages, raw, final) -> None:
  251. """A recovery exit whose text is already the unflushed tail row (the iteration-budget summary, a
  252. stop gate's answer kept as the budget fallback) stores the rewrite the user gets, not the model
  253. text (#44239). A held turn's hidden answer becomes the published reply; the model already saw
  254. it, so ``api_content`` keeps its bytes on the wire."""
  255. tail = messages[-1] if messages else None
  256. if not (isinstance(tail, dict) and tail.get("role") == "assistant" and tail.get("content") == raw
  257. and not tail.get(_DB_PERSISTED_MARKER)):
  258. return
  259. if tail.get("display_kind") == "hidden" and getattr(agent, "_hold_stream", False) is True:
  260. del tail["display_kind"]
  261. if final != raw:
  262. tail.setdefault("api_content", raw)
  263. tail["content"] = final
  264. def _micro_compact_after_turn(agent, messages, final_response, logger, task_id) -> None:
  265. """Post-turn micro-compaction: absorb the oldest uncompacted exchange into the
  266. rolling summary before persist, amortizing compression across turns."""
  267. try:
  268. _compressor = getattr(agent, "context_compressor", None)
  269. # Strict `is True` + callable gates: plugin context engines and MagicMock
  270. # compressors pass duck checks and would wipe the transcript. Never run while
  271. # compression.checkpoint_required is armed (no checkpoint hook here), nor for
  272. # persistence-isolated agents (background review fork): that burns an aux-LLM
  273. # call on a throwaway transcript and could compact the CANONICAL session rows.
  274. if (
  275. _compressor
  276. and getattr(_compressor, '_micro_compact_enabled', False) is True
  277. and callable(getattr(_compressor, '_micro_compact', None))
  278. and final_response
  279. and getattr(agent, "compression_checkpoint_required", False) is not True
  280. and not getattr(agent, "_persist_disabled", False)
  281. ):
  282. _before = len(messages)
  283. # A rotating compaction earlier in this turn may have moved the agent to a child session.
  284. _compacted = _compressor._micro_compact(
  285. messages, turn_session_id=getattr(agent, "_inflight_turn_session_id", None),
  286. )
  287. # Defrag rewrites the newest MICRO marker in place and pops _db_persisted;
  288. # the compressor flags us to invalidate the flush-scan cursor, else the
  289. # rewritten row is identity-skipped (stale).
  290. if getattr(_compressor, "_flush_scan_cursor_invalidated", False):
  291. _compressor._flush_scan_cursor_invalidated = False
  292. agent._db_flush_scan_prefix = None
  293. if isinstance(_compacted, list) and _compacted:
  294. _spliced = _compacted is not messages # no-op and defrag passes return the input
  295. messages[:] = _compacted
  296. if _spliced:
  297. # The splice summarized tool results away: a repeat read must serve them
  298. # again, not an "unchanged" stub pointing at a body that is gone (#32106).
  299. from agent.conversation_compression import _reset_read_dedup_caches
  300. _reset_read_dedup_caches(task_id, session_id=agent.session_id or "")
  301. if _before != len(messages):
  302. logger.info("Micro-compaction: %d -> %d messages", _before, len(messages))
  303. except Exception as _mc_err:
  304. logger.info("Micro-compaction failed: %s", _mc_err)
  305. def _log_turn_exit(agent, messages, final_response, api_call_count, _turn_exit_reason, interrupted, logger) -> None:
  306. """Always INFO so agent.log captures WHY every turn ended; WARNING when the last
  307. message is a tool result (the "just stops" scenario)."""
  308. _touch = getattr(agent, "_touch_activity", None)
  309. if callable(_touch):
  310. # Stamp the activity clock at the loop's end (#131740): the watchdog's stall
  311. # surface then names the finalizer ("turn end logged") instead of the last
  312. # API call, and a turn that wedges in the post-loop tail is measured — and
  313. # aborted — from when the loop actually finished, not from the last provider
  314. # response. Never raises into the finalizer.
  315. try:
  316. _touch("turn end logged")
  317. except Exception:
  318. logger.debug("turn-end activity stamp failed", exc_info=True)
  319. _last_msg_role = messages[-1].get("role") if messages else None
  320. _last_tool_name = None
  321. if _last_msg_role == "tool":
  322. # Walk back to the assistant message with the tool call.
  323. for _m in reversed(messages):
  324. if _m.get("role") == "assistant" and _m.get("tool_calls"):
  325. _tcs = _m["tool_calls"]
  326. if _tcs and isinstance(_tcs[0], dict):
  327. _last_tool_name = _tcs[-1].get("function", {}).get("name")
  328. break
  329. _turn_tool_count = sum(
  330. 1 for m in messages
  331. if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls")
  332. )
  333. # Fork turns (background review, side questions) carry ``_turn_origin``; tagging the
  334. # exit line keeps a fork's ``interrupted_during_api_call`` from reading as a killed
  335. # foreground stream — the fork shares the parent's session_id and often its model (#118693).
  336. _turn_origin = getattr(agent, "_turn_origin", None)
  337. _diag_msg = (
  338. "Turn ended: reason=%s model=%s api_calls=%d/%d budget=%d/%d "
  339. "tool_turns=%d last_msg_role=%s response_len=%d session=%s"
  340. + (" origin=%s" if _turn_origin else "")
  341. )
  342. _diag_args = (
  343. _turn_exit_reason, agent.model, api_call_count, agent.max_iterations,
  344. agent.iteration_budget.used if agent.iteration_budget else 0,
  345. agent.iteration_budget.max_total if agent.iteration_budget else 0,
  346. _turn_tool_count, _last_msg_role, len(final_response) if final_response else 0,
  347. agent.session_id or "none",
  348. *((_turn_origin,) if _turn_origin else ()),
  349. )
  350. if _last_msg_role == "tool" and not interrupted:
  351. logger.warning(
  352. "Turn ended with pending tool result (agent may appear stuck). "
  353. + _diag_msg + " last_tool=%s",
  354. *_diag_args, _last_tool_name,
  355. )
  356. else:
  357. logger.info(_diag_msg, *_diag_args)
  358. def _append_file_mutation_footer(agent, final_response, logger):
  359. """Append the verifier advisory when ``write_file`` / ``patch`` calls failed and were
  360. never superseded by a successful write to the same path (surfaces over-claiming)."""
  361. try:
  362. # File-mutation verifier footer. This catches the specific case — reported by Ben Eng
  363. # (#15524-adjacent) — where a model issues a batch of parallel patches, half of them fail with
  364. # "Could not find old_string", and the model summarises the turn claiming every file was edited. The
  365. # user then has to manually run ``git status`` to catch the lie. With this footer the truth is
  366. # surfaced on every turn, so over-claiming is structurally impossible past the model. Gate: only
  367. # applied when a real text response exists for this turn and the user didn't interrupt.
  368. # Empty/interrupted turns already have other surface text that shouldn't be augmented.
  369. _failed = getattr(agent, "_turn_failed_file_mutations", None) or {}
  370. if _failed and agent._file_mutation_verifier_enabled():
  371. _failed = agent._file_mutations_still_failed(_failed)
  372. footer = agent._format_file_mutation_failure_footer(_failed)
  373. if footer:
  374. final_response = final_response.rstrip() + "\n\n" + footer
  375. except Exception as _ver_err:
  376. logger.debug("file-mutation verifier footer failed: %s", _ver_err)
  377. return final_response
  378. def _explain_abnormal_exit(agent, final_response, _turn_exit_reason, preserved_verification_fallback, logger):
  379. """Turn-completion explainer: on abnormal exits, surface one explanation from
  380. ``_turn_exit_reason``. Only acts when no usable reply exists (empty, "(empty)",
  381. or a short unpunctuated fragment); ``text_response(...)`` exits stay silent."""
  382. try:
  383. if not agent._turn_completion_explainer_enabled():
  384. return final_response
  385. _stripped = (final_response or "").strip()
  386. _is_empty_terminal = _stripped in ("", "(empty)")
  387. # A short fragment not from a text_response exit and lacking sentence-ending
  388. # punctuation is treated as a truncated partial (#34452).
  389. _is_partial_fragment = (
  390. not _is_empty_terminal
  391. and not preserved_verification_fallback
  392. and not str(_turn_exit_reason).startswith("text_response")
  393. and _turn_exit_reason != "fallback_prior_turn_content"
  394. and len(_stripped) <= 24
  395. and _stripped[-1:] not in _SENTENCE_END
  396. )
  397. if _is_empty_terminal or _is_partial_fragment or str(_turn_exit_reason) == "partial_stream_recovery":
  398. _explanation = agent._format_turn_completion_explanation(
  399. _turn_exit_reason, getattr(agent, "_last_persistence_error_cause", None),
  400. db_path=getattr(getattr(agent, "_session_db", None), "db_path", None),
  401. model=str(getattr(agent, "model", "") or ""),
  402. )
  403. if _explanation:
  404. # Replace the bare sentinel; keep a partial fragment and append why.
  405. final_response = _explanation if _is_empty_terminal else _stripped + "\n\n" + _explanation
  406. except Exception as _exp_err:
  407. logger.debug("turn-completion explainer failed: %s", _exp_err)
  408. return final_response
  409. def _last_turn_reasoning(messages) -> Optional[Any]:
  410. """Reasoning from the CURRENT turn only: stop at this turn's user message (#17055),
  411. but take the most recent non-empty reasoning since many providers emit it on the
  412. tool-call step and leave the final step with reasoning=None."""
  413. for msg in reversed(messages):
  414. if msg.get("role") == "user":
  415. return None # turn boundary — don't cross into prior turns
  416. if msg.get("role") == "assistant" and msg.get("reasoning"):
  417. return msg["reasoning"]
  418. return None
  419. def _apply_output_hooks(
  420. agent, final_response, logger, *, platform, effective_task_id, turn_id, original_user_message,
  421. messages,
  422. ) -> tuple[Any, bool, Optional[Any]]:
  423. """Resolve the turn's ``transform_llm_output`` outcome, then fire ``post_llm_call`` once per
  424. turn after the tool loop. Returns ``(final_response, transformed, pre_transform_response)``.
  425. The transform itself normally already ran before the assistant row was first persisted
  426. (``apply_llm_output_transform`` from ``finish_text_response`` / ``_persist_step``); this
  427. call returns that recorded outcome, and only fires the hook here when no earlier seam saw a
  428. response (e.g. text that only appeared through ``_explain_abnormal_exit``)."""
  429. final_response, transformed, pre_transform = apply_llm_output_transform(
  430. agent, final_response, turn_id=turn_id, platform=platform, logger=logger,
  431. )
  432. # Detached forks are internal work and must not publish turns under the parent's session ID.
  433. if not getattr(agent, "_persist_disabled", False):
  434. # Leased on the turn thread so the capture holds this turn's context; None unless a
  435. # loaded plugin holds llm.side_question. The response must survive a failed capture.
  436. try:
  437. from agent.side_question_lease import issue_side_question_token
  438. side_question_token = issue_side_question_token(agent, messages, turn_id)
  439. except Exception:
  440. side_question_token = None
  441. logger.warning("side question lease failed", exc_info=True)
  442. _invoke_hook_safely(
  443. "post_llm_call", logger,
  444. session_id=agent.session_id,
  445. task_id=effective_task_id,
  446. turn_id=turn_id,
  447. user_message=original_user_message,
  448. assistant_response=final_response,
  449. conversation_history=list(messages),
  450. model=agent.model,
  451. platform=platform,
  452. side_question_token=side_question_token,
  453. )
  454. return final_response, transformed, pre_transform
  455. def apply_llm_output_transform(
  456. agent, final_response, *, turn_id, platform=None, logger=None,
  457. ) -> tuple[Any, bool, Optional[Any]]:
  458. """Fire ``transform_llm_output`` once per turn and return
  459. ``(final_response, transformed, pre_transform_response)``.
  460. Called BEFORE the final assistant row is first persisted — from ``finish_text_response``
  461. ahead of its durable flush, and from ``finalize_turn._persist_step`` ahead of the
  462. recovery-path tail close — so the text the user sees is the text stored in SQLite/JSON and
  463. replayed next turn (#44239). SQLite treats a non-blank assistant row as settled (a re-flush
  464. adopts the stored content rather than overwriting it), so transforming after that first
  465. write can never reach the durable store. Idempotent per ``turn_id``: later callers in the
  466. same turn get the recorded outcome instead of a second hook firing. Only the current
  467. turn's not-yet-written text is touched — earlier turns and the system prompt are never
  468. rewritten (prompt-cache invariant)."""
  469. if logger is None:
  470. from agent.conversation_loop import logger
  471. recorded = getattr(agent, "_llm_output_transform", None)
  472. if isinstance(recorded, tuple) and len(recorded) == 3 and recorded[0] == turn_id:
  473. _, transformed, pre_transform = recorded
  474. return final_response, transformed, pre_transform
  475. if not final_response:
  476. return final_response, False, None
  477. if platform is None:
  478. platform = getattr(agent, "platform", None) or ""
  479. transformed, pre_transform = False, None
  480. payload = dict(response_text=final_response, session_id=agent.session_id or "", model=agent.model,
  481. platform=platform, turn_id=turn_id) # turn_id: per-turn identity for the hook callback gate
  482. held = getattr(agent, "_held_transforms", None) if getattr(agent, "_hold_stream", False) else None
  483. if held:
  484. rewrite = _run_held_transforms(held, payload, logger)
  485. else:
  486. # First hook to return a string wins; None/empty leaves the text unchanged.
  487. rewrite = next((r for r in _invoke_hook_safely("transform_llm_output", logger, **payload)
  488. if isinstance(r, str) and r), None)
  489. if rewrite:
  490. pre_transform, final_response, transformed = final_response, rewrite, True
  491. agent._llm_output_transform = (turn_id, transformed, pre_transform)
  492. return final_response, transformed, pre_transform
  493. HELD_TRANSFORM_FAILED_REPLY = "This reply was withheld: the plugin that rewrites replies in this session failed."
  494. def _run_held_transforms(callbacks, payload: dict, logger) -> Optional[str]:
  495. """A held turn runs the transforms captured at its start (``held_stream_transforms``), not the live
  496. registry a mid-turn reload empties. When none rewrote and one raised or timed out, the user gets a
  497. fixed notice, never the text the plugin had to see first."""
  498. from hermes_cli.plugins import get_plugin_manager
  499. try:
  500. rewrite, failed = get_plugin_manager().run_held_transforms(callbacks, **payload)
  501. except Exception: # fail closed: whatever broke dispatch, the original stays withheld
  502. logger.warning("held transform_llm_output dispatch failed", exc_info=True)
  503. rewrite, failed = None, True
  504. return HELD_TRANSFORM_FAILED_REPLY if failed else rewrite
  505. def settle_held_final(agent, result: Any) -> None:
  506. """A held turn's reply leaves only through its transform snapshot. ``finish_text_response``,
  507. ``finalize_turn``, the Codex runtime and the truncation ceiling run it before persisting; every
  508. other exit (provider errors, retry exhaustion, interrupts, compression failures, ...) returns its
  509. text through ``run_conversation``, which calls this, so no path hands the user text the plugin
  510. did not see first, and a failed transform yields the withheld notice."""
  511. if getattr(agent, "_hold_stream", False) is not True or not isinstance(result, dict):
  512. return
  513. text, turn_id = result.get("final_response"), getattr(agent, "_current_turn_id", "") or ""
  514. recorded = getattr(agent, "_llm_output_transform", None)
  515. if not isinstance(text, str) or not text or (isinstance(recorded, tuple) and recorded[:1] == (turn_id,)):
  516. return
  517. final, transformed, _ = apply_llm_output_transform(agent, text, turn_id=turn_id)
  518. if result.get("error") == text: # keep the mirror an error card reads
  519. result["error"] = final
  520. result.update(final_response=final, response_transformed=transformed, pre_transform_response=None)
  521. def public_pre_transform(agent, pre_transform: Optional[Any]) -> Optional[Any]:
  522. """``pre_transform_response`` for a turn result: a held turn's original never leaves the agent
  523. (receipts, replay-gap recovery, post-stream suffix painting all see only the rewrite)."""
  524. return None if getattr(agent, "_hold_stream", False) else pre_transform
  525. def _mirror_turn_to_memory(
  526. agent, *, original_user_message, final_response, interrupted, messages, review_memory, review_skills, vetoed,
  527. ) -> None:
  528. """External-memory sync (+ next prefetch), then the background memory/skill review. A prompt a
  529. ``pre_llm_call`` hook vetoed never leaves the host, so neither runs for that turn."""
  530. if vetoed:
  531. return
  532. agent._sync_external_memory_for_turn(
  533. original_user_message=original_user_message, final_response=final_response,
  534. interrupted=interrupted, messages=messages,
  535. )
  536. # Background memory/skill review runs AFTER delivery so it never competes with the
  537. # user's task. Suppressed by skip_background_review (e.g. cron): the fork costs
  538. # ~30K tokens / event with no human-in-the-loop benefit. Best-effort; the review
  539. # clones the snapshot structurally so its sanitizers can't reach the live transcript.
  540. if (
  541. final_response
  542. and not interrupted
  543. and not getattr(agent, "skip_background_review", False)
  544. and (review_memory or review_skills)
  545. ):
  546. with suppress(Exception):
  547. agent._spawn_background_review(
  548. messages_snapshot=list(messages), review_memory=review_memory,
  549. review_skills=review_skills,
  550. )
  551. def finalize_turn(
  552. agent, *, final_response, api_call_count, interrupted, failed, messages, conversation_history,
  553. effective_task_id, turn_id, user_message, original_user_message, _should_review_memory,
  554. _turn_exit_reason, _pending_verification_response=None,
  555. _pending_verification_response_previewed=False,
  556. ):
  557. """Run the post-loop finalization and return the turn ``result`` dict."""
  558. from agent.conversation_loop import logger
  559. from agent.turn_context import PRE_LLM_BLOCK_EXIT_REASON
  560. vetoed = _turn_exit_reason == PRE_LLM_BLOCK_EXIT_REASON # a pre_llm_call block: no model call at all
  561. # Budget/error exits may never reach the final-text commit. Close admission
  562. # and preserve any accepted but unapplied correction for the recovery handoff.
  563. from contextlib import nullcontext
  564. with getattr(agent, "_pending_redirect_lock", None) or nullcontext():
  565. if getattr(agent, "_turn_redirect_open", False):
  566. agent._turn_redirect_open = False
  567. pending_redirect = agent._pending_redirect
  568. agent._pending_redirect = None
  569. if pending_redirect:
  570. agent.steer(pending_redirect)
  571. final_response, _turn_exit_reason, preserved_verification_fallback, interrupted = _resolve_budget_fallback(
  572. agent, final_response=final_response, api_call_count=api_call_count,
  573. interrupted=interrupted, failed=failed, messages=messages,
  574. _turn_exit_reason=_turn_exit_reason,
  575. _pending_verification_response=_pending_verification_response,
  576. _pending_verification_response_previewed=_pending_verification_response_previewed,
  577. logger=logger,
  578. )
  579. # A non-interrupted turn that fell out of the loop after a tool result, with no
  580. # follow-up assistant text, is the Desktop/TUI "silent stop" (#55316, #54756): the
  581. # composer returns to ready (or keeps spinning) while the durable transcript ends
  582. # at a raw ``tool`` row — the user never learns the turn stopped, and the next user
  583. # message lands as ``tool → user``. Interrupted tails keep
  584. # ``close_interrupted_tool_sequence``; this is the non-interrupt sibling. Mint the
  585. # exit reason, fail the turn, and synthesize the visible close so the tail close in
  586. # ``_persist_step`` persists an assistant row. A turn that already streamed text is
  587. # left alone: ``_recover_final_from_stream`` owns that recovery (#95514).
  588. if (
  589. not final_response
  590. and not interrupted
  591. and messages
  592. and isinstance(messages[-1], dict)
  593. and messages[-1].get("role") == "tool"
  594. and not (getattr(agent, "_current_streamed_assistant_text", "") or "").strip()
  595. ):
  596. _turn_exit_reason = "pending_tool_result"
  597. failed = True
  598. final_response = ""
  599. try:
  600. if agent._turn_completion_explainer_enabled():
  601. final_response = (
  602. agent._format_turn_completion_explanation("pending_tool_result", None) or ""
  603. )
  604. except Exception:
  605. final_response = ""
  606. if not final_response:
  607. # The turn-completion explainer opt-out must not reintroduce the silent stop.
  608. final_response = (
  609. "No reply: the turn stopped while a tool result was still pending. "
  610. "Send `continue` to let the model summarize."
  611. )
  612. # Loop exits that are failures in their own right (outer-loop error cap, shutdown, context
  613. # that could not be shrunk) carry the verdict the UI descriptor needs; a bare
  614. # ``turn_exit_reason`` collapsed to code="unknown", retryable=True on every surface.
  615. # Advisory verdicts (``fails_turn=False``) only add the code: ``failed``/``completed`` keep
  616. # the loop's values so cron, kanban and transcript persistence behave as before.
  617. _exit_failure = None if interrupted else exit_reason_failure(_turn_exit_reason)
  618. if _exit_failure is not None and _exit_failure.fails_turn:
  619. failed = True
  620. # Sibling producers (``turn_recovery``, ``codex_runtime``) return ``completed=False`` for an
  621. # interrupted turn; the gateway stream gate and the API run status rely on that contract.
  622. completed = (
  623. final_response is not None
  624. and not failed
  625. and not interrupted
  626. and (api_call_count < agent.max_iterations or str(_turn_exit_reason).startswith("text_response("))
  627. )
  628. _rollback_interrupted_preflight_display(agent, interrupted)
  629. from hermes_cli.observability.shared_metrics_harness import finish_turn
  630. finish_turn(agent, _turn_exit_reason, final_response, interrupted=interrupted, failed=failed)
  631. _cleanup_errors: list[str] = []
  632. # The model has answered (or the loop gave up): a title upgrade held back because it shares a
  633. # self-hosted endpoint with the main request (#117296) may go out now.
  634. from agent.turn_context import start_deferred_title_upgrade
  635. _guarded_cleanup("start_deferred_title_upgrade", lambda: start_deferred_title_upgrade(agent), _cleanup_errors, logger)
  636. # ``user_message`` may be a multimodal list of parts; the trajectory format wants a string.
  637. _guarded_cleanup(
  638. "save_trajectory",
  639. lambda: agent._save_trajectory(messages, _summarize_user_message_for_log(user_message), completed),
  640. _cleanup_errors, logger,
  641. )
  642. _guarded_cleanup(
  643. "cleanup_task_resources", lambda: agent._cleanup_task_resources(effective_task_id),
  644. _cleanup_errors, logger,
  645. )
  646. # Persist only after the transcript tail is shaped and scaffolding removed. Each
  647. # sub-step runs in the same order as the original inline block, and the
  648. # stream-recovered ``final_response`` is rebound the moment it is computed — BEFORE
  649. # the fallible tail-shaping / override / micro-compaction / persist calls — so a
  650. # raise in any of them can't drop text the user already saw (#95514, #8049).
  651. def _persist_step():
  652. nonlocal final_response
  653. _drop_transcript_scaffolding(agent, messages)
  654. final_response, _recovered_from_stream = _recover_final_from_stream(
  655. agent, final_response, interrupted, failed
  656. )
  657. # Recovery paths (stream-recovered / prior-turn text) reach here with a response no
  658. # earlier seam transformed; the normal text turn already did this before its flush and
  659. # gets the recorded outcome back. Either way the tail close below writes the text the
  660. # user will see, never the raw model text (#44239).
  661. if final_response and not interrupted:
  662. _raw = final_response
  663. final_response, _, _ = apply_llm_output_transform(agent, final_response, turn_id=turn_id, logger=logger)
  664. _store_rewrite_on_tail(agent, messages, _raw, final_response)
  665. _close_transcript_tail(agent, messages, final_response, interrupted, _recovered_from_stream)
  666. if not interrupted and not failed and not vetoed:
  667. _micro_compact_after_turn(agent, messages, final_response, logger, effective_task_id)
  668. agent._persist_session(messages, conversation_history)
  669. _guarded_cleanup("persist_session", _persist_step, _cleanup_errors, logger)
  670. # Keep the gateway's separate in-memory history snapshot current even on
  671. # cleanup error, so a later prompt isn't sent with a pre-turn snapshot.
  672. with suppress(Exception):
  673. agent._session_messages = messages
  674. _log_turn_exit(agent, messages, final_response, api_call_count, _turn_exit_reason, interrupted, logger)
  675. # Response transforms apply only to real, uninterrupted responses.
  676. if final_response and not interrupted:
  677. final_response = _append_file_mutation_footer(agent, final_response, logger)
  678. if not interrupted:
  679. final_response = _explain_abnormal_exit(
  680. agent, final_response, _turn_exit_reason, preserved_verification_fallback, logger,
  681. )
  682. _platform = getattr(agent, "platform", None) or ""
  683. _response_transformed = False
  684. _pre_transform_response = None
  685. if final_response and not interrupted:
  686. final_response, _response_transformed, _pre_transform_response = _apply_output_hooks(
  687. agent, final_response, logger, platform=_platform, effective_task_id=effective_task_id,
  688. turn_id=turn_id, original_user_message=original_user_message, messages=messages,
  689. )
  690. # Context engine observation hook: the turn finished with the finalized transcript.
  691. # Fail-open. ``_last_turn_usage`` is the last response's canonical usage dict, or
  692. # ``None`` on turns that never reached a provider response — by contract.
  693. try:
  694. from agent.conversation_loop import _notify_context_engine_turn_complete
  695. _notify_context_engine_turn_complete(
  696. agent, messages, usage=getattr(agent, "_last_turn_usage", None), logger=logger,
  697. turn_id=turn_id, task_id=effective_task_id, api_call_count=api_call_count,
  698. interrupted=interrupted, failed=failed, turn_exit_reason=_turn_exit_reason,
  699. )
  700. except Exception as exc:
  701. logger.warning("on_turn_complete notification failed: %s", exc)
  702. # Surrogate chokepoint: RAW SDK text with a lone UTF-16 surrogate crashes downstream
  703. # consumers (stdout, Telegram ``utf16_len``, JSON); scrub once where it leaves the loop.
  704. # Class-level surrogate chokepoint (#80366, #55143, #55309, #19819): ``final_response`` is often the RAW
  705. # SDK content (``assistant_message.content``), not the sanitized copy stored in history by
  706. # ``build_assistant_message``. Any lone UTF-16 surrogate (U+D800–U+DFFF) in it crashes downstream
  707. # consumers — oneshot stdout writes, Telegram's ``utf16_len`` length check, Signal formatting, JSON
  708. # envelope encodes — on every provider (Ollama, NVIDIA NIM, …). Scrub once here, where model text leaves
  709. # the conversation loop, so every delivery surface receives valid Unicode.
  710. if isinstance(final_response, str):
  711. final_response = _sanitize_surrogates(final_response)
  712. result = {
  713. "final_response": final_response,
  714. "last_reasoning": _last_turn_reasoning(messages),
  715. "messages": messages,
  716. "api_calls": api_call_count,
  717. "completed": completed,
  718. "turn_exit_reason": _turn_exit_reason,
  719. "failed": failed,
  720. "partial": False, # True only when stopped due to invalid tool calls
  721. "interrupted": interrupted,
  722. "response_transformed": _response_transformed,
  723. "pre_transform_response": public_pre_transform(agent, _pre_transform_response),
  724. "response_previewed": getattr(agent, "_response_was_previewed", False),
  725. # The final is byte-for-byte a response this turn already delivered (no footer or
  726. # explanation appended since): it carries no new text for the client to paint.
  727. "response_reused": bool(final_response) and final_response == getattr(agent, "_reused_response_text", None),
  728. "model": agent.model,
  729. # requested_model / served_model: proxy-reported deployment or Hermes' own fallback route.
  730. **result_model_fields(agent),
  731. "provider": agent.provider,
  732. "base_url": agent.base_url,
  733. **{key: getattr(agent, f"session_{key}") for key in _SESSION_TOKEN_KEYS},
  734. # Gateway SessionEntry persists an API reading, never the preflight display seed.
  735. "last_prompt_tokens": (
  736. getattr(agent.context_compressor, "last_real_prompt_tokens", agent.context_compressor.last_prompt_tokens)
  737. if getattr(agent.context_compressor, "last_prompt_tokens", 0) > 0
  738. else getattr(agent.context_compressor, "last_prompt_tokens", 0)
  739. ) or 0,
  740. **{key: getattr(agent, f"session_{key}") for key in _SESSION_COST_KEYS},
  741. # Requested service tier, for billing audits (`hermes -z --usage-file`).
  742. "service_tier": (
  743. (getattr(agent, "request_overrides", {}) or {}).get("extra_body") or {}
  744. ).get("service_tier"),
  745. "session_id": agent.session_id,
  746. }
  747. if agent._tool_guardrail_halt_decision is not None:
  748. result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata()
  749. # Persistence failures already set failed=True; also stamp `error` so the gateway
  750. # surfaces status="error" (desktop can toast) instead of a quiet complete frame, plus
  751. # the machine-readable cause 'session_persistence_failed:<locked|compression|...>'.
  752. if failed and str(_turn_exit_reason) == "session_persistence_failed":
  753. from hermes_constants import profile_cli_selector
  754. # Never rebind final_response here: the memory sync and the background-review gate
  755. # below must still see an empty response on a persistence-failed turn.
  756. result["error"] = final_response or (
  757. "session storage could not be written — check the state database "
  758. f"health (`hermes {profile_cli_selector()}doctor`), then send your message again"
  759. )
  760. _cause = getattr(agent, "_last_persistence_error_cause", None)
  761. result["failure_reason"] = "session_persistence_failed:" + (_cause or "unknown")
  762. elif _exit_failure is not None:
  763. if failed:
  764. result["error"] = final_response or str(_turn_exit_reason)
  765. stamp_failure(result, _exit_failure.reason, _exit_failure.retryable)
  766. # Cleanup failures are surfaced, but the response is returned either way (#8049).
  767. if _cleanup_errors:
  768. result["cleanup_errors"] = _cleanup_errors
  769. # A /steer landing after the final assistant turn has no tool batch to drain into;
  770. # hand it back so it becomes the next user turn instead of being lost. A gateway transfer
  771. # (ticketed) stays buffered: the claim's settlement returns it to its own queue place.
  772. _leftover_steer = (getattr(agent, "_drain_leftover_steer", None) or agent._drain_pending_steer)()
  773. if _leftover_steer:
  774. result["pending_steer"] = _leftover_steer
  775. agent._response_was_previewed = False
  776. agent._reused_response_text = None
  777. if interrupted and agent._interrupt_message:
  778. result["interrupt_message"] = agent._interrupt_message
  779. agent.clear_interrupt()
  780. agent._stream_callback = None # don't leak into future calls
  781. # A voice turn's model route ends with the turn: memory sync and the background review
  782. # below (and the next turn) run on the session's main model.
  783. from agent.voice_turn_route import end_voice_turn_route
  784. end_voice_turn_route(agent)
  785. # Skill trigger is checked NOW — based on how many tool iterations THIS turn used.
  786. _should_review_skills = (
  787. agent._skill_nudge_interval > 0
  788. and agent._iters_since_skill >= agent._skill_nudge_interval
  789. and "skill_manage" in agent.valid_tool_names
  790. )
  791. if _should_review_skills:
  792. agent._iters_since_skill = 0
  793. _mirror_turn_to_memory(
  794. agent, original_user_message=original_user_message, final_response=final_response,
  795. interrupted=interrupted, messages=messages, review_memory=_should_review_memory,
  796. review_skills=_should_review_skills, vetoed=vetoed,
  797. )
  798. # Opt-in (compression.prepare_ahead): near the trigger, write the next compaction summary
  799. # between turns. No-op otherwise; never raises. Not after a vetoed turn (no model calls).
  800. from agent.prepared_compaction import maybe_prepare
  801. if not vetoed:
  802. maybe_prepare(agent, messages, origin="turn-end")
  803. # Memory provider on_session_end()/shutdown_all() are NOT called here:
  804. # run_conversation() runs once per message; CLI/gateway own session-end cleanup.
  805. if not getattr(agent, "_persist_disabled", False):
  806. _invoke_hook_safely(
  807. "on_session_end", logger,
  808. session_id=agent.session_id,
  809. task_id=effective_task_id,
  810. turn_id=turn_id,
  811. completed=completed,
  812. failed=failed,
  813. interrupted=interrupted,
  814. turn_exit_reason=_turn_exit_reason,
  815. model=agent.model,
  816. platform=_platform,
  817. )
  818. agent._turn_preflight_display_snapshot = None
  819. agent._turn_received_provider_response = False
  820. return result