| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912 |
- """Post-loop turn finalization for ``run_conversation``.
- Budget summary, trajectory save, persist, diagnostics, response transforms, result
- assembly, steer drain, memory/skill review. Synchronous, single return. ``logger`` is
- imported lazily from ``agent.conversation_loop`` (no cycle, same logger name)."""
- from __future__ import annotations
- import logging
- import os
- import time
- from contextlib import suppress
- from typing import Any, Callable, List, Optional, Tuple
- from agent.codex_responses_adapter import _summarize_user_message_for_log
- from agent.delegation_context import is_dispatcher_owned_worker_context
- from agent.interrupt_control import interrupted_during_api_call_reason
- from agent.turn_failure_copy import exit_reason_failure, stamp_failure
- from agent.context_compressor import _DB_PERSISTED_MARKER
- from agent.message_content import flatten_message_text
- from agent.message_metadata import append_message, stamp_message_timestamp
- from agent.message_sanitization import _sanitize_surrogates
- from agent.served_model import result_model_fields
- # Verification-continuation nudges (verify-on-stop / pre_verify) must be stripped from
- # returned/live history to avoid role-alternation breaks; the assistant response is
- # real content and is not flagged. (#65919)
- _VERIFICATION_CONTINUATION_FLAGS = ("_verification_stop_synthetic", "_pre_verify_synthetic")
- _SENTENCE_END = {".", "!", "?", "。", "!", "?", "`", ")"}
- # ``result[key] = agent.session_<key>`` for the per-session usage/cost counters.
- _SESSION_TOKEN_KEYS = (
- "input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens",
- "reasoning_tokens", "prompt_tokens", "completion_tokens", "total_tokens",
- )
- _SESSION_COST_KEYS = ("estimated_cost_usd", "cost_status", "cost_source")
- def _assistant_row_missing_visible_text(msg: dict) -> bool:
- """True when an assistant row has no visible text (blank final or tool-only)."""
- if not isinstance(msg, dict) or msg.get("role") != "assistant":
- return False
- return not flatten_message_text(msg.get("content")).strip()
- def _record_kanban_budget_exhausted(
- kanban_task: str, api_call_count: int, max_iterations: int, logger: logging.Logger
- ) -> None:
- """Record a terminal ``timed_out`` outcome for a kanban worker out of budget.
- Routed via ``_record_task_failure`` (not ``kanban_block``) so it counts toward the
- consecutive-failure circuit breaker. Idempotent via the ``_end_run`` CAS
- (``WHERE ended_at IS NULL``), so safe from multiple exit paths.
- This is a bounded fallback (#87096): the CAS invariant in ``_end_run`` (``WHERE ended_at IS NULL``)
- guarantees idempotence — if another path already closed the run this is a no-op — so it is safe to call
- from multiple exit paths.
- """
- try:
- from hermes_cli import kanban_db as _kb
- from hermes_cli import kanban_db_connect as _kbc
- from hermes_cli import kanban_db_dispatch as _kbd
- _conn = _kbc.connect()
- try:
- _kbd._record_task_failure(
- _conn,
- kanban_task,
- error=(
- f"Iteration budget exhausted ({api_call_count}/{max_iterations}) — "
- "task could not complete within the allowed iterations"
- ),
- outcome="timed_out",
- release_claim=True,
- end_run=True,
- event_payload_extra={"budget_used": api_call_count, "budget_max": max_iterations},
- )
- finally:
- with suppress(Exception):
- _conn.close()
- except Exception:
- logger.warning(
- "Failed to record budget-exhausted failure for task %s", kanban_task, exc_info=True
- )
- def _drop_verification_continuation_scaffolding(messages) -> None:
- """Remove verification-continuation nudges in place; only the synthetic nudges carry
- these flags, so the real attempted final answer persisted to state.db survives."""
- messages[:] = [
- m for m in messages
- if not (isinstance(m, dict) and any(m.get(f) for f in _VERIFICATION_CONTINUATION_FLAGS))
- ]
- def _clone_background_review_messages(messages):
- """Copy the review input without aliasing the live transcript."""
- # Lazy: conversation_loop imports this module (cycle).
- from agent.conversation_loop import _clone_message_for_send
- return [_clone_message_for_send(message) for message in messages]
- def _invoke_hook_safely(name: str, logger: logging.Logger, **kwargs) -> list:
- """Fire a lifecycle plugin hook; a failing hook is logged, never fatal."""
- try:
- from hermes_cli.lifecycle import invoke_hook
- return invoke_hook(name, **kwargs)
- except Exception as exc:
- logger.warning("%s hook failed: %s", name, exc)
- return []
- def _guarded_cleanup(label: str, fn: Callable[[], Any], errors: list[str], logger) -> None:
- """Post-loop cleanup must never lose the response: each step is guarded
- independently and errors surface via ``cleanup_errors`` (#8049)."""
- try:
- fn()
- except Exception as err:
- errors.append(f"{label}: {err}")
- logger.error("finalize_turn: _%s failed: %s", label, err, exc_info=True)
- def _resolve_budget_fallback(
- agent, *, final_response, api_call_count, interrupted, failed, messages, _turn_exit_reason,
- _pending_verification_response, _pending_verification_response_previewed, logger,
- ) -> tuple[Any, Any, bool, Any]:
- """Iteration-budget exhaustion. Returns ``(final_response, _turn_exit_reason,
- preserved_verification_fallback, interrupted)``."""
- budget_exhausted = (
- api_call_count >= agent.max_iterations or agent.iteration_budget.remaining <= 0
- )
- preserved_verification_fallback = False
- if (
- final_response is None and budget_exhausted and not interrupted and not failed
- and str(_turn_exit_reason) in {"unknown", "budget_exhausted"}
- ):
- _turn_exit_reason = f"max_iterations_reached({api_call_count}/{agent.max_iterations})"
- if _pending_verification_response:
- # A verification gate withheld a composed answer, then the budget ran out:
- # preserve it rather than make another fallible call. The explicit pending
- # value is the provenance guard; unrelated error exits never enter here.
- # Previewed only if the reused candidate was actually streamed as interim.
- final_response = _pending_verification_response
- if _pending_verification_response_previewed:
- agent._response_was_previewed = True
- # Reuses the candidate the user already saw sealed as an interim (#130396).
- agent._reused_response_text = final_response
- preserved_verification_fallback = True
- else:
- # _handle_max_iterations makes one extra toolless request for a summary.
- agent._emit_diagnostic_status(
- f"⚠️ Iteration budget exhausted ({api_call_count}/{agent.max_iterations}) "
- "— asking model to summarise"
- )
- if not agent.quiet_mode:
- agent._safe_print(
- f"\n⚠️ Iteration budget exhausted ({api_call_count}/{agent.max_iterations}) "
- "— requesting summary...", diagnostic=True,
- )
- _summary_start = time.time()
- try:
- final_response = agent._handle_max_iterations(messages, api_call_count)
- except InterruptedError:
- # The turn ends interrupted, so the pending interrupt message is returned
- # for requeue instead of being cleared behind a fallback summary. A redirect
- # also ends it: the budget is spent, so there is no loop to restart into.
- from agent.conversation_loop import INTERRUPT_WAITING_FOR_MODEL_PREFIX
- interrupted = True
- _turn_exit_reason = interrupted_during_api_call_reason(agent)
- final_response = f"{INTERRUPT_WAITING_FOR_MODEL_PREFIX}{time.time() - _summary_start:.1f}s elapsed)."
- # A kanban worker must record a terminal outcome whether or not a fallback path
- # was eligible, so the dispatcher learns the worker could not complete. Only the
- # dispatcher-owned worker owns the task: an in-process delegate_task child or cron run
- # inherits ``HERMES_KANBAN_TASK`` via os.environ but exhausting ITS budget must not
- # close the parent's run and release its claim (#112817).
- _kanban_task = (
- os.environ.get("HERMES_KANBAN_TASK")
- if budget_exhausted and is_dispatcher_owned_worker_context() else None
- )
- # If running as a kanban worker, signal the dispatcher that the worker could not complete (rather than
- # treating it as a protocol violation). This applies whether the user-facing fallback came from the
- # summary call or an explicitly pending continuation; both exhausted the task budget and must advance
- # the failure circuit. We route through ``_record_task_failure(outcome="timed_out")`` rather than
- # ``kanban_block`` so this counts toward the dispatcher's consecutive-failure circuit breaker (#29747
- # gap 2).
- # Bounded fallback (#87096): budget was exhausted but none of the normal fallback paths were eligible
- # (interrupted / failed / anomalous exit_reason). If running as a kanban worker we must still record a
- # terminal outcome so the task does not remain in an ambiguous lifecycle state. The worker's run is
- # closed via ``_record_task_failure`` (compare-and-swap receipt path) which is a no-op if another path
- # closed it — the CAS invariant in ``_end_run`` (``WHERE ended_at IS NULL``) guarantees idempotence.
- if _kanban_task:
- _record_kanban_budget_exhausted(_kanban_task, api_call_count, agent.max_iterations, logger)
- return final_response, _turn_exit_reason, preserved_verification_fallback, interrupted
- def _rollback_interrupted_preflight_display(agent, interrupted) -> None:
- """Roll back the preflight-seeded display count only when an interrupt wins before
- any provider response; compaction state (incl. ``-1``) stays with the real-usage
- path. Type-pinned guards keep MagicMock/SimpleNamespace doubles inert."""
- _preflight_snapshot = getattr(agent, "_turn_preflight_display_snapshot", None)
- if (
- interrupted is True
- and isinstance(_preflight_snapshot, int)
- and not isinstance(_preflight_snapshot, bool)
- and getattr(agent, "_turn_received_provider_response", False) is not True
- and getattr(agent, "context_compressor", None) is not None
- ):
- _rollback_fn = getattr(
- agent.context_compressor, "rollback_interrupted_preflight_display_tokens", None
- )
- if callable(_rollback_fn):
- _rollback_fn(_preflight_snapshot)
- def _drop_transcript_scaffolding(agent, messages) -> None:
- """Strip private retry scaffolding first, or a later "continue" replays
- assistant("(empty)") / recovery nudges into the same empty-response loop. Only
- the synthetic verification nudges go; the assistant candidate persists (#65919)."""
- agent._drop_trailing_empty_response_scaffolding(messages)
- _drop_verification_continuation_scaffolding(messages)
- def _recover_final_from_stream(agent, final_response, interrupted, failed) -> tuple[Any, bool]:
- """An empty terminal completion is not authoritative when the stream already
- delivered text; recover before persist so a blank tail isn't frozen (#95514).
- Returns ``(final_response, recovered_from_stream)``. Called by the finalizer BEFORE
- the fallible tail-shaping/persist steps so the recovered text is already bound when
- one of them raises — a persist failure must not lose text the user already saw."""
- if interrupted or failed:
- return final_response, False
- _streamed = getattr(agent, "_current_streamed_assistant_text", "") or ""
- _streamed = _streamed.strip() if isinstance(_streamed, str) else ""
- if not (flatten_message_text(final_response).strip() if final_response else "") and _streamed:
- return _streamed, True
- return final_response, False
- def _close_transcript_tail(agent, messages, final_response, interrupted, _recovered_from_stream) -> None:
- """Shape the transcript tail before the durable snapshot (scaffolding already dropped
- and ``final_response`` already stream-recovered by the caller)."""
- # An interrupt can leave a tool result as the tail; close the sequence so strict
- # providers don't see ``tool → user`` (placeholder: final_response is usually empty).
- if interrupted:
- from agent.message_sanitization import close_interrupted_tool_sequence
- close_interrupted_tool_sequence(messages, final_response, stopped=True)
- # Recovery ``break`` sites can return a final_response with no closing assistant
- # row; enforce "delivered final_response ⇒ assistant row" here. Compare content,
- # not role, so a matching verification candidate isn't dup'd.
- if final_response and not interrupted:
- # Some recovery/fallback paths return a real final_response without adding a closing assistant
- # message to the transcript (e.g. the partial-stream and prior-turn-content recovery ``break`` sites
- # in ``conversation_loop``). If persisted as-is, the durable session can end at a tool/user message
- # even though the caller — and the gateway platform — already saw a completed assistant response.
- # The next turn then replays a user-only backlog and the model re-answers every "unanswered"
- # message. Close the durable turn at the source, at the single chokepoint every recovery ``break``
- # flows through, so the invariant "delivered final_response ⇒ assistant row in transcript" holds
- # regardless of which path produced it. (#43849 / #44100) Compare content (not just role) so a
- # verification candidate that matches the final response is not duplicated at budget exhaustion.
- # (#65919 §7)
- _tail = messages[-1] if messages else None
- if not isinstance(_tail, dict) or _tail.get("role") != "assistant":
- append_message(messages, {"role": "assistant", "content": final_response})
- elif (
- _tail.get("content") != final_response
- and _assistant_row_missing_visible_text(_tail)
- and (_tail.get("tool_calls") or _recovered_from_stream)
- ):
- # Pure tool-call turn or stream-recovered blank (#95514): fill the persisted
- # blank row's content rather than append a second row.
- _tail["content"] = final_response
- stamp_message_timestamp(_tail)
- _tail.pop(_DB_PERSISTED_MARKER, None)
- agent._db_flush_scan_prefix = None
- # Request is complete, so replace API-local voice/model/skill guidance with the
- # clean user input before the durable snapshot (earlier flushes still needed them).
- # Earlier turn-start flushes use the DB-only override because their messages are still needed for the
- # API request; this finalizer runs after that request is complete (#48677 / #63766).
- _apply_override = getattr(agent, "_apply_persist_user_message_override", None)
- if callable(_apply_override):
- _apply_override(messages)
- def _store_rewrite_on_tail(agent, messages, raw, final) -> None:
- """A recovery exit whose text is already the unflushed tail row (the iteration-budget summary, a
- stop gate's answer kept as the budget fallback) stores the rewrite the user gets, not the model
- text (#44239). A held turn's hidden answer becomes the published reply; the model already saw
- it, so ``api_content`` keeps its bytes on the wire."""
- tail = messages[-1] if messages else None
- if not (isinstance(tail, dict) and tail.get("role") == "assistant" and tail.get("content") == raw
- and not tail.get(_DB_PERSISTED_MARKER)):
- return
- if tail.get("display_kind") == "hidden" and getattr(agent, "_hold_stream", False) is True:
- del tail["display_kind"]
- if final != raw:
- tail.setdefault("api_content", raw)
- tail["content"] = final
- def _micro_compact_after_turn(agent, messages, final_response, logger, task_id) -> None:
- """Post-turn micro-compaction: absorb the oldest uncompacted exchange into the
- rolling summary before persist, amortizing compression across turns."""
- try:
- _compressor = getattr(agent, "context_compressor", None)
- # Strict `is True` + callable gates: plugin context engines and MagicMock
- # compressors pass duck checks and would wipe the transcript. Never run while
- # compression.checkpoint_required is armed (no checkpoint hook here), nor for
- # persistence-isolated agents (background review fork): that burns an aux-LLM
- # call on a throwaway transcript and could compact the CANONICAL session rows.
- if (
- _compressor
- and getattr(_compressor, '_micro_compact_enabled', False) is True
- and callable(getattr(_compressor, '_micro_compact', None))
- and final_response
- and getattr(agent, "compression_checkpoint_required", False) is not True
- and not getattr(agent, "_persist_disabled", False)
- ):
- _before = len(messages)
- # A rotating compaction earlier in this turn may have moved the agent to a child session.
- _compacted = _compressor._micro_compact(
- messages, turn_session_id=getattr(agent, "_inflight_turn_session_id", None),
- )
- # Defrag rewrites the newest MICRO marker in place and pops _db_persisted;
- # the compressor flags us to invalidate the flush-scan cursor, else the
- # rewritten row is identity-skipped (stale).
- if getattr(_compressor, "_flush_scan_cursor_invalidated", False):
- _compressor._flush_scan_cursor_invalidated = False
- agent._db_flush_scan_prefix = None
- if isinstance(_compacted, list) and _compacted:
- _spliced = _compacted is not messages # no-op and defrag passes return the input
- messages[:] = _compacted
- if _spliced:
- # The splice summarized tool results away: a repeat read must serve them
- # again, not an "unchanged" stub pointing at a body that is gone (#32106).
- from agent.conversation_compression import _reset_read_dedup_caches
- _reset_read_dedup_caches(task_id, session_id=agent.session_id or "")
- if _before != len(messages):
- logger.info("Micro-compaction: %d -> %d messages", _before, len(messages))
- except Exception as _mc_err:
- logger.info("Micro-compaction failed: %s", _mc_err)
- def _log_turn_exit(agent, messages, final_response, api_call_count, _turn_exit_reason, interrupted, logger) -> None:
- """Always INFO so agent.log captures WHY every turn ended; WARNING when the last
- message is a tool result (the "just stops" scenario)."""
- _touch = getattr(agent, "_touch_activity", None)
- if callable(_touch):
- # Stamp the activity clock at the loop's end (#131740): the watchdog's stall
- # surface then names the finalizer ("turn end logged") instead of the last
- # API call, and a turn that wedges in the post-loop tail is measured — and
- # aborted — from when the loop actually finished, not from the last provider
- # response. Never raises into the finalizer.
- try:
- _touch("turn end logged")
- except Exception:
- logger.debug("turn-end activity stamp failed", exc_info=True)
- _last_msg_role = messages[-1].get("role") if messages else None
- _last_tool_name = None
- if _last_msg_role == "tool":
- # Walk back to the assistant message with the tool call.
- for _m in reversed(messages):
- if _m.get("role") == "assistant" and _m.get("tool_calls"):
- _tcs = _m["tool_calls"]
- if _tcs and isinstance(_tcs[0], dict):
- _last_tool_name = _tcs[-1].get("function", {}).get("name")
- break
- _turn_tool_count = sum(
- 1 for m in messages
- if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls")
- )
- # Fork turns (background review, side questions) carry ``_turn_origin``; tagging the
- # exit line keeps a fork's ``interrupted_during_api_call`` from reading as a killed
- # foreground stream — the fork shares the parent's session_id and often its model (#118693).
- _turn_origin = getattr(agent, "_turn_origin", None)
- _diag_msg = (
- "Turn ended: reason=%s model=%s api_calls=%d/%d budget=%d/%d "
- "tool_turns=%d last_msg_role=%s response_len=%d session=%s"
- + (" origin=%s" if _turn_origin else "")
- )
- _diag_args = (
- _turn_exit_reason, agent.model, api_call_count, agent.max_iterations,
- agent.iteration_budget.used if agent.iteration_budget else 0,
- agent.iteration_budget.max_total if agent.iteration_budget else 0,
- _turn_tool_count, _last_msg_role, len(final_response) if final_response else 0,
- agent.session_id or "none",
- *((_turn_origin,) if _turn_origin else ()),
- )
- if _last_msg_role == "tool" and not interrupted:
- logger.warning(
- "Turn ended with pending tool result (agent may appear stuck). "
- + _diag_msg + " last_tool=%s",
- *_diag_args, _last_tool_name,
- )
- else:
- logger.info(_diag_msg, *_diag_args)
- def _append_file_mutation_footer(agent, final_response, logger):
- """Append the verifier advisory when ``write_file`` / ``patch`` calls failed and were
- never superseded by a successful write to the same path (surfaces over-claiming)."""
- try:
- # File-mutation verifier footer. This catches the specific case — reported by Ben Eng
- # (#15524-adjacent) — where a model issues a batch of parallel patches, half of them fail with
- # "Could not find old_string", and the model summarises the turn claiming every file was edited. The
- # user then has to manually run ``git status`` to catch the lie. With this footer the truth is
- # surfaced on every turn, so over-claiming is structurally impossible past the model. Gate: only
- # applied when a real text response exists for this turn and the user didn't interrupt.
- # Empty/interrupted turns already have other surface text that shouldn't be augmented.
- _failed = getattr(agent, "_turn_failed_file_mutations", None) or {}
- if _failed and agent._file_mutation_verifier_enabled():
- _failed = agent._file_mutations_still_failed(_failed)
- footer = agent._format_file_mutation_failure_footer(_failed)
- if footer:
- final_response = final_response.rstrip() + "\n\n" + footer
- except Exception as _ver_err:
- logger.debug("file-mutation verifier footer failed: %s", _ver_err)
- return final_response
- def _explain_abnormal_exit(agent, final_response, _turn_exit_reason, preserved_verification_fallback, logger):
- """Turn-completion explainer: on abnormal exits, surface one explanation from
- ``_turn_exit_reason``. Only acts when no usable reply exists (empty, "(empty)",
- or a short unpunctuated fragment); ``text_response(...)`` exits stay silent."""
- try:
- if not agent._turn_completion_explainer_enabled():
- return final_response
- _stripped = (final_response or "").strip()
- _is_empty_terminal = _stripped in ("", "(empty)")
- # A short fragment not from a text_response exit and lacking sentence-ending
- # punctuation is treated as a truncated partial (#34452).
- _is_partial_fragment = (
- not _is_empty_terminal
- and not preserved_verification_fallback
- and not str(_turn_exit_reason).startswith("text_response")
- and _turn_exit_reason != "fallback_prior_turn_content"
- and len(_stripped) <= 24
- and _stripped[-1:] not in _SENTENCE_END
- )
- if _is_empty_terminal or _is_partial_fragment or str(_turn_exit_reason) == "partial_stream_recovery":
- _explanation = agent._format_turn_completion_explanation(
- _turn_exit_reason, getattr(agent, "_last_persistence_error_cause", None),
- db_path=getattr(getattr(agent, "_session_db", None), "db_path", None),
- model=str(getattr(agent, "model", "") or ""),
- )
- if _explanation:
- # Replace the bare sentinel; keep a partial fragment and append why.
- final_response = _explanation if _is_empty_terminal else _stripped + "\n\n" + _explanation
- except Exception as _exp_err:
- logger.debug("turn-completion explainer failed: %s", _exp_err)
- return final_response
- def _last_turn_reasoning(messages) -> Optional[Any]:
- """Reasoning from the CURRENT turn only: stop at this turn's user message (#17055),
- but take the most recent non-empty reasoning since many providers emit it on the
- tool-call step and leave the final step with reasoning=None."""
- for msg in reversed(messages):
- if msg.get("role") == "user":
- return None # turn boundary — don't cross into prior turns
- if msg.get("role") == "assistant" and msg.get("reasoning"):
- return msg["reasoning"]
- return None
- def _apply_output_hooks(
- agent, final_response, logger, *, platform, effective_task_id, turn_id, original_user_message,
- messages,
- ) -> tuple[Any, bool, Optional[Any]]:
- """Resolve the turn's ``transform_llm_output`` outcome, then fire ``post_llm_call`` once per
- turn after the tool loop. Returns ``(final_response, transformed, pre_transform_response)``.
- The transform itself normally already ran before the assistant row was first persisted
- (``apply_llm_output_transform`` from ``finish_text_response`` / ``_persist_step``); this
- call returns that recorded outcome, and only fires the hook here when no earlier seam saw a
- response (e.g. text that only appeared through ``_explain_abnormal_exit``)."""
- final_response, transformed, pre_transform = apply_llm_output_transform(
- agent, final_response, turn_id=turn_id, platform=platform, logger=logger,
- )
- # Detached forks are internal work and must not publish turns under the parent's session ID.
- if not getattr(agent, "_persist_disabled", False):
- # Leased on the turn thread so the capture holds this turn's context; None unless a
- # loaded plugin holds llm.side_question. The response must survive a failed capture.
- try:
- from agent.side_question_lease import issue_side_question_token
- side_question_token = issue_side_question_token(agent, messages, turn_id)
- except Exception:
- side_question_token = None
- logger.warning("side question lease failed", exc_info=True)
- _invoke_hook_safely(
- "post_llm_call", logger,
- session_id=agent.session_id,
- task_id=effective_task_id,
- turn_id=turn_id,
- user_message=original_user_message,
- assistant_response=final_response,
- conversation_history=list(messages),
- model=agent.model,
- platform=platform,
- side_question_token=side_question_token,
- )
- return final_response, transformed, pre_transform
- def apply_llm_output_transform(
- agent, final_response, *, turn_id, platform=None, logger=None,
- ) -> tuple[Any, bool, Optional[Any]]:
- """Fire ``transform_llm_output`` once per turn and return
- ``(final_response, transformed, pre_transform_response)``.
- Called BEFORE the final assistant row is first persisted — from ``finish_text_response``
- ahead of its durable flush, and from ``finalize_turn._persist_step`` ahead of the
- recovery-path tail close — so the text the user sees is the text stored in SQLite/JSON and
- replayed next turn (#44239). SQLite treats a non-blank assistant row as settled (a re-flush
- adopts the stored content rather than overwriting it), so transforming after that first
- write can never reach the durable store. Idempotent per ``turn_id``: later callers in the
- same turn get the recorded outcome instead of a second hook firing. Only the current
- turn's not-yet-written text is touched — earlier turns and the system prompt are never
- rewritten (prompt-cache invariant)."""
- if logger is None:
- from agent.conversation_loop import logger
- recorded = getattr(agent, "_llm_output_transform", None)
- if isinstance(recorded, tuple) and len(recorded) == 3 and recorded[0] == turn_id:
- _, transformed, pre_transform = recorded
- return final_response, transformed, pre_transform
- if not final_response:
- return final_response, False, None
- if platform is None:
- platform = getattr(agent, "platform", None) or ""
- transformed, pre_transform = False, None
- payload = dict(response_text=final_response, session_id=agent.session_id or "", model=agent.model,
- platform=platform, turn_id=turn_id) # turn_id: per-turn identity for the hook callback gate
- held = getattr(agent, "_held_transforms", None) if getattr(agent, "_hold_stream", False) else None
- if held:
- rewrite = _run_held_transforms(held, payload, logger)
- else:
- # First hook to return a string wins; None/empty leaves the text unchanged.
- rewrite = next((r for r in _invoke_hook_safely("transform_llm_output", logger, **payload)
- if isinstance(r, str) and r), None)
- if rewrite:
- pre_transform, final_response, transformed = final_response, rewrite, True
- agent._llm_output_transform = (turn_id, transformed, pre_transform)
- return final_response, transformed, pre_transform
- HELD_TRANSFORM_FAILED_REPLY = "This reply was withheld: the plugin that rewrites replies in this session failed."
- def _run_held_transforms(callbacks, payload: dict, logger) -> Optional[str]:
- """A held turn runs the transforms captured at its start (``held_stream_transforms``), not the live
- registry a mid-turn reload empties. When none rewrote and one raised or timed out, the user gets a
- fixed notice, never the text the plugin had to see first."""
- from hermes_cli.plugins import get_plugin_manager
- try:
- rewrite, failed = get_plugin_manager().run_held_transforms(callbacks, **payload)
- except Exception: # fail closed: whatever broke dispatch, the original stays withheld
- logger.warning("held transform_llm_output dispatch failed", exc_info=True)
- rewrite, failed = None, True
- return HELD_TRANSFORM_FAILED_REPLY if failed else rewrite
- def settle_held_final(agent, result: Any) -> None:
- """A held turn's reply leaves only through its transform snapshot. ``finish_text_response``,
- ``finalize_turn``, the Codex runtime and the truncation ceiling run it before persisting; every
- other exit (provider errors, retry exhaustion, interrupts, compression failures, ...) returns its
- text through ``run_conversation``, which calls this, so no path hands the user text the plugin
- did not see first, and a failed transform yields the withheld notice."""
- if getattr(agent, "_hold_stream", False) is not True or not isinstance(result, dict):
- return
- text, turn_id = result.get("final_response"), getattr(agent, "_current_turn_id", "") or ""
- recorded = getattr(agent, "_llm_output_transform", None)
- if not isinstance(text, str) or not text or (isinstance(recorded, tuple) and recorded[:1] == (turn_id,)):
- return
- final, transformed, _ = apply_llm_output_transform(agent, text, turn_id=turn_id)
- if result.get("error") == text: # keep the mirror an error card reads
- result["error"] = final
- result.update(final_response=final, response_transformed=transformed, pre_transform_response=None)
- def public_pre_transform(agent, pre_transform: Optional[Any]) -> Optional[Any]:
- """``pre_transform_response`` for a turn result: a held turn's original never leaves the agent
- (receipts, replay-gap recovery, post-stream suffix painting all see only the rewrite)."""
- return None if getattr(agent, "_hold_stream", False) else pre_transform
- def _mirror_turn_to_memory(
- agent, *, original_user_message, final_response, interrupted, messages, review_memory, review_skills, vetoed,
- ) -> None:
- """External-memory sync (+ next prefetch), then the background memory/skill review. A prompt a
- ``pre_llm_call`` hook vetoed never leaves the host, so neither runs for that turn."""
- if vetoed:
- return
- agent._sync_external_memory_for_turn(
- original_user_message=original_user_message, final_response=final_response,
- interrupted=interrupted, messages=messages,
- )
- # Background memory/skill review runs AFTER delivery so it never competes with the
- # user's task. Suppressed by skip_background_review (e.g. cron): the fork costs
- # ~30K tokens / event with no human-in-the-loop benefit. Best-effort; the review
- # clones the snapshot structurally so its sanitizers can't reach the live transcript.
- if (
- final_response
- and not interrupted
- and not getattr(agent, "skip_background_review", False)
- and (review_memory or review_skills)
- ):
- with suppress(Exception):
- agent._spawn_background_review(
- messages_snapshot=list(messages), review_memory=review_memory,
- review_skills=review_skills,
- )
- def finalize_turn(
- agent, *, final_response, api_call_count, interrupted, failed, messages, conversation_history,
- effective_task_id, turn_id, user_message, original_user_message, _should_review_memory,
- _turn_exit_reason, _pending_verification_response=None,
- _pending_verification_response_previewed=False,
- ):
- """Run the post-loop finalization and return the turn ``result`` dict."""
- from agent.conversation_loop import logger
- from agent.turn_context import PRE_LLM_BLOCK_EXIT_REASON
- vetoed = _turn_exit_reason == PRE_LLM_BLOCK_EXIT_REASON # a pre_llm_call block: no model call at all
- # Budget/error exits may never reach the final-text commit. Close admission
- # and preserve any accepted but unapplied correction for the recovery handoff.
- from contextlib import nullcontext
- with getattr(agent, "_pending_redirect_lock", None) or nullcontext():
- if getattr(agent, "_turn_redirect_open", False):
- agent._turn_redirect_open = False
- pending_redirect = agent._pending_redirect
- agent._pending_redirect = None
- if pending_redirect:
- agent.steer(pending_redirect)
- final_response, _turn_exit_reason, preserved_verification_fallback, interrupted = _resolve_budget_fallback(
- agent, final_response=final_response, api_call_count=api_call_count,
- interrupted=interrupted, failed=failed, messages=messages,
- _turn_exit_reason=_turn_exit_reason,
- _pending_verification_response=_pending_verification_response,
- _pending_verification_response_previewed=_pending_verification_response_previewed,
- logger=logger,
- )
- # A non-interrupted turn that fell out of the loop after a tool result, with no
- # follow-up assistant text, is the Desktop/TUI "silent stop" (#55316, #54756): the
- # composer returns to ready (or keeps spinning) while the durable transcript ends
- # at a raw ``tool`` row — the user never learns the turn stopped, and the next user
- # message lands as ``tool → user``. Interrupted tails keep
- # ``close_interrupted_tool_sequence``; this is the non-interrupt sibling. Mint the
- # exit reason, fail the turn, and synthesize the visible close so the tail close in
- # ``_persist_step`` persists an assistant row. A turn that already streamed text is
- # left alone: ``_recover_final_from_stream`` owns that recovery (#95514).
- if (
- not final_response
- and not interrupted
- and messages
- and isinstance(messages[-1], dict)
- and messages[-1].get("role") == "tool"
- and not (getattr(agent, "_current_streamed_assistant_text", "") or "").strip()
- ):
- _turn_exit_reason = "pending_tool_result"
- failed = True
- final_response = ""
- try:
- if agent._turn_completion_explainer_enabled():
- final_response = (
- agent._format_turn_completion_explanation("pending_tool_result", None) or ""
- )
- except Exception:
- final_response = ""
- if not final_response:
- # The turn-completion explainer opt-out must not reintroduce the silent stop.
- final_response = (
- "No reply: the turn stopped while a tool result was still pending. "
- "Send `continue` to let the model summarize."
- )
- # Loop exits that are failures in their own right (outer-loop error cap, shutdown, context
- # that could not be shrunk) carry the verdict the UI descriptor needs; a bare
- # ``turn_exit_reason`` collapsed to code="unknown", retryable=True on every surface.
- # Advisory verdicts (``fails_turn=False``) only add the code: ``failed``/``completed`` keep
- # the loop's values so cron, kanban and transcript persistence behave as before.
- _exit_failure = None if interrupted else exit_reason_failure(_turn_exit_reason)
- if _exit_failure is not None and _exit_failure.fails_turn:
- failed = True
- # Sibling producers (``turn_recovery``, ``codex_runtime``) return ``completed=False`` for an
- # interrupted turn; the gateway stream gate and the API run status rely on that contract.
- completed = (
- final_response is not None
- and not failed
- and not interrupted
- and (api_call_count < agent.max_iterations or str(_turn_exit_reason).startswith("text_response("))
- )
- _rollback_interrupted_preflight_display(agent, interrupted)
- from hermes_cli.observability.shared_metrics_harness import finish_turn
- finish_turn(agent, _turn_exit_reason, final_response, interrupted=interrupted, failed=failed)
- _cleanup_errors: list[str] = []
- # The model has answered (or the loop gave up): a title upgrade held back because it shares a
- # self-hosted endpoint with the main request (#117296) may go out now.
- from agent.turn_context import start_deferred_title_upgrade
- _guarded_cleanup("start_deferred_title_upgrade", lambda: start_deferred_title_upgrade(agent), _cleanup_errors, logger)
- # ``user_message`` may be a multimodal list of parts; the trajectory format wants a string.
- _guarded_cleanup(
- "save_trajectory",
- lambda: agent._save_trajectory(messages, _summarize_user_message_for_log(user_message), completed),
- _cleanup_errors, logger,
- )
- _guarded_cleanup(
- "cleanup_task_resources", lambda: agent._cleanup_task_resources(effective_task_id),
- _cleanup_errors, logger,
- )
- # Persist only after the transcript tail is shaped and scaffolding removed. Each
- # sub-step runs in the same order as the original inline block, and the
- # stream-recovered ``final_response`` is rebound the moment it is computed — BEFORE
- # the fallible tail-shaping / override / micro-compaction / persist calls — so a
- # raise in any of them can't drop text the user already saw (#95514, #8049).
- def _persist_step():
- nonlocal final_response
- _drop_transcript_scaffolding(agent, messages)
- final_response, _recovered_from_stream = _recover_final_from_stream(
- agent, final_response, interrupted, failed
- )
- # Recovery paths (stream-recovered / prior-turn text) reach here with a response no
- # earlier seam transformed; the normal text turn already did this before its flush and
- # gets the recorded outcome back. Either way the tail close below writes the text the
- # user will see, never the raw model text (#44239).
- if final_response and not interrupted:
- _raw = final_response
- final_response, _, _ = apply_llm_output_transform(agent, final_response, turn_id=turn_id, logger=logger)
- _store_rewrite_on_tail(agent, messages, _raw, final_response)
- _close_transcript_tail(agent, messages, final_response, interrupted, _recovered_from_stream)
- if not interrupted and not failed and not vetoed:
- _micro_compact_after_turn(agent, messages, final_response, logger, effective_task_id)
- agent._persist_session(messages, conversation_history)
- _guarded_cleanup("persist_session", _persist_step, _cleanup_errors, logger)
- # Keep the gateway's separate in-memory history snapshot current even on
- # cleanup error, so a later prompt isn't sent with a pre-turn snapshot.
- with suppress(Exception):
- agent._session_messages = messages
- _log_turn_exit(agent, messages, final_response, api_call_count, _turn_exit_reason, interrupted, logger)
- # Response transforms apply only to real, uninterrupted responses.
- if final_response and not interrupted:
- final_response = _append_file_mutation_footer(agent, final_response, logger)
- if not interrupted:
- final_response = _explain_abnormal_exit(
- agent, final_response, _turn_exit_reason, preserved_verification_fallback, logger,
- )
- _platform = getattr(agent, "platform", None) or ""
- _response_transformed = False
- _pre_transform_response = None
- if final_response and not interrupted:
- final_response, _response_transformed, _pre_transform_response = _apply_output_hooks(
- agent, final_response, logger, platform=_platform, effective_task_id=effective_task_id,
- turn_id=turn_id, original_user_message=original_user_message, messages=messages,
- )
- # Context engine observation hook: the turn finished with the finalized transcript.
- # Fail-open. ``_last_turn_usage`` is the last response's canonical usage dict, or
- # ``None`` on turns that never reached a provider response — by contract.
- try:
- from agent.conversation_loop import _notify_context_engine_turn_complete
- _notify_context_engine_turn_complete(
- agent, messages, usage=getattr(agent, "_last_turn_usage", None), logger=logger,
- turn_id=turn_id, task_id=effective_task_id, api_call_count=api_call_count,
- interrupted=interrupted, failed=failed, turn_exit_reason=_turn_exit_reason,
- )
- except Exception as exc:
- logger.warning("on_turn_complete notification failed: %s", exc)
- # Surrogate chokepoint: RAW SDK text with a lone UTF-16 surrogate crashes downstream
- # consumers (stdout, Telegram ``utf16_len``, JSON); scrub once where it leaves the loop.
- # Class-level surrogate chokepoint (#80366, #55143, #55309, #19819): ``final_response`` is often the RAW
- # SDK content (``assistant_message.content``), not the sanitized copy stored in history by
- # ``build_assistant_message``. Any lone UTF-16 surrogate (U+D800–U+DFFF) in it crashes downstream
- # consumers — oneshot stdout writes, Telegram's ``utf16_len`` length check, Signal formatting, JSON
- # envelope encodes — on every provider (Ollama, NVIDIA NIM, …). Scrub once here, where model text leaves
- # the conversation loop, so every delivery surface receives valid Unicode.
- if isinstance(final_response, str):
- final_response = _sanitize_surrogates(final_response)
- result = {
- "final_response": final_response,
- "last_reasoning": _last_turn_reasoning(messages),
- "messages": messages,
- "api_calls": api_call_count,
- "completed": completed,
- "turn_exit_reason": _turn_exit_reason,
- "failed": failed,
- "partial": False, # True only when stopped due to invalid tool calls
- "interrupted": interrupted,
- "response_transformed": _response_transformed,
- "pre_transform_response": public_pre_transform(agent, _pre_transform_response),
- "response_previewed": getattr(agent, "_response_was_previewed", False),
- # The final is byte-for-byte a response this turn already delivered (no footer or
- # explanation appended since): it carries no new text for the client to paint.
- "response_reused": bool(final_response) and final_response == getattr(agent, "_reused_response_text", None),
- "model": agent.model,
- # requested_model / served_model: proxy-reported deployment or Hermes' own fallback route.
- **result_model_fields(agent),
- "provider": agent.provider,
- "base_url": agent.base_url,
- **{key: getattr(agent, f"session_{key}") for key in _SESSION_TOKEN_KEYS},
- # Gateway SessionEntry persists an API reading, never the preflight display seed.
- "last_prompt_tokens": (
- getattr(agent.context_compressor, "last_real_prompt_tokens", agent.context_compressor.last_prompt_tokens)
- if getattr(agent.context_compressor, "last_prompt_tokens", 0) > 0
- else getattr(agent.context_compressor, "last_prompt_tokens", 0)
- ) or 0,
- **{key: getattr(agent, f"session_{key}") for key in _SESSION_COST_KEYS},
- # Requested service tier, for billing audits (`hermes -z --usage-file`).
- "service_tier": (
- (getattr(agent, "request_overrides", {}) or {}).get("extra_body") or {}
- ).get("service_tier"),
- "session_id": agent.session_id,
- }
- if agent._tool_guardrail_halt_decision is not None:
- result["guardrail"] = agent._tool_guardrail_halt_decision.to_metadata()
- # Persistence failures already set failed=True; also stamp `error` so the gateway
- # surfaces status="error" (desktop can toast) instead of a quiet complete frame, plus
- # the machine-readable cause 'session_persistence_failed:<locked|compression|...>'.
- if failed and str(_turn_exit_reason) == "session_persistence_failed":
- from hermes_constants import profile_cli_selector
- # Never rebind final_response here: the memory sync and the background-review gate
- # below must still see an empty response on a persistence-failed turn.
- result["error"] = final_response or (
- "session storage could not be written — check the state database "
- f"health (`hermes {profile_cli_selector()}doctor`), then send your message again"
- )
- _cause = getattr(agent, "_last_persistence_error_cause", None)
- result["failure_reason"] = "session_persistence_failed:" + (_cause or "unknown")
- elif _exit_failure is not None:
- if failed:
- result["error"] = final_response or str(_turn_exit_reason)
- stamp_failure(result, _exit_failure.reason, _exit_failure.retryable)
- # Cleanup failures are surfaced, but the response is returned either way (#8049).
- if _cleanup_errors:
- result["cleanup_errors"] = _cleanup_errors
- # A /steer landing after the final assistant turn has no tool batch to drain into;
- # hand it back so it becomes the next user turn instead of being lost. A gateway transfer
- # (ticketed) stays buffered: the claim's settlement returns it to its own queue place.
- _leftover_steer = (getattr(agent, "_drain_leftover_steer", None) or agent._drain_pending_steer)()
- if _leftover_steer:
- result["pending_steer"] = _leftover_steer
- agent._response_was_previewed = False
- agent._reused_response_text = None
- if interrupted and agent._interrupt_message:
- result["interrupt_message"] = agent._interrupt_message
- agent.clear_interrupt()
- agent._stream_callback = None # don't leak into future calls
- # A voice turn's model route ends with the turn: memory sync and the background review
- # below (and the next turn) run on the session's main model.
- from agent.voice_turn_route import end_voice_turn_route
- end_voice_turn_route(agent)
- # Skill trigger is checked NOW — based on how many tool iterations THIS turn used.
- _should_review_skills = (
- agent._skill_nudge_interval > 0
- and agent._iters_since_skill >= agent._skill_nudge_interval
- and "skill_manage" in agent.valid_tool_names
- )
- if _should_review_skills:
- agent._iters_since_skill = 0
- _mirror_turn_to_memory(
- agent, original_user_message=original_user_message, final_response=final_response,
- interrupted=interrupted, messages=messages, review_memory=_should_review_memory,
- review_skills=_should_review_skills, vetoed=vetoed,
- )
- # Opt-in (compression.prepare_ahead): near the trigger, write the next compaction summary
- # between turns. No-op otherwise; never raises. Not after a vetoed turn (no model calls).
- from agent.prepared_compaction import maybe_prepare
- if not vetoed:
- maybe_prepare(agent, messages, origin="turn-end")
- # Memory provider on_session_end()/shutdown_all() are NOT called here:
- # run_conversation() runs once per message; CLI/gateway own session-end cleanup.
- if not getattr(agent, "_persist_disabled", False):
- _invoke_hook_safely(
- "on_session_end", logger,
- session_id=agent.session_id,
- task_id=effective_task_id,
- turn_id=turn_id,
- completed=completed,
- failed=failed,
- interrupted=interrupted,
- turn_exit_reason=_turn_exit_reason,
- model=agent.model,
- platform=_platform,
- )
- agent._turn_preflight_display_snapshot = None
- agent._turn_received_provider_response = False
- return result
|