| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966 |
- """The agent conversation loop — extracted from ``run_agent.AIAgent``.
- ``run_conversation(agent, ...)`` drives one user turn (model call, tool dispatch,
- retries, fallbacks, compression, post-turn hooks). Symbols that callers patch on
- ``run_agent`` (``handle_function_call``, ``_set_interrupt``, ``OpenAI``) resolve via
- ``_ra`` so those patches keep working."""
- from __future__ import annotations
- import inspect
- import json
- import logging
- import re
- import time
- from dataclasses import dataclass, field, fields
- from typing import Any, Dict, List, Optional
- from agent.agent_runtime_helpers_placeholders import hidden_interrupt_placeholder_row
- from agent.codex_responses_adapter import _summarize_user_message_for_log
- from agent.fast_mode import begin_turn as begin_fast_mode_turn
- from agent.message_metadata import append_message, without_persistence_fields
- from agent.message_sanitization import _repair_tool_call_arguments, _sanitize_surrogates
- from agent.model_metadata import MINIMUM_CONTEXT_LENGTH, _estimate_tools_tokens_rough
- from agent.process_bootstrap import _install_safe_stdio
- from agent.prompt_builder import RUNTIME_ENVIRONMENT_END, RUNTIME_ENVIRONMENT_HEADING
- from agent.prompt_refresh import rebuild_for_refresh, requested_epoch, stamp_prompt_epoch, stored_prompt_refresh_epoch
- from agent.prompt_caching import (
- build_prompt_cache_plan,
- effective_cache_ttl,
- strip_anthropic_cache_control,
- strip_anthropic_tool_cache_control,
- )
- from agent.repetition_guard import REPETITION_LOOP_INTERRUPTED, is_runaway_repetition
- from agent.runtime_cwd import resolve_agent_cwd
- from agent.surface_switch import (
- identity_line_value, note_inert_pinned_tools, runtime_host_value, stage_surface_switch_note,
- )
- from agent.turn_context import PRE_LLM_BLOCK_EXIT_REASON, PreflightCompressionTimedOut, build_turn_context
- from hermes_cli.observability.shared_metrics_efficiency import record_cache_break, record_prompt_rebuild
- from hermes_cli.plugins_dispatch import held_stream_transforms
- from agent.turn_retry_state import TurnRetryState
- from agent.turn_scripted_prelude import Prelude, play_prelude
- # Phase helpers of the turn loop, bound at import so a source-tree swap cannot load a
- # skewed phase mid-turn.
- from agent.turn_api_call import handle_api_interrupt, nous_rate_limit_guard, perform_api_call
- from agent.turn_api_error import handle_api_error
- from agent.turn_api_request import build_api_request
- from agent.turn_failure_copy import FAILED_TURN_DISPLAY_KIND, OPEN_TURN_TAIL_ROLES, failed_turn_notice, site_copy
- from agent.turn_final_response import finish_text_response
- from agent.turn_finalizer import finalize_turn
- from agent.turn_iteration_prep import (
- announce_api_call,
- apply_retry_restarts,
- begin_iteration,
- prepare_iteration,
- )
- from agent.turn_loop_errors import handle_outer_loop_error
- from agent.turn_preflight_gate import run_preflight_gate
- from agent.turn_request_assembly import assemble_api_request
- from agent.turn_response_check import check_api_response
- from agent.turn_response_intake import normalize_model_response
- from agent.turn_tool_round import run_tool_round
- from hermes_logging import set_session_context
- from tools.skill_provenance import set_current_write_origin
- from utils import base_url_host_matches
- logger = logging.getLogger(__name__)
- # Must mirror _STALE_TOOL_CALL_MARKER_RE in hermes_state.py; kept local so importing
- # hermes_state (module-level DEFAULT_DB_PATH) is not forced at load time.
- _STALE_MARKER_RE = re.compile(r"^\[[A-Za-z_][A-Za-z0-9_.-]*\]$")
- # Shared by _apply_active_turn_redirect and the api_messages ghost-row filter so both sites cannot drift.
- _INTERRUPT_SCAFFOLD_MARKER = "[This response was interrupted by a user correction.]"
- # One-time wrap-up notice appended when a wall-clock run budget (--run-budget) crosses 80%.
- RUN_BUDGET_WRAPUP_NOTICE = (
- "[SYSTEM NOTICE — run time budget nearly exhausted] Run time budget nearly exhausted. "
- "Stop new discovery/verification work now. Produce the required final deliverable "
- "(answer/JSON/summary) from the state you already have, completing only mandatory writes."
- )
- def _midturn_request_pressure_tokens(
- agent: Any, api_messages: list[dict[str, Any]], effective_system: str, approx_tokens: int
- ) -> int:
- """Token figure the mid-turn pre-API compression guard compares: the pruned
- native-Responses estimate when native compaction eligibility is proven (the generic
- estimate overstates the wire on compacted sessions, #96995), else messages+tools.
- The system prompt is counted exactly once.
- When the upcoming request is eligible for native Responses compaction the transport will
- checkpoint-prune the payload before sending, so the generic durable-history estimate overstates the wire
- by orders of magnitude on a compacted session and fires a 600s local compression the main request never
- needed (#96995).
- """
- try:
- from agent.codex_responses_adapter import estimate_native_responses_preflight_tokens
- native = estimate_native_responses_preflight_tokens(
- agent, api_messages, system_prompt=effective_system or "",
- tools=getattr(agent, "tools", None) or None,
- )
- if isinstance(native, int) and not isinstance(native, bool) and native >= 0:
- return native
- except Exception:
- logger.debug(
- "native Responses mid-turn estimate unavailable; using generic transcript estimate",
- exc_info=True,
- )
- return approx_tokens + (_estimate_tools_tokens_rough(agent.tools) if agent.tools else 0)
- def _review_input_budget_exhausted(agent: Any) -> bool:
- """True when a detached review fork has replayed its aggregate input budget.
- Only forks with an explicit ``_review_input_token_budget`` are gated (#93057). Fires
- at the top of the NEXT iteration, so the budget-crossing request completes first."""
- budget = getattr(agent, "_review_input_token_budget", None)
- if not isinstance(budget, int) or isinstance(budget, bool) or budget <= 0:
- return False
- used = getattr(agent, "session_input_tokens", 0)
- return isinstance(used, int) and not isinstance(used, bool) and used >= budget
- def _maybe_inject_run_budget_wrapup(agent: Any, messages: list[dict[str, Any]]) -> bool:
- """Inject the one-time wall-clock wrap-up notice when past 80% of budget.
- Appends to the NEWEST ``role:"tool"`` message (cache-safe, like /steer); latches
- ``_run_budget_wrapup_injected`` only on a successful append."""
- budget = getattr(agent, "run_budget_seconds", None)
- started = getattr(agent, "_run_budget_started_at", None)
- if not budget or not started or getattr(agent, "_run_budget_wrapup_injected", False) or (
- (time.time() - started) < 0.8 * float(budget)
- ):
- return False
- from agent.context_compressor import _DB_PERSISTED_MARKER
- for msg in reversed(messages):
- if isinstance(msg, dict) and msg.get("role") == "tool":
- # Only the current tool-result tail is mutable; an older turn may already be
- # cached (same contract as _maybe_inject_iteration_budget_warning).
- if msg.get(_DB_PERSISTED_MARKER):
- return False
- existing = msg.get("content", "")
- if isinstance(existing, str):
- msg["content"] = existing + f"\n\n{RUN_BUDGET_WRAPUP_NOTICE}"
- else: # multimodal content blocks — append a text block
- try:
- msg["content"] = [*(existing or []), {"type": "text", "text": RUN_BUDGET_WRAPUP_NOTICE}]
- except Exception:
- return False
- agent._run_budget_wrapup_injected = True
- logger.info(
- "Run budget wrap-up notice injected (budget=%.0fs, elapsed=%.0fs)",
- float(budget), time.time() - started,
- )
- return True
- return False
- def _restore_user_after_reference_handoff(
- messages: list[dict[str, Any]], user_message: Any
- ) -> bool:
- """Re-append this turn's real user ask when compaction left only a handoff (#80622).
- Returns True when a restore append happened."""
- if isinstance(user_message, str):
- restorable = bool(user_message.strip())
- else:
- restorable = isinstance(user_message, list) and bool(user_message)
- if not restorable:
- return False
- last = messages[-1] if messages else None
- if isinstance(last, dict) and last.get("role") == "user" and last.get("content") == user_message:
- return False
- append_message(messages, {"role": "user", "content": user_message})
- return True
- def _should_skip_model_call_for_reference_handoff(
- messages: list[dict[str, Any]], user_message: Any
- ) -> bool:
- """Guard post-compaction continues against sole-handoff active turns (#80622)."""
- from agent.context_compressor import reference_handoff_would_drive_next_model_call
- # A restored ask is an actionable non-synthetic user row appended after the
- # handoff — by construction the handoff no longer drives.
- return reference_handoff_would_drive_next_model_call(messages) and not (
- _restore_user_after_reference_handoff(messages, user_message)
- )
- # Fallback final_response for the sole-handoff skip (#80622); finalize_turn appends it as a
- # fresh assistant row, so it must not replay the last assistant text.
- # Deliberately NOT a replay of the last assistant text: finalize_turn's non-assistant-tail chokepoint
- # (#43849) appends final_response as a fresh assistant row, so recovering the previous turn's prose here
- # would duplicate it in the durable transcript AND re-deliver it to the user as if it were this turn's
- # answer. A short status is honest and idempotent.
- _HANDOFF_SKIP_FINAL_RESPONSE = (
- "Context was compacted. The previous response is complete — awaiting your next message."
- )
- # Terminal final_response when compression timed out while the request was still oversized (#98722).
- # Terminal final_response for a turn ended because context compression hit its host progress-aware timeout
- # while the request was still oversized (#98722, salvaged from #98741). Sending the unchanged request would
- # only bounce off the provider's overflow error and re-enter compression in the same turn.
- _COMPRESSION_TIMEOUT_FINAL_RESPONSE = (
- "Context compression timed out without reducing this conversation. No messages were "
- "dropped. Start a fresh session with /new, or check auxiliary.compression before retrying /compress."
- )
- # Stable prefix ACP/TUI match on to treat the text as cancellation metadata, not assistant prose.
- INTERRUPT_WAITING_FOR_MODEL_PREFIX = "Operation interrupted: waiting for model response ("
- def _should_rearm_compression_budget(
- compression_attempts: int, *, completed_compaction_pending: bool, prompt_tokens: int, threshold_tokens: int
- ) -> bool:
- """True once a provider proves a completed compaction worked: rough estimates cannot
- rearm the anti-thrash budget, only the completed-compaction latch plus a positive
- normalized prompt count below the threshold."""
- return bool(
- compression_attempts and completed_compaction_pending and 0 < prompt_tokens < threshold_tokens
- )
- # Modules whose presence in a traceback (without any API-call module) marks a
- # deterministic local bug not worth retrying. NEVER add "conversation_loop" or
- # "run_agent": every exception passes through them; _hit_local would be True (#66267)
- _LOCAL_PROCESSING_MODULES = frozenset({
- "agent_runtime_helpers",
- "message_content",
- "message_sanitization",
- "chat_completion_helpers", # only local when NOT also an API-call module
- })
- _API_CALL_MODULES = frozenset({"chat_completion_helpers"})
- # Max outer-loop exceptions per user turn before giving up; only exceptions that
- # ESCAPE the inner retry/fallback machinery count, so this can be small (#92450).
- _MAX_OUTER_LOOP_ERRORS = 8
- def _is_interpreter_shutdown_error(exc: Exception) -> bool:
- """True for a fatal interpreter-shutdown RuntimeError. The RuntimeError type gate
- stays here: a ValueError carrying similar text must not match (#93269)."""
- if isinstance(exc, RuntimeError):
- # ── Interpreter finalization: abandon immediately ── The process is exiting (TUI quit, SIGTERM,
- # one-shot done) while this turn — typically the post-turn review fork's daemon thread — is
- # mid-flight. Retries, credential rotation, and fallbacks are all futile ("cannot schedule new
- # futures..."), and the buffered ⚠️/❌ retry trace spams the shell after the TUI already exited. End
- # the turn with a single log line: no print, no traceback, no debug dump, no retry. Same class as
- # cron delivery (#55924/#58720) and concurrent tool submission — shared predicate.
- from tools.interpreter_shutdown import interpreter_shutting_down
- return interpreter_shutting_down(exc)
- return False
- def _moa_client_consumes_prepared_request(client: Any) -> bool:
- """True when ``client`` is the in-process MoA facade (only ``MoAChatCompletions`` exposes
- ``prepare()``; other clients raise TypeError on ``_moa_prepared_request`` even while
- ``agent.provider`` stays ``"moa"``)."""
- completions = getattr(getattr(client, "chat", None), "completions", None)
- return callable(getattr(completions, "prepare", None))
- _MIN_CONTINUATION_OVERLAP = 32
- def _continuation_overlap_length(previous: str, continuation: str) -> int:
- """Return the longest continuation prefix that repeats the previous suffix."""
- if len(previous) < _MIN_CONTINUATION_OVERLAP or len(continuation) < _MIN_CONTINUATION_OVERLAP:
- return 0
- prefix_lengths = [0] * len(continuation)
- matched = 0
- for index in range(1, len(continuation)):
- while matched and continuation[index] != continuation[matched]:
- matched = prefix_lengths[matched - 1]
- if continuation[index] == continuation[matched]:
- matched += 1
- prefix_lengths[index] = matched
- matched = 0
- last_index = len(previous) - 1
- for index, char in enumerate(previous):
- while matched and char != continuation[matched]:
- matched = prefix_lengths[matched - 1]
- if char == continuation[matched]:
- matched += 1
- if matched == len(continuation):
- if index == last_index:
- return matched
- matched = prefix_lengths[matched - 1]
- return matched if matched >= _MIN_CONTINUATION_OVERLAP else 0
- def _join_truncated_parts(parts: list[tuple[str, bool]]) -> str:
- """Join continuation fragments, deduping only interrupted-stream seams."""
- joined = ""
- previous_was_partial_stub = False
- for part, is_partial_stub in parts:
- if previous_was_partial_stub and joined and part:
- # Overlap can't exceed len(part): scan only that tail of ``joined``.
- part = part[_continuation_overlap_length(joined[-len(part):], part):]
- if joined and not joined[-1].isspace() and part and not part[0].isspace():
- joined += "\n"
- joined += part
- previous_was_partial_stub = is_partial_stub
- return joined
- def _moa_reference_metrics_for_hook(agent: Any) -> Any:
- """Per-advisor metrics for post_api_request, or None off the MoA path (a plugin only
- sees the aggregator generation; this carries the per-slot advisor spend)."""
- client = getattr(agent, "client", None)
- getter = getattr(client, "last_reference_metrics", None)
- if not callable(getter):
- return None
- try:
- return getter()
- except Exception:
- return None
- def _apply_active_turn_redirect(agent: Any, messages: list[dict[str, Any]], text: str) -> None:
- """Append a provider-safe checkpoint and correction to the live turn so role alternation
- holds and cached messages stay byte-identical. INVARIANTS: raw chain-of-thought never enters
- replayable content (inlined CoT reads as a prefill jailbreak and bricks the session with
- empty-response storms); the interruption scaffold is replay text carried only in the user
- correction's ``api_content``; an on-screen-empty placeholder is ``display_kind=hidden``."""
- visible = agent._strip_think_blocks(getattr(agent, "_current_streamed_assistant_text", "") or "").strip()
- checkpoint_parts = [_INTERRUPT_SCAFFOLD_MARKER]
- if is_runaway_repetition(visible):
- # Runaway shape only (a correct batch-style partial stays replayable): the looped bytes must
- # reach neither the replayed correction nor the placeholder below (empty ``visible`` takes
- # the hidden shape).
- checkpoint_parts.append(REPETITION_LOOP_INTERRUPTED)
- visible = ""
- elif visible:
- checkpoint_parts += ["Visible response before the interruption:", visible]
- checkpoint = "\n\n".join(checkpoint_parts)
- correction = f"[Context from the interrupted assistant response]\n{checkpoint}\n\n{text}"
- # The live tail is normally user or tool, so an assistant placeholder + correction
- # keeps strict alternation; if the tail is already assistant, the checkpoint is folded
- # into the user correction instead of creating assistant→assistant. The placeholder
- # preserves alternation only — scaffold bytes must never land in it, since api_content
- # is substituted back into content on replay (#81841).
- if not (messages and messages[-1].get("role") == "assistant"):
- # Hidden row with a neutral api_content (#88955). Never _INTERRUPT_SCAFFOLD_MARKER:
- # as assistant text the model echoes it (#81841).
- append_message(messages, {"role": "assistant", "content": visible} if visible else hidden_interrupt_placeholder_row())
- # Transcript shows the user's own words; the provider replays the scaffolded form.
- append_message(messages, {"role": "user", "content": text, "api_content": correction})
- # Stateful scrubber for <memory-context> spans split across stream deltas (#5719). sanitize_context()
- # alone can't survive chunk boundaries because the block regex needs both tags in one string.
- # Stateful scrubber for reasoning/thinking tags in streamed deltas (#17924). Replaces the per-delta
- # _strip_think_blocks regex that destroyed downstream state (e.g. MiniMax-M2.7 streaming '<think>' as
- # delta1 and 'Let me check' as delta2 — the regex erased delta1, so downstream state machines never
- # learned a block was open and leaked delta2 as content).
- agent._current_streamed_assistant_text = ""
- agent._stream_needs_break = True
- def _credential_refresh_profile(agent: Any) -> Any:
- """The profile owning the live credential's runtime refresh: the agent's provider by name, else
- (a ``custom`` route at a provider's host) the profile serving the base URL's host. Only profiles
- that override ``refresh_runtime_credentials`` qualify."""
- from agent.provider_route_hooks import route_profiles
- profiles = route_profiles("refresh_runtime_credentials", (getattr(agent, "provider", "") or "").strip().lower(),
- str(getattr(agent, "base_url", "") or ""))
- return profiles[0] if profiles else None
- def _pressure_with_real_floor(compressor: Any, rough_tokens: int) -> int:
- """Floor the ROUGH pre-API pressure estimate at the last REAL prompt size.
- Applied only on the fallback path -- when ``anchored_context_tokens`` has
- no valid anchor (first request, transcript rewritten under the anchor,
- provider never reported usage). A valid anchor is provider-exact and is
- used as-is; in particular on MoA turns the anchor deliberately uses the
- pre-fold aggregator usage while ``last_real_prompt_tokens`` holds the
- folded figure, so flooring an anchored value would re-add fan-out tokens
- the anchor exists to exclude.
- On the rough path, non-ASCII text (Cyrillic, Greek, Polish, ...)
- under-counts by up to ~2x, so a session can sit at the provider's real
- context ceiling while the rough figure stays under the compaction
- threshold -- on silent-clip providers (ollama /v1) that is a truncation
- death spiral the reactive overflow handler never sees (observed live:
- real prompts 64,842->64,995 against a 55,705 threshold). The provider's
- last reported prompt_tokens is authoritative; never let the rough figure
- fall below it. Skipped for exactly one turn after a compaction, when
- last_real_prompt_tokens still holds the stale pre-compression value
- (#36718's awaiting_real_usage_after_compression window).
- """
- last_real = int(getattr(compressor, "last_real_prompt_tokens", 0) or 0)
- if last_real > rough_tokens and not getattr(
- compressor, "awaiting_real_usage_after_compression", False
- ):
- return last_real
- return rough_tokens
- def _ollama_context_limit_error(agent: Any, request_tokens: int) -> Optional[str]:
- """Return a user-facing error when Ollama is loaded with too little context."""
- runtime_ctx = getattr(agent, "_ollama_num_ctx", None)
- if (
- not getattr(agent, "tools", None)
- or not isinstance(runtime_ctx, int)
- or not 0 < runtime_ctx < MINIMUM_CONTEXT_LENGTH
- ):
- return None
- model = getattr(agent, "model", "") or "the selected model"
- logger.warning(
- "Ollama runtime context too small for Hermes tool use: model=%s provider=%s base_url=%s "
- "runtime_context=%d minimum_context=%d estimated_request_tokens=%d tool_count=%d session=%s",
- model, getattr(agent, "provider", "") or "unknown",
- getattr(agent, "base_url", "") or "unknown base URL", runtime_ctx, MINIMUM_CONTEXT_LENGTH,
- request_tokens, len(getattr(agent, "tools", None) or []),
- getattr(agent, "session_id", None) or "none",
- )
- return (
- f"Ollama loaded `{model}` with only {runtime_ctx:,} tokens of runtime context, but Hermes "
- f"needs at least {MINIMUM_CONTEXT_LENGTH:,} tokens for reliable tool use.\n\n"
- "Increase the Ollama context for this model and restart/reload the model before trying "
- "again. A known-good starting point is 65,536 tokens. In Hermes config, set "
- "`model.ollama_num_ctx: 65536` (and `model.context_length: 65536` if you also override the "
- "displayed model context). If you manage the model through an Ollama Modelfile, set "
- "`PARAMETER num_ctx 65536` there instead."
- )
- def _maybe_grow_local_window(agent: Any, compressor: Any,
- request_tokens: int) -> Optional[int]:
- """Grow a managed local model's context window before compressing; returns the new
- window when the ladder granted one, else None."""
- provider = (getattr(agent, "provider", "") or "").strip().lower()
- base_url = getattr(agent, "base_url", "") or ""
- if provider not in ("llamacpp", "llama.cpp", "llama-cpp", "custom") or not (
- "127.0.0.1" in base_url or "localhost" in base_url
- ):
- return None
- try:
- from hermes_cli.local_runtime.growth import maybe_grow_window
- current_window = int(getattr(compressor, "context_length", 0) or 0)
- if current_window <= 0:
- return None
- return maybe_grow_window(
- getattr(agent, "model", "") or "", base_url=base_url,
- session_tokens=int(request_tokens), current_window=current_window,
- )
- except Exception as exc:
- logger.debug("local window growth check failed: %s", exc)
- return None
- def _ra():
- """Lazy ``run_agent`` reference so patches on ``run_agent.*`` reach this code path."""
- import run_agent
- return run_agent
- def _nous_entitlement_message(capability: str, *, billing: bool = False) -> str:
- """``billing``: the request hit a billing wall. A usage-window account then gets its own next
- step (top-up link); its plan is still live, so the entitlement copy alone would say nothing."""
- try:
- from hermes_cli.nous_account import (
- format_nous_portal_entitlement_message,
- format_usage_window_block_guidance,
- get_nous_portal_account_info,
- )
- account_info = get_nous_portal_account_info(force_fresh=True)
- if billing and (window_guidance := format_usage_window_block_guidance(account_info)):
- return window_guidance
- return format_nous_portal_entitlement_message(
- account_info, capability=capability, in_chat=True
- ) or ""
- except Exception:
- return ""
- def _print_guidance(agent, message: str) -> bool:
- """Print each line of ``message`` as a 💡 hint; False when there is nothing to print."""
- if not message:
- return False
- for line in message.splitlines():
- agent._vprint(f"{agent.log_prefix} 💡 {line}", force=True, diagnostic=True)
- return True
- def _print_nous_entitlement_guidance(agent, capability: str) -> bool:
- return _print_guidance(agent, _nous_entitlement_message(capability))
- def _system_prompt_for_hooks(api_kwargs: Any, request_messages: Any) -> Any:
- """System prompt as sent to the provider (``system`` / ``instructions`` / ``messages[0]``)
- for observability hooks; None when the request carries none."""
- system_prompt = api_kwargs.get("system")
- if system_prompt is None:
- system_prompt = api_kwargs.get("instructions")
- if system_prompt is None and isinstance(request_messages, list) and request_messages:
- first = request_messages[0]
- if isinstance(first, dict) and first.get("role") == "system":
- system_prompt = first.get("content")
- return system_prompt
- def _is_nous_inference_route(provider: str, base_url: str) -> bool:
- return (provider or "").strip().lower() == "nous" or base_url_host_matches(
- str(base_url or ""), "inference-api.nousresearch.com"
- )
- def _billing_or_entitlement_message(
- *, capability: str, provider: str, base_url: str, model: str, unverified: bool = False
- ) -> str:
- if _is_nous_inference_route(provider, base_url):
- return _nous_entitlement_message(capability, billing=True)
- provider_label = (provider or "").strip() or "the selected provider"
- model_label = (model or "").strip() or "the selected model"
- # Anthropic Pro/Max OAuth surfaces "extra usage" exhaustion as a hard 400 — "add credits"
- # does not apply. ``unverified`` (#82154): the same 400 is returned for a server-side
- # content-filter rejection, so hedge and name the other cause.
- if (provider or "").strip().lower() == "anthropic":
- switch = (
- "You can also switch to an Anthropic API key or another provider with "
- "/model <model> --provider <provider>."
- )
- if unverified:
- return "\n".join([
- f"{provider_label} reported that your Claude subscription usage may be exhausted for "
- f"{model_label} (included quota + extra-usage credits) — but this specific error is "
- "not proof of a billing problem.",
- "If https://claude.ai/settings/usage still shows quota remaining, this is probably NOT "
- "a billing problem: on a Claude subscription (OAuth) token Anthropic returns this same "
- "message when its content filter rejects part of the request — typically a phrase in "
- "the system prompt.",
- "If usage really is exhausted: wait for the billing cycle to reset, or add extra usage "
- "at https://claude.ai/settings/usage",
- switch,
- # The exhaustion latch replays the stored error without a request.
- "Retry with a fresh credential state: `hermes auth reset anthropic`. Until that "
- "cooldown clears, this error can be replayed from cache without contacting the API.",
- ])
- return "\n".join([
- f"{provider_label} reported that your Claude subscription usage is exhausted for "
- f"{model_label} (included quota + extra-usage credits).",
- "Options: wait for the billing cycle to reset, or add extra usage at https://claude.ai/settings/usage",
- switch,
- ])
- # Provider-agnostic billing URL so every text surface shows the same actionable link.
- try:
- from agent.billing_links import build_billing_block
- _link = build_billing_block(provider=provider, base_url=base_url, model=model)
- provider_label = _link.provider_label or provider_label
- billing_url = _link.billing_url
- except Exception:
- billing_url = None
- return "\n".join([
- f"{provider_label} reported that billing, credits, or account entitlement is exhausted for {model_label}.",
- "Add credits or update billing with that provider, then retry.",
- *([f"{provider_label} billing: {billing_url}"] if billing_url else []),
- "You can switch providers temporarily with /model <model> --provider <provider>.",
- ])
- def _billing_block_dict(provider, base_url, model, message="", *, unverified: bool = False) -> Optional[dict]:
- """Best-effort structured billing descriptor (None if billing_links is unavailable)."""
- try:
- from agent.billing_links import build_billing_block
- block = build_billing_block(
- provider=provider, base_url=str(base_url), model=model, message=message
- ).to_dict()
- except Exception:
- return None
- if block is not None and unverified:
- block["unverified"] = True # every surface rendering the block can hedge too (#82154)
- return block
- def _billing_terminal_label(summary: str, unverified: bool) -> str:
- """Terminal-failure prefix for a billing-classified error; ``unverified`` (#82154) must
- not assert exhaustion as fact."""
- if unverified:
- return (
- "Provider reported usage/credit exhaustion (unverified — the same "
- f"error can be a content-filter rejection, not billing): {summary}"
- )
- return f"Billing or credits exhausted: {summary}"
- def _usage_window_refusal(agent, classified, api_messages=None) -> tuple[str, Optional[float], Optional[dict]]:
- """A usage-window 402 in the words of the credits limit line ("You've hit your daily limit · resets
- 15:56 · /topup to add credits") plus when that allowance frees up (epoch
- seconds), so the chat and the error card say exactly what the notice says and the card can offer
- a retry at the reset. The third item is the structured limit (:class:`agent.plan_limits.LimitHit`
- payload: what ran out, the recommended models that still work, the credits option) every surface
- renders as buttons or a sentence. ``("", None, None)`` for every other billing failure."""
- context = getattr(classified, "error_context", None) or {}
- if not context.get("usage_window"):
- return "", None, None
- try:
- from agent.credits_tracker import _window_hard_stop, window_stop_resets_at_ms
- from agent.plan_limits import current_model_is_top_tier
- state = getattr(agent, "_credits_state", None)
- top_tier = current_model_is_top_tier(agent)
- stop = _window_hard_stop(state, model_is_free=False, model_is_top_tier=top_tier) if state is not None else None
- # The server may word the refusal itself (``display_message`` on the 402): shown verbatim, so a
- # reword ships with the API. Without it, the stop line Hermes builds from the gauges.
- display = context.get("display_message")
- server_message = str(getattr(classified, "message", "") or "").strip()
- if isinstance(display, str) and display.strip():
- message = display.strip()
- elif stop:
- message = stop.removeprefix("✕ ").strip()
- elif server_message:
- # Local state says nothing is spent (stale snapshot, or it thinks credits will pay) but the
- # server refused with the window code: its own sentence still beats the generic "out of
- # credits" wall, and keeps the turn on the window card (one place, Open billing) instead of
- # a billing toast + banner + Switch provider card.
- message = server_message
- else:
- return "", None, None
- resets_ms = window_stop_resets_at_ms(state, model_is_top_tier=top_tier) if state is not None else None
- limit = None
- try:
- from agent.model_metadata import estimate_messages_tokens_rough
- from agent.plan_limits import limit_hit
- hit = limit_hit(agent, message=message,
- needed_tokens=estimate_messages_tokens_rough(api_messages or []) if api_messages else None)
- limit = hit.to_payload() if hit is not None else None
- except Exception:
- logger.debug("Usage-window limit payload unavailable", exc_info=True)
- limit = None
- return (message, resets_ms / 1000 if isinstance(resets_ms, int) and resets_ms > 0 else None, limit)
- except Exception:
- logger.debug("Usage-window refusal parse failed", exc_info=True)
- return "", None, None
- def _billing_failure_result(
- *, classified, summary: str, messages, api_call_count: int, provider: str, base_url, model: str,
- guidance: Optional[str] = None,
- usage_window: tuple[str, Optional[float], Optional[dict]] = ("", None, None),
- ) -> dict:
- """Structured terminal result for a billing-classified failure — the single construction
- point for the non-retryable abort and max-retries paths (#82154). A usage-window refusal
- reads as its one stop-line sentence instead of the generic "billing exhausted" text."""
- unverified = bool(getattr(classified, "billing_unverified", False))
- if guidance is None and not (tuple(usage_window) + ("",))[0]: # a window refusal is its own message: no portal fetch
- guidance = _billing_or_entitlement_message(
- capability="model access", provider=provider, base_url=str(base_url), model=model,
- unverified=unverified,
- )
- usage_window_message, usage_window_resets_at, usage_limit = (tuple(usage_window) + (None, None, None))[:3]
- final = usage_window_message or (
- _billing_terminal_label(summary, unverified) + (f"\n\n{guidance}" if guidance else ""))
- extra: dict = {}
- if usage_window_message:
- extra["usage_window_message"] = usage_window_message
- if usage_window_resets_at is not None:
- extra["failure_resets_at"] = usage_window_resets_at
- if usage_limit:
- # Structured: the CLI and the messaging gateway read it off the result; the error surface
- # carries it to the TUI and the desktop card.
- extra["usage_limit"] = usage_limit
- return {
- **extra,
- "final_response": final, "messages": messages, "api_calls": api_call_count,
- "completed": False, "failed": True, "error": summary,
- "failure_reason": classified.reason.value,
- # Classifier's own retry verdict so the UI shows Retry only when a re-run can differ.
- "failure_retryable": bool(classified.retryable),
- "billing_unverified": unverified,
- "billing_block": _billing_block_dict(provider, base_url, model, guidance, unverified=unverified),
- }
- def _print_billing_or_entitlement_guidance(
- agent, *, capability: str, provider: str, base_url: str, model: str, unverified: bool = False
- ) -> bool:
- return _print_guidance(agent, _billing_or_entitlement_message(
- capability=capability, provider=provider, base_url=base_url, model=model,
- unverified=unverified,
- ))
- def _bot_chat_prompt_stale(agent, stored_prompt: str | None) -> bool:
- """Bot Chat capability epoch check for a stored prompt.
- The stored prompt embeds a capability fingerprint; a mismatch is a deliberate
- once-per-change rebuild. Unstamped prompts never match; probe failures fail closed
- to "reuse" so the cache is kept. Legacy upgrade: a Bot Chat prompt predating the
- epoch mechanism gets ONE title-gated migration rebuild; the stamped result cannot
- re-fire. A NULL or empty stored prompt already rebuilds every turn, so this probe
- is not a gate there and must not run.
- """
- if not stored_prompt:
- return False
- try:
- from tools.bot_mode_probe import (
- BOT_CHAT_TITLE,
- stored_bot_chat_prompt_needs_upgrade,
- stored_prompt_capability_stale,
- )
- home = None
- try:
- from agent.system_prompt import _agent_home
- home = _agent_home(agent)
- except Exception:
- pass
- if stored_prompt_capability_stale(stored_prompt, home):
- return True
- if not getattr(agent, "_bot_mode_protocol", True):
- return False
- title = str(getattr(agent, "_session_title_hint", "") or "").strip()
- if not title and agent._session_db and agent.session_id:
- try:
- title = str(agent._session_db.get_session_title(agent.session_id) or "").strip()
- except Exception:
- title = ""
- return title == BOT_CHAT_TITLE and bool(stored_bot_chat_prompt_needs_upgrade(stored_prompt, home))
- except Exception:
- return False
- def _persist_system_prompt(agent, failure_message: str, *, persist_tools: bool = False) -> None:
- """Persist ``agent._cached_system_prompt`` to the session row; failures log at WARNING
- (with ``failure_message``) because the gateway path (fresh AIAgent per turn) reads
- this row every turn, so a silent failure breaks prefix-cache reuse."""
- if not agent._session_db:
- return
- try:
- agent._session_db.update_system_prompt(agent.session_id, agent._cached_system_prompt)
- if persist_tools:
- from tools.mcp_tool_agent import persist_agent_tool_names
- persist_agent_tool_names(agent)
- except Exception as exc:
- logger.warning(failure_message, agent.session_id, exc)
- def _restore_pinned_tools(agent, session_row) -> list:
- """Pin ``agent.tools`` to the session's persisted array (tools freeze); returns the names
- this surface built BEFORE the pin merged a previous surface's tools back in."""
- from tools.mcp_tool_agent import agent_tool_names, persist_agent_tool_names, restore_agent_tool_prefix
- built_for_this_surface = agent_tool_names(agent)
- saved_tools = session_row.get("tool_names") if session_row else None
- try:
- pin = json.loads(saved_tools) if saved_tools else None
- except ValueError:
- pin = None # a pin hash whose row an older build's cleanup swept resolves to itself
- try:
- if pin:
- restore_agent_tool_prefix(agent, pin)
- elif session_row is not None and not getattr(agent, "_persist_disabled", False):
- # No usable pin (swept row, a session from before pins): pin what this turn sends,
- # or every later hop re-derives tools[] until the next compaction.
- persist_agent_tool_names(agent)
- except Exception:
- logger.debug("tool prefix restore skipped", exc_info=True)
- return built_for_this_surface
- def _refresh_bot_chat_tools(agent) -> None:
- """Rebuild ``agent.tools`` for a Bot Chat capability refresh through the builder that
- created the session, so the refreshed set is what a fresh desktop/TUI session gets
- (#124211). A canonical Bot Chat never forks, so its tools[] is otherwise a fossil of
- session creation: ``hermes tools enable/disable`` writes ``platform_toolsets`` and the
- automatic between-turns refresh reuses the build-time selection. Only the desktop/TUI
- gateway keeps agents alive across turns; every other surface builds a fresh agent whose
- tools[] already reflects config. No prefix preservation: a disabled toolset must drop,
- and the prompt rebuild this rides on already breaks the cache."""
- from gateway.session_policy import active_policy
- policy = active_policy()
- platform = getattr(agent, "platform", None)
- if policy is None and platform not in ("desktop", "tui"):
- return
- try:
- from tools.mcp_tool_agent import refresh_agent_mcp_tools
- if policy is not None:
- # Gateway-owned session (managed worker / owner turn): toolsets are frozen in its
- # launch policy, so live config edits never widen or narrow a running session.
- refresh_agent_mcp_tools(agent, enabled_override=list(policy.toolsets),
- quiet_mode=True, content_aware=True)
- return
- from tui_gateway.server import _load_disabled_toolsets, _load_enabled_toolsets
- refresh_agent_mcp_tools(
- agent, enabled_override=_load_enabled_toolsets(platform),
- disabled_override=_load_disabled_toolsets(), quiet_mode=True, content_aware=True)
- except Exception as exc:
- logger.warning("Bot Chat capability refresh kept the previous tools for session %s: %s",
- agent.session_id, exc)
- def _restore_or_build_system_prompt(agent, system_message, conversation_history):
- """Restore the cached system prompt from the session DB or build it fresh.
- Mutates ``agent._cached_system_prompt`` and persists a freshly-built prompt on first
- build. Row states ``missing``/``null``/``empty``/``present`` are logged and DB
- failures log at WARNING so silent prefix-cache misses show in ``agent.log``."""
- stored_prompt = None
- stored_state = "missing"
- session_row = None
- if conversation_history and agent._session_db:
- try:
- session_row = agent._session_db.get_session(agent.session_id)
- if session_row is not None:
- raw_prompt = session_row.get("system_prompt")
- stored_state = "null" if raw_prompt is None else ("empty" if raw_prompt == "" else "present")
- stored_prompt = raw_prompt or None
- except Exception as exc:
- logger.warning(
- "Session DB get_session failed for system-prompt restore (session=%s): %s. "
- "Falling back to fresh build — prefix cache will miss for this turn.",
- agent.session_id, exc,
- )
- if stored_prompt and _stored_prompt_matches_runtime(agent, stored_prompt):
- # NULL/empty rows never reach this probe: they already rebuild below.
- if _bot_chat_prompt_stale(agent, stored_prompt):
- logger.info(
- "Bot Chat capability epoch changed for session %s; rebuilding system prompt to "
- "adopt the new capability surface (one-time prefix-cache break).",
- agent.session_id,
- )
- agent._session_title_hint = "Bot Chat"
- # Read BEFORE the build: this rebuild also satisfies any pending prompt refresh.
- refresh_request = requested_epoch(agent)
- # The skills index cache (LRU + disk snapshot) does not watch the skills
- # dir; a capability refresh must rebuild THROUGH it or new skills are lost.
- try:
- from agent.prompt_builder import clear_skills_system_prompt_cache
- clear_skills_system_prompt_cache(clear_snapshot=True)
- except Exception:
- pass
- _refresh_bot_chat_tools(agent)
- agent._cached_system_prompt = agent._build_system_prompt(system_message)
- record_cache_break(agent, "toolset_change")
- stage_surface_switch_note(agent, agent._cached_system_prompt, conversation_history)
- # Persist so the NEXT turn restores the new bytes verbatim (cache break is
- # once per capability change). Tools re-pin too: without it the next
- # turn's pin-restore would resurrect the pre-refresh toolset (#124211).
- # on_session_start not re-fired: continuation.
- _persist_system_prompt(
- agent,
- "Session DB update_system_prompt failed after Bot Chat capability refresh "
- "(session=%s): %s. The refresh will re-fire next turn.",
- persist_tools=True,
- )
- stamp_prompt_epoch(agent, refresh_request)
- return
- refresh_epoch = stored_prompt_refresh_epoch(agent, session_row)
- if refresh_epoch:
- # `hermes sessions refresh-prompts` ran after this prompt was built. Pin tools[] first
- # (the refresh is about instructions; the frozen tool array stays), then rebuild once.
- _restore_pinned_tools(agent, session_row)
- rebuild_for_refresh(agent, system_message, conversation_history, refresh_epoch)
- return
- # Continuing session — reuse the exact system prompt from the
- # previous turn so the Anthropic cache prefix matches.
- agent._cached_system_prompt = stored_prompt
- # The reused bytes may describe the surface this conversation STARTED on; correct that
- # at the tail of the request instead of rebuilding the prompt in front of it (#104414).
- announced_switch = stage_surface_switch_note(agent, stored_prompt, conversation_history)
- # Same contract for tools[]: pin the array to the order this session already
- # sent (tools freeze) instead of re-probing every check_fn on a fresh AIAgent.
- # The pin holds ON the announcing turn too. tools[] is serialized AHEAD of the system
- # prompt this branch just preserved, so dropping the previous surface's toolset would
- # change the request at token 0 and re-prefill everything behind it — the exact cost
- # #104414 is about, paid on the exact turn we are here to make cheap. The merge still
- # ADDS what the new surface brought (a tui -> desktop switch pays a break no freeze can
- # avoid), and what it carries FORWARD is named in the note instead, so a tool that can
- # only answer ``tool_error("desktop only")`` here does not read as a live capability.
- built_for_this_surface = _restore_pinned_tools(agent, session_row)
- if announced_switch:
- note_inert_pinned_tools(agent, built_for_this_surface)
- # Prompt-section callbacks are new-session-only; recover their frozen bytes
- # from the persisted prompt so a compression rebuild keeps them. The static
- # prefix is not persisted either; rebuild it for the early cache breakpoint or
- # fresh-per-turn gateway agents fall back to the single-breakpoint layout
- # (reconstruct_static_prefix gates on _use_prompt_caching, fails open to legacy).
- from agent.system_prompt import reconstruct_static_prefix, restore_plugin_prompt_sections
- restore_plugin_prompt_sections(agent, stored_prompt)
- reconstruct_static_prefix(agent, system_message=system_message)
- return
- if stored_prompt:
- stored_state = "stale_runtime"
- logger.info(
- "Stored system prompt for session %s has stale runtime identity; "
- "rebuilding for model=%s provider=%s.",
- agent.session_id, getattr(agent, "model", "") or "", getattr(agent, "provider", "") or "",
- )
- if conversation_history and stored_state in ("null", "empty"):
- # Continuing session with an unusable stored prompt: every turn now rebuilds
- # and the prefix cache misses every time.
- logger.warning(
- "Stored system prompt for session %s is %s; rebuilding from scratch this turn. Prefix "
- "cache will miss until the rebuild persists. Investigate the previous turn's "
- "update_system_prompt write path.",
- agent.session_id, stored_state,
- )
- # First turn of a new session (or recovering from a broken stored prompt). Rebuilding an
- # EXISTING session's prompt (cwd drift, model switch) still keeps its pinned tools[]: this
- # surface's own build (the -q footprint, its tool_search catalog) would otherwise be
- # persisted over the pin below. Pinned first, so the prompt describes the tools sent.
- built_for_this_surface = _restore_pinned_tools(agent, session_row)
- # Read before the build: a refresh request landing mid-build still fires next turn.
- refresh_epoch = requested_epoch(agent)
- agent._cached_system_prompt = agent._build_system_prompt(system_message)
- if conversation_history:
- record_prompt_rebuild(agent, stored_prompt, stored_state, agent._cached_system_prompt)
- # The rebuilt prompt describes the CURRENT surface, but a surface note left in the
- # transcript by an earlier switch does not — retire it here too, or a rebuild for an
- # unrelated reason (a model switch) would leave the newest interface statement in the
- # request naming a surface the conversation has left (#104414).
- stage_surface_switch_note(agent, agent._cached_system_prompt, conversation_history)
- note_inert_pinned_tools(agent, built_for_this_surface)
- # Persistence-disabled forks share their parent's session ID and are not real sessions.
- if not getattr(agent, "_persist_disabled", False):
- try:
- from hermes_cli.lifecycle import invoke_hook as _invoke_hook
- _invoke_hook(
- "on_session_start", session_id=agent.session_id, model=agent.model,
- platform=getattr(agent, "platform", None) or "",
- )
- except Exception as exc:
- logger.warning("on_session_start hook failed: %s", exc)
- # Cold-start credits seed (L3) fallback for the first-turn path; TUI/desktop seed at
- # session open, so this is idempotent (skips when _credits_state exists). Fail-open.
- try:
- from agent.credits_tracker import seed_credits_at_session_start
- seed_credits_at_session_start(agent)
- except Exception:
- logger.debug("cold-start credits seed failed (fail-open)", exc_info=True)
- _persist_system_prompt(
- agent,
- "Session DB update_system_prompt failed for session %s: %s. Subsequent turns will "
- "rebuild the system prompt and miss the prefix cache.",
- persist_tools=True,
- )
- stamp_prompt_epoch(agent, refresh_epoch)
- def _stored_prompt_matches_runtime(agent, prompt: str) -> bool:
- """Return False when the persisted runtime-identity lines are stale."""
- # Model/provider identity, then cwd drift. A cwd change is a real content change (context
- # files, the workspace snapshot and the coding posture are all resolved from it), so it
- # still rebuilds; the runtime surface does not (agent/surface_switch.py).
- # The builder omits an empty trailer line, so stored-but-now-empty is a route change too;
- # the rebuilt prompt then carries no line and matches from the next turn on. Stored-empty
- # (pre-trailer prompts) keeps reusing.
- for label, attr in (("Model", "model"), ("Provider", "provider")):
- stored = identity_line_value(prompt, label)
- current = str(getattr(agent, attr, "") or "").strip()
- if stored and stored != current:
- return False
- # A prompt stamped for another session (a /branch child copies its parent's bytes) must not
- # tell the model a foreign Session ID. Checked only when the trailer is on: with it off, a
- # "Session ID:" line in project text would read as a mismatch and rebuild every turn.
- stored_sid = identity_line_value(prompt, "Session ID")
- if stored_sid and getattr(agent, "pass_session_id", False) and stored_sid != agent.session_id:
- return False
- # Compare against resolve_agent_cwd() — the SAME resolver used to build the
- # prompt — so TERMINAL_CWD sessions are not falsely rejected.
- stored_cwd = runtime_host_value(prompt, "Current working directory")
- if stored_cwd and stored_cwd != str(resolve_agent_cwd()):
- return False
- # Platform is deliberately NOT an identity field: a surface switch does not invalidate the
- # stored bytes, it only makes their interface section out of date, and that is corrected by
- # agent.surface_switch.stage_surface_switch_note without touching the cached prefix (#104414).
- return True
- # Named so _is_synthetic_compression_user_turn can recognize a crash-persisted nudge by
- # content (SessionDB projection strips the _length_continuation_nudge tag).
- _LENGTH_CONTINUATION_NETWORK_STUB = (
- "[System: The previous response was cut off by a network error mid-stream — a transport "
- "interruption, NOT a change in your capabilities. Your tools are still fully available; call "
- "them as normal and ignore any earlier claim that you lack tool access. Continue the task "
- "from where you left off. Do not restart or repeat prior text.]"
- )
- _LENGTH_CONTINUATION_OUTPUT_LIMIT = (
- "[System: Your previous response was truncated by the output length limit. Continue exactly "
- "where you left off. Do not restart or repeat prior text. Finish the answer directly.]"
- )
- # Pre-#74990 wording; kept so crash-persisted nudges from older sessions are still recognized.
- _LEGACY_LENGTH_CONTINUATION_NETWORK_STUB = (
- "[System: The previous response was cut off by a network error mid-stream. Continue exactly "
- "where you left off. Do not restart or repeat prior text. Finish the answer directly.]"
- )
- # The dropped-tools variant interpolates tool names; matched by prefix.
- _LENGTH_CONTINUATION_DROPPED_TOOLS_PREFIX = "[System: Your previous tool call "
- def _get_continuation_prompt(is_partial_stub: bool, dropped_tools: Optional[list[str]] = None) -> str:
- if is_partial_stub and dropped_tools:
- tool_list = ", ".join(dropped_tools[:3])
- return (
- f"{_LENGTH_CONTINUATION_DROPPED_TOOLS_PREFIX}({tool_list}) was too large and "
- "the stream timed out before it could be delivered. Do NOT retry the same tool call "
- "with the same large content. Instead, break the content into multiple smaller tool "
- "calls (e.g. use multiple patch calls or write smaller files). Each tool call's "
- "arguments must be under ~8K tokens to avoid stream timeouts. The cut was a transport "
- "interruption, not a capability change — your tools remain fully available.]"
- )
- return _LENGTH_CONTINUATION_NETWORK_STUB if is_partial_stub else _LENGTH_CONTINUATION_OUTPUT_LIMIT
- # Codex/Responses turns that returned only internal reasoning: a bare retry would be
- # byte-identical, so the model repeats it.
- _CODEX_INCOMPLETE_NUDGE = (
- "[System: Your previous response contained only internal reasoning and never produced a "
- "visible answer or tool call. Do not keep thinking. Produce your final answer as plain text "
- "now (or make the tool call you were planning).]"
- )
- # Re-prompt after an acknowledgment-only Codex/Responses reply.
- _CODEX_ACK_CONTINUATION_NUDGE = (
- "[System: Continue now. Execute the required tool calls and only send your final answer "
- "after completing the task.]"
- )
- # Re-prompt after a collapsed fragment ended a turn that had done real tool work (#103483). Asks
- # for the same answer again when it WAS complete, so a false positive costs one call, never the answer.
- _DEGENERATE_FINAL_NUDGE = (
- "[System: Your previous message ended the turn with a fragment that is not a usable answer. "
- "If the task is unfinished, continue it and then give the complete answer. If that fragment "
- "WAS your complete answer, send it again exactly as before.]"
- )
- # Re-prompt for finish_reason="tool_calls" with empty tool_calls (an interrupt mid-retry can persist it).
- _DROPPED_TOOLCALL_NUDGE_CONTENT = (
- "Your previous turn indicated a tool call but none was included. Do not narrate a plan or "
- "restate intent — issue the actual tool call now to continue the task."
- )
- # Re-prompt for an empty response after tool calls (#9400); the metadata flag does not
- # survive SessionDB projection, so it is matched by content.
- _EMPTY_TOOL_RESPONSE_NUDGE = (
- "You just executed tool calls but returned an empty response. Please process the tool "
- "results above and continue with the task."
- )
- # Memo for send-path tool-call argument canonicalization (re-run on every historical call
- # each iteration). Sound because canonicalization is pure; malformed strings raise before
- # being stored, so the repair fallback is never memoized. The byte budget exists because
- # argument strings can run 100KB+, so a count bound alone does not bound memory.
- _CANON_ARGS_CACHE: dict[str, str] = {}
- _CANON_ARGS_CACHE_MAX = 4096
- _CANON_ARGS_CACHE_MAX_BYTES = 32 * 1024 * 1024
- _canon_args_cache_bytes = 0
- def _canonicalize_tool_call_arguments(arg_str: str) -> str:
- """Canonical wire form of a tool-call arguments JSON string; raises on malformed input
- (the caller falls back to ``_repair_tool_call_arguments``)."""
- global _canon_args_cache_bytes
- cached = _CANON_ARGS_CACHE.get(arg_str)
- if cached is not None:
- return cached
- canonical = json.dumps(json.loads(arg_str), separators=(",", ":"), sort_keys=True)
- _CANON_ARGS_CACHE[arg_str] = canonical
- _canon_args_cache_bytes += len(arg_str) + len(canonical)
- while len(_CANON_ARGS_CACHE) > _CANON_ARGS_CACHE_MAX or (
- _canon_args_cache_bytes > _CANON_ARGS_CACHE_MAX_BYTES and len(_CANON_ARGS_CACHE) > 1
- ):
- try:
- evicted_key = next(iter(_CANON_ARGS_CACHE))
- _canon_args_cache_bytes -= len(evicted_key) + len(_CANON_ARGS_CACHE.pop(evicted_key))
- except (StopIteration, KeyError, RuntimeError):
- break
- return canonical
- def _clone_message_for_send(msg):
- """Structural clone (dicts/lists recursively, immutable leaves shared) of a history
- message for the per-call API copy, so send-path rewrites never reach the persisted
- transcript (#80498). Cheaper than deepcopy: messages are JSON-shaped and acyclic."""
- if isinstance(msg, dict):
- return {k: _clone_message_for_send(v) if isinstance(v, (dict, list)) else v for k, v in msg.items()}
- if isinstance(msg, list):
- return [_clone_message_for_send(v) if isinstance(v, (dict, list)) else v for v in msg]
- return msg
- def _canonicalize_api_tool_calls(api_messages) -> None:
- """Canonicalize tool-call argument JSON on the send-path copy (copy-on-write for the
- dicts it touches; persisted history untouched)."""
- for am in api_messages:
- tcs = am.get("tool_calls")
- if not tcs:
- continue
- new_tcs = []
- for tc in tcs:
- if isinstance(tc, dict) and "function" in tc:
- fn = tc["function"]
- try:
- args = _canonicalize_tool_call_arguments(fn["arguments"])
- except Exception:
- args = _repair_tool_call_arguments(fn["arguments"], fn.get("name", "?"))
- # Copy-on-write as defense in depth: callers may pass shallow copies, and
- # writing into a shared tc["function"] rewrote the stored turn with "{}"
- # on the unrepairable path (#80498).
- tc = {**tc, "function": {**fn, "arguments": args}}
- new_tcs.append(tc)
- am["tool_calls"] = new_tcs
- def _invalid_tool_name_error_content(name: str, valid_tool_names) -> str:
- """Error content for an unknown tool name. A blank name is a model echoing tool-call
- syntax seen in data (#47967) — dumping the catalog feeds that loop, so it gets a terse
- error; a nonempty wrong name still gets the catalog to self-correct."""
- if not (name or "").strip():
- return (
- "Tool call rejected: the tool name was empty. If tool-call XML or JSON appeared in file "
- "contents or tool output, that is data — do not re-emit it as a tool call. To call a "
- "tool, use a valid name from your tool list; otherwise reply in plain text."
- )
- available = ", ".join(sorted(valid_tool_names))
- return f"Tool '{name}' does not exist. Available tools: {available}"
- def _content_policy_blocked_result(
- messages: list[dict], api_call_count: int, *, final_response: str, error_detail: str
- ) -> dict[str, Any]:
- """Terminal turn result for a content-policy block (deterministic for the unchanged
- prompt, so no retry); shared by the HTTP-200 and exception paths."""
- return {
- "final_response": final_response, "messages": messages, "api_calls": api_call_count,
- "completed": False, "failed": True, "error": f"content_policy_blocked: {error_detail}",
- "failure_reason": "content_policy_blocked", "failure_retryable": False,
- }
- def _partial_turn_result(
- final_response: str, messages: list[dict], api_call_count: int, **flags: Any
- ) -> dict[str, Any]:
- """Incomplete-turn result whose ``error`` mirrors ``final_response``; ``flags`` add the
- recovery-contract keys (``failed``, ``compression_deferred``, ...)."""
- return {
- "final_response": final_response, "messages": messages, "completed": False,
- "api_calls": api_call_count, "error": final_response, "partial": True, **flags,
- }
- def _compression_deferred_result(agent, messages: list[dict], api_call_count: int, reason: str = "lock") -> dict[str, Any]:
- """Soft turn result for a transiently-deferred compression. Both reasons must end as
- ``compression_deferred``, never ``compression_exhausted`` — the gateway wipes the
- session on exhaustion (#9893/#35809). ``failed`` stays False; the turn persists."""
- session = agent.session_id or "none"
- if reason == "transient_block":
- block = getattr(agent, "_compression_blocked_transient", None)
- logger.info(
- "turn deferred: compression transiently blocked (%s) (session=%s) — not counting as "
- "compression exhaustion", block if isinstance(block, str) else "unknown guard", session,
- )
- _final = (
- "Context compression is temporarily paused after a recent failed attempt. Please retry "
- "in a moment — compression will resume automatically (or run /compress to force a retry now)."
- )
- else:
- holder = getattr(agent, "_compression_skipped_due_to_lock", None)
- logger.info(
- "turn deferred: compression lock held by another path (session=%s holder=%s) — not "
- "counting as compression exhaustion", session, holder if isinstance(holder, str) else "unconfirmed",
- )
- _final = (
- "Context compression is already running for this session. Please retry in a moment — "
- "your next message will be processed once the concurrent compression finishes."
- )
- try:
- agent._flush_status_buffer()
- except Exception:
- pass
- return _partial_turn_result(
- _final, messages, api_call_count,
- failed=False, compression_deferred=True, session_id=agent.session_id,
- )
- def _provider_overflow_exhausted_result(
- agent, messages: list[dict], conversation_history, api_call_count: int,
- request_pressure_tokens: int, max_compression_attempts: int,
- ) -> dict[str, Any]:
- """Fail closed when a rebuilt request is still too large after recovery."""
- agent._flush_status_buffer()
- logger.error(
- "%sContext compression failed after %d attempts; rebuilt request "
- "remains over threshold at ~%s tokens.",
- agent.log_prefix, max_compression_attempts, f"{request_pressure_tokens:,}",
- )
- # Host progress-aware timeout (#98722, salvaged from #98741): the provider proved the request does not
- # fit, but this recovery pass spent the full wait budget without a committed summary. Re-sending the
- # unchanged request would bounce off the same overflow error and re-enter compression in the same turn.
- # End the turn with the typed recovery contract instead — transcript intact, no further doomed provider
- # sends.
- # Prior <3 retries (or an earlier successful tool batch) leave a tool-result tail. Closing it here
- # matches interrupt aborts (#48879 / #52592) so the next user turn is not tool→user for strict
- # providers.
- agent._persist_session(messages, conversation_history)
- return _partial_turn_result(
- site_copy("context_overflow", model=agent.model),
- messages, api_call_count, failed=True, compression_exhausted=True,
- turn_exit_reason="context_compression_exhausted",
- failure_reason="context_overflow", failure_retryable=False,
- )
- def _rewrite_system_content_blocks(system_message: dict, effective: str) -> bool:
- """Rewrite a cache-decorated system message in place, keeping its blocks (a bare string
- over the ``[static prefix, volatile tail]`` list would drop both cache_control
- breakpoints). Returns False when the shape cannot be safely patched."""
- content = system_message.get("content")
- if not isinstance(content, list) or not content or not all(
- isinstance(part, dict) and part.get("type") == "text" for part in content
- ):
- return False
- if len(content) == 1:
- content[0]["text"] = effective
- return True
- if len(content) == 2:
- head = content[0].get("text") or ""
- if head and effective.startswith(head) and effective[len(head):]:
- content[1]["text"] = effective[len(head):]
- return True
- return False
- def _sync_failover_system_message(agent, api_messages, active_system_prompt):
- """Refresh the in-flight system message after a provider failover: ``api_messages`` were
- built pre-failover and are reused each retry. Returns the new ``active_system_prompt``."""
- sp = getattr(agent, "_cached_system_prompt", None)
- if not isinstance(sp, str) or not sp:
- return active_system_prompt
- if api_messages and api_messages[0].get("role") == "system":
- effective = (sp + "\n\n" + agent.ephemeral_system_prompt).strip() if agent.ephemeral_system_prompt else sp
- if not _rewrite_system_content_blocks(api_messages[0], effective):
- api_messages[0]["content"] = effective
- return sp
- def _arm_fallback_restart(agent, api_messages, active_system_prompt, _retry):
- """After a successful fallback activation: sync the system message and arm
- ``restart_with_rebuilt_messages``. Callers also zero ``retry_count`` /
- ``compression_attempts`` and ``break`` the retry loop."""
- active_system_prompt = _sync_failover_system_message(
- agent, api_messages, active_system_prompt)
- _retry.primary_recovery_attempted = False
- _retry.restart_with_rebuilt_messages = True
- return active_system_prompt
- def _ensure_cached_system_prompt_static(agent, system_message=None) -> None:
- """Rebuild ``_cached_system_prompt_static`` when caching becomes active (#72626): sessions
- restored under a cache-off primary would otherwise fall back to the legacy layout after
- failover to a cache-on provider."""
- from agent.system_prompt import reconstruct_static_prefix
- reconstruct_static_prefix(agent, system_message=system_message, log_label="failover redecoration")
- def _peel_moa_guidance(messages: list[dict[str, Any]], guidance: Any) -> list[dict[str, Any]]:
- """Remove MoA reference guidance attached by ``_attach_reference_guidance``."""
- from agent.moa_loop import peel_reference_guidance
- return peel_reference_guidance(messages, guidance)
- def _redecorate_prompt_cache_for_provider(
- agent, api_messages: list[dict[str, Any]], *, system_message=None,
- moa_prepared: Optional[dict[str, Any]] = None, tools_for_api: Optional[list[dict[str, Any]]] = None,
- ) -> tuple[list[dict[str, Any]], Optional[dict[str, Any]], list[dict[str, Any]]]:
- """Strip and re-apply cache_control for the *current* provider policy — failover
- ``continue`` paths reuse ``api_messages`` (#72626). MoA guidance is peeled and rebased."""
- messages: list[dict[str, Any]] = [dict(m) if isinstance(m, dict) else m for m in (api_messages or [])]
- prepared = moa_prepared
- guidance = prepared.get("guidance") if isinstance(prepared, dict) else None
- if guidance:
- messages = _peel_moa_guidance(messages, guidance)
- strip_anthropic_cache_control(messages)
- planned_tools = strip_anthropic_tool_cache_control(
- tools_for_api if tools_for_api is not None else getattr(agent, "tools", [])
- )
- if prepared is not None and getattr(agent, "provider", None) == "moa":
- # Prepared MoA state is canonical: the synchronous acting-aggregator
- # sender owns its destination-local cache plan after it resolves the slot.
- completions = getattr(getattr(agent.client, "chat", None), "completions", None)
- rebase = getattr(completions, "rebase_prepared_request", None)
- if callable(rebase):
- prepared = rebase(prepared, messages)
- messages = prepared["messages"]
- # Direct attribute access, not getattr: the flags are always initialized on
- # AIAgent, and a default would mask a real init bug as silent cache-off.
- elif agent._use_prompt_caching:
- _ensure_cached_system_prompt_static(agent, system_message=system_message)
- static = getattr(agent, "_cached_system_prompt_static", None)
- from agent.prompt_caching import envelope_tool_part_cache_markers_supported
- plan = build_prompt_cache_plan(
- messages,
- planned_tools,
- # Clamp per-destination: a configured 1h regresses to 5m on
- # Qwen/Alibaba routes, whose context cache is 5m-only (#84733).
- cache_ttl=effective_cache_ttl(agent._cache_ttl, provider=agent.provider, model=agent.model),
- native_anthropic=agent._use_native_cache_layout,
- static_system_prefix=static if isinstance(static, str) else None,
- direct_native_tool_cache=getattr(
- agent, "_direct_native_anthropic_tool_cache_capability", lambda: False
- )(),
- # LiteLLM-style envelope routes forward part-level markers into
- # tool_result.content[] → non-retryable 400 (#89886).
- tool_part_markers=envelope_tool_part_cache_markers_supported(
- getattr(agent, "provider", ""), getattr(agent, "base_url", "")
- ),
- )
- messages, planned_tools = plan.messages, plan.tools
- return messages, prepared, planned_tools
- def _engine_overrides_hook(engine: Any, name: str) -> bool:
- """True when ``engine`` implements ContextEngine hook ``name`` itself.
- Non-implementing engines must pay nothing per turn; ``hasattr`` is not enough because
- the ABC defines a no-op default. Lazy import avoids a cycle with agent.context_engine."""
- hook = getattr(engine, name, None)
- if engine is None or not callable(hook):
- return False
- try:
- from agent.context_engine import ContextEngine as _CE
- return getattr(hook, "__func__", None) is not getattr(_CE, name)
- except Exception:
- return True
- def _apply_context_engine_selection(
- agent: Any, api_messages: list[dict[str, Any]], conversation_messages: list[dict[str, Any]],
- incoming_message: Optional[dict[str, Any]], *, logger: Any,
- ) -> list[dict[str, Any]]:
- """Run the optional per-turn ``ContextEngine.select_context()`` hook, fail-open: any
- exception or invalid return yields ``api_messages`` unchanged; history is never mutated."""
- engine = getattr(agent, "context_compressor", None)
- if not _engine_overrides_hook(engine, "select_context"):
- return api_messages
- session_label = getattr(agent, "session_id", None) or "-"
- # Structural clones: the engine must not be able to write through nested
- # containers into persisted history; only the request list is acted on (#80498).
- try:
- selected = engine.select_context(
- api_messages,
- conversation_messages=(
- [_clone_message_for_send(m) for m in conversation_messages]
- if conversation_messages is not None else None
- ),
- incoming_message=(
- _clone_message_for_send(incoming_message)
- if isinstance(incoming_message, dict) else incoming_message
- ),
- budget_tokens=getattr(engine, "context_length", 0) or 0,
- )
- except Exception:
- logger.warning(
- "Context engine select_context hook failed; using unmodified request messages (session=%s)",
- session_label, exc_info=True,
- )
- return api_messages
- if selected is None:
- return api_messages
- # Require a NON-EMPTY list of dicts: ``all([])`` is ``True``, so a ``[]`` from a
- # buggy engine would otherwise replace the request instead of failing open.
- if isinstance(selected, list) and selected and all(isinstance(m, dict) for m in selected):
- # The engine may hand back the ``conversation_messages`` clones (or its own dicts) that still
- # carry persistence-only fields; the request copy was stripped BEFORE this hook, so strip the
- # selection too or those fields reach the provider. Dicts without them pass through as-is.
- stripped = [without_persistence_fields(m) for m in selected]
- return selected if all(a is b for a, b in zip(stripped, selected)) else stripped
- logger.warning(
- "Context engine select_context returned an invalid value "
- "(not a non-empty list of dicts); ignoring (session=%s)", session_label,
- )
- return api_messages
- def _notify_context_engine_turn_complete(
- agent: Any, messages: list[dict[str, Any]], *, usage: Optional[dict[str, Any]] = None, logger: Any, **meta: Any
- ) -> None:
- """Notify the active context engine that a user turn has finished (fail-open; the engine
- gets a copy so it cannot mutate the persisted transcript)."""
- engine = getattr(agent, "context_compressor", None)
- if not _engine_overrides_hook(engine, "on_turn_complete"):
- return
- try:
- # Structural clones: dict(m) would let a hook write into nested containers of the
- # persisted transcript (#80498).
- engine.on_turn_complete([_clone_message_for_send(m) for m in messages], usage=usage, **meta)
- except Exception:
- logger.warning(
- "Context engine on_turn_complete hook failed (session=%s)",
- getattr(agent, "session_id", None) or "-", exc_info=True,
- )
- def _decode_inline_moa_turn(user_message, persist_user_message):
- """Decode a MoA preset encoded into ``user_message``; returns ``(user_message,
- moa_config, persist_user_message)``, unchanged with ``moa_config=None`` otherwise."""
- try:
- from hermes_cli.moa_config import decode_moa_turn
- _decoded_message, _decoded_moa_config = decode_moa_turn(user_message)
- if _decoded_moa_config is not None:
- if persist_user_message is None:
- persist_user_message = _decoded_message
- return _decoded_message, _decoded_moa_config, persist_user_message
- except Exception:
- pass
- return user_message, None, persist_user_message
- def _preflight_timeout_result(agent, exc, conversation_history) -> dict[str, Any]:
- """Typed recovery result when turn-start preflight compression timed out (#98424): no
- provider call was sent, and surfaces would otherwise hide the actionable guidance."""
- logger.warning(
- "Turn-start preflight compression timed out — ending turn with typed recovery result: %s", exc,
- )
- # Clear the tripwire slot note_turn_start registered (the early return skips the persist
- # funnel). The user row is deliberately NOT persisted (#7100).
- from agent.agent_runtime_helpers import note_turn_persisted
- note_turn_persisted(agent)
- # Not _COMPRESSION_TIMEOUT_FINAL_RESPONSE — that describes a different state
- # (compression ran, could not reduce); the exception text carries the guidance.
- return _partial_turn_result(
- str(exc), list(conversation_history or []), 0,
- failed=True, compression_exhausted=True, turn_exit_reason="context_compression_timeout",
- failure_reason="context_overflow", failure_retryable=False,
- )
- @dataclass
- class _LoopState:
- """Every local the turn loop threads through the phase helpers in ``agent/turn_*.py``.
- Helpers take the loop locals they need as keyword arguments named like these fields and
- return a verdict whose non-``action``/``result`` fields carry the same names;
- :func:`_run_phase` passes and copies them back by name, so a new helper input/output
- needs a field here and nothing else. Per-iteration slots are rebound by the phases
- before any later phase reads them, exactly as the former inline locals were."""
- # Fixed for the turn.
- user_message: Any
- system_message: Any
- moa_config: Any
- original_user_message: Any
- conversation_history: Any
- effective_task_id: Any
- turn_id: Any
- _should_review_memory: Any
- _plugin_user_context: Any
- _ext_prefetch_cache: Any
- # Turn-scoped state (rebound by the phases).
- messages: Any
- active_system_prompt: Any
- current_turn_user_idx: Any
- _preflight_compression_blocked: Any
- # Compression attempt cap shared by the pre-API gate, 413 handlers and post-tool compaction:
- # a consecutive-ineffective-attempt backstop, rearmed only after a provider response
- # reports a prompt below threshold.
- max_compression_attempts: Any
- api_call_count: int = 0
- final_response: Any = None
- interrupted: bool = False
- failed: bool = False
- codex_ack_continuations: int = 0
- length_continue_retries: int = 0
- # Backstop for the refunding restarts (redirect / rebuilt-for-fallback). Unlike
- # ``retry_count`` (rebound to 0 each iteration) this survives across iterations until a
- # response arrives, so a runaway interrupt/redirect that keeps re-arming a restart flag
- # cannot refund the iteration budget forever and hold the turn lease indefinitely.
- restart_count: int = 0
- _outer_error_count: int = 0 # outer-loop exceptions this turn (#92450), see _MAX_OUTER_LOOP_ERRORS
- truncated_tool_call_retries: int = 0
- truncated_response_parts: list[tuple[str, bool]] = field(default_factory=list)
- compression_attempts: int = 0
- _last_preflight_pressure: Optional[int] = None
- # A provider overflow outweighs the rough-estimate calibration that defers preflight after
- # compaction: stays armed until the rebuilt request is below the threshold.
- _provider_overflow_recovery_pending: bool = False
- # A compression host-timeout ended the turn; finalize reuses the gateway context-recovery
- # contract (error/partial/compression_exhausted) (#98722).
- _compression_timeout_exhausted: bool = False
- _turn_exit_reason: str = "unknown" # diagnostic: why the loop ended
- # Answer held back by a verification gate (best user-facing result if the continuation
- # exhausts the budget) and whether it was streamed as interim; ``_response_was_previewed``
- # is set ONLY if it becomes the final response (#65919).
- _pending_verification_response: Any = None
- _pending_verification_response_previewed: bool = False
- # MoA guidance retained across a pre-API compression, rebased next iteration (no second fan-out).
- pending_moa_prepared_request: Any = None
- # Per-iteration slots.
- request_logger: Any = None
- api_messages: Any = None
- tools_for_api: Any = None
- _moa_prepared_request: Any = None
- approx_tokens: Any = None
- request_pressure_tokens: Any = None
- total_chars: Any = None
- thinking_spinner: Any = None
- api_start_time: Any = None
- retry_count: int = 0
- max_retries: Any = None
- _retry: Any = None
- finish_reason: str = "stop"
- response: Any = None # None when every retry failed
- api_kwargs: Any = None # None until built; read by the except handlers
- api_request_id: Any = None
- _original_api_kwargs: Any = None
- _llm_middleware_trace: Any = None
- api_duration: Any = None
- assistant_message: Any = None
- # _LoopState fields seeded from TurnContext (same name minus the leading underscore).
- _CTX_FIELDS = frozenset({
- "user_message", "original_user_message", "conversation_history", "effective_task_id", "turn_id",
- "_should_review_memory", "_plugin_user_context", "_ext_prefetch_cache", "messages",
- "active_system_prompt", "current_turn_user_idx", "_preflight_compression_blocked",
- })
- # Keyword names each phase helper takes (minus ``agent``), cached per function object.
- _PHASE_PARAMS: dict[Any, tuple] = {}
- # Verdict fields the loop latches (only ever sets True) instead of copying back:
- # ``handle_api_error`` reports overflow recovery per call and must not clear an earlier arm.
- _LATCHED_VERDICT_FIELDS = {"handle_api_error": frozenset({"_provider_overflow_recovery_pending"})}
- def _run_phase(fn, agent, state: _LoopState, **extra):
- """Call phase helper ``fn`` with the loop locals it names, copy its verdict fields back.
- ``extra`` supplies non-state arguments (the caught exception). Returns the verdict so
- the caller can act on ``.action`` / ``.result``."""
- params = _PHASE_PARAMS.get(fn)
- if params is None:
- params = _PHASE_PARAMS[fn] = tuple(p for p in inspect.signature(fn).parameters if p != "agent")
- verdict = fn(agent, **{n: extra[n] if n in extra else getattr(state, n) for n in params})
- latched = _LATCHED_VERDICT_FIELDS.get(getattr(fn, "__name__", ""), ())
- for f in fields(verdict):
- if f.name in ("action", "result"):
- continue
- value = getattr(verdict, f.name)
- if f.name not in latched:
- setattr(state, f.name, value)
- elif value:
- setattr(state, f.name, True)
- return verdict
- def _run_api_retry_loop(agent, s: _LoopState) -> Optional[dict[str, Any]]:
- """One API call with its retry/recovery loop (guard → build → call → check, error handlers).
- Returns a turn result dict when a phase ends the turn, else None once the loop is left
- (success, a restart armed on ``s._retry``, interrupt, or retries exhausted)."""
- while s.retry_count < s.max_retries:
- _ng = _run_phase(nous_rate_limit_guard, agent, s)
- if _ng.action == "return":
- return _ng.result
- if _ng.action == "break":
- return None
- try:
- _run_phase(build_api_request, agent, s)
- if _run_phase(perform_api_call, agent, s).action == "break":
- return None
- _rc = _run_phase(check_api_response, agent, s)
- if _rc.action == "return":
- return _rc.result
- if _rc.action == "break":
- return None
- except InterruptedError:
- if _run_phase(handle_api_interrupt, agent, s).action == "break":
- return None
- except Exception as api_error:
- _ae = _run_phase(handle_api_error, agent, s, api_error=api_error)
- if _ae.action == "return":
- return _ae.result
- if _ae.action == "break":
- return None
- return None
- def _finalize_from_state(agent: Any, s: _LoopState) -> dict[str, Any]:
- """Post-loop finalization (agent/turn_finalizer.finalize_turn) fed from the loop state."""
- return finalize_turn(agent, **{
- name: getattr(s, name)
- for name in inspect.signature(finalize_turn).parameters if name != "agent"
- })
- def _turn_without_loop(agent: Any, s: _LoopState, block_message: str) -> Optional[dict[str, Any]]:
- """The turn's result when it never enters the generic loop, else None. A ``pre_llm_call``
- veto finalizes with no inference request (the block message closes the transcript pair);
- the opt-in codex app-server runtime runs the whole turn in its subprocess
- (agent/transports/codex_app_server_session.py)."""
- if block_message:
- s.final_response, s._turn_exit_reason = block_message, PRE_LLM_BLOCK_EXIT_REASON
- return _finalize_from_state(agent, s)
- return _codex_app_server_turn(agent, s) if agent.api_mode == "codex_app_server" else None
- def _codex_app_server_turn(agent: Any, s: Any) -> Optional[dict[str, Any]]:
- """The codex app-server's result for this turn, or None when its failure activated a fallback and the
- generic loop retries the same user turn."""
- codex_result = agent._run_codex_app_server_turn(
- user_message=s.user_message, original_user_message=s.original_user_message,
- messages=s.messages, effective_task_id=s.effective_task_id,
- should_review_memory=s._should_review_memory, turn_id=s.turn_id,
- )
- from agent.turn_recovery import activate_codex_app_server_fallback
- if not activate_codex_app_server_fallback(agent, codex_result):
- return codex_result
- # Fallback activation rewrote provider/model/api_mode: retry this same user turn on the generic
- # loop below, keeping codex's projected rows and its failed API call in the turn's accounting.
- s.api_call_count = int(codex_result.get("api_calls") or 0)
- s.active_system_prompt = _sync_failover_system_message(agent, None, s.active_system_prompt)
- return None
- def _run_conversation_turn(
- agent,
- user_message: Any,
- system_message: str | None = None,
- conversation_history: list[dict[str, Any]] | None = None,
- task_id: str | None = None,
- stream_callback: Optional[callable] = None,
- persist_user_message: Optional[Any] = None,
- persist_user_timestamp: Optional[float] = None,
- persist_user_display_kind: Optional[str] = None,
- persist_user_display_metadata: Optional[dict[str, Any]] = None,
- persist_user_platform_id: Optional[str] = None,
- turn_author: Optional[dict[str, Any]] = None,
- moa_config: Optional[dict[str, Any]] = None,
- title_user_message: Optional[str] = None,
- prelude: Optional[Prelude] = None,
- ) -> dict[str, Any]:
- """Run a complete conversation with tool calling until completion; returns the result dict.
- ``stream_callback``: per-text-delta callback (TTS). ``persist_user_message``: clean text to
- store when ``user_message`` carries API-only synthetic prefixes; timestamp / platform id are
- stored as metadata (platform id lets restart drain recovery dedup).
- ``title_user_message``: optional pre-injection text for titles only (None uses the
- model-facing message; an empty string suppresses titling for this turn).
- ``persist_user_display_*``:
- display-only event rendering; the model still receives the message unchanged.
- ``prelude``: scripted tool calls played before the first model call (``agent/turn_scripted_prelude.py``)."""
- if moa_config is None:
- user_message, moa_config, persist_user_message = _decode_inline_moa_turn(
- user_message, persist_user_message
- )
- # The gateway caches agents across turns; compression state is per-turn, or a stale
- # in-place boundary would make a later uncompressed result look compacted.
- agent._last_compaction_in_place = agent._last_compression_attempt_recorded = False
- agent._last_compression_attempt_in_place = None
- begin_fast_mode_turn(agent, conversation_history)
- # Adopt ~/.hermes/.env credential/base-url edits made since the last turn — a
- # Settings save updates .env, not this worker's client (#67821). No-op if unchanged.
- try:
- agent._try_refresh_env_client_credentials()
- except Exception:
- logger.debug("per-turn env credential refresh failed", exc_info=True)
- # A loaded ``hold_stream`` transform in this profile: the user sees only its rewritten final, made
- # by the transforms live now (a plugin reload mid-turn cannot drop them before the final). Set
- # before any exit, so an early one never reads the previous turn's hold.
- agent._held_transforms = held_stream_transforms()
- agent._hold_stream = bool(agent._held_transforms)
- # Per-turn setup: build_turn_context mutates ``agent`` and returns the locals the loop reads.
- try:
- _ctx = build_turn_context(
- agent, user_message, system_message, conversation_history, task_id,
- stream_callback, persist_user_message, persist_user_timestamp,
- persist_user_display_kind=persist_user_display_kind,
- persist_user_display_metadata=persist_user_display_metadata,
- persist_user_platform_id=persist_user_platform_id,
- turn_author=turn_author,
- restore_or_build_system_prompt=_restore_or_build_system_prompt,
- install_safe_stdio=_install_safe_stdio,
- sanitize_surrogates=_sanitize_surrogates,
- summarize_user_message_for_log=_summarize_user_message_for_log,
- set_session_context=set_session_context,
- set_current_write_origin=set_current_write_origin,
- ra=_ra,
- # MoA turns append per-call aggregated context to the API copy of the
- # user message, so no byte-stable api_content sidecar can be stamped.
- moa_active=bool(moa_config),
- title_user_message=title_user_message,
- )
- except PreflightCompressionTimedOut as _preflight_timeout_exc:
- return _preflight_timeout_result(agent, _preflight_timeout_exc, conversation_history)
- # Voice turns may run on auxiliary.voice_chat: bound after the prompt/row/compaction were settled
- # against the main model, undone in finalize_turn (and run_conversation's finally on early exits).
- from agent.voice_turn_route import begin_voice_turn_route
- _ctx.active_system_prompt = begin_voice_turn_route(agent, _ctx.messages, _ctx.active_system_prompt)
- # Per-turn agent state (the gateway caches agents across turns, so none of this may
- # leak into the next message): interim-commentary dedup spans the whole turn but not
- # the next; a SessionDB append failure (and its classified cause) halts only this turn;
- # a failed compression-tip adoption is reported only against its own turn; the
- # thinking-only-truncation one-shot must not survive an interrupted turn; credential-
- # pool refresh tallies cap same-entry refreshes on a persistent 401 (#26080); usage
- # for on_turn_complete() stays None on turns that never reach a response.
- agent._delivered_interim_texts = set()
- agent._incremental_persistence_failed = False
- agent._last_persistence_error_cause = None
- agent._compression_adoption_failed = False
- agent._ephemeral_reasoning_off = False
- agent._auth_pool_refresh_counts = {}
- agent._last_turn_usage = None
- s = _LoopState(
- system_message=system_message, moa_config=moa_config,
- max_compression_attempts=getattr(agent, "max_compression_attempts", 3),
- **{f.name: getattr(_ctx, f.name.lstrip("_")) for f in fields(_LoopState) if f.name in _CTX_FIELDS},
- )
- if (early_result := _turn_without_loop(agent, s, _ctx.plugin_block_message)) is not None:
- return early_result
- _prelude_action, _prelude_result = play_prelude(agent, s, prelude)
- if _prelude_action == "return":
- return _prelude_result
- while _prelude_action != "break" and (
- (s.api_call_count < agent.max_iterations and agent.iteration_budget.remaining > 0) or agent._budget_grace_call
- ):
- if _run_phase(begin_iteration, agent, s).action == "break":
- break
- _run_phase(prepare_iteration, agent, s)
- _run_phase(assemble_api_request, agent, s)
- _pg = _run_phase(run_preflight_gate, agent, s)
- if _pg.action == "return":
- return _pg.result
- if _pg.action == "break":
- break
- if _pg.action == "continue":
- continue
- _run_phase(announce_api_call, agent, s)
- s.api_start_time, s.retry_count, s.max_retries = time.time(), 0, agent._api_max_retries
- s._retry, s.finish_reason, s.response, s.api_kwargs = TurnRetryState(), "stop", None, None
- s.api_request_id = agent._current_api_request_id = f"{s.turn_id}:api:{s.api_call_count}"
- early_result = _run_api_retry_loop(agent, s)
- if early_result is not None:
- return early_result
- _rs = _run_phase(apply_retry_restarts, agent, s)
- if _rs.action == "break":
- break
- if _rs.action == "continue":
- continue
- try:
- _ri = _run_phase(normalize_model_response, agent, s)
- if _ri.action == "return":
- return _ri.result
- if _ri.action == "continue":
- continue
- _v = _run_phase(
- run_tool_round if s.assistant_message.tool_calls else finish_text_response, agent, s
- )
- if _v.action == "return":
- return _v.result
- if _v.action == "redirect":
- s._retry.restart_with_redirected_messages = True
- if _run_phase(apply_retry_restarts, agent, s).action == "break":
- break
- continue
- if _v.action == "break":
- break
- if _v.action == "continue":
- continue
- except Exception as e:
- if _run_phase(handle_outer_loop_error, agent, s, e=e).action == "break":
- break
- result = _finalize_from_state(agent, s)
- if s._compression_timeout_exhausted:
- # Reuse the gateway's context-recovery contract: transcript stays intact while
- # future input can move to a clean session (#98722).
- result.update(error=_COMPRESSION_TIMEOUT_FINAL_RESPONSE, partial=True, compression_exhausted=True)
- return result
- def run_conversation(
- agent,
- user_message: Any,
- system_message: str | None = None,
- conversation_history: list[dict[str, Any]] | None = None,
- task_id: str | None = None,
- stream_callback: Optional[callable] = None,
- persist_user_message: Optional[Any] = None,
- persist_user_timestamp: Optional[float] = None,
- persist_user_display_kind: Optional[str] = None,
- persist_user_display_metadata: Optional[dict[str, Any]] = None,
- persist_user_platform_id: Optional[str] = None,
- moa_config: Optional[dict[str, Any]] = None,
- turn_author: Optional[dict[str, Any]] = None,
- title_user_message: Optional[str] = None,
- prelude: Optional[Prelude] = None,
- ) -> dict[str, Any]:
- """Run one turn (see ``_run_conversation_turn``) and export the current-turn boundary.
- Every envelope that leaves the loop — success, partial/error, interrupt, retry-exhausted,
- tool-limit, preflight timeout, codex runtime — passes through here, so the
- ``{turn_id, current_turn_user_idx}`` pair is stamped beside the exact ``messages`` it
- addresses, after every history rewrite including post-turn micro-compaction.
- """
- from agent.turn_context import export_current_turn_boundary
- from agent.turn_finalizer import settle_held_final
- from agent.voice_turn_route import end_voice_turn_route
- from tools.vision_tools_history_budget import native_turn_images
- # Steer / redirect / interrupt set this mid-turn; the result reports whether the turn ran untouched.
- agent._turn_user_intervened = bool(
- getattr(agent, "_turn_redirect_open", False) and agent._has_pending_redirect()
- )
- # Images attached natively to this user turn stay visible to vision_analyze for the turn, so
- # it does not embed the same pixels a second time into the same request (#76411).
- with native_turn_images(user_message):
- try:
- result = _run_conversation_turn(
- agent,
- user_message,
- system_message=system_message,
- conversation_history=conversation_history,
- task_id=task_id,
- stream_callback=stream_callback,
- persist_user_message=persist_user_message,
- persist_user_timestamp=persist_user_timestamp,
- persist_user_display_kind=persist_user_display_kind,
- persist_user_display_metadata=persist_user_display_metadata,
- persist_user_platform_id=persist_user_platform_id,
- moa_config=moa_config,
- turn_author=turn_author,
- title_user_message=title_user_message,
- prelude=prelude,
- )
- finally:
- # Early returns and failures also end the worker's opt-in admission.
- from contextlib import nullcontext
- with getattr(agent, "_pending_redirect_lock", None) or nullcontext():
- agent._turn_redirect_open = False
- end_voice_turn_route(agent)
- settle_held_final(agent, result)
- result = export_current_turn_boundary(agent, result, user_message)
- if isinstance(result, dict):
- result["user_intervened"] = bool(getattr(agent, "_turn_user_intervened", False))
- _close_durable_failed_turn(agent, result)
- return result
- _FAILED_TURN_ERROR_MAX_CHARS = 2000
- def _failed_turn_display_metadata(agent, result: dict) -> dict:
- """The error text and ``error_surface`` a client needs to redraw the failed turn's error
- card from the transcript, after the live ``message.complete`` frame is gone. Every
- string is redacted with ``force=True``: the error came from a provider/tool, so a
- secret echoed in it must not reach the durable store (the redaction e2e boundary)."""
- from agent.error_surface import build_error_surface_from_result
- try:
- surface = build_error_surface_from_result(
- result, provider=agent.provider or "", model=agent.model or ""
- )
- except Exception:
- logger.debug("failed-turn error surface unavailable", exc_info=True)
- surface = None
- error = str(result.get("error") or "").strip()[:_FAILED_TURN_ERROR_MAX_CHARS]
- metadata = {k: v for k, v in (("error", error), ("error_surface", surface)) if v}
- return _redact_display_metadata(metadata)
- def _redact_display_metadata(metadata: dict) -> dict:
- """Force-redact every string in a display_metadata payload (dicts and lists included).
- display_metadata is persisted via ``SessionDB.append_message`` and re-delivered to
- clients, so it sits downstream of the turn's own content redaction: an error string
- that escaped a provider or tool would otherwise reach the 'store' and 'export' sinks
- verbatim. ``force=True`` keeps the boundary closed even when ``security.redact_secrets``
- is off, matching the compressor's persistence boundary."""
- from agent.redact import redact_sensitive_text
- def _redact(value):
- if isinstance(value, str):
- return redact_sensitive_text(value, force=True)
- if isinstance(value, dict):
- return {k: _redact(v) for k, v in value.items()}
- if isinstance(value, list):
- return [_redact(v) for v in value]
- return value
- return {k: _redact(v) for k, v in metadata.items()}
- def _close_durable_failed_turn(agent, result: Any) -> None:
- """Append a Hermes-authored assistant boundary when a failed turn left its durable tail open
- (``OPEN_TURN_TAIL_ROLES``: the accepted ``user`` row, or a ``tool`` row when a call ran before
- the failure), in place on ``result["messages"]`` and in SessionDB.
- The terminal-failure paths (content-policy refusal, ``_Trunc.end_turn``, retry exhaustion,
- interrupt before any assistant text) persist the accepted user row and return without
- reaching ``finalize_turn``; the next prompt then appends a second user row and
- ``repair_message_sequence`` merges the failed request into the new one. The gateway
- compensates with ``_hmwa_close_failed_turn``; CLI, TUI/Desktop and ACP hosts hand
- ``result["messages"]`` straight back as history, so the seam is here.
- Excluded: the context-pressure classes (``compression_exhausted``, ``compression_deferred``,
- ``failure_reason == "context_overflow"``) — appending to an already-oversized session is the
- #1630 growth loop; their repair is rotation or a retry. Idempotence is keyed on the DURABLE
- tail (``SessionDB.latest_conversation_role``), so a redelivery or a tail already closed by
- another writer is a no-op, and the gateway's own closer then no-ops in turn.
- """
- try:
- if not isinstance(result, dict) or result.get("completed") is True:
- return
- if (
- result.get("compression_exhausted") or result.get("compression_deferred")
- or result.get("failure_reason") == "context_overflow"
- ):
- return
- messages = result.get("messages")
- db, session_id = getattr(agent, "_session_db", None), getattr(agent, "session_id", None)
- if not isinstance(messages, list) or not messages or db is None or not session_id:
- return
- if getattr(agent, "_persist_disabled", False) or db.latest_conversation_role(session_id) not in OPEN_TURN_TAIL_ROLES:
- return
- # Scope the "did a tool run" scan to this turn when its boundary is proven; otherwise
- # hedge over the whole list rather than under-report a possible side effect.
- start = result.get("current_turn_user_idx")
- turn_messages = messages[start:] if isinstance(start, int) and 0 <= start < len(messages) else messages
- boundary = {
- "role": "assistant", "content": failed_turn_notice(turn_messages), "display_kind": FAILED_TURN_DISPLAY_KIND,
- }
- if failure := _failed_turn_display_metadata(agent, result):
- boundary["display_metadata"] = failure
- append_message(messages, boundary)
- agent._flush_messages_to_session_db(messages)
- except Exception:
- logger.debug("failed-turn boundary not written", exc_info=True)
- __all__ = ["run_conversation"]
|