| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435 |
- #!/usr/bin/env python3
- """File Tools Module - LLM agent file manipulation tools.
- Companions: ``file_tools_paths`` (task-aware resolution), ``file_tools_write_guards``
- (write-side guards), ``file_tools_read_tracking`` (per-task dedup / loop-detection /
- staleness state).
- """
- import base64
- import errno
- import json
- import logging
- import os
- import re
- import stat
- import threading
- import time
- from contextlib import ExitStack
- from pathlib import Path
- from agent.file_safety import get_nt_namespace_error, get_read_block_error
- from agent.tool_result_classification import GUARDRAIL_REFUSAL_KEY
- from tools.binary_extensions import has_binary_extension
- from tools.skill_provenance import is_background_review
- from tools.file_operations import (
- ShellFileOperations, normalize_read_pagination, normalize_search_pagination)
- from tools.file_operations_common import DEFAULT_READ_LIMIT, count_conflict_blocks
- from tools import file_state
- from hermes_platform.host import facts
- from agent.redact import _is_secret_file_arg, redact_sensitive_text
- from tools.file_tools_paths import (
- _expand_tilde, _path_resolution_warning, _resolve_base_dir, _resolve_entry_for_task,
- _resolve_path_for_task)
- from tools.file_tools_write_guards import (
- _READ_DEDUP_STATUS_MESSAGE, _check_approval_required_write, _check_binary_document_write,
- _check_cross_profile_path, _check_protected_instruction_write, _check_sensitive_path,
- _is_internal_file_tool_content, _stale_overwrite_blocker, _stale_write_refusal)
- from tools.file_tools_read_tracking import (
- _bump_consecutive, _cap_read_tracker_data, _carry_full_write_baselines, _check_file_staleness,
- _check_not_found_cache, _known_full_content_sha256,
- _file_metadata, _file_version,
- _mark_full_write_baseline, _mark_verification_stale, _note_read_coverage, _patch_failure_lock,
- _patch_failure_tracker, _read_tracker, _read_tracker_lock, _record_not_found,
- _record_patch_failure, _reset_patch_failures, _task_data, _update_read_timestamp)
- logger = logging.getLogger(__name__)
- _EXPECTED_WRITE_ERRNOS = {errno.EACCES, errno.EPERM, errno.EROFS}
- # Read-size guard. Model-agnostic, so characters proxy tokens: 100K chars is
- # ~25-35K tokens across typical tokenisers. Configurable: file_read_max_chars.
- _DEFAULT_MAX_READ_CHARS = 100_000
- def _get_max_read_chars() -> int:
- """Return ``file_read_max_chars`` from config.yaml (default on missing/invalid). No module
- cache: ``load_config_readonly`` is already mtime+path cached, and a process-lifetime slot
- would pin the launch profile's value under the multiplexed gateway."""
- try:
- from hermes_cli.config import load_config_readonly
- val = load_config_readonly().get("file_read_max_chars")
- except Exception:
- val = None
- valid = isinstance(val, (int, float)) and val > 0
- return int(val) if valid else _DEFAULT_MAX_READ_CHARS
- def _truncate_to_char_budget(content: str, max_chars: int) -> tuple[str, int, bool]:
- """Trim line-numbered ``read_file`` content to the last COMPLETE line within *max_chars*.
- Returns ``(kept_text, lines_kept, truncated)`` so the caller can offer a
- ``next_offset`` instead of rejecting the read. If not even the first line
- fits it is clamped mid-line so the read is never empty and the cursor advances.
- Ported in spirit from nearai/ironclaw#5029 (dual line/byte cap on ``read_file``). Where hermes
- previously hard-rejected an oversized read (forcing the model to guess a smaller ``limit`` and burn a
- round-trip returning nothing), this trims the content to the last *complete line* that fits within
- ``max_chars`` and reports how many lines were kept so the caller can offer a ``next_offset``
- continuation.
- """
- if len(content) <= max_chars:
- return content, (content.count("\n") + 1 if content else 0), False
- lines = content.split("\n")
- kept: list[str] = []
- running = 0
- for line in lines:
- addition = len(line) + (1 if kept else 0) # +1 for the rejoining "\n"
- if running + addition > max_chars:
- break
- kept.append(line)
- running += addition
- if not kept:
- kept.append(lines[0][:max_chars])
- return "\n".join(kept), len(kept), True
- def _apply_char_budget(result_dict: dict, content: str, offset: int, total_lines, max_chars: int) -> str:
- """Trim *content* to the char budget, annotate *result_dict* with the
- continuation hint, and return the trimmed text."""
- trimmed, lines_kept, _ = _truncate_to_char_budget(content, max_chars)
- next_offset = offset + lines_kept
- result_dict["content"] = trimmed
- result_dict["truncated"] = True
- result_dict["truncated_by"] = "bytes"
- result_dict["next_offset"] = next_offset
- result_dict["hint"] = (
- f"Output truncated at the {max_chars:,}-char read budget after "
- f"{lines_kept} line(s) (showing lines {offset}-{next_offset - 1} of "
- f"{total_lines}). Use offset={next_offset} to continue.")
- if len(trimmed.split("\n", 1)[0]) >= max_chars:
- result_dict["truncated_lines"] = True
- result_dict["hint"] += (
- " Note: the first line alone exceeded the budget and was "
- "clamped mid-line; its remainder is not retrievable via offset.")
- return trimmed
- # Above this size, a wide read (limit > 200) gets a hint toward targeted reads.
- _LARGE_FILE_HINT_BYTES = 512_000
- # Device/fd paths whose reads hang the process. Checked by path only — no I/O.
- _BLOCKED_DEVICE_PATHS = frozenset({
- "/dev/zero", "/dev/random", "/dev/urandom", "/dev/full", # never reach EOF
- "/dev/stdin", "/dev/tty", "/dev/console", # block on input
- "/dev/stdout", "/dev/stderr", # nonsensical to read
- "/dev/fd/0", "/dev/fd/1", "/dev/fd/2", # fd aliases
- })
- # /proc/<pid>/... (and /proc/<pid>/task/<tid>/...) files that leak secrets,
- # argv, memory layout (ASLR oracle: maps family, auxv, pagemap) or raw memory.
- _BLOCKED_PROC_SUFFIXES = (
- "/fd/0", "/fd/1", "/fd/2", # stdio aliases
- "/environ", "/cmdline", "/maps", "/smaps", "/smaps_rollup", "/numa_maps",
- "/mem", "/auxv", "/pagemap")
- def _file_ops_uses_host_paths(file_ops) -> bool:
- """True when *file_ops* targets the host filesystem (only then may we stat paths
- or rewrite V4A headers to host-absolute paths; sandboxes have their own namespace)."""
- env = getattr(file_ops, "env", None)
- if env is None:
- return True
- from tools.environments.base import env_uses_host_filesystem
- return env_uses_host_filesystem(env)
- # V4A file headers: group 1 = header prefix, 2 = op, 3 = path. ``\s*`` after
- # ``***`` mirrors patch_parser's leniency (``***Update File:`` applies, so it
- # must be checked).
- _V4A_SINGLE_HEADER_RE = re.compile(r'^(\*\*\*\s*(Update|Add|Delete)\s+File:\s*)(.+)$', re.MULTILINE)
- _V4A_MOVE_HEADER_RE = re.compile(r'^(\*\*\*\s*Move\s+File:\s*)(.+?)\s*->\s*(.+)$', re.MULTILINE)
- def _rewrite_v4a_patch_paths_for_host(patch: str, path_to_resolved: dict, path_to_entry: dict,
- file_ops) -> str:
- """Rewrite V4A file headers to the resolved host paths (host backends only).
- The shell layer must patch the SAME files ``patch_tool`` resolved for
- locking/staleness, not re-resolve a relative header against its own cwd
- (which can differ — the git-worktree cwd bug). Delete and Move headers take
- ``path_to_entry``: they act on a symlink itself, never on its target.
- """
- if not _file_ops_uses_host_paths(file_ops):
- return patch
- def _res(raw: str, entry: bool = False) -> str:
- raw = raw.strip()
- return (path_to_entry if entry else path_to_resolved).get(raw) or raw
- patch = _V4A_SINGLE_HEADER_RE.sub(
- lambda m: f"{m.group(1)}{_res(m.group(3), entry=m.group(2) == 'Delete')}", patch)
- return _V4A_MOVE_HEADER_RE.sub(
- lambda m: f"{m.group(1)}{_res(m.group(2), entry=True)} -> {_res(m.group(3), entry=True)}", patch)
- def _is_blocked_device_path(path: str) -> bool:
- """Return True for concrete device/fd/proc paths that can hang reads or leak process state."""
- normalized = os.path.normpath(_expand_tilde(path))
- if normalized in _BLOCKED_DEVICE_PATHS:
- return True
- return normalized.startswith("/proc/") and normalized.endswith(_BLOCKED_PROC_SUFFIXES)
- def _is_blocked_device(filepath: str, base_dir: str | Path | None = None) -> bool:
- """True if the path (literal, any symlink hop, or final realpath) is a blocked device.
- Literal first so /dev/stdin is caught before resolving to a terminal path;
- every symlink hop is checked so an alias cannot bypass the guard.
- """
- expanded = _expand_tilde(filepath)
- if base_dir is not None and not os.path.isabs(expanded):
- expanded = os.path.join(os.fspath(base_dir), expanded)
- normalized = os.path.normpath(expanded)
- if _is_blocked_device_path(normalized):
- return True
- seen: set[str] = set()
- current = normalized
- for _ in range(20):
- try:
- target = os.readlink(current)
- except OSError:
- break
- if not os.path.isabs(target):
- target = os.path.join(os.path.dirname(current), target)
- target = os.path.normpath(target)
- if _is_blocked_device_path(target):
- return True
- if target in seen:
- break
- seen.add(target)
- current = target
- try:
- resolved = os.path.normpath(os.path.realpath(normalized))
- except (OSError, ValueError):
- return False
- return _is_blocked_device_path(resolved)
- def _resolved_match_path(path: str, task_id: str) -> str:
- """Best-effort task-cwd resolution of a search hit's path.
- Search backends may return cwd-relative paths while the process cwd differs, so both the
- read-block filter and the redaction classifier must resolve against the task cwd. An
- unresolvable path is used as-is (the raw path is still worth classifying).
- """
- try:
- return str(_resolve_path_for_task(path, task_id))
- except (OSError, ValueError, RuntimeError):
- return path
- def _filter_read_blocked_search_results(result, task_id: str = "default") -> int:
- """Remove credential/cache/env paths from a SearchResult in-place; return the omitted count.
- Each path is resolved against the task cwd first (search backends may
- return cwd-relative paths; the process cwd can differ).
- """
- omitted = 0
- def _allowed(path: str) -> bool:
- nonlocal omitted
- if get_read_block_error(_resolved_match_path(path, task_id)):
- omitted += 1
- return False
- return True
- if getattr(result, "matches", None):
- result.matches = [m for m in result.matches if _allowed(m.path)]
- if getattr(result, "files", None):
- result.files = [f for f in result.files if _allowed(f)]
- if getattr(result, "counts", None):
- result.counts = {f: c for f, c in result.counts.items() if _allowed(f)}
- return omitted
- def _is_expected_write_exception(exc: Exception) -> bool:
- """Return True for expected write denials that should not hit error logs."""
- return isinstance(exc, PermissionError) or (
- isinstance(exc, OSError) and exc.errno in _EXPECTED_WRITE_ERRNOS)
- # ── ShellFileOperations per terminal environment ─────────────────────────
- _file_ops_lock = threading.Lock()
- _file_ops_cache: dict = {}
- def _create_terminal_env_for_file_ops(raw_task_id: str, task_id: str):
- """Build the terminal environment for *task_id* via the shared ``_create_configured_env``,
- so a file tool that runs before any terminal command still gets the configured backend."""
- from tools.terminal_tool_config import _is_container_backend, coerce_ssh_remote_cwd
- from tools.terminal_tool import (
- _create_configured_env, _get_env_config, _is_mounted_host_cwd, _is_unusable_container_cwd,
- _resolve_task_host_cwd, _select_image, get_session_cwd, resolve_task_overrides)
- config = _get_env_config()
- env_type = config["env_type"]
- overrides = resolve_task_overrides(raw_task_id)
- try:
- recorded_cwd = get_session_cwd(raw_task_id)
- except Exception:
- recorded_cwd = None
- cwd = coerce_ssh_remote_cwd(overrides.get("cwd") or recorded_cwd or config["cwd"], env_type)
- # Re-apply the container cwd guard: a gateway/TUI/ACP override is a raw HOST
- # path and ``docker run -w <host-path>`` makes search_files & co silently
- # return nothing. Valid in-container overrides (/workspace, /root) pass.
- # Re-apply the container cwd guard that _get_env_config() already ran on config["cwd"] (see #50636). A
- # per-task cwd override registered by the gateway/TUI/ACP for workspace tracking is a raw host path
- # (e.g. a Desktop session's /Users/<me>/workspace or C:\\Users\\<me>). On a container backend that
- # reaches ``docker run -w <host-path>`` and the container starts in a directory that doesn't exist
- # inside the sandbox, so search_files and friends silently return empty results (#54447). Sanitize it
- # back to the already-validated config["cwd"] so the override can't bypass the guard.
- host_cwd = _resolve_task_host_cwd(config, raw_task_id)
- if _is_container_backend(env_type) and _is_unusable_container_cwd(cwd, mounted_host=host_cwd):
- fallback = "/workspace" if _is_mounted_host_cwd(cwd, host_cwd) else config["cwd"]
- if cwd != fallback:
- logger.info(
- "Ignoring host/relative cwd override %r for %s backend "
- "(won't exist in sandbox). Using %r instead.",
- cwd, env_type, fallback)
- cwd = fallback
- logger.info("Creating new %s environment for task %s...", env_type, task_id[:8])
- terminal_env = _create_configured_env(
- config, env_type, image=_select_image(env_type, overrides, config), cwd=cwd,
- timeout=config["timeout"], task_id=task_id,
- host_cwd=host_cwd,
- )
- return env_type, terminal_env
- def _get_file_ops(task_id: str = "default") -> ShellFileOperations:
- """Get or create ShellFileOperations for the task's terminal environment.
- Uses terminal_tool's per-task creation locks (no duplicate sandboxes).
- Subagent task_ids collapse to "default" (``_resolve_container_task_id``) so
- delegate_task children share the parent's container; RL/benchmark task_ids
- with a registered env override keep their isolation.
- """
- from tools.terminal_tool import (
- _active_environments, _env_lock, _last_activity, _start_cleanup_thread,
- _creation_locks, _creation_locks_lock, _resolve_container_task_id,
- get_session_cwd, record_session_cwd)
- raw_task_id = task_id or "default"
- task_id = _resolve_container_task_id(raw_task_id)
- # Fast path: cached AND the environment is still alive (cleanup thread may have killed it).
- with _file_ops_lock:
- cached = _file_ops_cache.get(task_id)
- if cached is not None:
- with _env_lock:
- if task_id in _active_environments:
- _last_activity[task_id] = time.time()
- return cached
- # Env was cleaned up: rescue its cwd into the session record FILL-ONLY
- # (``cached.cwd`` is the SHARED env's cwd, not this session's own).
- # Environment was cleaned up -- preserve the old cwd in the session record before invalidating
- # the stale cache entry (fixes #26211: silent file-creation failures in long-running
- # conversations). Usually a no-op: every completed command already recorded its cwd. Fill-only:
- # ``cached.cwd`` is a snapshot of the SHARED env's cwd at cache-build time, so it is not
- # attributable to this session (same class as the interrupted-command bug, #85658). Rescue a
- # session that has no record, but never overwrite a record the session wrote for itself.
- old_cwd = getattr(cached, "cwd", None)
- if old_cwd:
- try:
- if get_session_cwd(raw_task_id) is None:
- record_session_cwd(raw_task_id, old_cwd)
- except Exception:
- pass
- with _file_ops_lock:
- _file_ops_cache.pop(task_id, None)
- with _creation_locks_lock:
- task_lock = _creation_locks.setdefault(task_id, threading.Lock())
- with task_lock:
- # Double-check: another thread may have created it while we waited.
- with _env_lock:
- terminal_env = _active_environments.get(task_id)
- if terminal_env is not None:
- _last_activity[task_id] = time.time()
- if terminal_env is None:
- env_type, terminal_env = _create_terminal_env_for_file_ops(raw_task_id, task_id)
- with _env_lock:
- _active_environments[task_id] = terminal_env
- _last_activity[task_id] = time.time()
- _start_cleanup_thread()
- logger.info("%s environment ready for task %s", env_type, task_id[:8])
- file_ops = ShellFileOperations(terminal_env)
- with _file_ops_lock:
- _file_ops_cache[task_id] = file_ops
- return file_ops
- def clear_file_ops_cache(task_id: str | None = None):
- """Clear file-operation state for a finished task, or all tasks."""
- with _file_ops_lock:
- if task_id:
- _file_ops_cache.pop(task_id, None)
- else:
- _file_ops_cache.clear()
- with _read_tracker_lock:
- if task_id:
- _read_tracker.pop(task_id, None)
- else:
- _read_tracker.clear()
- with _patch_failure_lock:
- if task_id:
- _patch_failure_tracker.pop(task_id, None)
- else:
- _patch_failure_tracker.clear()
- if task_id:
- file_state.get_registry().forget_task(task_id)
- else:
- file_state.get_registry().clear()
- _SPECIAL_FILE_KINDS = (
- (stat.S_ISFIFO, "a FIFO (named pipe)"),
- (stat.S_ISSOCK, "a socket"),
- (stat.S_ISCHR, "a character device"),
- (stat.S_ISBLK, "a block device"))
- def _special_file_kind(path) -> str | None:
- """Human name for a non-regular file type that would hang a read, else None.
- Stat-based sibling of ``_is_blocked_device``: a FIFO/socket in a workspace
- hangs like ``/dev/zero`` but has no recognizable name. Host filesystems
- only; unstattable paths return None and flow to the normal read path.
- """
- try:
- mode = os.stat(os.fspath(path)).st_mode # follows symlinks, matching a real read
- except OSError:
- return None
- if stat.S_ISREG(mode) or stat.S_ISDIR(mode):
- return None
- return next((label for predicate, label in _SPECIAL_FILE_KINDS if predicate(mode)),
- "a special (non-regular) file")
- def _read_extracted_document(path: str, _resolved, offset: int, limit: int, task_id: str) -> str | None:
- """Render an extractable document (.docx/.xlsx/.pdf/...) as paginated text.
- Returns the JSON result, a tool_error for an actionable extraction failure
- (size cap, encrypted, malformed), or ``None`` to fall through to the normal
- read path. Runs BEFORE the binary-extension guard.
- """
- from tools.read_extract import (
- ANYDOC_EXTENSIONS, EXTRACTABLE_EXTENSIONS, MAX_DOCUMENT_BYTES, ExtractionError,
- extract_document_bytes, is_extractable_document)
- if not is_extractable_document(str(_resolved)):
- return None
- file_ops = _get_file_ops(task_id)
- try:
- binary = file_ops.read_file_bytes(str(_resolved), max_bytes=MAX_DOCUMENT_BYTES)
- if binary.error or binary.base64_content is None:
- raise ExtractionError(binary.error or "Document bytes unavailable")
- document_bytes = base64.b64decode(binary.base64_content, validate=True)
- extracted_text = extract_document_bytes(document_bytes, str(_resolved))
- except (ExtractionError, ValueError, base64.binascii.Error) as exc:
- logger.debug("document extraction failed for %s", path, exc_info=True)
- # Binary formats surface the specific failure (fallthrough would only
- # give a generic binary error); .ipynb and byte-transport errors fall through.
- _doc_ext = _resolved.suffix.lower()
- _binary_doc = _doc_ext in ANYDOC_EXTENSIONS or (
- _doc_ext in EXTRACTABLE_EXTENSIONS and _doc_ext != ".ipynb")
- if (_binary_doc and isinstance(exc, ExtractionError)
- and not str(exc).startswith("Unsupported document type")):
- return tool_error(
- f"Cannot read '{path}' ({_doc_ext}): document "
- f"extraction failed — {exc}. Use terminal utilities "
- "to inspect or convert the file.")
- return None
- lines = extracted_text.splitlines()
- total_lines = len(lines)
- end_line = offset + limit - 1
- page_text = "\n".join(lines[offset - 1:end_line])
- from tools.tool_output_limits import get_max_line_length
- max_line_length = get_max_line_length()
- truncated_lines = any(len(line) > max_line_length for line in page_text.split('\n'))
- result_dict = {
- "content": file_ops._add_line_numbers(page_text, offset) if page_text else "",
- "total_lines": total_lines,
- "file_size": binary.file_size,
- "truncated": total_lines > end_line,
- "extracted_document": True}
- if result_dict["truncated"]:
- result_dict["hint"] = (
- f"Use offset={end_line + 1} to continue reading "
- f"(showing {offset}-{min(end_line, total_lines)} of {total_lines} lines)")
- max_chars = _get_max_read_chars()
- if len(result_dict["content"]) > max_chars:
- _apply_char_budget(result_dict, result_dict["content"], offset, total_lines, max_chars)
- if result_dict["content"]:
- rendered = result_dict["content"]
- result_dict["content"] = redact_sensitive_text(
- rendered, file_read=True,
- secret_file=_is_secret_file_arg(str(_resolved)))
- redacted = result_dict["content"] != rendered
- else:
- redacted = False
- if (offset == 1 and not result_dict["truncated"] and not redacted
- and not truncated_lines and not result_dict.get("truncated_lines")):
- # The whole document was shown, so a text-authorable format (.ipynb)
- # may later be overwritten by write_file; the binary-container guard
- # keeps refusing .docx/.xlsx/.pdf regardless of this baseline.
- _mark_full_write_baseline(str(_resolved), task_id)
- _update_read_timestamp(str(_resolved), task_id)
- file_state.record_read(task_id, str(_resolved))
- return json.dumps(result_dict, ensure_ascii=False)
- def _dedup_stub_or_block(task_data: dict, dedup_key: tuple, path: str) -> str:
- """Return the "unchanged" stub for a repeated identical read, escalating to a
- hard BLOCK after 2 stubs so weak tool-followers don't loop forever."""
- with _read_tracker_lock:
- hits = task_data["dedup_hits"].get(dedup_key, 0) + 1
- task_data["dedup_hits"][dedup_key] = hits
- _cap_read_tracker_data(task_data)
- if hits >= 2:
- return tool_error(
- f"BLOCKED: You have called read_file on this "
- f"exact region {hits + 1} times and the file "
- "has NOT changed. STOP calling read_file for "
- "this path — the content from your earlier "
- "read_file result in this conversation is "
- "still current. Proceed with your task using "
- "the information you already have.",
- path=path,
- already_read=hits + 1,
- # A REFUSAL the harness chose, not a failure the tool hit: without the
- # marker the failure classifiers count the block and a repeated read
- # escalates to `repeated_exact_failure_block` over calls that never failed.
- **{GUARDRAIL_REFUSAL_KEY: True})
- return json.dumps({
- "status": "unchanged",
- "message": _READ_DEDUP_STATUS_MESSAGE,
- "path": path,
- "dedup": True,
- "content_returned": False,
- }, ensure_ascii=False)
- def _record_successful_read(task_data: dict, task_id: str, path: str, resolved_str: str,
- offset: int, limit: int, dedup_key: tuple, *, partial: bool,
- redacted: bool = False, end_line: int | None = None,
- total_lines=None, version_before=None, snapshot=None) -> int:
- """Bookkeeping after a real (non-stub) read; returns the consecutive-read count.
- Per-task tracker under the lock (stub counter, history, consecutive count,
- mtime for dedup + staleness, page coverage, and the write_file baseline once
- the task has seen every line UNREDACTED — in one page or by paging
- contiguously through a file too big for one; a redacted page returned a
- non-round-trippable ``«redacted:…»`` sentinel, so it must not bless an
- overwrite that would persist the sentinel into a credential file). Then
- OUTSIDE our lock (no nested locking): the cross-agent registry, and the
- background-review read-mark (a FULL read of a skill file counts like
- skill_view so a follow-up skill_manage(patch) is accepted).
- """
- version = (snapshot or _file_version(resolved_str)) if version_before is not None else None
- stable = version is not None and version[:-1] == version_before == _file_metadata(resolved_str)
- complete = False
- with _read_tracker_lock:
- task_data["dedup_hits"].pop(dedup_key, None)
- task_data["dedup_generation_reads"].add(dedup_key)
- task_data["read_history"].add((path, offset, limit))
- count = _bump_consecutive(task_data, ("read", path, offset, limit))
- try:
- _mtime_now = os.path.getmtime(resolved_str)
- task_data.setdefault("read_timestamps", {})[resolved_str] = _mtime_now
- except OSError:
- pass
- baselines = task_data["full_write_baselines"]
- if stable and version is not None and count < 4:
- task_data["dedup"][dedup_key] = version_before
- # A narrower view does not undo knowledge of these same bytes. Do
- # not revive a baseline after a partial read of a different version.
- complete = baselines.get(resolved_str) == version
- if not complete:
- complete = not partial
- if partial and end_line is not None:
- complete, redacted = _note_read_coverage(
- task_data, resolved_str, version, offset, end_line, total_lines, redacted)
- complete = complete and not redacted
- if complete:
- baselines[resolved_str] = version
- task_data.setdefault("blind_patches", {}).pop(resolved_str, None)
- if not complete:
- baselines.pop(resolved_str, None)
- if not stable or count >= 4:
- task_data["dedup"].pop(dedup_key, None)
- task_data["dedup_generation_reads"].discard(dedup_key)
- _cap_read_tracker_data(task_data)
- try:
- file_state.record_read(task_id, resolved_str, partial=not complete)
- except Exception:
- logger.debug("file_state.record_read failed", exc_info=True)
- if complete:
- try:
- # Background-review read-before-write guard integration (#61521): when the self-improvement
- # review fork reads a skill file with read_file (now whitelisted dispatch-side), register the
- # read the same way skill_view does, so a follow-up skill_manage(action='patch') on the loaded
- # file is accepted. A partial read doesn't count — the guard requires the CURRENT full content
- # to have been seen. No-op outside review forks (mark_background_review_skill_read gates on
- # is_background_review).
- from tools.skill_manager_guards import mark_background_review_skill_read
- mark_background_review_skill_read(Path(resolved_str))
- except Exception:
- logger.debug("background-review read-mark failed", exc_info=True)
- return count
- def read_file_tool(path: str, offset: int = 1, limit: int = DEFAULT_READ_LIMIT, task_id: str = "default") -> str:
- """Read a file with pagination and line numbers.
- Guard order: NT/device-namespace prefix (raw string, no resolution) →
- device-path blocklist (no I/O) → stat-based special-file guard (host only)
- → Hermes internal denylist → document extraction → binary-extension guard
- → negative-result cache → dedup stub → real read.
- """
- try:
- offset, limit = normalize_read_pagination(offset, limit)
- # On the RAW model-supplied string, before any expanduser()/resolve():
- # on Windows resolving \??\UNC\host\share already sends SMB auth (NTLM
- # leak); on POSIX the task-base join would anchor the prefix as a
- # relative segment and hide it from every resolved-path check below.
- nt_err = get_nt_namespace_error(path, verb="Read")
- if nt_err:
- return tool_error(nt_err)
- device_base = None if Path(path).expanduser().is_absolute() else _resolve_base_dir(task_id)
- if _is_blocked_device(path, base_dir=device_base):
- return tool_error(
- f"Cannot read '{path}': this is a device file that would "
- "block or produce infinite output.")
- _resolved = _resolve_path_for_task(path, task_id)
- # A read on a FIFO/socket blocks until the exec timeout: a self-shipped DoS.
- if _file_ops_uses_host_paths(_get_file_ops(task_id)):
- kind = _special_file_kind(_resolved)
- if kind is not None:
- return json.dumps({
- "success": False,
- "note": (
- f"'{path}' is {kind}, not a regular file — reading "
- "it would block indefinitely, so no read was "
- "attempted. Use terminal utilities if you need to "
- "interact with it.")})
- # Hermes internal denylist (prompt injection via catalog metadata,
- # credential stores). Runs BEFORE document extraction so a
- # protected SQLite store (state.db) cannot be read through the extractor. Pass the RESOLVED path: the denylist's own
- # resolve() uses the process cwd and would miss a relative "auth.json".
- block_error = get_read_block_error(str(_resolved))
- if block_error:
- return tool_error(block_error)
- extracted = _read_extracted_document(path, _resolved, offset, limit, task_id)
- if extracted is not None:
- return extracted
- # The extension is a claim, so this message names only the extension;
- # the content-sniffing path names the actual magic-byte type.
- if has_binary_extension(str(_resolved)):
- return tool_error(
- f"Cannot read binary file '{path}' ({_resolved.suffix.lower()}). "
- "Use vision_analyze for images, or terminal to inspect binary files.")
- resolved_str = str(_resolved)
- cached_not_found = _check_not_found_cache("read", resolved_str, task_id)
- if cached_not_found is not None:
- return cached_not_found
- # Dedup: identical (path, offset, limit) on an unchanged file returns a
- # lightweight stub instead of re-sending the content.
- dedup_key = (resolved_str, offset, limit)
- with _read_tracker_lock:
- task_data = _task_data(task_id)
- cached_version = task_data["dedup"].get(dedup_key)
- # First unchanged read after a compaction boundary serves full content
- # (the summary may have dropped exact bytes); later ones get the stub.
- content_served_in_generation = dedup_key in task_data["dedup_generation_reads"]
- # Same rule as skill_view: the review fork shares the parent's task_id and its
- # read-before-write guard needs a real read, which the stub path never records (#95976).
- file_ops = _get_file_ops(task_id)
- version_before = _file_metadata(resolved_str) if _file_ops_uses_host_paths(file_ops) else None
- if (cached_version is not None and not is_background_review()
- and version_before == cached_version and content_served_in_generation):
- return _dedup_stub_or_block(task_data, dedup_key, path)
- result = file_ops.read_file(resolved_str if _file_ops_uses_host_paths(file_ops) else path, offset, limit)
- result_dict = result.to_dict()
- # Failed reads cannot establish whole-file knowledge.
- _err = result_dict.get("error") or ""
- if isinstance(_err, str) and _err.startswith("File not found:"):
- from tools.tool_result_storage import expired_archive_fields
- result_dict.update(expired_archive_fields(path))
- _record_not_found("read", resolved_str, task_id, json.dumps(result_dict, ensure_ascii=False))
- if _err or result_dict.get("is_binary"):
- return json.dumps(result_dict, ensure_ascii=False)
- # Char budget on the FORMATTED content (what enters context), BEFORE
- # redaction (skip the regex pass on huge content); truncate gracefully
- # with a next_offset instead of rejecting.
- file_size = result_dict.get("file_size", 0)
- max_chars = _get_max_read_chars()
- if len(result.content or "") > max_chars:
- result.content = _apply_char_budget(
- result_dict, result.content or "", offset,
- result_dict.get("total_lines", "unknown"), max_chars)
- redacted = False
- if result.content:
- unredacted = result.content
- result.content = redact_sensitive_text(
- unredacted, file_read=True, secret_file=_is_secret_file_arg(resolved_str))
- redacted = result.content != unredacted
- result_dict["content"] = result.content
- if result.content:
- conflicts = count_conflict_blocks(result.content)
- if conflicts:
- result_dict["conflict_blocks"] = conflicts
- result_dict["_hint"] = (
- f"{conflicts} unresolved git merge-conflict block(s) (<<<<<<< / ======= / >>>>>>>) in this "
- "range. Resolve them (keep one side or combine, delete the markers) before editing around them.")
- if (file_size and file_size > _LARGE_FILE_HINT_BYTES
- and limit > 200 and result_dict.get("truncated")):
- result_dict.setdefault("_hint", (
- f"This file is large ({file_size:,} bytes). "
- "Consider reading only the section you need with offset and limit "
- "to keep context usage efficient."))
- total_lines = result_dict.get("total_lines")
- if result_dict.get("truncated_by") == "bytes":
- end_line = int(result_dict.get("next_offset", offset)) - 1
- else:
- end_line = offset + limit - 1
- if isinstance(total_lines, int) and total_lines > 0:
- end_line = min(end_line, total_lines)
- count = _record_successful_read(task_data, task_id, path, resolved_str, offset, limit,
- dedup_key, partial=(offset > 1) or bool(result_dict.get("truncated")),
- redacted=redacted or bool(result_dict.get("truncated_lines")),
- end_line=end_line, total_lines=total_lines,
- version_before=version_before,
- snapshot=getattr(result, "_snapshot", None))
- if count >= 4:
- return tool_error(
- f"BLOCKED: You have read this exact file region {count} times in a row. "
- "The content has NOT changed. You already have this information. "
- "STOP re-reading and proceed with your task.",
- path=path,
- already_read=count,
- **{GUARDRAIL_REFUSAL_KEY: True})
- if count >= 3:
- result_dict["_warning"] = (
- f"You have read this exact file region {count} times consecutively. "
- "The content has not changed since your last read. Use the information you already have. "
- "If you are stuck in a loop, stop reading and proceed with writing or responding.")
- return json.dumps(result_dict, ensure_ascii=False)
- except Exception as e:
- return tool_error(str(e))
- # ── Shared write/patch plumbing ──────────────────────────────────────────
- def _resolve_or_none(filepath: str, task_id: str, *, entry: bool = False) -> str | None:
- """Task-resolved path string, or None when resolution fails for any reason.
- ``entry``: keep a symlink in the last component (``_resolve_entry_for_task``)."""
- try:
- return str((_resolve_entry_for_task if entry else _resolve_path_for_task)(filepath, task_id))
- except Exception:
- return None
- def _write_precheck_error(paths: list[str], content_paths: list[str], task_id: str,
- cross_profile: bool) -> str | None:
- """Run the shared write/patch guards in order; return the first error string.
- Order matters: hard denies (sensitive path, mirror) and the corruption
- guard run before anything that could prompt the user, and ONE approval
- prompt covers every path of a multi-file patch.
- """
- for p in paths:
- err = _check_sensitive_path(p, task_id) or (
- None if cross_profile else _check_cross_profile_path(p, task_id))
- if err:
- return err
- for p in content_paths:
- err = _check_binary_document_write(p, task_id)
- if err:
- return err
- return (_check_protected_instruction_write(paths, task_id)
- or _check_approval_required_write(paths, task_id))
- def _edit_warnings(paths: list[str], path_to_resolved: dict, task_id: str) -> list[str]:
- """One pre-edit warning per path, in priority order: cross-agent registry
- (names the sibling subagent) > per-task staleness > workspace divergence
- (relative path resolving outside the terminal's cwd — the worktree-cwd bug)."""
- warnings: list[str] = []
- for p in paths:
- r = path_to_resolved.get(p)
- w = (file_state.check_stale(task_id, r) if r else None) or _check_file_staleness(p, task_id)
- if not w and r:
- w = _path_resolution_warning(p, Path(r), task_id)
- if w:
- warnings.append(w)
- return warnings
- def _note_edited(task_id: str, paths: list[str], path_to_resolved: dict, session_id: str | None) -> None:
- """Post-success bookkeeping: verification-stale marker, then per path refresh
- the read stamp (no false staleness on the next edit) and record the write."""
- _mark_verification_stale(task_id, [path_to_resolved.get(p) or p for p in paths], session_id=session_id)
- for p in paths:
- _update_read_timestamp(p, task_id)
- if path_to_resolved.get(p):
- file_state.note_write(task_id, path_to_resolved[p])
- # Whole-file rewrite hint: an overwrite of an existing file this large whose new content keeps at least
- # this fraction of the old lines is a patch written the expensive way. In one 1,393-agent run 661 such
- # rewrites of >20k-char files cost ~25M output chars (~$155) where `patch` averaged 1.3k chars/call.
- _REWRITE_HINT_MIN_CHARS = 20_000
- _REWRITE_HINT_MIN_UNCHANGED = 0.80
- # Above this the line diff is skipped: SequenceMatcher on pathological repeated-line files is
- # quadratic (a 460 KB same-line file took ~22 s under the write lock).
- _REWRITE_HINT_MAX_CHARS = 400_000
- def _whole_file_rewrite_hint(task_id: str, resolved: str | None, new_content: str) -> str | None:
- """Return a hint when ``new_content`` mostly re-sends what is already at ``resolved``.
- Reads the OLD content through the task's own file ops (``read_file_raw``, the sandbox/remote
- backend the write targets), never the host path: on a remote backend the host file is a
- different file, and a host FIFO at that path would block the write lock forever. Bounded size
- and a line multiset comparison (linear) instead of a sequence diff (quadratic on repeated lines)."""
- if not resolved or not (_REWRITE_HINT_MIN_CHARS <= len(new_content) <= _REWRITE_HINT_MAX_CHARS):
- return None
- try:
- result = _get_file_ops(task_id).read_file_raw(resolved)
- old = getattr(result, "content", None)
- if getattr(result, "error", None) or not isinstance(old, str):
- return None
- except Exception:
- return None
- if not (_REWRITE_HINT_MIN_CHARS <= len(old) <= _REWRITE_HINT_MAX_CHARS):
- return None
- old_lines, new_lines = old.splitlines(), new_content.splitlines()
- if not old_lines:
- return None
- from collections import Counter
- unchanged = sum((Counter(old_lines) & Counter(new_lines)).values())
- ratio = unchanged / max(len(old_lines), len(new_lines))
- if ratio < _REWRITE_HINT_MIN_UNCHANGED:
- return None
- changed = max(len(old_lines), len(new_lines)) - unchanged
- return (
- f"{unchanged:,} of {len(new_lines):,} lines were already on disk ({ratio:.0%} unchanged); ~{changed:,} "
- f"line(s) actually changed. Re-sending a {len(new_content):,}-char file costs output tokens for every "
- "unchanged line; for edits like this use patch (old_string/new_string), which sends only the changed region."
- )
- def write_file_tool(path: str, content: str, task_id: str = "default",
- cross_profile: bool = False,
- session_id: str | None = None) -> str:
- """Write content to a file.
- ``cross_profile`` bypasses the sandbox-mirror lost-write guards only
- (unadvertised in the schema; the mirror rejection error teaches it — the
- cross-PROFILE guard it was named for no longer exists).
- """
- # write_file checks the binary-document guard before the mirror guard.
- err = (_check_sensitive_path(path, task_id)
- or _check_binary_document_write(path, task_id)
- or _check_protected_instruction_write([path], task_id)
- or _check_approval_required_write([path], task_id)
- or (None if cross_profile else _check_cross_profile_path(path, task_id)))
- if not err and _is_internal_file_tool_content(content):
- err = ("Refusing to write internal read_file display text as file content. "
- "Strip read_file line-number prefixes or reconstruct the intended "
- "file contents before writing.")
- if err:
- return tool_error(err)
- try:
- # Resolution failure falls back to the legacy unlocked path (the write
- # still proceeds; the per-task staleness check still runs).
- _resolved = _resolve_or_none(path, task_id)
- path_to_resolved = {path: _resolved}
- with ExitStack() as _lock:
- if _resolved:
- # Per-path lock serializes read→modify→write across concurrent
- # subagents; different paths stay fully parallel.
- _lock.enter_context(file_state.lock_path(_resolved))
- # A whole-file overwrite of content this task never saw, or that
- # changed since, is refused HERE — before the write — instead of
- # warning after the clobber (#65604). Nothing below runs.
- blocker = _stale_overwrite_blocker(path, _resolved, task_id)
- if blocker:
- return json.dumps(_stale_write_refusal(path, blocker, _resolved), ensure_ascii=False)
- warnings = _edit_warnings([path], path_to_resolved, task_id)
- rewrite_hint = _whole_file_rewrite_hint(task_id, _resolved, content)
- result = _get_file_ops(task_id).write_file(_resolved or path, content)
- result_dict = result.to_dict()
- if warnings:
- result_dict["_warning"] = warnings[0]
- if rewrite_hint and not result_dict.get("error"):
- result_dict["hint"] = rewrite_hint
- if _resolved:
- # Always report the ABSOLUTE path written so a wrong-cwd mismatch
- # is visible in the response instead of silently landing elsewhere.
- result_dict["resolved_path"] = _resolved
- if result_dict.get("error"):
- _update_read_timestamp(path, task_id)
- else:
- if _resolved:
- result_dict["files_modified"] = [_resolved]
- # Own write = current whole-file content: consecutive
- # same-task writes stay unblocked (patch_tool carries it).
- _mark_full_write_baseline(_resolved, task_id, getattr(result, "_content_sha256", None))
- _note_edited(task_id, [path], path_to_resolved, session_id)
- return json.dumps(result_dict, ensure_ascii=False)
- except Exception as e:
- if _is_expected_write_exception(e):
- logger.debug("write_file expected denial: %s: %s", type(e).__name__, e)
- else:
- logger.error("write_file error: %s: %s", type(e).__name__, e, exc_info=True)
- return tool_error(str(e))
- def _collect_v4a_header_paths(patch: str) -> tuple[list[str], list[str], list[str]] | str:
- """Extract every path named in V4A headers, rejecting ``..`` traversal.
- Returns ``(all_paths, content_write_paths, entry_paths)`` or a tool_error
- string. Header paths come from patch CONTENT (more attacker-influenceable
- than ``path=``, which keeps its legitimate ``..`` use). Move headers check
- BOTH endpoints; only Update/Add write text and feed the binary-document
- guard; Delete and Move act on the directory entry (``entry_paths``).
- """
- from tools.path_security import has_traversal_component
- headers = [(m.group(3), m.group(2) in ("Update", "Add")) for m in _V4A_SINGLE_HEADER_RE.finditer(patch)]
- headers += [(g, False) for m in _V4A_MOVE_HEADER_RE.finditer(patch) for g in (m.group(2), m.group(3))]
- paths: list[str] = []
- content_paths: list[str] = []
- entry_paths: list[str] = []
- for raw, writes_text in headers:
- v4a_path = raw.strip()
- if has_traversal_component(v4a_path):
- return tool_error(
- f"V4A patch header contains '..' traversal: {v4a_path!r}. "
- "Use the agent's cwd-relative path (no '..') or an absolute "
- "path in '*** Update File:' / '*** Add File:' / "
- "'*** Delete File:' / '*** Move File:' headers.")
- paths.append(v4a_path)
- (content_paths if writes_text else entry_paths).append(v4a_path)
- return paths, content_paths, entry_paths
- def patch_tool(mode: str = "replace", path: str | None = None, old_string: str | None = None,
- new_string: str | None = None, replace_all: bool = False, patch: str | None = None,
- task_id: str = "default", cross_profile: bool = False,
- session_id: str | None = None) -> str:
- """Patch a file using replace mode or V4A patch format.
- ``cross_profile``: same semantics as ``write_file``'s flag (mirror-guard
- bypass only; unadvertised).
- """
- _paths_to_check = [path] if path else []
- _content_write_paths = list(_paths_to_check)
- _entry_paths: list[str] = []
- if mode == "patch" and patch:
- collected = _collect_v4a_header_paths(patch)
- if isinstance(collected, str):
- return collected
- _paths_to_check += collected[0]
- _content_write_paths += collected[1]
- _entry_paths = collected[2]
- precheck_err = _write_precheck_error(_paths_to_check, _content_write_paths, task_id, cross_profile)
- if precheck_err:
- return tool_error(precheck_err)
- try:
- # Lock paths in sorted, deduplicated order so concurrent callers with
- # overlapping multi-file patches can't deadlock (every caller locks in
- # the same order). An unresolvable path is simply not locked.
- _path_to_resolved: dict[str, str] = {_p: _resolve_or_none(_p, task_id) for _p in _paths_to_check}
- _path_to_entry: dict[str, str] = {_p: _resolve_or_none(_p, task_id, entry=True) for _p in _entry_paths}
- with ExitStack() as _locks:
- for _r in sorted({_r for _r in (*_path_to_resolved.values(), *_path_to_entry.values()) if _r}):
- _locks.enter_context(file_state.lock_path(_r))
- stale_warnings = _edit_warnings(_paths_to_check, _path_to_resolved, task_id)
- file_ops = _get_file_ops(task_id)
- # Whole-file knowledge held BEFORE the patch reads, under the same locks. Like
- # every baseline check it stats the host path, so a sandbox path carries nothing.
- _known = _known_full_content_sha256(_path_to_resolved.values(), task_id)
- # Hand the shell layer the RESOLVED targets so both layers agree on
- # which file is edited even when the shell's cwd differs.
- if mode == "replace":
- if not path:
- return tool_error("path required")
- if old_string is None or new_string is None:
- return tool_error("old_string and new_string required")
- _replace_target = _path_to_resolved.get(path) or path
- result = file_ops.patch_replace(_replace_target, old_string, new_string, replace_all)
- elif mode == "patch":
- if not patch:
- return tool_error("patch content required")
- result = file_ops.patch_v4a(
- _rewrite_v4a_patch_paths_for_host(patch, _path_to_resolved, _path_to_entry, file_ops))
- else:
- return tool_error(f"Unknown mode: {mode}")
- result_dict = result.to_dict()
- if stale_warnings:
- result_dict["_warning"] = " | ".join(stale_warnings)
- if not result_dict.get("error"):
- # Report the ABSOLUTE path(s) actually patched so a wrong-cwd
- # mismatch is visible instead of silently landing elsewhere.
- _resolved_modified = [_path_to_entry.get(_p) or _path_to_resolved.get(_p) or _p
- for _p in _paths_to_check]
- result_dict["files_modified"] = _resolved_modified
- if len(_resolved_modified) == 1:
- result_dict["resolved_path"] = _resolved_modified[0]
- _note_edited(task_id, _paths_to_check, _path_to_resolved, session_id)
- _carry_full_write_baselines(task_id, _known, getattr(result, "_writes", ()), _path_to_resolved)
- # Clear failure counters so a future miss starts a fresh count.
- _reset_patch_failures(task_id, [_r for _r in _path_to_resolved.values() if _r])
- # old_string-not-found hint. Failure escalation is tracked for replace
- # mode only (V4A misses are rare); the generic hint is suppressed when
- # patch_replace already attached a richer "Did you mean?" snippet.
- if result_dict.get("error") and "Could not find" in str(result_dict["error"]):
- failure_count = 0
- if mode == "replace" and path:
- failure_count = _record_patch_failure(task_id, _path_to_resolved.get(path) or path)
- if failure_count >= 3:
- result_dict["_hint"] = (
- f"This is failure #{failure_count} patching {path!r}. "
- "Stop retrying with variations of the same old_string. "
- "Either: (1) re-read the file fresh to verify current "
- "content, (2) use a longer / more unique old_string with "
- "surrounding context lines, or (3) re-read the file in "
- "full, then use write_file to replace it if the targeted "
- "region is hard to anchor.")
- elif "Did you mean one of these sections?" not in str(result_dict["error"]):
- result_dict["_hint"] = (
- "old_string not found. Use read_file to verify the current "
- "content, or search_files to locate the text.")
- return json.dumps(result_dict, ensure_ascii=False)
- except Exception as e:
- return tool_error(str(e))
- def search_tool(pattern: str, target: str = "content", path: str = ".",
- file_glob: str | None = None, limit: int = 50, offset: int = 0,
- output_mode: str = "content", context: int = 0,
- order: str = "discovery",
- task_id: str = "default") -> str:
- """Search for content or files."""
- try:
- offset, limit = normalize_search_pagination(offset, limit)
- # Pagination args (and order) are part of the key so paging through truncated
- # results doesn't trip the repeated-search guard.
- search_key = ("search", pattern, target, str(path), file_glob or "", limit, offset, order)
- with _read_tracker_lock:
- task_data = _read_tracker.setdefault(task_id, {
- "last_key": None, "consecutive": 0, "read_history": set()})
- count = _bump_consecutive(task_data, search_key)
- if count >= 4:
- return tool_error(
- f"BLOCKED: You have run this exact search {count} times in a row. "
- "The results have NOT changed. You already have this information. "
- "STOP re-searching and proceed with your task.",
- pattern=pattern,
- already_searched=count,
- **{GUARDRAIL_REFUSAL_KEY: True})
- # Raw string before _resolve_path_for_task: resolving is the NTLM-leak
- # trigger and the task-base join would hide the prefix (see read_file_tool).
- nt_err = get_nt_namespace_error(path, verb="Search")
- if nt_err:
- return tool_error(nt_err)
- try:
- resolved_search_path = str(_resolve_path_for_task(path, task_id))
- except (OSError, ValueError, RuntimeError) as exc:
- resolved_search_path = path
- # A RuntimeError still surfaces as the tool error unless the raw
- # path is itself denylisted (that error wins).
- if isinstance(exc, RuntimeError) and not get_read_block_error(path):
- raise
- block_error = get_read_block_error(resolved_search_path)
- if block_error:
- return tool_error(block_error)
- # A missing search root costs two shells (search + parent listing for
- # "Similar paths"); cache the miss so a retry skips both.
- cached_search_nf = _check_not_found_cache("search", resolved_search_path, task_id)
- if cached_search_nf is not None:
- return cached_search_nf
- result = _get_file_ops(task_id).search(
- pattern=pattern, path=path, target=target, file_glob=file_glob,
- limit=limit, offset=offset, output_mode=output_mode, context=context, order=order)
- omitted = _filter_read_blocked_search_results(result, task_id)
- for m in getattr(result, "matches", None) or ():
- if getattr(m, "content", None):
- m.content = redact_sensitive_text(
- m.content, file_read=True,
- secret_file=_is_secret_file_arg(_resolved_match_path(m.path, task_id)))
- result_dict = result.to_dict(densify=True)
- if omitted:
- result_dict["_omitted"] = (
- f"{omitted} result(s) omitted because they target credential, "
- "token, cache, or secret-bearing environment files.")
- # No early return on a cached miss — same rationale as the read path.
- _search_err = result_dict.get("error") or ""
- if isinstance(_search_err, str) and _search_err.startswith("Path not found:"):
- _record_not_found("search", resolved_search_path, task_id, json.dumps(result_dict, ensure_ascii=False))
- if count >= 3:
- result_dict["_warning"] = (
- f"You have run this exact search {count} times consecutively. "
- "The results have not changed. Use the information you already have.")
- # Structured like ``_warning`` above: text appended after the JSON
- # breaks every json.loads consumer (execute_code RPC, strict tool-message
- # providers) — #90322.
- if result_dict.get("truncated"):
- result_dict["_hint"] = (
- f"Results truncated. Use offset={offset + limit} to see more, "
- "or narrow with a more specific pattern or file_glob."
- )
- return json.dumps(result_dict, ensure_ascii=False)
- except Exception as e:
- return tool_error(str(e))
- # ---------------------------------------------------------------------------
- # Schemas + Registry
- # ---------------------------------------------------------------------------
- from tools.registry import registry, tool_error
- def _check_file_reqs():
- """Lazy wrapper to avoid circular import with tools/__init__.py."""
- from tools import check_file_requirements
- return check_file_requirements()
- READ_FILE_SCHEMA = {
- "name": "read_file",
- # Document formats are stated unconditionally: firecrawl-anydoc is a
- # core dependency (bundled), so its absence is a broken install, not a
- # configuration — the teaching error in read_extract handles that rare
- # case with the PM repair hint. The ONE dynamic word: "PDF (text
- # layer)" upgrades to "PDF (scanned or text)" when hosted OCR has a
- # route we trust (_read_file_schema_overrides). Scanned-page coverage
- # teaching lives in the response-time NEEDS-OCR warning
- # (read_extract.py); the schema doesn't pre-teach it.
- "description": "Read a text file with line numbers and pagination. Use this instead of cat/head/tail in terminal. Output format: 'LINE_NUM|CONTENT'. Suggests similar filenames if not found. Use offset and limit for large files. Reads exceeding ~100K characters are truncated on a line boundary and return a next_offset; continue with offset to read the rest. Documents auto-extract to readable text: .ipynb, Office (.docx/.xlsx/.pptx and legacy .doc/.ppt/.xls), PDF (text layer), OpenDocument, RTF, EPUB, SQLite (.db/.sqlite: schema, row counts, first rows). Cannot read images/binary — use vision_analyze for images.",
- "parameters": {
- "type": "object",
- "properties": {
- "path": {"type": "string", "description": "Path to the file to read (absolute, relative, or ~/path)"},
- "offset": {"type": "integer", "description": "Line number to start reading from (1-indexed, default: 1)", "default": 1, "minimum": 1},
- "limit": {"type": "integer", "description": "Maximum number of lines to read (default: 2000, max: 2000). Reads are additionally capped at a ~100K-character budget with a next_offset continuation.", "default": DEFAULT_READ_LIMIT, "maximum": 2000}
- },
- "required": ["path"]
- }
- }
- WRITE_FILE_SCHEMA = {
- "name": "write_file",
- "description": "Write content to a file, completely replacing existing content. Use this instead of echo/cat heredoc in terminal. Creates parent directories automatically. OVERWRITES the entire file — use 'patch' for targeted edits. For an EXISTING file, call read_file first: write_file refuses (file untouched) when this task has no current full read/write of the file or the file changed on disk since; on refusal, read_file, merge, then retry. Auto-runs syntax checks on .py/.json/.yaml/.toml and other linted languages; only NEW errors introduced by this write are surfaced (pre-existing errors are filtered out). The result's verified:true means the on-disk content hash was confirmed — do NOT re-read the file to check the write landed.",
- "parameters": {
- "type": "object",
- "properties": {
- "path": {"type": "string", "description": "Path to the file to write (will be created if it doesn't exist, overwritten if it does)"},
- "content": {"type": "string", "description": "Complete content to write to the file"},
- # NOTE: the handler still accepts `cross_profile` (bool) — it now
- # bypasses only the #32049 sandbox-mirror lost-write guards, whose
- # rejection error teaches it. Unadvertised: the cross-PROFILE
- # guard it was named for was removed (profiles are not isolated,
- # maintainer decision), and mirror hits are rare + self-teaching.
- },
- "required": ["path", "content"]
- }
- }
- PATCH_SCHEMA = {
- "name": "patch",
- # BASE = replace-only (what nearly every model family was trained on).
- # The V4A patch mode (mode + patch params, dual-mode description) is
- # LAYERED ON dynamically for OpenAI-family mains only — V4A is the
- # OpenAI apply_patch dialect their models emit natively; advertising
- # it to everyone cost every other session ~148 tok/call
- # (_patch_schema_overrides below). The handler accepts BOTH shapes
- # from any model regardless (replay compat + strong models that know
- # V4A anyway): mode defaults to 'replace' when omitted.
- "description": (
- "Targeted find-and-replace edits in files. Use this instead of sed/awk in terminal. "
- "Uses fuzzy matching (9 strategies) so minor whitespace/indentation differences won't break it. "
- "Returns a unified diff. Auto-runs syntax checks after editing. "
- "Finds a unique string and replaces it."
- ),
- "parameters": {
- "type": "object",
- "properties": {
- "path": {
- "type": "string",
- "description": "File path to edit.",
- },
- "old_string": {
- "type": "string",
- "description": "Exact text to find and replace. Must be unique in the file unless replace_all=true. Include surrounding context lines to ensure uniqueness.",
- },
- "new_string": {
- "type": "string",
- "description": "Changed replacement text; it must differ from old_string. Pass empty string '' to delete the matched text.",
- },
- "replace_all": {
- "type": "boolean",
- "description": "Replace all occurrences instead of requiring a unique match (default: false)",
- "default": False,
- },
- # NOTE: handler still accepts `cross_profile` — see write_file's
- # NOTE (mirror-guard bypass only; unadvertised by design).
- # NOTE: handler still accepts `mode` + `patch` (V4A) from ANY
- # model — the schema just doesn't advertise them off-family.
- },
- "required": ["path", "old_string", "new_string"],
- },
- }
- # V4A layer, rendered only for OpenAI-family main models (see PATCH_SCHEMA
- # comment). Kept as data so the override composes it deterministically.
- _PATCH_V4A_DESCRIPTION = (
- "Targeted find-and-replace edits in files. Use this instead of sed/awk in terminal. "
- "Uses fuzzy matching (9 strategies) so minor whitespace/indentation differences won't break it. "
- "Returns a unified diff. Auto-runs syntax checks after editing.\n\n"
- "REPLACE MODE (mode='replace', default): find a unique string and replace it. "
- "REQUIRED PARAMETERS: mode, path, old_string, new_string.\n"
- "PATCH MODE (mode='patch'): apply V4A multi-file patches for bulk changes. "
- "REQUIRED PARAMETERS: mode, patch."
- )
- _PATCH_V4A_PARAMS = {
- "mode": {
- "type": "string",
- "enum": ["replace", "patch"],
- "description": "Edit mode. 'replace' (default): requires path + old_string + new_string. 'patch': requires patch content only.",
- "default": "replace",
- },
- "patch": {
- "type": "string",
- "description": "REQUIRED when mode='patch'. V4A format patch content. Format:\n*** Begin Patch\n*** Update File: path/to/file\n@@ context hint @@\n context line\n-removed line\n+added line\n*** End Patch",
- },
- }
- def _is_openai_family_main() -> bool:
- """Whether the active main provider/model is the OpenAI/codex family —
- the population trained on the V4A apply_patch dialect.
- Provider-family-coarse on purpose (no per-model training-diet table to
- go stale): direct OpenAI providers always qualify; on aggregators
- (openrouter/nous/azure...) the MODEL slug decides (gpt-*/o-series/
- codex). Fail-closed to the universal replace-only schema.
- """
- try:
- from agent.auxiliary_client import _read_main_model, _read_main_provider
- provider = (_read_main_provider() or "").strip().lower()
- model = (_read_main_model() or "").strip().lower()
- except Exception:
- return False
- if provider in {"openai", "openai-chat", "openai-codex", "azure-openai", "codex"}:
- return True
- # Aggregators: the model slug carries the family.
- slug = model.split("/", 1)[-1]
- if slug.startswith(("gpt-", "gpt.", "chatgpt", "codex", "o1", "o3", "o4", "o5")):
- return True
- return "openai/" in model
- # Pruning happens only on a macOS host (``_macos_protected_search_exclusions``), so only that
- # host's schema mentions it; other hosts would pay for a sentence about an OS they are not on.
- _MACOS_SEARCH_NOTE = (" On macOS, broad searches above the user home automatically skip TCC-protected folders (Desktop, "
- "Documents, Downloads, Library, Movies, Music, Pictures); target one directly when access is intentional."
- if facts.os_family() == "darwin" else "")
- SEARCH_FILES_SCHEMA = {
- "name": "search_files",
- "description": "Search file contents or find files by name. Use this instead of grep/rg/find/ls in terminal. Ripgrep-backed, faster than shell equivalents." + _MACOS_SEARCH_NOTE + "\n\nContent search (target='content'): Regex search inside files. Output modes: full matches with line numbers, file paths only, or match counts.\n\nFile search (target='files'): Find files by glob pattern (e.g., '*.py', '*config*'). Also use this instead of ls. Discovery order is the fast bounded default; exact global newest-first order is an explicit opt-in and may scan the full tree.",
- "parameters": {
- "type": "object",
- "properties": {
- "pattern": {"type": "string", "description": "Regex pattern for content search, or glob pattern (e.g., '*.py') for file search"},
- "target": {"type": "string", "enum": ["content", "files"], "description": "'content' searches inside file contents, 'files' searches for files by name", "default": "content"},
- "path": {"type": "string", "description": "Directory or file to search in (default: current working directory)", "default": "."},
- "file_glob": {"type": "string", "description": "Filter files by pattern in grep mode (e.g., '*.py' to only search Python files)"},
- "limit": {"type": "integer", "description": "Maximum number of results to return (default: 50)", "default": 50},
- "offset": {"type": "integer", "description": "Skip first N results for pagination (default: 0)", "default": 0},
- "order": {"type": "string", "enum": ["discovery", "modified"], "description": "File-search order: 'discovery' is fast bounded traversal order; 'modified' is exact global newest-first and may scan the full tree; ignored for content", "default": "discovery"},
- "output_mode": {"type": "string", "enum": ["content", "files_only", "count"], "description": "Output format for grep mode: 'content' shows matching lines with line numbers, 'files_only' lists file paths, 'count' shows match counts per file", "default": "content"},
- "context": {"type": "integer", "description": "Number of context lines before and after each match (grep mode only)", "default": 0}
- },
- "required": ["pattern"]
- }
- }
- def _handle_read_file(args, **kw):
- tid = kw.get("task_id") or "default"
- return read_file_tool(path=args.get("path", ""), offset=args.get("offset", 1), limit=args.get("limit", DEFAULT_READ_LIMIT), task_id=tid)
- def _handle_write_file(args, **kw):
- tid = kw.get("task_id") or "default"
- if not args.get("path") or not isinstance(args.get("path"), str):
- return tool_error(
- "write_file: missing required field 'path'. Re-emit the tool call with "
- "both 'path' and 'content' set."
- )
- if "content" not in args:
- return tool_error(
- "write_file: missing required field 'content'. The tool call included a "
- "path but no content argument — this is almost always a dropped-arg bug "
- "under context pressure. Re-emit the tool call with the full content "
- "payload, or use execute_code with hermes_tools.write_file() for very "
- "large files."
- )
- if not isinstance(args["content"], str):
- return tool_error(
- f"write_file: 'content' must be a string, got "
- f"{type(args['content']).__name__}."
- )
- from hermes_cli.observability.shared_metrics_harness import record_file_edit
- return record_file_edit("write_file", "whole_file", lambda: _owned(kw.get("session_id") or tid, write_file_tool,
- path=args["path"], content=args["content"], task_id=tid,
- cross_profile=bool(args.get("cross_profile", False)),
- session_id=kw.get("session_id"),
- ))
- def _owned(owner: str, fn, **call_kwargs):
- """Run an edit tool with its late LSP verdicts attributed to ``owner``, the calling session
- (``agent.lsp.deferred``): they ride that session's next tool result."""
- from agent.lsp.deferred import bind_owner
- with bind_owner(owner):
- return fn(**call_kwargs)
- def _handle_patch(args, **kw):
- from hermes_cli.observability.shared_metrics_harness import record_file_edit
- tid = kw.get("task_id") or "default"
- mode = args.get("mode", "replace")
- return record_file_edit("patch", mode, lambda: _owned(kw.get("session_id") or tid, patch_tool,
- mode=mode, path=args.get("path"),
- old_string=args.get("old_string"), new_string=args.get("new_string"),
- replace_all=args.get("replace_all", False), patch=args.get("patch"), task_id=tid,
- cross_profile=bool(args.get("cross_profile", False)),
- session_id=kw.get("session_id"),
- ))
- def _handle_search_files(args, **kw):
- tid = kw.get("task_id") or "default"
- target_map = {"grep": "content", "find": "files"}
- raw_target = args.get("target", "content")
- target = target_map.get(raw_target, raw_target)
- # The schema documents path='.'; a present-but-blank (or JSON null) value
- # is not a missing key for dict.get, so apply the default here (#112424).
- path = args.get("path", ".")
- if path is None or (isinstance(path, str) and not path.strip()):
- path = "."
- return search_tool(
- pattern=args.get("pattern", ""), target=target, path=path,
- file_glob=args.get("file_glob"), limit=args.get("limit", 50), offset=args.get("offset", 0),
- output_mode=args.get("output_mode", "content"), context=args.get("context", 0),
- order=args.get("order", "discovery"), task_id=tid)
- def _read_file_schema_overrides():
- """One-word capability upgrade: "PDF (text layer)" → "PDF (scanned or
- text)" when hosted OCR has a trusted route (see
- read_extract.hosted_ocr_available). Config/env probe only — no
- network at schema-build time. Compaction's tool refresh (#97073)
- picks up a key added mid-session.
- """
- try:
- from tools.read_extract import hosted_ocr_available
- if hosted_ocr_available():
- return {
- "description": READ_FILE_SCHEMA["description"].replace(
- "PDF (text layer)", "PDF (scanned or text)"
- )
- }
- except Exception:
- pass
- return {}
- registry.register(name="read_file", toolset="file", schema=READ_FILE_SCHEMA, handler=_handle_read_file, check_fn=_check_file_reqs, emoji="📖", max_result_size_chars=100_000, dynamic_schema_overrides=_read_file_schema_overrides)
- registry.register(name="write_file", toolset="file", schema=WRITE_FILE_SCHEMA, handler=_handle_write_file, check_fn=_check_file_reqs, emoji="✍️", max_result_size_chars=100_000)
- def _patch_schema_overrides():
- """Layer the V4A patch mode onto the base replace-only schema for
- OpenAI-family mains (see PATCH_SCHEMA comment). Config/context probe
- only — no I/O at schema-build time; compaction's tool refresh
- (#97073) re-evaluates on model switches."""
- try:
- if not _is_openai_family_main():
- return {}
- params = {
- "type": "object",
- "properties": {
- "mode": _PATCH_V4A_PARAMS["mode"],
- **PATCH_SCHEMA["parameters"]["properties"],
- "patch": _PATCH_V4A_PARAMS["patch"],
- },
- "required": ["mode"],
- }
- return {"description": _PATCH_V4A_DESCRIPTION, "parameters": params}
- except Exception:
- return {}
- registry.register(name="patch", toolset="file", schema=PATCH_SCHEMA, handler=_handle_patch, check_fn=_check_file_reqs, emoji="🔧", max_result_size_chars=100_000, dynamic_schema_overrides=_patch_schema_overrides)
- registry.register(name="search_files", toolset="file", schema=SEARCH_FILES_SCHEMA, handler=_handle_search_files, check_fn=_check_file_reqs, emoji="🔎", max_result_size_chars=100_000)
|