file_operations.py 92 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718
  1. #!/usr/bin/env python3
  2. """File operations (read, write, patch, search) over any terminal backend.
  3. Every operation is a shell command run through the backend's ``execute()``, so one
  4. implementation serves every environment (local, docker, ssh, modal, ...). Companions:
  5. ``file_operations_common`` (result dataclasses, text helpers), ``file_operations_lint``
  6. (LintMixin), ``file_operations_search`` (SearchMixin).
  7. """
  8. import base64
  9. import binascii
  10. import os
  11. import re
  12. import sys
  13. import difflib
  14. import hashlib
  15. import json
  16. import logging
  17. import secrets
  18. import time
  19. import unicodedata
  20. from abc import ABC, abstractmethod
  21. from typing import Optional, Dict
  22. from pathlib import Path
  23. from tools.binary_extensions import has_binary_extension
  24. from agent.file_safety import get_write_denied_error
  25. from tools.file_operations_common import (
  26. ExecuteResult, PatchResult, ReadResult, SearchResult, WriteResult,
  27. _UTF8_BOM, _detect_line_ending, _has_bom, _normalize_line_endings, _strip_bom,
  28. _strip_terminal_fence_leaks, normalize_read_pagination, normalize_search_pagination)
  29. from tools.file_operations_lint import LINTERS_INPROC, LintMixin, _FAIL_CLOSED_INPROC_EXTS
  30. from tools.file_operations_search import SearchMixin
  31. from tools import file_operations_native as _native_io
  32. logger = logging.getLogger(__name__)
  33. # Controller home; SearchMixin reads it (tests monkeypatch it here).
  34. _HOME = str(Path.home())
  35. # --- Binary-content identification -------------------------------------------
  36. _MAGIC_SIGNATURES: tuple = (
  37. # (prefix bytes, human name) — ordered, first match wins. Longest
  38. # prefixes for a shared first byte come first.
  39. (b"\x89PNG\r\n\x1a\n", "PNG image data"),
  40. (b"\xff\xd8\xff", "JPEG image data"),
  41. (b"GIF87a", "GIF image data"),
  42. (b"GIF89a", "GIF image data"),
  43. (b"RIFF", "RIFF container (WAV/AVI/WebP family)"),
  44. (b"%PDF-", "PDF document"),
  45. (b"PK\x03\x04", "ZIP archive (also docx/xlsx/jar/apk)"),
  46. (b"PK\x05\x06", "ZIP archive (empty)"),
  47. (b"\x1f\x8b", "gzip compressed data"),
  48. (b"BZh", "bzip2 compressed data"),
  49. (b"\xfd7zXZ\x00", "xz compressed data"),
  50. (b"7z\xbc\xaf\x27\x1c", "7-Zip archive"),
  51. (b"\x7fELF", "ELF executable"),
  52. (b"MZ", "Windows PE executable"),
  53. (b"\xcf\xfa\xed\xfe", "Mach-O executable (64-bit)"),
  54. (b"\xca\xfe\xba\xbe", "Mach-O universal binary / Java class"),
  55. (b"SQLite format 3\x00", "SQLite database"),
  56. (b"OggS", "Ogg container"),
  57. (b"fLaC", "FLAC audio"),
  58. (b"ID3", "MP3 audio (ID3 tag)"),
  59. (b"\x00\x00\x00", "ISO media container (MP4/MOV family)"), # ftyp at +4
  60. (b"BM", "BMP image data"),
  61. (b"II*\x00", "TIFF image data (little-endian)"),
  62. (b"MM\x00*", "TIFF image data (big-endian)"),
  63. )
  64. def identify_binary_bytes(sample: bytes) -> str:
  65. """Best-effort human name for binary content from its magic bytes; never raises.
  66. The ISO-media entry additionally requires ``ftyp`` at offset 4 (three leading
  67. NULs alone are too weak a signature)."""
  68. for prefix, name in _MAGIC_SIGNATURES:
  69. if sample.startswith(prefix):
  70. if name.startswith("ISO media") and sample[4:8] != b"ftyp":
  71. continue
  72. return name
  73. return "unknown binary"
  74. def describe_binary_file(sample: Optional[bytes], file_size: int) -> str:
  75. """One-line binary-file refusal naming the TYPE ("PNG image data, 4.1 KB"), so the
  76. model gets what-is-this in one read instead of hunting for tools it may lack."""
  77. kind = identify_binary_bytes(sample or b"")
  78. if file_size >= 1024 * 1024:
  79. size = f"{file_size / (1024 * 1024):.1f} MB"
  80. elif file_size >= 1024:
  81. size = f"{file_size / 1024:.1f} KB"
  82. else:
  83. size = f"{file_size} bytes"
  84. return f"Binary file ({kind}, {size}) — cannot display as text."
  85. class FileOperations(ABC):
  86. """Abstract interface for file operations across terminal backends."""
  87. @abstractmethod
  88. def read_file(self, path: str, offset: int = 1, limit: int = 2000) -> ReadResult:
  89. """Read a file with pagination support."""
  90. @abstractmethod
  91. def read_file_raw(self, path: str) -> ReadResult:
  92. """Whole file as a plain string: no pagination, line numbers or clamping."""
  93. @abstractmethod
  94. def write_file(self, path: str, content: str, pre_content: Optional[str] = None) -> WriteResult:
  95. """Write content to a file, creating directories as needed."""
  96. @abstractmethod
  97. def patch_replace(self, path: str, old_string: str, new_string: str,
  98. replace_all: bool = False) -> PatchResult:
  99. """Replace text in a file using fuzzy matching."""
  100. @abstractmethod
  101. def patch_v4a(self, patch_content: str) -> PatchResult:
  102. """Apply a V4A format patch."""
  103. @abstractmethod
  104. def delete_file(self, path: str) -> WriteResult:
  105. """Delete a file. Returns WriteResult with .error set on failure."""
  106. @abstractmethod
  107. def move_file(self, src: str, dst: str) -> WriteResult:
  108. """Move/rename a file. Returns WriteResult with .error set on failure."""
  109. @abstractmethod
  110. def search(self, pattern: str, path: str = ".", target: str = "content",
  111. file_glob: Optional[str] = None, limit: int = 50, offset: int = 0,
  112. output_mode: str = "content", context: int = 0,
  113. order: str = "discovery") -> SearchResult:
  114. """Search for content or files."""
  115. # --- Shell-based implementation ----------------------------------------------
  116. # Image extensions (subset of binary that we can return as base64)
  117. IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.ico'}
  118. # Echoed by the size probe when the path exists but is not a regular file.
  119. # `wc -c` prints only digits, so this can never collide with a real size.
  120. NOT_REGULAR_SENTINEL = "__hermes_not_regular__"
  121. # Echoed by the compound read/write probes when the path does not exist. A
  122. # compound command only reports its *last* exit status, so the missing-file
  123. # signal that ``_probe_regular_file`` carries in ``exit 1`` travels in-band.
  124. MISSING_SENTINEL = "__hermes_missing__"
  125. _READ_SENTINEL_PREFIX = "__HERMES_RF_"
  126. _WRITE_SENTINEL_PREFIX = "__HERMES_WF_"
  127. _BYTES_SENTINEL_PREFIX = "__HERMES_RB_"
  128. def _new_sentinel(prefix: str) -> str:
  129. """Per-call separator line for a compound shell probe. 128 random bits make a
  130. collision with file content negligible; the underscores keep the token outside
  131. the base64 alphabet, so a sentinel leaking into a sample segment fails base64
  132. validation instead of decoding into bytes."""
  133. return f"{prefix}{secrets.token_hex(16)}__"
  134. def _split_segments(output: str, sentinel: str) -> list[str]:
  135. """Split compound-probe stdout on its sentinel lines. Every producer (``wc``,
  136. ``base64``, ``cut``) newline-terminates or prints nothing, so the separator is
  137. always ``sentinel + "\n"`` on its own line; the text after the final sentinel
  138. is the status segment."""
  139. return output.split(sentinel + "\n")
  140. def _json_nonstandard_constant(text: str) -> Optional[str]:
  141. """First NaN/Infinity/-Infinity in ``text`` when it is otherwise valid JSON,
  142. else None. ``json.loads`` accepts these JavaScript extensions by default."""
  143. if "NaN" not in text and "Infinity" not in text:
  144. return None
  145. found: list[str] = []
  146. def note_constant(value: str) -> float:
  147. found.append(value)
  148. return float("nan")
  149. try:
  150. json.loads(_strip_bom(text)[0], parse_constant=note_constant)
  151. except (ValueError, RecursionError): # unparseable text has no constant to report
  152. return None
  153. return found[0] if found else None
  154. def _refuse_introduced_json_constant(path: str, content: str,
  155. pre_content: Optional[str]) -> Optional[WriteResult]:
  156. """Refuse a JSON write that INTRODUCES a nonstandard constant (strict JSON
  157. consumers reject them). A file that already holds one keeps accepting
  158. unrelated edits, which is why the lenient syntax gate can't do this check."""
  159. constant = _json_nonstandard_constant(content)
  160. if constant is None:
  161. return None
  162. if pre_content is not None and _json_nonstandard_constant(pre_content) is not None:
  163. return None
  164. return WriteResult(error=(
  165. f"Refusing to write '{path}': candidate content uses {constant}, which is "
  166. "not valid JSON. The file was NOT created or modified. Use null or a "
  167. "string instead and retry."))
  168. class ShellFileOperations(LintMixin, SearchMixin, FileOperations):
  169. """File operations over any terminal backend exposing ``execute(command, cwd)``
  170. returning ``{"output": str, "returncode": int}``.
  171. cwd rule: every ``_exec`` prefers the LIVE ``env.cwd`` so a ``cd`` run via the
  172. terminal tool is picked up immediately; the init-time ``self.cwd`` is only a
  173. fallback for envs that don't track cwd (using it for every call once made
  174. patches "succeed" with a plausible diff while landing in the wrong directory).
  175. """
  176. def __init__(self, terminal_env, cwd: str | None = None):
  177. self.env = terminal_env
  178. # Never os.getcwd(): that is the HOST path, absent inside container backends.
  179. self.cwd = cwd or getattr(terminal_env, 'cwd', None) or \
  180. getattr(getattr(terminal_env, 'config', None), 'cwd', None) or "/"
  181. # Ordinary executables: bool cache (hits AND misses). rg is special — it has
  182. # an off-PATH resolver and may be installed mid-session — so only successful
  183. # rg resolutions are cached (see SearchMixin._resolve_command).
  184. self._command_cache: dict[str, bool] = {}
  185. self._rg_resolution_cache: dict[str, str] = {}
  186. self._rg_modified_capability: dict[str, Optional[str]] = {}
  187. def _exec(self, command: str, cwd: str | None = None, timeout: int | None = None,
  188. stdin_data: str | None = None) -> ExecuteResult:
  189. """Run ``command`` on the backend. cwd: explicit arg → live ``env.cwd`` →
  190. init-time ``self.cwd``. ``stdin_data`` is piped (bypasses ARG_MAX)."""
  191. kwargs = {}
  192. if timeout:
  193. kwargs['timeout'] = timeout
  194. if stdin_data is not None:
  195. kwargs['stdin_data'] = stdin_data
  196. effective_cwd = cwd or getattr(self.env, 'cwd', None) or self.cwd
  197. result = self.env.execute(command, cwd=effective_cwd, **kwargs)
  198. exit_code = result.get("returncode", 0)
  199. output = result.get("output", "")
  200. # The command wrapper's own ``builtin cd -- <cwd> || exit 126`` failed: the
  201. # working directory does not exist on this backend (typically ``terminal.cwd``
  202. # is a host path and the backend is a container). Name that, or the raw
  203. # ``cd:`` line reads like a sandbox/mount fault at the requested path.
  204. cwd_error = ""
  205. if exit_code == 126 and "cd: " in output:
  206. from tools.terminal_tool_config import _is_container_backend
  207. env_type = getattr(self.env, "env_type", None)
  208. hint = ("; for container backends use a path inside the container, e.g. /workspace"
  209. if env_type and _is_container_backend(env_type) else "")
  210. cwd_error = output = (
  211. f"working directory {effective_cwd!r} does not exist on the active terminal "
  212. f"backend (check terminal.cwd or the session cwd{hint}). {output.strip()}")
  213. # A stdin write failure with a clean child exit is still a failure: the
  214. # child never received the input.
  215. if result.get("stdin_error") and exit_code == 0:
  216. exit_code = 1
  217. return ExecuteResult(stdout=output, exit_code=exit_code, cwd_error=cwd_error)
  218. def _has_command(self, cmd: str) -> bool:
  219. """Check if a command exists in the environment (cached); rg goes through
  220. the resolver so a mid-session install becomes visible."""
  221. if cmd == "rg":
  222. return self._resolve_command(cmd) is not None
  223. if cmd not in self._command_cache:
  224. result = self._exec(f"command -v {cmd} >/dev/null 2>&1 && echo 'yes'")
  225. if result.cwd_error: # the probe never ran: no verdict to cache
  226. return False
  227. self._command_cache[cmd] = result.stdout.strip() == 'yes'
  228. return self._command_cache[cmd]
  229. def _cat(self, path: str) -> ExecuteResult:
  230. """``cat`` the file with stderr silenced (missing file → non-zero exit)."""
  231. return self._exec(f"cat {self._escape_shell_arg(path)} 2>/dev/null")
  232. def _head(self, path: str, nbytes: int) -> ExecuteResult:
  233. return self._exec(f"head -c {nbytes} {self._escape_shell_arg(path)} 2>/dev/null")
  234. def _run_python_snippet(self, snippet: str) -> ExecuteResult:
  235. """Run ``snippet`` via the backend's ``python3``, retrying with ``python``
  236. when only that name exists (Windows / older systems)."""
  237. result = self._exec(f"python3 -c {self._escape_shell_arg(snippet)}")
  238. if result.exit_code != 0 and "python3" in (result.stdout or ""):
  239. result = self._exec(f"python -c {self._escape_shell_arg(snippet)}")
  240. return result
  241. def _fenced_read(self, body: str, *more: str) -> tuple[Optional[list[str]], Optional[int], ExecuteResult]:
  242. """Run BODY, then each of MORE, each in its own sentinel-delimited segment; return (those
  243. segments, BODY's exit status, reply).
  244. The transport merges the backend's own stdout with the command's, and every caller here
  245. decodes a segment into file bytes, so the payload has to be delimited rather than taken to
  246. be the whole reply: a remote shell announcing ``TERM`` is four base64 characters that would
  247. otherwise join the payload and decode to ``b"LDL"`` at the head of it. The fence drops noise
  248. OUTSIDE it only; output emitted while BODY runs (a ``BASH_ENV`` DEBUG hook) lands inside the
  249. payload, so a caller that writes the bytes back must verify them independently (MORE). The
  250. status rides in its own trailing segment so a failed BODY is still told apart from an empty
  251. file. ``(None, None, reply)`` when no fenced reply came back — the command never ran as
  252. written.
  253. """
  254. sentinel = _new_sentinel(_BYTES_SENTINEL_PREFIX)
  255. mark = f"echo {sentinel}"
  256. rest = "".join(f"{mark}; {cmd}; " for cmd in more)
  257. # xtrace off first: a traced ``+ echo <sentinel>`` line is an extra separator, and the
  258. # traces of the transport commands would land inside the payload segments.
  259. result = self._exec(f"{{ set +x; }} 2>/dev/null; {mark}; {body}; __hb=$?; {rest}{mark}; echo $__hb")
  260. segments = _split_segments(result.stdout or "", sentinel)
  261. if len(segments) != len(more) + 3:
  262. return None, None, result
  263. try:
  264. return segments[1:-1], int(_strip_terminal_fence_leaks(segments[-1]).split()[0]), result
  265. except (IndexError, ValueError):
  266. return segments[1:-1], None, result
  267. @staticmethod
  268. def _matches_size(data: bytes, size_segment: str) -> bool:
  269. """Whether DATA is exactly as long as the file's own ``wc -c``. Noise inside the payload
  270. only ever ADDS text, and any addition that still decodes adds bytes, so equal length is
  271. the check; noise in the size segment breaks its single-integer shape instead."""
  272. tokens = _strip_terminal_fence_leaks(size_segment).split()
  273. return len(tokens) == 1 and tokens[0].isdigit() and int(tokens[0]) == len(data)
  274. def _sample_file_bytes(self, path: str, length: int = 1000):
  275. """First ``length`` raw bytes, base64-wrapped so they survive the terminal
  276. transport (which decodes stdout with ``errors="replace"`` and manufactures
  277. U+FFFD for every undecodable byte, including a multibyte char cut in half
  278. by ``head -c``). None when no clean base64 came back (no ``base64`` binary);
  279. callers then fall back to the text heuristic.
  280. Wrapping the sample in base64 lets the original bytes survive the transport, so binary detection can
  281. happen at the byte layer where it is well-defined (#80308 and friends). Fenced like the
  282. byte-exact read below: this sample is the binary-admission gate in FRONT of that read, so
  283. backend noise decoded into it decides whether a file is editable at all.
  284. """
  285. segments, read_rc, _ = self._fenced_read(
  286. f"head -c {length} {self._escape_shell_arg(path)} 2>/dev/null | base64")
  287. if segments is None or read_rc != 0:
  288. return None
  289. return self._decode_base64_sample(segments[0])
  290. def _read_exact_bytes(self, path: str) -> tuple[Optional[bytes], Optional[ExecuteResult]]:
  291. """The file's bytes exactly, for the edit paths that write back every line they did not touch.
  292. The text transport cannot carry them: it decodes with errors="replace", so a byte UTF-8 cannot
  293. decode comes back as U+FFFD and the edit then persists it. A native read on the local POSIX host,
  294. else base64 over the transport; ``(None, result)`` hands back the failed shell read for the
  295. caller's message. Only a regular file gets a native open (a FIFO would block this thread); the
  296. rest take the shell path and its timeout, as before."""
  297. if self._native_read_enabled():
  298. import stat as _stat
  299. full = path if os.path.isabs(path) else os.path.join(
  300. getattr(self.env, "cwd", None) or self.cwd, path)
  301. try:
  302. # One lookup, not two: a stat-then-open pair can have the path swapped for a FIFO in
  303. # between, and that open blocks this thread forever (no backend timeout covers it).
  304. # O_NONBLOCK returns a descriptor for a FIFO instead of waiting, and fstat judges THAT
  305. # descriptor, so a non-regular file is rejected rather than read.
  306. fd = os.open(full, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
  307. try:
  308. if _stat.S_ISREG(os.fstat(fd).st_mode):
  309. with open(fd, "rb", closefd=False) as fh:
  310. return fh.read(), None
  311. finally:
  312. os.close(fd)
  313. except OSError:
  314. pass # missing/unreadable/would-block: the shell read below reports it the usual way
  315. # Fenced like the compound read probe, and for the same reason: a backend whose merged
  316. # stdout carries login-shell noise (a remote shell announcing TERM, a banner) would
  317. # otherwise have it whitespace-joined onto the payload and decoded INTO the file's bytes,
  318. # which the edit paths then write back. The file's own byte count travels beside it: output
  319. # INSIDE the fence decodes too, so only a read that matches it is ever handed to a writer.
  320. arg = self._escape_shell_arg(path)
  321. segments, read_rc, result = self._fenced_read(f"base64 < {arg}", f"wc -c < {arg}")
  322. garbled = ExecuteResult(stdout=f"{path}: the backend returned a garbled byte-exact read", exit_code=1)
  323. if segments is None:
  324. # No fenced reply: the command never ran as written (a wrapper ``cd`` failed, the backend
  325. # refused it). Hand the backend's own text back so the caller reports what it said.
  326. return None, result if result.exit_code != 0 else garbled
  327. if read_rc is None:
  328. return None, garbled
  329. if read_rc == 127: # no base64 on this backend (busybox, distroless): try the hex transport
  330. return self._read_exact_bytes_hex(path)
  331. payload, size = segments
  332. if read_rc != 0:
  333. # stderr is merged into the fenced segment, so that segment holds base64's own diagnostic
  334. # ("No such file or directory", "Permission denied"): keep it for the caller's message.
  335. return None, self._failed_read(path, payload, read_rc)
  336. data = self._decode_base64_sample(payload)
  337. if data is None or not self._matches_size(data, size): # stray output: refuse, never echo it back
  338. return None, garbled
  339. return data, None
  340. @staticmethod
  341. def _failed_read(path: str, payload: str, read_rc: int) -> ExecuteResult:
  342. """The backend's own diagnostic for a read that ran and failed, else a bare exit status."""
  343. return ExecuteResult(stdout=_strip_terminal_fence_leaks(payload).strip() or f"{path}: exit {read_rc}",
  344. exit_code=read_rc)
  345. def _read_exact_bytes_hex(self, path: str) -> tuple[Optional[bytes], Optional[ExecuteResult]]:
  346. """``od`` fallback for a backend without ``base64``, fenced the same way.
  347. ``read_file_raw`` is the edit paths' source read AND, through ``_apply_add``, their
  348. existence check, so a transport that simply is not installed must not read as "no such
  349. file" — that clobbers the file the Add was refusing to overwrite. ``od`` is POSIX and
  350. present in busybox; when it is missing too the caller gets a transport error, never a
  351. not-found."""
  352. arg = self._escape_shell_arg(path)
  353. segments, read_rc, result = self._fenced_read(f"od -An -v -tx1 < {arg}", f"wc -c < {arg}")
  354. unavailable = ExecuteResult(
  355. stdout=f"{path}: this backend has neither base64 nor od, so a byte-exact read is unavailable",
  356. exit_code=1)
  357. if segments is None:
  358. return None, result if result.exit_code != 0 else unavailable
  359. if read_rc is None or read_rc == 127:
  360. return None, unavailable
  361. payload, size = segments
  362. if read_rc != 0:
  363. return None, self._failed_read(path, payload, read_rc)
  364. try:
  365. data = bytes.fromhex("".join(_strip_terminal_fence_leaks(payload).split()))
  366. except ValueError:
  367. data = None
  368. if data is None or not self._matches_size(data, size):
  369. return None, ExecuteResult(stdout=f"{path}: the backend returned a garbled byte-exact read",
  370. exit_code=1)
  371. return data, None
  372. @staticmethod
  373. def _decode_base64_sample(text: str) -> Optional[bytes]:
  374. """Decode one ``base64`` transport reply (a ``head -c N`` sample or a whole file). Whitespace-joins
  375. the whole text first (``base64`` wraps at 76 columns), so callers hand over exactly one
  376. segment; anything else fails validation → None."""
  377. encoded = "".join(_strip_terminal_fence_leaks(text).split())
  378. if not encoded:
  379. return b""
  380. if not re.fullmatch(r"[A-Za-z0-9+/]+={0,2}", encoded):
  381. return None
  382. try:
  383. return base64.b64decode(encoded, validate=True)
  384. except (binascii.Error, ValueError):
  385. return None
  386. @staticmethod
  387. def _is_likely_binary_bytes(sample: bytes) -> bool:
  388. """Byte-layer binary detection: text iff valid UTF-8, allowing one incomplete
  389. multibyte sequence at the very end (artifact of the byte-boundary cut).
  390. NUL bytes or mid-stream invalid UTF-8 stay read-only so a read→edit→write
  391. round-trip never rewrites undecodable bytes as U+FFFD; a file that
  392. legitimately CONTAINS U+FFFD is valid UTF-8 and reads as text.
  393. See #80308.
  394. """
  395. if not sample:
  396. return False
  397. if b"\x00" in sample:
  398. return True
  399. try:
  400. sample.decode("utf-8")
  401. return False
  402. except UnicodeDecodeError as exc:
  403. # UTF-8 sequences are at most 4 bytes: an error starting in the
  404. # last 3 bytes with a clean prefix is a boundary cut, not binary.
  405. if exc.start >= len(sample) - 3:
  406. try:
  407. sample[: exc.start].decode("utf-8")
  408. return False
  409. except UnicodeDecodeError:
  410. pass
  411. return True
  412. def _is_likely_binary(self, path: str, content_sample: str | None = None) -> bool:
  413. """Legacy text-layer binary check: extension, else >30% non-printable chars."""
  414. if has_binary_extension(path):
  415. return True
  416. if content_sample:
  417. # Undecodable bytes arrive as U+FFFD ("printable", so the ratio misses
  418. # them); treat as binary so a round-trip can't write back mojibake.
  419. if "\ufffd" in content_sample[:1000]:
  420. return True
  421. non_printable = sum(1 for c in content_sample[:1000] if ord(c) < 32 and c not in '\n\r\t')
  422. return non_printable / min(len(content_sample), 1000) > 0.30
  423. return False
  424. def _is_image(self, path: str) -> bool:
  425. return os.path.splitext(path)[1].lower() in IMAGE_EXTENSIONS
  426. def _add_line_numbers(self, content: str, start_line: int = 1) -> str:
  427. """Prefix each line with a compact ``<n>|`` gutter, clamping long lines. Not
  428. fixed-width: padding cost ~16% more tokens per line for no accuracy gain in
  429. A/B, while dropping numbers regressed line-referencing."""
  430. from tools.tool_output_limits import get_max_line_length
  431. max_line_length = get_max_line_length()
  432. # A trailing newline terminates the final line — it does not start a new,
  433. # empty one. Splitting without dropping it rendered a phantom "<N+1>|"
  434. # gutter line on every newline-terminated file (`cat -n` semantics).
  435. # Exactly ONE terminator is dropped, so a genuinely selected trailing
  436. # blank line in a page keeps its own number.
  437. content = content.removesuffix('\n')
  438. return '\n'.join(
  439. f"{i}|{line if len(line) <= max_line_length else line[:max_line_length] + '... [truncated]'}"
  440. for i, line in enumerate(content.split('\n'), start=start_line))
  441. def _expand_path(self, path: str) -> str:
  442. """Expand ``~`` / ``~user`` via the backend's shell (its HOME, not the
  443. host's). A host path under the configured workspace mount is rewritten
  444. to that container path first, so a Windows drive path is readable
  445. inside Docker. Must run BEFORE shell escaping — ~ doesn't expand in quotes."""
  446. from tools.terminal_tool_config import translate_mounted_host_path
  447. host_root = getattr(self.env, "host_cwd", None)
  448. container_root = getattr(self.env, "host_cwd_mount", None) or "/workspace"
  449. translated = translate_mounted_host_path(path, host_root or "", container_root)
  450. if translated:
  451. return translated
  452. if not path or not path.startswith('~'):
  453. return path
  454. result = self._exec("echo $HOME")
  455. if result.exit_code == 0 and result.stdout.strip():
  456. home = result.stdout.strip()
  457. if path == '~':
  458. return home
  459. if path.startswith('~/'):
  460. return home + path[1:]
  461. # ~username: validate and expand ONLY that token, so neither "~; rm -rf /"
  462. # nor "~user/$(malicious)" reaches the shell.
  463. rest = path[1:]
  464. slash_idx = rest.find('/')
  465. username = rest[:slash_idx] if slash_idx >= 0 else rest
  466. if username and re.fullmatch(r'[a-zA-Z0-9._-]+', username):
  467. expand_result = self._exec(f"echo ~{username}")
  468. if expand_result.exit_code == 0 and expand_result.stdout.strip():
  469. return expand_result.stdout.strip() + path[1 + len(username):]
  470. return path
  471. def _escape_shell_arg(self, arg: str, translate_path: bool = True) -> str:
  472. """Escape a string for safe use in shell commands.
  473. On Windows native drive paths (``C:\\Users\\x`` / ``C:/Users/x``)
  474. and mixed MSYS leftovers (``/c/Users\\x``) are rewritten to the
  475. Git Bash ``/c/Users/x`` form via ``_bash_safe_path``: bash eats
  476. backslashes and MSYS otherwise mangles drive paths into the
  477. ``Directory \\drivers\\etc does not exist`` failure class. Reuses
  478. the env-layer translator so shell file ops and the terminal ``cd``
  479. agree on the path form. No-op off Windows and for plain POSIX paths.
  480. ``translate_path=False`` skips that translation for non-path values
  481. such as regex patterns. Backslash compensation applies only to the
  482. local Windows argv transport. Serialized shell text stays literal.
  483. """
  484. from tools.environments.local import _IS_WINDOWS, _bash_safe_path
  485. if translate_path:
  486. arg = _bash_safe_path(arg)
  487. elif _IS_WINDOWS and getattr(self.env, "is_local", False):
  488. arg = arg.replace("\\", "\\\\")
  489. # Use single quotes and escape any single quotes in the string
  490. return "'" + arg.replace("'", "'\"'\"'") + "'"
  491. def _escape_native_tool_arg(self, arg: str) -> str:
  492. """Quote a path for a NATIVE Windows binary (rg, node, git ...): those don't
  493. understand the MSYS ``/c/...`` form and Hermes disables MSYS argument
  494. conversion, so nothing translates it back (→ ``os error 3``). ``C:/Users/x``
  495. is accepted by every layer. Identical to ``_escape_shell_arg`` off Windows."""
  496. from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
  497. if _IS_WINDOWS and arg:
  498. arg = _msys_to_windows_path(arg).replace("\\", "/")
  499. return "'" + arg.replace("'", "'\"'\"'") + "'"
  500. def _atomic_write(self, path: str, content: str) -> ExecuteResult:
  501. """Write ``content`` atomically: stdin → temp file in the SAME directory →
  502. ``mv -f`` (same-FS rename; cross-device ``mv`` is copy+unlink, NOT atomic).
  503. ``mkdir -p`` folded in. Exit 0 = swap happened; non-zero = original intact.
  504. Symlink targets are resolved first (replacing the link would orphan the
  505. target) and the temp dir recomputed from the RESOLVED target. Existing
  506. target: mode copied via ``stat`` (GNU ``-c%a`` / BSD ``-f%Lp``) + ``chmod``
  507. (``chmod --reference`` is GNU-only). New target: ``chmod "=rw"`` AFTER cat
  508. gives umask-default perms instead of mktemp's 0600 — not ``$(umask)``
  509. arithmetic (zsh parses leading-zero constants as decimal), quoted so zsh
  510. doesn't =word-expand. ``trap ... EXIT`` removes the temp on every failure.
  511. The local POSIX backend does the same in-process (``tools.file_operations_native``).
  512. """
  513. if self._native_read_enabled():
  514. error = _native_io.atomic_write(self._native_path(path), content.encode("utf-8", "surrogateescape"))
  515. return ExecuteResult(stdout=error or "", exit_code=1 if error else 0)
  516. q_path = self._escape_shell_arg(path)
  517. q_parent = self._escape_shell_arg(os.path.dirname(path) or ".")
  518. tmpl = self._escape_shell_arg(".hermes-tmp.XXXXXX")
  519. script = (
  520. "set -e; "
  521. # One shell script, fully quoted. Notes: - `mkdir -p "$d"` is folded in here so the parent
  522. # directory is created in the same subprocess that writes the temp file — saves one entire
  523. # subprocess spawn vs. a separate mkdir call. - `mktemp` lands the temp in the target's own dir
  524. # (-p) so `mv` is same-FS atomic; we fall back to a PID-stamped name if the backend lacks mktemp
  525. # (rare; busybox/macOS/Linux all ship it). - `chmod --reference` is GNU-only, so we read the
  526. # octal mode with `stat` (GNU `-c%a` or BSD `-f%Lp`) and `chmod` it explicitly; silent
  527. # best-effort — a perms-copy failure must not abort the write (the file then lands at mktemp's
  528. # 0600, same as pre-fix). - brand-new targets get `chmod "=rw"` — the POSIX who-less symbolic
  529. # form, which sets rw minus the process umask (e.g. 0644 under umask 022) instead of mktemp's
  530. # hardcoded 0600 (#70856). Deliberately NOT shell arithmetic on `$(umask)`: zsh (reachable via
  531. # _find_bash's $SHELL fallback) parses leading-zero constants as decimal and silently computes a
  532. # garbage mode, while `chmod "=rw"` is spec-identical in bash/dash/ash/zsh and degrades to 0600
  533. # (pre-fix behavior) if an exotic chmod rejects it. - `trap ... EXIT` guarantees the temp is
  534. # removed on every error path (cat failure, mv failure, signal) but NOT after a successful mv
  535. # (the temp no longer exists by then). - we `cat >` the temp, then `mv -f` it over the target.
  536. f"d={q_parent}; t={q_path}; "
  537. 'if [ -L "$t" ]; then '
  538. 'rt="$(readlink -f "$t" 2>/dev/null || realpath "$t" 2>/dev/null || true)"; '
  539. '[ -n "$rt" ] && { t="$rt"; d="$(dirname "$t")"; }; '
  540. "fi; "
  541. 'mkdir -p "$d"; '
  542. 'tmp="$(mktemp -p "$d" ' + tmpl + ' 2>/dev/null '
  543. '|| mktemp "$d/.hermes-tmp.$$.XXXXXX" 2>/dev/null '
  544. '|| { tmp="$d/.hermes-tmp.$$"; : > "$tmp" && echo "$tmp"; })"; '
  545. '[ -n "$tmp" ] || { echo "atomic write: could not create temp file" >&2; exit 1; }; '
  546. "trap 'rm -f \\\"$tmp\\\"' EXIT; "
  547. 'if [ -e "$t" ]; then '
  548. 'm="$(stat -c%a "$t" 2>/dev/null || stat -f%Lp "$t" 2>/dev/null || true)"; '
  549. '[ -n "$m" ] && chmod "$m" "$tmp" 2>/dev/null || true; '
  550. "fi; "
  551. 'cat > "$tmp"; '
  552. # new file: umask-default perms instead of mktemp's 0600 (#70856). Runs AFTER cat so a
  553. # write-masking umask can't EACCES the stream; quoted "=rw" so zsh doesn't =word-expand it.
  554. 'if [ ! -e "$t" ]; then chmod "=rw" "$tmp" 2>/dev/null || true; fi; '
  555. 'mv -f "$tmp" "$t"; '
  556. "trap - EXIT")
  557. return self._exec(script, stdin_data=content)
  558. def _file_has_bom(self, path: str, pre_content: Optional[str] = None) -> bool:
  559. """Whether the on-disk file starts with a UTF-8 BOM. ALWAYS probes disk:
  560. ``pre_content`` usually comes from ``read_file_raw``, which strips BOMs, so
  561. trusting it would silently drop the marker on rewrite. Missing → False."""
  562. head_result = self._head(path, 3)
  563. return head_result.exit_code == 0 and _has_bom(head_result.stdout)
  564. def _unified_diff(self, old_content: str, new_content: str, filename: str) -> str:
  565. return ''.join(difflib.unified_diff(
  566. old_content.splitlines(keepends=True), new_content.splitlines(keepends=True),
  567. fromfile=f"a/{filename}", tofile=f"b/{filename}"))
  568. # --- READ ---------------------------------------------------------------
  569. @staticmethod
  570. def _not_regular_error(path: str) -> ReadResult:
  571. """Error for a path that exists but would block if read."""
  572. return ReadResult(error=(
  573. f"Cannot read '{path}': not a regular file (directory, dangling symlink, "
  574. "FIFO, socket, or device). Reading it could block indefinitely."))
  575. def _probe_regular_file(self, path: str) -> tuple[int, str]:
  576. """Byte size of a REGULAR file: ``(file_size, status)`` with status ``"ok"``,
  577. ``"missing"``, ``"not_regular"``, ``"bad_size"`` (unparseable ``wc``),
  578. ``"env_unavailable"``, or the named working-directory error when the exec
  579. wrapper itself failed (``_env_unavailable_error`` surfaces it verbatim).
  580. ``wc -c <`` on a writer-less FIFO/socket//dev/zero blocks forever and a
  581. name-based blocklist can't cover a FIFO (a file TYPE at any path); ``[ -f ]``
  582. is a stat (symlinks followed) so it answers without touching content. A dangling
  583. symlink is ``not_regular``, never ``missing``: the entry exists, and a writer
  584. told the path is free would follow the link and create its target."""
  585. arg = self._escape_shell_arg(path)
  586. # A missing path ECHOES its sentinel: a non-zero exit with no sentinel means the shell itself did
  587. # not run (container still starting, removed out-of-band, transport down) — not a missing file.
  588. # Reporting that as "File not found" made the model trust a false negative for the whole session.
  589. stat_result = self._exec(
  590. f"if [ -f {arg} ]; then wc -c < {arg} 2>/dev/null; "
  591. f"elif [ -e {arg} ] || [ -L {arg} ]; then echo {NOT_REGULAR_SENTINEL}; "
  592. f"else echo {MISSING_SENTINEL}; fi")
  593. stat_output = _strip_terminal_fence_leaks(stat_result.stdout).strip()
  594. if stat_output == MISSING_SENTINEL:
  595. return 0, "missing"
  596. if stat_output == NOT_REGULAR_SENTINEL:
  597. return 0, "not_regular"
  598. if stat_result.exit_code != 0:
  599. return 0, stat_result.cwd_error or "env_unavailable"
  600. try:
  601. return int(stat_output), "ok"
  602. except ValueError:
  603. return 0, "bad_size"
  604. def _env_unavailable_error(self, path: str, status: str = "env_unavailable") -> ReadResult:
  605. if status != "env_unavailable":
  606. return ReadResult(error=status)
  607. return ReadResult(error=(f"Terminal environment unavailable: could not stat {path} "
  608. "(the sandbox may still be starting or was removed). Retry shortly."))
  609. def _detect_binary(self, path: str) -> tuple[bool, Optional[bytes]]:
  610. """``(is_binary, sample_bytes)`` — byte-layer detection when the transport
  611. allows (base64 sample), else the legacy text heuristic (sample is None)."""
  612. sample_bytes = self._sample_file_bytes(path)
  613. if sample_bytes is not None:
  614. ext_binary = has_binary_extension(path)
  615. return ext_binary or self._is_likely_binary_bytes(sample_bytes), sample_bytes
  616. sample_output = _strip_terminal_fence_leaks(self._head(path, 1000).stdout)
  617. return self._is_likely_binary(path, sample_output), None
  618. # UTF-16 rescue: trust a BOM first, then zero-byte PARITY (not density, so
  619. # mixed Latin/CJK still detects): zeros at odd indices → UTF-16 LE, at even
  620. # → BE; both parities or a single zero → real binary. Legacy 8-bit
  621. # encodings (GBK, Big5) are never guessed — a wrong silent guess is worse
  622. # than a clear refusal.
  623. # UTF-16 rescue constants (ported from MoonshotAI/kimi-code#2647, detection derived from VS Code's
  624. # encoding sniffer): sample the leading bytes; trust a BOM first, then a zero-byte parity heuristic —
  625. # zeros clustering at odd indices mean UTF-16 LE (`0xAA 0x00`), at even indices UTF-16 BE (`0x00 0xAA`).
  626. _UTF16_MAX_BYTES = 10 * 1024 * 1024
  627. _UTF16_SAMPLE_BYTES = 512
  628. def _python_interpreter_cmd(self) -> str:
  629. """Return a shell-safe Python interpreter for the terminal backend.
  630. On the local backend ``sys.executable`` is always a working
  631. interpreter — and on Windows it dodges the Microsoft Store
  632. ``python``/``python3`` alias stub (exit 49, "Python was not
  633. found") that otherwise breaks inline ``-c`` snippets. Remote
  634. backends (docker/ssh/...) don't have the agent's interpreter, so
  635. they fall back to ``python3`` on their own PATH (the ``python``
  636. fallback is handled at the call site).
  637. """
  638. if self._lsp_local_only():
  639. return self._escape_shell_arg(sys.executable)
  640. return "python3"
  641. def _exec_python_snippet(self, snippet: str, py: str | None = None) -> ExecuteResult:
  642. """Run a Python ``snippet`` in the terminal backend's interpreter.
  643. Base64-encodes the snippet so it survives every shell/quoting layer
  644. as pure ASCII: Windows ``subprocess`` list-arg quoting and ``bash``
  645. double-quote processing both eat backslashes, which otherwise
  646. corrupts Windows paths (``C:\\Users\\x``) and byte literals
  647. (``b'\\xfe\\xff'``) embedded in a ``-c`` program. ``exec`` decodes
  648. and runs it unchanged.
  649. """
  650. encoded = base64.b64encode(snippet.encode("utf-8")).decode("ascii")
  651. if py is None:
  652. py = self._python_interpreter_cmd()
  653. return self._exec(
  654. f"{py} -c \"import base64; exec(base64.b64decode('{encoded}').decode())\""
  655. )
  656. def _try_read_utf16(self, path: str, offset: int, limit: int,
  657. file_size: int) -> Optional[ReadResult]:
  658. """Read ``path`` as UTF-16 transcoded to UTF-8, or None (caller falls back
  659. to the binary-file error). Skips known-binary extensions and files over
  660. 10 MiB. ``path`` must already be expanded."""
  661. if has_binary_extension(path) or file_size > self._UTF16_MAX_BYTES:
  662. return None
  663. snippet = (
  664. "import sys, json, os\n"
  665. f"p = {json.dumps(path)}\n"
  666. f"offset = {int(offset)}\n"
  667. f"limit = {int(limit)}\n"
  668. f"MAX = {self._UTF16_MAX_BYTES}\n"
  669. f"SAMPLE = {self._UTF16_SAMPLE_BYTES}\n"
  670. "try:\n"
  671. " size = os.path.getsize(p)\n"
  672. " if size > MAX:\n"
  673. " print('HERMES_UTF16:NO'); sys.exit(0)\n"
  674. " with open(p, 'rb') as f:\n"
  675. " data = f.read()\n"
  676. " sample = data[:SAMPLE]\n"
  677. " enc = None\n"
  678. " if sample[:2] == b'\\xfe\\xff':\n"
  679. " enc = 'utf-16-be'\n"
  680. " elif sample[:2] == b'\\xff\\xfe':\n"
  681. " enc = 'utf-16-le'\n"
  682. " else:\n"
  683. " odd = sum(1 for i in range(1, len(sample), 2) if sample[i] == 0)\n"
  684. " even = sum(1 for i in range(0, len(sample), 2) if sample[i] == 0)\n"
  685. " if even == 0 and odd >= 2:\n"
  686. " enc = 'utf-16-le'\n"
  687. " elif odd == 0 and even >= 2:\n"
  688. " enc = 'utf-16-be'\n"
  689. " if enc is None:\n"
  690. " print('HERMES_UTF16:NO'); sys.exit(0)\n"
  691. " text = data.decode(enc, 'replace')\n"
  692. " if text[:1] == '\\ufeff':\n"
  693. " text = text[1:]\n"
  694. " text = text.replace('\\r\\n', '\\n')\n"
  695. " lines = text.split('\\n')\n"
  696. " total = len(lines)\n"
  697. " sel = lines[offset - 1: offset - 1 + limit]\n"
  698. " out = {'total_lines': total, 'encoding': enc,\n"
  699. " 'content': '\\n'.join(sel)}\n"
  700. " print('HERMES_UTF16:OK')\n"
  701. " print(json.dumps(out, ensure_ascii=True))\n"
  702. "except Exception:\n"
  703. " print('HERMES_UTF16:NO'); sys.exit(0)\n"
  704. )
  705. result = self._exec_python_snippet(snippet)
  706. if result.exit_code != 0 and "python3" in (result.stdout or ""):
  707. result = self._exec_python_snippet(snippet, py="python")
  708. stdout = _strip_terminal_fence_leaks(result.stdout or "")
  709. marker = stdout.find("HERMES_UTF16:OK")
  710. if result.exit_code != 0 or marker < 0:
  711. return None
  712. payload = stdout[marker + len("HERMES_UTF16:OK"):].strip()
  713. try:
  714. data = json.loads(payload.split("\n", 1)[0] if "\n" in payload else payload)
  715. content = data["content"]
  716. total_lines = int(data["total_lines"])
  717. encoding = str(data.get("encoding", "utf-16"))
  718. except (ValueError, KeyError, TypeError):
  719. return None
  720. end_line = offset + limit - 1
  721. truncated = total_lines > end_line
  722. hint_parts = [f"Transcoded from {encoding.upper()} to UTF-8 for display. "
  723. "Text edits via patch/write_file would re-encode as UTF-8."]
  724. if truncated:
  725. hint_parts.append(
  726. f"Use offset={end_line + 1} to continue reading "
  727. f"(showing {offset}-{end_line} of {total_lines} lines)")
  728. from tools.tool_output_limits import get_max_line_length
  729. max_line_length = get_max_line_length()
  730. truncated_lines = any(len(line) > max_line_length for line in content.split('\n'))
  731. return ReadResult(
  732. content=self._add_line_numbers(content, offset), total_lines=total_lines,
  733. file_size=file_size, truncated=truncated, hint=" ".join(hint_parts),
  734. truncated_lines=True if truncated_lines else None)
  735. def read_file(self, path: str, offset: int = 1, limit: int = 2000) -> ReadResult:
  736. """Read a file with pagination, binary detection, and line numbers.
  737. ``offset`` is 1-indexed; ``limit`` is clamped by ``normalize_read_pagination``.
  738. One shell round-trip answers every question the read needs (existence, size,
  739. binary sample, page, line count, trailing newline; see ``_read_probe_cmd``).
  740. An unparseable reply falls back to ``_read_file_sequential`` (one probe per
  741. question), so an exotic shell can never do worse than before. On a local
  742. POSIX environment the read never touches the shell (``_read_file_native``).
  743. """
  744. path = self._expand_path(path) # before shell escaping: ~ doesn't expand in quotes
  745. offset, limit = normalize_read_pagination(offset, limit)
  746. if self._native_read_enabled():
  747. return self._read_file_native(path, offset, limit)
  748. # Images / known-binary extensions never inline content; the sequential
  749. # path stops at the probes for them, so don't stream their bytes.
  750. if self._is_image(path) or has_binary_extension(path):
  751. return self._read_file_sequential(path, offset, limit)
  752. from tools.tool_output_limits import get_max_line_length
  753. line_clamp_bytes = 4 * get_max_line_length() + 1
  754. end_line = offset + limit - 1
  755. sentinel = _new_sentinel(_READ_SENTINEL_PREFIX)
  756. probe = self._exec(self._read_probe_cmd(path, offset, end_line, line_clamp_bytes, sentinel))
  757. output = probe.stdout or ""
  758. if sentinel not in output:
  759. # Single-line replies: the path is missing or not a regular file.
  760. marker = _strip_terminal_fence_leaks(output).strip()
  761. if marker == MISSING_SENTINEL:
  762. return self._read_file_missing(path, offset, limit)
  763. if marker == NOT_REGULAR_SENTINEL:
  764. return self._not_regular_error(path)
  765. logger.debug(
  766. "read_file: compound probe reply for %s has no sentinel "
  767. "(exit %s, %d chars); falling back to sequential probes",
  768. path, probe.exit_code, len(output))
  769. return self._read_file_sequential(path, offset, limit)
  770. segments = _split_segments(output, sentinel)
  771. if probe.exit_code != 0 or len(segments) != 6:
  772. logger.debug(
  773. "read_file: compound probe for %s returned exit %s with %d "
  774. "segments (want 6); falling back to sequential probes",
  775. path, probe.exit_code, len(segments))
  776. return self._read_file_sequential(path, offset, limit)
  777. size_seg, sample_seg, page_seg, wc_seg, tail_seg, status_seg = segments
  778. status = _strip_terminal_fence_leaks(status_seg).split()
  779. try:
  780. sample_rc, read_rc = int(status[0]), int(status[1])
  781. except (IndexError, ValueError):
  782. logger.debug(
  783. "read_file: compound probe for %s has unparseable status %r; "
  784. "falling back to sequential probes", path, status_seg[-40:])
  785. return self._read_file_sequential(path, offset, limit)
  786. try:
  787. file_size = int(_strip_terminal_fence_leaks(size_seg).strip())
  788. except ValueError:
  789. file_size = 0
  790. # Byte-layer binary detection when base64 was available, else the legacy
  791. # text heuristic over a plain sample (one extra round-trip, shells without base64).
  792. sample_bytes = self._decode_base64_sample(sample_seg) if sample_rc == 0 else None
  793. if sample_bytes is not None:
  794. is_binary = self._is_likely_binary_bytes(sample_bytes)
  795. else:
  796. logger.debug(
  797. "read_file: no usable base64 sample for %s (base64 exit %s); "
  798. "paying one extra round-trip for the text heuristic", path, sample_rc)
  799. sample_output = _strip_terminal_fence_leaks(self._head(path, 1000).stdout)
  800. is_binary = self._is_likely_binary(path, sample_output)
  801. if is_binary:
  802. return self._read_binary_file(path, offset, limit, file_size, sample_bytes)
  803. if read_rc != 0:
  804. return ReadResult(error=f"Failed to read file: {_strip_terminal_fence_leaks(page_seg)}")
  805. read_output = _strip_terminal_fence_leaks(page_seg)
  806. try:
  807. total_lines = int(_strip_terminal_fence_leaks(wc_seg).strip())
  808. except ValueError:
  809. total_lines = 0
  810. tail_flag = _strip_terminal_fence_leaks(tail_seg).strip()
  811. file_ends_with_newline = tail_flag == "1" if tail_flag in ("0", "1") else None
  812. return self._assemble_read_result(
  813. read_output, offset=offset, end_line=end_line, total_lines=total_lines,
  814. file_size=file_size, file_ends_with_newline=file_ends_with_newline)
  815. def _native_read_enabled(self) -> bool:
  816. """Whether ``read_file`` and ``search_files`` may bypass the shell: only POSIX + a backend whose
  817. files are the host's (``uses_host_filesystem``: the file is on this host, path already native;
  818. Windows keeps the shell path since file_operations holds Git-Bash-style paths there).
  819. ``HERMES_NATIVE_FILE_READ=0`` turns the fast path off."""
  820. flag = os.environ.get("HERMES_NATIVE_FILE_READ", "1").strip().lower()
  821. if flag in ("0", "false", "no", "off"):
  822. return False
  823. # Same "are these the host's files" test the LSP path uses; an attribute read
  824. # and self.env is never rebound, so nothing to memoize.
  825. return sys.platform != "win32" and self._lsp_local_only()
  826. def _native_path(self, path: str) -> str:
  827. """``path`` resolved the way the shell resolves it: against the live backend cwd."""
  828. return path if os.path.isabs(path) else os.path.join(getattr(self.env, "cwd", None) or self.cwd, path)
  829. def _read_file_native(self, path: str, offset: int, limit: int) -> ReadResult:
  830. """``read_file`` without a shell — same contract as the shell path, byte for
  831. byte. ``os.stat`` is the ``[ -f ]`` guard (a stat, never an open, so FIFOs and
  832. devices are refused before anything touches them); the first 1000 bytes drive
  833. the byte-layer binary check; the page is produced exactly as
  834. ``sed -n 'a,bp' | cut -b1-N`` prints it (each line clamped to N bytes and
  835. newline-terminated) then decoded with errors="replace" like the transport.
  836. One chunked pass counts lines and collects the page, so neither the file nor
  837. a pathological line is ever held whole. ``path`` is already expanded; any
  838. unexpected OSError hands over to the shell path."""
  839. import stat as _stat
  840. full = path if os.path.isabs(path) else os.path.join(
  841. getattr(self.env, "cwd", None) or self.cwd, path)
  842. try:
  843. st = os.stat(full)
  844. except (FileNotFoundError, NotADirectoryError):
  845. if os.path.islink(full): # dangling: an entry, not an absent path (``_probe_regular_file``)
  846. return self._not_regular_error(path)
  847. return self._read_file_missing(path, offset, limit)
  848. except OSError:
  849. return self._read_file_sequential(path, offset, limit)
  850. if not _stat.S_ISREG(st.st_mode):
  851. return self._not_regular_error(path)
  852. file_size = st.st_size
  853. if self._is_image(path):
  854. return self._image_redirect_result(file_size)
  855. from tools.tool_output_limits import get_max_line_length
  856. clamp = 4 * get_max_line_length() + 1
  857. end_line = offset + limit - 1
  858. page: list[bytes] = []
  859. total_lines = 0
  860. lineno = 1 # the line currently being scanned
  861. kept = bytearray() # first ``clamp`` bytes of that line
  862. have_partial = False # that line has bytes but no newline yet
  863. last_byte = b""
  864. digest = hashlib.sha256()
  865. try:
  866. with open(full, "rb") as fh:
  867. sample = fh.read(1000)
  868. ext_binary = has_binary_extension(path)
  869. if ext_binary or self._is_likely_binary_bytes(sample):
  870. return self._read_binary_file(path, offset, limit, file_size, sample)
  871. fh.seek(0)
  872. while True:
  873. chunk = fh.read(1 << 20)
  874. if not chunk:
  875. break
  876. digest.update(chunk)
  877. last_byte = chunk[-1:]
  878. if lineno > end_line:
  879. # Past the window: only the line count and trailing byte
  880. # are needed, so let memchr do it instead of per-line work.
  881. total_lines += chunk.count(b"\n")
  882. have_partial = chunk[-1:] != b"\n"
  883. continue
  884. pos, n = 0, len(chunk)
  885. while pos < n:
  886. nl = chunk.find(b"\n", pos)
  887. in_page = offset <= lineno <= end_line
  888. if nl < 0:
  889. if in_page and len(kept) < clamp:
  890. kept += chunk[pos:pos + (clamp - len(kept))]
  891. have_partial = True
  892. break
  893. if in_page:
  894. if len(kept) < clamp:
  895. kept += chunk[pos:min(nl, pos + (clamp - len(kept)))]
  896. page.append(bytes(kept) + b"\n")
  897. kept = bytearray()
  898. have_partial = False
  899. total_lines += 1
  900. lineno += 1
  901. pos = nl + 1
  902. except OSError:
  903. return self._read_file_sequential(path, offset, limit)
  904. if have_partial and offset <= lineno <= end_line:
  905. # ``sed`` prints a final line that lacks a newline; ``cut`` adds one.
  906. page.append(bytes(kept) + b"\n")
  907. read_output = _strip_terminal_fence_leaks(b"".join(page).decode("utf-8", errors="replace"))
  908. result = self._assemble_read_result(
  909. read_output, offset=offset, end_line=end_line, total_lines=total_lines,
  910. file_size=file_size,
  911. file_ends_with_newline=(last_byte == b"\n") if file_size else None)
  912. result._snapshot = (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns, digest.digest())
  913. return result
  914. @staticmethod
  915. def _image_redirect_result(file_size: int) -> ReadResult:
  916. return ReadResult(
  917. is_image=True, is_binary=True, file_size=file_size,
  918. hint=(
  919. "Image file detected. Automatically redirected to vision_analyze tool. "
  920. "Use vision_analyze with this file path to inspect the image contents."))
  921. def _read_probe_cmd(self, path: str, offset: int, end_line: int,
  922. line_clamp_bytes: int, sentinel: str) -> str:
  923. """One shell command answering every question ``read_file`` asks: six
  924. segments each closed by a ``sentinel`` line — byte size, base64 of the first
  925. 1000 bytes, the ``sed | cut`` page, ``wc -l``, whether the last byte is a
  926. newline, then the base64 and page pipeline statuses. Probes run only inside
  927. ``[ -f ]`` (stat-not-open, like ``_probe_regular_file``) so a FIFO/device never
  928. reaches ``head``/``sed``. A missing path echoes ``MISSING_SENTINEL`` (a compound
  929. command only reports its last status). Every stage silences stderr: the local
  930. backend merges stderr into stdout and a stray diagnostic would land inside a
  931. segment. The byte clamp is ``4 * max_line_length + 1``; see ``_read_file_sequential``."""
  932. arg = self._escape_shell_arg(path)
  933. mark = f"echo {sentinel}"
  934. return (
  935. f"if [ -f {arg} ]; then "
  936. f"wc -c < {arg} 2>/dev/null; {mark}; "
  937. f"head -c 1000 {arg} 2>/dev/null | base64 2>/dev/null; __hs=$?; {mark}; "
  938. f"sed -n '{offset},{end_line}p' {arg} 2>/dev/null"
  939. f" | cut -b1-{line_clamp_bytes} 2>/dev/null; __hr=$?; {mark}; "
  940. f"wc -l < {arg} 2>/dev/null; {mark}; "
  941. f"tail -c 1 {arg} 2>/dev/null | wc -l; {mark}; "
  942. f'echo "$__hs $__hr"; '
  943. f"elif [ -e {arg} ] || [ -L {arg} ]; then echo {NOT_REGULAR_SENTINEL}; "
  944. f"else echo {MISSING_SENTINEL}; fi")
  945. def _read_file_missing(self, path: str, offset: int, limit: int) -> ReadResult:
  946. """Not-found recovery shared by every read path. Unicode-equivalent spellings
  947. (NFC/NFD, confusable spaces/quotes) render identically, so the model can never
  948. discover the byte mismatch by retyping — retrying is the tool's job. No
  949. equivalent spelling → suggest similar files."""
  950. variant = self._unicode_variant_match(path)
  951. if variant is not None:
  952. result = self.read_file(variant, offset=offset, limit=limit)
  953. note = (
  954. f"Note: '{path}' not found byte-for-byte; resolved to "
  955. f"the unicode-equivalent file '{variant}' (invisible "
  956. "encoding difference: NFC/NFD or special space/quote "
  957. "characters).")
  958. result.hint = f"{note} {result.hint}" if result.hint else note
  959. return result
  960. return self._suggest_similar_files(path)
  961. def _read_binary_file(self, path: str, offset: int, limit: int,
  962. file_size: int, sample_bytes: Optional[bytes]) -> ReadResult:
  963. """Binary branch shared by every read path: UTF-16 text (Notepad, PowerShell
  964. ``>``) trips the binary guard; transcode it, else refuse with the type name.
  965. UTF-16 rescue (ported from MoonshotAI/kimi-code#2647): the terminal env decodes stdout as UTF-8 with
  966. errors="replace", so a UTF-16 text file (Windows Notepad .txt, PowerShell `>` redirects) arrives
  967. mangled with U+FFFD and trips the binary guard. Probe the raw bytes via the backend's Python and
  968. transcode to UTF-8 when a BOM or the zero-byte parity heuristic identifies UTF-16.
  969. """
  970. utf16_result = self._try_read_utf16(path, offset, limit, file_size)
  971. if utf16_result is not None:
  972. return utf16_result
  973. return ReadResult(
  974. is_binary=True, file_size=file_size,
  975. error=describe_binary_file(sample_bytes, file_size))
  976. def _read_file_sequential(self, path: str, offset: int, limit: int) -> ReadResult:
  977. """One-probe-per-call read: the pre-compound form, kept as fallback for
  978. image / known-binary extensions and unparseable compound replies. ``path`` is
  979. already expanded and ``offset``/``limit`` normalized."""
  980. file_size, status = self._probe_regular_file(path)
  981. if status == "missing":
  982. return self._read_file_missing(path, offset, limit)
  983. if status == "not_regular":
  984. return self._not_regular_error(path)
  985. if status not in ("ok", "bad_size"):
  986. return self._env_unavailable_error(path, status)
  987. if self._is_image(path): # never inlined — redirect to the vision tool
  988. return self._image_redirect_result(file_size)
  989. is_binary, sample_bytes = self._detect_binary(path)
  990. if is_binary:
  991. return self._read_binary_file(path, offset, limit, file_size, sample_bytes)
  992. # Clamp each line to a byte budget IN THE SHELL so a 400MB single-line file
  993. # never crosses the exec transport. 4*max+1 BYTES (not max+1): ``cut -b`` can
  994. # split a multibyte codepoint, and a tighter byte clamp would yield fewer
  995. # CHARS than max so the Python clamp in _add_line_numbers would never fire
  996. # (silent truncation). UTF-8 codepoints are ≤4 bytes, so every over-long
  997. # line still trips the char clamp, which also drops a boundary-split U+FFFD.
  998. from tools.tool_output_limits import get_max_line_length
  999. line_clamp_bytes = 4 * get_max_line_length() + 1
  1000. end_line = offset + limit - 1
  1001. read_result = self._exec(
  1002. f"sed -n '{offset},{end_line}p' {self._escape_shell_arg(path)}"
  1003. f" | cut -b1-{line_clamp_bytes}")
  1004. if read_result.exit_code != 0:
  1005. return ReadResult(error=f"Failed to read file: {read_result.stdout}")
  1006. read_output = _strip_terminal_fence_leaks(read_result.stdout)
  1007. wc_result = self._exec(f"wc -l < {self._escape_shell_arg(path)}")
  1008. try:
  1009. total_lines = int(_strip_terminal_fence_leaks(wc_result.stdout).strip())
  1010. except ValueError:
  1011. total_lines = 0
  1012. # Only the page reaching the file's final line can carry the ``cut`` newline
  1013. # artifact (see _assemble_read_result); probe the last byte just for that case.
  1014. file_ends_with_newline: Optional[bool] = None
  1015. if not total_lines > end_line and read_output.endswith('\n'):
  1016. tail_result = self._exec(f"tail -c 1 {self._escape_shell_arg(path)} | wc -l")
  1017. if tail_result.exit_code == 0:
  1018. file_ends_with_newline = _strip_terminal_fence_leaks(tail_result.stdout).strip() != "0"
  1019. return self._assemble_read_result(
  1020. read_output, offset=offset, end_line=end_line, total_lines=total_lines,
  1021. file_size=file_size, file_ends_with_newline=file_ends_with_newline)
  1022. def _assemble_read_result(self, read_output: str, *, offset: int, end_line: int,
  1023. total_lines: int, file_size: int,
  1024. file_ends_with_newline: Optional[bool]) -> ReadResult:
  1025. """Turn a raw ``sed | cut`` page into the final ``ReadResult``. Shared by every
  1026. read path so the BOM strip, pagination hint, ``cut`` newline-artifact fix and
  1027. the ambiguous-silence guards never drift apart. ``file_ends_with_newline`` is
  1028. None when the caller could not tell (artifact left alone, as before)."""
  1029. # ``wc -l`` counts newlines, not lines: a nonempty file whose last byte
  1030. # is not a newline holds one more line than the count (#3907). Adjust
  1031. # here — the single choke point — so total_lines, truncation, and the
  1032. # past-EOF guard agree on every read path (compound, sequential, native).
  1033. if file_size > 0 and file_ends_with_newline is False:
  1034. total_lines += 1
  1035. if offset == 1: # only the first chunk can carry a BOM (byte 0)
  1036. read_output, _ = _strip_bom(read_output)
  1037. truncated = total_lines > end_line
  1038. hint = None
  1039. if truncated:
  1040. hint = f"Use offset={end_line + 1} to continue reading (showing {offset}-{end_line} of {total_lines} lines)"
  1041. # ``cut`` always newline-terminates, so a file without a trailing newline
  1042. # would grow a phantom empty last line; strip it when the last byte says so.
  1043. if not truncated and read_output.endswith('\n') and file_ends_with_newline is False:
  1044. read_output = read_output[:-1]
  1045. # Empty content is indistinguishable from a broken tool: name the dead end.
  1046. if file_size == 0:
  1047. return ReadResult(content="", total_lines=0, file_size=0, hint="File is empty (0 bytes).")
  1048. if offset > total_lines > 0:
  1049. return ReadResult(
  1050. content="", total_lines=total_lines, file_size=file_size,
  1051. hint=(
  1052. f"Note: offset {offset} is beyond the end of the file "
  1053. f"({total_lines} lines total). Retry with offset <= "
  1054. f"{total_lines}."))
  1055. from tools.tool_output_limits import get_max_line_length
  1056. max_line_length = get_max_line_length()
  1057. truncated_lines = any(len(line) > max_line_length for line in read_output.split('\n'))
  1058. return ReadResult(
  1059. content=self._add_line_numbers(read_output, offset), total_lines=total_lines,
  1060. file_size=file_size, truncated=truncated, hint=hint,
  1061. truncated_lines=True if truncated_lines else None)
  1062. # Confusable characters seen in real filenames, collapsed after NFC.
  1063. _CONFUSABLES = (
  1064. ("\u202f", " "), # narrow no-break space (macOS screenshots)
  1065. ("\u00a0", " "), # no-break space
  1066. ("\u2019", "'"), # right single quotation mark (Finder)
  1067. ("\u2018", "'"), # left single quotation mark
  1068. )
  1069. def _unicode_variant_match(self, path: str) -> Optional[str]:
  1070. """On-disk spelling of a file whose name is unicode-equivalent to ``path``
  1071. (NFC/NFD, confusable spaces/quotes). Returns the entry only when EXACTLY one
  1072. matches — several candidates = homoglyph collision, guessing would read the
  1073. wrong file."""
  1074. dir_path = os.path.dirname(path) or "."
  1075. filename = os.path.basename(path)
  1076. if not filename:
  1077. return None
  1078. def _canon(name: str) -> str:
  1079. out = unicodedata.normalize("NFC", name)
  1080. for src, dst in self._CONFUSABLES:
  1081. out = out.replace(src, dst)
  1082. return out
  1083. target = _canon(filename)
  1084. ls_result = self._exec(f"ls -1 {self._escape_shell_arg(dir_path)} 2>/dev/null")
  1085. if ls_result.exit_code != 0 or not ls_result.stdout.strip():
  1086. return None
  1087. candidates = [
  1088. entry for entry in _strip_terminal_fence_leaks(ls_result.stdout).splitlines()
  1089. if entry and entry != filename and _canon(entry) == target]
  1090. if len(candidates) == 1:
  1091. return os.path.join(dir_path, candidates[0]) if dir_path != "." or "/" in path else candidates[0]
  1092. return None
  1093. def _suggest_similar_files(self, path: str) -> ReadResult:
  1094. """"File not found" result listing up to 5 similar names from the same directory."""
  1095. dir_path = os.path.dirname(path) or "."
  1096. filename = os.path.basename(path)
  1097. basename_no_ext = os.path.splitext(filename)[0].lower()
  1098. ext = os.path.splitext(filename)[1].lower()
  1099. lower_name = filename.lower()
  1100. ls_result = self._exec(f"ls -1 {self._escape_shell_arg(dir_path)} 2>/dev/null | head -50")
  1101. scored: list = [] # (score, filepath) — higher is better
  1102. if ls_result.exit_code == 0 and ls_result.stdout.strip():
  1103. for f in ls_result.stdout.strip().split('\n'):
  1104. if not f:
  1105. continue
  1106. lf = f.lower()
  1107. score = 0
  1108. if lf == lower_name:
  1109. score = 100
  1110. elif os.path.splitext(f)[0].lower() == basename_no_ext: # config.yml vs config.yaml
  1111. score = 90
  1112. elif lf.startswith(lower_name) or lower_name.startswith(lf):
  1113. score = 70
  1114. elif lower_name in lf:
  1115. score = 60
  1116. elif lf in lower_name and len(lf) > 2:
  1117. score = 40
  1118. elif ext and os.path.splitext(f)[1].lower() == ext:
  1119. common = set(lower_name) & set(lf)
  1120. if len(common) >= max(len(lower_name), len(lf)) * 0.4:
  1121. score = 30
  1122. # Near-miss spelling (AGENT.md -> AGENTS.md) the substring checks miss.
  1123. if score == 0 and difflib.SequenceMatcher(None, lower_name, lf).ratio() >= 0.8:
  1124. score = 50
  1125. if score > 0:
  1126. scored.append((score, os.path.join(dir_path, f)))
  1127. scored.sort(key=lambda x: -x[0])
  1128. return ReadResult(error=f"File not found: {path}", not_found=True,
  1129. similar_files=[fp for _, fp in scored[:5]])
  1130. def read_file_raw(self, path: str) -> ReadResult:
  1131. """Whole file as a plain string (no pagination/line numbers/clamping)."""
  1132. path = self._expand_path(path)
  1133. file_size, status = self._probe_regular_file(path)
  1134. if status == "missing":
  1135. return self._suggest_similar_files(path)
  1136. if status == "not_regular":
  1137. return self._not_regular_error(path)
  1138. if status not in ("ok", "bad_size"):
  1139. return self._env_unavailable_error(path, status)
  1140. if self._is_image(path):
  1141. return ReadResult(is_image=True, is_binary=True, file_size=file_size)
  1142. is_binary, sample_bytes = self._detect_binary(path)
  1143. if is_binary:
  1144. return ReadResult(is_binary=True, file_size=file_size, error=describe_binary_file(sample_bytes, file_size))
  1145. data, failed = self._read_exact_bytes(path)
  1146. if data is None:
  1147. return ReadResult(error=f"Failed to read file: {failed.stdout}")
  1148. # V4A writes this back, so no display cleanup (nothing has emitted the __HERMES_FENCE_ wrapper it
  1149. # targets since d684d7ee7e; it can only eat the file's own escape bytes), and surrogateescape
  1150. # so write_file's encode restores any byte past the sample that UTF-8 cannot decode (#79178).
  1151. # Strip a leading BOM (a phantom U+FEFF defeats an exact first-line match);
  1152. # write_file re-probes disk and restores it.
  1153. raw_content, _ = _strip_bom(data.decode("utf-8", "surrogateescape"))
  1154. return ReadResult(content=raw_content, file_size=file_size,
  1155. _content_sha256=hashlib.sha256(data).hexdigest())
  1156. def read_file_bytes(self, path: str, max_bytes: Optional[int] = None) -> ReadResult:
  1157. """Read binary-safe bytes (as base64) from any shell-backed environment."""
  1158. path = self._expand_path(path)
  1159. file_size, status = self._probe_regular_file(path)
  1160. if status == "missing":
  1161. return ReadResult(error=f"File not found: {path}", not_found=True)
  1162. if status == "not_regular":
  1163. return self._not_regular_error(path)
  1164. if status not in ("ok", "bad_size"):
  1165. return self._env_unavailable_error(path, status)
  1166. if status == "bad_size":
  1167. return ReadResult(error=f"Could not determine file size: {path}")
  1168. if max_bytes is not None and file_size > max_bytes:
  1169. return ReadResult(
  1170. file_size=file_size,
  1171. error=f"File is too large ({file_size:,} bytes, limit is {max_bytes:,})")
  1172. encoded = self._exec(f"base64 < {self._escape_shell_arg(path)}")
  1173. if encoded.exit_code != 0:
  1174. return ReadResult(error=f"Failed to read binary file: {encoded.stdout}")
  1175. compact = "".join(_strip_terminal_fence_leaks(encoded.stdout).split())
  1176. try:
  1177. base64.b64decode(compact, validate=True)
  1178. except (ValueError, base64.binascii.Error):
  1179. return ReadResult(error=f"Backend returned invalid binary data for: {path}")
  1180. return ReadResult(base64_content=compact, file_size=file_size, is_binary=True)
  1181. def delete_file(self, path: str) -> WriteResult:
  1182. """Delete a single file (directories rejected) via the backend's ``python -c``
  1183. so one code path works on local/docker/ssh AND Windows shells (no ``rm``)."""
  1184. path = self._expand_path(path)
  1185. # Delete removes the directory entry (a symlink itself, not its target), so
  1186. # the guards vet the entry as well as the target it resolves to.
  1187. denied = get_write_denied_error(path, verb="Delete", entry=True)
  1188. if denied:
  1189. return WriteResult(error=denied)
  1190. # Path baked in via repr() for shell-independent quoting; no
  1191. # ``unlink(missing_ok=True)`` (a 3.7 remote interpreter lacks it).
  1192. snippet = (
  1193. "import shutil, pathlib, sys\n"
  1194. f"p = pathlib.Path({json.dumps(path)})\n"
  1195. "recursive = False\n"
  1196. "try:\n"
  1197. " if p.is_dir() and not p.is_symlink():\n"
  1198. " if recursive:\n"
  1199. " shutil.rmtree(p)\n"
  1200. " else:\n"
  1201. " print('is a directory: ' + str(p), file=sys.stderr); sys.exit(2)\n"
  1202. " else:\n"
  1203. " p.unlink()\n"
  1204. "except FileNotFoundError:\n"
  1205. " pass\n"
  1206. "except Exception as exc:\n"
  1207. " print(str(exc), file=sys.stderr); sys.exit(1)\n"
  1208. )
  1209. result = self._exec_python_snippet(snippet)
  1210. # Fall back to ``python`` (remote backends / older systems where there's no
  1211. # ``python3`` symlink but a ``python`` binary is on PATH).
  1212. if result.exit_code != 0 and "python3" in (result.stdout or ""):
  1213. result = self._exec_python_snippet(snippet, py="python")
  1214. if result.exit_code != 0:
  1215. return WriteResult(error=f"Failed to delete {path}: {(result.stdout or '').strip() or 'unknown error'}")
  1216. return WriteResult()
  1217. def move_file(self, src: str, dst: str) -> WriteResult:
  1218. src = self._expand_path(src)
  1219. dst = self._expand_path(dst)
  1220. # Entry-level op like delete_file: vet both entries, not just their targets.
  1221. for p in (src, dst):
  1222. denied = get_write_denied_error(p, verb="Move", entry=True)
  1223. if denied:
  1224. return WriteResult(error=denied)
  1225. result = self._exec(f"mv {self._escape_shell_arg(src)} {self._escape_shell_arg(dst)}")
  1226. if result.exit_code != 0:
  1227. return WriteResult(error=f"Failed to move {src} -> {dst}: {result.stdout}")
  1228. return WriteResult()
  1229. # --- WRITE --------------------------------------------------------------
  1230. # Lone surrogates OUTSIDE the surrogateescape range (U+DC80-U+DCFF round-trips
  1231. # through the pipe; anything else can't be encoded at all).
  1232. _LONE_SURROGATE_RE = re.compile(r"[\ud800-\udc7f\udd00-\udfff]")
  1233. def _reject_unencodable(self, path: str, content: str) -> Optional[WriteResult]:
  1234. """Refuse content with a lone surrogate BEFORE any subprocess: letting it
  1235. reach the pipe spawns a child that hangs or truncates the target via
  1236. empty-stdin ``cat``. A regex scan needs no encode."""
  1237. m = self._LONE_SURROGATE_RE.search(content)
  1238. if m:
  1239. return WriteResult(error=(
  1240. f"Refusing to write '{path}': content contains a lone "
  1241. f"surrogate character ({m.group(0)!r}) that cannot be "
  1242. "encoded as UTF-8. The file was NOT created or modified."))
  1243. return None
  1244. @staticmethod
  1245. def _fail_closed_syntax_error(path: str, ext: str, content: str) -> Optional[WriteResult]:
  1246. """Fail-closed pre-write gate for ``_FAIL_CLOSED_INPROC_EXTS`` (JSON/YAML/TOML):
  1247. a structured-format write that doesn't parse is a corrupt write, so refuse
  1248. before any bytes touch disk. Checked against the RAW content, before the
  1249. BOM/CRLF shims (post-shim linting would false-positive on a BOM-marked file)."""
  1250. linter = LINTERS_INPROC.get(ext) if ext in _FAIL_CLOSED_INPROC_EXTS else None
  1251. if linter is None:
  1252. return None
  1253. ok, err = linter(content)
  1254. if ok or err == "__SKIP__":
  1255. return None
  1256. return WriteResult(error=(
  1257. f"Refusing to write '{path}': candidate content fails "
  1258. f"{ext} syntax validation ({err}). The file was "
  1259. "NOT created or modified. Fix the content and retry."))
  1260. def _write_probe_cmd(self, path: str, sentinel: str, body: Optional[str]) -> str:
  1261. """One shell command for the on-disk questions ``write_file`` asks. Two
  1262. segments closed by a ``sentinel`` line: base64 of the first three bytes (BOM
  1263. detection at the byte layer, same on-disk truth as ``_file_has_bom``), then
  1264. ``body``: ``"cat"`` for the full text when pre-content is wanted, ``"sample"``
  1265. for the 4 KB line-ending sample, or None for nothing. Gated on ``[ -f ]`` so a
  1266. FIFO/device never reaches ``head``/``cat``; a missing path echoes ``MISSING_SENTINEL``."""
  1267. arg = self._escape_shell_arg(path)
  1268. if body == "cat":
  1269. body_cmd = f"cat {arg} 2>/dev/null"
  1270. elif body == "sample":
  1271. body_cmd = f"head -c 4096 {arg} 2>/dev/null"
  1272. else:
  1273. body_cmd = ":"
  1274. return (
  1275. f"if [ -f {arg} ]; then "
  1276. f"head -c 3 {arg} 2>/dev/null | base64 2>/dev/null; echo {sentinel}; "
  1277. f"{body_cmd}; "
  1278. f"else echo {MISSING_SENTINEL}; fi")
  1279. def _probe_write_target(self, path: str, pre_content: Optional[str], want_pre: bool,
  1280. ) -> tuple[bool, Optional[str], Optional[str]]:
  1281. """``(has_bom, pre_content, original_line_ending)`` for ``path`` in ONE
  1282. round-trip (replaces ``cat`` when pre-content is wanted, a ``head -c 4096``
  1283. line-ending sample and a ``head -c 3`` BOM check). Semantics unchanged:
  1284. pre-content is read only when wanted and not supplied; the line ending comes
  1285. from pre-content when there is any, else from the sample; the BOM always comes
  1286. from disk. An unparseable reply falls back to the separate probes."""
  1287. if want_pre and pre_content is None:
  1288. body_mode: Optional[str] = "cat"
  1289. elif not pre_content:
  1290. body_mode = "sample"
  1291. else:
  1292. body_mode = None
  1293. native = _native_io.probe(self._native_path(path), body_mode == "cat") if self._native_read_enabled() else None
  1294. if native is not None:
  1295. has_bom, raw = native
  1296. # Decoded as the shell transport decodes (errors="replace"): same pre_content either way.
  1297. body = raw.decode("utf-8", "replace") if raw else ""
  1298. if body_mode == "cat" and body:
  1299. pre_content = body
  1300. sample = pre_content or (body if body_mode == "sample" else "")
  1301. return has_bom, pre_content, _detect_line_ending(sample) if sample else None
  1302. sentinel = _new_sentinel(_WRITE_SENTINEL_PREFIX)
  1303. probe = self._exec(self._write_probe_cmd(path, sentinel, body_mode))
  1304. output = probe.stdout or ""
  1305. if sentinel not in output:
  1306. if _strip_terminal_fence_leaks(output).strip() == MISSING_SENTINEL:
  1307. ending = _detect_line_ending(pre_content) if pre_content else None
  1308. return False, pre_content, ending
  1309. logger.debug(
  1310. "write_file: pre-write probe reply for %s has no sentinel "
  1311. "(exit %s, %d chars); falling back to sequential probes",
  1312. path, probe.exit_code, len(output))
  1313. return self._probe_write_target_sequential(path, pre_content, want_pre)
  1314. segments = _split_segments(output, sentinel)
  1315. if probe.exit_code != 0 or len(segments) != 2:
  1316. logger.debug(
  1317. "write_file: pre-write probe for %s returned exit %s with %d "
  1318. "segments (want 2); falling back to sequential probes",
  1319. path, probe.exit_code, len(segments))
  1320. return self._probe_write_target_sequential(path, pre_content, want_pre)
  1321. head_seg, body = segments
  1322. head_bytes = self._decode_base64_sample(head_seg)
  1323. if head_bytes is None:
  1324. # No clean base64 on this shell; ask the way we used to.
  1325. logger.debug(
  1326. "write_file: no usable base64 head for %s; paying one extra "
  1327. "round-trip for the BOM probe", path)
  1328. has_bom = self._file_has_bom(path, pre_content)
  1329. else:
  1330. has_bom = head_bytes.startswith(_UTF8_BOM.encode("utf-8"))
  1331. if body_mode == "cat" and body:
  1332. pre_content = body
  1333. if pre_content:
  1334. ending = _detect_line_ending(pre_content)
  1335. elif body_mode == "sample" and body:
  1336. ending = _detect_line_ending(body)
  1337. else:
  1338. ending = None
  1339. return has_bom, pre_content, ending
  1340. def _probe_write_target_sequential(self, path: str, pre_content: Optional[str], want_pre: bool,
  1341. ) -> tuple[bool, Optional[str], Optional[str]]:
  1342. """Pre-compound form of ``_probe_write_target``: one exec per question. A
  1343. failed ``cat`` leaves pre_content None so the lint-delta and LSP consumers
  1344. degrade gracefully."""
  1345. if want_pre and pre_content is None:
  1346. read_result = self._cat(path)
  1347. if read_result.exit_code == 0 and read_result.stdout:
  1348. pre_content = read_result.stdout
  1349. if pre_content:
  1350. ending = _detect_line_ending(pre_content)
  1351. else:
  1352. head = self._head(path, 4096)
  1353. ending = _detect_line_ending(head.stdout) if head.exit_code == 0 and head.stdout else None
  1354. return self._file_has_bom(path, pre_content), pre_content, ending
  1355. def _verify_written_hash(self, path: str, content_bytes: bytes) -> tuple[Optional[bool], Optional[WriteResult]]:
  1356. """Compare the on-disk sha256 to the intended bytes: ``(verified, error)``.
  1357. The explicit flag saves the model a confirming re-read; a mismatch is a hard
  1358. error. ``verified`` is None when the hash could not be taken."""
  1359. try:
  1360. disk_sha = _native_io.sha256_of(self._native_path(path)) if self._native_read_enabled() else None
  1361. hash_result = None if disk_sha else self._exec(f"sha256sum {self._escape_shell_arg(path)} 2>/dev/null")
  1362. if hash_result is not None and hash_result.exit_code == 0 and hash_result.stdout.strip():
  1363. disk_sha = hash_result.stdout.strip().split()[0]
  1364. if disk_sha:
  1365. if disk_sha != hashlib.sha256(content_bytes).hexdigest():
  1366. return False, WriteResult(error=(
  1367. f"Post-write verification failed for {path}: on-disk "
  1368. "content hash differs from the intended write. The "
  1369. "write did not persist correctly — re-read the file "
  1370. "and retry."))
  1371. return True, None
  1372. except Exception:
  1373. pass
  1374. return None, None
  1375. def write_file(self, path: str, content: str, pre_content: Optional[str] = None) -> WriteResult:
  1376. """Write content atomically, creating parent directories as needed.
  1377. Order: deny list → lone-surrogate refusal → fail-closed syntax gate on the
  1378. CANDIDATE content (JSON/YAML/TOML) → one compound on-disk probe
  1379. (pre-content when wanted, CRLF, BOM; see ``_probe_write_target``) →
  1380. JSON NaN/Infinity refusal when the write introduces one → CRLF/BOM
  1381. preservation → atomic write (content rides stdin: no ARG_MAX limit) →
  1382. sha256 verification → lint delta → LSP check when syntax is clean
  1383. (non-blocking; the pre-edit content is its baseline, see ``_maybe_lsp_diagnostics``). ``pre_content``: pre-edit content the
  1384. caller already has (skips the read); BOM detection always probes disk.
  1385. """
  1386. started = time.monotonic()
  1387. path = self._expand_path(path)
  1388. denied = get_write_denied_error(path)
  1389. if denied:
  1390. return WriteResult(error=denied)
  1391. refused = self._reject_unencodable(path, content)
  1392. if refused is not None:
  1393. return refused
  1394. ext = os.path.splitext(path)[1].lower()
  1395. refused = self._fail_closed_syntax_error(path, ext, content)
  1396. if refused is not None:
  1397. return refused
  1398. # Pre-content is read only for extensions in the UNION of in-process lint and
  1399. # LSP coverage (keeps the hot path fast for binaries).
  1400. want_pre = ext in LINTERS_INPROC or self._lsp_handles_extension(ext)
  1401. has_bom, pre_content, original_ending = self._probe_write_target(path, pre_content, want_pre)
  1402. if ext == ".json":
  1403. refused = _refuse_introduced_json_constant(path, content, pre_content)
  1404. if refused is not None:
  1405. return refused
  1406. # read_file strips the BOM and models send bare-LF text, so a round-trip would
  1407. # otherwise normalize CRLF files and drop the BOM (prepend only when absent).
  1408. if original_ending == "\r\n":
  1409. content = _normalize_line_endings(content, "\r\n")
  1410. if has_bom and not _has_bom(content):
  1411. content = _UTF8_BOM + content
  1412. # ``dirs_created`` means "parent dirs ensured" (mkdir -p is folded into
  1413. # _atomic_write; its failure surfaces as the atomic-write error below).
  1414. dirs_created = bool(os.path.dirname(path))
  1415. # surrogateescape is the exact inverse of the decode that may have produced
  1416. # this content, so these are the bytes on disk; the early rejection above
  1417. # guarantees this cannot raise.
  1418. content_bytes = content.encode("utf-8", "surrogateescape")
  1419. write_result = self._atomic_write(path, content)
  1420. if write_result.exit_code != 0:
  1421. return WriteResult(error=f"Failed to write file: {write_result.stdout}")
  1422. content_verified, verify_error = self._verify_written_hash(path, content_bytes)
  1423. if verify_error is not None:
  1424. return verify_error
  1425. lint_result = self._check_lint_delta(path, pre_content=pre_content, post_content=content)
  1426. # LSP diagnostics are a separate channel, fired only when the syntax tier is
  1427. # clean (no point asking an LSP about a file that won't parse).
  1428. lsp_diagnostics: Optional[str] = None
  1429. if lint_result.success or lint_result.skipped:
  1430. lsp_diagnostics = self._maybe_lsp_diagnostics(
  1431. path, pre_content=pre_content, post_content=content, started=started) or None
  1432. return WriteResult(
  1433. bytes_written=len(content_bytes), dirs_created=dirs_created, verified=content_verified,
  1434. _content_sha256=hashlib.sha256(content_bytes).hexdigest(),
  1435. lint=lint_result.to_dict() if lint_result else None, lsp_diagnostics=lsp_diagnostics)
  1436. # --- PATCH (replace mode) -----------------------------------------------
  1437. def _no_match_result(self, path: str, content: str, old_string: str,
  1438. new_string: str, match_count: int, error: Optional[str]) -> PatchResult:
  1439. """PatchResult for a failed fuzzy match. Already-applied detection first: the
  1440. most common production failure is a re-send of an edit that already landed,
  1441. and a success-shaped no-op stops the model burning turns on re-reads.
  1442. Otherwise attach a best-effort "Did you mean?" snippet to the error."""
  1443. from tools.fuzzy_match import format_no_match_hint, is_already_applied
  1444. if is_already_applied(content, old_string, new_string):
  1445. return PatchResult(
  1446. success=True, no_change=True,
  1447. note=(
  1448. f"File already contains the target text — the edit "
  1449. f"appears to be already applied to {path}. No write "
  1450. "performed; do not re-send this patch."))
  1451. err_msg = error or f"Could not find match for old_string in {path}"
  1452. try:
  1453. err_msg += format_no_match_hint(err_msg, match_count, old_string, content)
  1454. except Exception:
  1455. pass
  1456. return PatchResult(error=err_msg)
  1457. def _verify_patch_persisted(self, path: str, new_content: str) -> Optional[PatchResult]:
  1458. """Re-read ``path`` and confirm the intended bytes landed; error result or None.
  1459. Catches silent persistence failures (FS oddities, races, truncated pipe).
  1460. Line endings are normalized first (Windows text-mode ``open()`` writes LF as
  1461. CRLF) and the re-read's BOM stripped (``new_content`` is the BOM-less
  1462. string we matched against)."""
  1463. data, _failed = self._read_exact_bytes(path)
  1464. if data is None:
  1465. return PatchResult(error=f"Post-write verification failed: could not re-read {path}")
  1466. bomless, _ = _strip_bom(data.decode("utf-8", "surrogateescape"))
  1467. on_disk = bomless.replace("\r\n", "\n").replace("\r", "\n")
  1468. intended = new_content.replace("\r\n", "\n").replace("\r", "\n")
  1469. if on_disk != intended:
  1470. return PatchResult(error=(
  1471. f"Post-write verification failed for {path}: on-disk content "
  1472. f"differs from intended write "
  1473. f"(wrote {len(intended)} chars, read back "
  1474. f"{len(on_disk)} chars after normalizing line endings). "
  1475. "The patch did not persist. Re-read the file and try again."))
  1476. return None
  1477. def patch_replace(self, path: str, old_string: str, new_string: str,
  1478. replace_all: bool = False) -> PatchResult:
  1479. """Replace text in a file using fuzzy matching (``old_string`` must be
  1480. unique unless ``replace_all``). Returns a PatchResult with diff + lint."""
  1481. path = self._expand_path(path)
  1482. denied = get_write_denied_error(path)
  1483. if denied:
  1484. return PatchResult(error=denied)
  1485. data, failed = self._read_exact_bytes(path)
  1486. if data is None:
  1487. return PatchResult(error=failed.cwd_error or f"Failed to read file: {path}")
  1488. # Every line the replacement does not touch is written back, so read the exact bytes;
  1489. # surrogateescape lets write_file restore any byte UTF-8 cannot decode (#79178).
  1490. # Match and diff on BOM-stripped content (a phantom U+FEFF defeats an exact
  1491. # first-line match); the raw read becomes write_file's pre_content.
  1492. raw_content = data.decode("utf-8", "surrogateescape")
  1493. content, _ = _strip_bom(raw_content)
  1494. from tools.fuzzy_match import boundary_note, fuzzy_find_and_replace
  1495. correction_notes = []
  1496. new_content, match_count, strategy, error = fuzzy_find_and_replace(
  1497. content, old_string, new_string, replace_all, correction_notes=correction_notes)
  1498. if error or match_count == 0:
  1499. return self._no_match_result(path, content, old_string, new_string, match_count, error)
  1500. # Models send bare-LF old/new strings; normalize the substituted region to
  1501. # the file's ending so CRLF files stay consistent.
  1502. file_ending = _detect_line_ending(content)
  1503. if file_ending:
  1504. new_content = _normalize_line_endings(new_content, file_ending)
  1505. write_result = self.write_file(path, new_content, pre_content=raw_content)
  1506. if write_result.error:
  1507. return PatchResult(error=f"Failed to write changes: {write_result.error}")
  1508. verify_error = self._verify_patch_persisted(path, new_content)
  1509. if verify_error is not None:
  1510. return verify_error
  1511. # write_file already ran the lint delta on these bytes (pre = the raw read).
  1512. return PatchResult(
  1513. success=True, diff=self._unified_diff(content, new_content, path), files_modified=[path],
  1514. lint=write_result.lint,
  1515. note="\n".join(n for n in (*correction_notes, boundary_note([strategy])) if n) or None,
  1516. # From the internal write_file call, whose baseline was the pre-patch content.
  1517. lsp_diagnostics=write_result.lsp_diagnostics,
  1518. _writes=[(path, hashlib.sha256(data).hexdigest(), write_result._content_sha256)])
  1519. def patch_v4a(self, patch_content: str) -> PatchResult:
  1520. """Apply a V4A format patch (``*** Begin Patch`` / ``*** Update File:`` /
  1521. ``@@ hint @@`` hunks / ``*** End Patch``)."""
  1522. from tools.patch_parser import parse_v4a_patch, apply_v4a_operations
  1523. operations, parse_error = parse_v4a_patch(patch_content)
  1524. if parse_error:
  1525. return PatchResult(error=f"Failed to parse patch: {parse_error}")
  1526. return apply_v4a_operations(operations, self)
  1527. # --- SEARCH -------------------------------------------------------------
  1528. def search(self, pattern: str, path: str = ".", target: str = "content",
  1529. file_glob: Optional[str] = None, limit: int = 50, offset: int = 0,
  1530. output_mode: str = "content", context: int = 0,
  1531. order: str = "discovery") -> SearchResult:
  1532. """Search for content (regex, ``target="content"``) or files (glob,
  1533. ``target="files"``). ``output_mode``: "content", "files_only" or "count";
  1534. ``context``: lines of context around matches; ``order``: file-search
  1535. ordering — fast "discovery" or exact "modified" time."""
  1536. offset, limit = normalize_search_pagination(offset, limit)
  1537. if target == "files" and order not in {"discovery", "modified"}:
  1538. return SearchResult(
  1539. error=(f"Invalid file search order {order!r}; expected "
  1540. "'discovery' or 'modified'."))
  1541. path = self._expand_path(path)
  1542. probe = self._path_exists_probe(path)
  1543. exists_probe = probe.stdout
  1544. if probe.cwd_error:
  1545. return SearchResult(error=probe.cwd_error)
  1546. if "exists" not in exists_probe and "not_found" not in exists_probe:
  1547. return SearchResult(error=(f"Terminal environment unavailable: could not stat {path} "
  1548. "(the sandbox may still be starting or was removed). Retry shortly."))
  1549. if "not_found" in exists_probe:
  1550. # Models often pass several paths in one string: search the parts that exist.
  1551. multi = self._try_multi_path_search(
  1552. pattern, path, target, file_glob, limit, offset, output_mode, context, order)
  1553. if multi is not None:
  1554. return multi
  1555. return self._path_not_found_result(path)
  1556. result = self._dispatch_search(pattern, path, target, file_glob, limit, offset,
  1557. output_mode, context, order)
  1558. exclusions = self._macos_search_exclusions(path)
  1559. if exclusions and not result.error:
  1560. skipped = ", ".join(item.split("/")[-1] for item in exclusions)
  1561. protected = (
  1562. "Skipped macOS protected folders during broad search to avoid "
  1563. f"an unattended privacy prompt: {skipped}. Search a protected "
  1564. "folder directly when access is intentional.")
  1565. result.warning = f"{result.warning} {protected}" if result.warning else protected
  1566. return result