file_operations_common.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313
  1. """Result dataclasses and pure text helpers shared by ``tools.file_operations``
  2. and its search/lint mixins. Leaf module (imports nothing from ``tools`` at module
  3. scope) so nothing cycles. The ``to_dict`` output of every class here IS tool
  4. behavior — key names, order and omission rules are pinned by tests and read by
  5. the model.
  6. """
  7. import re
  8. from dataclasses import dataclass, field
  9. from typing import Any, ClassVar, Dict, List, Optional, Tuple
  10. @dataclass
  11. class ReadResult:
  12. """Result from reading a file."""
  13. content: str = ""
  14. total_lines: int = 0
  15. file_size: int = 0
  16. truncated: bool = False
  17. truncated_lines: Optional[bool] = None
  18. hint: Optional[str] = None
  19. is_binary: bool = False
  20. is_image: bool = False
  21. base64_content: Optional[str] = None
  22. mime_type: Optional[str] = None
  23. dimensions: Optional[str] = None # For images: "WIDTHxHEIGHT"
  24. error: Optional[str] = None
  25. #: True only when the path is genuinely absent. An error with this False is a read that
  26. #: FAILED (transport down, no byte transport installed); callers deciding whether a path
  27. #: is free must not read that as "absent". See patch_parser._apply_add.
  28. not_found: bool = False
  29. similar_files: list[str] = field(default_factory=list)
  30. _snapshot: Optional[tuple] = None
  31. # sha256 of the exact on-disk bytes ``read_file_raw`` decoded (BOM included).
  32. _content_sha256: Optional[str] = None
  33. def to_dict(self) -> dict:
  34. return {k: v for k, v in self.__dict__.items() if not k.startswith("_") and v is not None and v != []}
  35. @dataclass
  36. class WriteResult:
  37. """Result from writing a file."""
  38. bytes_written: int = 0
  39. dirs_created: bool = False
  40. # True when the on-disk sha256 matched the intended content; None when the
  41. # backend couldn't verify (no sha256sum). A mismatch is a hard error, never a flag.
  42. verified: Optional[bool] = None
  43. _content_sha256: Optional[str] = None
  44. lint: Optional[dict[str, Any]] = None
  45. # LSP semantic diagnostics, kept separate from ``lint`` (syntax) so the model
  46. # reads the two as independent signals. None when LSP is off/inapplicable.
  47. lsp_diagnostics: Optional[str] = None
  48. error: Optional[str] = None
  49. warning: Optional[str] = None
  50. def to_dict(self) -> dict:
  51. return {k: v for k, v in self.__dict__.items() if not k.startswith("_") and v is not None}
  52. @dataclass
  53. class PatchResult:
  54. """Result from patching a file."""
  55. success: bool = False
  56. diff: str = ""
  57. files_modified: list[str] = field(default_factory=list)
  58. files_created: list[str] = field(default_factory=list)
  59. files_deleted: list[str] = field(default_factory=list)
  60. lint: Optional[dict[str, Any]] = None
  61. lsp_diagnostics: Optional[str] = None # see WriteResult.lsp_diagnostics
  62. error: Optional[str] = None
  63. # Success-shaped no-op: the edit was already present, nothing written; ``note`` says why.
  64. no_change: bool = False
  65. note: Optional[str] = None
  66. # One ``(path, read_sha256, written_sha256)`` per file write, in order: the sha256 of the
  67. # bytes the edit read ("" when it created the file, None when unknown) and of the bytes it
  68. # wrote (None when unknown). patch_tool carries a task's full-file baseline across them.
  69. _writes: list[tuple[str, Optional[str], Optional[str]]] = field(default_factory=list)
  70. # Emission order is part of the output contract.
  71. _DICT_FIELDS: ClassVar[tuple] = (
  72. "diff", "files_modified", "files_created", "files_deleted",
  73. "lint", "lsp_diagnostics", "error",
  74. )
  75. def to_dict(self) -> dict:
  76. result: dict[str, Any] = {"success": self.success}
  77. if self.no_change:
  78. result["no_change"] = True
  79. if self.note:
  80. result["note"] = self.note
  81. for key in self._DICT_FIELDS:
  82. value = getattr(self, key)
  83. if value:
  84. result[key] = value
  85. return result
  86. @dataclass
  87. class SearchMatch:
  88. """A single search match."""
  89. path: str
  90. line_number: int
  91. content: str
  92. mtime: float = 0.0 # Modification time for sorting
  93. @dataclass
  94. class SearchResult:
  95. """Result from searching."""
  96. matches: list[SearchMatch] = field(default_factory=list)
  97. files: list[str] = field(default_factory=list)
  98. counts: dict[str, int] = field(default_factory=dict)
  99. total_count: int = 0
  100. truncated: bool = False
  101. limit_reason: Optional[str] = None
  102. warning: Optional[str] = None
  103. error: Optional[str] = None
  104. # Unreadable paths behind the "Skipped N unreadable path(s)" sentence in ``warning``,
  105. # plus the diagnostics dropped past the runner's cap: a multi-root merge recomposes
  106. # one sentence from this data instead of parsing the text. Never serialized.
  107. skipped_paths: list[str] = field(default_factory=list)
  108. skipped_omitted: int = 0
  109. # Below this many matches the verbose array is already compact enough that
  110. # a path-grouping header would cost more tokens than it saves.
  111. _DENSIFY_MIN_MATCHES: ClassVar[int] = 5
  112. def _densify_matches(self) -> Optional[str]:
  113. """Lossless path-grouped text block: path once, then `` <line>: <content>``
  114. rows. Relies on rg/grep emitting a file's hits consecutively. None when
  115. too few matches to be worth it."""
  116. if len(self.matches) < self._DENSIFY_MIN_MATCHES:
  117. return None
  118. lines: list[str] = []
  119. current_path: Optional[str] = None
  120. for m in self.matches:
  121. if m.path != current_path:
  122. lines.append(m.path)
  123. current_path = m.path
  124. # rstrip only: leading indentation is meaningful code and kept verbatim.
  125. lines.append(f" {m.line_number}: {m.content.rstrip()}")
  126. return "\n".join(lines)
  127. def to_dict(self, densify: bool = False) -> dict:
  128. result: dict[str, object] = {"total_count": self.total_count}
  129. if self.matches:
  130. dense = self._densify_matches() if densify else None
  131. if dense is not None:
  132. # Self-describing so the model never guesses the block's shape.
  133. result["matches_format"] = (
  134. "path-grouped: each file path on its own line, followed by "
  135. "indented '<line>: <content>' rows for matches in that file"
  136. )
  137. result["matches_text"] = dense
  138. else:
  139. result["matches"] = [
  140. {"path": m.path, "line": m.line_number, "content": m.content} for m in self.matches
  141. ]
  142. if self.files:
  143. result["files"] = self.files
  144. if self.counts:
  145. result["counts"] = self.counts
  146. if self.truncated:
  147. result["truncated"] = True
  148. result["total_count_is_lower_bound"] = True
  149. for key in ("limit_reason", "warning", "error"):
  150. value = getattr(self, key)
  151. if value:
  152. result[key] = value
  153. return result
  154. @dataclass
  155. class LintResult:
  156. """Result from linting a file."""
  157. success: bool = True
  158. skipped: bool = False
  159. output: str = ""
  160. message: str = ""
  161. def to_dict(self) -> dict:
  162. if self.skipped:
  163. return {"status": "skipped", "message": self.message}
  164. result = {"status": "ok" if self.success else "error", "output": self.output}
  165. if self.message:
  166. result["message"] = self.message
  167. return result
  168. @dataclass
  169. class ExecuteResult:
  170. """Result from executing a shell command."""
  171. stdout: str = ""
  172. exit_code: int = 0
  173. # Set when the backend's own ``builtin cd -- <cwd> || exit 126`` wrapper failed:
  174. # the command never ran, so the verdict is about the working directory, not
  175. # the requested path (callers must neither cache it nor blame the path).
  176. cwd_error: str = ""
  177. # ---------------------------------------------------------------------------
  178. # Pure text helpers (no I/O)
  179. # ---------------------------------------------------------------------------
  180. _OSC_SEQUENCE_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
  181. _FENCE_MARKER_RE = re.compile(r"'?\x07?__HERMES_FENCE_[A-Za-z0-9]+__\x07?'?")
  182. _CONFLICT_OPEN = re.compile(r"^\s*\d+\|<<<<<<< ", re.MULTILINE)
  183. _CONFLICT_CLOSE = re.compile(r"^\s*\d+\|>>>>>>> ", re.MULTILINE)
  184. def count_conflict_blocks(formatted_content: str) -> int:
  185. """Unresolved git merge-conflict blocks in a ``LINE|CONTENT`` read; 0 when the page has no
  186. balanced ``<<<<<<< `` / ``>>>>>>> `` pair (a lone marker in prose or a test fixture is not a
  187. conflict). Reported on read so the model resolves the conflict instead of editing around it."""
  188. opens = len(_CONFLICT_OPEN.findall(formatted_content))
  189. return min(opens, len(_CONFLICT_CLOSE.findall(formatted_content))) if opens else 0
  190. def _strip_terminal_fence_leaks(text: str) -> str:
  191. """Strip leaked terminal fence wrappers (OSC sequences, fence markers) from
  192. command output; drops lines that were nothing but wrapper."""
  193. if not text:
  194. return text
  195. cleaned_lines: list[str] = []
  196. for line in text.splitlines(keepends=True):
  197. had_terminal_wrapper = "__HERMES_FENCE_" in line or "\x1b]" in line
  198. cleaned = _FENCE_MARKER_RE.sub("", _OSC_SEQUENCE_RE.sub("", line)).replace("\x07", "")
  199. if had_terminal_wrapper and cleaned.strip("'\r\n\t ") == "":
  200. continue
  201. cleaned_lines.append(cleaned)
  202. return "".join(cleaned_lines)
  203. def _detect_line_ending(sample: str) -> Optional[str]:
  204. """Dominant line ending of ``sample`` (``\\r\\n`` if any CRLF in the first 4KB,
  205. else ``\\n``), or None for empty/single-line content. Preserves a file's
  206. endings across write_file/patch: bare-LF tool args would otherwise silently
  207. normalize CRLF files, and patch would produce mixed endings."""
  208. head = sample[:4096] if sample else ""
  209. if "\r\n" in head:
  210. return "\r\n"
  211. if "\n" in head:
  212. return "\n"
  213. return None
  214. def _normalize_line_endings(text: str, target: str) -> str:
  215. """Convert every line ending (CRLF, lone CR, LF) in ``text`` to ``target``.
  216. Idempotent. Collapses to LF first — separate replacements would
  217. double-convert CRLF → LFLF."""
  218. lf_normalized = text.replace("\r\n", "\n").replace("\r", "\n")
  219. if target == "\n":
  220. return lf_normalized
  221. if target == "\r\n":
  222. return lf_normalized.replace("\n", "\r\n")
  223. return text
  224. # UTF-8 BOM (EF BB BF == U+FEFF), prepended by some Windows editors. Stripped on
  225. # read so the model never sees a phantom first character (and patch's first-line
  226. # match works), restored on write when the on-disk file had one.
  227. _UTF8_BOM = "\ufeff"
  228. def _strip_bom(text: str) -> tuple[str, bool]:
  229. """Return (text-without-leading-BOM, had_bom). Only a leading BOM is
  230. stripped; mid-content U+FEFF is legitimate data."""
  231. if _has_bom(text):
  232. return text[len(_UTF8_BOM):], True
  233. return text, False
  234. def _has_bom(text: Optional[str]) -> bool:
  235. """True if ``text`` begins with a UTF-8 BOM."""
  236. return bool(text) and text.startswith(_UTF8_BOM)
  237. # ---------------------------------------------------------------------------
  238. # Pagination clamps
  239. # ---------------------------------------------------------------------------
  240. DEFAULT_READ_OFFSET = 1
  241. DEFAULT_READ_LIMIT = 2000
  242. DEFAULT_SEARCH_OFFSET = 0
  243. DEFAULT_SEARCH_LIMIT = 50
  244. def _coerce_int(value: Any, default: int) -> int:
  245. """Best-effort integer coercion for tool pagination inputs."""
  246. try:
  247. return int(value)
  248. except (TypeError, ValueError):
  249. return default
  250. def normalize_read_pagination(offset: Any = DEFAULT_READ_OFFSET,
  251. limit: Any = DEFAULT_READ_LIMIT) -> tuple[int, int]:
  252. """Clamp read_file pagination so invalid values can never reach a sed range
  253. like ``0,-1p`` (schemas declare bounds, but not every caller enforces them).
  254. The ``limit`` ceiling is ``tool_output.max_lines`` from config.yaml."""
  255. from tools.tool_output_limits import get_max_lines
  256. normalized_offset = max(1, _coerce_int(offset, DEFAULT_READ_OFFSET))
  257. normalized_limit = max(1, min(_coerce_int(limit, DEFAULT_READ_LIMIT), get_max_lines()))
  258. return normalized_offset, normalized_limit
  259. def normalize_search_pagination(offset: Any = DEFAULT_SEARCH_OFFSET,
  260. limit: Any = DEFAULT_SEARCH_LIMIT) -> tuple[int, int]:
  261. """Return safe search pagination bounds for shell head/tail pipelines."""
  262. return max(0, _coerce_int(offset, DEFAULT_SEARCH_OFFSET)), max(1, _coerce_int(limit, DEFAULT_SEARCH_LIMIT))