file_operations_search.py 74 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432
  1. """Content/file search tier for ``tools.file_operations``.
  2. ``ShellFileOperations`` inherits ``SearchMixin``; module-level helpers are pure
  3. (no I/O).
  4. """
  5. import contextlib
  6. import os
  7. import posixpath
  8. import re
  9. import shlex
  10. import subprocess
  11. import sys
  12. import threading
  13. import time
  14. from pathlib import Path
  15. from typing import Any, List, Optional
  16. from agent.search_policy import SEARCH_PRUNE_DIR_NAMES
  17. from tools import interrupt as tool_interrupt
  18. from tools.file_operations_common import ExecuteResult, SearchMatch, SearchResult
  19. _MACOS_TCC_PROTECTED_HOME_DIRS = (
  20. "Desktop", "Documents", "Downloads", "Library", "Movies", "Music", "Pictures",
  21. )
  22. def _macos_protected_search_exclusions(
  23. path: str, *, cwd: Optional[str] = None, home: Optional[str] = None, platform: Optional[str] = None,
  24. ) -> list[str]:
  25. """Protected home dirs (relative to ``path``) below a broad macOS search root.
  26. Only an ANCESTOR search (``$HOME``, ``/Users``) gets exclusions, so recursive
  27. tools never trigger unattended TCC prompts; a search rooted inside a
  28. protected dir stays allowed.
  29. """
  30. if (platform or sys.platform) != "darwin":
  31. return []
  32. root = Path(path).expanduser()
  33. if not root.is_absolute():
  34. root = Path(cwd or os.getcwd()) / root
  35. root = Path(os.path.normpath(str(root)))
  36. home_path = Path(os.path.normpath(str(Path(home or Path.home()).expanduser())))
  37. exclusions: list[str] = []
  38. for dirname in _MACOS_TCC_PROTECTED_HOME_DIRS:
  39. try:
  40. relative = (home_path / dirname).relative_to(root)
  41. except ValueError:
  42. continue
  43. if relative.parts:
  44. exclusions.append(relative.as_posix())
  45. return exclusions
  46. # --- Filename-walk admission: one walk per (backend, root) at a time --------------
  47. _FILENAME_SEARCH_ADMISSION = threading.Condition()
  48. _ACTIVE_FILENAME_SEARCH_ROOTS: set[tuple[str, str, str]] = set()
  49. _FILENAME_SEARCH_WAIT_SECONDS = 0.05
  50. def _normalized_filename_search_root(env: Any, root: str, fallback_cwd: str) -> str:
  51. """Normalize a filename-walk root without resolving remote paths locally."""
  52. from tools.environments.base import env_uses_host_filesystem
  53. from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
  54. cwd = getattr(env, "cwd", None) or fallback_cwd
  55. if env_uses_host_filesystem(env):
  56. if _IS_WINDOWS:
  57. root = _msys_to_windows_path(root)
  58. cwd = _msys_to_windows_path(cwd)
  59. if not os.path.isabs(root):
  60. root = os.path.join(cwd, root)
  61. return os.path.normcase(os.path.abspath(os.path.normpath(root)))
  62. if not posixpath.isabs(root):
  63. root = posixpath.join(cwd, root)
  64. return posixpath.normpath(root)
  65. def _filename_search_root_keys(env: Any, roots: list[str], fallback_cwd: str) -> tuple[tuple[str, str, str], ...]:
  66. """Unique backend/root admission keys in deterministic order."""
  67. env_type = type(env)
  68. return tuple(sorted({
  69. (env_type.__module__, env_type.__qualname__, _normalized_filename_search_root(env, root, fallback_cwd))
  70. for root in roots}))
  71. def _acquire_filename_search_roots(keys: tuple[tuple[str, str, str], ...]) -> bool:
  72. """Atomically claim every key, polling for thread-scoped interruption."""
  73. with _FILENAME_SEARCH_ADMISSION:
  74. while any(key in _ACTIVE_FILENAME_SEARCH_ROOTS for key in keys):
  75. if tool_interrupt.is_interrupted():
  76. return False
  77. _FILENAME_SEARCH_ADMISSION.wait(_FILENAME_SEARCH_WAIT_SECONDS)
  78. if tool_interrupt.is_interrupted():
  79. return False
  80. if tool_interrupt.is_interrupted():
  81. return False
  82. return tool_interrupt.run_if_not_interrupted(lambda: _ACTIVE_FILENAME_SEARCH_ROOTS.update(keys))
  83. def _release_filename_search_roots(keys: tuple[tuple[str, str, str], ...]) -> None:
  84. """Release a completed walk and leave no idle per-root state behind."""
  85. with _FILENAME_SEARCH_ADMISSION:
  86. _ACTIVE_FILENAME_SEARCH_ROOTS.difference_update(keys)
  87. _FILENAME_SEARCH_ADMISSION.notify_all()
  88. _ADMISSION_INTERRUPTED_ERROR = (
  89. "File search was interrupted while waiting for another filename "
  90. "search on the same root. Retry when ready.")
  91. _SEARCH_TIMEOUT_MARKER_RE = re.compile(r"\n?\[Command timed out after \d+s\]\s*$")
  92. def _search_stdout_and_limit(result: ExecuteResult) -> tuple[str, Optional[str]]:
  93. """Return stdout cleaned for parsing and a limit reason for search timeouts."""
  94. if result.exit_code == 124:
  95. return _SEARCH_TIMEOUT_MARKER_RE.sub("", result.stdout), "search_timeout"
  96. return result.stdout, None
  97. def _bundled_rg() -> Optional[str]:
  98. """Hermes's pinned ripgrep, forward-slashed for the shell transports, or None."""
  99. import pm
  100. try:
  101. installed = pm.installed_package("ripgrep")
  102. except (OSError, RuntimeError, ValueError):
  103. return None
  104. binary = installed.binary if installed is not None else None
  105. return str(binary).replace("\\", "/") if binary is not None and binary.is_file() else None
  106. # A real rg/grep output line is a whitespace-free path token followed by ``:``
  107. # (match/count), ``-`` (context), or nothing (files_only); tool diagnostics
  108. # ("rg: ...", indented carets) never match.
  109. _SEARCH_OUTPUT_RE = re.compile(r'^([A-Za-z]:)?[^\s:][^\n]*?[:\-]\d|^[^\s:][^\s]*$')
  110. def _split_tool_diagnostics(output: str) -> tuple[str, str]:
  111. """Separate rg/grep diagnostic lines from real match output → ``(diagnostics, payload)``.
  112. ``_exec`` merges stderr into stdout; classifying by SHAPE lets the exit-2 guard
  113. tell a pure failure (no payload) from a partial one (one unreadable file, others
  114. matched) and guarantees error text is never parsed as a match."""
  115. diagnostics: list[str] = []
  116. payload: list[str] = []
  117. for line in output.split('\n'):
  118. if not line.strip():
  119. continue
  120. # Prefix check first: a match path can contain "-<digit>" (".../pytest-686/...").
  121. if line.lstrip().startswith(("rg: ", "grep: ", "find: ")):
  122. diagnostics.append(line)
  123. elif line == "--" or _SEARCH_OUTPUT_RE.match(line):
  124. payload.append(line)
  125. else:
  126. diagnostics.append(line)
  127. return '\n'.join(diagnostics), '\n'.join(payload)
  128. # One unreadable-path diagnostic, in every shape the search lanes produce:
  129. # rg: ``rg: <path>: Permission denied (os error 13)``
  130. # rg modified-sort walk: ``rg: <path>: IO error for operation on <path>: Permission denied (os error 13)``
  131. # grep: ``grep: <path>: Permission denied`` (no ``(os error N)`` suffix)
  132. # find: ``find: '<path>': Permission denied`` (pruned grep lane; GNU find quotes the
  133. # path with ASCII ``'`` or, in a UTF-8 locale, with typographic quotes)
  134. # The ``(os error N)`` suffix is ripgrep's Rust io::Error spelling, so it is optional.
  135. # The tool prefix is required: the shell's own ``bash: <path>/rg: Permission denied``
  136. # (the engine binary itself is not executable) is a failed invocation, not a skip.
  137. _PERMISSION_DENIED_RE = re.compile(
  138. r"^(?P<tool>rg|grep|find): (?P<path>.+?): "
  139. r"(?:IO error for operation on .+?: )?Permission denied(?: \(os error \d+\))?$")
  140. _SHELL_EXEC_FAILURE_CODES = frozenset({126, 127})
  141. _FIND_QUOTES_RE = re.compile(r"^['\u2018]|['\u2019]$")
  142. # The native rg runner keeps at most this much stderr (a huge tree of unreadable dirs
  143. # would otherwise be held in memory whole) and appends one tally line for the rest.
  144. _STDERR_RETAIN_BYTES = 64 * 1024
  145. _OMITTED_DIAGNOSTICS_RE = re.compile(r"^rg: \.\.\. (\d+) more diagnostic line\(s\) omitted$")
  146. def _omitted_diagnostics_line(count: int) -> str:
  147. return f"rg: ... {count} more diagnostic line(s) omitted"
  148. def _omitted_diagnostic_count(diagnostics: str) -> int:
  149. return sum(int(m.group(1)) for line in diagnostics.splitlines()
  150. if (m := _OMITTED_DIAGNOSTICS_RE.match(line.strip())))
  151. def _is_permission_only_error(diagnostics: str) -> bool:
  152. """True when every diagnostic line reports an unreadable path (rg or grep).
  153. That is a partial-success condition — the tool still emits results for the
  154. readable remainder — not a broken invocation like a bad glob or regex. The
  155. native runner's omitted-lines tally is judged by the lines it kept."""
  156. lines = [line.strip() for line in diagnostics.splitlines()
  157. if line.strip() and not _OMITTED_DIAGNOSTICS_RE.match(line.strip())]
  158. return bool(lines) and all(_PERMISSION_DENIED_RE.match(line) for line in lines)
  159. def _extract_unreadable_paths(diagnostics: str) -> list[str]:
  160. """Paths reported unreadable by rg or grep, deduplicated in first-seen order."""
  161. paths: list[str] = []
  162. seen: set[str] = set()
  163. for line in diagnostics.splitlines():
  164. match = _PERMISSION_DENIED_RE.match(line.strip())
  165. if not match:
  166. continue
  167. path = match.group("path").strip()
  168. if match.group("tool") == "find":
  169. path = _FIND_QUOTES_RE.sub("", path)
  170. if path and path not in seen:
  171. seen.add(path)
  172. paths.append(path)
  173. return paths
  174. def _compact_unreadable_paths(paths: list[str], keep: int = 3, omitted: int = 0) -> str:
  175. shown = ", ".join(paths[:keep])
  176. extra = max(len(paths) - keep, 0) + omitted
  177. return shown + (f" (+{extra} more)" if extra > 0 else "")
  178. def _skipped_unreadable_warning(paths: list[str], omitted: int = 0) -> str:
  179. """``omitted`` counts diagnostics the native runner dropped past its retention cap."""
  180. return ("Skipped " + str(len(paths) + omitted) + " unreadable path(s) with restricted "
  181. "permissions: " + _compact_unreadable_paths(paths, omitted=omitted)
  182. + ". Results may be incomplete; fix permissions or run as a user that "
  183. "can read them to include them.")
  184. def _skipped_fields(diagnostics: str) -> dict:
  185. """SearchResult skip fields from rg/grep/find diagnostics (nothing skipped → empty lists)."""
  186. paths = _extract_unreadable_paths(diagnostics)
  187. return {"skipped_paths": paths, "skipped_omitted": _omitted_diagnostic_count(diagnostics) if paths else 0}
  188. def _skipped_sentence(result: SearchResult) -> Optional[str]:
  189. """The skipped-paths sentence ``result.warning`` carries, rebuilt from its structured data."""
  190. if not result.skipped_paths:
  191. return None
  192. return _skipped_unreadable_warning(result.skipped_paths, result.skipped_omitted)
  193. def _prepend_skipped_paths_warning(skip: dict, existing: Optional[str]) -> Optional[str]:
  194. """Warning naming the skipped unreadable paths, prepended to any existing warning."""
  195. if not skip["skipped_paths"]:
  196. return existing
  197. warning = _skipped_unreadable_warning(skip["skipped_paths"], skip["skipped_omitted"])
  198. return warning if not existing else f"{warning} {existing}"
  199. _PERMISSION_SEARCH_HINT = "Nothing matched in the readable part of the search scope."
  200. def _merge_child_warnings(children: list[SearchResult], has_results: bool) -> list[str]:
  201. """Combine per-root search warnings into one list: the skipped-path sentences collapse
  202. into one sentence composed from each child's structured skip data (summed count), the
  203. rest of the text is deduplicated in order. The nothing-matched hint is dropped once
  204. any root produced results."""
  205. total, shown, rest = 0, [], []
  206. for child in children:
  207. if not child.warning:
  208. continue
  209. warning, sentence = child.warning, _skipped_sentence(child)
  210. if sentence:
  211. total += len(child.skipped_paths) + child.skipped_omitted
  212. for path in child.skipped_paths: # only three are displayed; never scan thousands
  213. if len(shown) >= 3:
  214. break
  215. if path not in shown:
  216. shown.append(path)
  217. warning = warning.replace(sentence, "\0")
  218. for piece in warning.split("\0"):
  219. piece = piece.strip()
  220. if has_results:
  221. piece = piece.replace(_PERMISSION_SEARCH_HINT, "").strip()
  222. if piece and piece not in rest:
  223. rest.append(piece)
  224. if not total:
  225. return rest
  226. sentence = _skipped_unreadable_warning(shown, max(total - len(shown), 0))
  227. return [sentence, *rest]
  228. _FIND_MODIFIED_ORDER_ERROR = (
  229. "Exact modification-time order requires GNU find with "
  230. "-printf support; install ripgrep 14+ or use order='discovery'.")
  231. def _split_find_output(stdout: str, order: str) -> tuple[list[str], list[str]]:
  232. """``(paths, diagnostics)`` from a bounded find run; ``find: `` lines are stderr."""
  233. raw_files: list[str] = []
  234. diagnostics: list[str] = []
  235. for line in stdout.splitlines():
  236. if line.startswith("find: "):
  237. diagnostics.append(line)
  238. elif order == "modified":
  239. parts = line.split(" ", 1)
  240. if len(parts) == 2 and parts[0].replace(".", "", 1).isdigit():
  241. raw_files.append(parts[1])
  242. elif line:
  243. raw_files.append(line)
  244. return raw_files, diagnostics
  245. def _find_exit_verdict(exit_code: int, raw_files: list[str], diagnostics: list[str],
  246. fetch_limit: int, order: str) -> tuple[Optional[SearchResult], Optional[str]]:
  247. """``(terminal result or None, warning)`` for a bounded find run.
  248. Exit 141 is benign only when the payload proves head closed the pipe at the
  249. bound. find exits 1 when traversal hit unreadable dirs (EACCES) but still prints
  250. what it could read: a partial success, named from its diagnostics. Exit 1 with no
  251. payload is a real failure unless every diagnostic is such a skip (bad expressions
  252. and BSD find's missing -printf also exit 1)."""
  253. bounded_sigpipe = exit_code == 141 and len(raw_files) >= fetch_limit
  254. if exit_code not in {0, 1, 124} and not bounded_sigpipe:
  255. if order == "modified":
  256. return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None
  257. return SearchResult(error="File search failed while running bounded find traversal."), None
  258. diagnostic_text = "\n".join(diagnostics)
  259. skipped = _extract_unreadable_paths(diagnostic_text)
  260. warning = _skipped_unreadable_warning(skipped) if skipped else None
  261. if exit_code != 1:
  262. return None, warning
  263. if not raw_files and _is_permission_only_error(diagnostic_text):
  264. return SearchResult(total_count=0, warning=f"{warning} {_PERMISSION_SEARCH_HINT}",
  265. skipped_paths=skipped), None
  266. if not raw_files:
  267. if order == "modified":
  268. return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None
  269. detail = f": {diagnostics[0]}" if diagnostics else "."
  270. return SearchResult(error=f"File search failed while running bounded find traversal{detail}"), None
  271. # A non-permission diagnostic (a dir removed mid-walk) must not be reported as a permission
  272. # problem: name it, plus any permission skips, so the user chases the real cause.
  273. other = next((d.strip() for d in diagnostics if d.strip() and not _PERMISSION_DENIED_RE.match(d.strip())), None)
  274. if other:
  275. return None, f"Some paths could not be read: {other}. Results may be incomplete." + (
  276. f" {warning}" if warning else "")
  277. return None, warning or ("find traversal skipped one or more unreadable paths (restricted "
  278. "permissions); results are partial.")
  279. def _drain_stderr_capped(pipe, chunks: list[bytes], omitted: list[int]) -> None:
  280. """Read ``pipe`` to EOF (rg must never block on a full stderr pipe) but keep only the
  281. first ``_STDERR_RETAIN_BYTES``; the first line is always kept so a lone huge diagnostic
  282. still names the failure. ``omitted[0]`` counts the dropped lines. Past the cap the first
  283. non-permission line is still kept, so a real failure behind a flood of permission
  284. errors is never classified as a permission-only partial success."""
  285. kept = 0
  286. kept_other = False
  287. for raw in pipe or ():
  288. if chunks and kept + len(raw) > _STDERR_RETAIN_BYTES:
  289. line = raw.decode("utf-8", "replace").strip()
  290. if not kept_other and line and not _PERMISSION_DENIED_RE.match(line):
  291. chunks.append(raw)
  292. kept_other = True
  293. continue
  294. omitted[0] += 1
  295. continue
  296. chunks.append(raw)
  297. kept += len(raw)
  298. def _parse_search_context_line(line: str) -> tuple[str, int, str] | None:
  299. """Parse a ``path-line-content`` context line using the RIGHTMOST numeric
  300. separator (filenames may contain ``-<digits>-`` segments):
  301. ``dir/file-12-name.py-8-context`` → (``dir/file-12-name.py``, 8, ``context``)."""
  302. if not line or line == "--":
  303. return None
  304. match = None
  305. for candidate in re.finditer(r'-(\d+)-', line):
  306. match = candidate
  307. if match is None or match.start() == 0:
  308. return None
  309. return line[:match.start()], int(match.group(1)), line[match.end():]
  310. _REGEX_NEWLINE_ESCAPE_RE = re.compile(r"(?<!\\)(?:\\\\)*\\n")
  311. def _pattern_has_regex_newline(pattern: str) -> bool:
  312. """True when a content regex wants to match a newline: a literal newline or a
  313. ``\\n`` escape with an ODD number of backslashes (``\\\\n`` is a literal
  314. backslash+n and must not count)."""
  315. return "\n" in pattern or bool(_REGEX_NEWLINE_ESCAPE_RE.search(pattern))
  316. def _is_line_oriented_newline_error(error: Optional[str]) -> bool:
  317. """Return True for rg's hard error when multiline mode is required."""
  318. return bool(error) and "literal \"\\n\" is not allowed" in error and "--multiline" in error
  319. def _maybe_warn_line_oriented_newline_pattern(result: SearchResult, pattern: str) -> SearchResult:
  320. """Attach a newline-regex warning only when search found no usable results."""
  321. if result.total_count != 0 or not _pattern_has_regex_newline(pattern):
  322. return result
  323. if result.error and not _is_line_oriented_newline_error(result.error):
  324. return result
  325. result.error = None
  326. result.skipped_paths, result.skipped_omitted = [], 0
  327. result.warning = (
  328. "0 results found. Note: search_files content search is line-oriented "
  329. "and does not run ripgrep with -U/--multiline, so `\\n` in the regex "
  330. "does not match line breaks. Use context=N to inspect neighboring "
  331. "lines, or escape as `\\\\n` when searching for a literal backslash+n.")
  332. return result
  333. # Match lines are "file:lineno:content". Windows paths carry a drive letter
  334. # ("C:\path"), so a naive split(":") breaks — the regex handles both.
  335. _MATCH_LINE_RE = re.compile(r'^([A-Za-z]:)?(.*?):(\d+):(.*)$')
  336. # Output-mode → engine flag (identical for rg and grep).
  337. _OUTPUT_MODE_FLAGS = {"files_only": "-l", "count": "-c"}
  338. def _parse_search_output(result, output_mode: str, limit: int, offset: int,
  339. context: int, warning: Optional[str] = None) -> SearchResult:
  340. """Parse rg/grep ``| head`` output into a SearchResult (shared by both engines).
  341. Exit codes: 0=matches, 1=none, 2=error — but both tools return 2 on PARTIAL
  342. errors (one unreadable file), so an error is surfaced only when exit==2 AND no
  343. usable payload remains. ``warning`` is attached to files_only/content results."""
  344. stdout, limit_reason = _search_stdout_and_limit(result)
  345. diagnostics, payload = _split_tool_diagnostics(stdout)
  346. if result.exit_code in _SHELL_EXEC_FAILURE_CODES:
  347. # The shell could not run the engine at all (126 not executable, 127 not found).
  348. return SearchResult(error=f"Search failed: {result.stdout.strip() or 'Search error'}",
  349. total_count=0)
  350. if result.exit_code == 2 and not payload.strip():
  351. error_msg = diagnostics.strip() or result.stdout.strip() or "Search error"
  352. if _is_permission_only_error(error_msg):
  353. # Pure unreadable-path failure: rg read nothing but the invocation was
  354. # sound. Degrade to an explicit empty result instead of failing — the
  355. # model can still act on the named paths.
  356. skip = _skipped_fields(error_msg)
  357. warning = _prepend_skipped_paths_warning(skip, warning)
  358. if not warning:
  359. warning = _PERMISSION_SEARCH_HINT
  360. elif _PERMISSION_SEARCH_HINT not in warning:
  361. warning = f"{warning} {_PERMISSION_SEARCH_HINT}"
  362. return SearchResult(total_count=0, warning=warning, **skip)
  363. return SearchResult(error=f"Search failed: {error_msg}", total_count=0)
  364. skip = _skipped_fields(diagnostics)
  365. skipped_warning = _prepend_skipped_paths_warning(skip, None)
  366. # A non-permission error beside the skips (a path removed mid-search) is named too, not
  367. # folded into the permission warning.
  368. other_errors = [ln.strip() for ln in diagnostics.splitlines()
  369. if ln.lstrip().startswith(("rg: ", "grep: "))
  370. and not _PERMISSION_DENIED_RE.match(ln.strip())
  371. and not _OMITTED_DIAGNOSTICS_RE.match(ln.strip())]
  372. if result.exit_code == 2 and other_errors:
  373. error_warning = f"The search reported errors for some paths; results may be incomplete ({other_errors[0]})."
  374. skipped_warning = f"{error_warning} {skipped_warning}" if skipped_warning else error_warning
  375. if skipped_warning:
  376. warning = skipped_warning if not warning else f"{skipped_warning} {warning}"
  377. lines = [ln for ln in payload.strip().split('\n') if ln]
  378. if output_mode == "files_only":
  379. return SearchResult(
  380. files=lines[offset:offset + limit], total_count=len(lines),
  381. truncated=bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip)
  382. if output_mode == "count":
  383. counts = {}
  384. for line in lines:
  385. if ':' in line:
  386. path, n = line.rsplit(':', 1)
  387. try:
  388. counts[path] = int(n)
  389. except ValueError:
  390. pass
  391. return SearchResult(
  392. counts=counts, total_count=sum(counts.values()),
  393. truncated=bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip)
  394. matches = []
  395. for line in lines:
  396. if line == "--":
  397. continue
  398. m = _MATCH_LINE_RE.match(line)
  399. if m:
  400. matches.append(SearchMatch(
  401. path=(m.group(1) or '') + m.group(2), line_number=int(m.group(3)), content=m.group(4)[:500],
  402. ))
  403. continue
  404. # Context lines only when requested, to avoid false positives on dashy paths.
  405. if context > 0:
  406. parsed = _parse_search_context_line(line)
  407. if parsed:
  408. matches.append(SearchMatch(path=parsed[0], line_number=parsed[1], content=parsed[2][:500]))
  409. total = len(matches)
  410. return SearchResult(
  411. matches=matches[offset:offset + limit], total_count=total,
  412. truncated=total > offset + limit or bool(limit_reason), limit_reason=limit_reason, warning=warning,
  413. **skip,
  414. )
  415. def _posix_roots(roots: list[str]) -> bool:
  416. """Darwin-only: every root is POSIX-shaped (no drive letter / backslash)."""
  417. return sys.platform == "darwin" and all(
  418. not re.match(r"^[A-Za-z]:[\\/]", root) and "\\" not in root for root in roots)
  419. def _find_literal_path_expressions(roots: list[str]) -> list[str]:
  420. """Escape each search root for find's ``-path`` test, which matches its PATTERN
  421. (glob) against the path find echoes — the operand verbatim. The operand reaches
  422. the shell already single-quoted; ``-path`` must compare literally, so ``*?[]``
  423. in a root's own name need escaping (a directory literally named ``a[1]`` must
  424. be excluded by identity, not treated as a character class). Backslashes are
  425. left alone: find does not read them as escapes inside a bracket-free pattern,
  426. and doubling them would break matching on backslash-shaped roots."""
  427. return [re.sub(r"([*?\[\]])", r"\\\1", root) for root in roots]
  428. class SearchMixin:
  429. """File-name and content search via rg with find/grep fallbacks. Requires
  430. ``_exec``, ``_has_command``, ``_expand_path``, ``_escape_shell_arg``,
  431. ``_escape_native_tool_arg``, ``env``, ``cwd``, ``_command_cache``,
  432. ``_rg_resolution_cache`` and ``_rg_modified_capability`` from the host class."""
  433. # --- rg resolution --------------------------------------------------------
  434. def _resolve_command(self, cmd: str) -> Optional[str]:
  435. """Resolve an executable in the command host's namespace. Ordinary commands
  436. keep the bool hit/miss cache; rg alone caches successful resolved paths and
  437. re-probes misses so a mid-session install becomes visible (with off-PATH
  438. Windows candidates: cargo, scoop, winget)."""
  439. if cmd != "rg":
  440. return cmd if self._has_command(cmd) else None
  441. cached = self._rg_resolution_cache.get(cmd)
  442. if cached:
  443. return cached
  444. from tools.environments.base import env_uses_host_filesystem
  445. from tools.environments.local import _IS_WINDOWS
  446. # A host search is Hermes's own work, so it runs the pinned ripgrep, never the
  447. # first rg on the terminal PATH, where the user's tools come first.
  448. on_host = env_uses_host_filesystem(self.env)
  449. bundled = _bundled_rg() if on_host else None
  450. if bundled:
  451. self._rg_resolution_cache[cmd] = bundled
  452. return bundled
  453. result = self._exec("command -v rg 2>/dev/null")
  454. if result.exit_code == 0 and result.stdout.strip():
  455. resolved = result.stdout.strip().splitlines()[0]
  456. if resolved == "yes": # compatibility with old boolean-probe fakes
  457. resolved = "rg"
  458. self._rg_resolution_cache[cmd] = resolved
  459. return resolved
  460. if _IS_WINDOWS and on_host:
  461. user_profile = os.environ.get("USERPROFILE") or str(Path.home())
  462. local_app_data = os.environ.get("LOCALAPPDATA")
  463. scoop = os.environ.get("SCOOP") or os.path.join(user_profile, "scoop")
  464. candidates = [
  465. os.path.join(user_profile, ".cargo", "bin", "rg.exe"),
  466. os.path.join(scoop, "shims", "rg.exe"),
  467. ]
  468. if local_app_data:
  469. candidates.append(os.path.join(local_app_data, "Microsoft", "WinGet", "Links", "rg.exe"))
  470. for candidate in candidates:
  471. if os.path.isfile(candidate):
  472. resolved = candidate.replace("\\", "/")
  473. self._rg_resolution_cache[cmd] = resolved
  474. return resolved
  475. return None
  476. _RG_VERSION_RE = re.compile(
  477. r"(?m)^ripgrep\s+((?:0|[1-9]\d*))\."
  478. r"(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)"
  479. r"(?:-(?:(?:0|[1-9]\d*)|(?:[0-9A-Za-z-]*[A-Za-z-]"
  480. r"[0-9A-Za-z-]*))(?:\.(?:(?:0|[1-9]\d*)|"
  481. r"(?:[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)))*)?"
  482. r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?"
  483. r"(?:\s+\(rev [^)]+\))?\s*$")
  484. def _modified_rg_capability_error(self, executable: str) -> Optional[str]:
  485. """Cached actionable error unless rg can sort exactly (full SemVer, >= 14)."""
  486. if executable in self._rg_modified_capability:
  487. return self._rg_modified_capability[executable]
  488. result = self._exec(f"{self._quote_executable(executable)} --version", timeout=10)
  489. match = self._RG_VERSION_RE.search(result.stdout or "")
  490. if result.exit_code == 0 and match and int(match.group(1)) >= 14:
  491. error = None
  492. else:
  493. error = ("Exact modification-time order requires ripgrep 14 or newer; "
  494. "upgrade ripgrep or use order='discovery'.")
  495. self._rg_modified_capability[executable] = error
  496. return error
  497. # --- native rg transport (local POSIX) --------------------------------------
  498. def _run_rg_native(self, argv: list[str], fetch_limit: int, timeout: int,
  499. merge_stderr: bool = False) -> ExecuteResult:
  500. """Run ``argv`` (shell-quoted rg words) natively and stop reading after
  501. ``fetch_limit`` lines — the ``| head -n`` of the shell pipeline without the
  502. two bash spawns. ``shlex.split`` undoes the escaping the builders apply for
  503. the shell path, so both transports see identical arguments. Exit code and
  504. stdout follow the shell contract (rg 0/1/2; 124 on timeout with partial
  505. output; 130 on interrupt), so ``_parse_search_output`` is shared. Once the
  506. bound is reached rg is killed like ``head`` closing the pipe would.
  507. ``merge_stderr`` mirrors the shell path's stderr handling: kept for real
  508. searches (diagnostics name unreadable paths or the error), discarded
  509. (``2>/dev/null``) for probes. Like the shell's ``2>&1``-free ``| head``, kept
  510. stderr is drained on its own pipe so diagnostics never use up the
  511. ``fetch_limit`` lines meant for results; it is prepended to stdout."""
  512. from tools.environments.local import _kill_process_group_posix, _make_run_env
  513. cwd = getattr(self.env, "cwd", None) or self.cwd
  514. args = shlex.split(" ".join(argv))
  515. try:
  516. proc = subprocess.Popen(
  517. args, cwd=cwd, env=_make_run_env(self.env.env), stdin=subprocess.DEVNULL,
  518. stdout=subprocess.PIPE, stderr=subprocess.PIPE if merge_stderr else subprocess.DEVNULL,
  519. # detached-spawn: ok — awaited rg under a deadline; the group is killed on timeout or interrupt
  520. start_new_session=True)
  521. except OSError as exc:
  522. return ExecuteResult(stdout=f"rg: {exc}", exit_code=2)
  523. # Record the group at spawn, as LocalEnvironment._run_bash does: rg can exit between the
  524. # poll and the kill below, and on macOS a not-yet-reaped child has no pgid any more.
  525. with contextlib.suppress(ProcessLookupError):
  526. proc._hermes_pgid = os.getpgid(proc.pid)
  527. from gateway.status import get_process_start_time
  528. proc._hermes_pgid_start = get_process_start_time(proc.pid)
  529. # Drain on a thread so a silent rg (huge tree, no hits yet) cannot pin the
  530. # caller past the deadline or past a /stop; the waiter below owns both.
  531. lines: list[bytes] = []
  532. bounded = threading.Event()
  533. def _drain() -> None:
  534. for raw in proc.stdout:
  535. lines.append(raw)
  536. if len(lines) >= fetch_limit:
  537. bounded.set()
  538. break
  539. drainer = threading.Thread(target=_drain, daemon=True)
  540. drainer.start()
  541. err_chunks: list[bytes] = []
  542. err_omitted = [0]
  543. err_drainer = None
  544. if merge_stderr:
  545. err_pipe = proc.stderr
  546. err_drainer = threading.Thread( # health: allow HX012 -- pipe drainer, reads no profile-scoped state
  547. target=_drain_stderr_capped, args=(err_pipe, err_chunks, err_omitted), daemon=True)
  548. err_drainer.start()
  549. deadline = time.monotonic() + timeout
  550. exit_code: Optional[int] = None
  551. while True:
  552. drainer.join(0.05)
  553. if not drainer.is_alive() or bounded.is_set():
  554. break
  555. if tool_interrupt.is_interrupted():
  556. exit_code = 130
  557. break
  558. if time.monotonic() > deadline:
  559. exit_code = 124
  560. break
  561. if proc.poll() is None:
  562. _kill_process_group_posix(proc) # native lane is POSIX-only (gate above)
  563. proc.wait()
  564. drainer.join()
  565. proc.stdout.close()
  566. stdout = b"".join(lines).decode("utf-8", errors="replace")
  567. if err_drainer is not None:
  568. err_drainer.join()
  569. if proc.stderr is not None:
  570. proc.stderr.close()
  571. err_text = b"".join(err_chunks).decode("utf-8", errors="replace")
  572. if err_omitted[0]:
  573. err_text = err_text.rstrip("\n") + "\n" + _omitted_diagnostics_line(err_omitted[0]) + "\n"
  574. stdout = err_text + stdout
  575. if exit_code == 124:
  576. return ExecuteResult(stdout=stdout + f"\n[Command timed out after {timeout}s]", exit_code=124)
  577. if exit_code == 130:
  578. return ExecuteResult(stdout=stdout + "\n[Command interrupted]", exit_code=130)
  579. # A killed-at-bound rg reports a signal (negative returncode); head would have
  580. # left the pipeline at 0 unless rg itself already failed.
  581. return ExecuteResult(stdout=stdout, exit_code=0 if bounded.is_set() else proc.returncode)
  582. def _run_rg_bounded(self, words: list[str], fetch_limit: int, timeout: int, *,
  583. merge_stderr: bool = False, native_ok: bool = True,
  584. shell_prefix: str = "") -> ExecuteResult:
  585. """Run an rg command (shell-quoted words) and keep the first ``fetch_limit``
  586. lines: natively on a local POSIX host, else through the backend shell as
  587. ``<prefix><words> | head -n N``. ``native_ok=False`` keeps a form the native
  588. lane cannot express (the multi-root ``cd`` prefix); ``shell_prefix`` is shell-only."""
  589. if native_ok and self._native_read_enabled():
  590. return self._run_rg_native(words, fetch_limit, timeout, merge_stderr=merge_stderr)
  591. stderr = "" if merge_stderr else " 2>/dev/null"
  592. return self._exec(f"{shell_prefix}{' '.join(words)}{stderr} | head -n {fetch_limit}", timeout=timeout)
  593. def _quote_executable(self, executable: str) -> str:
  594. """Quote an executable without leaking controller path semantics."""
  595. if re.fullmatch(r"[A-Za-z0-9_.-]+", executable):
  596. return executable
  597. from tools.environments.base import env_uses_host_filesystem
  598. if env_uses_host_filesystem(self.env):
  599. return self._escape_native_tool_arg(executable)
  600. return "'" + executable.replace("'", "'\"'\"'") + "'"
  601. # --- macOS protected-folder exclusions --------------------------------------
  602. def _macos_search_exclusions(self, path: str) -> list[str]:
  603. """Protected descendants to prune for this search root, if any. Gated on
  604. ``env.is_local``: ``sys.platform``/``_HOME`` describe the CONTROLLER, but the
  605. search runs on ``env``'s host. Envs without the flag default to local
  606. semantics; pruning is a warning-carrying skip, never data loss."""
  607. env = getattr(self, "env", None)
  608. if env is not None and getattr(env, "is_local", True) is False:
  609. return []
  610. from tools import file_operations as _fo # lazy: _HOME is monkeypatched there
  611. cwd = getattr(self.env, "cwd", None) or self.cwd
  612. return _macos_protected_search_exclusions(path, cwd=cwd, home=_fo._HOME, platform=sys.platform)
  613. def _protected_prune_paths(self, path: str) -> list[str]:
  614. """Absolute-ish protected paths for find's ``-path ... -prune``."""
  615. return [os.path.normpath(os.path.join(path, item)) for item in self._macos_search_exclusions(path)]
  616. def _effective_macos_search_exclusions(self, roots: list[str]) -> list[tuple[str, str, str]]:
  617. """Unique ``(root, relative, absolute)`` exclusions across ``roots``, never
  618. pruning a root the caller chose explicitly."""
  619. cwd = getattr(self.env, "cwd", None) or self.cwd
  620. use_posix_paths = _posix_roots(roots)
  621. def normalized(root: str) -> str:
  622. if use_posix_paths:
  623. return posixpath.normpath(root if posixpath.isabs(root) else posixpath.join(cwd, root))
  624. return os.path.normcase(os.path.abspath(os.path.normpath(root)))
  625. normalized_roots = [normalized(root) for root in roots]
  626. explicit_roots = set(normalized_roots)
  627. seen = set()
  628. effective = []
  629. for root, normalized_root in zip(roots, normalized_roots):
  630. for relative in self._macos_search_exclusions(root):
  631. if use_posix_paths:
  632. absolute = key = posixpath.normpath(posixpath.join(normalized_root, relative))
  633. else:
  634. absolute = os.path.normpath(os.path.join(root, relative))
  635. key = os.path.normcase(os.path.abspath(absolute))
  636. if key in explicit_roots or key in seen:
  637. continue
  638. seen.add(key)
  639. effective.append((root, relative, absolute))
  640. return effective
  641. @staticmethod
  642. def _macos_protected_search_warning(paths: list[str]) -> str:
  643. skipped = ", ".join(os.path.basename(item) for item in paths)
  644. return ("Skipped macOS protected folders during broad search to avoid "
  645. f"an unattended privacy prompt: {skipped}. Search a protected "
  646. "folder directly when access is intentional.")
  647. @staticmethod
  648. def _hidden_prune_expr(q_roots: list[str]) -> str:
  649. """find clause pruning hidden dirs while keeping an explicitly selected dot-named root
  650. (dir or single file) — find echoes each start point as given, so ``! -path`` matches it."""
  651. exemptions = "".join(f" ! -path {root}" for root in q_roots)
  652. return f"\\( -type d -name '.*'{exemptions} \\) -prune"
  653. def _prune_expr(self, protected_paths: list[str]) -> str:
  654. """find ``\\( -path A -o -path B \\) -prune`` clause for the protected dirs."""
  655. terms = " -o ".join(f"-path {self._escape_shell_arg(item)}" for item in protected_paths)
  656. return f"\\( {terms} \\) -prune"
  657. def _root_under_hidden_dir(self, path: str) -> bool:
  658. """True when the search root or any ancestor is dot-named (``~/.hermes/skills``)."""
  659. root = _normalized_filename_search_root(self.env, path or ".", self.cwd)
  660. return any(part.startswith(".") and part not in (".", "..") for part in root.replace("\\", "/").split("/"))
  661. def _rg_exclusion_globs(self, path: str) -> list[str]:
  662. """``--glob '!<dir>/**'`` pairs excluding protected dirs from an rg run."""
  663. out: list[str] = []
  664. for item in self._macos_search_exclusions(path):
  665. out.extend(["--glob", self._escape_shell_arg(f"!{item}/**")])
  666. return out
  667. def _path_exists_probe(self, path: str) -> ExecuteResult:
  668. """Existence probe; stdout contains "exists" or "not_found" (or the probe's
  669. ``cwd_error`` when the exec wrapper itself failed)."""
  670. if self._native_read_enabled():
  671. full = path if os.path.isabs(path) else os.path.join(getattr(self.env, "cwd", None) or self.cwd, path)
  672. return ExecuteResult(stdout="exists" if os.path.exists(full) else "not_found")
  673. return self._exec(f"test -e {self._escape_shell_arg(path)} && echo exists || echo not_found")
  674. def _dispatch_search(self, pattern: str, path: str, target: str,
  675. file_glob: Optional[str], limit: int, offset: int,
  676. output_mode: str, context: int, order: str = "discovery") -> SearchResult:
  677. if target == "files":
  678. return self._search_files(pattern, path, limit, offset, order)
  679. return self._search_content(pattern, path, file_glob, limit, offset, output_mode, context)
  680. def _path_not_found_result(self, path: str) -> SearchResult:
  681. """Error result for a missing search root, with nearby-entry suggestions."""
  682. parent = os.path.dirname(path) or "."
  683. basename_query = os.path.basename(path)
  684. hint_parts = [f"Path not found: {path}"]
  685. parent_check = self._exec(f"test -d {self._escape_shell_arg(parent)} && echo yes || echo no")
  686. if "yes" in parent_check.stdout and basename_query:
  687. ls_result = self._exec(f"ls -1 {self._escape_shell_arg(parent)} 2>/dev/null | head -20")
  688. if ls_result.exit_code == 0 and ls_result.stdout.strip():
  689. lq = basename_query.lower()
  690. candidates = [
  691. os.path.join(parent, e) for e in ls_result.stdout.strip().split('\n')
  692. if e and (lq in e.lower() or e.lower() in lq or e.lower().startswith(lq[:3]))]
  693. if candidates:
  694. hint_parts.append("Similar paths: " + ", ".join(candidates[:5]))
  695. return SearchResult(error=". ".join(hint_parts), total_count=0)
  696. def _try_multi_path_search(self, pattern: str, path: str, target: str,
  697. file_glob: Optional[str], limit: int, offset: int,
  698. output_mode: str, context: int,
  699. order: str = "discovery") -> Optional[SearchResult]:
  700. """Recover a not-found ``path`` that is really several paths in one string.
  701. Commas explicitly delimit paths (internal spaces preserved); without commas
  702. split on whitespace. Search every existing part, merge, and note skipped
  703. parts. None when it doesn't look like a multi-path string."""
  704. if "," in path:
  705. parts = [part.strip() for part in path.split(",") if part.strip()]
  706. else:
  707. parts = path.split()
  708. if len(parts) < 2:
  709. return None
  710. existing, missing = [], []
  711. for p in parts:
  712. expanded = self._expand_path(p)
  713. (existing if "exists" in self._path_exists_probe(expanded).stdout else missing).append(expanded)
  714. if not existing:
  715. return None
  716. if target == "files":
  717. # One global traversal across roots so modified ordering and pagination
  718. # are exact; root admission wraps the actual rg/find invocation.
  719. merged = self._search_files(pattern, existing, limit, offset, order)
  720. children = [merged]
  721. else:
  722. merged, children = SearchResult(), []
  723. for root in existing:
  724. sub = self._search_content(pattern, root, file_glob, limit, offset, output_mode, context)
  725. if sub.error:
  726. return sub
  727. children.append(sub)
  728. merged.matches.extend(sub.matches)
  729. merged.files.extend(sub.files)
  730. merged.counts.update(sub.counts)
  731. merged.total_count += sub.total_count
  732. merged.truncated = merged.truncated or sub.truncated
  733. merged.matches = merged.matches[:limit]
  734. merged.files = merged.files[:limit]
  735. note = f"path contained {len(parts)} entries; searched {len(existing)} that exist"
  736. if missing:
  737. note += "; skipped missing: " + ", ".join(missing[:3])
  738. if len(missing) > 3:
  739. note += f" (+{len(missing) - 3} more)"
  740. has_results = bool(merged.total_count or merged.matches or merged.files or merged.counts)
  741. warning_parts = [*_merge_child_warnings(children, has_results), note]
  742. if not merged.error:
  743. protected_paths = [absolute for _r, _rel, absolute in self._effective_macos_search_exclusions(existing)]
  744. if protected_paths:
  745. warning_parts.append(self._macos_protected_search_warning(protected_paths))
  746. merged.warning = " ".join(warning_parts)
  747. return merged
  748. def _search_prune_glob_args(self) -> str:
  749. """rg globs pruning known heavyweight recursive subtrees. Both forms are
  750. needed: globs are relative to each rg root, so ``**/name/**`` alone misses an
  751. explicitly selected ``name/`` root. Names come from the shared scan policy —
  752. no second search-only list."""
  753. globs = []
  754. for dirname in sorted(SEARCH_PRUNE_DIR_NAMES):
  755. for prefix in ("", "**/"):
  756. globs.extend(("--glob", self._escape_shell_arg(f"!{prefix}{dirname}/**")))
  757. return " ".join(globs)
  758. # (rg flags, message template) probes for a 0-match content search, in order.
  759. # The fixed-string probe only runs when the pattern has regex metacharacters.
  760. _ZERO_MATCH_PROBES = (
  761. ("-i", "0 exact matches, but {total} case-insensitive match(es) in {n} file(s): "
  762. "{paths} — the pattern's casing may be wrong."),
  763. # rg skips dotdirs and .gitignore'd files by default; say so instead of a bare zero.
  764. ("--hidden --no-ignore", "0 matches in visible files, but {total} match(es) in {n} "
  765. "hidden or gitignored file(s): {paths} — these are excluded by default."),
  766. ("-F", "0 regex matches, but {total} literal match(es) in {n} file(s): {paths} — the "
  767. "pattern contains regex metacharacters that likely need escaping "
  768. "(or pass a simpler substring)."),
  769. )
  770. def _zero_match_probe(self, pattern: str, path: str, file_glob: Optional[str]) -> Optional[str]:
  771. """Steering hint for a 0-match content search, or None: a bare zero gives the
  772. model nothing to act on, so run cheap count-only rg probes (case-insensitive,
  773. hidden/ignored, fixed-string) and report the first that hits."""
  774. rg_executable = self._resolve_command('rg')
  775. if not rg_executable:
  776. return None
  777. rg = self._quote_executable(rg_executable)
  778. has_meta = bool(re.search(r"[.\[\](){}?*+^$\\|]", pattern))
  779. glob_expr = f" --glob {self._escape_shell_arg(file_glob)}" if file_glob else ""
  780. for flags, template in self._ZERO_MATCH_PROBES:
  781. if flags == "-F" and not has_meta:
  782. continue
  783. # The hidden/ignored probe keeps --no-ignore so project-local ignored
  784. # files stay diagnosable, but prunes heavyweight trees before rg recurses.
  785. if flags.startswith("--hidden"):
  786. glob_expr_probe = f"{glob_expr} {self._search_prune_glob_args()}"
  787. else:
  788. glob_expr_probe = glob_expr
  789. probe_words = [rg, flags, "--count-matches", glob_expr_probe,
  790. self._escape_shell_arg(pattern, translate_path=False), self._escape_native_tool_arg(path)]
  791. probe = self._run_rg_bounded(probe_words, 50, timeout=30)
  792. total, per_file = 0, []
  793. for line in (probe.stdout or "").strip().splitlines():
  794. p, _sep, n = line.rpartition(":")
  795. if n.isdigit():
  796. total += int(n)
  797. per_file.append(p)
  798. if total > 0:
  799. extra = len(per_file) - 5
  800. paths = ", ".join(per_file[:5]) + (f" (+{extra} more)" if extra > 0 else "")
  801. return template.format(total=total, n=len(per_file), paths=paths)
  802. return None
  803. def _is_broad_local_search_root(self, path: str) -> bool:
  804. """Whether a no-rg LOCAL root (filesystem root, $HOME or an ancestor of it) is
  805. unsafe for recursive find. Controller paths never classify remotes."""
  806. from tools.environments.base import env_uses_host_filesystem
  807. from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
  808. if not env_uses_host_filesystem(self.env):
  809. return False
  810. def normalized(value: str) -> str:
  811. if _IS_WINDOWS:
  812. value = _msys_to_windows_path(value).replace("\\", "/")
  813. if not os.path.isabs(value):
  814. value = os.path.join(getattr(self.env, "cwd", None) or self.cwd, value)
  815. # Classify the linked target, not the link: ``find -H`` now follows an
  816. # operand symlink, so a link pointing at $HOME (or at the filesystem root)
  817. # must not slip a recursive find past this guard (#116270). Local-only by
  818. # the isinstance check above, so this resolves on the host that runs find.
  819. return os.path.normcase(os.path.realpath(value))
  820. from tools import file_operations as _fo # lazy: _HOME is monkeypatched there
  821. root = normalized(path)
  822. home = normalized(_fo._HOME)
  823. drive = os.path.splitdrive(root)[0]
  824. anchor = drive + os.sep if drive else os.path.abspath(os.sep)
  825. if root == os.path.normcase(anchor):
  826. return True
  827. try:
  828. common = os.path.commonpath([root, home])
  829. except ValueError:
  830. return False
  831. return root == home or common == root
  832. def _filter_hidden_descendants(self, paths: list[str], root: Path) -> list[str]:
  833. """Drop hidden descendants while allowing an explicit hidden search root."""
  834. normalized_root = root.resolve()
  835. filtered = []
  836. for item_path in paths:
  837. try:
  838. rel_parts = Path(item_path).resolve().relative_to(normalized_root).parts
  839. except ValueError:
  840. rel_parts = Path(item_path).parts
  841. if any(part not in {".", ".."} and part.startswith(".") for part in rel_parts):
  842. continue
  843. filtered.append(item_path)
  844. return filtered
  845. def _filter_gitignored_paths(self, paths: list[str], root: str) -> list[str]:
  846. """Drop paths ignored by git when *root* is inside a git worktree."""
  847. if not paths:
  848. return paths
  849. escaped_root = self._escape_shell_arg(root)
  850. probe = self._exec(
  851. f"git -C {escaped_root} rev-parse --is-inside-work-tree >/dev/null 2>&1",
  852. timeout=10,
  853. )
  854. if probe.exit_code != 0:
  855. return paths
  856. result = self._exec(
  857. f"git -C {escaped_root} check-ignore --stdin",
  858. timeout=10,
  859. stdin_data="\n".join(paths),
  860. )
  861. ignored = set(result.stdout.splitlines())
  862. return [item_path for item_path in paths if item_path not in ignored]
  863. def _sort_paths_by_mtime(self, paths: list[str]) -> list[str]:
  864. """Sort paths newest-first using metadata from the active backend."""
  865. if len(paths) < 2:
  866. return paths
  867. script = (
  868. 'while IFS= read -r item || [ -n "$item" ]; do '
  869. 'mtime="$(stat -c%Y "$item" 2>/dev/null '
  870. '|| stat -f%m "$item" 2>/dev/null || true)"; '
  871. "printf '%s\\n' \"${mtime:-0}\"; "
  872. "done"
  873. )
  874. result = self._exec(script, timeout=60, stdin_data="\n".join(paths))
  875. mtimes = []
  876. for line in result.stdout.splitlines()[:len(paths)]:
  877. try:
  878. mtimes.append(float(line))
  879. except ValueError:
  880. mtimes.append(0.0)
  881. # A path can disappear between traversal and stat. Keep such paths at
  882. # the end, preserving their existing relative order as a stable fallback.
  883. mtimes.extend([0.0] * (len(paths) - len(mtimes)))
  884. return [
  885. item
  886. for _, item in sorted(
  887. zip(mtimes, paths), key=lambda pair: pair[0], reverse=True
  888. )
  889. ]
  890. def _search_directories_with_find(
  891. self, pattern: str, path: str, fetch_limit: int
  892. ) -> tuple[list[str], Optional[str]]:
  893. """Return matching descendant directories using find, if available.
  894. ``rg --files`` and ``find -type f`` both enumerate files only, so
  895. directories — especially empty ones — are invisible (#54347). rg stays
  896. authoritative for which *files* survive ignore processing; this adds
  897. the matching directory entries so the tool honours its documented
  898. ls-replacement contract.
  899. """
  900. if fetch_limit <= 0 or not self._has_command("find"):
  901. return [], None
  902. escaped_path = self._escape_shell_arg(path)
  903. escaped_pattern = self._escape_shell_arg(pattern)
  904. q_roots = [escaped_path]
  905. hidden_prune = self._hidden_prune_expr(q_roots)
  906. cmd = (
  907. f"find -H {escaped_path} -mindepth 1 {hidden_prune} -o -type d "
  908. f"! -name '.*' -name {escaped_pattern} -printf '%T@ %p\\n' "
  909. f"2>/dev/null | sort -rn"
  910. )
  911. result = self._exec(cmd, timeout=30)
  912. stdout, limit_reason = _search_stdout_and_limit(result)
  913. if not stdout.strip() and not limit_reason:
  914. # Try without -printf (BSD find compatibility -- macOS)
  915. cmd_simple = (
  916. f"find -H {escaped_path} -mindepth 1 {hidden_prune} -o -type d "
  917. f"! -name '.*' -name {escaped_pattern} 2>/dev/null"
  918. )
  919. result = self._exec(cmd_simple, timeout=30)
  920. stdout, limit_reason = _search_stdout_and_limit(result)
  921. dirs = []
  922. for line in stdout.strip().split('\n'):
  923. if not line:
  924. continue
  925. parts = line.split(' ', 1)
  926. if len(parts) == 2 and parts[0].replace('.', '').isdigit():
  927. dirs.append(parts[1])
  928. else:
  929. dirs.append(line)
  930. if dirs:
  931. # An explicitly selected hidden root keeps its (hidden) subtree;
  932. # any other hidden descendants stay invisible, matching rg.
  933. if self._root_under_hidden_dir(path):
  934. dirs = self._filter_hidden_descendants(dirs, Path(path))
  935. dirs = self._filter_gitignored_paths(dirs, path)
  936. return self._sort_paths_by_mtime(dirs)[:fetch_limit], limit_reason
  937. def _search_files(self, pattern: str, path: str | list[str], limit: int, offset: int,
  938. order: str = "discovery") -> SearchResult:
  939. """Search for files by name (glob-like) across one or more roots: rg --files,
  940. else a bounded find. ``order``: "discovery" (fast, bounded) or "modified"
  941. (exact global newest-first; needs rg 14+ or GNU find)."""
  942. search_pattern = pattern if (not pattern.startswith('**/') and '/' not in pattern) \
  943. else pattern.split('/')[-1]
  944. roots = [path] if isinstance(path, str) else path
  945. if not roots:
  946. return SearchResult(error="File search requires at least one search root in 'path'.")
  947. # Prefer ripgrep: bounded parallel traversal with ignore semantics. Resolve
  948. # the engine and exact-order capability BEFORE admission so a queued request
  949. # does not occupy a root while doing command discovery.
  950. if self._has_command("rg"):
  951. rg_executable = self._resolve_command("rg") or "rg"
  952. if order == "modified":
  953. capability_error = self._modified_rg_capability_error(rg_executable)
  954. if capability_error:
  955. return SearchResult(error=capability_error)
  956. keys = _filename_search_root_keys(self.env, roots, self.cwd)
  957. if not _acquire_filename_search_roots(keys):
  958. return SearchResult(error=_ADMISSION_INTERRUPTED_ERROR)
  959. try:
  960. return self._search_files_rg(search_pattern, path, limit, offset, order,
  961. rg_executable=rg_executable)
  962. finally:
  963. _release_filename_search_roots(keys)
  964. # A local find rooted at/above $HOME or a filesystem root can take minutes and
  965. # prompt on protected paths: refuse before invoking find.
  966. if any(self._is_broad_local_search_root(root) for root in roots):
  967. return SearchResult(error=(
  968. "Broad local file search without ripgrep is disabled because "
  969. "find cannot keep this traversal safely bounded. Install "
  970. "ripgrep or search a narrower directory."))
  971. if not self._has_command("find"):
  972. return SearchResult(
  973. error="File search requires 'rg' (ripgrep) or 'find'. "
  974. "Install ripgrep for best results: "
  975. "https://github.com/BurntSushi/ripgrep#installation")
  976. # Prune hidden descendant dirs (and hidden files, matching rg's default) while
  977. # still allowing an explicitly selected hidden root; dash-prefixed roots get
  978. # ``./`` so find doesn't parse them as options.
  979. find_roots = [f"./{root}" if root.startswith("-") else root for root in roots]
  980. q_roots = [self._escape_shell_arg(root) for root in find_roots]
  981. hidden_prune = f" {self._hidden_prune_expr(q_roots)} -o"
  982. protected_paths = [absolute for _r, _rel, absolute in self._effective_macos_search_exclusions(roots)]
  983. protected_prune = f" {self._prune_expr(protected_paths)} -o" if protected_paths else ""
  984. fetch_limit = offset + limit + 1
  985. # ``-H`` follows a symlink handed in as an OPERAND, and only an operand: without
  986. # it ``find <link> -type f`` tests the link itself, so ``target="files"`` listed
  987. # nothing at all for a symlinked root - total_count: 0, no error, no warning,
  988. # indistinguishable from an empty directory - while ``rg --files`` followed the
  989. # same argument (#116270). Following the operand inside the command is also what
  990. # covers a link that only exists on the execution host (SSH/container), with no
  991. # probe of its own. That contract forbids ``-mindepth 1`` here: with ``-H`` the
  992. # followed operand IS depth 0, so it would exclude the root itself — the only
  993. # match when the root is a symlink to a file. The root directory itself is
  994. # excluded by path identity instead, inside the ``-type d`` arm (the ``-type f``
  995. # arm must keep the depth-0 match). ``-path`` matching is literal against the
  996. # path as find echoes it, which is the operand verbatim, so the operand needs
  997. # its glob metacharacters escaped.
  998. root_exemptions = "".join(
  999. f" ! -path {self._escape_shell_arg(root)}"
  1000. for root in _find_literal_path_expressions(find_roots)
  1001. )
  1002. base = (f"find -H {' '.join(q_roots)}{protected_prune}{hidden_prune} "
  1003. f"\\( -type f -o \\( -type d{root_exemptions} \\) \\) "
  1004. f"! -name '.*' -name {self._escape_shell_arg(search_pattern)}")
  1005. # find's stderr is not piped into sort/head, so ``find: '<dir>': Permission
  1006. # denied`` lines reach the merged output without using up result rows.
  1007. if order == "modified":
  1008. cmd = "set -o pipefail; " + base + f" -printf '%T@ %p\\n' | sort -rn | head -n {fetch_limit}"
  1009. else:
  1010. cmd = "set -o pipefail; " + base + f" -print | head -n {fetch_limit}"
  1011. keys = _filename_search_root_keys(self.env, roots, self.cwd)
  1012. if not _acquire_filename_search_roots(keys):
  1013. return SearchResult(error=_ADMISSION_INTERRUPTED_ERROR)
  1014. try:
  1015. result = self._exec(cmd, timeout=60)
  1016. finally:
  1017. _release_filename_search_roots(keys)
  1018. stdout, limit_reason = _search_stdout_and_limit(result)
  1019. raw_files, diagnostics = _split_find_output(stdout, order)
  1020. failure, partial_warning = _find_exit_verdict(result.exit_code, raw_files, diagnostics,
  1021. fetch_limit, order)
  1022. if failure is not None:
  1023. return failure
  1024. from tools.environments.base import env_uses_host_filesystem
  1025. from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
  1026. if _IS_WINDOWS and env_uses_host_filesystem(self.env):
  1027. raw_files = [_msys_to_windows_path(file_path) for file_path in raw_files]
  1028. return SearchResult(
  1029. files=raw_files[offset:offset + limit], total_count=len(raw_files),
  1030. truncated=len(raw_files) > offset + limit or bool(limit_reason),
  1031. limit_reason=limit_reason, warning=partial_warning,
  1032. skipped_paths=_extract_unreadable_paths("\n".join(diagnostics)))
  1033. def _search_files_rg(self, pattern: str, path: str | list[str], limit: int, offset: int,
  1034. order: str = "discovery", rg_executable: Optional[str] = None) -> SearchResult:
  1035. """File-name search via ``rg --files`` (respects .gitignore, skips hidden dirs,
  1036. parallel walk). Discovery order stays bounded and fast; exact modification-time
  1037. ordering is explicit because it scans globally."""
  1038. # Wrap bare names so -g matches at any depth (equivalent to find -name).
  1039. glob_pattern = f"*{pattern}" if ('/' not in pattern and not pattern.startswith('*')) else pattern
  1040. roots = [path] if isinstance(path, str) else path
  1041. fetch_limit = limit + offset + 1
  1042. effective_exclusions = self._effective_macos_search_exclusions(roots)
  1043. scoped_common = None
  1044. command_roots = roots
  1045. if len(roots) > 1 and effective_exclusions and _posix_roots(roots):
  1046. # Several roots: rg globs are root-relative, so cd to the common ancestor
  1047. # and express roots + exclusions relative to it.
  1048. cwd = getattr(self.env, "cwd", None) or self.cwd
  1049. absolute_roots = [
  1050. posixpath.normpath(root if posixpath.isabs(root) else posixpath.join(cwd, root))
  1051. for root in roots]
  1052. scoped_common = posixpath.commonpath(absolute_roots)
  1053. command_roots = [posixpath.relpath(root, scoped_common) for root in absolute_roots]
  1054. exclusion_terms = [
  1055. f"--glob {self._escape_shell_arg(f'!{posixpath.relpath(absolute, scoped_common)}/**')}"
  1056. for _r, _rel, absolute in effective_exclusions]
  1057. else:
  1058. exclusion_terms = [
  1059. f"--glob {self._escape_shell_arg(f'!{relative}/**')}"
  1060. for _r, relative, _abs in effective_exclusions]
  1061. exclusion_globs = " ".join(dict.fromkeys(exclusion_terms))
  1062. exclusion_args = f" {exclusion_globs}" if exclusion_globs else ""
  1063. rg_executable = rg_executable or self._resolve_command("rg")
  1064. if not rg_executable:
  1065. return SearchResult(error="File search requires ripgrep (rg).")
  1066. if order == "modified":
  1067. capability_error = self._modified_rg_capability_error(rg_executable)
  1068. if capability_error:
  1069. return SearchResult(error=capability_error)
  1070. rg = self._quote_executable(rg_executable)
  1071. sort_arg = " --sortr=modified" if order == "modified" else ""
  1072. root_args = " ".join(self._escape_native_tool_arg(root) for root in command_roots)
  1073. cd_prefix = f"cd {self._escape_shell_arg(scoped_common)} && " if scoped_common else ""
  1074. # ``--`` terminates options so a dash-prefixed root is never parsed as a flag.
  1075. rg_cmd = (f"{rg} --files{sort_arg} -g {self._escape_shell_arg(glob_pattern)}"
  1076. f"{exclusion_args} -- {root_args}")
  1077. result = self._run_rg_bounded([rg_cmd], fetch_limit, timeout=60, native_ok=not scoped_common,
  1078. merge_stderr=True,
  1079. shell_prefix=f"set -o pipefail; {cd_prefix}")
  1080. stdout, limit_reason = _search_stdout_and_limit(result)
  1081. # Files-mode transport mixes stderr into stdout, but file paths may contain
  1082. # spaces — the content-mode shape matcher (_SEARCH_OUTPUT_RE) would drop
  1083. # them as "diagnostics". rg/grep diagnostic lines always begin with
  1084. # "rg: "/"grep: " (continuation lines only matter on the fail-closed path),
  1085. # so classify by that prefix instead.
  1086. stdout_lines = stdout.splitlines()
  1087. diagnostics = "\n".join(
  1088. line for line in stdout_lines if line.lstrip().startswith(("rg: ", "grep: ")))
  1089. all_files = [
  1090. line for line in stdout_lines
  1091. if line and not line.lstrip().startswith(("rg: ", "grep: "))]
  1092. # Permission-denied diagnostics are partial-success signals independent of
  1093. # the exit code: exit 2 = traversal hit unreadable paths (degrade below),
  1094. # exit 124 = timed out while ALSO skipping them (warning rides along).
  1095. skip = _skipped_fields(diagnostics) if _is_permission_only_error(diagnostics) \
  1096. else {"skipped_paths": [], "skipped_omitted": 0}
  1097. permission_only = result.exit_code == 2 and bool(skip["skipped_paths"])
  1098. warning = _prepend_skipped_paths_warning(skip, None)
  1099. if scoped_common:
  1100. all_files = [
  1101. f if posixpath.isabs(f) else posixpath.normpath(posixpath.join(scoped_common, f))
  1102. for f in all_files]
  1103. bounded_sigpipe = result.exit_code == 141 and len(all_files) >= fetch_limit
  1104. if (result.exit_code not in {0, 1, 124} and not bounded_sigpipe
  1105. and not permission_only):
  1106. detail = diagnostics.strip() or "no diagnostics"
  1107. if order == "modified":
  1108. # ripgrep 14+ was already verified by the capability gate, so this
  1109. # failure is an invocation/traversal error, not a version problem.
  1110. return SearchResult(error=(
  1111. "Exact modification-time order search failed (ripgrep 14+ "
  1112. "required for --sortr=modified; the installed rg passed the "
  1113. f"version check, so this is an invocation failure: {detail}). "
  1114. "Use order='discovery'."))
  1115. return SearchResult(error=(
  1116. f"File search failed while running ripgrep: {detail}"))
  1117. # rg --files lists files only; supplement with matching directories so
  1118. # empty directories stay discoverable (#54347). Skip the extra traversal
  1119. # when rg already timed out — partial file results must not be presented
  1120. # as a coherent page. Only the single-root path supplements (a multi-root
  1121. # walk is already the merged view across roots).
  1122. if limit_reason != "search_timeout" and isinstance(path, str) and self._has_command("find"):
  1123. dirs, directory_limit_reason = self._search_directories_with_find(glob_pattern, path, fetch_limit)
  1124. if dirs:
  1125. existing = set(all_files)
  1126. merged_entries = self._sort_paths_by_mtime(
  1127. all_files + [d for d in dirs if d not in existing])
  1128. # Under exact modified order rg is authoritative for files;
  1129. # the merged view keeps the newest-first contract globally.
  1130. all_files = merged_entries
  1131. limit_reason = limit_reason or directory_limit_reason
  1132. if permission_only and not all_files:
  1133. return SearchResult(files=[], total_count=0,
  1134. warning=f"{warning} {_PERMISSION_SEARCH_HINT}", **skip)
  1135. return SearchResult(
  1136. files=all_files[offset:offset + limit], total_count=len(all_files),
  1137. truncated=len(all_files) > offset + limit or bool(limit_reason),
  1138. limit_reason=limit_reason, warning=warning, **skip)
  1139. def _search_content(self, pattern: str, path: str, file_glob: Optional[str],
  1140. limit: int, offset: int, output_mode: str, context: int) -> SearchResult:
  1141. """Content search: rg, else grep; attaches zero-match steering hints."""
  1142. used_rg = self._has_command('rg')
  1143. if used_rg:
  1144. result = self._search_with_rg(pattern, path, file_glob, limit, offset, output_mode, context,
  1145. rg_executable=self._resolve_command("rg") or "rg")
  1146. elif self._has_command('grep'):
  1147. result = self._search_with_grep(pattern, path, file_glob, limit, offset, output_mode, context)
  1148. else:
  1149. return SearchResult(
  1150. error="Content search requires ripgrep (rg) or grep. "
  1151. "Install ripgrep: https://github.com/BurntSushi/ripgrep#installation")
  1152. if (not result.error and result.total_count == 0
  1153. and not result.matches and not result.files and not result.counts):
  1154. try:
  1155. hint = self._zero_match_probe(pattern, path, file_glob)
  1156. except Exception:
  1157. hint = None
  1158. if hint:
  1159. result.warning = hint if not result.warning else f"{result.warning} {hint}"
  1160. # rg auto-enables --multiline for \n patterns, so the line-oriented
  1161. # explanation only applies to the grep fallback.
  1162. if used_rg:
  1163. return result
  1164. return _maybe_warn_line_oriented_newline_pattern(result, pattern)
  1165. def _run_search_pipeline(self, cmd_parts: list[str], output_mode: str, limit: int,
  1166. offset: int, context: int, warning: Optional[str] = None,
  1167. line_cap: bool = False) -> SearchResult:
  1168. """Run ``cmd_parts | head -n <fetch_limit>`` under pipefail and parse. Extra
  1169. rows report the true total (context mode also emits "--" separators, so
  1170. grab 200 more). pipefail keeps the engine's exit 2 alive across ``| head``
  1171. (a truncating head makes rg exit 0 / grep 141, which the ==2 guard ignores).
  1172. ``line_cap`` appends ``| cut -c1-2000`` for engines without --max-columns
  1173. (grep): bounds giant single-line matches at the pipe layer; skipped for
  1174. files_only/count where lines are paths/counts."""
  1175. fetch_limit = limit + offset + (200 if context > 0 else 0)
  1176. if line_cap: # grep/find pipelines: shell only, with the column cap
  1177. parts = cmd_parts + ["|", "head", "-n", str(fetch_limit)]
  1178. if output_mode not in ("files_only", "count"):
  1179. parts += ["|", "cut", "-c1-2000"]
  1180. result = self._exec("set -o pipefail; " + " ".join(parts), timeout=60)
  1181. else:
  1182. result = self._run_rg_bounded(cmd_parts, fetch_limit, timeout=60, merge_stderr=True,
  1183. shell_prefix="set -o pipefail; ")
  1184. return _parse_search_output(result, output_mode, limit, offset, context, warning=warning)
  1185. def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str],
  1186. limit: int, offset: int, output_mode: str, context: int,
  1187. rg_executable: Optional[str] = None) -> SearchResult:
  1188. """Search using ripgrep."""
  1189. rg_executable = rg_executable or self._resolve_command("rg")
  1190. if not rg_executable:
  1191. return SearchResult(error="Content search requires ripgrep (rg).")
  1192. cmd_parts = [self._quote_executable(rg_executable), "--line-number", "--no-heading", "--with-filename"]
  1193. # Giant-single-line containment (cline#13525): a match inside a multi-MB
  1194. # single-line dump makes rg emit the ENTIRE line (``head -n`` counts lines).
  1195. # --max-columns bounds each printed line at the rg layer; --max-columns-preview
  1196. # keeps a truncated prefix so the model still sees the hit. 2000 cols exceeds
  1197. # the 500-char content clamp, so nothing previously visible is lost.
  1198. if output_mode not in ("files_only", "count"):
  1199. cmd_parts.extend(["--max-columns", "2000", "--max-columns-preview"])
  1200. # A regex \n hard-errors in line-oriented mode; enable -U up front and say so.
  1201. multiline = _pattern_has_regex_newline(pattern)
  1202. if multiline:
  1203. cmd_parts.append("--multiline")
  1204. if context > 0:
  1205. cmd_parts.extend(["-C", str(context)])
  1206. cmd_parts.extend(self._rg_exclusion_globs(path))
  1207. if file_glob:
  1208. cmd_parts.extend(["--glob", self._escape_shell_arg(file_glob)])
  1209. if output_mode in _OUTPUT_MODE_FLAGS:
  1210. cmd_parts.append(_OUTPUT_MODE_FLAGS[output_mode])
  1211. cmd_parts.append(self._escape_shell_arg(pattern, translate_path=False))
  1212. # rg is a native Windows binary (winget/cargo/choco): needs C:/... not MSYS /c/...
  1213. cmd_parts.append(self._escape_native_tool_arg(path))
  1214. ml_note = (
  1215. "Pattern contains \\n — multiline mode (-U) was enabled automatically "
  1216. "so the regex can match across line boundaries."
  1217. ) if multiline else None
  1218. return self._run_search_pipeline(cmd_parts, output_mode, limit, offset, context, warning=ml_note)
  1219. def _grep_cmd(self, head: list[str], pattern: str, output_mode: str, context: int,
  1220. file_glob: Optional[str] = None) -> list[str]:
  1221. """``head`` + context/include/mode flags + quoted pattern (argument order is fixed)."""
  1222. parts = list(head)
  1223. if context > 0:
  1224. parts.extend(["-C", str(context)])
  1225. if file_glob:
  1226. parts.extend(["--include", self._escape_shell_arg(file_glob)])
  1227. if output_mode in _OUTPUT_MODE_FLAGS:
  1228. parts.append(_OUTPUT_MODE_FLAGS[output_mode])
  1229. parts.append(self._escape_shell_arg(pattern, translate_path=False))
  1230. return parts
  1231. def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str],
  1232. limit: int, offset: int, output_mode: str, context: int) -> SearchResult:
  1233. """Fallback search using grep."""
  1234. # grep's --exclude-dir matches BASENAMES anywhere, so it can't express "only
  1235. # the home-level Downloads"; route pruning through find's path-scoped -prune.
  1236. protected_paths = self._protected_prune_paths(path)
  1237. # grep applies --exclude-dir='.*' to the command-line root too (GNU grep: to
  1238. # every component of it), so a search rooted under a hidden dir such as
  1239. # ~/.hermes returns nothing (#18473); find's -prune only sees descendants.
  1240. if protected_paths or self._root_under_hidden_dir(path):
  1241. return self._search_with_grep_pruned(
  1242. pattern, path, file_glob, limit, offset, output_mode, context, protected_paths)
  1243. # -H forces filenames; -E matches rg regex behavior; --exclude-dir='.*'
  1244. # mirrors rg's hidden-dir default (.git/, .hub/index-cache/, ...).
  1245. cmd_parts = self._grep_cmd(["grep", "-rnHE", "--exclude-dir='.*'"], pattern, output_mode, context, file_glob)
  1246. # --exclude-dir applies to the root too, so "." would be excluded by '.*';
  1247. # anchor relative paths at the shell's live $PWD.
  1248. is_absolute = path.startswith(("/", "\\\\")) or bool(re.match(r"^[A-Za-z]:[\\/]", path))
  1249. if is_absolute:
  1250. search_root = self._escape_shell_arg(path)
  1251. else:
  1252. relative_path = path.removeprefix("./")
  1253. search_root = '"$PWD"'
  1254. if relative_path not in {"", "."}:
  1255. search_root += f"/{self._escape_shell_arg(relative_path)}"
  1256. cmd_parts.append(search_root)
  1257. return self._run_search_pipeline(cmd_parts, output_mode, limit, offset, context, line_cap=True)
  1258. def _search_with_grep_pruned(self, pattern: str, path: str, file_glob: Optional[str],
  1259. limit: int, offset: int, output_mode: str, context: int,
  1260. protected_paths: list[str]) -> SearchResult:
  1261. """grep fallback via ``find ... -prune -exec grep {} +``, used when the root needs
  1262. path-scoped pruning (macOS protected dirs) or is itself under a dot-directory
  1263. (#18473: grep's ``--exclude-dir='.*'`` would drop the root). Trade-off: find folds
  1264. grep's exit code, so a hard grep error surfaces as an empty result."""
  1265. grep_parts = self._grep_cmd(["grep", "-nHE"], pattern, output_mode, context)
  1266. q_root = self._escape_shell_arg(path or ".")
  1267. # ``-H``: follow a symlink handed in as the OPERAND (and only the operand). Without
  1268. # it ``find <link> -type f`` tests the link itself and hands grep nothing, so a
  1269. # symlinked root answered a confident ``total_count: 0`` on every platform (#116270).
  1270. find_parts = ["find", "-H", q_root]
  1271. if protected_paths:
  1272. find_parts.extend([self._prune_expr(protected_paths), "-o"])
  1273. find_parts.extend([self._hidden_prune_expr([q_root]), "-o", "-type f"])
  1274. if file_glob:
  1275. find_parts.extend(["-name", self._escape_shell_arg(file_glob)])
  1276. # stderr stays attached (``_exec`` merges it into stdout): find's
  1277. # ``Permission denied`` lines must reach the parser so an unreadable
  1278. # subtree beside a readable match is reported, not silently dropped.
  1279. find_parts.extend(["-exec", *grep_parts, "{}", "+"])
  1280. return self._run_search_pipeline(find_parts, output_mode, limit, offset, context, line_cap=True)