"""Content/file search tier for ``tools.file_operations``. ``ShellFileOperations`` inherits ``SearchMixin``; module-level helpers are pure (no I/O). """ import contextlib import os import posixpath import re import shlex import subprocess import sys import threading import time from pathlib import Path from typing import Any, List, Optional from agent.search_policy import SEARCH_PRUNE_DIR_NAMES from tools import interrupt as tool_interrupt from tools.file_operations_common import ExecuteResult, SearchMatch, SearchResult _MACOS_TCC_PROTECTED_HOME_DIRS = ( "Desktop", "Documents", "Downloads", "Library", "Movies", "Music", "Pictures", ) def _macos_protected_search_exclusions( path: str, *, cwd: Optional[str] = None, home: Optional[str] = None, platform: Optional[str] = None, ) -> list[str]: """Protected home dirs (relative to ``path``) below a broad macOS search root. Only an ANCESTOR search (``$HOME``, ``/Users``) gets exclusions, so recursive tools never trigger unattended TCC prompts; a search rooted inside a protected dir stays allowed. """ if (platform or sys.platform) != "darwin": return [] root = Path(path).expanduser() if not root.is_absolute(): root = Path(cwd or os.getcwd()) / root root = Path(os.path.normpath(str(root))) home_path = Path(os.path.normpath(str(Path(home or Path.home()).expanduser()))) exclusions: list[str] = [] for dirname in _MACOS_TCC_PROTECTED_HOME_DIRS: try: relative = (home_path / dirname).relative_to(root) except ValueError: continue if relative.parts: exclusions.append(relative.as_posix()) return exclusions # --- Filename-walk admission: one walk per (backend, root) at a time -------------- _FILENAME_SEARCH_ADMISSION = threading.Condition() _ACTIVE_FILENAME_SEARCH_ROOTS: set[tuple[str, str, str]] = set() _FILENAME_SEARCH_WAIT_SECONDS = 0.05 def _normalized_filename_search_root(env: Any, root: str, fallback_cwd: str) -> str: """Normalize a filename-walk root without resolving remote paths locally.""" from tools.environments.base import env_uses_host_filesystem from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path cwd = getattr(env, "cwd", None) or fallback_cwd if env_uses_host_filesystem(env): if _IS_WINDOWS: root = _msys_to_windows_path(root) cwd = _msys_to_windows_path(cwd) if not os.path.isabs(root): root = os.path.join(cwd, root) return os.path.normcase(os.path.abspath(os.path.normpath(root))) if not posixpath.isabs(root): root = posixpath.join(cwd, root) return posixpath.normpath(root) def _filename_search_root_keys(env: Any, roots: list[str], fallback_cwd: str) -> tuple[tuple[str, str, str], ...]: """Unique backend/root admission keys in deterministic order.""" env_type = type(env) return tuple(sorted({ (env_type.__module__, env_type.__qualname__, _normalized_filename_search_root(env, root, fallback_cwd)) for root in roots})) def _acquire_filename_search_roots(keys: tuple[tuple[str, str, str], ...]) -> bool: """Atomically claim every key, polling for thread-scoped interruption.""" with _FILENAME_SEARCH_ADMISSION: while any(key in _ACTIVE_FILENAME_SEARCH_ROOTS for key in keys): if tool_interrupt.is_interrupted(): return False _FILENAME_SEARCH_ADMISSION.wait(_FILENAME_SEARCH_WAIT_SECONDS) if tool_interrupt.is_interrupted(): return False if tool_interrupt.is_interrupted(): return False return tool_interrupt.run_if_not_interrupted(lambda: _ACTIVE_FILENAME_SEARCH_ROOTS.update(keys)) def _release_filename_search_roots(keys: tuple[tuple[str, str, str], ...]) -> None: """Release a completed walk and leave no idle per-root state behind.""" with _FILENAME_SEARCH_ADMISSION: _ACTIVE_FILENAME_SEARCH_ROOTS.difference_update(keys) _FILENAME_SEARCH_ADMISSION.notify_all() _ADMISSION_INTERRUPTED_ERROR = ( "File search was interrupted while waiting for another filename " "search on the same root. Retry when ready.") _SEARCH_TIMEOUT_MARKER_RE = re.compile(r"\n?\[Command timed out after \d+s\]\s*$") def _search_stdout_and_limit(result: ExecuteResult) -> tuple[str, Optional[str]]: """Return stdout cleaned for parsing and a limit reason for search timeouts.""" if result.exit_code == 124: return _SEARCH_TIMEOUT_MARKER_RE.sub("", result.stdout), "search_timeout" return result.stdout, None def _bundled_rg() -> Optional[str]: """Hermes's pinned ripgrep, forward-slashed for the shell transports, or None.""" import pm try: installed = pm.installed_package("ripgrep") except (OSError, RuntimeError, ValueError): return None binary = installed.binary if installed is not None else None return str(binary).replace("\\", "/") if binary is not None and binary.is_file() else None # A real rg/grep output line is a whitespace-free path token followed by ``:`` # (match/count), ``-`` (context), or nothing (files_only); tool diagnostics # ("rg: ...", indented carets) never match. _SEARCH_OUTPUT_RE = re.compile(r'^([A-Za-z]:)?[^\s:][^\n]*?[:\-]\d|^[^\s:][^\s]*$') def _split_tool_diagnostics(output: str) -> tuple[str, str]: """Separate rg/grep diagnostic lines from real match output → ``(diagnostics, payload)``. ``_exec`` merges stderr into stdout; classifying by SHAPE lets the exit-2 guard tell a pure failure (no payload) from a partial one (one unreadable file, others matched) and guarantees error text is never parsed as a match.""" diagnostics: list[str] = [] payload: list[str] = [] for line in output.split('\n'): if not line.strip(): continue # Prefix check first: a match path can contain "-" (".../pytest-686/..."). if line.lstrip().startswith(("rg: ", "grep: ", "find: ")): diagnostics.append(line) elif line == "--" or _SEARCH_OUTPUT_RE.match(line): payload.append(line) else: diagnostics.append(line) return '\n'.join(diagnostics), '\n'.join(payload) # One unreadable-path diagnostic, in every shape the search lanes produce: # rg: ``rg: : Permission denied (os error 13)`` # rg modified-sort walk: ``rg: : IO error for operation on : Permission denied (os error 13)`` # grep: ``grep: : Permission denied`` (no ``(os error N)`` suffix) # find: ``find: '': Permission denied`` (pruned grep lane; GNU find quotes the # path with ASCII ``'`` or, in a UTF-8 locale, with typographic quotes) # The ``(os error N)`` suffix is ripgrep's Rust io::Error spelling, so it is optional. # The tool prefix is required: the shell's own ``bash: /rg: Permission denied`` # (the engine binary itself is not executable) is a failed invocation, not a skip. _PERMISSION_DENIED_RE = re.compile( r"^(?Prg|grep|find): (?P.+?): " r"(?:IO error for operation on .+?: )?Permission denied(?: \(os error \d+\))?$") _SHELL_EXEC_FAILURE_CODES = frozenset({126, 127}) _FIND_QUOTES_RE = re.compile(r"^['\u2018]|['\u2019]$") # The native rg runner keeps at most this much stderr (a huge tree of unreadable dirs # would otherwise be held in memory whole) and appends one tally line for the rest. _STDERR_RETAIN_BYTES = 64 * 1024 _OMITTED_DIAGNOSTICS_RE = re.compile(r"^rg: \.\.\. (\d+) more diagnostic line\(s\) omitted$") def _omitted_diagnostics_line(count: int) -> str: return f"rg: ... {count} more diagnostic line(s) omitted" def _omitted_diagnostic_count(diagnostics: str) -> int: return sum(int(m.group(1)) for line in diagnostics.splitlines() if (m := _OMITTED_DIAGNOSTICS_RE.match(line.strip()))) def _is_permission_only_error(diagnostics: str) -> bool: """True when every diagnostic line reports an unreadable path (rg or grep). That is a partial-success condition — the tool still emits results for the readable remainder — not a broken invocation like a bad glob or regex. The native runner's omitted-lines tally is judged by the lines it kept.""" lines = [line.strip() for line in diagnostics.splitlines() if line.strip() and not _OMITTED_DIAGNOSTICS_RE.match(line.strip())] return bool(lines) and all(_PERMISSION_DENIED_RE.match(line) for line in lines) def _extract_unreadable_paths(diagnostics: str) -> list[str]: """Paths reported unreadable by rg or grep, deduplicated in first-seen order.""" paths: list[str] = [] seen: set[str] = set() for line in diagnostics.splitlines(): match = _PERMISSION_DENIED_RE.match(line.strip()) if not match: continue path = match.group("path").strip() if match.group("tool") == "find": path = _FIND_QUOTES_RE.sub("", path) if path and path not in seen: seen.add(path) paths.append(path) return paths def _compact_unreadable_paths(paths: list[str], keep: int = 3, omitted: int = 0) -> str: shown = ", ".join(paths[:keep]) extra = max(len(paths) - keep, 0) + omitted return shown + (f" (+{extra} more)" if extra > 0 else "") def _skipped_unreadable_warning(paths: list[str], omitted: int = 0) -> str: """``omitted`` counts diagnostics the native runner dropped past its retention cap.""" return ("Skipped " + str(len(paths) + omitted) + " unreadable path(s) with restricted " "permissions: " + _compact_unreadable_paths(paths, omitted=omitted) + ". Results may be incomplete; fix permissions or run as a user that " "can read them to include them.") def _skipped_fields(diagnostics: str) -> dict: """SearchResult skip fields from rg/grep/find diagnostics (nothing skipped → empty lists).""" paths = _extract_unreadable_paths(diagnostics) return {"skipped_paths": paths, "skipped_omitted": _omitted_diagnostic_count(diagnostics) if paths else 0} def _skipped_sentence(result: SearchResult) -> Optional[str]: """The skipped-paths sentence ``result.warning`` carries, rebuilt from its structured data.""" if not result.skipped_paths: return None return _skipped_unreadable_warning(result.skipped_paths, result.skipped_omitted) def _prepend_skipped_paths_warning(skip: dict, existing: Optional[str]) -> Optional[str]: """Warning naming the skipped unreadable paths, prepended to any existing warning.""" if not skip["skipped_paths"]: return existing warning = _skipped_unreadable_warning(skip["skipped_paths"], skip["skipped_omitted"]) return warning if not existing else f"{warning} {existing}" _PERMISSION_SEARCH_HINT = "Nothing matched in the readable part of the search scope." def _merge_child_warnings(children: list[SearchResult], has_results: bool) -> list[str]: """Combine per-root search warnings into one list: the skipped-path sentences collapse into one sentence composed from each child's structured skip data (summed count), the rest of the text is deduplicated in order. The nothing-matched hint is dropped once any root produced results.""" total, shown, rest = 0, [], [] for child in children: if not child.warning: continue warning, sentence = child.warning, _skipped_sentence(child) if sentence: total += len(child.skipped_paths) + child.skipped_omitted for path in child.skipped_paths: # only three are displayed; never scan thousands if len(shown) >= 3: break if path not in shown: shown.append(path) warning = warning.replace(sentence, "\0") for piece in warning.split("\0"): piece = piece.strip() if has_results: piece = piece.replace(_PERMISSION_SEARCH_HINT, "").strip() if piece and piece not in rest: rest.append(piece) if not total: return rest sentence = _skipped_unreadable_warning(shown, max(total - len(shown), 0)) return [sentence, *rest] _FIND_MODIFIED_ORDER_ERROR = ( "Exact modification-time order requires GNU find with " "-printf support; install ripgrep 14+ or use order='discovery'.") def _split_find_output(stdout: str, order: str) -> tuple[list[str], list[str]]: """``(paths, diagnostics)`` from a bounded find run; ``find: `` lines are stderr.""" raw_files: list[str] = [] diagnostics: list[str] = [] for line in stdout.splitlines(): if line.startswith("find: "): diagnostics.append(line) elif order == "modified": parts = line.split(" ", 1) if len(parts) == 2 and parts[0].replace(".", "", 1).isdigit(): raw_files.append(parts[1]) elif line: raw_files.append(line) return raw_files, diagnostics def _find_exit_verdict(exit_code: int, raw_files: list[str], diagnostics: list[str], fetch_limit: int, order: str) -> tuple[Optional[SearchResult], Optional[str]]: """``(terminal result or None, warning)`` for a bounded find run. Exit 141 is benign only when the payload proves head closed the pipe at the bound. find exits 1 when traversal hit unreadable dirs (EACCES) but still prints what it could read: a partial success, named from its diagnostics. Exit 1 with no payload is a real failure unless every diagnostic is such a skip (bad expressions and BSD find's missing -printf also exit 1).""" bounded_sigpipe = exit_code == 141 and len(raw_files) >= fetch_limit if exit_code not in {0, 1, 124} and not bounded_sigpipe: if order == "modified": return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None return SearchResult(error="File search failed while running bounded find traversal."), None diagnostic_text = "\n".join(diagnostics) skipped = _extract_unreadable_paths(diagnostic_text) warning = _skipped_unreadable_warning(skipped) if skipped else None if exit_code != 1: return None, warning if not raw_files and _is_permission_only_error(diagnostic_text): return SearchResult(total_count=0, warning=f"{warning} {_PERMISSION_SEARCH_HINT}", skipped_paths=skipped), None if not raw_files: if order == "modified": return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None detail = f": {diagnostics[0]}" if diagnostics else "." return SearchResult(error=f"File search failed while running bounded find traversal{detail}"), None # A non-permission diagnostic (a dir removed mid-walk) must not be reported as a permission # problem: name it, plus any permission skips, so the user chases the real cause. other = next((d.strip() for d in diagnostics if d.strip() and not _PERMISSION_DENIED_RE.match(d.strip())), None) if other: return None, f"Some paths could not be read: {other}. Results may be incomplete." + ( f" {warning}" if warning else "") return None, warning or ("find traversal skipped one or more unreadable paths (restricted " "permissions); results are partial.") def _drain_stderr_capped(pipe, chunks: list[bytes], omitted: list[int]) -> None: """Read ``pipe`` to EOF (rg must never block on a full stderr pipe) but keep only the first ``_STDERR_RETAIN_BYTES``; the first line is always kept so a lone huge diagnostic still names the failure. ``omitted[0]`` counts the dropped lines. Past the cap the first non-permission line is still kept, so a real failure behind a flood of permission errors is never classified as a permission-only partial success.""" kept = 0 kept_other = False for raw in pipe or (): if chunks and kept + len(raw) > _STDERR_RETAIN_BYTES: line = raw.decode("utf-8", "replace").strip() if not kept_other and line and not _PERMISSION_DENIED_RE.match(line): chunks.append(raw) kept_other = True continue omitted[0] += 1 continue chunks.append(raw) kept += len(raw) def _parse_search_context_line(line: str) -> tuple[str, int, str] | None: """Parse a ``path-line-content`` context line using the RIGHTMOST numeric separator (filenames may contain ``--`` segments): ``dir/file-12-name.py-8-context`` → (``dir/file-12-name.py``, 8, ``context``).""" if not line or line == "--": return None match = None for candidate in re.finditer(r'-(\d+)-', line): match = candidate if match is None or match.start() == 0: return None return line[:match.start()], int(match.group(1)), line[match.end():] _REGEX_NEWLINE_ESCAPE_RE = re.compile(r"(? bool: """True when a content regex wants to match a newline: a literal newline or a ``\\n`` escape with an ODD number of backslashes (``\\\\n`` is a literal backslash+n and must not count).""" return "\n" in pattern or bool(_REGEX_NEWLINE_ESCAPE_RE.search(pattern)) def _is_line_oriented_newline_error(error: Optional[str]) -> bool: """Return True for rg's hard error when multiline mode is required.""" return bool(error) and "literal \"\\n\" is not allowed" in error and "--multiline" in error def _maybe_warn_line_oriented_newline_pattern(result: SearchResult, pattern: str) -> SearchResult: """Attach a newline-regex warning only when search found no usable results.""" if result.total_count != 0 or not _pattern_has_regex_newline(pattern): return result if result.error and not _is_line_oriented_newline_error(result.error): return result result.error = None result.skipped_paths, result.skipped_omitted = [], 0 result.warning = ( "0 results found. Note: search_files content search is line-oriented " "and does not run ripgrep with -U/--multiline, so `\\n` in the regex " "does not match line breaks. Use context=N to inspect neighboring " "lines, or escape as `\\\\n` when searching for a literal backslash+n.") return result # Match lines are "file:lineno:content". Windows paths carry a drive letter # ("C:\path"), so a naive split(":") breaks — the regex handles both. _MATCH_LINE_RE = re.compile(r'^([A-Za-z]:)?(.*?):(\d+):(.*)$') # Output-mode → engine flag (identical for rg and grep). _OUTPUT_MODE_FLAGS = {"files_only": "-l", "count": "-c"} def _parse_search_output(result, output_mode: str, limit: int, offset: int, context: int, warning: Optional[str] = None) -> SearchResult: """Parse rg/grep ``| head`` output into a SearchResult (shared by both engines). Exit codes: 0=matches, 1=none, 2=error — but both tools return 2 on PARTIAL errors (one unreadable file), so an error is surfaced only when exit==2 AND no usable payload remains. ``warning`` is attached to files_only/content results.""" stdout, limit_reason = _search_stdout_and_limit(result) diagnostics, payload = _split_tool_diagnostics(stdout) if result.exit_code in _SHELL_EXEC_FAILURE_CODES: # The shell could not run the engine at all (126 not executable, 127 not found). return SearchResult(error=f"Search failed: {result.stdout.strip() or 'Search error'}", total_count=0) if result.exit_code == 2 and not payload.strip(): error_msg = diagnostics.strip() or result.stdout.strip() or "Search error" if _is_permission_only_error(error_msg): # Pure unreadable-path failure: rg read nothing but the invocation was # sound. Degrade to an explicit empty result instead of failing — the # model can still act on the named paths. skip = _skipped_fields(error_msg) warning = _prepend_skipped_paths_warning(skip, warning) if not warning: warning = _PERMISSION_SEARCH_HINT elif _PERMISSION_SEARCH_HINT not in warning: warning = f"{warning} {_PERMISSION_SEARCH_HINT}" return SearchResult(total_count=0, warning=warning, **skip) return SearchResult(error=f"Search failed: {error_msg}", total_count=0) skip = _skipped_fields(diagnostics) skipped_warning = _prepend_skipped_paths_warning(skip, None) # A non-permission error beside the skips (a path removed mid-search) is named too, not # folded into the permission warning. other_errors = [ln.strip() for ln in diagnostics.splitlines() if ln.lstrip().startswith(("rg: ", "grep: ")) and not _PERMISSION_DENIED_RE.match(ln.strip()) and not _OMITTED_DIAGNOSTICS_RE.match(ln.strip())] if result.exit_code == 2 and other_errors: error_warning = f"The search reported errors for some paths; results may be incomplete ({other_errors[0]})." skipped_warning = f"{error_warning} {skipped_warning}" if skipped_warning else error_warning if skipped_warning: warning = skipped_warning if not warning else f"{skipped_warning} {warning}" lines = [ln for ln in payload.strip().split('\n') if ln] if output_mode == "files_only": return SearchResult( files=lines[offset:offset + limit], total_count=len(lines), truncated=bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip) if output_mode == "count": counts = {} for line in lines: if ':' in line: path, n = line.rsplit(':', 1) try: counts[path] = int(n) except ValueError: pass return SearchResult( counts=counts, total_count=sum(counts.values()), truncated=bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip) matches = [] for line in lines: if line == "--": continue m = _MATCH_LINE_RE.match(line) if m: matches.append(SearchMatch( path=(m.group(1) or '') + m.group(2), line_number=int(m.group(3)), content=m.group(4)[:500], )) continue # Context lines only when requested, to avoid false positives on dashy paths. if context > 0: parsed = _parse_search_context_line(line) if parsed: matches.append(SearchMatch(path=parsed[0], line_number=parsed[1], content=parsed[2][:500])) total = len(matches) return SearchResult( matches=matches[offset:offset + limit], total_count=total, truncated=total > offset + limit or bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip, ) def _posix_roots(roots: list[str]) -> bool: """Darwin-only: every root is POSIX-shaped (no drive letter / backslash).""" return sys.platform == "darwin" and all( not re.match(r"^[A-Za-z]:[\\/]", root) and "\\" not in root for root in roots) def _find_literal_path_expressions(roots: list[str]) -> list[str]: """Escape each search root for find's ``-path`` test, which matches its PATTERN (glob) against the path find echoes — the operand verbatim. The operand reaches the shell already single-quoted; ``-path`` must compare literally, so ``*?[]`` in a root's own name need escaping (a directory literally named ``a[1]`` must be excluded by identity, not treated as a character class). Backslashes are left alone: find does not read them as escapes inside a bracket-free pattern, and doubling them would break matching on backslash-shaped roots.""" return [re.sub(r"([*?\[\]])", r"\\\1", root) for root in roots] class SearchMixin: """File-name and content search via rg with find/grep fallbacks. Requires ``_exec``, ``_has_command``, ``_expand_path``, ``_escape_shell_arg``, ``_escape_native_tool_arg``, ``env``, ``cwd``, ``_command_cache``, ``_rg_resolution_cache`` and ``_rg_modified_capability`` from the host class.""" # --- rg resolution -------------------------------------------------------- def _resolve_command(self, cmd: str) -> Optional[str]: """Resolve an executable in the command host's namespace. Ordinary commands keep the bool hit/miss cache; rg alone caches successful resolved paths and re-probes misses so a mid-session install becomes visible (with off-PATH Windows candidates: cargo, scoop, winget).""" if cmd != "rg": return cmd if self._has_command(cmd) else None cached = self._rg_resolution_cache.get(cmd) if cached: return cached from tools.environments.base import env_uses_host_filesystem from tools.environments.local import _IS_WINDOWS # A host search is Hermes's own work, so it runs the pinned ripgrep, never the # first rg on the terminal PATH, where the user's tools come first. on_host = env_uses_host_filesystem(self.env) bundled = _bundled_rg() if on_host else None if bundled: self._rg_resolution_cache[cmd] = bundled return bundled result = self._exec("command -v rg 2>/dev/null") if result.exit_code == 0 and result.stdout.strip(): resolved = result.stdout.strip().splitlines()[0] if resolved == "yes": # compatibility with old boolean-probe fakes resolved = "rg" self._rg_resolution_cache[cmd] = resolved return resolved if _IS_WINDOWS and on_host: user_profile = os.environ.get("USERPROFILE") or str(Path.home()) local_app_data = os.environ.get("LOCALAPPDATA") scoop = os.environ.get("SCOOP") or os.path.join(user_profile, "scoop") candidates = [ os.path.join(user_profile, ".cargo", "bin", "rg.exe"), os.path.join(scoop, "shims", "rg.exe"), ] if local_app_data: candidates.append(os.path.join(local_app_data, "Microsoft", "WinGet", "Links", "rg.exe")) for candidate in candidates: if os.path.isfile(candidate): resolved = candidate.replace("\\", "/") self._rg_resolution_cache[cmd] = resolved return resolved return None _RG_VERSION_RE = re.compile( r"(?m)^ripgrep\s+((?:0|[1-9]\d*))\." r"(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)" r"(?:-(?:(?:0|[1-9]\d*)|(?:[0-9A-Za-z-]*[A-Za-z-]" r"[0-9A-Za-z-]*))(?:\.(?:(?:0|[1-9]\d*)|" r"(?:[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)))*)?" r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" r"(?:\s+\(rev [^)]+\))?\s*$") def _modified_rg_capability_error(self, executable: str) -> Optional[str]: """Cached actionable error unless rg can sort exactly (full SemVer, >= 14).""" if executable in self._rg_modified_capability: return self._rg_modified_capability[executable] result = self._exec(f"{self._quote_executable(executable)} --version", timeout=10) match = self._RG_VERSION_RE.search(result.stdout or "") if result.exit_code == 0 and match and int(match.group(1)) >= 14: error = None else: error = ("Exact modification-time order requires ripgrep 14 or newer; " "upgrade ripgrep or use order='discovery'.") self._rg_modified_capability[executable] = error return error # --- native rg transport (local POSIX) -------------------------------------- def _run_rg_native(self, argv: list[str], fetch_limit: int, timeout: int, merge_stderr: bool = False) -> ExecuteResult: """Run ``argv`` (shell-quoted rg words) natively and stop reading after ``fetch_limit`` lines — the ``| head -n`` of the shell pipeline without the two bash spawns. ``shlex.split`` undoes the escaping the builders apply for the shell path, so both transports see identical arguments. Exit code and stdout follow the shell contract (rg 0/1/2; 124 on timeout with partial output; 130 on interrupt), so ``_parse_search_output`` is shared. Once the bound is reached rg is killed like ``head`` closing the pipe would. ``merge_stderr`` mirrors the shell path's stderr handling: kept for real searches (diagnostics name unreadable paths or the error), discarded (``2>/dev/null``) for probes. Like the shell's ``2>&1``-free ``| head``, kept stderr is drained on its own pipe so diagnostics never use up the ``fetch_limit`` lines meant for results; it is prepended to stdout.""" from tools.environments.local import _kill_process_group_posix, _make_run_env cwd = getattr(self.env, "cwd", None) or self.cwd args = shlex.split(" ".join(argv)) try: proc = subprocess.Popen( args, cwd=cwd, env=_make_run_env(self.env.env), stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE if merge_stderr else subprocess.DEVNULL, # detached-spawn: ok — awaited rg under a deadline; the group is killed on timeout or interrupt start_new_session=True) except OSError as exc: return ExecuteResult(stdout=f"rg: {exc}", exit_code=2) # Record the group at spawn, as LocalEnvironment._run_bash does: rg can exit between the # poll and the kill below, and on macOS a not-yet-reaped child has no pgid any more. with contextlib.suppress(ProcessLookupError): proc._hermes_pgid = os.getpgid(proc.pid) from gateway.status import get_process_start_time proc._hermes_pgid_start = get_process_start_time(proc.pid) # Drain on a thread so a silent rg (huge tree, no hits yet) cannot pin the # caller past the deadline or past a /stop; the waiter below owns both. lines: list[bytes] = [] bounded = threading.Event() def _drain() -> None: for raw in proc.stdout: lines.append(raw) if len(lines) >= fetch_limit: bounded.set() break drainer = threading.Thread(target=_drain, daemon=True) drainer.start() err_chunks: list[bytes] = [] err_omitted = [0] err_drainer = None if merge_stderr: err_pipe = proc.stderr err_drainer = threading.Thread( # health: allow HX012 -- pipe drainer, reads no profile-scoped state target=_drain_stderr_capped, args=(err_pipe, err_chunks, err_omitted), daemon=True) err_drainer.start() deadline = time.monotonic() + timeout exit_code: Optional[int] = None while True: drainer.join(0.05) if not drainer.is_alive() or bounded.is_set(): break if tool_interrupt.is_interrupted(): exit_code = 130 break if time.monotonic() > deadline: exit_code = 124 break if proc.poll() is None: _kill_process_group_posix(proc) # native lane is POSIX-only (gate above) proc.wait() drainer.join() proc.stdout.close() stdout = b"".join(lines).decode("utf-8", errors="replace") if err_drainer is not None: err_drainer.join() if proc.stderr is not None: proc.stderr.close() err_text = b"".join(err_chunks).decode("utf-8", errors="replace") if err_omitted[0]: err_text = err_text.rstrip("\n") + "\n" + _omitted_diagnostics_line(err_omitted[0]) + "\n" stdout = err_text + stdout if exit_code == 124: return ExecuteResult(stdout=stdout + f"\n[Command timed out after {timeout}s]", exit_code=124) if exit_code == 130: return ExecuteResult(stdout=stdout + "\n[Command interrupted]", exit_code=130) # A killed-at-bound rg reports a signal (negative returncode); head would have # left the pipeline at 0 unless rg itself already failed. return ExecuteResult(stdout=stdout, exit_code=0 if bounded.is_set() else proc.returncode) def _run_rg_bounded(self, words: list[str], fetch_limit: int, timeout: int, *, merge_stderr: bool = False, native_ok: bool = True, shell_prefix: str = "") -> ExecuteResult: """Run an rg command (shell-quoted words) and keep the first ``fetch_limit`` lines: natively on a local POSIX host, else through the backend shell as `` | head -n N``. ``native_ok=False`` keeps a form the native lane cannot express (the multi-root ``cd`` prefix); ``shell_prefix`` is shell-only.""" if native_ok and self._native_read_enabled(): return self._run_rg_native(words, fetch_limit, timeout, merge_stderr=merge_stderr) stderr = "" if merge_stderr else " 2>/dev/null" return self._exec(f"{shell_prefix}{' '.join(words)}{stderr} | head -n {fetch_limit}", timeout=timeout) def _quote_executable(self, executable: str) -> str: """Quote an executable without leaking controller path semantics.""" if re.fullmatch(r"[A-Za-z0-9_.-]+", executable): return executable from tools.environments.base import env_uses_host_filesystem if env_uses_host_filesystem(self.env): return self._escape_native_tool_arg(executable) return "'" + executable.replace("'", "'\"'\"'") + "'" # --- macOS protected-folder exclusions -------------------------------------- def _macos_search_exclusions(self, path: str) -> list[str]: """Protected descendants to prune for this search root, if any. Gated on ``env.is_local``: ``sys.platform``/``_HOME`` describe the CONTROLLER, but the search runs on ``env``'s host. Envs without the flag default to local semantics; pruning is a warning-carrying skip, never data loss.""" env = getattr(self, "env", None) if env is not None and getattr(env, "is_local", True) is False: return [] from tools import file_operations as _fo # lazy: _HOME is monkeypatched there cwd = getattr(self.env, "cwd", None) or self.cwd return _macos_protected_search_exclusions(path, cwd=cwd, home=_fo._HOME, platform=sys.platform) def _protected_prune_paths(self, path: str) -> list[str]: """Absolute-ish protected paths for find's ``-path ... -prune``.""" return [os.path.normpath(os.path.join(path, item)) for item in self._macos_search_exclusions(path)] def _effective_macos_search_exclusions(self, roots: list[str]) -> list[tuple[str, str, str]]: """Unique ``(root, relative, absolute)`` exclusions across ``roots``, never pruning a root the caller chose explicitly.""" cwd = getattr(self.env, "cwd", None) or self.cwd use_posix_paths = _posix_roots(roots) def normalized(root: str) -> str: if use_posix_paths: return posixpath.normpath(root if posixpath.isabs(root) else posixpath.join(cwd, root)) return os.path.normcase(os.path.abspath(os.path.normpath(root))) normalized_roots = [normalized(root) for root in roots] explicit_roots = set(normalized_roots) seen = set() effective = [] for root, normalized_root in zip(roots, normalized_roots): for relative in self._macos_search_exclusions(root): if use_posix_paths: absolute = key = posixpath.normpath(posixpath.join(normalized_root, relative)) else: absolute = os.path.normpath(os.path.join(root, relative)) key = os.path.normcase(os.path.abspath(absolute)) if key in explicit_roots or key in seen: continue seen.add(key) effective.append((root, relative, absolute)) return effective @staticmethod def _macos_protected_search_warning(paths: list[str]) -> str: skipped = ", ".join(os.path.basename(item) for item in paths) return ("Skipped macOS protected folders during broad search to avoid " f"an unattended privacy prompt: {skipped}. Search a protected " "folder directly when access is intentional.") @staticmethod def _hidden_prune_expr(q_roots: list[str]) -> str: """find clause pruning hidden dirs while keeping an explicitly selected dot-named root (dir or single file) — find echoes each start point as given, so ``! -path`` matches it.""" exemptions = "".join(f" ! -path {root}" for root in q_roots) return f"\\( -type d -name '.*'{exemptions} \\) -prune" def _prune_expr(self, protected_paths: list[str]) -> str: """find ``\\( -path A -o -path B \\) -prune`` clause for the protected dirs.""" terms = " -o ".join(f"-path {self._escape_shell_arg(item)}" for item in protected_paths) return f"\\( {terms} \\) -prune" def _root_under_hidden_dir(self, path: str) -> bool: """True when the search root or any ancestor is dot-named (``~/.hermes/skills``).""" root = _normalized_filename_search_root(self.env, path or ".", self.cwd) return any(part.startswith(".") and part not in (".", "..") for part in root.replace("\\", "/").split("/")) def _rg_exclusion_globs(self, path: str) -> list[str]: """``--glob '!/**'`` pairs excluding protected dirs from an rg run.""" out: list[str] = [] for item in self._macos_search_exclusions(path): out.extend(["--glob", self._escape_shell_arg(f"!{item}/**")]) return out def _path_exists_probe(self, path: str) -> ExecuteResult: """Existence probe; stdout contains "exists" or "not_found" (or the probe's ``cwd_error`` when the exec wrapper itself failed).""" if self._native_read_enabled(): full = path if os.path.isabs(path) else os.path.join(getattr(self.env, "cwd", None) or self.cwd, path) return ExecuteResult(stdout="exists" if os.path.exists(full) else "not_found") return self._exec(f"test -e {self._escape_shell_arg(path)} && echo exists || echo not_found") def _dispatch_search(self, pattern: str, path: str, target: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int, order: str = "discovery") -> SearchResult: if target == "files": return self._search_files(pattern, path, limit, offset, order) return self._search_content(pattern, path, file_glob, limit, offset, output_mode, context) def _path_not_found_result(self, path: str) -> SearchResult: """Error result for a missing search root, with nearby-entry suggestions.""" parent = os.path.dirname(path) or "." basename_query = os.path.basename(path) hint_parts = [f"Path not found: {path}"] parent_check = self._exec(f"test -d {self._escape_shell_arg(parent)} && echo yes || echo no") if "yes" in parent_check.stdout and basename_query: ls_result = self._exec(f"ls -1 {self._escape_shell_arg(parent)} 2>/dev/null | head -20") if ls_result.exit_code == 0 and ls_result.stdout.strip(): lq = basename_query.lower() candidates = [ os.path.join(parent, e) for e in ls_result.stdout.strip().split('\n') if e and (lq in e.lower() or e.lower() in lq or e.lower().startswith(lq[:3]))] if candidates: hint_parts.append("Similar paths: " + ", ".join(candidates[:5])) return SearchResult(error=". ".join(hint_parts), total_count=0) def _try_multi_path_search(self, pattern: str, path: str, target: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int, order: str = "discovery") -> Optional[SearchResult]: """Recover a not-found ``path`` that is really several paths in one string. Commas explicitly delimit paths (internal spaces preserved); without commas split on whitespace. Search every existing part, merge, and note skipped parts. None when it doesn't look like a multi-path string.""" if "," in path: parts = [part.strip() for part in path.split(",") if part.strip()] else: parts = path.split() if len(parts) < 2: return None existing, missing = [], [] for p in parts: expanded = self._expand_path(p) (existing if "exists" in self._path_exists_probe(expanded).stdout else missing).append(expanded) if not existing: return None if target == "files": # One global traversal across roots so modified ordering and pagination # are exact; root admission wraps the actual rg/find invocation. merged = self._search_files(pattern, existing, limit, offset, order) children = [merged] else: merged, children = SearchResult(), [] for root in existing: sub = self._search_content(pattern, root, file_glob, limit, offset, output_mode, context) if sub.error: return sub children.append(sub) merged.matches.extend(sub.matches) merged.files.extend(sub.files) merged.counts.update(sub.counts) merged.total_count += sub.total_count merged.truncated = merged.truncated or sub.truncated merged.matches = merged.matches[:limit] merged.files = merged.files[:limit] note = f"path contained {len(parts)} entries; searched {len(existing)} that exist" if missing: note += "; skipped missing: " + ", ".join(missing[:3]) if len(missing) > 3: note += f" (+{len(missing) - 3} more)" has_results = bool(merged.total_count or merged.matches or merged.files or merged.counts) warning_parts = [*_merge_child_warnings(children, has_results), note] if not merged.error: protected_paths = [absolute for _r, _rel, absolute in self._effective_macos_search_exclusions(existing)] if protected_paths: warning_parts.append(self._macos_protected_search_warning(protected_paths)) merged.warning = " ".join(warning_parts) return merged def _search_prune_glob_args(self) -> str: """rg globs pruning known heavyweight recursive subtrees. Both forms are needed: globs are relative to each rg root, so ``**/name/**`` alone misses an explicitly selected ``name/`` root. Names come from the shared scan policy — no second search-only list.""" globs = [] for dirname in sorted(SEARCH_PRUNE_DIR_NAMES): for prefix in ("", "**/"): globs.extend(("--glob", self._escape_shell_arg(f"!{prefix}{dirname}/**"))) return " ".join(globs) # (rg flags, message template) probes for a 0-match content search, in order. # The fixed-string probe only runs when the pattern has regex metacharacters. _ZERO_MATCH_PROBES = ( ("-i", "0 exact matches, but {total} case-insensitive match(es) in {n} file(s): " "{paths} — the pattern's casing may be wrong."), # rg skips dotdirs and .gitignore'd files by default; say so instead of a bare zero. ("--hidden --no-ignore", "0 matches in visible files, but {total} match(es) in {n} " "hidden or gitignored file(s): {paths} — these are excluded by default."), ("-F", "0 regex matches, but {total} literal match(es) in {n} file(s): {paths} — the " "pattern contains regex metacharacters that likely need escaping " "(or pass a simpler substring)."), ) def _zero_match_probe(self, pattern: str, path: str, file_glob: Optional[str]) -> Optional[str]: """Steering hint for a 0-match content search, or None: a bare zero gives the model nothing to act on, so run cheap count-only rg probes (case-insensitive, hidden/ignored, fixed-string) and report the first that hits.""" rg_executable = self._resolve_command('rg') if not rg_executable: return None rg = self._quote_executable(rg_executable) has_meta = bool(re.search(r"[.\[\](){}?*+^$\\|]", pattern)) glob_expr = f" --glob {self._escape_shell_arg(file_glob)}" if file_glob else "" for flags, template in self._ZERO_MATCH_PROBES: if flags == "-F" and not has_meta: continue # The hidden/ignored probe keeps --no-ignore so project-local ignored # files stay diagnosable, but prunes heavyweight trees before rg recurses. if flags.startswith("--hidden"): glob_expr_probe = f"{glob_expr} {self._search_prune_glob_args()}" else: glob_expr_probe = glob_expr probe_words = [rg, flags, "--count-matches", glob_expr_probe, self._escape_shell_arg(pattern, translate_path=False), self._escape_native_tool_arg(path)] probe = self._run_rg_bounded(probe_words, 50, timeout=30) total, per_file = 0, [] for line in (probe.stdout or "").strip().splitlines(): p, _sep, n = line.rpartition(":") if n.isdigit(): total += int(n) per_file.append(p) if total > 0: extra = len(per_file) - 5 paths = ", ".join(per_file[:5]) + (f" (+{extra} more)" if extra > 0 else "") return template.format(total=total, n=len(per_file), paths=paths) return None def _is_broad_local_search_root(self, path: str) -> bool: """Whether a no-rg LOCAL root (filesystem root, $HOME or an ancestor of it) is unsafe for recursive find. Controller paths never classify remotes.""" from tools.environments.base import env_uses_host_filesystem from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path if not env_uses_host_filesystem(self.env): return False def normalized(value: str) -> str: if _IS_WINDOWS: value = _msys_to_windows_path(value).replace("\\", "/") if not os.path.isabs(value): value = os.path.join(getattr(self.env, "cwd", None) or self.cwd, value) # Classify the linked target, not the link: ``find -H`` now follows an # operand symlink, so a link pointing at $HOME (or at the filesystem root) # must not slip a recursive find past this guard (#116270). Local-only by # the isinstance check above, so this resolves on the host that runs find. return os.path.normcase(os.path.realpath(value)) from tools import file_operations as _fo # lazy: _HOME is monkeypatched there root = normalized(path) home = normalized(_fo._HOME) drive = os.path.splitdrive(root)[0] anchor = drive + os.sep if drive else os.path.abspath(os.sep) if root == os.path.normcase(anchor): return True try: common = os.path.commonpath([root, home]) except ValueError: return False return root == home or common == root def _filter_hidden_descendants(self, paths: list[str], root: Path) -> list[str]: """Drop hidden descendants while allowing an explicit hidden search root.""" normalized_root = root.resolve() filtered = [] for item_path in paths: try: rel_parts = Path(item_path).resolve().relative_to(normalized_root).parts except ValueError: rel_parts = Path(item_path).parts if any(part not in {".", ".."} and part.startswith(".") for part in rel_parts): continue filtered.append(item_path) return filtered def _filter_gitignored_paths(self, paths: list[str], root: str) -> list[str]: """Drop paths ignored by git when *root* is inside a git worktree.""" if not paths: return paths escaped_root = self._escape_shell_arg(root) probe = self._exec( f"git -C {escaped_root} rev-parse --is-inside-work-tree >/dev/null 2>&1", timeout=10, ) if probe.exit_code != 0: return paths result = self._exec( f"git -C {escaped_root} check-ignore --stdin", timeout=10, stdin_data="\n".join(paths), ) ignored = set(result.stdout.splitlines()) return [item_path for item_path in paths if item_path not in ignored] def _sort_paths_by_mtime(self, paths: list[str]) -> list[str]: """Sort paths newest-first using metadata from the active backend.""" if len(paths) < 2: return paths script = ( 'while IFS= read -r item || [ -n "$item" ]; do ' 'mtime="$(stat -c%Y "$item" 2>/dev/null ' '|| stat -f%m "$item" 2>/dev/null || true)"; ' "printf '%s\\n' \"${mtime:-0}\"; " "done" ) result = self._exec(script, timeout=60, stdin_data="\n".join(paths)) mtimes = [] for line in result.stdout.splitlines()[:len(paths)]: try: mtimes.append(float(line)) except ValueError: mtimes.append(0.0) # A path can disappear between traversal and stat. Keep such paths at # the end, preserving their existing relative order as a stable fallback. mtimes.extend([0.0] * (len(paths) - len(mtimes))) return [ item for _, item in sorted( zip(mtimes, paths), key=lambda pair: pair[0], reverse=True ) ] def _search_directories_with_find( self, pattern: str, path: str, fetch_limit: int ) -> tuple[list[str], Optional[str]]: """Return matching descendant directories using find, if available. ``rg --files`` and ``find -type f`` both enumerate files only, so directories — especially empty ones — are invisible (#54347). rg stays authoritative for which *files* survive ignore processing; this adds the matching directory entries so the tool honours its documented ls-replacement contract. """ if fetch_limit <= 0 or not self._has_command("find"): return [], None escaped_path = self._escape_shell_arg(path) escaped_pattern = self._escape_shell_arg(pattern) q_roots = [escaped_path] hidden_prune = self._hidden_prune_expr(q_roots) cmd = ( f"find -H {escaped_path} -mindepth 1 {hidden_prune} -o -type d " f"! -name '.*' -name {escaped_pattern} -printf '%T@ %p\\n' " f"2>/dev/null | sort -rn" ) result = self._exec(cmd, timeout=30) stdout, limit_reason = _search_stdout_and_limit(result) if not stdout.strip() and not limit_reason: # Try without -printf (BSD find compatibility -- macOS) cmd_simple = ( f"find -H {escaped_path} -mindepth 1 {hidden_prune} -o -type d " f"! -name '.*' -name {escaped_pattern} 2>/dev/null" ) result = self._exec(cmd_simple, timeout=30) stdout, limit_reason = _search_stdout_and_limit(result) dirs = [] for line in stdout.strip().split('\n'): if not line: continue parts = line.split(' ', 1) if len(parts) == 2 and parts[0].replace('.', '').isdigit(): dirs.append(parts[1]) else: dirs.append(line) if dirs: # An explicitly selected hidden root keeps its (hidden) subtree; # any other hidden descendants stay invisible, matching rg. if self._root_under_hidden_dir(path): dirs = self._filter_hidden_descendants(dirs, Path(path)) dirs = self._filter_gitignored_paths(dirs, path) return self._sort_paths_by_mtime(dirs)[:fetch_limit], limit_reason def _search_files(self, pattern: str, path: str | list[str], limit: int, offset: int, order: str = "discovery") -> SearchResult: """Search for files by name (glob-like) across one or more roots: rg --files, else a bounded find. ``order``: "discovery" (fast, bounded) or "modified" (exact global newest-first; needs rg 14+ or GNU find).""" search_pattern = pattern if (not pattern.startswith('**/') and '/' not in pattern) \ else pattern.split('/')[-1] roots = [path] if isinstance(path, str) else path if not roots: return SearchResult(error="File search requires at least one search root in 'path'.") # Prefer ripgrep: bounded parallel traversal with ignore semantics. Resolve # the engine and exact-order capability BEFORE admission so a queued request # does not occupy a root while doing command discovery. if self._has_command("rg"): rg_executable = self._resolve_command("rg") or "rg" if order == "modified": capability_error = self._modified_rg_capability_error(rg_executable) if capability_error: return SearchResult(error=capability_error) keys = _filename_search_root_keys(self.env, roots, self.cwd) if not _acquire_filename_search_roots(keys): return SearchResult(error=_ADMISSION_INTERRUPTED_ERROR) try: return self._search_files_rg(search_pattern, path, limit, offset, order, rg_executable=rg_executable) finally: _release_filename_search_roots(keys) # A local find rooted at/above $HOME or a filesystem root can take minutes and # prompt on protected paths: refuse before invoking find. if any(self._is_broad_local_search_root(root) for root in roots): return SearchResult(error=( "Broad local file search without ripgrep is disabled because " "find cannot keep this traversal safely bounded. Install " "ripgrep or search a narrower directory.")) if not self._has_command("find"): return SearchResult( error="File search requires 'rg' (ripgrep) or 'find'. " "Install ripgrep for best results: " "https://github.com/BurntSushi/ripgrep#installation") # Prune hidden descendant dirs (and hidden files, matching rg's default) while # still allowing an explicitly selected hidden root; dash-prefixed roots get # ``./`` so find doesn't parse them as options. find_roots = [f"./{root}" if root.startswith("-") else root for root in roots] q_roots = [self._escape_shell_arg(root) for root in find_roots] hidden_prune = f" {self._hidden_prune_expr(q_roots)} -o" protected_paths = [absolute for _r, _rel, absolute in self._effective_macos_search_exclusions(roots)] protected_prune = f" {self._prune_expr(protected_paths)} -o" if protected_paths else "" fetch_limit = offset + limit + 1 # ``-H`` follows a symlink handed in as an OPERAND, and only an operand: without # it ``find -type f`` tests the link itself, so ``target="files"`` listed # nothing at all for a symlinked root - total_count: 0, no error, no warning, # indistinguishable from an empty directory - while ``rg --files`` followed the # same argument (#116270). Following the operand inside the command is also what # covers a link that only exists on the execution host (SSH/container), with no # probe of its own. That contract forbids ``-mindepth 1`` here: with ``-H`` the # followed operand IS depth 0, so it would exclude the root itself — the only # match when the root is a symlink to a file. The root directory itself is # excluded by path identity instead, inside the ``-type d`` arm (the ``-type f`` # arm must keep the depth-0 match). ``-path`` matching is literal against the # path as find echoes it, which is the operand verbatim, so the operand needs # its glob metacharacters escaped. root_exemptions = "".join( f" ! -path {self._escape_shell_arg(root)}" for root in _find_literal_path_expressions(find_roots) ) base = (f"find -H {' '.join(q_roots)}{protected_prune}{hidden_prune} " f"\\( -type f -o \\( -type d{root_exemptions} \\) \\) " f"! -name '.*' -name {self._escape_shell_arg(search_pattern)}") # find's stderr is not piped into sort/head, so ``find: '': Permission # denied`` lines reach the merged output without using up result rows. if order == "modified": cmd = "set -o pipefail; " + base + f" -printf '%T@ %p\\n' | sort -rn | head -n {fetch_limit}" else: cmd = "set -o pipefail; " + base + f" -print | head -n {fetch_limit}" keys = _filename_search_root_keys(self.env, roots, self.cwd) if not _acquire_filename_search_roots(keys): return SearchResult(error=_ADMISSION_INTERRUPTED_ERROR) try: result = self._exec(cmd, timeout=60) finally: _release_filename_search_roots(keys) stdout, limit_reason = _search_stdout_and_limit(result) raw_files, diagnostics = _split_find_output(stdout, order) failure, partial_warning = _find_exit_verdict(result.exit_code, raw_files, diagnostics, fetch_limit, order) if failure is not None: return failure from tools.environments.base import env_uses_host_filesystem from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path if _IS_WINDOWS and env_uses_host_filesystem(self.env): raw_files = [_msys_to_windows_path(file_path) for file_path in raw_files] return SearchResult( files=raw_files[offset:offset + limit], total_count=len(raw_files), truncated=len(raw_files) > offset + limit or bool(limit_reason), limit_reason=limit_reason, warning=partial_warning, skipped_paths=_extract_unreadable_paths("\n".join(diagnostics))) def _search_files_rg(self, pattern: str, path: str | list[str], limit: int, offset: int, order: str = "discovery", rg_executable: Optional[str] = None) -> SearchResult: """File-name search via ``rg --files`` (respects .gitignore, skips hidden dirs, parallel walk). Discovery order stays bounded and fast; exact modification-time ordering is explicit because it scans globally.""" # Wrap bare names so -g matches at any depth (equivalent to find -name). glob_pattern = f"*{pattern}" if ('/' not in pattern and not pattern.startswith('*')) else pattern roots = [path] if isinstance(path, str) else path fetch_limit = limit + offset + 1 effective_exclusions = self._effective_macos_search_exclusions(roots) scoped_common = None command_roots = roots if len(roots) > 1 and effective_exclusions and _posix_roots(roots): # Several roots: rg globs are root-relative, so cd to the common ancestor # and express roots + exclusions relative to it. cwd = getattr(self.env, "cwd", None) or self.cwd absolute_roots = [ posixpath.normpath(root if posixpath.isabs(root) else posixpath.join(cwd, root)) for root in roots] scoped_common = posixpath.commonpath(absolute_roots) command_roots = [posixpath.relpath(root, scoped_common) for root in absolute_roots] exclusion_terms = [ f"--glob {self._escape_shell_arg(f'!{posixpath.relpath(absolute, scoped_common)}/**')}" for _r, _rel, absolute in effective_exclusions] else: exclusion_terms = [ f"--glob {self._escape_shell_arg(f'!{relative}/**')}" for _r, relative, _abs in effective_exclusions] exclusion_globs = " ".join(dict.fromkeys(exclusion_terms)) exclusion_args = f" {exclusion_globs}" if exclusion_globs else "" rg_executable = rg_executable or self._resolve_command("rg") if not rg_executable: return SearchResult(error="File search requires ripgrep (rg).") if order == "modified": capability_error = self._modified_rg_capability_error(rg_executable) if capability_error: return SearchResult(error=capability_error) rg = self._quote_executable(rg_executable) sort_arg = " --sortr=modified" if order == "modified" else "" root_args = " ".join(self._escape_native_tool_arg(root) for root in command_roots) cd_prefix = f"cd {self._escape_shell_arg(scoped_common)} && " if scoped_common else "" # ``--`` terminates options so a dash-prefixed root is never parsed as a flag. rg_cmd = (f"{rg} --files{sort_arg} -g {self._escape_shell_arg(glob_pattern)}" f"{exclusion_args} -- {root_args}") result = self._run_rg_bounded([rg_cmd], fetch_limit, timeout=60, native_ok=not scoped_common, merge_stderr=True, shell_prefix=f"set -o pipefail; {cd_prefix}") stdout, limit_reason = _search_stdout_and_limit(result) # Files-mode transport mixes stderr into stdout, but file paths may contain # spaces — the content-mode shape matcher (_SEARCH_OUTPUT_RE) would drop # them as "diagnostics". rg/grep diagnostic lines always begin with # "rg: "/"grep: " (continuation lines only matter on the fail-closed path), # so classify by that prefix instead. stdout_lines = stdout.splitlines() diagnostics = "\n".join( line for line in stdout_lines if line.lstrip().startswith(("rg: ", "grep: "))) all_files = [ line for line in stdout_lines if line and not line.lstrip().startswith(("rg: ", "grep: "))] # Permission-denied diagnostics are partial-success signals independent of # the exit code: exit 2 = traversal hit unreadable paths (degrade below), # exit 124 = timed out while ALSO skipping them (warning rides along). skip = _skipped_fields(diagnostics) if _is_permission_only_error(diagnostics) \ else {"skipped_paths": [], "skipped_omitted": 0} permission_only = result.exit_code == 2 and bool(skip["skipped_paths"]) warning = _prepend_skipped_paths_warning(skip, None) if scoped_common: all_files = [ f if posixpath.isabs(f) else posixpath.normpath(posixpath.join(scoped_common, f)) for f in all_files] bounded_sigpipe = result.exit_code == 141 and len(all_files) >= fetch_limit if (result.exit_code not in {0, 1, 124} and not bounded_sigpipe and not permission_only): detail = diagnostics.strip() or "no diagnostics" if order == "modified": # ripgrep 14+ was already verified by the capability gate, so this # failure is an invocation/traversal error, not a version problem. return SearchResult(error=( "Exact modification-time order search failed (ripgrep 14+ " "required for --sortr=modified; the installed rg passed the " f"version check, so this is an invocation failure: {detail}). " "Use order='discovery'.")) return SearchResult(error=( f"File search failed while running ripgrep: {detail}")) # rg --files lists files only; supplement with matching directories so # empty directories stay discoverable (#54347). Skip the extra traversal # when rg already timed out — partial file results must not be presented # as a coherent page. Only the single-root path supplements (a multi-root # walk is already the merged view across roots). if limit_reason != "search_timeout" and isinstance(path, str) and self._has_command("find"): dirs, directory_limit_reason = self._search_directories_with_find(glob_pattern, path, fetch_limit) if dirs: existing = set(all_files) merged_entries = self._sort_paths_by_mtime( all_files + [d for d in dirs if d not in existing]) # Under exact modified order rg is authoritative for files; # the merged view keeps the newest-first contract globally. all_files = merged_entries limit_reason = limit_reason or directory_limit_reason if permission_only and not all_files: return SearchResult(files=[], total_count=0, warning=f"{warning} {_PERMISSION_SEARCH_HINT}", **skip) return SearchResult( files=all_files[offset:offset + limit], total_count=len(all_files), truncated=len(all_files) > offset + limit or bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip) def _search_content(self, pattern: str, path: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int) -> SearchResult: """Content search: rg, else grep; attaches zero-match steering hints.""" used_rg = self._has_command('rg') if used_rg: result = self._search_with_rg(pattern, path, file_glob, limit, offset, output_mode, context, rg_executable=self._resolve_command("rg") or "rg") elif self._has_command('grep'): result = self._search_with_grep(pattern, path, file_glob, limit, offset, output_mode, context) else: return SearchResult( error="Content search requires ripgrep (rg) or grep. " "Install ripgrep: https://github.com/BurntSushi/ripgrep#installation") if (not result.error and result.total_count == 0 and not result.matches and not result.files and not result.counts): try: hint = self._zero_match_probe(pattern, path, file_glob) except Exception: hint = None if hint: result.warning = hint if not result.warning else f"{result.warning} {hint}" # rg auto-enables --multiline for \n patterns, so the line-oriented # explanation only applies to the grep fallback. if used_rg: return result return _maybe_warn_line_oriented_newline_pattern(result, pattern) def _run_search_pipeline(self, cmd_parts: list[str], output_mode: str, limit: int, offset: int, context: int, warning: Optional[str] = None, line_cap: bool = False) -> SearchResult: """Run ``cmd_parts | head -n `` under pipefail and parse. Extra rows report the true total (context mode also emits "--" separators, so grab 200 more). pipefail keeps the engine's exit 2 alive across ``| head`` (a truncating head makes rg exit 0 / grep 141, which the ==2 guard ignores). ``line_cap`` appends ``| cut -c1-2000`` for engines without --max-columns (grep): bounds giant single-line matches at the pipe layer; skipped for files_only/count where lines are paths/counts.""" fetch_limit = limit + offset + (200 if context > 0 else 0) if line_cap: # grep/find pipelines: shell only, with the column cap parts = cmd_parts + ["|", "head", "-n", str(fetch_limit)] if output_mode not in ("files_only", "count"): parts += ["|", "cut", "-c1-2000"] result = self._exec("set -o pipefail; " + " ".join(parts), timeout=60) else: result = self._run_rg_bounded(cmd_parts, fetch_limit, timeout=60, merge_stderr=True, shell_prefix="set -o pipefail; ") return _parse_search_output(result, output_mode, limit, offset, context, warning=warning) def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int, rg_executable: Optional[str] = None) -> SearchResult: """Search using ripgrep.""" rg_executable = rg_executable or self._resolve_command("rg") if not rg_executable: return SearchResult(error="Content search requires ripgrep (rg).") cmd_parts = [self._quote_executable(rg_executable), "--line-number", "--no-heading", "--with-filename"] # Giant-single-line containment (cline#13525): a match inside a multi-MB # single-line dump makes rg emit the ENTIRE line (``head -n`` counts lines). # --max-columns bounds each printed line at the rg layer; --max-columns-preview # keeps a truncated prefix so the model still sees the hit. 2000 cols exceeds # the 500-char content clamp, so nothing previously visible is lost. if output_mode not in ("files_only", "count"): cmd_parts.extend(["--max-columns", "2000", "--max-columns-preview"]) # A regex \n hard-errors in line-oriented mode; enable -U up front and say so. multiline = _pattern_has_regex_newline(pattern) if multiline: cmd_parts.append("--multiline") if context > 0: cmd_parts.extend(["-C", str(context)]) cmd_parts.extend(self._rg_exclusion_globs(path)) if file_glob: cmd_parts.extend(["--glob", self._escape_shell_arg(file_glob)]) if output_mode in _OUTPUT_MODE_FLAGS: cmd_parts.append(_OUTPUT_MODE_FLAGS[output_mode]) cmd_parts.append(self._escape_shell_arg(pattern, translate_path=False)) # rg is a native Windows binary (winget/cargo/choco): needs C:/... not MSYS /c/... cmd_parts.append(self._escape_native_tool_arg(path)) ml_note = ( "Pattern contains \\n — multiline mode (-U) was enabled automatically " "so the regex can match across line boundaries." ) if multiline else None return self._run_search_pipeline(cmd_parts, output_mode, limit, offset, context, warning=ml_note) def _grep_cmd(self, head: list[str], pattern: str, output_mode: str, context: int, file_glob: Optional[str] = None) -> list[str]: """``head`` + context/include/mode flags + quoted pattern (argument order is fixed).""" parts = list(head) if context > 0: parts.extend(["-C", str(context)]) if file_glob: parts.extend(["--include", self._escape_shell_arg(file_glob)]) if output_mode in _OUTPUT_MODE_FLAGS: parts.append(_OUTPUT_MODE_FLAGS[output_mode]) parts.append(self._escape_shell_arg(pattern, translate_path=False)) return parts def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int) -> SearchResult: """Fallback search using grep.""" # grep's --exclude-dir matches BASENAMES anywhere, so it can't express "only # the home-level Downloads"; route pruning through find's path-scoped -prune. protected_paths = self._protected_prune_paths(path) # grep applies --exclude-dir='.*' to the command-line root too (GNU grep: to # every component of it), so a search rooted under a hidden dir such as # ~/.hermes returns nothing (#18473); find's -prune only sees descendants. if protected_paths or self._root_under_hidden_dir(path): return self._search_with_grep_pruned( pattern, path, file_glob, limit, offset, output_mode, context, protected_paths) # -H forces filenames; -E matches rg regex behavior; --exclude-dir='.*' # mirrors rg's hidden-dir default (.git/, .hub/index-cache/, ...). cmd_parts = self._grep_cmd(["grep", "-rnHE", "--exclude-dir='.*'"], pattern, output_mode, context, file_glob) # --exclude-dir applies to the root too, so "." would be excluded by '.*'; # anchor relative paths at the shell's live $PWD. is_absolute = path.startswith(("/", "\\\\")) or bool(re.match(r"^[A-Za-z]:[\\/]", path)) if is_absolute: search_root = self._escape_shell_arg(path) else: relative_path = path.removeprefix("./") search_root = '"$PWD"' if relative_path not in {"", "."}: search_root += f"/{self._escape_shell_arg(relative_path)}" cmd_parts.append(search_root) return self._run_search_pipeline(cmd_parts, output_mode, limit, offset, context, line_cap=True) def _search_with_grep_pruned(self, pattern: str, path: str, file_glob: Optional[str], limit: int, offset: int, output_mode: str, context: int, protected_paths: list[str]) -> SearchResult: """grep fallback via ``find ... -prune -exec grep {} +``, used when the root needs path-scoped pruning (macOS protected dirs) or is itself under a dot-directory (#18473: grep's ``--exclude-dir='.*'`` would drop the root). Trade-off: find folds grep's exit code, so a hard grep error surfaces as an empty result.""" grep_parts = self._grep_cmd(["grep", "-nHE"], pattern, output_mode, context) q_root = self._escape_shell_arg(path or ".") # ``-H``: follow a symlink handed in as the OPERAND (and only the operand). Without # it ``find -type f`` tests the link itself and hands grep nothing, so a # symlinked root answered a confident ``total_count: 0`` on every platform (#116270). find_parts = ["find", "-H", q_root] if protected_paths: find_parts.extend([self._prune_expr(protected_paths), "-o"]) find_parts.extend([self._hidden_prune_expr([q_root]), "-o", "-type f"]) if file_glob: find_parts.extend(["-name", self._escape_shell_arg(file_glob)]) # stderr stays attached (``_exec`` merges it into stdout): find's # ``Permission denied`` lines must reach the parser so an unreadable # subtree beside a readable match is reported, not silently dropped. find_parts.extend(["-exec", *grep_parts, "{}", "+"]) return self._run_search_pipeline(find_parts, output_mode, limit, offset, context, line_cap=True)