| 1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432 |
- """Content/file search tier for ``tools.file_operations``.
- ``ShellFileOperations`` inherits ``SearchMixin``; module-level helpers are pure
- (no I/O).
- """
- import contextlib
- import os
- import posixpath
- import re
- import shlex
- import subprocess
- import sys
- import threading
- import time
- from pathlib import Path
- from typing import Any, List, Optional
- from agent.search_policy import SEARCH_PRUNE_DIR_NAMES
- from tools import interrupt as tool_interrupt
- from tools.file_operations_common import ExecuteResult, SearchMatch, SearchResult
- _MACOS_TCC_PROTECTED_HOME_DIRS = (
- "Desktop", "Documents", "Downloads", "Library", "Movies", "Music", "Pictures",
- )
- def _macos_protected_search_exclusions(
- path: str, *, cwd: Optional[str] = None, home: Optional[str] = None, platform: Optional[str] = None,
- ) -> list[str]:
- """Protected home dirs (relative to ``path``) below a broad macOS search root.
- Only an ANCESTOR search (``$HOME``, ``/Users``) gets exclusions, so recursive
- tools never trigger unattended TCC prompts; a search rooted inside a
- protected dir stays allowed.
- """
- if (platform or sys.platform) != "darwin":
- return []
- root = Path(path).expanduser()
- if not root.is_absolute():
- root = Path(cwd or os.getcwd()) / root
- root = Path(os.path.normpath(str(root)))
- home_path = Path(os.path.normpath(str(Path(home or Path.home()).expanduser())))
- exclusions: list[str] = []
- for dirname in _MACOS_TCC_PROTECTED_HOME_DIRS:
- try:
- relative = (home_path / dirname).relative_to(root)
- except ValueError:
- continue
- if relative.parts:
- exclusions.append(relative.as_posix())
- return exclusions
- # --- Filename-walk admission: one walk per (backend, root) at a time --------------
- _FILENAME_SEARCH_ADMISSION = threading.Condition()
- _ACTIVE_FILENAME_SEARCH_ROOTS: set[tuple[str, str, str]] = set()
- _FILENAME_SEARCH_WAIT_SECONDS = 0.05
- def _normalized_filename_search_root(env: Any, root: str, fallback_cwd: str) -> str:
- """Normalize a filename-walk root without resolving remote paths locally."""
- from tools.environments.base import env_uses_host_filesystem
- from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
- cwd = getattr(env, "cwd", None) or fallback_cwd
- if env_uses_host_filesystem(env):
- if _IS_WINDOWS:
- root = _msys_to_windows_path(root)
- cwd = _msys_to_windows_path(cwd)
- if not os.path.isabs(root):
- root = os.path.join(cwd, root)
- return os.path.normcase(os.path.abspath(os.path.normpath(root)))
- if not posixpath.isabs(root):
- root = posixpath.join(cwd, root)
- return posixpath.normpath(root)
- def _filename_search_root_keys(env: Any, roots: list[str], fallback_cwd: str) -> tuple[tuple[str, str, str], ...]:
- """Unique backend/root admission keys in deterministic order."""
- env_type = type(env)
- return tuple(sorted({
- (env_type.__module__, env_type.__qualname__, _normalized_filename_search_root(env, root, fallback_cwd))
- for root in roots}))
- def _acquire_filename_search_roots(keys: tuple[tuple[str, str, str], ...]) -> bool:
- """Atomically claim every key, polling for thread-scoped interruption."""
- with _FILENAME_SEARCH_ADMISSION:
- while any(key in _ACTIVE_FILENAME_SEARCH_ROOTS for key in keys):
- if tool_interrupt.is_interrupted():
- return False
- _FILENAME_SEARCH_ADMISSION.wait(_FILENAME_SEARCH_WAIT_SECONDS)
- if tool_interrupt.is_interrupted():
- return False
- if tool_interrupt.is_interrupted():
- return False
- return tool_interrupt.run_if_not_interrupted(lambda: _ACTIVE_FILENAME_SEARCH_ROOTS.update(keys))
- def _release_filename_search_roots(keys: tuple[tuple[str, str, str], ...]) -> None:
- """Release a completed walk and leave no idle per-root state behind."""
- with _FILENAME_SEARCH_ADMISSION:
- _ACTIVE_FILENAME_SEARCH_ROOTS.difference_update(keys)
- _FILENAME_SEARCH_ADMISSION.notify_all()
- _ADMISSION_INTERRUPTED_ERROR = (
- "File search was interrupted while waiting for another filename "
- "search on the same root. Retry when ready.")
- _SEARCH_TIMEOUT_MARKER_RE = re.compile(r"\n?\[Command timed out after \d+s\]\s*$")
- def _search_stdout_and_limit(result: ExecuteResult) -> tuple[str, Optional[str]]:
- """Return stdout cleaned for parsing and a limit reason for search timeouts."""
- if result.exit_code == 124:
- return _SEARCH_TIMEOUT_MARKER_RE.sub("", result.stdout), "search_timeout"
- return result.stdout, None
- def _bundled_rg() -> Optional[str]:
- """Hermes's pinned ripgrep, forward-slashed for the shell transports, or None."""
- import pm
- try:
- installed = pm.installed_package("ripgrep")
- except (OSError, RuntimeError, ValueError):
- return None
- binary = installed.binary if installed is not None else None
- return str(binary).replace("\\", "/") if binary is not None and binary.is_file() else None
- # A real rg/grep output line is a whitespace-free path token followed by ``:``
- # (match/count), ``-`` (context), or nothing (files_only); tool diagnostics
- # ("rg: ...", indented carets) never match.
- _SEARCH_OUTPUT_RE = re.compile(r'^([A-Za-z]:)?[^\s:][^\n]*?[:\-]\d|^[^\s:][^\s]*$')
- def _split_tool_diagnostics(output: str) -> tuple[str, str]:
- """Separate rg/grep diagnostic lines from real match output → ``(diagnostics, payload)``.
- ``_exec`` merges stderr into stdout; classifying by SHAPE lets the exit-2 guard
- tell a pure failure (no payload) from a partial one (one unreadable file, others
- matched) and guarantees error text is never parsed as a match."""
- diagnostics: list[str] = []
- payload: list[str] = []
- for line in output.split('\n'):
- if not line.strip():
- continue
- # Prefix check first: a match path can contain "-<digit>" (".../pytest-686/...").
- if line.lstrip().startswith(("rg: ", "grep: ", "find: ")):
- diagnostics.append(line)
- elif line == "--" or _SEARCH_OUTPUT_RE.match(line):
- payload.append(line)
- else:
- diagnostics.append(line)
- return '\n'.join(diagnostics), '\n'.join(payload)
- # One unreadable-path diagnostic, in every shape the search lanes produce:
- # rg: ``rg: <path>: Permission denied (os error 13)``
- # rg modified-sort walk: ``rg: <path>: IO error for operation on <path>: Permission denied (os error 13)``
- # grep: ``grep: <path>: Permission denied`` (no ``(os error N)`` suffix)
- # find: ``find: '<path>': Permission denied`` (pruned grep lane; GNU find quotes the
- # path with ASCII ``'`` or, in a UTF-8 locale, with typographic quotes)
- # The ``(os error N)`` suffix is ripgrep's Rust io::Error spelling, so it is optional.
- # The tool prefix is required: the shell's own ``bash: <path>/rg: Permission denied``
- # (the engine binary itself is not executable) is a failed invocation, not a skip.
- _PERMISSION_DENIED_RE = re.compile(
- r"^(?P<tool>rg|grep|find): (?P<path>.+?): "
- r"(?:IO error for operation on .+?: )?Permission denied(?: \(os error \d+\))?$")
- _SHELL_EXEC_FAILURE_CODES = frozenset({126, 127})
- _FIND_QUOTES_RE = re.compile(r"^['\u2018]|['\u2019]$")
- # The native rg runner keeps at most this much stderr (a huge tree of unreadable dirs
- # would otherwise be held in memory whole) and appends one tally line for the rest.
- _STDERR_RETAIN_BYTES = 64 * 1024
- _OMITTED_DIAGNOSTICS_RE = re.compile(r"^rg: \.\.\. (\d+) more diagnostic line\(s\) omitted$")
- def _omitted_diagnostics_line(count: int) -> str:
- return f"rg: ... {count} more diagnostic line(s) omitted"
- def _omitted_diagnostic_count(diagnostics: str) -> int:
- return sum(int(m.group(1)) for line in diagnostics.splitlines()
- if (m := _OMITTED_DIAGNOSTICS_RE.match(line.strip())))
- def _is_permission_only_error(diagnostics: str) -> bool:
- """True when every diagnostic line reports an unreadable path (rg or grep).
- That is a partial-success condition — the tool still emits results for the
- readable remainder — not a broken invocation like a bad glob or regex. The
- native runner's omitted-lines tally is judged by the lines it kept."""
- lines = [line.strip() for line in diagnostics.splitlines()
- if line.strip() and not _OMITTED_DIAGNOSTICS_RE.match(line.strip())]
- return bool(lines) and all(_PERMISSION_DENIED_RE.match(line) for line in lines)
- def _extract_unreadable_paths(diagnostics: str) -> list[str]:
- """Paths reported unreadable by rg or grep, deduplicated in first-seen order."""
- paths: list[str] = []
- seen: set[str] = set()
- for line in diagnostics.splitlines():
- match = _PERMISSION_DENIED_RE.match(line.strip())
- if not match:
- continue
- path = match.group("path").strip()
- if match.group("tool") == "find":
- path = _FIND_QUOTES_RE.sub("", path)
- if path and path not in seen:
- seen.add(path)
- paths.append(path)
- return paths
- def _compact_unreadable_paths(paths: list[str], keep: int = 3, omitted: int = 0) -> str:
- shown = ", ".join(paths[:keep])
- extra = max(len(paths) - keep, 0) + omitted
- return shown + (f" (+{extra} more)" if extra > 0 else "")
- def _skipped_unreadable_warning(paths: list[str], omitted: int = 0) -> str:
- """``omitted`` counts diagnostics the native runner dropped past its retention cap."""
- return ("Skipped " + str(len(paths) + omitted) + " unreadable path(s) with restricted "
- "permissions: " + _compact_unreadable_paths(paths, omitted=omitted)
- + ". Results may be incomplete; fix permissions or run as a user that "
- "can read them to include them.")
- def _skipped_fields(diagnostics: str) -> dict:
- """SearchResult skip fields from rg/grep/find diagnostics (nothing skipped → empty lists)."""
- paths = _extract_unreadable_paths(diagnostics)
- return {"skipped_paths": paths, "skipped_omitted": _omitted_diagnostic_count(diagnostics) if paths else 0}
- def _skipped_sentence(result: SearchResult) -> Optional[str]:
- """The skipped-paths sentence ``result.warning`` carries, rebuilt from its structured data."""
- if not result.skipped_paths:
- return None
- return _skipped_unreadable_warning(result.skipped_paths, result.skipped_omitted)
- def _prepend_skipped_paths_warning(skip: dict, existing: Optional[str]) -> Optional[str]:
- """Warning naming the skipped unreadable paths, prepended to any existing warning."""
- if not skip["skipped_paths"]:
- return existing
- warning = _skipped_unreadable_warning(skip["skipped_paths"], skip["skipped_omitted"])
- return warning if not existing else f"{warning} {existing}"
- _PERMISSION_SEARCH_HINT = "Nothing matched in the readable part of the search scope."
- def _merge_child_warnings(children: list[SearchResult], has_results: bool) -> list[str]:
- """Combine per-root search warnings into one list: the skipped-path sentences collapse
- into one sentence composed from each child's structured skip data (summed count), the
- rest of the text is deduplicated in order. The nothing-matched hint is dropped once
- any root produced results."""
- total, shown, rest = 0, [], []
- for child in children:
- if not child.warning:
- continue
- warning, sentence = child.warning, _skipped_sentence(child)
- if sentence:
- total += len(child.skipped_paths) + child.skipped_omitted
- for path in child.skipped_paths: # only three are displayed; never scan thousands
- if len(shown) >= 3:
- break
- if path not in shown:
- shown.append(path)
- warning = warning.replace(sentence, "\0")
- for piece in warning.split("\0"):
- piece = piece.strip()
- if has_results:
- piece = piece.replace(_PERMISSION_SEARCH_HINT, "").strip()
- if piece and piece not in rest:
- rest.append(piece)
- if not total:
- return rest
- sentence = _skipped_unreadable_warning(shown, max(total - len(shown), 0))
- return [sentence, *rest]
- _FIND_MODIFIED_ORDER_ERROR = (
- "Exact modification-time order requires GNU find with "
- "-printf support; install ripgrep 14+ or use order='discovery'.")
- def _split_find_output(stdout: str, order: str) -> tuple[list[str], list[str]]:
- """``(paths, diagnostics)`` from a bounded find run; ``find: `` lines are stderr."""
- raw_files: list[str] = []
- diagnostics: list[str] = []
- for line in stdout.splitlines():
- if line.startswith("find: "):
- diagnostics.append(line)
- elif order == "modified":
- parts = line.split(" ", 1)
- if len(parts) == 2 and parts[0].replace(".", "", 1).isdigit():
- raw_files.append(parts[1])
- elif line:
- raw_files.append(line)
- return raw_files, diagnostics
- def _find_exit_verdict(exit_code: int, raw_files: list[str], diagnostics: list[str],
- fetch_limit: int, order: str) -> tuple[Optional[SearchResult], Optional[str]]:
- """``(terminal result or None, warning)`` for a bounded find run.
- Exit 141 is benign only when the payload proves head closed the pipe at the
- bound. find exits 1 when traversal hit unreadable dirs (EACCES) but still prints
- what it could read: a partial success, named from its diagnostics. Exit 1 with no
- payload is a real failure unless every diagnostic is such a skip (bad expressions
- and BSD find's missing -printf also exit 1)."""
- bounded_sigpipe = exit_code == 141 and len(raw_files) >= fetch_limit
- if exit_code not in {0, 1, 124} and not bounded_sigpipe:
- if order == "modified":
- return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None
- return SearchResult(error="File search failed while running bounded find traversal."), None
- diagnostic_text = "\n".join(diagnostics)
- skipped = _extract_unreadable_paths(diagnostic_text)
- warning = _skipped_unreadable_warning(skipped) if skipped else None
- if exit_code != 1:
- return None, warning
- if not raw_files and _is_permission_only_error(diagnostic_text):
- return SearchResult(total_count=0, warning=f"{warning} {_PERMISSION_SEARCH_HINT}",
- skipped_paths=skipped), None
- if not raw_files:
- if order == "modified":
- return SearchResult(error=_FIND_MODIFIED_ORDER_ERROR), None
- detail = f": {diagnostics[0]}" if diagnostics else "."
- return SearchResult(error=f"File search failed while running bounded find traversal{detail}"), None
- # A non-permission diagnostic (a dir removed mid-walk) must not be reported as a permission
- # problem: name it, plus any permission skips, so the user chases the real cause.
- other = next((d.strip() for d in diagnostics if d.strip() and not _PERMISSION_DENIED_RE.match(d.strip())), None)
- if other:
- return None, f"Some paths could not be read: {other}. Results may be incomplete." + (
- f" {warning}" if warning else "")
- return None, warning or ("find traversal skipped one or more unreadable paths (restricted "
- "permissions); results are partial.")
- def _drain_stderr_capped(pipe, chunks: list[bytes], omitted: list[int]) -> None:
- """Read ``pipe`` to EOF (rg must never block on a full stderr pipe) but keep only the
- first ``_STDERR_RETAIN_BYTES``; the first line is always kept so a lone huge diagnostic
- still names the failure. ``omitted[0]`` counts the dropped lines. Past the cap the first
- non-permission line is still kept, so a real failure behind a flood of permission
- errors is never classified as a permission-only partial success."""
- kept = 0
- kept_other = False
- for raw in pipe or ():
- if chunks and kept + len(raw) > _STDERR_RETAIN_BYTES:
- line = raw.decode("utf-8", "replace").strip()
- if not kept_other and line and not _PERMISSION_DENIED_RE.match(line):
- chunks.append(raw)
- kept_other = True
- continue
- omitted[0] += 1
- continue
- chunks.append(raw)
- kept += len(raw)
- def _parse_search_context_line(line: str) -> tuple[str, int, str] | None:
- """Parse a ``path-line-content`` context line using the RIGHTMOST numeric
- separator (filenames may contain ``-<digits>-`` segments):
- ``dir/file-12-name.py-8-context`` → (``dir/file-12-name.py``, 8, ``context``)."""
- if not line or line == "--":
- return None
- match = None
- for candidate in re.finditer(r'-(\d+)-', line):
- match = candidate
- if match is None or match.start() == 0:
- return None
- return line[:match.start()], int(match.group(1)), line[match.end():]
- _REGEX_NEWLINE_ESCAPE_RE = re.compile(r"(?<!\\)(?:\\\\)*\\n")
- def _pattern_has_regex_newline(pattern: str) -> bool:
- """True when a content regex wants to match a newline: a literal newline or a
- ``\\n`` escape with an ODD number of backslashes (``\\\\n`` is a literal
- backslash+n and must not count)."""
- return "\n" in pattern or bool(_REGEX_NEWLINE_ESCAPE_RE.search(pattern))
- def _is_line_oriented_newline_error(error: Optional[str]) -> bool:
- """Return True for rg's hard error when multiline mode is required."""
- return bool(error) and "literal \"\\n\" is not allowed" in error and "--multiline" in error
- def _maybe_warn_line_oriented_newline_pattern(result: SearchResult, pattern: str) -> SearchResult:
- """Attach a newline-regex warning only when search found no usable results."""
- if result.total_count != 0 or not _pattern_has_regex_newline(pattern):
- return result
- if result.error and not _is_line_oriented_newline_error(result.error):
- return result
- result.error = None
- result.skipped_paths, result.skipped_omitted = [], 0
- result.warning = (
- "0 results found. Note: search_files content search is line-oriented "
- "and does not run ripgrep with -U/--multiline, so `\\n` in the regex "
- "does not match line breaks. Use context=N to inspect neighboring "
- "lines, or escape as `\\\\n` when searching for a literal backslash+n.")
- return result
- # Match lines are "file:lineno:content". Windows paths carry a drive letter
- # ("C:\path"), so a naive split(":") breaks — the regex handles both.
- _MATCH_LINE_RE = re.compile(r'^([A-Za-z]:)?(.*?):(\d+):(.*)$')
- # Output-mode → engine flag (identical for rg and grep).
- _OUTPUT_MODE_FLAGS = {"files_only": "-l", "count": "-c"}
- def _parse_search_output(result, output_mode: str, limit: int, offset: int,
- context: int, warning: Optional[str] = None) -> SearchResult:
- """Parse rg/grep ``| head`` output into a SearchResult (shared by both engines).
- Exit codes: 0=matches, 1=none, 2=error — but both tools return 2 on PARTIAL
- errors (one unreadable file), so an error is surfaced only when exit==2 AND no
- usable payload remains. ``warning`` is attached to files_only/content results."""
- stdout, limit_reason = _search_stdout_and_limit(result)
- diagnostics, payload = _split_tool_diagnostics(stdout)
- if result.exit_code in _SHELL_EXEC_FAILURE_CODES:
- # The shell could not run the engine at all (126 not executable, 127 not found).
- return SearchResult(error=f"Search failed: {result.stdout.strip() or 'Search error'}",
- total_count=0)
- if result.exit_code == 2 and not payload.strip():
- error_msg = diagnostics.strip() or result.stdout.strip() or "Search error"
- if _is_permission_only_error(error_msg):
- # Pure unreadable-path failure: rg read nothing but the invocation was
- # sound. Degrade to an explicit empty result instead of failing — the
- # model can still act on the named paths.
- skip = _skipped_fields(error_msg)
- warning = _prepend_skipped_paths_warning(skip, warning)
- if not warning:
- warning = _PERMISSION_SEARCH_HINT
- elif _PERMISSION_SEARCH_HINT not in warning:
- warning = f"{warning} {_PERMISSION_SEARCH_HINT}"
- return SearchResult(total_count=0, warning=warning, **skip)
- return SearchResult(error=f"Search failed: {error_msg}", total_count=0)
- skip = _skipped_fields(diagnostics)
- skipped_warning = _prepend_skipped_paths_warning(skip, None)
- # A non-permission error beside the skips (a path removed mid-search) is named too, not
- # folded into the permission warning.
- other_errors = [ln.strip() for ln in diagnostics.splitlines()
- if ln.lstrip().startswith(("rg: ", "grep: "))
- and not _PERMISSION_DENIED_RE.match(ln.strip())
- and not _OMITTED_DIAGNOSTICS_RE.match(ln.strip())]
- if result.exit_code == 2 and other_errors:
- error_warning = f"The search reported errors for some paths; results may be incomplete ({other_errors[0]})."
- skipped_warning = f"{error_warning} {skipped_warning}" if skipped_warning else error_warning
- if skipped_warning:
- warning = skipped_warning if not warning else f"{skipped_warning} {warning}"
- lines = [ln for ln in payload.strip().split('\n') if ln]
- if output_mode == "files_only":
- return SearchResult(
- files=lines[offset:offset + limit], total_count=len(lines),
- truncated=bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip)
- if output_mode == "count":
- counts = {}
- for line in lines:
- if ':' in line:
- path, n = line.rsplit(':', 1)
- try:
- counts[path] = int(n)
- except ValueError:
- pass
- return SearchResult(
- counts=counts, total_count=sum(counts.values()),
- truncated=bool(limit_reason), limit_reason=limit_reason, warning=warning, **skip)
- matches = []
- for line in lines:
- if line == "--":
- continue
- m = _MATCH_LINE_RE.match(line)
- if m:
- matches.append(SearchMatch(
- path=(m.group(1) or '') + m.group(2), line_number=int(m.group(3)), content=m.group(4)[:500],
- ))
- continue
- # Context lines only when requested, to avoid false positives on dashy paths.
- if context > 0:
- parsed = _parse_search_context_line(line)
- if parsed:
- matches.append(SearchMatch(path=parsed[0], line_number=parsed[1], content=parsed[2][:500]))
- total = len(matches)
- return SearchResult(
- matches=matches[offset:offset + limit], total_count=total,
- truncated=total > offset + limit or bool(limit_reason), limit_reason=limit_reason, warning=warning,
- **skip,
- )
- def _posix_roots(roots: list[str]) -> bool:
- """Darwin-only: every root is POSIX-shaped (no drive letter / backslash)."""
- return sys.platform == "darwin" and all(
- not re.match(r"^[A-Za-z]:[\\/]", root) and "\\" not in root for root in roots)
- def _find_literal_path_expressions(roots: list[str]) -> list[str]:
- """Escape each search root for find's ``-path`` test, which matches its PATTERN
- (glob) against the path find echoes — the operand verbatim. The operand reaches
- the shell already single-quoted; ``-path`` must compare literally, so ``*?[]``
- in a root's own name need escaping (a directory literally named ``a[1]`` must
- be excluded by identity, not treated as a character class). Backslashes are
- left alone: find does not read them as escapes inside a bracket-free pattern,
- and doubling them would break matching on backslash-shaped roots."""
- return [re.sub(r"([*?\[\]])", r"\\\1", root) for root in roots]
- class SearchMixin:
- """File-name and content search via rg with find/grep fallbacks. Requires
- ``_exec``, ``_has_command``, ``_expand_path``, ``_escape_shell_arg``,
- ``_escape_native_tool_arg``, ``env``, ``cwd``, ``_command_cache``,
- ``_rg_resolution_cache`` and ``_rg_modified_capability`` from the host class."""
- # --- rg resolution --------------------------------------------------------
- def _resolve_command(self, cmd: str) -> Optional[str]:
- """Resolve an executable in the command host's namespace. Ordinary commands
- keep the bool hit/miss cache; rg alone caches successful resolved paths and
- re-probes misses so a mid-session install becomes visible (with off-PATH
- Windows candidates: cargo, scoop, winget)."""
- if cmd != "rg":
- return cmd if self._has_command(cmd) else None
- cached = self._rg_resolution_cache.get(cmd)
- if cached:
- return cached
- from tools.environments.base import env_uses_host_filesystem
- from tools.environments.local import _IS_WINDOWS
- # A host search is Hermes's own work, so it runs the pinned ripgrep, never the
- # first rg on the terminal PATH, where the user's tools come first.
- on_host = env_uses_host_filesystem(self.env)
- bundled = _bundled_rg() if on_host else None
- if bundled:
- self._rg_resolution_cache[cmd] = bundled
- return bundled
- result = self._exec("command -v rg 2>/dev/null")
- if result.exit_code == 0 and result.stdout.strip():
- resolved = result.stdout.strip().splitlines()[0]
- if resolved == "yes": # compatibility with old boolean-probe fakes
- resolved = "rg"
- self._rg_resolution_cache[cmd] = resolved
- return resolved
- if _IS_WINDOWS and on_host:
- user_profile = os.environ.get("USERPROFILE") or str(Path.home())
- local_app_data = os.environ.get("LOCALAPPDATA")
- scoop = os.environ.get("SCOOP") or os.path.join(user_profile, "scoop")
- candidates = [
- os.path.join(user_profile, ".cargo", "bin", "rg.exe"),
- os.path.join(scoop, "shims", "rg.exe"),
- ]
- if local_app_data:
- candidates.append(os.path.join(local_app_data, "Microsoft", "WinGet", "Links", "rg.exe"))
- for candidate in candidates:
- if os.path.isfile(candidate):
- resolved = candidate.replace("\\", "/")
- self._rg_resolution_cache[cmd] = resolved
- return resolved
- return None
- _RG_VERSION_RE = re.compile(
- r"(?m)^ripgrep\s+((?:0|[1-9]\d*))\."
- r"(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)"
- r"(?:-(?:(?:0|[1-9]\d*)|(?:[0-9A-Za-z-]*[A-Za-z-]"
- r"[0-9A-Za-z-]*))(?:\.(?:(?:0|[1-9]\d*)|"
- r"(?:[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)))*)?"
- r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?"
- r"(?:\s+\(rev [^)]+\))?\s*$")
- def _modified_rg_capability_error(self, executable: str) -> Optional[str]:
- """Cached actionable error unless rg can sort exactly (full SemVer, >= 14)."""
- if executable in self._rg_modified_capability:
- return self._rg_modified_capability[executable]
- result = self._exec(f"{self._quote_executable(executable)} --version", timeout=10)
- match = self._RG_VERSION_RE.search(result.stdout or "")
- if result.exit_code == 0 and match and int(match.group(1)) >= 14:
- error = None
- else:
- error = ("Exact modification-time order requires ripgrep 14 or newer; "
- "upgrade ripgrep or use order='discovery'.")
- self._rg_modified_capability[executable] = error
- return error
- # --- native rg transport (local POSIX) --------------------------------------
- def _run_rg_native(self, argv: list[str], fetch_limit: int, timeout: int,
- merge_stderr: bool = False) -> ExecuteResult:
- """Run ``argv`` (shell-quoted rg words) natively and stop reading after
- ``fetch_limit`` lines — the ``| head -n`` of the shell pipeline without the
- two bash spawns. ``shlex.split`` undoes the escaping the builders apply for
- the shell path, so both transports see identical arguments. Exit code and
- stdout follow the shell contract (rg 0/1/2; 124 on timeout with partial
- output; 130 on interrupt), so ``_parse_search_output`` is shared. Once the
- bound is reached rg is killed like ``head`` closing the pipe would.
- ``merge_stderr`` mirrors the shell path's stderr handling: kept for real
- searches (diagnostics name unreadable paths or the error), discarded
- (``2>/dev/null``) for probes. Like the shell's ``2>&1``-free ``| head``, kept
- stderr is drained on its own pipe so diagnostics never use up the
- ``fetch_limit`` lines meant for results; it is prepended to stdout."""
- from tools.environments.local import _kill_process_group_posix, _make_run_env
- cwd = getattr(self.env, "cwd", None) or self.cwd
- args = shlex.split(" ".join(argv))
- try:
- proc = subprocess.Popen(
- args, cwd=cwd, env=_make_run_env(self.env.env), stdin=subprocess.DEVNULL,
- stdout=subprocess.PIPE, stderr=subprocess.PIPE if merge_stderr else subprocess.DEVNULL,
- # detached-spawn: ok — awaited rg under a deadline; the group is killed on timeout or interrupt
- start_new_session=True)
- except OSError as exc:
- return ExecuteResult(stdout=f"rg: {exc}", exit_code=2)
- # Record the group at spawn, as LocalEnvironment._run_bash does: rg can exit between the
- # poll and the kill below, and on macOS a not-yet-reaped child has no pgid any more.
- with contextlib.suppress(ProcessLookupError):
- proc._hermes_pgid = os.getpgid(proc.pid)
- from gateway.status import get_process_start_time
- proc._hermes_pgid_start = get_process_start_time(proc.pid)
- # Drain on a thread so a silent rg (huge tree, no hits yet) cannot pin the
- # caller past the deadline or past a /stop; the waiter below owns both.
- lines: list[bytes] = []
- bounded = threading.Event()
- def _drain() -> None:
- for raw in proc.stdout:
- lines.append(raw)
- if len(lines) >= fetch_limit:
- bounded.set()
- break
- drainer = threading.Thread(target=_drain, daemon=True)
- drainer.start()
- err_chunks: list[bytes] = []
- err_omitted = [0]
- err_drainer = None
- if merge_stderr:
- err_pipe = proc.stderr
- err_drainer = threading.Thread( # health: allow HX012 -- pipe drainer, reads no profile-scoped state
- target=_drain_stderr_capped, args=(err_pipe, err_chunks, err_omitted), daemon=True)
- err_drainer.start()
- deadline = time.monotonic() + timeout
- exit_code: Optional[int] = None
- while True:
- drainer.join(0.05)
- if not drainer.is_alive() or bounded.is_set():
- break
- if tool_interrupt.is_interrupted():
- exit_code = 130
- break
- if time.monotonic() > deadline:
- exit_code = 124
- break
- if proc.poll() is None:
- _kill_process_group_posix(proc) # native lane is POSIX-only (gate above)
- proc.wait()
- drainer.join()
- proc.stdout.close()
- stdout = b"".join(lines).decode("utf-8", errors="replace")
- if err_drainer is not None:
- err_drainer.join()
- if proc.stderr is not None:
- proc.stderr.close()
- err_text = b"".join(err_chunks).decode("utf-8", errors="replace")
- if err_omitted[0]:
- err_text = err_text.rstrip("\n") + "\n" + _omitted_diagnostics_line(err_omitted[0]) + "\n"
- stdout = err_text + stdout
- if exit_code == 124:
- return ExecuteResult(stdout=stdout + f"\n[Command timed out after {timeout}s]", exit_code=124)
- if exit_code == 130:
- return ExecuteResult(stdout=stdout + "\n[Command interrupted]", exit_code=130)
- # A killed-at-bound rg reports a signal (negative returncode); head would have
- # left the pipeline at 0 unless rg itself already failed.
- return ExecuteResult(stdout=stdout, exit_code=0 if bounded.is_set() else proc.returncode)
- def _run_rg_bounded(self, words: list[str], fetch_limit: int, timeout: int, *,
- merge_stderr: bool = False, native_ok: bool = True,
- shell_prefix: str = "") -> ExecuteResult:
- """Run an rg command (shell-quoted words) and keep the first ``fetch_limit``
- lines: natively on a local POSIX host, else through the backend shell as
- ``<prefix><words> | head -n N``. ``native_ok=False`` keeps a form the native
- lane cannot express (the multi-root ``cd`` prefix); ``shell_prefix`` is shell-only."""
- if native_ok and self._native_read_enabled():
- return self._run_rg_native(words, fetch_limit, timeout, merge_stderr=merge_stderr)
- stderr = "" if merge_stderr else " 2>/dev/null"
- return self._exec(f"{shell_prefix}{' '.join(words)}{stderr} | head -n {fetch_limit}", timeout=timeout)
- def _quote_executable(self, executable: str) -> str:
- """Quote an executable without leaking controller path semantics."""
- if re.fullmatch(r"[A-Za-z0-9_.-]+", executable):
- return executable
- from tools.environments.base import env_uses_host_filesystem
- if env_uses_host_filesystem(self.env):
- return self._escape_native_tool_arg(executable)
- return "'" + executable.replace("'", "'\"'\"'") + "'"
- # --- macOS protected-folder exclusions --------------------------------------
- def _macos_search_exclusions(self, path: str) -> list[str]:
- """Protected descendants to prune for this search root, if any. Gated on
- ``env.is_local``: ``sys.platform``/``_HOME`` describe the CONTROLLER, but the
- search runs on ``env``'s host. Envs without the flag default to local
- semantics; pruning is a warning-carrying skip, never data loss."""
- env = getattr(self, "env", None)
- if env is not None and getattr(env, "is_local", True) is False:
- return []
- from tools import file_operations as _fo # lazy: _HOME is monkeypatched there
- cwd = getattr(self.env, "cwd", None) or self.cwd
- return _macos_protected_search_exclusions(path, cwd=cwd, home=_fo._HOME, platform=sys.platform)
- def _protected_prune_paths(self, path: str) -> list[str]:
- """Absolute-ish protected paths for find's ``-path ... -prune``."""
- return [os.path.normpath(os.path.join(path, item)) for item in self._macos_search_exclusions(path)]
- def _effective_macos_search_exclusions(self, roots: list[str]) -> list[tuple[str, str, str]]:
- """Unique ``(root, relative, absolute)`` exclusions across ``roots``, never
- pruning a root the caller chose explicitly."""
- cwd = getattr(self.env, "cwd", None) or self.cwd
- use_posix_paths = _posix_roots(roots)
- def normalized(root: str) -> str:
- if use_posix_paths:
- return posixpath.normpath(root if posixpath.isabs(root) else posixpath.join(cwd, root))
- return os.path.normcase(os.path.abspath(os.path.normpath(root)))
- normalized_roots = [normalized(root) for root in roots]
- explicit_roots = set(normalized_roots)
- seen = set()
- effective = []
- for root, normalized_root in zip(roots, normalized_roots):
- for relative in self._macos_search_exclusions(root):
- if use_posix_paths:
- absolute = key = posixpath.normpath(posixpath.join(normalized_root, relative))
- else:
- absolute = os.path.normpath(os.path.join(root, relative))
- key = os.path.normcase(os.path.abspath(absolute))
- if key in explicit_roots or key in seen:
- continue
- seen.add(key)
- effective.append((root, relative, absolute))
- return effective
- @staticmethod
- def _macos_protected_search_warning(paths: list[str]) -> str:
- skipped = ", ".join(os.path.basename(item) for item in paths)
- return ("Skipped macOS protected folders during broad search to avoid "
- f"an unattended privacy prompt: {skipped}. Search a protected "
- "folder directly when access is intentional.")
- @staticmethod
- def _hidden_prune_expr(q_roots: list[str]) -> str:
- """find clause pruning hidden dirs while keeping an explicitly selected dot-named root
- (dir or single file) — find echoes each start point as given, so ``! -path`` matches it."""
- exemptions = "".join(f" ! -path {root}" for root in q_roots)
- return f"\\( -type d -name '.*'{exemptions} \\) -prune"
- def _prune_expr(self, protected_paths: list[str]) -> str:
- """find ``\\( -path A -o -path B \\) -prune`` clause for the protected dirs."""
- terms = " -o ".join(f"-path {self._escape_shell_arg(item)}" for item in protected_paths)
- return f"\\( {terms} \\) -prune"
- def _root_under_hidden_dir(self, path: str) -> bool:
- """True when the search root or any ancestor is dot-named (``~/.hermes/skills``)."""
- root = _normalized_filename_search_root(self.env, path or ".", self.cwd)
- return any(part.startswith(".") and part not in (".", "..") for part in root.replace("\\", "/").split("/"))
- def _rg_exclusion_globs(self, path: str) -> list[str]:
- """``--glob '!<dir>/**'`` pairs excluding protected dirs from an rg run."""
- out: list[str] = []
- for item in self._macos_search_exclusions(path):
- out.extend(["--glob", self._escape_shell_arg(f"!{item}/**")])
- return out
- def _path_exists_probe(self, path: str) -> ExecuteResult:
- """Existence probe; stdout contains "exists" or "not_found" (or the probe's
- ``cwd_error`` when the exec wrapper itself failed)."""
- if self._native_read_enabled():
- full = path if os.path.isabs(path) else os.path.join(getattr(self.env, "cwd", None) or self.cwd, path)
- return ExecuteResult(stdout="exists" if os.path.exists(full) else "not_found")
- return self._exec(f"test -e {self._escape_shell_arg(path)} && echo exists || echo not_found")
- def _dispatch_search(self, pattern: str, path: str, target: str,
- file_glob: Optional[str], limit: int, offset: int,
- output_mode: str, context: int, order: str = "discovery") -> SearchResult:
- if target == "files":
- return self._search_files(pattern, path, limit, offset, order)
- return self._search_content(pattern, path, file_glob, limit, offset, output_mode, context)
- def _path_not_found_result(self, path: str) -> SearchResult:
- """Error result for a missing search root, with nearby-entry suggestions."""
- parent = os.path.dirname(path) or "."
- basename_query = os.path.basename(path)
- hint_parts = [f"Path not found: {path}"]
- parent_check = self._exec(f"test -d {self._escape_shell_arg(parent)} && echo yes || echo no")
- if "yes" in parent_check.stdout and basename_query:
- ls_result = self._exec(f"ls -1 {self._escape_shell_arg(parent)} 2>/dev/null | head -20")
- if ls_result.exit_code == 0 and ls_result.stdout.strip():
- lq = basename_query.lower()
- candidates = [
- os.path.join(parent, e) for e in ls_result.stdout.strip().split('\n')
- if e and (lq in e.lower() or e.lower() in lq or e.lower().startswith(lq[:3]))]
- if candidates:
- hint_parts.append("Similar paths: " + ", ".join(candidates[:5]))
- return SearchResult(error=". ".join(hint_parts), total_count=0)
- def _try_multi_path_search(self, pattern: str, path: str, target: str,
- file_glob: Optional[str], limit: int, offset: int,
- output_mode: str, context: int,
- order: str = "discovery") -> Optional[SearchResult]:
- """Recover a not-found ``path`` that is really several paths in one string.
- Commas explicitly delimit paths (internal spaces preserved); without commas
- split on whitespace. Search every existing part, merge, and note skipped
- parts. None when it doesn't look like a multi-path string."""
- if "," in path:
- parts = [part.strip() for part in path.split(",") if part.strip()]
- else:
- parts = path.split()
- if len(parts) < 2:
- return None
- existing, missing = [], []
- for p in parts:
- expanded = self._expand_path(p)
- (existing if "exists" in self._path_exists_probe(expanded).stdout else missing).append(expanded)
- if not existing:
- return None
- if target == "files":
- # One global traversal across roots so modified ordering and pagination
- # are exact; root admission wraps the actual rg/find invocation.
- merged = self._search_files(pattern, existing, limit, offset, order)
- children = [merged]
- else:
- merged, children = SearchResult(), []
- for root in existing:
- sub = self._search_content(pattern, root, file_glob, limit, offset, output_mode, context)
- if sub.error:
- return sub
- children.append(sub)
- merged.matches.extend(sub.matches)
- merged.files.extend(sub.files)
- merged.counts.update(sub.counts)
- merged.total_count += sub.total_count
- merged.truncated = merged.truncated or sub.truncated
- merged.matches = merged.matches[:limit]
- merged.files = merged.files[:limit]
- note = f"path contained {len(parts)} entries; searched {len(existing)} that exist"
- if missing:
- note += "; skipped missing: " + ", ".join(missing[:3])
- if len(missing) > 3:
- note += f" (+{len(missing) - 3} more)"
- has_results = bool(merged.total_count or merged.matches or merged.files or merged.counts)
- warning_parts = [*_merge_child_warnings(children, has_results), note]
- if not merged.error:
- protected_paths = [absolute for _r, _rel, absolute in self._effective_macos_search_exclusions(existing)]
- if protected_paths:
- warning_parts.append(self._macos_protected_search_warning(protected_paths))
- merged.warning = " ".join(warning_parts)
- return merged
- def _search_prune_glob_args(self) -> str:
- """rg globs pruning known heavyweight recursive subtrees. Both forms are
- needed: globs are relative to each rg root, so ``**/name/**`` alone misses an
- explicitly selected ``name/`` root. Names come from the shared scan policy —
- no second search-only list."""
- globs = []
- for dirname in sorted(SEARCH_PRUNE_DIR_NAMES):
- for prefix in ("", "**/"):
- globs.extend(("--glob", self._escape_shell_arg(f"!{prefix}{dirname}/**")))
- return " ".join(globs)
- # (rg flags, message template) probes for a 0-match content search, in order.
- # The fixed-string probe only runs when the pattern has regex metacharacters.
- _ZERO_MATCH_PROBES = (
- ("-i", "0 exact matches, but {total} case-insensitive match(es) in {n} file(s): "
- "{paths} — the pattern's casing may be wrong."),
- # rg skips dotdirs and .gitignore'd files by default; say so instead of a bare zero.
- ("--hidden --no-ignore", "0 matches in visible files, but {total} match(es) in {n} "
- "hidden or gitignored file(s): {paths} — these are excluded by default."),
- ("-F", "0 regex matches, but {total} literal match(es) in {n} file(s): {paths} — the "
- "pattern contains regex metacharacters that likely need escaping "
- "(or pass a simpler substring)."),
- )
- def _zero_match_probe(self, pattern: str, path: str, file_glob: Optional[str]) -> Optional[str]:
- """Steering hint for a 0-match content search, or None: a bare zero gives the
- model nothing to act on, so run cheap count-only rg probes (case-insensitive,
- hidden/ignored, fixed-string) and report the first that hits."""
- rg_executable = self._resolve_command('rg')
- if not rg_executable:
- return None
- rg = self._quote_executable(rg_executable)
- has_meta = bool(re.search(r"[.\[\](){}?*+^$\\|]", pattern))
- glob_expr = f" --glob {self._escape_shell_arg(file_glob)}" if file_glob else ""
- for flags, template in self._ZERO_MATCH_PROBES:
- if flags == "-F" and not has_meta:
- continue
- # The hidden/ignored probe keeps --no-ignore so project-local ignored
- # files stay diagnosable, but prunes heavyweight trees before rg recurses.
- if flags.startswith("--hidden"):
- glob_expr_probe = f"{glob_expr} {self._search_prune_glob_args()}"
- else:
- glob_expr_probe = glob_expr
- probe_words = [rg, flags, "--count-matches", glob_expr_probe,
- self._escape_shell_arg(pattern, translate_path=False), self._escape_native_tool_arg(path)]
- probe = self._run_rg_bounded(probe_words, 50, timeout=30)
- total, per_file = 0, []
- for line in (probe.stdout or "").strip().splitlines():
- p, _sep, n = line.rpartition(":")
- if n.isdigit():
- total += int(n)
- per_file.append(p)
- if total > 0:
- extra = len(per_file) - 5
- paths = ", ".join(per_file[:5]) + (f" (+{extra} more)" if extra > 0 else "")
- return template.format(total=total, n=len(per_file), paths=paths)
- return None
- def _is_broad_local_search_root(self, path: str) -> bool:
- """Whether a no-rg LOCAL root (filesystem root, $HOME or an ancestor of it) is
- unsafe for recursive find. Controller paths never classify remotes."""
- from tools.environments.base import env_uses_host_filesystem
- from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
- if not env_uses_host_filesystem(self.env):
- return False
- def normalized(value: str) -> str:
- if _IS_WINDOWS:
- value = _msys_to_windows_path(value).replace("\\", "/")
- if not os.path.isabs(value):
- value = os.path.join(getattr(self.env, "cwd", None) or self.cwd, value)
- # Classify the linked target, not the link: ``find -H`` now follows an
- # operand symlink, so a link pointing at $HOME (or at the filesystem root)
- # must not slip a recursive find past this guard (#116270). Local-only by
- # the isinstance check above, so this resolves on the host that runs find.
- return os.path.normcase(os.path.realpath(value))
- from tools import file_operations as _fo # lazy: _HOME is monkeypatched there
- root = normalized(path)
- home = normalized(_fo._HOME)
- drive = os.path.splitdrive(root)[0]
- anchor = drive + os.sep if drive else os.path.abspath(os.sep)
- if root == os.path.normcase(anchor):
- return True
- try:
- common = os.path.commonpath([root, home])
- except ValueError:
- return False
- return root == home or common == root
- def _filter_hidden_descendants(self, paths: list[str], root: Path) -> list[str]:
- """Drop hidden descendants while allowing an explicit hidden search root."""
- normalized_root = root.resolve()
- filtered = []
- for item_path in paths:
- try:
- rel_parts = Path(item_path).resolve().relative_to(normalized_root).parts
- except ValueError:
- rel_parts = Path(item_path).parts
- if any(part not in {".", ".."} and part.startswith(".") for part in rel_parts):
- continue
- filtered.append(item_path)
- return filtered
- def _filter_gitignored_paths(self, paths: list[str], root: str) -> list[str]:
- """Drop paths ignored by git when *root* is inside a git worktree."""
- if not paths:
- return paths
- escaped_root = self._escape_shell_arg(root)
- probe = self._exec(
- f"git -C {escaped_root} rev-parse --is-inside-work-tree >/dev/null 2>&1",
- timeout=10,
- )
- if probe.exit_code != 0:
- return paths
- result = self._exec(
- f"git -C {escaped_root} check-ignore --stdin",
- timeout=10,
- stdin_data="\n".join(paths),
- )
- ignored = set(result.stdout.splitlines())
- return [item_path for item_path in paths if item_path not in ignored]
- def _sort_paths_by_mtime(self, paths: list[str]) -> list[str]:
- """Sort paths newest-first using metadata from the active backend."""
- if len(paths) < 2:
- return paths
- script = (
- 'while IFS= read -r item || [ -n "$item" ]; do '
- 'mtime="$(stat -c%Y "$item" 2>/dev/null '
- '|| stat -f%m "$item" 2>/dev/null || true)"; '
- "printf '%s\\n' \"${mtime:-0}\"; "
- "done"
- )
- result = self._exec(script, timeout=60, stdin_data="\n".join(paths))
- mtimes = []
- for line in result.stdout.splitlines()[:len(paths)]:
- try:
- mtimes.append(float(line))
- except ValueError:
- mtimes.append(0.0)
- # A path can disappear between traversal and stat. Keep such paths at
- # the end, preserving their existing relative order as a stable fallback.
- mtimes.extend([0.0] * (len(paths) - len(mtimes)))
- return [
- item
- for _, item in sorted(
- zip(mtimes, paths), key=lambda pair: pair[0], reverse=True
- )
- ]
- def _search_directories_with_find(
- self, pattern: str, path: str, fetch_limit: int
- ) -> tuple[list[str], Optional[str]]:
- """Return matching descendant directories using find, if available.
- ``rg --files`` and ``find -type f`` both enumerate files only, so
- directories — especially empty ones — are invisible (#54347). rg stays
- authoritative for which *files* survive ignore processing; this adds
- the matching directory entries so the tool honours its documented
- ls-replacement contract.
- """
- if fetch_limit <= 0 or not self._has_command("find"):
- return [], None
- escaped_path = self._escape_shell_arg(path)
- escaped_pattern = self._escape_shell_arg(pattern)
- q_roots = [escaped_path]
- hidden_prune = self._hidden_prune_expr(q_roots)
- cmd = (
- f"find -H {escaped_path} -mindepth 1 {hidden_prune} -o -type d "
- f"! -name '.*' -name {escaped_pattern} -printf '%T@ %p\\n' "
- f"2>/dev/null | sort -rn"
- )
- result = self._exec(cmd, timeout=30)
- stdout, limit_reason = _search_stdout_and_limit(result)
- if not stdout.strip() and not limit_reason:
- # Try without -printf (BSD find compatibility -- macOS)
- cmd_simple = (
- f"find -H {escaped_path} -mindepth 1 {hidden_prune} -o -type d "
- f"! -name '.*' -name {escaped_pattern} 2>/dev/null"
- )
- result = self._exec(cmd_simple, timeout=30)
- stdout, limit_reason = _search_stdout_and_limit(result)
- dirs = []
- for line in stdout.strip().split('\n'):
- if not line:
- continue
- parts = line.split(' ', 1)
- if len(parts) == 2 and parts[0].replace('.', '').isdigit():
- dirs.append(parts[1])
- else:
- dirs.append(line)
- if dirs:
- # An explicitly selected hidden root keeps its (hidden) subtree;
- # any other hidden descendants stay invisible, matching rg.
- if self._root_under_hidden_dir(path):
- dirs = self._filter_hidden_descendants(dirs, Path(path))
- dirs = self._filter_gitignored_paths(dirs, path)
- return self._sort_paths_by_mtime(dirs)[:fetch_limit], limit_reason
- def _search_files(self, pattern: str, path: str | list[str], limit: int, offset: int,
- order: str = "discovery") -> SearchResult:
- """Search for files by name (glob-like) across one or more roots: rg --files,
- else a bounded find. ``order``: "discovery" (fast, bounded) or "modified"
- (exact global newest-first; needs rg 14+ or GNU find)."""
- search_pattern = pattern if (not pattern.startswith('**/') and '/' not in pattern) \
- else pattern.split('/')[-1]
- roots = [path] if isinstance(path, str) else path
- if not roots:
- return SearchResult(error="File search requires at least one search root in 'path'.")
- # Prefer ripgrep: bounded parallel traversal with ignore semantics. Resolve
- # the engine and exact-order capability BEFORE admission so a queued request
- # does not occupy a root while doing command discovery.
- if self._has_command("rg"):
- rg_executable = self._resolve_command("rg") or "rg"
- if order == "modified":
- capability_error = self._modified_rg_capability_error(rg_executable)
- if capability_error:
- return SearchResult(error=capability_error)
- keys = _filename_search_root_keys(self.env, roots, self.cwd)
- if not _acquire_filename_search_roots(keys):
- return SearchResult(error=_ADMISSION_INTERRUPTED_ERROR)
- try:
- return self._search_files_rg(search_pattern, path, limit, offset, order,
- rg_executable=rg_executable)
- finally:
- _release_filename_search_roots(keys)
- # A local find rooted at/above $HOME or a filesystem root can take minutes and
- # prompt on protected paths: refuse before invoking find.
- if any(self._is_broad_local_search_root(root) for root in roots):
- return SearchResult(error=(
- "Broad local file search without ripgrep is disabled because "
- "find cannot keep this traversal safely bounded. Install "
- "ripgrep or search a narrower directory."))
- if not self._has_command("find"):
- return SearchResult(
- error="File search requires 'rg' (ripgrep) or 'find'. "
- "Install ripgrep for best results: "
- "https://github.com/BurntSushi/ripgrep#installation")
- # Prune hidden descendant dirs (and hidden files, matching rg's default) while
- # still allowing an explicitly selected hidden root; dash-prefixed roots get
- # ``./`` so find doesn't parse them as options.
- find_roots = [f"./{root}" if root.startswith("-") else root for root in roots]
- q_roots = [self._escape_shell_arg(root) for root in find_roots]
- hidden_prune = f" {self._hidden_prune_expr(q_roots)} -o"
- protected_paths = [absolute for _r, _rel, absolute in self._effective_macos_search_exclusions(roots)]
- protected_prune = f" {self._prune_expr(protected_paths)} -o" if protected_paths else ""
- fetch_limit = offset + limit + 1
- # ``-H`` follows a symlink handed in as an OPERAND, and only an operand: without
- # it ``find <link> -type f`` tests the link itself, so ``target="files"`` listed
- # nothing at all for a symlinked root - total_count: 0, no error, no warning,
- # indistinguishable from an empty directory - while ``rg --files`` followed the
- # same argument (#116270). Following the operand inside the command is also what
- # covers a link that only exists on the execution host (SSH/container), with no
- # probe of its own. That contract forbids ``-mindepth 1`` here: with ``-H`` the
- # followed operand IS depth 0, so it would exclude the root itself — the only
- # match when the root is a symlink to a file. The root directory itself is
- # excluded by path identity instead, inside the ``-type d`` arm (the ``-type f``
- # arm must keep the depth-0 match). ``-path`` matching is literal against the
- # path as find echoes it, which is the operand verbatim, so the operand needs
- # its glob metacharacters escaped.
- root_exemptions = "".join(
- f" ! -path {self._escape_shell_arg(root)}"
- for root in _find_literal_path_expressions(find_roots)
- )
- base = (f"find -H {' '.join(q_roots)}{protected_prune}{hidden_prune} "
- f"\\( -type f -o \\( -type d{root_exemptions} \\) \\) "
- f"! -name '.*' -name {self._escape_shell_arg(search_pattern)}")
- # find's stderr is not piped into sort/head, so ``find: '<dir>': Permission
- # denied`` lines reach the merged output without using up result rows.
- if order == "modified":
- cmd = "set -o pipefail; " + base + f" -printf '%T@ %p\\n' | sort -rn | head -n {fetch_limit}"
- else:
- cmd = "set -o pipefail; " + base + f" -print | head -n {fetch_limit}"
- keys = _filename_search_root_keys(self.env, roots, self.cwd)
- if not _acquire_filename_search_roots(keys):
- return SearchResult(error=_ADMISSION_INTERRUPTED_ERROR)
- try:
- result = self._exec(cmd, timeout=60)
- finally:
- _release_filename_search_roots(keys)
- stdout, limit_reason = _search_stdout_and_limit(result)
- raw_files, diagnostics = _split_find_output(stdout, order)
- failure, partial_warning = _find_exit_verdict(result.exit_code, raw_files, diagnostics,
- fetch_limit, order)
- if failure is not None:
- return failure
- from tools.environments.base import env_uses_host_filesystem
- from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
- if _IS_WINDOWS and env_uses_host_filesystem(self.env):
- raw_files = [_msys_to_windows_path(file_path) for file_path in raw_files]
- return SearchResult(
- files=raw_files[offset:offset + limit], total_count=len(raw_files),
- truncated=len(raw_files) > offset + limit or bool(limit_reason),
- limit_reason=limit_reason, warning=partial_warning,
- skipped_paths=_extract_unreadable_paths("\n".join(diagnostics)))
- def _search_files_rg(self, pattern: str, path: str | list[str], limit: int, offset: int,
- order: str = "discovery", rg_executable: Optional[str] = None) -> SearchResult:
- """File-name search via ``rg --files`` (respects .gitignore, skips hidden dirs,
- parallel walk). Discovery order stays bounded and fast; exact modification-time
- ordering is explicit because it scans globally."""
- # Wrap bare names so -g matches at any depth (equivalent to find -name).
- glob_pattern = f"*{pattern}" if ('/' not in pattern and not pattern.startswith('*')) else pattern
- roots = [path] if isinstance(path, str) else path
- fetch_limit = limit + offset + 1
- effective_exclusions = self._effective_macos_search_exclusions(roots)
- scoped_common = None
- command_roots = roots
- if len(roots) > 1 and effective_exclusions and _posix_roots(roots):
- # Several roots: rg globs are root-relative, so cd to the common ancestor
- # and express roots + exclusions relative to it.
- cwd = getattr(self.env, "cwd", None) or self.cwd
- absolute_roots = [
- posixpath.normpath(root if posixpath.isabs(root) else posixpath.join(cwd, root))
- for root in roots]
- scoped_common = posixpath.commonpath(absolute_roots)
- command_roots = [posixpath.relpath(root, scoped_common) for root in absolute_roots]
- exclusion_terms = [
- f"--glob {self._escape_shell_arg(f'!{posixpath.relpath(absolute, scoped_common)}/**')}"
- for _r, _rel, absolute in effective_exclusions]
- else:
- exclusion_terms = [
- f"--glob {self._escape_shell_arg(f'!{relative}/**')}"
- for _r, relative, _abs in effective_exclusions]
- exclusion_globs = " ".join(dict.fromkeys(exclusion_terms))
- exclusion_args = f" {exclusion_globs}" if exclusion_globs else ""
- rg_executable = rg_executable or self._resolve_command("rg")
- if not rg_executable:
- return SearchResult(error="File search requires ripgrep (rg).")
- if order == "modified":
- capability_error = self._modified_rg_capability_error(rg_executable)
- if capability_error:
- return SearchResult(error=capability_error)
- rg = self._quote_executable(rg_executable)
- sort_arg = " --sortr=modified" if order == "modified" else ""
- root_args = " ".join(self._escape_native_tool_arg(root) for root in command_roots)
- cd_prefix = f"cd {self._escape_shell_arg(scoped_common)} && " if scoped_common else ""
- # ``--`` terminates options so a dash-prefixed root is never parsed as a flag.
- rg_cmd = (f"{rg} --files{sort_arg} -g {self._escape_shell_arg(glob_pattern)}"
- f"{exclusion_args} -- {root_args}")
- result = self._run_rg_bounded([rg_cmd], fetch_limit, timeout=60, native_ok=not scoped_common,
- merge_stderr=True,
- shell_prefix=f"set -o pipefail; {cd_prefix}")
- stdout, limit_reason = _search_stdout_and_limit(result)
- # Files-mode transport mixes stderr into stdout, but file paths may contain
- # spaces — the content-mode shape matcher (_SEARCH_OUTPUT_RE) would drop
- # them as "diagnostics". rg/grep diagnostic lines always begin with
- # "rg: "/"grep: " (continuation lines only matter on the fail-closed path),
- # so classify by that prefix instead.
- stdout_lines = stdout.splitlines()
- diagnostics = "\n".join(
- line for line in stdout_lines if line.lstrip().startswith(("rg: ", "grep: ")))
- all_files = [
- line for line in stdout_lines
- if line and not line.lstrip().startswith(("rg: ", "grep: "))]
- # Permission-denied diagnostics are partial-success signals independent of
- # the exit code: exit 2 = traversal hit unreadable paths (degrade below),
- # exit 124 = timed out while ALSO skipping them (warning rides along).
- skip = _skipped_fields(diagnostics) if _is_permission_only_error(diagnostics) \
- else {"skipped_paths": [], "skipped_omitted": 0}
- permission_only = result.exit_code == 2 and bool(skip["skipped_paths"])
- warning = _prepend_skipped_paths_warning(skip, None)
- if scoped_common:
- all_files = [
- f if posixpath.isabs(f) else posixpath.normpath(posixpath.join(scoped_common, f))
- for f in all_files]
- bounded_sigpipe = result.exit_code == 141 and len(all_files) >= fetch_limit
- if (result.exit_code not in {0, 1, 124} and not bounded_sigpipe
- and not permission_only):
- detail = diagnostics.strip() or "no diagnostics"
- if order == "modified":
- # ripgrep 14+ was already verified by the capability gate, so this
- # failure is an invocation/traversal error, not a version problem.
- return SearchResult(error=(
- "Exact modification-time order search failed (ripgrep 14+ "
- "required for --sortr=modified; the installed rg passed the "
- f"version check, so this is an invocation failure: {detail}). "
- "Use order='discovery'."))
- return SearchResult(error=(
- f"File search failed while running ripgrep: {detail}"))
- # rg --files lists files only; supplement with matching directories so
- # empty directories stay discoverable (#54347). Skip the extra traversal
- # when rg already timed out — partial file results must not be presented
- # as a coherent page. Only the single-root path supplements (a multi-root
- # walk is already the merged view across roots).
- if limit_reason != "search_timeout" and isinstance(path, str) and self._has_command("find"):
- dirs, directory_limit_reason = self._search_directories_with_find(glob_pattern, path, fetch_limit)
- if dirs:
- existing = set(all_files)
- merged_entries = self._sort_paths_by_mtime(
- all_files + [d for d in dirs if d not in existing])
- # Under exact modified order rg is authoritative for files;
- # the merged view keeps the newest-first contract globally.
- all_files = merged_entries
- limit_reason = limit_reason or directory_limit_reason
- if permission_only and not all_files:
- return SearchResult(files=[], total_count=0,
- warning=f"{warning} {_PERMISSION_SEARCH_HINT}", **skip)
- return SearchResult(
- files=all_files[offset:offset + limit], total_count=len(all_files),
- truncated=len(all_files) > offset + limit or bool(limit_reason),
- limit_reason=limit_reason, warning=warning, **skip)
- def _search_content(self, pattern: str, path: str, file_glob: Optional[str],
- limit: int, offset: int, output_mode: str, context: int) -> SearchResult:
- """Content search: rg, else grep; attaches zero-match steering hints."""
- used_rg = self._has_command('rg')
- if used_rg:
- result = self._search_with_rg(pattern, path, file_glob, limit, offset, output_mode, context,
- rg_executable=self._resolve_command("rg") or "rg")
- elif self._has_command('grep'):
- result = self._search_with_grep(pattern, path, file_glob, limit, offset, output_mode, context)
- else:
- return SearchResult(
- error="Content search requires ripgrep (rg) or grep. "
- "Install ripgrep: https://github.com/BurntSushi/ripgrep#installation")
- if (not result.error and result.total_count == 0
- and not result.matches and not result.files and not result.counts):
- try:
- hint = self._zero_match_probe(pattern, path, file_glob)
- except Exception:
- hint = None
- if hint:
- result.warning = hint if not result.warning else f"{result.warning} {hint}"
- # rg auto-enables --multiline for \n patterns, so the line-oriented
- # explanation only applies to the grep fallback.
- if used_rg:
- return result
- return _maybe_warn_line_oriented_newline_pattern(result, pattern)
- def _run_search_pipeline(self, cmd_parts: list[str], output_mode: str, limit: int,
- offset: int, context: int, warning: Optional[str] = None,
- line_cap: bool = False) -> SearchResult:
- """Run ``cmd_parts | head -n <fetch_limit>`` under pipefail and parse. Extra
- rows report the true total (context mode also emits "--" separators, so
- grab 200 more). pipefail keeps the engine's exit 2 alive across ``| head``
- (a truncating head makes rg exit 0 / grep 141, which the ==2 guard ignores).
- ``line_cap`` appends ``| cut -c1-2000`` for engines without --max-columns
- (grep): bounds giant single-line matches at the pipe layer; skipped for
- files_only/count where lines are paths/counts."""
- fetch_limit = limit + offset + (200 if context > 0 else 0)
- if line_cap: # grep/find pipelines: shell only, with the column cap
- parts = cmd_parts + ["|", "head", "-n", str(fetch_limit)]
- if output_mode not in ("files_only", "count"):
- parts += ["|", "cut", "-c1-2000"]
- result = self._exec("set -o pipefail; " + " ".join(parts), timeout=60)
- else:
- result = self._run_rg_bounded(cmd_parts, fetch_limit, timeout=60, merge_stderr=True,
- shell_prefix="set -o pipefail; ")
- return _parse_search_output(result, output_mode, limit, offset, context, warning=warning)
- def _search_with_rg(self, pattern: str, path: str, file_glob: Optional[str],
- limit: int, offset: int, output_mode: str, context: int,
- rg_executable: Optional[str] = None) -> SearchResult:
- """Search using ripgrep."""
- rg_executable = rg_executable or self._resolve_command("rg")
- if not rg_executable:
- return SearchResult(error="Content search requires ripgrep (rg).")
- cmd_parts = [self._quote_executable(rg_executable), "--line-number", "--no-heading", "--with-filename"]
- # Giant-single-line containment (cline#13525): a match inside a multi-MB
- # single-line dump makes rg emit the ENTIRE line (``head -n`` counts lines).
- # --max-columns bounds each printed line at the rg layer; --max-columns-preview
- # keeps a truncated prefix so the model still sees the hit. 2000 cols exceeds
- # the 500-char content clamp, so nothing previously visible is lost.
- if output_mode not in ("files_only", "count"):
- cmd_parts.extend(["--max-columns", "2000", "--max-columns-preview"])
- # A regex \n hard-errors in line-oriented mode; enable -U up front and say so.
- multiline = _pattern_has_regex_newline(pattern)
- if multiline:
- cmd_parts.append("--multiline")
- if context > 0:
- cmd_parts.extend(["-C", str(context)])
- cmd_parts.extend(self._rg_exclusion_globs(path))
- if file_glob:
- cmd_parts.extend(["--glob", self._escape_shell_arg(file_glob)])
- if output_mode in _OUTPUT_MODE_FLAGS:
- cmd_parts.append(_OUTPUT_MODE_FLAGS[output_mode])
- cmd_parts.append(self._escape_shell_arg(pattern, translate_path=False))
- # rg is a native Windows binary (winget/cargo/choco): needs C:/... not MSYS /c/...
- cmd_parts.append(self._escape_native_tool_arg(path))
- ml_note = (
- "Pattern contains \\n — multiline mode (-U) was enabled automatically "
- "so the regex can match across line boundaries."
- ) if multiline else None
- return self._run_search_pipeline(cmd_parts, output_mode, limit, offset, context, warning=ml_note)
- def _grep_cmd(self, head: list[str], pattern: str, output_mode: str, context: int,
- file_glob: Optional[str] = None) -> list[str]:
- """``head`` + context/include/mode flags + quoted pattern (argument order is fixed)."""
- parts = list(head)
- if context > 0:
- parts.extend(["-C", str(context)])
- if file_glob:
- parts.extend(["--include", self._escape_shell_arg(file_glob)])
- if output_mode in _OUTPUT_MODE_FLAGS:
- parts.append(_OUTPUT_MODE_FLAGS[output_mode])
- parts.append(self._escape_shell_arg(pattern, translate_path=False))
- return parts
- def _search_with_grep(self, pattern: str, path: str, file_glob: Optional[str],
- limit: int, offset: int, output_mode: str, context: int) -> SearchResult:
- """Fallback search using grep."""
- # grep's --exclude-dir matches BASENAMES anywhere, so it can't express "only
- # the home-level Downloads"; route pruning through find's path-scoped -prune.
- protected_paths = self._protected_prune_paths(path)
- # grep applies --exclude-dir='.*' to the command-line root too (GNU grep: to
- # every component of it), so a search rooted under a hidden dir such as
- # ~/.hermes returns nothing (#18473); find's -prune only sees descendants.
- if protected_paths or self._root_under_hidden_dir(path):
- return self._search_with_grep_pruned(
- pattern, path, file_glob, limit, offset, output_mode, context, protected_paths)
- # -H forces filenames; -E matches rg regex behavior; --exclude-dir='.*'
- # mirrors rg's hidden-dir default (.git/, .hub/index-cache/, ...).
- cmd_parts = self._grep_cmd(["grep", "-rnHE", "--exclude-dir='.*'"], pattern, output_mode, context, file_glob)
- # --exclude-dir applies to the root too, so "." would be excluded by '.*';
- # anchor relative paths at the shell's live $PWD.
- is_absolute = path.startswith(("/", "\\\\")) or bool(re.match(r"^[A-Za-z]:[\\/]", path))
- if is_absolute:
- search_root = self._escape_shell_arg(path)
- else:
- relative_path = path.removeprefix("./")
- search_root = '"$PWD"'
- if relative_path not in {"", "."}:
- search_root += f"/{self._escape_shell_arg(relative_path)}"
- cmd_parts.append(search_root)
- return self._run_search_pipeline(cmd_parts, output_mode, limit, offset, context, line_cap=True)
- def _search_with_grep_pruned(self, pattern: str, path: str, file_glob: Optional[str],
- limit: int, offset: int, output_mode: str, context: int,
- protected_paths: list[str]) -> SearchResult:
- """grep fallback via ``find ... -prune -exec grep {} +``, used when the root needs
- path-scoped pruning (macOS protected dirs) or is itself under a dot-directory
- (#18473: grep's ``--exclude-dir='.*'`` would drop the root). Trade-off: find folds
- grep's exit code, so a hard grep error surfaces as an empty result."""
- grep_parts = self._grep_cmd(["grep", "-nHE"], pattern, output_mode, context)
- q_root = self._escape_shell_arg(path or ".")
- # ``-H``: follow a symlink handed in as the OPERAND (and only the operand). Without
- # it ``find <link> -type f`` tests the link itself and hands grep nothing, so a
- # symlinked root answered a confident ``total_count: 0`` on every platform (#116270).
- find_parts = ["find", "-H", q_root]
- if protected_paths:
- find_parts.extend([self._prune_expr(protected_paths), "-o"])
- find_parts.extend([self._hidden_prune_expr([q_root]), "-o", "-type f"])
- if file_glob:
- find_parts.extend(["-name", self._escape_shell_arg(file_glob)])
- # stderr stays attached (``_exec`` merges it into stdout): find's
- # ``Permission denied`` lines must reach the parser so an unreadable
- # subtree beside a readable match is reported, not silently dropped.
- find_parts.extend(["-exec", *grep_parts, "{}", "+"])
- return self._run_search_pipeline(find_parts, output_mode, limit, offset, context, line_cap=True)
|