subdirectory_hints.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. """Progressive subdirectory hint discovery: as the agent navigates into
  2. subdirectories via tool calls, load project context files (AGENTS.md, CLAUDE.md,
  3. .cursorrules) from them and append to the tool result — context arrives without
  4. touching the system prompt (prompt caching preserved). Complements the startup
  5. CWD-only loading in ``prompt_builder.py``."""
  6. import hashlib
  7. import logging
  8. import os
  9. import shlex
  10. from pathlib import Path
  11. from typing import Dict, Any, Optional, Set
  12. from agent.prompt_builder import _read_text_with_timeout, _scan_context_content, _truncate_content
  13. from agent.search_policy import SEARCH_PRUNE_DIR_NAMES
  14. logger = logging.getLogger(__name__)
  15. # Same filenames as prompt_builder.py, in priority order (first match wins per dir).
  16. _HINT_FILENAMES = ["AGENTS.override.md", "AGENTS.md", "agents.md", "CLAUDE.md", "claude.md", ".cursorrules"]
  17. # Per-file ceiling for on-demand subdirectory hints. 32 KiB matches Codex's `project_doc_max_bytes` default
  18. # (Claude Code and Cursor apply none); it is a guard against a stray huge CLAUDE.md in a vendored tree, not a
  19. # target — keep area AGENTS.md files well under it (~8k) because this text lands in a tool result on the first
  20. # touch of that directory. Over the ceiling: head+tail kept, marker with the path so the agent can read_file it,
  21. # and a WARNING in the log (the old 8k silent tail-chop cut apps/desktop/AGENTS.md for months unnoticed).
  22. _MAX_HINT_CHARS = 32_000
  23. _PATH_ARG_KEYS = {"path", "file_path", "workdir"}
  24. _COMMAND_TOOLS = {"terminal"}
  25. _MAX_ANCESTOR_WALK = 5 # ancestor levels walked per path — bounds deep-path scans
  26. # Shared with broad recursive search probes so context discovery and search never drift into
  27. # different dependency/cache/build trees (those hold *copies* of context files, never authoritative ones).
  28. _EXCLUDED_DIR_NAMES = SEARCH_PRUNE_DIR_NAMES
  29. def _digest(content: str) -> str:
  30. return hashlib.sha256(content.encode("utf-8")).hexdigest()
  31. def _resolved_hint_target(hint_path: Path, working_dir: Path) -> Optional[Path]:
  32. """Resolved hint-file target, or None when the file must not be loaded.
  33. ``is_file()`` and ``read_text`` follow symlinks, so a checked-in
  34. ``sub/AGENTS.md -> ~/.aws/credentials`` would inject an out-of-tree file
  35. into the tool result. The resolved target must stay inside the resolved
  36. working dir (same containment ``_within_working_dir`` applies to the
  37. directory itself) and pass the canonical read deny-list
  38. (``context_references`` applies it to explicit @-references), which also
  39. catches in-tree targets like a symlink to the project ``.env``.
  40. """
  41. try:
  42. resolved = hint_path.resolve()
  43. except (OSError, RuntimeError):
  44. return None
  45. try:
  46. inside = resolved.is_relative_to(working_dir)
  47. except (OSError, ValueError, RuntimeError):
  48. inside = False
  49. if not inside:
  50. return None
  51. try:
  52. from agent.file_safety import get_read_block_error
  53. blocked = get_read_block_error(str(resolved)) is not None
  54. except Exception:
  55. # Mirror context_references: a deny-list lookup that fails re-opens the
  56. # exact hole the check closes, so fail closed.
  57. return None
  58. return None if blocked else resolved
  59. def _first_hint_file(directory: Path):
  60. """``(path, stripped content)`` of the first readable non-empty hint file
  61. in *directory* (priority order), or None. Unreadable files are skipped."""
  62. from agent.safe_worker_policy import safe_worker_enabled
  63. if safe_worker_enabled():
  64. return None
  65. for filename in _HINT_FILENAMES:
  66. candidate = directory / filename
  67. try:
  68. if not candidate.is_file() or (target := _resolved_hint_target(candidate, directory)) is None:
  69. continue
  70. # Read the resolved target (not the link path) so a symlink swapped
  71. # between check and read still lands on the vetted file.
  72. content = target.read_text(encoding="utf-8-sig").strip()
  73. except (OSError, UnicodeDecodeError):
  74. continue
  75. return candidate, content
  76. return None
  77. _NAV_COMMANDS = frozenset({"cd", "pushd"})
  78. _SHELL_OPERATORS = frozenset({"&&", "||", "|", ";", "&", ";;", "|&", "(", ")"})
  79. def _nav_targets(cmd: str) -> list:
  80. """Operands of `cd` / `pushd` that begin a shell segment. `cd -` and bare `cd` yield nothing."""
  81. lexer = shlex.shlex(cmd, posix=True, punctuation_chars=True)
  82. lexer.whitespace_split = True
  83. try:
  84. tokens = list(lexer)
  85. except ValueError:
  86. return []
  87. targets, segment_start = [], True
  88. for idx, token in enumerate(tokens):
  89. if token in _SHELL_OPERATORS:
  90. segment_start = True
  91. continue
  92. if segment_start and token in _NAV_COMMANDS:
  93. operand = next((t for t in tokens[idx + 1:] if t in _SHELL_OPERATORS or not t.startswith("-")), None)
  94. if operand and operand not in _SHELL_OPERATORS:
  95. targets.append(operand)
  96. segment_start = False
  97. return targets
  98. # A container cwd is not a host path: a docker/local session whose working dir is the backend
  99. # container's default /home (or /root) home must not start hint discovery either — the same
  100. # vacuous-containment hazard as the desktop home pin, just container-shaped (#76902).
  101. _CONTAINER_HOMES = frozenset({Path("/home"), Path("/root"), Path("/workspace")})
  102. def _is_home_like_working_dir(working_dir: Path) -> bool:
  103. """True when ``working_dir`` is a home the desktop/container fell back to — not a project."""
  104. home = Path.home()
  105. if working_dir == home or working_dir in _CONTAINER_HOMES or working_dir.parent in _CONTAINER_HOMES:
  106. return True
  107. # Also the local-sandbox home mount: a docker backend binds the host home under a
  108. # container path whose basename stays the user name (e.g. /mnt/host-home/brooklyn).
  109. for part in working_dir.parts:
  110. if part in (".home", "host-home", "host_home"):
  111. return True
  112. return False
  113. class SubdirectoryHintTracker:
  114. """Track which directories the agent visits and load hints on first access.
  115. Usage: after each tool call, ``hints = tracker.check_tool_call(name, args)``
  116. and append the returned text to the tool result.
  117. """
  118. def __init__(self, working_dir: Optional[str] = None, *, enabled: bool = True):
  119. # ``enabled=False`` mirrors ``skip_context_files``: a session that opted out of
  120. # AGENTS.md/CLAUDE.md injection at startup must not get the same files spliced into
  121. # tool results later — cron jobs relaying exact stdout leaked them to chat (#9441).
  122. self.enabled = enabled
  123. self.working_dir = Path(working_dir or os.getcwd()).resolve()
  124. # $HOME is not a project (#76902): the packaged Desktop app with no default project
  125. # dir configured resolves its cwd (and TERMINAL_CWD) to home, and the containment
  126. # check is then vacuously true for the whole home subtree — every AGENTS.md /
  127. # CLAUDE.md / .cursorrules under ~ would be injected on tool calls. Hint discovery
  128. # runs only inside a real project workspace; a home cwd only resumes when the
  129. # session adopts a real project (see ``rebind_working_dir``).
  130. self._home_is_working_dir = _is_home_like_working_dir(self.working_dir)
  131. # The working dir is pre-marked loaded (startup context handles it).
  132. self._loaded_dirs: set[Path] = {self.working_dir}
  133. # Content digests already injected: the same file reached through
  134. # symlinks/hardlinks/copies is never re-sent. Seeded with the CWD hint
  135. # file prompt_builder already loaded.
  136. self._loaded_digests: set[str] = set()
  137. found = _first_hint_file(self.working_dir)
  138. if found and found[1]:
  139. self._loaded_digests.add(_digest(found[1]))
  140. def rebind_working_dir(self, working_dir: str) -> None:
  141. """Re-anchor hint discovery to *working_dir* (a session workspace adoption)."""
  142. try:
  143. new_dir = Path(working_dir).expanduser().resolve()
  144. if not new_dir.is_dir() or _is_home_like_working_dir(new_dir):
  145. return
  146. except (OSError, ValueError, RuntimeError):
  147. return
  148. if new_dir == self.working_dir:
  149. self._home_is_working_dir = False
  150. return
  151. self.working_dir = new_dir
  152. self._home_is_working_dir = False
  153. # Fresh anchor, fresh bookkeeping: the new project's directories get their
  154. # own first-visit hints (startup never loaded context files for it).
  155. self._loaded_dirs = {self.working_dir}
  156. self._loaded_digests = set()
  157. found = _first_hint_file(self.working_dir)
  158. if found and found[1]:
  159. self._loaded_digests.add(_digest(found[1]))
  160. def check_tool_call(self, tool_name: str, tool_args: dict[str, Any]) -> Optional[str]:
  161. """Return formatted hint text for newly visited directories, or None."""
  162. from agent.safe_worker_policy import safe_worker_enabled
  163. if not self.enabled or safe_worker_enabled():
  164. return None
  165. if self._home_is_working_dir:
  166. return None
  167. all_hints = [h for d in self._extract_directories(tool_name, tool_args) if (h := self._load_hints_for_directory(d))]
  168. return "\n\n" + "\n\n".join(all_hints) if all_hints else None
  169. def _extract_directories(self, tool_name: str, args: dict[str, Any]) -> list:
  170. """Extract directory paths from tool call arguments."""
  171. candidates: set[Path] = set()
  172. for key in _PATH_ARG_KEYS:
  173. val = args.get(key)
  174. if isinstance(val, str) and val.strip():
  175. self._add_path_candidate(val, candidates)
  176. cmd = args.get("command", "") if tool_name in _COMMAND_TOOLS else None
  177. if isinstance(cmd, str):
  178. self._extract_paths_from_command(cmd, candidates)
  179. return list(candidates)
  180. def _add_path_candidate(self, raw_path: str, candidates: set[Path]):
  181. """Add a raw path's directory and its ancestors (up to ``_MAX_ANCESTOR_WALK``
  182. levels, stopping at the first already-loaded dir) so reading
  183. ``project/src/main.py`` still discovers ``project/AGENTS.md``."""
  184. try:
  185. p = Path(raw_path).expanduser()
  186. if not p.is_absolute():
  187. p = self.working_dir / p
  188. p = p.resolve()
  189. if p.suffix or (p.exists() and p.is_file()):
  190. p = p.parent
  191. for _ in range(_MAX_ANCESTOR_WALK):
  192. if p in self._loaded_dirs:
  193. break
  194. if self._is_valid_subdir(p):
  195. candidates.add(p)
  196. if p.parent == p:
  197. break # filesystem root
  198. p = p.parent
  199. except (OSError, ValueError, RuntimeError):
  200. pass
  201. def _extract_paths_from_command(self, cmd: str, candidates: set[Path]):
  202. """Extract path-like tokens (contain / or .; not flags or URLs) from a shell command."""
  203. try:
  204. tokens = shlex.split(cmd)
  205. except ValueError:
  206. tokens = cmd.split()
  207. # `cd backend && ls`: a bare directory name has no `/` or `.`, so the generic filter below drops
  208. # it; the operand of a navigation command is a path by construction (#11032). Only a `cd` at the
  209. # START of a shell segment counts (`echo cd backend` is prose); punctuation-aware tokenizing keeps
  210. # a quoted `'backend;'` literal while splitting bare `backend;ls` at the operator.
  211. for target in _nav_targets(cmd):
  212. self._add_path_candidate(target, candidates)
  213. for token in tokens:
  214. if token.startswith(("-", "http://", "https://", "git@")) or ("/" not in token and "." not in token):
  215. continue
  216. self._add_path_candidate(token, candidates)
  217. def _within_working_dir(self, path: Path) -> bool:
  218. """Reject paths outside the working-dir tree: loading ~/.codex/AGENTS.md
  219. or ~/.claude/CLAUDE.md would mix another agent's instructions into this
  220. session. Falls back to an ancestor check when ``is_relative_to`` fails."""
  221. try:
  222. return path.is_relative_to(self.working_dir)
  223. except (OSError, ValueError):
  224. try:
  225. path.relative_to(self.working_dir)
  226. return True
  227. except ValueError:
  228. return False
  229. def _is_valid_subdir(self, path: Path) -> bool:
  230. """Directory inside the working-dir tree, not yet loaded, not an excluded copy dir."""
  231. try:
  232. if not path.is_dir():
  233. return False
  234. except OSError:
  235. return False
  236. return path not in self._loaded_dirs and self._within_working_dir(path) and not self._is_excluded(path)
  237. def _is_excluded(self, path: Path) -> bool:
  238. """True when a segment *below* the working dir is an excluded copy dir
  239. (a user deliberately working inside ``vendor/`` keeps that segment legitimate)."""
  240. try:
  241. rel_parts = path.relative_to(self.working_dir).parts
  242. except ValueError:
  243. return True # outside the tree — already rejected upstream
  244. return any(part in _EXCLUDED_DIR_NAMES for part in rel_parts)
  245. def _load_hints_for_directory(self, directory: Path) -> Optional[str]:
  246. """Load the first hint file in *directory*; formatted text or None."""
  247. self._loaded_dirs.add(directory)
  248. if self._home_is_working_dir or not self._within_working_dir(directory):
  249. logger.debug("Skipping hint files in %s — outside working_dir %s", directory, self.working_dir)
  250. return None
  251. for filename in _HINT_FILENAMES:
  252. hint_path = directory / filename
  253. try:
  254. if not hint_path.is_file():
  255. continue
  256. except OSError:
  257. continue
  258. if (target := _resolved_hint_target(hint_path, self.working_dir)) is None:
  259. continue
  260. try:
  261. content = (_read_text_with_timeout(target) or "").strip()
  262. if not content:
  263. continue
  264. digest = _digest(content)
  265. if digest in self._loaded_digests:
  266. logger.debug("Skipping duplicate hint content at %s (digest %s)", hint_path, digest[:12])
  267. return None
  268. self._loaded_digests.add(digest)
  269. # Same security scan as startup context loading.
  270. content = _scan_context_content(content, filename)
  271. rel_path = self._display_path(hint_path)
  272. content = _truncate_content(
  273. content, filename, max_chars=_MAX_HINT_CHARS, read_path=rel_path, queue_warning=False,
  274. )
  275. logger.debug("Loaded subdirectory hints from %s: %s", directory, [rel_path])
  276. return f"[Subdirectory context discovered: {rel_path}]\n{content}" # first match wins per directory
  277. except Exception as exc:
  278. logger.debug("Could not read %s: %s", hint_path, exc)
  279. return None
  280. def _display_path(self, hint_path: Path) -> str:
  281. """Working-dir-relative, else ``~/``-relative (POSIX rendering so Windows
  282. never shows ``~/AppData\\Local\\...`` chimeras), else absolute."""
  283. try:
  284. return str(hint_path.relative_to(self.working_dir))
  285. except (ValueError, RuntimeError):
  286. pass
  287. try:
  288. return "~/" + hint_path.relative_to(Path.home()).as_posix()
  289. except (ValueError, RuntimeError):
  290. return str(hint_path)