context_references.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702
  1. """@-reference expansion (``@file:``, ``@folder:``, ``@diff``, ``@git:``, ``@url:`` + plugin prefixes)."""
  2. from __future__ import annotations
  3. import asyncio
  4. import inspect
  5. import json
  6. import mimetypes
  7. import os
  8. import re
  9. import subprocess
  10. import threading
  11. from abc import ABC, abstractmethod
  12. from dataclasses import dataclass, field
  13. from pathlib import Path
  14. from typing import Awaitable, Callable
  15. from agent.file_safety import HOME_CREDENTIAL_DIRS
  16. from agent.model_metadata import CHARS_PER_TOKEN, estimate_tokens_rough
  17. from hermes_cli._subprocess_compat import IS_WINDOWS, harden_git_argv, noninteractive_git_env, windows_hide_flags
  18. from hermes_cli.sqlite_safe_read import LiveConnectionError, offline_file_access
  19. from hermes_cli.sizefmt import format_bytes
  20. # ── Plugin context-reference provider API ────────────────────────────────────
  21. # --------------------------------------------------------------------------- Plugin context-reference
  22. # provider API (Issue #26193) ---------------------------------------------------------------------------
  23. BUILTIN_PREFIXES = frozenset({"diff", "staged", "file", "folder", "git", "url"})
  24. _context_reference_providers: dict[str, ContextReferenceProvider] = {}
  25. @dataclass(slots=True)
  26. class ContextCompletionItem:
  27. """A single autocomplete result from a context reference provider (a dataclass, so it crosses
  28. the plugin-host wire as data)."""
  29. text: str
  30. display: str = ""
  31. meta: str = ""
  32. def __post_init__(self) -> None:
  33. self.display = self.display or self.text
  34. class ContextReferenceProvider(ABC):
  35. """Base class for plugin @-prefix providers, registered via ``PluginContext.register_context_reference()``."""
  36. prefix: str = "" # e.g. "issue", "channel", "doc"
  37. description: str = "" # shown in autocomplete meta column
  38. @abstractmethod
  39. async def autocomplete(self, query: str, *, limit: int = 10) -> list[ContextCompletionItem]:
  40. """Return autocomplete items for the given query string."""
  41. @abstractmethod
  42. async def expand(self, target: str) -> str | None:
  43. """Expand *target* to prompt content. Return ``None`` to skip."""
  44. def register_context_reference_provider(provider: ContextReferenceProvider) -> None:
  45. """Register a plugin context reference provider."""
  46. if not isinstance(provider, ContextReferenceProvider):
  47. raise TypeError("provider must be a ContextReferenceProvider instance")
  48. prefix = provider.prefix.lower().strip()
  49. if not prefix:
  50. raise ValueError("prefix must be a non-empty string")
  51. if prefix in BUILTIN_PREFIXES:
  52. raise ValueError(f"prefix '{prefix}' is reserved for built-in references")
  53. if prefix in _context_reference_providers:
  54. raise ValueError(f"prefix '{prefix}' is already registered")
  55. _context_reference_providers[prefix] = provider
  56. def get_context_reference_providers() -> dict[str, ContextReferenceProvider]:
  57. """Return a snapshot of all registered plugin providers."""
  58. return dict(_context_reference_providers)
  59. _QUOTED_REFERENCE_VALUE = r'(?:`[^`\n]+`|"[^"\n]+"|\'[^\'\n]+\')'
  60. REFERENCE_PATTERN = re.compile(
  61. rf"(?<![\w/])@(?:(?P<simple>diff|staged)\b|(?P<kind>file|folder|git|url):(?P<value>{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+))"
  62. )
  63. # Plugin fallback: any @<word>:<value> the built-in regex did not claim.
  64. _PLUGIN_REFERENCE_PATTERN = re.compile(
  65. rf"(?<![\w/])@(?P<kind>[a-zA-Z][a-zA-Z0-9_-]*):(?P<value>{_QUOTED_REFERENCE_VALUE}(?::\d+(?:-\d+)?)?|\S+)"
  66. )
  67. # ``@file:`` value: quoted path or bare path, each with an optional ``:start[-end]`` range.
  68. _FILE_VALUE_PATTERN = re.compile(
  69. r'^(?:(?P<quote>`|"|\')(?P<qpath>.+?)(?P=quote)|(?P<path>.+?))(?::(?P<start>\d+)(?:-(?P<end>\d+))?)?$'
  70. )
  71. TRAILING_PUNCTUATION = ",.;!?"
  72. _OPENERS = {")": "(", "]": "[", "}": "{"}
  73. _NEEDS_QUOTING = re.compile(r"""[\s()\[\]{}<>"'`]""")
  74. _SENSITIVE_HERMES_DIRS = (Path("skills") / ".hub",)
  75. _SENSITIVE_HOME_FILES = tuple(Path(p) for p in (
  76. ".ssh/authorized_keys", ".ssh/id_rsa", ".ssh/id_ed25519", ".ssh/config", ".bashrc", ".zshrc",
  77. ".profile", ".bash_profile", ".zprofile", ".netrc", ".pgpass", ".npmrc", ".pypirc",
  78. ))
  79. _TEXT_EXTENSIONS = (".py", ".md", ".txt", ".json", ".yaml", ".yml", ".toml", ".js", ".ts")
  80. # Bound the work one message can force: each expanded ref reads at most a bounded prefix /
  81. # window, and at most this many refs are expanded per message.
  82. _MAX_EXPANDED_REFERENCES = 16
  83. # Folder-listing line counts stop paying I/O past this size; bigger files report bytes only.
  84. _LINE_COUNT_MAX_BYTES = 4 * 1024 * 1024
  85. # Per-command stdout/stderr ceiling for the ``git``/``rg`` helpers; past it the child is
  86. # killed and the nonzero returncode routes the caller to its fallback path.
  87. _MAX_QUIET_OUTPUT_BYTES = 4 * 1024 * 1024
  88. _OUTPUT_CAP_EXCEEDED_RETURNCODE = 137 # 128 + SIGKILL, the code a shell reports for a killed child
  89. _FENCE_LANGUAGES = {
  90. ".py": "python", ".js": "javascript", ".ts": "typescript", ".tsx": "tsx", ".jsx": "jsx",
  91. ".json": "json", ".md": "markdown", ".sh": "bash", ".yml": "yaml", ".yaml": "yaml", ".toml": "toml",
  92. }
  93. @dataclass(frozen=True)
  94. class ContextReference:
  95. raw: str
  96. kind: str
  97. target: str
  98. start: int
  99. end: int
  100. line_start: int | None = None
  101. line_end: int | None = None
  102. @dataclass
  103. class ContextReferenceResult:
  104. message: str
  105. original_message: str
  106. references: list[ContextReference] = field(default_factory=list)
  107. warnings: list[str] = field(default_factory=list)
  108. injected_tokens: int = 0
  109. expanded: bool = False
  110. blocked: bool = False
  111. UrlFetcher = Callable[[str], str | Awaitable[str]] | None
  112. Expansion = tuple[str | None, str | None] # (warning, block) — exactly one side is set
  113. def format_reference_value(value: str) -> str:
  114. """Quote a value so ``REFERENCE_PATTERN`` (bare alternative ``\\S+``) reads it back whole.
  115. Mirrors ``formatRefValue`` in the desktop's directive-text.tsx."""
  116. if not _NEEDS_QUOTING.search(value):
  117. return value
  118. for quote in ("`", '"', "'"):
  119. if quote not in value:
  120. return f"{quote}{value}{quote}"
  121. return value
  122. def parse_context_references(message: str, *, plugins: bool = True) -> list[ContextReference]:
  123. """``plugins=False`` reads built-in kinds only (a safe-mode / config-bypass session loads no plugins)."""
  124. refs: list[ContextReference] = []
  125. if not message:
  126. return refs
  127. for match in REFERENCE_PATTERN.finditer(message):
  128. kind = match.group("simple") or match.group("kind")
  129. value = _strip_trailing_punctuation(match.group("value") or "")
  130. if match.group("simple"):
  131. target, line_start, line_end = "", None, None
  132. elif kind == "file":
  133. target, line_start, line_end = _parse_file_reference_value(value)
  134. else:
  135. target, line_start, line_end = _strip_reference_wrappers(value), None, None
  136. refs.append(ContextReference(match.group(0), kind, target, match.start(), match.end(), line_start, line_end))
  137. # Second pass: plugin-registered prefixes the built-in pattern missed.
  138. for match in _PLUGIN_REFERENCE_PATTERN.finditer(message) if plugins and _context_reference_providers else ():
  139. kind = match.group("kind")
  140. if kind in BUILTIN_PREFIXES or kind not in _context_reference_providers:
  141. continue
  142. if any(r.kind == kind and r.start == match.start() for r in refs):
  143. continue
  144. target = _strip_reference_wrappers(_strip_trailing_punctuation(match.group("value") or ""))
  145. refs.append(ContextReference(match.group(0), kind, target, match.start(), match.end()))
  146. return refs
  147. def preprocess_context_references(
  148. message: str, *, cwd: str | Path, context_length: int, url_fetcher: UrlFetcher = None,
  149. allowed_root: str | Path | None = None,
  150. ) -> ContextReferenceResult:
  151. """Sync wrapper; safe both without a loop (CLI) and inside a running loop (gateway)."""
  152. coro = preprocess_context_references_async(
  153. message, cwd=cwd, context_length=context_length, url_fetcher=url_fetcher, allowed_root=allowed_root
  154. )
  155. try:
  156. asyncio.get_running_loop()
  157. except RuntimeError:
  158. return asyncio.run(coro)
  159. import concurrent.futures
  160. import contextvars
  161. # The side thread starts with an empty Context: without the caller's copy the served profile's
  162. # HERMES_HOME override is lost and the credential-path guard checks the launch profile's .env.
  163. with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
  164. return pool.submit(contextvars.copy_context().run, asyncio.run, coro).result()
  165. async def preprocess_context_references_async(
  166. message: str, *, cwd: str | Path, context_length: int, url_fetcher: UrlFetcher = None,
  167. allowed_root: str | Path | None = None, plugins: bool = True,
  168. ) -> ContextReferenceResult:
  169. refs = parse_context_references(message, plugins=plugins)
  170. if not refs:
  171. return ContextReferenceResult(message=message, original_message=message)
  172. cwd_path = Path(cwd).expanduser().resolve()
  173. # Default root = cwd so @ references cannot escape the workspace unless a caller widens it.
  174. allowed_root_path = Path(allowed_root).expanduser().resolve() if allowed_root is not None else cwd_path
  175. # Expand concurrently (each ref is independent; several @url: refs would otherwise
  176. # serialize web_extract round-trips). gather preserves order, so warnings/blocks
  177. # are assembled in ref order; the token-budget check runs once afterwards.
  178. hard_limit = max(1, int(context_length * 0.50))
  179. soft_limit = max(1, int(context_length * 0.25))
  180. tasks = (
  181. _expand_reference(ref, cwd_path, url_fetcher=url_fetcher, allowed_root=allowed_root_path,
  182. max_inline_tokens=hard_limit)
  183. for ref in refs[:_MAX_EXPANDED_REFERENCES]
  184. )
  185. expanded = await asyncio.gather(*tasks)
  186. warnings = [warning for warning, _ in expanded if warning]
  187. warnings.extend(
  188. f"{ref.raw}: not expanded (maximum {_MAX_EXPANDED_REFERENCES} @-references per message)"
  189. for ref in refs[_MAX_EXPANDED_REFERENCES:]
  190. )
  191. blocks = [block for _, block in expanded if block]
  192. injected_tokens = sum(estimate_tokens_rough(block) for block in blocks)
  193. result = ContextReferenceResult(
  194. message=message, original_message=message, references=refs, warnings=warnings, injected_tokens=injected_tokens
  195. )
  196. if injected_tokens > hard_limit:
  197. warnings.append(f"@ context injection refused: {injected_tokens} tokens exceeds the 50% hard limit ({hard_limit}).")
  198. result.blocked = True
  199. return result
  200. if injected_tokens > soft_limit:
  201. warnings.append(f"@ context injection warning: {injected_tokens} tokens exceeds the 25% soft limit ({soft_limit}).")
  202. # The `@file:`/`@folder:` tokens stay where the user typed them: the token IS the
  203. # reference (clients render it as an inline chip); stripping it left a hole in the
  204. # sentence and forced the desktop to re-derive refs from the attached block.
  205. final = message
  206. if warnings:
  207. final = f"{final}\n\n--- Context Warnings ---\n" + "\n".join(f"- {warning}" for warning in warnings)
  208. if blocks:
  209. final = f"{final}\n\n--- Attached Context ---\n\n" + "\n\n".join(blocks)
  210. result.message = final.strip()
  211. result.expanded = bool(blocks or warnings)
  212. return result
  213. # Git-backed reference kinds -> f(ref) -> git argv (the label is "git " + argv).
  214. _GIT_REFERENCE_ARGS: dict[str, Callable[[ContextReference], list[str]]] = {
  215. "diff": lambda ref: ["diff"],
  216. "staged": lambda ref: ["diff", "--staged"],
  217. "git": lambda ref: ["log", f"-{max(1, min(int(ref.target or '1'), 10))}", "-p"],
  218. }
  219. async def _expand_reference(
  220. ref: ContextReference, cwd: Path, *, url_fetcher: UrlFetcher = None, allowed_root: Path | None = None,
  221. max_inline_tokens: int | None = None,
  222. ) -> Expansion:
  223. try:
  224. if ref.kind in ("file", "folder"):
  225. return _expand_path_reference(ref, cwd, allowed_root=allowed_root, max_inline_tokens=max_inline_tokens)
  226. if ref.kind in _GIT_REFERENCE_ARGS:
  227. git_args = _GIT_REFERENCE_ARGS[ref.kind](ref)
  228. return _expand_git_reference(ref, cwd, git_args, "git " + " ".join(git_args))
  229. if ref.kind == "url":
  230. content = await _fetch_url_content(ref.target, url_fetcher=url_fetcher)
  231. if not content:
  232. return f"{ref.raw}: no content extracted", None
  233. return None, f"🌐 {ref.raw} ({estimate_tokens_rough(content)} tokens)\n{content}"
  234. except Exception as exc:
  235. return f"{ref.raw}: {exc}", None
  236. provider = _context_reference_providers.get(ref.kind)
  237. if provider is not None:
  238. try:
  239. plugin_content = await provider.expand(ref.target)
  240. if plugin_content is not None:
  241. return None, f"📌 {ref.raw} ({estimate_tokens_rough(plugin_content)} tokens)\n{plugin_content}"
  242. except Exception as exc:
  243. return f"{ref.raw}: plugin expansion error: {exc}", None
  244. return f"{ref.raw}: unsupported reference type", None
  245. def _expand_path_reference(ref: ContextReference, cwd: Path, *, allowed_root: Path | None = None,
  246. max_inline_tokens: int | None = None) -> Expansion:
  247. """``@file:`` / ``@folder:``: resolve, allow-check, then inline text / binary stub / listing."""
  248. is_folder = ref.kind == "folder"
  249. path = _resolve_path(cwd, ref.target, allowed_root=allowed_root)
  250. _ensure_reference_path_allowed(path)
  251. if not path.exists():
  252. return f"{ref.raw}: {ref.kind} not found", None
  253. if not (path.is_dir() if is_folder else path.is_file()):
  254. return f"{ref.raw}: path is not a {ref.kind}", None
  255. if is_folder:
  256. listing = _build_folder_listing(path, cwd, display_base=allowed_root)
  257. return None, f"📁 {ref.raw} ({estimate_tokens_rough(listing)} tokens)\n{listing}"
  258. try:
  259. # Keep admission through every sniff, stat and text read (a connection can start
  260. # between a check and a later open otherwise), but release it before token
  261. # counting/formatting: the registry lock blocks every tracked connect/close.
  262. with offline_file_access(path, what="preview context reference"):
  263. early, text = _read_file_reference(ref, path, max_inline_tokens)
  264. except LiveConnectionError:
  265. return None, _on_disk_reference_block(
  266. ref, path, descriptor="live SQLite database file",
  267. reason="not previewed: raw access would cancel SQLite's POSIX locks.",
  268. guidance="Do not open this file directly while its database connection is live.",
  269. )
  270. return early or _format_file_reference(ref, path, text, max_inline_tokens)
  271. def _read_file_reference(
  272. ref: ContextReference, path: Path, max_inline_tokens: int | None,
  273. ) -> tuple[Expansion | None, str]:
  274. """Raw file I/O for an @file ref: ``(early, text)`` where ``early`` is a refusal block
  275. (then ``text`` is empty) or ``None`` with the text to inline."""
  276. if _is_binary_file(path):
  277. # A bare "not supported" warning was a dead end (the model gave up); the file IS
  278. # on disk where the agent's tools run, so hand it an actionable block instead.
  279. return (None, _binary_reference_block(ref, path)), ""
  280. if ref.line_start is not None:
  281. # A ranged ref wants a slice, not the file: stream to the window so a GB-scale
  282. # file serves :1-5 without being materialized. Lines are read in bounded pieces
  283. # (budget + 1) so even a single-line giant (minified JSON, one-line logs) cannot
  284. # force an unbounded string into memory; skipped lines are discarded, not kept.
  285. char_budget = None if max_inline_tokens is None else max_inline_tokens * CHARS_PER_TOKEN
  286. line_cap = None if char_budget is None else char_budget + 1
  287. def _next_line(fh, collect: bool, remaining: int | None = None) -> str | None:
  288. """One line ("" when skipping), None at EOF. While collecting, stop as soon as the
  289. pieces exceed ``remaining``: the caller returns the oversized block and never needs
  290. the rest of a giant line, so it is never materialized."""
  291. pieces, seen, collected = [], False, 0
  292. while True:
  293. piece = fh.readline() if line_cap is None else fh.readline(line_cap)
  294. if not piece:
  295. break
  296. seen = True
  297. if collect:
  298. pieces.append(piece)
  299. collected += len(piece)
  300. if remaining is not None and collected > remaining:
  301. break
  302. if piece.endswith("\n"):
  303. break
  304. return ("".join(pieces) if collect else "") if seen else None
  305. parts, total_chars = [], 0
  306. with path.open(encoding="utf-8") as fh:
  307. for _ in range(max(ref.line_start - 1, 0)):
  308. if _next_line(fh, collect=False) is None:
  309. break
  310. for _ in range((ref.line_end or ref.line_start) - ref.line_start + 1):
  311. line = _next_line(fh, collect=True, remaining=None if char_budget is None else char_budget - total_chars)
  312. if line is None:
  313. break
  314. total_chars += len(line)
  315. if char_budget is not None and total_chars > char_budget:
  316. return (None, _oversized_text_reference_block(ref, path, total_chars // CHARS_PER_TOKEN)), ""
  317. parts.append(line)
  318. text = "".join(parts)
  319. else:
  320. # estimate_tokens_rough >= bytes/CHARS_PER_TOKEN for every encoding mix, so a
  321. # file past that byte ceiling is certainly oversized; refuse without reading it.
  322. size = path.stat().st_size
  323. if max_inline_tokens is not None and size > max_inline_tokens * CHARS_PER_TOKEN:
  324. return (None, _oversized_text_reference_block(ref, path, size // CHARS_PER_TOKEN)), ""
  325. text = path.read_text(encoding="utf-8-sig")
  326. return None, text
  327. def _format_file_reference(ref: ContextReference, path: Path, text: str, max_inline_tokens: int | None) -> Expansion:
  328. lang = _FENCE_LANGUAGES.get(path.suffix.lower(), "")
  329. text_tokens = estimate_tokens_rough(text)
  330. # Check BEFORE building the fenced block: an oversized file is not going to be
  331. # inlined, so don't build a second MB-scale string just to discard it.
  332. if max_inline_tokens is not None and text_tokens > max_inline_tokens:
  333. # One oversized file used to poison the aggregate check and refuse the whole
  334. # turn (#61987); the file stays readable via the agent's tools instead. The
  335. # block alone carries the message (same shape as the binary path) — a warning
  336. # would duplicate it in "--- Context Warnings ---".
  337. return None, _oversized_text_reference_block(ref, path, text_tokens)
  338. return None, f"📄 {ref.raw} ({text_tokens} tokens)\n```{lang}\n{text}\n```"
  339. def _run_quiet(cmd: list[str], cwd: Path, timeout: int, env: dict | None = None) -> subprocess.CompletedProcess:
  340. """Captured text output with bounded pipes, no stdin, and no console flash on Windows.
  341. ``capture_output=True`` buffers the child's entire stdout — an unbounded ``git diff``
  342. or ``rg --files`` would materialize a hostile-size stream in memory. Each pipe is
  343. drained up to ``_MAX_QUIET_OUTPUT_BYTES``; past it the child is killed and the result
  344. carries a nonzero returncode that routes callers to their existing fallback paths. Set
  345. explicitly: a child that flushed everything and exited 0 before the drain thread crossed
  346. the cap is not killable and would otherwise report success with truncated output.
  347. """
  348. popen_kwargs: dict = {"creationflags": windows_hide_flags()} if IS_WINDOWS else {}
  349. proc = subprocess.Popen(
  350. cmd, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  351. **popen_kwargs, **({} if env is None else {"env": env}))
  352. truncated = threading.Event()
  353. def _drain(stream, sink: list[bytes]) -> None:
  354. total = 0
  355. while True:
  356. chunk = stream.read1(1 << 16)
  357. if not chunk:
  358. return
  359. total += len(chunk)
  360. if total <= _MAX_QUIET_OUTPUT_BYTES:
  361. sink.append(chunk)
  362. else:
  363. truncated.set()
  364. proc.kill()
  365. return
  366. sinks: list[list[bytes]] = [[], []]
  367. threads = [
  368. threading.Thread(target=_drain, args=(stream, sink), daemon=True)
  369. for stream, sink in zip((proc.stdout, proc.stderr), sinks)
  370. ]
  371. for thread in threads:
  372. thread.start()
  373. try:
  374. proc.wait(timeout=timeout)
  375. except subprocess.TimeoutExpired:
  376. proc.kill()
  377. proc.wait()
  378. raise subprocess.TimeoutExpired(cmd, timeout)
  379. for thread in threads:
  380. thread.join()
  381. stdout, stderr = (b"".join(sink).decode("utf-8", "replace") for sink in sinks)
  382. returncode = proc.returncode
  383. if truncated.is_set() and returncode == 0:
  384. returncode = _OUTPUT_CAP_EXCEEDED_RETURNCODE
  385. return subprocess.CompletedProcess(cmd, returncode, stdout, stderr)
  386. def _expand_git_reference(ref: ContextReference, cwd: Path, args: list[str], label: str) -> Expansion:
  387. try:
  388. # Repo-supplied config/attributes must never execute code (GHSA-7x36-8jrh-v4pw).
  389. result = _run_quiet(["git", *harden_git_argv(args)], cwd, 30, env=noninteractive_git_env())
  390. except subprocess.TimeoutExpired:
  391. return f"{ref.raw}: git command timed out (30s)", None
  392. if result.returncode != 0:
  393. return f"{ref.raw}: {(result.stderr or '').strip() or 'git command failed'}", None
  394. content = result.stdout.strip() or "(no output)"
  395. return None, f"🧾 {label} ({estimate_tokens_rough(content)} tokens)\n```diff\n{content}\n```"
  396. async def _fetch_url_content(url: str, *, url_fetcher: UrlFetcher = None) -> str:
  397. content = (url_fetcher or _default_url_fetcher)(url)
  398. if inspect.isawaitable(content):
  399. content = await content
  400. return str(content or "").strip()
  401. async def _default_url_fetcher(url: str) -> str:
  402. from tools.web_tools import web_extract_tool
  403. docs = json.loads(await web_extract_tool([url], format="markdown")).get("results", [])
  404. return str(docs[0].get("content") or docs[0].get("raw_content") or "").strip() if docs else ""
  405. def _is_under(path: Path, root: Path) -> bool:
  406. try:
  407. path.relative_to(root)
  408. except ValueError:
  409. return False
  410. return True
  411. # Desktop persists a large plain-text paste as a `.txt` under this Hermes-managed
  412. # directory (apps/desktop/electron/composer-paste.ts) and attaches it as `@file:`.
  413. # The chat's cwd is rarely an ancestor of it, so it is the one anchored root the
  414. # workspace guard admits besides `allowed_root` itself.
  415. COMPOSER_PASTES_DIRNAME = "composer-pastes"
  416. def _composer_paste_roots() -> list[Path]:
  417. from agent.file_safety import _hermes_dirs
  418. return [hermes_dir / COMPOSER_PASTES_DIRNAME for hermes_dir in _hermes_dirs()]
  419. def _agent_staged_path(path: Path) -> bool:
  420. """True when *path* sits in a Hermes dir the gateway stages for the agent.
  421. Those are the ``_CACHE_DIRS`` roots — ``attachments/`` (file drops),
  422. ``images/`` (image uploads), ``cache/*`` (platform downloads) and now
  423. ``composer-pastes/`` (large text pastes) — the gateway's OWN payload, never
  424. a workspace escape, and they live outside the workspace by construction: on
  425. a remote execution backend (ssh and friends) the workspace root is a path
  426. on THAT host (#110174), so the workspace check below rejected every staged
  427. attachment there. The bytes are staged to (or already live on) the gateway
  428. either way, so the ref still expands — text inlines; binaries point at the
  429. backend-visible path via ``to_agent_visible_cache_path``.
  430. """
  431. try:
  432. from tools.credential_files import get_cache_directory_mounts
  433. return any(_is_under(path, Path(entry["host_path"]).expanduser().resolve())
  434. for entry in get_cache_directory_mounts())
  435. except Exception:
  436. return False
  437. def _resolve_path(cwd: Path, target: str, *, allowed_root: Path | None = None) -> Path:
  438. from agent.file_safety import is_nt_namespace_path
  439. if is_nt_namespace_path(target): # raw-string check: resolving such a path is the NTLM-leak trigger
  440. raise ValueError("path uses a Windows NT/device namespace prefix and cannot be attached")
  441. resolved = (cwd / Path(os.path.expanduser(target))).resolve() # `/` keeps an absolute target as-is
  442. if (
  443. allowed_root is not None
  444. and not _is_under(resolved, allowed_root)
  445. and not any(_is_under(resolved, root) for root in _composer_paste_roots())
  446. and not _agent_staged_path(resolved)
  447. ):
  448. raise ValueError("path is outside the allowed workspace")
  449. return resolved
  450. def _ensure_reference_path_allowed(path: Path) -> None:
  451. """Refuse credential/internal paths. Fails CLOSED: the gateway feeds untrusted remote text here."""
  452. from hermes_constants import get_hermes_home
  453. home, hermes_home = Path(os.path.expanduser("~")).resolve(), get_hermes_home().resolve()
  454. blocked_exact = {home / rel for rel in _SENSITIVE_HOME_FILES} | {hermes_home / ".env"}
  455. blocked_dirs = [home / rel for rel in HOME_CREDENTIAL_DIRS] + [hermes_home / rel for rel in _SENSITIVE_HERMES_DIRS]
  456. if path in blocked_exact:
  457. raise ValueError("path is a sensitive credential file and cannot be attached")
  458. if any(_is_under(path, blocked_dir) for blocked_dir in blocked_dirs):
  459. raise ValueError("path is a sensitive credential or internal Hermes path and cannot be attached")
  460. # Anchor to the canonical read deny-list (agent/file_safety.get_read_block_error): the
  461. # narrow list above never caught auth.json, .anthropic_oauth.json, mcp-tokens/, webhook
  462. # secrets or project .env files, and it grows automatically with that deny-list.
  463. try:
  464. from agent.file_safety import get_read_block_error
  465. blocked = get_read_block_error(str(path)) is not None
  466. except ValueError:
  467. raise
  468. except Exception:
  469. # If the canonical lookup fails, falling through would re-open the exact hole this
  470. # guard closes; a spurious block is recoverable, a leaked credential is not.
  471. raise ValueError("path could not be verified against the credential deny-list and cannot be attached")
  472. if blocked:
  473. raise ValueError("path is a sensitive credential or internal Hermes path and cannot be attached")
  474. def _strip_trailing_punctuation(value: str) -> str:
  475. stripped = value.rstrip(TRAILING_PUNCTUATION)
  476. # Drop unbalanced closers so "(see @file:x.py)" does not swallow the ")".
  477. while stripped.endswith((")", "]", "}")) and stripped.count(stripped[-1]) > stripped.count(_OPENERS[stripped[-1]]):
  478. stripped = stripped[:-1]
  479. return stripped
  480. def _strip_reference_wrappers(value: str) -> str:
  481. return value[1:-1] if len(value) >= 2 and value[0] == value[-1] and value[0] in "`\"'" else value
  482. def _parse_file_reference_value(value: str) -> tuple[str, int | None, int | None]:
  483. m = _FILE_VALUE_PATTERN.match(value)
  484. start = m and m.group("start")
  485. if not start: # no line range: the whole value is the (possibly quoted) path
  486. return _strip_reference_wrappers(value), None, None
  487. return m.group("qpath") or m.group("path"), int(start), int(m.group("end") or start)
  488. def _is_binary_file(path: Path) -> bool:
  489. mime = mimetypes.guess_type(path.name)[0]
  490. if mime and not mime.startswith("text/") and not path.name.endswith(_TEXT_EXTENSIONS):
  491. return True
  492. with path.open("rb") as fh: # sniff only; read_bytes() materialized the whole file
  493. return b"\x00" in fh.read(4096)
  494. def _build_folder_listing(path: Path, cwd: Path, limit: int = 200, display_base: Path | None = None) -> str:
  495. # The target may sit outside cwd when the caller widened allowed_root: show it relative to
  496. # cwd when possible, else relative to the allowed root, else the absolute path.
  497. shown: str | None = None
  498. for base in (cwd, display_base):
  499. if base is None:
  500. continue
  501. try:
  502. shown = f"{path.relative_to(base)}/"
  503. break
  504. except ValueError:
  505. continue
  506. if shown is None:
  507. shown = f"{path}/"
  508. lines = [shown]
  509. entries = _iter_visible_entries(path, cwd, limit=limit)
  510. for entry in entries:
  511. indent = " " * max(len(entry.relative_to(path).parts) - 1, 0)
  512. lines.append(f"{indent}- {entry.name}/" if entry.is_dir() else f"{indent}- {entry.name} ({_file_metadata(entry)})")
  513. if len(entries) >= limit:
  514. lines.append("- ...")
  515. return "\n".join(lines)
  516. def _iter_visible_entries(path: Path, cwd: Path, limit: int) -> list[Path]:
  517. """Files under ``path`` via ``rg --files`` (honours ignore files), else an os.walk fallback."""
  518. try:
  519. # Absolute path arg: rg echoes it as the output prefix, so results stay correct even
  520. # when the folder is outside cwd (a widened allowed_root target).
  521. rg = _run_quiet(["rg", "--files", str(path)], cwd, 10)
  522. except (FileNotFoundError, OSError, subprocess.TimeoutExpired):
  523. rg = None
  524. if rg is not None and rg.returncode == 0:
  525. output: list[Path] = []
  526. seen_dirs: set[Path] = set()
  527. for line in [ln.strip() for ln in rg.stdout.splitlines() if ln.strip()][:limit]:
  528. full = cwd / Path(line) # absolute lines pass through unchanged; defensive for relative
  529. for parent in full.parents:
  530. if parent in seen_dirs or path not in {parent, *parent.parents}:
  531. continue
  532. seen_dirs.add(parent)
  533. output.append(parent)
  534. output.append(full)
  535. return sorted({p for p in output if p.exists()}, key=lambda p: (not p.is_dir(), str(p)))
  536. output = []
  537. for root, dirs, files in os.walk(path):
  538. dirs[:] = sorted(d for d in dirs if not d.startswith(".") and d != "__pycache__")
  539. files = sorted(f for f in files if not f.startswith("."))
  540. for name in dirs + files:
  541. output.append(Path(root) / name)
  542. if len(output) >= limit:
  543. return output
  544. return output
  545. def _agent_visible_path(path: Path) -> str:
  546. # Under a container backend the host path dangles inside the sandbox: translate staged
  547. # files to their auto-mounted cache path; fall back to the host path (local backend /
  548. # translation failure). Run the idempotent TERMINAL_ENV bridge first so in-process
  549. # gateways that never bridged terminal.* config still see the active backend.
  550. try:
  551. from tools.terminal_tool import _ensure_terminal_env_bridged
  552. _ensure_terminal_env_bridged()
  553. from tools.credential_files import to_agent_visible_cache_path
  554. return to_agent_visible_cache_path(str(path))
  555. except Exception:
  556. return str(path)
  557. def _on_disk_reference_block(ref: ContextReference, path: Path, descriptor: str, reason: str, guidance: str) -> str:
  558. """Shared 📎 shape: the file was not inlined, but it IS on disk where the agent's
  559. tools run — hand the model the path and a nudge instead of a dead-end warning."""
  560. try:
  561. size = format_bytes(path.stat().st_size)
  562. except OSError:
  563. size = "unknown size"
  564. return (
  565. f"📎 {ref.raw} ({descriptor}, {size}) — {reason} "
  566. f"It is available on disk at `{_agent_visible_path(path)}`. {guidance}"
  567. )
  568. def _binary_reference_block(ref: ContextReference, path: Path) -> str:
  569. mime = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
  570. return _on_disk_reference_block(
  571. ref, path,
  572. descriptor=mime,
  573. reason="binary file, not inlined as text.",
  574. guidance="Use your tools to work with it (read or convert it, extract its text, "
  575. "or view/render it as needed); do not tell the user the file type is unsupported.",
  576. )
  577. def _oversized_text_reference_block(ref: ContextReference, path: Path, text_tokens: int) -> str:
  578. return _on_disk_reference_block(
  579. ref, path,
  580. descriptor=f"text file, approximately {text_tokens} tokens",
  581. reason="too large to inline safely.",
  582. guidance="Use read_file with a narrow line range, search_files, or terminal/code tools "
  583. "to inspect only the relevant parts; do not load the entire file into context.",
  584. )
  585. def _file_metadata(path: Path) -> str:
  586. try:
  587. size = path.stat().st_size
  588. except OSError:
  589. return "unknown size"
  590. try:
  591. # A directory preview inspects each entry separately; the registry lock
  592. # must cover both its binary sniff and optional line-count read.
  593. with offline_file_access(path, what="inspect folder entry"):
  594. # A listing line is a summary, not content: past the cap, byte size conveys
  595. # the same "how big is this" without a full scan per entry.
  596. if _is_binary_file(path) or size > _LINE_COUNT_MAX_BYTES:
  597. return f"{size} bytes"
  598. with path.open("rb") as fh:
  599. # UTF-8 never embeds 0x0A inside a multibyte sequence, so counting bytes
  600. # matches a decoded newline while streaming instead of read_text.
  601. lines = sum(chunk.count(b"\n") for chunk in iter(lambda: fh.read(1 << 20), b""))
  602. return f"{lines + 1} lines"
  603. except (LiveConnectionError, OSError):
  604. return f"{size} bytes"