file_operations_lint.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417
  1. """Syntax-lint and LSP-diagnostics tier for ``tools.file_operations``.
  2. ``ShellFileOperations`` inherits ``LintMixin``; module constants and in-process
  3. linters are pure functions importable from this module.
  4. """
  5. import ast
  6. import json
  7. import logging
  8. import os
  9. import tomllib
  10. from typing import Callable, Dict, Optional
  11. from tools.file_operations_common import ExecuteResult, LintResult
  12. logger = logging.getLogger(__name__)
  13. # Shell linters by extension (external toolchain). ``.tsx`` is deliberately absent:
  14. # it hits the "No linter" skip and LSP covers it when enabled.
  15. LINTERS = {
  16. '.py': 'python -m py_compile {file} 2>&1',
  17. '.js': 'node --check {file} 2>&1',
  18. '.ts': 'npx tsc --noEmit {file} 2>&1',
  19. '.go': 'go vet {file} 2>&1',
  20. '.rs': 'rustfmt --check {file} 2>&1',
  21. }
  22. # Node linters Hermes runs on the host (local backend) under its PM-managed Node,
  23. # never the user's: the terminal PATH puts the user's dirs first, and ``npx tsc``
  24. # re-execs ``env node`` through that PATH.
  25. _MANAGED_NODE_LINTERS = {'.js': ('node', '--check'), '.ts': ('npx', 'tsc', '--noEmit')}
  26. # Per-file shell linters that flood phantom errors on real projects (single-file
  27. # ``tsc`` ignores tsconfig, ``go vet`` fails outside a module, ``rustfmt --check``
  28. # is style-only): skipped when an LSP server claims the file. py_compile /
  29. # node --check are file-local and correct so always run.
  30. _SHELL_LINTER_LSP_REDUNDANT = frozenset({'.ts', '.go', '.rs'})
  31. # Shell linters that resolve the toolchain from the checkout they run in: ``npx tsc`` runs the
  32. # repo's ``node_modules/.bin/tsc`` (or installs from the registry its ``.npmrc`` names) and
  33. # rustup honours its ``rust-toolchain.toml``. Skipped on a local backend in an untrusted workspace.
  34. _SHELL_LINTER_RUNS_PROJECT_TOOLCHAIN = frozenset({'.ts', '.rs'})
  35. # Output substrings (case-insensitive) meaning the linter binary exists but could
  36. # not run → ``skipped`` so the write isn't flagged and the LSP tier still runs.
  37. _LINTER_UNUSABLE_PATTERNS = {
  38. 'npx': (
  39. 'this is not the tsc command you are looking for', # tsc not installed locally
  40. 'could not determine executable to run',
  41. 'not found in npm registry',
  42. ),
  43. 'rustfmt': (
  44. 'no input filename given', # outside a Cargo project
  45. 'error: not a workspace',
  46. ),
  47. 'go': (
  48. 'cannot find package', # outside a module / GOPATH
  49. 'go: cannot find main module',
  50. ),
  51. }
  52. def _looks_like_linter_unusable(base_cmd: str, output: str) -> bool:
  53. """True iff ``output`` from ``base_cmd`` (first word of the linter cmd) says the tool itself couldn't run."""
  54. patterns = _LINTER_UNUSABLE_PATTERNS.get(base_cmd)
  55. if not patterns:
  56. return False
  57. lower = output.lower()
  58. return any(p in lower for p in patterns)
  59. def _lint_json_inproc(content: str) -> tuple[bool, str]:
  60. """In-process JSON syntax check. Returns (ok, error_message)."""
  61. try:
  62. json.loads(content)
  63. return True, ""
  64. except json.JSONDecodeError as e:
  65. return False, f"JSONDecodeError: {e.msg} (line {e.lineno}, column {e.colno})"
  66. except Exception as e:
  67. return False, f"{type(e).__name__}: {e}"
  68. def _lint_yaml_inproc(content: str) -> tuple[bool, str]:
  69. """In-process YAML syntax check; ``__SKIP__`` when ruamel.yaml is missing. Syntax-only
  70. (``YAML.parse``), NOT ``safe_load``: loading rejects valid multi-doc streams and
  71. app tags (``!Sub``, ``!vault``), and this is a fail-closed WRITE gate."""
  72. try:
  73. from ruamel.yaml import YAML
  74. from ruamel.yaml.error import YAMLError
  75. except ImportError:
  76. return True, "__SKIP__"
  77. try:
  78. for _event in YAML(typ="safe").parse(content):
  79. pass
  80. return True, ""
  81. except YAMLError as e:
  82. return False, f"YAMLError: {e}"
  83. except Exception as e:
  84. return False, f"{type(e).__name__}: {e}"
  85. def _lint_toml_inproc(content: str) -> tuple[bool, str]:
  86. """In-process TOML syntax check (stdlib tomllib)."""
  87. try:
  88. tomllib.loads(content)
  89. return True, ""
  90. except Exception as e: # TOMLDecodeError is a ValueError subclass
  91. return False, f"{type(e).__name__}: {e}"
  92. def _lint_python_inproc(content: str) -> tuple[bool, str]:
  93. """In-process Python syntax check via ast.parse (py_compile's scope, no subprocess). Parses the
  94. bytes, as py_compile does, so a legacy ``coding:`` cookie is honoured for a file the edit paths
  95. read with surrogateescape."""
  96. try:
  97. ast.parse(content.encode("utf-8", "surrogateescape"))
  98. return True, ""
  99. except SyntaxError as e:
  100. loc = f" (line {e.lineno}, column {e.offset})" if e.lineno else ""
  101. return False, f"{type(e).__name__}: {e.msg}{loc}"
  102. except Exception as e:
  103. return False, f"{type(e).__name__}: {e}"
  104. # In-process linters, preferred over shell linters (no subprocess). Each returns
  105. # (ok, error); error ``"__SKIP__"`` = unavailable dependency, counts as "no linter".
  106. LINTERS_INPROC: dict[str, Callable[[str], tuple[bool, str]]] = {
  107. '.py': _lint_python_inproc,
  108. '.json': _lint_json_inproc,
  109. '.yaml': _lint_yaml_inproc,
  110. '.yml': _lint_yaml_inproc,
  111. '.toml': _lint_toml_inproc,
  112. }
  113. # Extensions where write_file REFUSES on a parse failure. ``.py`` is excluded on
  114. # purpose: test fixtures use ``*.py`` paths as a stand-in for arbitrary text, so
  115. # Python keeps the non-blocking lint-delta report.
  116. _FAIL_CLOSED_INPROC_EXTS = frozenset({'.json', '.yaml', '.yml', '.toml'})
  117. class LintMixin:
  118. """Post-write syntax lint + LSP diagnostics. Requires ``_exec``,
  119. ``_has_command``, ``_escape_shell_arg``, ``_escape_native_tool_arg`` and
  120. ``env`` from the host class."""
  121. def _check_lint(self, path: str, content: Optional[str] = None) -> LintResult:
  122. """Syntax-check ``path``: in-process linter when one matches the
  123. extension (``content`` avoids a re-read), else the shell linter table."""
  124. ext = os.path.splitext(path)[1].lower()
  125. inproc = LINTERS_INPROC.get(ext)
  126. if inproc is not None:
  127. if content is None:
  128. read_result = self._exec(f"cat {self._escape_shell_arg(path)} 2>/dev/null")
  129. if read_result.exit_code != 0:
  130. return LintResult(skipped=True, message=f"Failed to read {path} for lint")
  131. content = read_result.stdout
  132. ok, err = inproc(content)
  133. if err == "__SKIP__":
  134. return LintResult(skipped=True, message=f"No linter available for {ext} (missing dependency)")
  135. return LintResult(success=ok, output="" if ok else err)
  136. if ext not in LINTERS:
  137. return LintResult(skipped=True, message=f"No linter for {ext} files")
  138. # Single-file tsc can't read tsconfig.json and floods phantom TS2307/TS2339
  139. # errors the delta filter misreports as "pre-existing"; let the LSP tier speak.
  140. if ext == '.ts' and self._has_ancestor_tsconfig(path):
  141. return LintResult(skipped=True, message=(
  142. "Project tsconfig.json detected — per-file tsc skipped "
  143. "(single-file tsc can't resolve project aliases/globals; "
  144. "use the LSP tier or `tsc -p tsconfig.json` for real "
  145. "diagnostics)."
  146. ))
  147. if ext in _SHELL_LINTER_LSP_REDUNDANT and self._lsp_will_handle(path):
  148. return LintResult(skipped=True, message=f"LSP server handles {ext} — shell linter skipped")
  149. if ext in _SHELL_LINTER_RUNS_PROJECT_TOOLCHAIN and self._local_workspace_untrusted():
  150. return LintResult(skipped=True, message=(
  151. f"{LINTERS[ext].split()[0]} skipped: untrusted workspace (add it to lsp.trusted_workspaces to lint {ext} here)"))
  152. if ext in _MANAGED_NODE_LINTERS and self._lsp_local_only():
  153. base_cmd = _MANAGED_NODE_LINTERS[ext][0]
  154. result = self._run_managed_node_linter(ext, path)
  155. if result is None:
  156. return LintResult(skipped=True, message=f"{base_cmd} not available (Hermes-managed Node not installed)")
  157. else:
  158. linter_cmd = LINTERS[ext]
  159. base_cmd = linter_cmd.split()[0]
  160. if not self._has_command(base_cmd):
  161. return LintResult(skipped=True, message=f"{base_cmd} not available")
  162. # Native Windows binaries need C:/... not MSYS /c/... (→ phantom ENOENT).
  163. result = self._exec(linter_cmd.replace("{file}", self._escape_native_tool_arg(path)), timeout=30)
  164. if result.exit_code != 0 and _looks_like_linter_unusable(base_cmd, result.stdout):
  165. from tools.ansi_strip import strip_ansi
  166. cleaned = strip_ansi(result.stdout).strip()
  167. # Collapse to one line — the npx banner is multi-line ASCII art.
  168. first_line = next((ln.strip() for ln in cleaned.splitlines() if ln.strip()), cleaned[:120])
  169. return LintResult(skipped=True, message=f"{base_cmd} not usable: {first_line[:200]}")
  170. return LintResult(success=result.exit_code == 0, output=result.stdout.strip())
  171. def _run_managed_node_linter(self, ext: str, path: str) -> Optional[ExecuteResult]:
  172. """Run the ``ext`` Node linter on the host under PM's Node; None when PM has none."""
  173. import shutil
  174. import subprocess
  175. from hermes_cli._subprocess_compat import windows_hide_flags
  176. from hermes_constants import with_hermes_node_path
  177. from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path, hermes_subprocess_env
  178. tool, *args = _MANAGED_NODE_LINTERS[ext]
  179. executable = shutil.which(tool, path=with_hermes_node_path({"PATH": ""})["PATH"])
  180. if executable is None:
  181. return None
  182. cwd = getattr(self.env, "cwd", None) or self.cwd
  183. if _IS_WINDOWS:
  184. path, cwd = _msys_to_windows_path(path), cwd and _msys_to_windows_path(cwd)
  185. try:
  186. proc = subprocess.run(
  187. [executable, *args, path], cwd=cwd or None, env=with_hermes_node_path(hermes_subprocess_env()),
  188. stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
  189. text=True, encoding="utf-8", errors="replace", timeout=30, creationflags=windows_hide_flags(), check=False)
  190. except subprocess.TimeoutExpired:
  191. return ExecuteResult(stdout=f"{tool} timed out after 30s", exit_code=124)
  192. return ExecuteResult(stdout=proc.stdout or "", exit_code=proc.returncode)
  193. def _check_lint_delta(self, path: str, pre_content: Optional[str],
  194. post_content: Optional[str] = None) -> LintResult:
  195. """Post-write lint; when it fails and ``pre_content`` is known, report only
  196. errors this edit introduced (pre-existing lines filtered out). Semantic
  197. (LSP) diagnostics are a separate channel — see ``_maybe_lsp_diagnostics``."""
  198. post = self._check_lint(path, content=post_content)
  199. if post.success or post.skipped or pre_content is None:
  200. return post
  201. pre = self._check_lint(path, content=pre_content)
  202. if pre.success or pre.skipped or not pre.output:
  203. return post # pre-write was clean (or unlintable): all post errors are new
  204. # Single-error parsers stop at the first error, so if every post error already
  205. # existed we can't prove the edit is clean — say nothing new was introduced.
  206. pre_lines = {ln.strip() for ln in pre.output.splitlines() if ln.strip()}
  207. post_lines = [ln for ln in post.output.splitlines() if ln.strip() and ln.strip() not in pre_lines]
  208. if not post_lines:
  209. return LintResult(success=False, output=post.output, message=(
  210. "Pre-existing lint errors — this edit didn't introduce new ones but the file is still broken."))
  211. return LintResult(success=False, output=(
  212. "New lint errors introduced by this edit "
  213. "(pre-existing errors filtered out):\n" + "\n".join(post_lines)
  214. ))
  215. def _lsp_local_only(self) -> bool:
  216. """True iff the backend's files are the host's (``uses_host_filesystem``). LSP servers run on
  217. the host and can't see files inside Docker/Modal/SSH/Daytona or a confining local sandbox."""
  218. from tools.environments.base import env_uses_host_filesystem
  219. return env_uses_host_filesystem(getattr(self, "env", None)) # tests may build via __new__
  220. def _local_workspace_untrusted(self) -> bool:
  221. """True iff on a local backend the linter's cwd lies outside every trusted workspace
  222. (``workspace.is_trusted_workspace``): ``npx`` and rustup resolve the toolchain from there, not
  223. from the linted file's directory. Sandboxed backends answer False: their toolchain is not the host's."""
  224. if not self._lsp_local_only():
  225. return False
  226. from agent.lsp.manager import parse_trusted_workspaces
  227. from agent.lsp.workspace import is_trusted_workspace, operator_workspace_roots
  228. from hermes_cli.config import load_config_readonly
  229. lsp_cfg = load_config_readonly().get("lsp")
  230. trusted = parse_trusted_workspaces(lsp_cfg.get("trusted_workspaces") if isinstance(lsp_cfg, dict) else None)
  231. return not is_trusted_workspace(getattr(self.env, "cwd", None) or self.cwd, trusted, operator_workspace_roots())
  232. def _lsp_service(self):
  233. """The active LSPService, or None on a non-local backend / any failure.
  234. LSP is an enrichment layer and must never break a write."""
  235. if not self._lsp_local_only():
  236. return None
  237. try:
  238. from agent.lsp import get_service
  239. return get_service()
  240. except Exception:
  241. return None
  242. def _lsp_handles_extension(self, ext: str) -> bool:
  243. """True iff the active service (config-declared servers included) or, without one,
  244. the static registry claims ``ext``. Decides whether pre-write content is worth
  245. capturing for the line-shift map."""
  246. if not ext:
  247. return False
  248. try:
  249. svc = self._lsp_service()
  250. if svc is not None:
  251. return svc.handles_extension(ext)
  252. from agent.lsp.servers import SERVERS
  253. except Exception:
  254. return False
  255. return any(ext.lower() in srv.extensions for srv in SERVERS)
  256. def _has_ancestor_tsconfig(self, path: str) -> bool:
  257. """True iff a tsconfig.json exists in ``path``'s directory or any ancestor.
  258. Host-side walk, local backend only: on a remote backend this answers False so
  259. the shell linter still runs — never suppress lint on a probe that couldn't answer."""
  260. if not self._lsp_local_only():
  261. return False
  262. try:
  263. d = os.path.dirname(os.path.abspath(path))
  264. while not os.path.isfile(os.path.join(d, "tsconfig.json")):
  265. parent = os.path.dirname(d)
  266. if parent == d:
  267. return False
  268. d = parent
  269. return True
  270. except Exception:
  271. return False
  272. def _lsp_call(self, method: str, path: str, default):
  273. """``svc.<method>(path)`` on the active service; ``default`` when there is
  274. no service or the call raises (LSP never breaks a write)."""
  275. svc = self._lsp_service()
  276. if svc is None:
  277. return default
  278. try:
  279. return getattr(svc, method)(path)
  280. except Exception:
  281. return default
  282. def _lsp_will_handle(self, path: str) -> bool:
  283. """True iff the LSP service is active AND ``enabled_for(path)`` (workspace
  284. detection, disabled-server set, broken-pair short-circuit). Any failure →
  285. False so the shell linter still runs."""
  286. return bool(self._lsp_call("enabled_for", path, False))
  287. def _maybe_lsp_diagnostics(self, path: str, *, pre_content: Optional[str] = None,
  288. post_content: Optional[str] = None, started: Optional[float] = None) -> str:
  289. """Formatted LSP diagnostics this edit introduced, or "" when LSP is unavailable/disabled/clean
  290. OR the verdict is not ready yet. Never blocks the write on the language server: the check
  291. (``agent.lsp.edit_check``) diagnoses ``post_content`` (the bytes this write produced) against
  292. ``pre_content`` (None = new file) on the LSP loop: those snapshots, not a later read of the
  293. file, which another session may already have rewritten. The write waits for the verdict only until ``inline_deadline(started)`` (what is
  294. left of the write's latency budget; ``started`` = ``time.monotonic()`` at write entry); a slower
  295. verdict reaches the model with a later tool result of the same session (``agent.lsp.deferred``)."""
  296. svc = self._lsp_service()
  297. if svc is None or not self._lsp_will_handle(path):
  298. return ""
  299. try:
  300. from agent.lsp import deferred
  301. from agent.lsp.edit_check import inline_wait
  302. abs_path = os.path.abspath(path)
  303. owner = deferred.current_owner()
  304. # This session's undelivered verdict for the file: the new check takes it over (diffs
  305. # against its baseline) and hands it back in ``result.carry`` only if it gets no verdict.
  306. older = deferred.take(owner, abs_path)
  307. carry = (older,) if older is not None else ()
  308. future = svc.start_edit_check(path, pre_text=pre_content, post_text=post_content, owner=owner,
  309. carry=carry)
  310. except Exception:
  311. return ""
  312. if future is None:
  313. deferred.restore(owner, abs_path, carry)
  314. return ""
  315. import concurrent.futures
  316. try:
  317. result = future.result(timeout=inline_wait(started))
  318. except concurrent.futures.TimeoutError:
  319. if owner:
  320. token = deferred.track_pending(owner)
  321. future.add_done_callback(lambda f: _post_deferred(owner, abs_path, f, carry, token))
  322. return ""
  323. except Exception: # LSP never breaks a write
  324. logger.debug("LSP edit check failed for %s", path, exc_info=True)
  325. deferred.restore(owner, abs_path, carry)
  326. return ""
  327. if result.superseded:
  328. return ""
  329. if result.diagnostics is None:
  330. deferred.restore(owner, abs_path, result.carry)
  331. return ""
  332. return _format_lsp_block(path, result.diagnostics, "LSP diagnostics introduced by this edit:\n")
  333. def _format_lsp_block(path: str, diagnostics, header: str) -> str:
  334. if not diagnostics:
  335. return ""
  336. try:
  337. from agent.lsp.reporter import report_for_file, truncate
  338. block = report_for_file(path, diagnostics)
  339. return truncate(header + block) if block else ""
  340. except Exception: # LSP never breaks a write
  341. logger.debug("formatting LSP diagnostics for %s failed", path, exc_info=True)
  342. return ""
  343. def _post_deferred(owner: str, abs_path: str, future, carry, token) -> None:
  344. """Done-callback (LSP loop thread): queue a late verdict for the session that made the edit, then
  345. release a turn end waiting on it (``agent.lsp.deferred.settle_for_agent``)."""
  346. from agent.lsp import deferred
  347. try:
  348. _queue_late_verdict(owner, abs_path, future, carry)
  349. finally:
  350. deferred.settled(owner, token)
  351. def _queue_late_verdict(owner: str, abs_path: str, future, carry) -> None:
  352. """A superseded check posts nothing: the newer check owns its verdict AND the notes it carried.
  353. No verdict hands back only what the check still holds (``result.carry``), never the caller's copy."""
  354. from agent.lsp import deferred
  355. try:
  356. result = None if future.cancelled() else future.result()
  357. except Exception: # a failed check reports nothing; the older note goes back
  358. logger.debug("deferred LSP edit check failed for %s", abs_path, exc_info=True)
  359. deferred.restore(owner, abs_path, carry)
  360. return
  361. if result is None:
  362. deferred.restore(owner, abs_path, carry)
  363. return
  364. if result.superseded:
  365. return
  366. if result.diagnostics is None:
  367. deferred.restore(owner, abs_path, result.carry)
  368. return
  369. text = _format_lsp_block(abs_path, result.diagnostics, (
  370. f"LSP diagnostics introduced by your earlier edit of {abs_path} "
  371. "(the language server answered after that tool result returned):\n"))
  372. if text:
  373. deferred.post(owner, abs_path, text, result.baseline_text)