shell_heredoc.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. """Conservative heredoc masking for shell-command scanners ('&' guard, blocked-command checks,
  2. cron lifecycle_guard) that false-positive on heredoc *bodies*. Stripping every body is unsafe the
  3. other way (a fake ``<<`` in quotes can swallow an operator; unquoted bodies expand; ``bash <<'EOF'``
  4. executes), so a body is masked ONLY when every delimiter is quoted, every heredoc has an exact
  5. terminator line, the owning simple command is an allowlisted non-shell interpreter, and no list
  6. operator follows the heredoc. Otherwise the command is returned untouched: a false positive is
  7. acceptable, hiding shell syntax from a guard is not.
  8. Masked bodies keep their newline count (re.MULTILINE)."""
  9. from __future__ import annotations
  10. import re
  11. # Non-shell interpreters whose quoted heredoc bodies are data for THAT interpreter; optional
  12. # VAR=... assignments, ``env`` and a path prefix allowed. Narrow on purpose: unmatched = visible.
  13. _INERT_HEREDOC_CONSUMER_RE = re.compile(
  14. r"^\s*(?:[A-Z_][A-Z0-9_]*=\S+\s+)*(?:env\s+)?(?:[A-Za-z0-9_./-]+/)?"
  15. r"(?:python(?:3(?:\.\d+)*)?|osascript|cat)(?=\s|$)",
  16. re.IGNORECASE)
  17. def _span_end(command: str, cursor: int, closer: str) -> int:
  18. """Index just past the backslash-aware span opened at ``cursor``."""
  19. end = cursor + 1
  20. while end < len(command):
  21. if command[end] == closer:
  22. return end + 1
  23. end += 2 if command[end] == "\\" and end + 1 < len(command) else 1
  24. return end
  25. def _mask_simple_quotes(command: str) -> str:
  26. """Blank inert quoted spans; keep ``$(``/backtick-bearing ones visible."""
  27. result = []
  28. cursor = 0
  29. while cursor < len(command):
  30. char = command[cursor]
  31. if char in "'\"": # single quotes have no escapes; double quotes are backslash-aware
  32. end = (command.find("'", cursor + 1) + 1 if char == "'"
  33. else _span_end(command, cursor, '"'))
  34. segment = command[cursor:end]
  35. if not segment.endswith(char):
  36. result.append(command[cursor:])
  37. break
  38. keep = char == '"' and ("$(" in segment or "`" in segment)
  39. result.append(segment if keep else char * 2)
  40. cursor = end
  41. elif char == "`":
  42. end = _span_end(command, cursor, "`")
  43. result.append(command[cursor:end])
  44. cursor = end
  45. else:
  46. result.append(char)
  47. cursor += 1
  48. return "".join(result)
  49. def _parse_heredoc_operator(command: str, index: int):
  50. """Parse one ``<<`` opener -> ``(end_index, delimiter, strip_tabs, quoted)`` or None."""
  51. if not command.startswith("<<", index) or command.startswith("<<<", index):
  52. return None
  53. strip_tabs = command.startswith("-", index + 2)
  54. cursor = index + 3 if strip_tabs else index + 2
  55. while cursor < len(command) and command[cursor] in " \t":
  56. cursor += 1
  57. if cursor >= len(command) or command[cursor] in "\r\n":
  58. return None
  59. delimiter: list[str] = []
  60. quoted = False
  61. while cursor < len(command) and not (command[cursor].isspace() or command[cursor] in ";&|<>()"):
  62. char = command[cursor]
  63. if char == "\\": # backslash-escaped char: quoted, literal
  64. if cursor + 1 >= len(command) or command[cursor + 1] in "\r\n":
  65. return None
  66. quoted = True
  67. delimiter.append(command[cursor + 1])
  68. cursor += 2
  69. elif char in "'\"":
  70. quoted = True
  71. cursor += 1
  72. while cursor < len(command) and command[cursor] != char:
  73. current = command[cursor]
  74. if current in "\r\n":
  75. return None
  76. if char == '"' and current == "\\":
  77. if cursor + 1 >= len(command):
  78. return None
  79. if command[cursor + 1] in '$`"\\\n': # else backslash is literal in dquotes
  80. cursor += 1
  81. current = command[cursor]
  82. delimiter.append(current)
  83. cursor += 1
  84. if cursor >= len(command): # unterminated quote
  85. return None
  86. cursor += 1
  87. else:
  88. delimiter.append(char)
  89. cursor += 1
  90. if not delimiter and not quoted:
  91. return None
  92. return cursor, "".join(delimiter), strip_tabs, quoted
  93. def _is_fd_redirect_ampersand(command: str, index: int) -> bool:
  94. """Return whether ``&`` at ``index`` belongs to ``>&``/``<&``/``&>`` redirection."""
  95. before = command[index - 1] if index else ""
  96. after = command[index + 1] if index + 1 < len(command) else ""
  97. return before in "<>" or after == ">"
  98. def _scan_heredoc_command_unit(command: str, start: int):
  99. """Scan one logical command.
  100. Return ``(end, specs, unknown_operator, post_heredoc_list_operator, owner_start)``.
  101. List operators before the first heredoc select the simple command that owns it. A list
  102. operator after a heredoc keeps the body visible because another command may consume it.
  103. """
  104. cursor = start
  105. quote = None
  106. comment = False
  107. specs = []
  108. unknown_operator = False
  109. post_heredoc_list_operator = False
  110. owner_start = start
  111. while cursor < len(command):
  112. char = command[cursor]
  113. if char == "\n" and (comment or quote is None):
  114. break
  115. # Backslash escapes (incl. line continuations) outside single quotes skip the next char.
  116. escaped = char == "\\" and quote != "'" and not comment and cursor + 1 < len(command)
  117. if comment or quote is not None or escaped:
  118. if char == quote:
  119. quote = None
  120. cursor += 2 if escaped else 1
  121. elif char in "'\"`":
  122. quote = char
  123. cursor += 1
  124. elif char == "#" and (cursor == start or command[cursor - 1].isspace()
  125. or command[cursor - 1] in ";&|()"):
  126. comment = True
  127. cursor += 1
  128. elif command.startswith("<<<", cursor):
  129. cursor += 3
  130. elif command.startswith("<<", cursor):
  131. parsed = _parse_heredoc_operator(command, cursor)
  132. if parsed is None:
  133. unknown_operator = True
  134. cursor += 2
  135. else:
  136. cursor, delimiter, strip_tabs, quoted = parsed
  137. specs.append((delimiter, strip_tabs, quoted))
  138. else:
  139. if char in ";|&" and not (
  140. char == "&" and _is_fd_redirect_ampersand(command, cursor)
  141. ):
  142. if specs:
  143. post_heredoc_list_operator = True
  144. else:
  145. owner_start = cursor + 1
  146. cursor += 1
  147. return cursor, specs, unknown_operator, post_heredoc_list_operator, owner_start
  148. def _find_heredoc_close(
  149. command: str, body_start: int, delimiter: str, strip_tabs: bool) -> int | None:
  150. """Return the position after an exact shell heredoc terminator line."""
  151. cursor = body_start
  152. while True:
  153. newline = command.find("\n", cursor)
  154. after = len(command) if newline == -1 else newline + 1
  155. line = command[cursor:after].removesuffix("\n").removesuffix("\r")
  156. candidate = line.lstrip("\t") if strip_tabs else line
  157. if candidate == delimiter:
  158. return after
  159. if newline == -1:
  160. return None
  161. cursor = after
  162. def strip_inert_heredoc_bodies(command: str) -> str:
  163. """Mask heredoc bodies that are provably inert data (see module docstring)."""
  164. # Runs on every terminal call: skip the state machine when no '<<' exists; stop past the last.
  165. if "<<" not in command:
  166. return command
  167. last_opener_index = command.rfind("<<")
  168. ranges: list[tuple[int, int]] = []
  169. command_start = 0
  170. while command_start <= last_opener_index:
  171. (
  172. command_end,
  173. specs,
  174. unknown_operator,
  175. post_heredoc_list_operator,
  176. owner_start,
  177. ) = _scan_heredoc_command_unit(command, command_start)
  178. if unknown_operator:
  179. return command
  180. if not specs:
  181. if command_end >= len(command):
  182. break
  183. command_start = command_end + 1
  184. continue
  185. if command_end >= len(command):
  186. return command # opener with no body line: unterminated — leave visible
  187. body_cursor = command_end + 1
  188. body_ranges: list[tuple[int, int]] = []
  189. for delimiter, strip_tabs, _quoted in specs:
  190. close_end = _find_heredoc_close(command, body_cursor, delimiter, strip_tabs)
  191. if close_end is None:
  192. return command # unterminated
  193. body_ranges.append((body_cursor, close_end))
  194. body_cursor = close_end
  195. if (
  196. all(quoted for _delimiter, _strip_tabs, quoted in specs)
  197. and not post_heredoc_list_operator
  198. ):
  199. masked_opener = _mask_simple_quotes(command[command_start:command_end])
  200. masked_owner = _mask_simple_quotes(command[owner_start:command_end])
  201. if not any(
  202. marker in masked_opener
  203. for marker in ("$(", "`", "<(", ">(", "(", ")", "{", "}")
  204. ) and _INERT_HEREDOC_CONSUMER_RE.search(masked_owner):
  205. ranges.extend(body_ranges)
  206. command_start = body_cursor
  207. # Single-pass rebuild (ranges are sorted and non-overlapping), bodies -> their newlines only.
  208. parts: list[str] = []
  209. previous = 0
  210. for start, end in ranges:
  211. parts += [command[previous:start], "\n" * command.count("\n", start, end)]
  212. previous = end
  213. return "".join(parts) + command[previous:]