debug_redaction.py 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216
  1. """Fail-closed redaction for text and structures leaving debug/support flows."""
  2. from __future__ import annotations
  3. import re
  4. from agent.redact import (
  5. _SENSITIVE_QUERY_PARAMS,
  6. REDACTION_UNAVAILABLE,
  7. SECRET_HEADER_NAME_LIST,
  8. _canonical_url_param_name,
  9. redact_sensitive_text,
  10. )
  11. _REDACTED = "[REDACTED]"
  12. _EMAIL_ADDRESS_RE = re.compile(
  13. r"(?<![A-Za-z0-9._%+-])"
  14. r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
  15. r"(?![A-Za-z0-9._%+-])"
  16. )
  17. # Support uploads are a strict opt-in surface: agent.redact's query-param names
  18. # plus a few support-only extras, compared by compact name so snake_case,
  19. # kebab-case and camelCase match without widening ordinary tool-output redaction.
  20. _SENSITIVE_URL_PARAM_NAMES = frozenset(
  21. "".join(ch for ch in name if ch.isalnum()) for name in _SENSITIVE_QUERY_PARAMS
  22. ) | {"authtoken", "privatekey", "secretkey", "xamzsecuritytoken", "securitytoken"}
  23. _RAW_URL_PARAM_RE = re.compile(
  24. r"(?P<sep>[?#&;])(?P<key>[A-Za-z0-9_.~+%\-]+)="
  25. r"(?P<value>[^?#&;\s\"'<>]*)"
  26. )
  27. # ``%25`` prefixes the double-encoded forms (``%253F``/``%2526``/``%253D``) that a
  28. # URL nested inside an already-encoded redirect value carries.
  29. _ENCODED_URL_SEP = r"%(?:25)?(?:3f|23|26|3b)"
  30. _ENCODED_URL_PARAM_RE = re.compile(
  31. rf"(?P<sep>{_ENCODED_URL_SEP})"
  32. rf"(?P<key>(?:(?!%(?:25)?3d|{_ENCODED_URL_SEP})[^\s\"'<>]){{1,160}}?)"
  33. r"(?P<eq>%(?:25)?3d|=)"
  34. r"(?P<value>.*?)"
  35. rf"(?={_ENCODED_URL_SEP}|[\s\"'<>]|$)",
  36. re.IGNORECASE,
  37. )
  38. _SENSITIVE_HEADER_NAMES = (
  39. "authorization",
  40. "proxy-authorization",
  41. "cookie",
  42. "set-cookie",
  43. *SECRET_HEADER_NAME_LIST,
  44. )
  45. _HEADER_NAME_PATTERN = "|".join(re.escape(name) for name in _SENSITIVE_HEADER_NAMES)
  46. _QUOTED_HEADER_RE = re.compile(
  47. rf"(?P<prefix>['\"]?(?:{_HEADER_NAME_PATTERN})['\"]?\s*[:=]\s*)"
  48. r"(?P<quote>['\"])(?P<value>.*?)(?P=quote)",
  49. re.IGNORECASE,
  50. )
  51. # A key may carry inner colons (``sk-abc:xyz``, ``user:pass``); a trailing colon
  52. # is prose punctuation (``see: docs``). A letters-only word (``docs``, ``Note``,
  53. # ``HTTP``) is prose rather than a key: keys mix case, digits or symbols. The
  54. # word test is case-sensitive even inside the IGNORECASE regexes below. A
  55. # single-case run of 20+ letters is no English word, so it is a key.
  56. _KEY_CHARS = r"A-Za-z0-9._~+/\-"
  57. # One key-end boundary shared by "is it a key" and "is it a word" so they cannot drift.
  58. _KEY_END = rf"(?![{_KEY_CHARS}=]|:[{_KEY_CHARS}])"
  59. _KEY = rf"[{_KEY_CHARS}]+(?::[{_KEY_CHARS}]+)*=*{_KEY_END}"
  60. _NOT_WORD = rf"(?!(?-i:[A-Z]?[a-z]{{1,19}}|[A-Z]{{1,19}}){_KEY_END})"
  61. # agent.redact masks the scheme of ``x-api-key: Basic <key>`` to ``***`` and
  62. # keeps the key, so ``***`` is a scheme too and the key is the value when it
  63. # is 8+ key chars (any case, letters-only included: fail closed), key-shaped
  64. # (not a word; 4+ chars before any colon, or 6+ mixing letters and digits) or
  65. # a short word ending the header (``Basic short``). Shorter prose after a
  66. # masked value stays.
  67. _SCHEME = (
  68. r"(?:Bearer|Basic|Digest)\s+|\*\*\*[ \t]+"
  69. rf"(?=[{_KEY_CHARS}]{{8,}}=*(?![{_KEY_CHARS}=])|{_NOT_WORD}(?=[^\s:]{{4}}|(?=\S*[A-Za-z])(?=\S*[0-9])\S{{6}}){_KEY}|[A-Za-z]{{4,7}}[ \t]*(?:[\r\n'\",;}}\]]|$))"
  70. rf"(?!(?:{_HEADER_NAME_PATTERN})\b\s*[:=])"
  71. )
  72. _BARE_HEADER_RE = re.compile(
  73. rf"(?P<prefix>\b(?:{_HEADER_NAME_PATTERN})\b\s*[:=]\s*)"
  74. rf"(?P<value>(?>(?:{_SCHEME})?)(?!\[REDACTED\])[^\s,}}\]]+)",
  75. re.IGNORECASE,
  76. )
  77. _SECRET_ARG_FLAG = (
  78. r"--(?:api[-_]?key|access[-_]?token|refresh[-_]?token|id[-_]?token|"
  79. r"auth[-_]?token|client[-_]?secret|private[-_]?key|secret[-_]?key|"
  80. r"password|passwd|credential|token|secret)"
  81. )
  82. _QUOTED_ARGV_RE = re.compile(
  83. rf"(?P<flag_quote>['\"])(?P<flag>{_SECRET_ARG_FLAG})(?P=flag_quote)"
  84. r"(?P<sep>\s*,\s*)"
  85. r"(?P<value_quote>['\"])(?P<value>.*?)(?P=value_quote)",
  86. re.IGNORECASE,
  87. )
  88. _ARG_EQUALS_RE = re.compile(
  89. rf"(?P<flag>{_SECRET_ARG_FLAG})(?P<sep>\s*=\s*)"
  90. r"(?P<quote>['\"]?)(?!\[REDACTED\])(?P<value>[^\s,}\]'\"]+)(?P=quote)",
  91. re.IGNORECASE,
  92. )
  93. _ARG_OPERAND_RE = re.compile(
  94. rf"(?P<flag>{_SECRET_ARG_FLAG})(?P<sep>\s+)"
  95. r"(?P<quote>['\"]?)(?!\[REDACTED\])(?P<value>[^\s,}\]'\"]+)(?P=quote)",
  96. re.IGNORECASE,
  97. )
  98. _HEADER_ARGV_RE = re.compile(
  99. r"(?P<head_quote>['\"])--header(?P=head_quote)\s*,\s*"
  100. rf"(?P<name_quote>['\"])(?P<name>{_HEADER_NAME_PATTERN})(?P=name_quote)\s*,\s*"
  101. r"(?:(?P<scheme_quote>['\"])(?P<scheme>Bearer|Basic|Digest)(?P=scheme_quote)\s*,\s*)?"
  102. r"(?P<value_quote>['\"])(?P<value>.*?)(?P=value_quote)",
  103. re.IGNORECASE,
  104. )
  105. _PLAIN_HEADER_ARG_RE = re.compile(
  106. rf"(?P<prefix>--header\s+(?:{_HEADER_NAME_PATTERN})\s+)"
  107. rf"(?>(?:{_SCHEME})?)(?!\[REDACTED\])(?P<value>[^\s,}}\]]+)",
  108. re.IGNORECASE,
  109. )
  110. _SUPPORT_BEARER_RE = re.compile(
  111. # Free text has no header boundary to stop at: consume the whole value,
  112. # inner ``=`` padding included (``abcd1234==x``), not just a key-shaped prefix.
  113. # An RFC 6750/7235 challenge (``Bearer realm="api"``, ``Bearer error=...``)
  114. # is auth-params, not a token: only a known param name stays, so a key-shaped
  115. # ``<letters>="..."`` is still scrubbed (fail closed).
  116. rf"(?P<prefix>\bBearer\s+|\bBasic\s+{_NOT_WORD})"
  117. r"(?!(?:realm|scope|error|error_description|error_uri)=)"
  118. rf"(?=[^\s]{{8}})"
  119. rf"[{_KEY_CHARS}=]+(?::[{_KEY_CHARS}=]+)*",
  120. re.IGNORECASE,
  121. )
  122. def _decoded_compact_name(value: object) -> str:
  123. return "".join(ch for ch in _canonical_url_param_name(str(value or "")) if ch.isalnum())
  124. def _redact_url_params(text: str) -> str:
  125. def _raw_sub(match: re.Match[str]) -> str:
  126. if _decoded_compact_name(match.group("key")) not in _SENSITIVE_URL_PARAM_NAMES:
  127. return match.group(0)
  128. return f"{match.group('sep')}{match.group('key')}=***"
  129. def _encoded_sub(match: re.Match[str]) -> str:
  130. if _decoded_compact_name(match.group("key")) not in _SENSITIVE_URL_PARAM_NAMES:
  131. return match.group(0)
  132. eq = match.group("eq")
  133. star = f"{eq[:-2]}2A" if eq != "=" else "*" # %2A, or %252A when double-encoded
  134. return f"{match.group('sep')}{match.group('key')}{eq}{star * 3}"
  135. # Running until stable handles a raw URL nested inside another query value
  136. # without recursive parsing. The pass cap keeps malformed input bounded.
  137. for _ in range(4):
  138. redacted = _ENCODED_URL_PARAM_RE.sub(_encoded_sub, _RAW_URL_PARAM_RE.sub(_raw_sub, text))
  139. if redacted == text:
  140. break
  141. text = redacted
  142. return text
  143. def _redact_headers_and_argv(text: str) -> str:
  144. text = _QUOTED_HEADER_RE.sub(
  145. lambda match: f"{match.group('prefix')}{match.group('quote')}{_REDACTED}{match.group('quote')}",
  146. text,
  147. )
  148. text = _BARE_HEADER_RE.sub(lambda match: f"{match.group('prefix')}{_REDACTED}", text)
  149. text = _HEADER_ARGV_RE.sub(
  150. lambda match: (
  151. f"{match.group('head_quote')}--header{match.group('head_quote')}, "
  152. f"{match.group('name_quote')}{match.group('name')}{match.group('name_quote')}, "
  153. + (
  154. f"{match.group('scheme_quote')}{match.group('scheme')}{match.group('scheme_quote')}, "
  155. if match.group("scheme")
  156. else ""
  157. )
  158. + f"{match.group('value_quote')}{_REDACTED}{match.group('value_quote')}"
  159. ),
  160. text,
  161. )
  162. text = _QUOTED_ARGV_RE.sub(
  163. lambda match: (
  164. f"{match.group('flag_quote')}{match.group('flag')}{match.group('flag_quote')}"
  165. f"{match.group('sep')}{match.group('value_quote')}{_REDACTED}{match.group('value_quote')}"
  166. ),
  167. text,
  168. )
  169. text = _ARG_EQUALS_RE.sub(
  170. lambda match: f"{match.group('flag')}{match.group('sep')}{match.group('quote')}{_REDACTED}{match.group('quote')}",
  171. text,
  172. )
  173. text = _ARG_OPERAND_RE.sub(
  174. lambda match: f"{match.group('flag')}{match.group('sep')}{match.group('quote')}{_REDACTED}{match.group('quote')}",
  175. text,
  176. )
  177. text = _PLAIN_HEADER_ARG_RE.sub(
  178. lambda match: f"{match.group('prefix')}{_REDACTED}", text
  179. )
  180. return _SUPPORT_BEARER_RE.sub(
  181. lambda match: f"{match.group('prefix')}{_REDACTED}", text
  182. )
  183. def redact_debug_support_text(value: object, *, max_chars: int | None = None) -> str:
  184. """Strictly scrub one debug/support value, then apply an optional character cap.
  185. This is deliberately stricter than ordinary model/tool output: support text is
  186. leaving the process for a human or remote diagnostics store. Any redactor
  187. failure returns the shared fail-closed sentinel rather than the original.
  188. """
  189. text = "" if value is None else str(value)
  190. if not text:
  191. return text
  192. try:
  193. text = redact_sensitive_text(text, force=True, redact_url_credentials=True)
  194. text = _redact_url_params(text)
  195. text = _redact_headers_and_argv(text)
  196. text = _EMAIL_ADDRESS_RE.sub("[REDACTED_EMAIL]", text)
  197. except Exception:
  198. return REDACTION_UNAVAILABLE
  199. return text[:max_chars] if max_chars is not None else text