redact.py 103 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950
  1. """Regex-based secret redaction for logs and tool output.
  2. Short tokens (< 18 chars) are fully masked; longer ones keep the first 6 and
  3. last 4 characters for debuggability.
  4. """
  5. import logging
  6. import os
  7. import re
  8. import shlex
  9. import threading
  10. import unicodedata
  11. from urllib.parse import unquote_plus
  12. # Shared with agent/file_safety's read-block list so the two defenses can't
  13. # drift: a blocked file_tools read that falls back to ``cat`` is still caught.
  14. from agent.file_safety import _BLOCKED_PROJECT_ENV_BASENAMES as _ENV_FILE_BASENAMES
  15. from agent.redact_connstr import _KV_CONNSTR_GATE_RE, _redact_kv_connection_strings
  16. logger = logging.getLogger(__name__)
  17. # Browser and CDP results can be a page-controlled JSON tree. The walkers stop
  18. # at this depth and replace the remainder, so a nested payload cannot raise
  19. # RecursionError on the way to the model.
  20. REDACT_WALK_MAX_DEPTH = 320
  21. REDACT_WALK_DEPTH_MARKER = "[redacted: nesting limit]"
  22. # ---------------------------------------------------------------------------
  23. # Vault-value redaction registry (profile-scoped, bounded)
  24. # ---------------------------------------------------------------------------
  25. # Exact secret values that transited a server-side vault fill (browser_vault). Generic
  26. # credential-shaped regexes cannot catch an arbitrary user password, so the fill path registers
  27. # the exact bytes and every browser_* tool result (including browser_cdp Runtime.evaluate
  28. # passthrough) is scrubbed against them before it can reach the model. Memory only: never
  29. # persisted or logged. Keyed by profile home so a multiplex gateway never scrubs profile B's
  30. # output with profile A's passwords (which would also confirm to B that the bytes exist), and
  31. # bounded per profile: a fill-heavy session evicts its oldest entries rather than growing forever.
  32. _VAULT_REDACTION_MAX_PER_PROFILE = 64
  33. _VAULT_REDACTION_VALUES: dict = {} # profile home → ordered {value: None}
  34. _VAULT_REDACTION_LOCK = threading.Lock()
  35. def _vault_scope() -> str:
  36. from hermes_constants import get_hermes_home
  37. return str(get_hermes_home())
  38. def register_vault_redaction_value(value) -> None:
  39. """Register an exact vault secret value for model-facing redaction.
  40. Called by the vault fill path BEFORE the injection happens, so no later browser tool result
  41. can echo the value back into model context. Also registers the form a text input normalizes
  42. it to (CR/LF stripped), since that is what the page holds.
  43. """
  44. if not isinstance(value, str) or not value:
  45. return
  46. normalized = value.replace("\r", "").replace("\n", "")
  47. with _VAULT_REDACTION_LOCK:
  48. bucket = _VAULT_REDACTION_VALUES.setdefault(_vault_scope(), {})
  49. for v in (value, normalized):
  50. if v:
  51. bucket.pop(v, None) # re-registering refreshes recency
  52. bucket[v] = None
  53. while len(bucket) > _VAULT_REDACTION_MAX_PER_PROFILE:
  54. del bucket[next(iter(bucket))]
  55. def clear_vault_redaction_values() -> None:
  56. """Drop the current profile's registered values (profile teardown / explicit lock)."""
  57. with _VAULT_REDACTION_LOCK:
  58. _VAULT_REDACTION_VALUES.pop(_vault_scope(), None)
  59. def redact_registered_vault_values(text: str) -> str:
  60. """Exact-substring scrub of every vault secret value registered for the current profile."""
  61. if not isinstance(text, str) or not text:
  62. return text
  63. with _VAULT_REDACTION_LOCK:
  64. bucket = _VAULT_REDACTION_VALUES.get(_vault_scope())
  65. values = sorted(bucket, key=len, reverse=True) if bucket else () # longest first: a substring never shadows its superstring
  66. for value in values:
  67. if value in text:
  68. text = text.replace(value, "«redacted-vault-secret»")
  69. return text
  70. # Sensitive query-string param names (case-insensitive): opaque tokens / OAuth
  71. # codes / pre-signed signatures with no vendor prefix.
  72. # Ported from nearai/ironclaw#2529 — catches tokens whose values don't match any known vendor prefix regex
  73. # (e.g. opaque tokens, short OAuth codes).
  74. _SENSITIVE_QUERY_PARAMS = frozenset({
  75. "access_token", "refresh_token", "id_token", "token", "api_key", "apikey",
  76. "client_secret", "password", "auth", "jwt", "session", "secret", "key",
  77. "code", "signature", "x-amz-signature",
  78. "x-goog-signature", "sig", # GCS V4 signed URLs, Azure SAS tokens
  79. })
  80. # Snapshot at import time so runtime env mutations (e.g. an LLM-generated
  81. # `export HERMES_REDACT_SECRETS=false`) cannot disable redaction mid-session.
  82. # ON by default; `security.redact_secrets: false` bridges to this env var.
  83. # ON by default — secure default per issue #17691. Users who need raw credential values in tool output (e.g.
  84. # working on the redactor itself) can opt out via `security.redact_secrets: false` in config.yaml (bridged
  85. # to this env var in hermes_cli/main.py, gateway/run.py, and cli.py) or `HERMES_REDACT_SECRETS=false` in
  86. # ~/.hermes/.env. An opt-out warning is logged at gateway and CLI startup so operators see the downgrade —
  87. # see `_log_redaction_status()` in gateway/run.py and cli.py.
  88. _REDACT_ENABLED = os.getenv("HERMES_REDACT_SECRETS", "true").lower() in {"1", "true", "yes", "on"}
  89. # Routed multiplex profiles: the import-time snapshot above is the LAUNCH profile's policy. A profile
  90. # served under a HERMES_HOME override follows its own ``security.redact_secrets`` (its ``.env`` value
  91. # first, like the standalone bridge in hermes_cli/main.py). The formatter never reads config or files
  92. # for it (it runs under the logging handler's lock, per record): the values are recorded when that
  93. # profile's .env / config is loaded (``note_env_policy`` / ``note_config_policy``) or when log routing
  94. # is set up for it (``resolve_profile_policy``). Until then the profile redacts (the secure default).
  95. _ENV_POLICY_BY_HOME: dict = {} # home key -> raw HERMES_REDACT_SECRETS from its .env, or None
  96. _CONFIG_POLICY_BY_HOME: dict = {} # home key -> raw security.redact_secrets from its config, or None
  97. def note_env_policy(home_key: str, raw) -> None:
  98. _ENV_POLICY_BY_HOME[home_key] = None if raw is None else str(raw)
  99. def note_config_policy(home_key: str, raw) -> None:
  100. _CONFIG_POLICY_BY_HOME[home_key] = None if raw is None else str(raw)
  101. def resolve_profile_policy(home) -> None:
  102. """Record ``home``'s redaction policy now (log-routing setup), so its records never wait for it."""
  103. from hermes_constants import hermes_home_key, reset_hermes_home_override, set_hermes_home_override
  104. token = set_hermes_home_override(home)
  105. try:
  106. key = hermes_home_key()
  107. from hermes_cli.config import load_config_readonly, load_env
  108. note_env_policy(key, load_env().get("HERMES_REDACT_SECRETS"))
  109. note_config_policy(key, (load_config_readonly().get("security") or {}).get("redact_secrets"))
  110. except (Exception, SystemExit) as exc: # health: allow BLE001 -- unreadable now: stays unknown, which redacts
  111. logging.getLogger(__name__).debug("redaction policy for %s not read: %s", home, exc)
  112. finally:
  113. reset_hermes_home_override(token)
  114. def _redact_enabled() -> bool:
  115. """Effective redaction switch for the active profile (launch snapshot when no override). Reads
  116. only in-memory state: the live secret scope and the recorded per-profile policy."""
  117. from hermes_constants import get_hermes_home_override, hermes_home_key
  118. if get_hermes_home_override() is None:
  119. return _REDACT_ENABLED
  120. home_key = hermes_home_key()
  121. from agent.secret_scope import current_secret_scope
  122. scope = current_secret_scope()
  123. raw = scope.get("HERMES_REDACT_SECRETS") if scope else None
  124. if raw is None and scope is None:
  125. # No live scope (the log listener thread formats routed records): the profile's own .env, as
  126. # its scope would have it. Not recorded yet: redact, since .env may opt the profile back in.
  127. if home_key not in _ENV_POLICY_BY_HOME:
  128. return True
  129. raw = _ENV_POLICY_BY_HOME[home_key]
  130. if raw is None:
  131. raw = _CONFIG_POLICY_BY_HOME.get(home_key)
  132. if raw is None:
  133. return True
  134. return str(raw).strip().lower() in {"1", "true", "yes", "on"}
  135. # Known API key prefixes -- match the prefix + contiguous token chars.
  136. # Every pattern MUST start with a literal prefix: _PREFIX_SUBSTRINGS (the cheap
  137. # pre-screen gate) is derived from these literals and must stay false-negative-free.
  138. _PREFIX_PATTERNS = [
  139. # Some provider-issued ``sk-`` keys carry dot-delimited body segments (Alibaba
  140. # ``sk-sp-…``/``sk-ws-…``). Each unit is one body char optionally preceded by
  141. # a single dot, so the body ends on its last non-dot char (sentence punctuation
  142. # is never consumed) and can never span ``..``: the ``sk-pro...EFGH`` display
  143. # mask is left alone by a second redaction pass instead of collapsing to
  144. # ``***``. Kept free of nested unbounded repeats so the pattern passes the
  145. # same structural gate plugins must.
  146. r"sk-[A-Za-z0-9_-](?:\.?[A-Za-z0-9_-]){9,}",
  147. # Context7 API keys: a fixed vendor prefix (``ctx7sk-``) plus an opaque body.
  148. # The generic ``sk-`` family cannot catch it: the matcher's lookbehind
  149. # rejects a prefix preceded by a word char, and here the ``sk-`` sits inside
  150. # ``ctx7sk-`` - so a bare Context7 key passed through unmasked (#142576).
  151. r"ctx7sk-[A-Za-z0-9_-]{10,}",
  152. r"ghp_[A-Za-z0-9]{10,}", # GitHub PAT (classic)
  153. r"github_pat_[A-Za-z0-9_]{10,}", # GitHub PAT (fine-grained)
  154. r"gho_[A-Za-z0-9]{10,}", # GitHub OAuth access token
  155. r"ghu_[A-Za-z0-9]{10,}", # GitHub user-to-server token
  156. r"ghs_[A-Za-z0-9]{10,}", # GitHub server-to-server token
  157. r"ghr_[A-Za-z0-9]{10,}", # GitHub refresh token
  158. r"xapp-\d+-[A-Za-z0-9-]{10,}", # Slack app-Level token
  159. r"xox[baprs]-[A-Za-z0-9-]{10,}", # Slack bot/app/user tokens
  160. r"AIza[A-Za-z0-9_-]{30,}", # Google API keys
  161. r"pplx-[A-Za-z0-9]{10,}", # Perplexity
  162. r"fal_[A-Za-z0-9_-]{10,}", # Fal.ai
  163. r"fc-[A-Za-z0-9]{10,}", # Firecrawl
  164. r"bb_live_[A-Za-z0-9_-]{10,}", # BrowserBase
  165. r"gAAAA[A-Za-z0-9_=-]{20,}", # Codex encrypted tokens
  166. r"AKIA[A-Z0-9]{16}", # AWS Access Key ID
  167. r"sk_live_[A-Za-z0-9]{10,}", # Stripe secret key (live)
  168. r"sk_test_[A-Za-z0-9]{10,}", # Stripe secret key (test)
  169. r"rk_live_[A-Za-z0-9]{10,}", # Stripe restricted key
  170. r"SG\.[A-Za-z0-9_-]{10,}", # SendGrid API key
  171. r"hf_[A-Za-z0-9]{10,}", # HuggingFace token
  172. r"r8_[A-Za-z0-9]{10,}", # Replicate API token
  173. r"npm_[A-Za-z0-9]{10,}", # npm access token
  174. r"pypi-[A-Za-z0-9_-]{10,}", # PyPI API token
  175. r"dop_v1_[A-Za-z0-9]{10,}", # DigitalOcean PAT
  176. r"doo_v1_[A-Za-z0-9]{10,}", # DigitalOcean OAuth
  177. # AgentMail API key: ``am_`` / ``am_org_`` + an opaque alphanumeric body. The body has no ``_``/``-``,
  178. # which is what separates it from ``am_example_identifier_123`` (#10983); public docs pin only the
  179. # prefix, so the charset stays broad and the length floor does the discriminating.
  180. r"am_(?:org_)?[A-Za-z0-9]{20,}",
  181. r"sk_[A-Za-z0-9_]{10,}", # ElevenLabs TTS key (sk_ underscore, not sk- dash)
  182. r"tvly-[A-Za-z0-9]{10,}", # Tavily search API key
  183. r"exa_[A-Za-z0-9]{10,}", # Exa search API key
  184. r"gsk_[A-Za-z0-9]{10,}", # Groq Cloud API key
  185. r"syt_[A-Za-z0-9]{10,}", # Matrix access token
  186. r"retaindb_[A-Za-z0-9]{10,}", # RetainDB API key
  187. r"hsk-[A-Za-z0-9]{10,}", # Hindsight API key
  188. r"mem0_[A-Za-z0-9]{10,}", # Mem0 Platform API key
  189. r"brv_[A-Za-z0-9]{10,}", # ByteRover API key
  190. r"xai-[A-Za-z0-9]{30,}", # xAI (Grok) API key
  191. r"ntn_[A-Za-z0-9]{10,}", # Notion internal integration token
  192. r"fw-[A-Za-z0-9]{30,}", # Fireworks AI API key
  193. r"fw_[A-Za-z0-9]{30,}", # Fireworks AI API key
  194. r"fpk_[A-Za-z0-9]{30,}", # Fireworks AI project key
  195. # GitLab token families (each keeps a full literal prefix for the pre-screen).
  196. # Ported from openclaw/openclaw#112954; follow-up invited in #4541.
  197. r"glpat-[A-Za-z0-9_\-]{10,}", # GitLab personal access token
  198. r"gloas-[A-Za-z0-9_\-]{10,}", # GitLab OAuth application secret
  199. r"gldt-[A-Za-z0-9_\-]{10,}", # GitLab deploy token
  200. r"glrt-[A-Za-z0-9_.\-]{10,}", # GitLab runner authentication token (routable tokens are dotted)
  201. r"glrtr-[A-Za-z0-9_.\-]{10,}", # GitLab runner registration token (routable)
  202. r"glcbt-[A-Za-z0-9_\-]{10,}", # GitLab CI/CD job token
  203. r"glptt-[A-Za-z0-9_\-]{10,}", # GitLab pipeline trigger token
  204. r"glft-[A-Za-z0-9_\-]{10,}", # GitLab feed token
  205. r"glimt-[A-Za-z0-9_\-]{10,}", # GitLab incoming mail token
  206. r"glagent-[A-Za-z0-9_\-]{10,}", # GitLab agent (KAS) token
  207. r"glsoat-[A-Za-z0-9_\-]{10,}", # GitLab service-account access token
  208. r"glffct-[A-Za-z0-9_\-]{10,}", # GitLab feature-flags client token
  209. r"glwt-[A-Za-z0-9_\-]{10,}", # GitLab workspace token
  210. r"GR1348941[A-Za-z0-9_\-]{10,}", # GitLab legacy runner registration token
  211. r"pk-lf-[A-Za-z0-9\-]{8,}", # Langfuse public key (sk-lf- already covered by sk- pattern)
  212. ]
  213. # ENV assignment: KEY=value where KEY carries a secret-like name. Uppercase keys
  214. # tolerate spaces around "=" and allow the keyword embedded anywhere
  215. # (``MYTOKEN=…``) — an all-caps key is almost never prose. Bare ``KEY``/``PASS``/
  216. # ``PW`` suffixes are included; _key_has_secret_keyword rejects ``KEYBOARD=``.
  217. # The regex is IGNORECASE so lowercase env names (``openai_key=…``) are caught here too. The secret name
  218. # must sit at a word boundary (``_``-delimited or whole-word) so generic prose words (``password=``,
  219. # ``token=``, ``KEYBOARD=``, ``PASSAGE=``) do not match — those are handled by the config/form/URL paths,
  220. # and a bare ``password=…`` in a form body must not be swallowed greedily by ``\S+``. See #77484.
  221. _SECRET_ENV_NAMES = r"(?:API_?KEY|KEY|TOKEN|SECRET|PASSWORD|PASSWD|PASS|PW|CREDENTIAL|AUTH)"
  222. _ENV_ASSIGN_RE = re.compile(rf"([A-Z0-9_]{{0,50}}{_SECRET_ENV_NAMES}[A-Z0-9_]{{0,50}})\s*=\s*(['\"]?)(\S+)\2")
  223. # Command-line credentials. ``--password=…``, ``--token …`` and ``docker login -p …`` reach
  224. # the approval card and the chat egress verbatim because the ENV passes only look at
  225. # ``NAME=value`` (issue #143982). The flag name is kept so the reader still knows what was
  226. # passed; only the value is masked. ``-p`` is scoped to a container ``login`` because
  227. # everywhere else it is a port or a path.
  228. # Each quote style is matched on its own, so ``"Abc'def ghi"`` is consumed whole instead of
  229. # falling back to ``\S+`` at the apostrophe and leaving the tail visible. A double-quoted
  230. # value honours backslash escapes (``"a\"b c"``) and a single-quoted one the shell's
  231. # ``'\''`` splice, and either may span a newline. The body is capped and possessive: an
  232. # unterminated or over-long quote masks through the end of its line instead of only its
  233. # first word, and never rescans the rest of the text from every flag.
  234. _QUOTED_FLAG_VALUE = (r"\"(?:\\[\s\S]|[^\"\\]){1,512}+\"|'(?:'\\''|[^']){1,512}+'"
  235. r"|[\"'][^\n]*+|\S+")
  236. _CLI_SECRET_FLAG_RE = re.compile(
  237. r"(?P<flag>(?<![\w-])--?(?:password|passwd|passphrase|pass|token|api[-_]?key|apikey"
  238. r"|client[-_]?secret|access[-_]?key|account[-_]?key|auth[-_]?token|session[-_]?token|secret|bearer"
  239. r"|credential)s?)"
  240. r"(?P<sep>\s*=|\s+)"
  241. rf"(?P<value>{_QUOTED_FLAG_VALUE})",
  242. re.IGNORECASE,
  243. )
  244. _CLI_LOGIN_P_RE = re.compile(
  245. r"(?P<flag>\b(?:docker|podman|nerdctl)\s+login\b[^\n;|&]*?\s-p\s*)"
  246. rf"(?P<value>{_QUOTED_FLAG_VALUE})"
  247. )
  248. # Azure connection-string keys. They are ``Name=value`` inside a quoted string, so
  249. # every ENV pass declines them: the name is mixed case and carries no underscore boundary.
  250. # The key must open a connection-string segment (start, ``;``, ``,``, a URL ``?``/``&``, a quote or whitespace before
  251. # it, no space around ``=``): ``this.accountKey = x`` / ``dt.PrimaryKey = new …`` in source
  252. # read through read_file or a terminal stays byte-identical, or the agent cannot patch it.
  253. # A spaced ``AccountKey = …`` (ini files, PowerShell) only counts when the value is a
  254. # 16+ char base64 run carrying a digit, which an identifier like ``config.Key`` or
  255. # ``StorageAccountKeyFromVault`` is not.
  256. _CONNSTR_SECRET_KEY_RE = re.compile(
  257. r"(?P<flag>(?<![^;,?&'\"\s])(?:AccountKey|SharedAccessKey|(?P<sas>SharedAccessSignature))"
  258. r"(?:=|[ \t]*+=[ \t]*+(?=[A-Za-z+/]*[0-9])(?=[A-Za-z0-9+/]{16,}={0,2}(?![\w.(]))))"
  259. # A SAS value is itself a query string (``sv=…&sig=…``): it runs through the ``sig=``
  260. # parameter and stops there, or at a ``;`` / ``)`` / ``,`` / ``|`` / backtick delimiter.
  261. r"(?P<value>(?(sas)(?:(?!sig=)[^;\s'\"&),`|]++&)*+(?:sig=)?[^;\s'\"&),`|]*+|[^;\s'\"&]+))",
  262. re.IGNORECASE,
  263. )
  264. # Cheap pre-gates: every flag the regexes above can match contains one of these.
  265. _CLI_SECRET_FLAG_GATE = re.compile(r"-{1,2}[A-Za-z]")
  266. _CLI_LOGIN_GATE = re.compile(r"(?:docker|podman|nerdctl)\s+login\b", re.IGNORECASE)
  267. _CONNSTR_GATE = re.compile(r"(?:AccountKey|SharedAccessKey|SharedAccessSignature)", re.IGNORECASE)
  268. # Lowercase env names: only underscore-boundary forms (``openai_key=``) — NOT
  269. # bare ``password=``/``token=``, which appear in prose, URLs, and form bodies.
  270. # The lookbehind anchors each attempt to the start of an identifier run; without
  271. # it re.sub retries the greedy prefix at every byte of a long opaque payload.
  272. # See #77484.
  273. _ENV_ASSIGN_LOWER_RE = re.compile(
  274. r"(?<![a-z0-9_])([a-z0-9_]+(?:_|^)(?:key|pass|pw|token|secret|password|passwd|credential|auth)(?=[^a-z0-9_]|$))\s*=\s*(['\"]?)(\S+)\2",
  275. re.IGNORECASE,
  276. )
  277. # Lowercase / dotted config-file keys (``spring.datasource.password=x``,
  278. # line-start ``password=x``). Carve-outs vs prose/code/URLs: values stop at
  279. # whitespace AND ``&`` (form bodies go pair-by-pair via _redact_form_body);
  280. # _CFG_DOTTED_RE needs a NAMESPACED key; _CFG_ANCHORED_RE needs line start
  281. # (optionally after ``export``). The ``://`` URL guard lives at the call site.
  282. # The uppercase _ENV_ASSIGN_RE above never matched these, so config-file passwords leaked verbatim (issue
  283. # #16413). These run only in a config-file context, NOT in prose, code, or URLs — three carve-outs preserved
  284. # from the original design (#4367 + the documented web-URL passthrough below): 1. The value is bounded by
  285. # ``[^\s&]`` (stops at whitespace AND ``&``) so form-urlencoded bodies are handled pair-by-pair (by
  286. # _redact_form_body), not greedily swallowed. 2. _CFG_DOTTED_RE only matches when the key is NAMESPACED
  287. # (contains a dot), which is unambiguously a config key — never a prose word. 3. _CFG_ANCHORED_RE matches a
  288. # bare secret-word key only at line start (optionally after ``export``), so conversational ``I have
  289. # password=foo`` mid-sentence is left alone.
  290. _SECRET_CFG_NAMES = r"(?:api[ _.\-]?key|token|secret|passwd|password|credential|auth)"
  291. # Rendered line-number prefix: ``5|line`` (read_file), ``6:line`` (grep -n), ``7-line`` (grep -A/-B/-C
  292. # context lines) and `` 8\tline`` (cat -n / nl: right-aligned number + TAB). Callers put the ONLY
  293. # leading ``[ \t]*`` in front of it — stacking a second whitespace run around an optional gutter made
  294. # the anchored passes quadratic on long indented lines (2s per 5k spaces).
  295. _LINE_NUMBER_GUTTER = r"(?:[0-9]+(?:[|:\-]|\t)[ \t]*)?"
  296. _CFG_VALUE = r"(['\"]?)([^\s&]+?)\2(?=[\s&]|$)"
  297. # Linear pre-gate for the _CFG_*_RE subs: no secret keyword => neither can match.
  298. _CFG_SECRET_WORD_RE = re.compile(_SECRET_CFG_NAMES, re.IGNORECASE)
  299. # Programmatic env lookups (``os.getenv(...)``, ``process.env.X``, ``$ENV{X}``)
  300. # as the VALUE of a KEY=... match name a variable; they are not a leaked secret.
  301. _ENV_LOOKUP_VALUE_RE = re.compile(r"^(?:os\.(?:getenv|environ)|process\.env|\$ENV\{)")
  302. # Namespaced key: the secret word may sit anywhere in a dotted path.
  303. # NOTE(perf): possessive quantifiers (nested ``(?:[...]+\.)+`` backtracked
  304. # exponentially); the ``*`` runs bordering {_SECRET_CFG_NAMES} must stay
  305. # backtrackable (``app.api.key=``). The lookbehind anchors each attempt to a key
  306. # run start so re.sub is not quadratic; the match set is unchanged.
  307. _CFG_DOTTED_RE = re.compile(
  308. rf"(?<![A-Za-z0-9_.\-])"
  309. rf"([A-Za-z0-9_\-]++\.[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*+"
  310. rf"|[A-Za-z0-9_.\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_.\-]*\.[A-Za-z0-9_.\-]++)"
  311. rf"={_CFG_VALUE}",
  312. re.IGNORECASE,
  313. )
  314. # Line-anchored bare key: ``password=…`` / ``export api_key=…`` at start of line.
  315. # ``{_LINE_NUMBER_GUTTER}``: line-numbered dumps put the key behind a rendered gutter —
  316. # ``read_file`` emits ``5| ADS_API_TOKEN: …``, ``grep -n`` emits ``6: ADS_API_TOKEN: …``
  317. # and ``cat -n`` emits `` 7\tADS_API_TOKEN: …``. Anchored at ``^`` without it, none of those
  318. # matched, so the rendered read of a secret-bearing file leaked what the raw text masked.
  319. _CFG_ANCHORED_RE = re.compile(
  320. rf"(^[ \t]*{_LINE_NUMBER_GUTTER}(?:export[ \t]+)?[A-Za-z0-9_\-]*{_SECRET_CFG_NAMES}[A-Za-z0-9_\-]*)={_CFG_VALUE}",
  321. re.IGNORECASE | re.MULTILINE,
  322. )
  323. # YAML / colon config: bare-key quoted scalars cannot match _JSON_FIELD_RE.
  324. # Match escaped double quotes, doubled single quotes, or an unquoted token.
  325. # An unterminated quote masks to end-of-line; no branch crosses LF or CRLF.
  326. # NOTE(perf): possessive where the successor is disjoint; the leading class
  327. # stays backtrackable (see _CFG_DOTTED_RE).
  328. _YAML_CFG_NAMES = (
  329. r"(?:api[ _.\-]?key|token|secret|passwd|password|credential"
  330. r"|client[ _.\-]?key[ _.\-]?data|(?:private|signing|encryption|master|hmac)[ _.\-]?key)"
  331. )
  332. _YAML_ASSIGN_RE = re.compile(
  333. rf"(^[ \t]*+{_LINE_NUMBER_GUTTER}[A-Za-z0-9_.\-]*{_YAML_CFG_NAMES}[A-Za-z0-9_.\-]*+)(:[ \t]*+)"
  334. r"(\"(?:\\[^\r\n]|[^\"\\\r\n])*\"|'(?:''|[^'\r\n])*'|['\"][^\r\n]*|[^\s&]++)",
  335. re.IGNORECASE | re.MULTILINE,
  336. )
  337. _YAML_CFG_WORD_RE = re.compile(_YAML_CFG_NAMES, re.IGNORECASE)
  338. # Word-boundary validation for the key patterns above: their classes allow
  339. # arbitrary affixes (``client_secret``, ``s3.secret-key``), which also matched
  340. # prose CONTAINING a keyword (``Secretary:``, ``tokenizer:``). A keyword counts
  341. # only at a word boundary: key edge, next to a non-letter, or a camelCase
  342. # transition (``clientSecret``, ``APIToken``); trailing plural ``s`` is part of
  343. # it. Concatenations match via explicit alternatives (``authtoken``, ``apikey``).
  344. # The side effect: ordinary prose/document words that merely CONTAIN a keyword also matched — ``Secretary:
  345. # J.Smith`` (secret), ``tokenizer: cl100k_base`` (token), ``author=Smith`` (auth) — mangling legitimate
  346. # content on the surfaces that run these passes (browser snapshots, log lines, kanban summaries, CLI-echoed
  347. # command output). Ported from nearai/ironclaw#6129, where the same substring false positive ("Secretary of
  348. # the Treasury" matching the ``secret`` marker) scrubbed legitimate tool results from the replayed
  349. # transcript and sent the model into a re-fetch loop. Common concatenated compounds keep matching via
  350. # explicit alternatives (``authtoken`` ngrok, ``authkey`` tailscale, ``secretkey`` minio, ``apikey``).
  351. # Embedded occurrences inside a larger word (``secretary``, ``tokenizer``, ``authored``, ``credentialing``)
  352. # no longer match. ALL-CAPS keys keep the legacy embedded matching (``MYTOKEN=…``) — an all-caps key is
  353. # almost never prose, the same rationale as _ENV_ASSIGN_RE.
  354. _KEY_KEYWORD_RE = re.compile(
  355. r"(?:api|auth|access|refresh|session|secret)[ _.\\-]?(?:key|token)"
  356. r"|token|secret|passwd|password|pass|pw|credential|auth|key",
  357. re.IGNORECASE,
  358. )
  359. # Key names that are credential-specific even when their values are short or
  360. # human-readable. Bare ``token`` / ``key`` are intentionally absent: they also
  361. # describe model limits, tensor names, and cache keys, so those assignments
  362. # are gated on value shape (_looks_like_opaque_credential).
  363. _STRONG_KEY_KEYWORD_RE = re.compile(
  364. r"(?:api|auth|access|refresh|session|id|bearer)[ _.\\-]?(?:key|token)"
  365. r"|key[ _.\\-]?material|secret|passwd|password|pass|pw|credential|auth|bearer",
  366. re.IGNORECASE,
  367. )
  368. # Password-class keys mask any literal value; for other keys a value that starts like ``$HOME/...``,
  369. # ``/usr/...`` or ``~/...`` references a variable or a path, not a credential, even under a strong key
  370. # (``SSH_AUTH_SOCK=$HOME/.ssh/agent.sock``, ``DOCKER_AUTH_CONFIG=/home/u/.docker``).
  371. _PASSWORD_KEY_RE = re.compile(r"passwd|password|pass|pw", re.IGNORECASE)
  372. # Anchored on both ends: the whole value must be a ``$VAR``/``${VAR}`` reference, a ``~/``
  373. # path, or an absolute path — not merely a string whose FIRST character is one of those.
  374. # A 40-char AWS secret key starts with '/' ~1 in 64 times and argon2/bcrypt digests always
  375. # start with '$'; an unanchored class let those secrets skip every check below.
  376. # Further ``$VAR`` interpolations may appear anywhere in the path (``/run/user/$UID/ssh``,
  377. # ``$XDG_RUNTIME_DIR/agent.$USER.sock``, ``$A:$B`` lists); crypt digests never parse as one
  378. # because their ``$`` fields start with a digit or carry ``=``/``,``.
  379. # A leading ``$(`` is a command substitution (``SSH_AUTH_SOCK=$(gpgconf --list-dirs
  380. # agent-ssh-socket)``): the value token stops at whitespace, so only ``$(gpgconf`` is seen.
  381. _SHELL_VAR_REF = r"\$(?:\{[A-Za-z_]\w*[^}]*\}|[A-Za-z_]\w*)"
  382. _PATH_OR_VAR_VALUE_RE = re.compile(rf"^(?:{_SHELL_VAR_REF}|\$\(|~|/)(?:[\w./:-]|{_SHELL_VAR_REF})*$")
  383. # ``$VAR`` / ``$(cmd`` are unambiguous references. A ``/``- or ``~``-led value is a path only
  384. # while every segment reads like one: a 16+ char segment mixing case and digits with no ``.``
  385. # (``/wJalrXUtnFEMIK7MDENG/bPxRf…``) is a secret that happens to start with a path character,
  386. # whereas ``/home/u/.docker`` / ``~/.ssh/id_rsa`` / ``S.gpg-agent.ssh`` never clear that bar.
  387. _OPAQUE_PATH_SEGMENT_RE = re.compile(r"(?=[^.]*[a-z])(?=[^.]*[A-Z])(?=[^.]*[0-9])[^.]{16,}")
  388. def _is_word_start(s: str, i: int) -> bool:
  389. """True if position ``i`` in ``s`` begins a word (edge, non-letter before, camelCase/acronym boundary)."""
  390. if i == 0:
  391. return True
  392. prev, cur = s[i - 1], s[i]
  393. if not prev.isalpha() or (cur.isupper() and prev.islower()): # clientSecret
  394. return True
  395. return cur.isupper() and prev.isupper() and i + 1 < len(s) and s[i + 1].islower() # APIToken
  396. def _is_word_end(s: str, j: int, *, allow_plural: bool = True) -> bool:
  397. """True if position ``j`` (exclusive end) in ``s`` ends a word; one trailing ``s`` is absorbed."""
  398. if j >= len(s):
  399. return True
  400. cur = s[j]
  401. if not cur.isalpha() or (cur.isupper() and s[j - 1].islower()): # secretKey
  402. return True
  403. return allow_plural and cur in "sS" and _is_word_end(s, j + 1, allow_plural=False)
  404. def _has_word_bounded_keyword(key: str, keyword_re: re.Pattern[str]) -> bool:
  405. """True if ``keyword_re`` matches ``key`` at a word boundary (see _KEY_KEYWORD_RE)."""
  406. return any(_is_word_start(key, m.start()) and _is_word_end(key, m.end()) for m in keyword_re.finditer(key))
  407. _PASSWORD_WORD_RE = re.compile(r"passwd|password", re.IGNORECASE)
  408. def _has_password_keyword(key: str) -> bool:
  409. """``password``/``passwd`` ending a word, wherever it starts: no prose word ends in them, so
  410. concatenated keys (``dbpassword``, ``rootpasswd``) are credentials, while ``passwordless``
  411. is not. Requiring a word START too passed ``dbpassword: <v>`` through in clear."""
  412. return any(_is_word_end(key, m.end()) for m in _PASSWORD_WORD_RE.finditer(key))
  413. # Tool-defined env names that glue the keyword onto a prefix with no separator, so the
  414. # word-boundary test can never see it: libpq's ``PGPASSWORD``, MySQL's ``MYSQL_PWD`` (bare
  415. # ``PWD`` is the shell's working directory, so only this exact name), sshpass's ``SSHPASS``.
  416. _GLUED_CREDENTIAL_ENV_RE = re.compile(r"PGPASSWORD|MYSQL_PWD|SSHPASS", re.IGNORECASE)
  417. def _is_glued_credential_env(key: str) -> bool:
  418. return _GLUED_CREDENTIAL_ENV_RE.fullmatch(key.strip("\"'")) is not None
  419. def _key_has_secret_keyword(key: str) -> bool:
  420. """Post-match key validator: ``API_KEY``/``DB_PW`` count, ``KEYBOARD``/``secretary`` do not."""
  421. return (_has_word_bounded_keyword(key, _KEY_KEYWORD_RE) or _has_password_keyword(key)
  422. or _is_glued_credential_env(key))
  423. def _looks_like_opaque_credential(value: str) -> bool:
  424. """Credential-like shape test for ambiguous ``token``/``key`` values, so short
  425. technical scalars (``CPU``, ``local``) are not masked merely for their key name."""
  426. if value == "***" or value.startswith("«redacted:"):
  427. return True
  428. if len(value) >= 16 and re.fullmatch(r"[A-Fa-f0-9]+", value):
  429. return True
  430. if len(value) >= 20 and re.fullmatch(r"[A-Za-z0-9_./+=-]+", value):
  431. return True
  432. if len(value) < 12:
  433. return False
  434. return sum(bool(re.search(p, value)) for p in (r"[a-z]", r"[A-Z]", r"[0-9]")) >= 2
  435. def _should_redact_assignment(key: str, value: str, *, check_keyword: bool) -> bool:
  436. """Shared gate for the ENV / JSON / YAML assignment passes: skip programmatic env
  437. lookups used as values, optionally require a word-bounded keyword in the key,
  438. then redact when the key is unambiguously credential-bearing or the value looks opaque."""
  439. # Programmatic env lookups reference variable *names*, not secret values — masking them corrupts code
  440. # snippets in prose/log contexts (issue #2852): ``KEY=os.getenv('X')``.
  441. # Same programmatic-env-lookup exception as _redact_env above (issue #2852): "apiKey": "os.getenv('X')"
  442. # is a code snippet, not a leaked secret value.
  443. # Same programmatic-env-lookup exception as _redact_env above (issue #2852): api_key: os.getenv('X') is
  444. # a code snippet, not a leaked secret value.
  445. if _ENV_LOOKUP_VALUE_RE.match(value):
  446. return False
  447. # An earlier pass already masked this value (``***`` or the ``«redacted:…»`` sentinel). Masking it
  448. # again only erases what the sentinel deliberately kept — the vendor label (``Digest ***`` →
  449. # ``***``, ``«redacted:ghp_…»`` → ``«redacted-secret»``). Same guard _redact_python_repr_fields uses.
  450. if value == "***" or value.startswith("«redacted"):
  451. return False
  452. if check_keyword and not _key_has_secret_keyword(key):
  453. return False
  454. # A shell rc's ``SSH_AUTH_SOCK=$HOME/.ssh/agent.sock`` is configuration the agent must keep
  455. # readable; only password-class keys mask a path/variable reference.
  456. password_key = _has_word_bounded_keyword(key, _PASSWORD_KEY_RE) or _has_password_keyword(key)
  457. if _PATH_OR_VAR_VALUE_RE.match(value) and not password_key:
  458. # ``$VAR`` is an unambiguous reference. A bare ``/...`` or ``~...`` is not:
  459. # ``/home/u/.docker`` and ``/8f3kd9sKd0als...`` have the same shape, so every
  460. # segment has to look like a path (see _OPAQUE_PATH_SEGMENT_RE) before the
  461. # value is treated as configuration.
  462. if value[0] == "$" or not any(_OPAQUE_PATH_SEGMENT_RE.fullmatch(seg) for seg in value.split("/")):
  463. return False
  464. # ``signing_key`` / ``privateKey`` are credentials at any length, the same policy
  465. # is_secret_field_name applies to mappings, so JSON text and dicts agree.
  466. return (password_key or _has_word_bounded_keyword(key, _STRONG_KEY_KEYWORD_RE)
  467. or _is_glued_credential_env(key)
  468. or (_is_qualified_key_name(key) and not _CONFIG_PLACEHOLDER_VALUE_RE.fullmatch(value))
  469. or _looks_like_opaque_credential(value))
  470. # JSON field patterns: "apiKey": "value", "token": "value", etc.
  471. # ``key``-suffixed credential names (``aws_secret_access_key``, ``SECRET_KEY``,
  472. # ``privateKey``, ``signing_key``) and kubeconfig's ``client-key-data`` (a base64 PEM).
  473. # The bare word ``key`` stays out: it names cache keys and model limits. A qualifier
  474. # word has to sit in front of it, which is what the alternation spells out.
  475. _KEY_QUALIFIED_NAME = (
  476. r"[\w.\-]*(?:secret|private|signing|encryption|encrypt|master|hmac|symmetric|access)"
  477. r"[_.\-]?(?:key|Key|KEY)"
  478. )
  479. _JSON_KEY_NAMES = (
  480. r"(?:api_?[Kk]ey|token|secret|password|pgpassword|mysql_pwd|sshpass|access_token|refresh_token|auth_token|bearer"
  481. r"|secret_value|raw_secret|secret_input|key_material|client[_.\-]?key[_.\-]?data"
  482. rf"|{_KEY_QUALIFIED_NAME})"
  483. )
  484. _JSON_FIELD_RE = re.compile(rf'("{_JSON_KEY_NAMES}")\s*:\s*"([^"]+)"', re.IGNORECASE)
  485. # The same field inside a JSON-encoded string (``{"output": "{\\"api_key\\": \\"…\\"}"}``): every tool result
  486. # that wraps a config dump in JSON escapes the quotes, so the rule above never saw the value (#115104).
  487. # The backreference keeps the key's and value's escape depth equal.
  488. _JSON_FIELD_ESCAPED_RE = re.compile(rf'((\\+)"{_JSON_KEY_NAMES}\2")\s*:\s*\2"([^"\\]+)\2"', re.IGNORECASE)
  489. # Any other identifier-shaped key (``"API_TOKEN"``, ``"DB_PASSWORD"``, ``"client_secret"``): the closed
  490. # list above misses affixed names that the ENV and YAML passes already accept (#147240). The callback
  491. # applies the YAML pass's word set at a word boundary, then the shared assignment gate.
  492. # The value may carry JSON escapes (``"s3cret\\value"``): ``\\.`` consumes each escape pair whole.
  493. _JSON_ANY_KEY_FIELD_RE = re.compile(r'("([A-Za-z0-9_.\-]{1,128}+)")\s*+:\s*+"((?:[^"\\\n]|\\.)++)"')
  494. # The same field inside a JSON-encoded string (``{"output": "{\\"DB_PASSWORD\\": \\"…\\"}"}``),
  495. # like _JSON_FIELD_ESCAPED_RE does for the closed key list: group 2 is the escape run.
  496. _JSON_ANY_KEY_FIELD_ESCAPED_RE = re.compile(
  497. r'((\\++)"([A-Za-z0-9_.\-]{1,128}+)\2")\s*+:\s*+\2"([^"\\\n]++)\2"')
  498. _JSON_KEY_NAMES_FULL_RE = re.compile(_JSON_KEY_NAMES, re.IGNORECASE)
  499. # Python ``repr`` uses single-quoted mapping fields, so opaque credentials in
  500. # tracebacks and pytest failure introspection bypass the double-quoted JSON rule
  501. # above: ``{'BRAVE_API_KEY': 'opaque-value'}``. Capture identifier-shaped keys
  502. # here, then apply the canonical high-confidence key policy in the callback.
  503. _PYTHON_REPR_SECRET_KEYS = frozenset({
  504. "access_token",
  505. "refresh_token",
  506. "id_token",
  507. "auth_token",
  508. "token",
  509. "api_key",
  510. "apikey",
  511. "client_secret",
  512. "secret",
  513. "password",
  514. "passwd",
  515. "private_key",
  516. "credential",
  517. "credentials",
  518. "authorization",
  519. "bearer",
  520. "secret_value",
  521. "raw_secret",
  522. "secret_input",
  523. "key_material",
  524. })
  525. _PYTHON_REPR_ENV_SUFFIXES = (
  526. "_API_KEY",
  527. "_TOKEN",
  528. "_SECRET",
  529. "_PASSWORD",
  530. "_PASSWD",
  531. "_CREDENTIAL",
  532. "_CREDENTIALS",
  533. )
  534. # Casefolded credential suffixes for mixed/camel-case key names
  535. # (``UserPassword``, ``sessionToken``, ``clientApiKey``). Suffix-only so
  536. # ``token_count`` / ``password_policy`` metadata keys never match. Widened per
  537. # OpenHands/software-agent-sdk#4508.
  538. _PYTHON_REPR_CREDENTIAL_SUFFIXES = (
  539. "apikey",
  540. "api_key",
  541. "token",
  542. "secret",
  543. "password",
  544. "passwd",
  545. "credential",
  546. "credentials",
  547. )
  548. _PYTHON_REPR_FIELD_RE = re.compile(
  549. r"'(?P<key>[A-Za-z_][A-Za-z0-9_]*)'(?P<sep>\s*:\s*)"
  550. r"(?:"
  551. r"(?P<single_prefix>[bB]?)'(?P<single_value>(?:\\.|[^'\\])+)'"
  552. r"|(?P<double_prefix>[bB]?)\"(?P<double_value>(?:\\.|[^\"\\])+)\""
  553. r")"
  554. )
  555. # Terminal/process output normally uses ``code_file=True`` to preserve source.
  556. # Add repr masking only to high-confidence diagnostic lines: pytest assertion
  557. # introspection (``E ...``) and final Python exception lines.
  558. _PYTEST_DIAGNOSTIC_LINE_RE = re.compile(r"^(?P<prefix>[ \t]*E[ \t]{2,})(?P<body>.*)$")
  559. _PYTHON_EXCEPTION_LINE_RE = re.compile(
  560. r"^(?P<prefix>(?:[A-Za-z_]\w*\.)*[A-Za-z_]\w*"
  561. r"(?:Error|Exception|Warning):[ \t]*)(?P<body>.*)$"
  562. )
  563. # Authorization / Proxy-Authorization, any scheme or bare credential; header
  564. # name and scheme word preserved. The credential class excludes quotes: pulling
  565. # a closing quote into the mask turns value corruption into SYNTAX corruption
  566. # (unterminated quote → shell EOF / SyntaxError).
  567. _AUTH_HEADER_RE = re.compile(r"((?:Proxy-)?Authorization:\s*)([A-Za-z][\w.+-]*\s+)?([^\s\"']+)", re.IGNORECASE)
  568. # API-key style headers (single opaque value, no scheme word): non-vendor-prefix
  569. # values would otherwise leak when a curl command is echoed into tool output.
  570. SECRET_HEADER_NAME_LIST = ("x-api-key", "x-goog-api-key", "api-key", "apikey", "x-api-token", "x-auth-token", "x-access-token")
  571. _SECRET_HEADER_NAMES = rf"(?:{'|'.join(SECRET_HEADER_NAME_LIST)})"
  572. _SECRET_HEADER_RE = re.compile(rf"({_SECRET_HEADER_NAMES}\s*:\s*)(\S+)", re.IGNORECASE)
  573. # Telegram bot tokens: [bot]<digits>:<token>, token >= 30 chars. The lookbehind
  574. # anchors the id at the start of its digit run: without it, a long run of digits
  575. # with no ":<token>" after it (a Unity/YAML ``_typelessdata`` blob in a 2 MB PR
  576. # diff, hex/decimal dumps) retried the greedy ``\d{8,}`` from every digit —
  577. # quadratic, one core at 100% for hours while holding the GIL.
  578. _TELEGRAM_RE = re.compile(r"(?<!\d)(bot)?(\d{8,}):([-A-Za-z0-9_]{30,})")
  579. _PRIVATE_KEY_RE = re.compile(r"-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----")
  580. # A base64-encoded PEM private key: kubeconfig's ``client-key-data`` and the ``*_b64``
  581. # fields around it (``kubectl config view --raw``, cloud-init, CI secrets). The name is
  582. # not reliable — kubeconfig writes ``client-key-data`` in YAML and JSON alike, and the
  583. # terminal path deliberately skips the assignment passes on source-shaped output — so the
  584. # rule keys on the payload: base64 that decodes to a ``BEGIN … PRIVATE KEY`` header.
  585. _B64_PEM_RE = re.compile(r"LS0tLS1CRUdJTi[A-Za-z0-9+/=]{60,}")
  586. # Private-key formats that are not PEM. All three reach the model on their own: PGP's
  587. # armour carries a \` BLOCK\` suffix so the PEM pattern never matched it, PuTTY's .ppk is
  588. # base64 under a \`Private-Lines:\` header (the usual SSH key format on Windows), and age
  589. # writes a bech32 AGE-SECRET-KEY-1… string.
  590. _PGP_PRIVATE_KEY_RE = re.compile(
  591. r"-----BEGIN PGP PRIVATE KEY BLOCK-----[\s\S]*?-----END PGP PRIVATE KEY BLOCK-----"
  592. )
  593. # Line ends may be CRLF (Windows writes .ppk that way) or a JSON-escaped ``\\n``.
  594. _PPK_EOL = r"(?:\r?\n|\\r\\n|\\n)"
  595. # Body lines may carry read_file's `` 4|`` line-number gutter.
  596. _PPK_BODY_GUTTER = r"[ \t]*+(?:[0-9]++(?:[|:]|\t)[ \t]*+)?"
  597. _PUTTY_PRIVATE_LINES_RE = re.compile(
  598. rf"(?<![\w-])(Private-Lines:[ \t]*+[0-9]++[ \t]*+(?P<eol>{_PPK_EOL}))"
  599. rf"(?:{_PPK_BODY_GUTTER}[A-Za-z0-9+/=]{{16,}}+[ \t]*+{_PPK_EOL}?)++"
  600. )
  601. _AGE_SECRET_KEY_RE = re.compile(r"AGE-SECRET-KEY-1[0-9A-Z]{20,}")
  602. def _is_b64_pem_key(blob: str) -> bool:
  603. """True when ``blob`` is base64 of a PEM private key block."""
  604. import base64
  605. import binascii
  606. padded = blob + "=" * (-len(blob) % 4)
  607. try:
  608. decoded = base64.b64decode(padded, validate=False)
  609. except (binascii.Error, ValueError):
  610. return False
  611. text = decoded[:64]
  612. return text.startswith(b"-----BEGIN") and b"PRIVATE KEY" in text
  613. # Database connection strings: protocol://user:PASSWORD@host. The userinfo and
  614. # password groups forbid whitespace so a match can never span a line break (a
  615. # greedy ``[^@]+`` once ran to a decorator's ``@`` on the next code line).
  616. # Database connection strings: protocol://user:PASSWORD@host Catches postgres, mysql, mongodb, redis, amqp
  617. # URLs and redacts the password. A real DSN password never contains whitespace; without this bound the
  618. # greedy [^@]+ would scan past the end of a code line to the next stray "@" (e.g. a Python decorator),
  619. # swallowing intervening lines and corrupting tool OUTPUT for any source containing a postgresql:// f-string
  620. # template. See issue #33801.
  621. _DB_CONNSTR_RE = re.compile(
  622. r"((?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|rediss?|amqps?)://[^:\s]*+:)((?:[^@\s:]++|:(?!//))++)(@)",
  623. re.IGNORECASE,
  624. )
  625. # Bare-token URL credential ``scheme://TOKEN@host`` (``git remote set-url
  626. # https://PASSWORD@github.com/...``): unambiguously a secret, since round-trip
  627. # URLs (OAuth callbacks, magic links) carry tokens in the QUERY STRING, never
  628. # bare userinfo. ``user:pass@`` passes through (class forbids ``:``); DB schemes
  629. # belong to _DB_CONNSTR_RE. 8+ char floor skips short usernames; the class
  630. # forbids ``/`` so an ``@`` in a path/query (``?q=user@example.com``) never counts.
  631. # The class also forbids ``"``: a double quote is not URL userinfo, and
  632. # ``https://schema.org","@type`` is a JSON string boundary, not a credential
  633. # (src#903). A comma stays legal userinfo.
  634. # This is the ``git remote set-url origin https://PASSWORD@github.com/...`` shape from issue #6396 — a
  635. # single opaque credential in the userinfo position with NO ``user:pass`` colon. The colon form's
  636. # PASSWORD is masked separately by _redact_userinfo_passwords (issue #125664) — #34029's pass-through
  637. # was about query-string round-trip tokens (magic links / OAuth callbacks / pre-signed URLs live in
  638. # the query, never in userinfo), and a userinfo password is a credential, not a workflow token. DB
  639. # schemes are handled by _DB_CONNSTR_RE above and excluded here. Guards against false positives:
  640. _URL_BARE_TOKEN_RE = re.compile(
  641. r"((?:https?|wss?|git|ssh|ftp|ftps|sftp)://)" # scheme
  642. r"([^\s:@/\"]{8,})" # bare token, 8+ chars; no quote
  643. r"(@[^\s]+)", # @host...
  644. re.IGNORECASE,
  645. )
  646. # JWTs always start with "eyJ" (base64 "{"); 1-, 2- and 3-part forms.
  647. _JWT_RE = re.compile(r"eyJ[A-Za-z0-9_-]{10,}(?:\.[A-Za-z0-9_=-]{4,}){0,2}")
  648. # Links ``hermes webapp`` prints that sign a browser in as the user: the gateway's browser credential
  649. # after ``hermes-session=`` and the one-use ticket of a ``/webapp/launch`` link. Both values are
  650. # URL-safe base64 (43 characters); the 20-character floor leaves placeholders and code (``${token}``).
  651. _WEBAPP_LINK_CREDENTIAL_RE = re.compile(
  652. r"((?<![A-Za-z0-9_-])hermes-session=|/webapp/launch#(?:[^\s#]*?&)?ticket=)[A-Za-z0-9_-]{20,}")
  653. # E.164 phone numbers, 7-15 digits; the lookahead rejects hex strings / identifiers.
  654. _SIGNAL_PHONE_RE = re.compile(r"(\+[1-9]\d{6,14})(?![A-Za-z0-9])")
  655. # CDP-URL path: web URLs with a query string / with ``user:password@`` userinfo
  656. # (DB protocols are covered by _DB_CONNSTR_RE).
  657. _URL_WITH_QUERY_RE = re.compile(r"(https?|wss?|ftp)://([^\s/?#]+)([^\s?#]*)\?([^\s#]+)(#\S*)?")
  658. _URL_USERINFO_RE = re.compile(r"(https?|wss?|ftp)://([^/\s:@]+):([^/\s@]+)@")
  659. # Strict provider-egress URL redaction: delimiters stay in capture groups so the
  660. # query/fragment layout is preserved byte-for-byte; the key is decoded
  661. # separately for classification. Values stop at ``&``/``;`` (both valid).
  662. _STRICT_URL_PARAM_RE = re.compile(r"([?#&;])([A-Za-z0-9_.~+%\-]+)=([^#&;\s\"'<>]*)")
  663. # Userinfo in absolute and network-path (``//user:pass@host``) references; the
  664. # authority stops at path/query/fragment delimiters. Anchored on the mandatory
  665. # ``//`` — an optional-scheme prefix backtracked O(n²) on long alphanumeric runs
  666. # (~55s per sub() on a 320KB compaction payload).
  667. # No ``"``: ``"https://a.com","u:pw@y"`` is two JSON strings, not ``a.com","u`` userinfo.
  668. _STRICT_URL_USERINFO_RE = re.compile(r"//[^/\s?#@\"]+@")
  669. # The same authority span with the delimiter and userinfo captured, for _redact_userinfo_passwords.
  670. _URL_USERINFO_PAIR_RE = re.compile(r"(//)([^/\s?#@\"]+)@")
  671. # Scheme-less network path (``mount //user:pass@host/share``), masked under force only: ``//`` must
  672. # start a token (comments are ``// note``, protocol-relative URLs carry no ``@``) and the password
  673. # needs 8+ characters, so ``//user:note@x`` stays as written.
  674. _NETWORK_PATH_USERINFO_RE = re.compile(r"(?<![^\s\"'=(`,\[])(//)([^/\s?#@:\"]+):([^/\s?#@\"]{8,})@")
  675. # ``NAME=user:pass`` where NAME carries no secret keyword — an HTTP Basic auth pair is often stored
  676. # this way (#125664). Shape-only on arbitrary names, so it runs force-only (persistence boundary,
  677. # debug dumps, session export all pass force=True): ordinary prose/config keeps today's behaviour.
  678. # Guards against non-secret colon values: the password needs an 8-char floor AND a letter (ports and
  679. # digit-only tags fail it), the password group is a single token (no whitespace, quotes or ``/`` —
  680. # so ``registry.example.com:5000/repo`` keeps its path, and an ISO timestamp's ``:00:00`` tail
  681. # truncates the group to digits), and a version-shaped tail (``24.04-noble``, ``v1.2.3-alpine``)
  682. # is rejected outright. The value is the WHOLE assignment (``user`` cannot contain ``:``/``@``/``/``),
  683. # and the ``:(?!//)`` lookahead keeps ``scheme://`` values (``MY_URL=https://example.com``) out.
  684. # ``user`` must also be at least 2 characters: a single-letter ``user:`` is a Windows drive
  685. # (``SRC=C:\Users\...``), and masking the whole path as ``SRC=C:***`` at a force boundary is
  686. # unrecoverable data loss — no real Basic pair has a one-letter user, and a secret-keyword NAME
  687. # still masks via the ENV pass (#125680 review).
  688. # ``@`` is not a password character here either: a ``user:pass@host``-shaped value belongs to the
  689. # userinfo pass below, which knows the f-string ``{...}`` template exemption; excluding it lets that
  690. # pass see the pure brace expression (``BASIC="user:{pw}@host"`` keeps its template verbatim).
  691. _BASIC_PAIR_ASSIGN_RE = re.compile(
  692. r"(?<![A-Za-z0-9_.\-])([A-Za-z][A-Za-z0-9_.\-]{0,63})=([\"']?)"
  693. r"([A-Za-z0-9._~\-]{2,})(?::(?!//)([^\s'\"/:@]+))\2(?![^\s;,&|)\]}\"'`])"
  694. )
  695. # ``24.04-noble`` / ``v1.2.3-alpine`` image tags: digit-led dotted versions, optionally ``v``-prefixed
  696. # — long and letter-bearing enough to pass the floor, yet never a credential.
  697. _VERSION_TAG_RE = re.compile(r"^[vV]?\d+(\.\d+)+")
  698. # ``nginx:stable-alpine`` / ``myrepo:latest-release``: lower-case words joined by ``-``/``.``
  699. # are image tags and labels; a password carries a digit, a capital or a symbol.
  700. _WORD_TAG_RE = re.compile(r"[a-z]++(?:[.\-][a-z]++)*+")
  701. # ``CHECKSUM=sha256:1234…`` digest labels: the hex body is long and letter-bearing, but a hash
  702. # algorithm name as the ``user`` half marks the pair as an algorithm:digest, not a credential
  703. # (#125680 review). No real Basic pair has one of these as its username.
  704. _HASH_ALGO_NAMES = frozenset({
  705. "md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha3-224", "sha3-256",
  706. "sha3-384", "sha3-512", "blake2b", "blake2s", "ripemd160", "crc32", "crc32c",
  707. })
  708. # ``NAME=user:pass`` is only a Basic pair when NAME says it holds one: ``IMAGE=redis:7-alpine``,
  709. # ``BRANCH=origin:feature2``, ``LS_COLORS=...:mh=00:...`` and ``Server=tcp:db,1433`` are colon
  710. # values under names that never carry credentials (#149435 review).
  711. _BASIC_PAIR_NAME_RE = re.compile(r"auth|basic|cred|login|proxy|userpass|userinfo", re.IGNORECASE)
  712. def _is_basic_pair_password(password: str) -> bool:
  713. """8+ chars with a letter and not a version-shaped tail — a secret, not a tag or timestamp."""
  714. if _VERSION_TAG_RE.search(password) or _WORD_TAG_RE.fullmatch(password):
  715. return False
  716. return len(password) >= 8 and any(c.isalpha() for c in password)
  717. # Form-urlencoded body: only when the ENTIRE text is a k=v&k=v string.
  718. _FORM_BODY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*=[^&\s]*(?:&[A-Za-z_][A-Za-z0-9_.-]*=[^&\s]*)+$")
  719. # Control / zero-width characters that can split a token body (``sk-abc\x1bdef``,
  720. # ``ghp_abc\n123``) and escape the contiguous prefix regexes.
  721. _CONTROL_CHARS_RE = re.compile(r"[\x00-\x1f\x7f\u200b-\u200f\u2028-\u202f\u2060\ufeff]")
  722. # Union of every _PREFIX_PATTERNS body class: a control-stripped match may only
  723. # span token-body or control chars. ``=`` is excluded so a KEY=value separator
  724. # never lets a match span unrelated text.
  725. _TOKEN_BODY_CHARS = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-.")
  726. def _compile_prefix_matcher(patterns: list) -> re.Pattern[str]:
  727. return re.compile(r"(?<![A-Za-z0-9_-])(" + "|".join(patterns) + r")(?![A-Za-z0-9_-])")
  728. def _compile_prefix_sub_matcher(patterns: list) -> re.Pattern[str]:
  729. """_PREFIX_RE for the masking pass, also admitting a key glued to an identifier by ``-``/``_``
  730. (``backup-sk-…``, ``x-ghp_…``), which _PREFIX_RE rejects so ``scikit-learn`` stays whole. One
  731. fixed-width lookbehind keeps main's single scan; _mask_prefix_match masks a glued match only
  732. when its body is opaque, so ``backup-sk-learn`` stays whole too (#147240)."""
  733. # Branches grouped by their literal first char (every pattern opens on one, see
  734. # register_redaction_patterns): admitting ``-``/``_`` before a key makes every separator a
  735. # candidate, and a flat 60-way alternation per candidate doubled the cost of a ``_ghp_`` flood.
  736. # Groups are disjoint by first char, so leftmost-first match order is unchanged.
  737. by_first: dict = {}
  738. for pattern in patterns:
  739. by_first.setdefault(pattern[0], []).append(pattern[1:])
  740. body = "|".join(f"{re.escape(c)}(?:{'|'.join(rest)})" for c, rest in by_first.items())
  741. return re.compile(r"(?<![A-Za-z0-9])(" + body + r")(?![A-Za-z0-9_-])")
  742. _PREFIX_RE = _compile_prefix_matcher(_PREFIX_PATTERNS)
  743. _PREFIX_SUB_RE = _compile_prefix_sub_matcher(_PREFIX_PATTERNS)
  744. # A real key body has a 16+ char run with no separator mixing two of lower/upper/digit;
  745. # ``sk-prod-deployment-01`` and ``fc-feature-flags-v2`` do not.
  746. _LONG_ALNUM_RUN_RE = re.compile(r"[A-Za-z0-9]{16,}+")
  747. def _is_opaque_prefixed_token(token: str) -> bool:
  748. for run in _LONG_ALNUM_RUN_RE.finditer(token):
  749. run = run.group(0)
  750. if (run.isalpha() and (run.islower() or run.isupper())) or run.isdigit():
  751. continue
  752. return True
  753. return False
  754. def _mask_prefix_match(m: "re.Match[str]", mask) -> str:
  755. glued = m.start() > 0 and m.string[m.start() - 1] in "-_"
  756. return m.group(0) if glued and not _is_opaque_prefixed_token(m.group(1)) else mask(m.group(1))
  757. # Zhipu API keys use an unprefixed ``id.secret`` form. Keep this deliberately
  758. # provider-shaped instead of applying a generic high-entropy dotted-token rule:
  759. # the ID is exactly 32 lowercase hex chars and the credential suffix is a run of
  760. # at least 16 alphanumerics, so content-hash filenames (``<sha>.bundle``,
  761. # ``<md5>.sqlite3``) never match.
  762. _ZHIPU_API_KEY_RE = re.compile(
  763. r"(?<![A-Za-z0-9_.-])([0-9a-f]{32}\.[A-Za-z0-9]{16,})(?![A-Za-z0-9_.-])"
  764. )
  765. def _mask_control_split_tokens(text: str, mask_fn) -> str:
  766. """Mask tokens whose body is split by control/zero-width characters.
  767. Match on a control-stripped copy, then mask the corresponding span in the
  768. ORIGINAL — only when that span holds solely token-body and control chars, so
  769. a match can never cross into another line's unrelated text.
  770. A credential like ``sk-abc\\x1bdef456…`` or ``ghp_abc\\n123def…`` has its token body interrupted, so the
  771. contiguous _PREFIX_RE cannot match it and the secret leaks verbatim (issue #77484). ``EXA_API_KEY=*** is
  772. rejected).
  773. """
  774. stripped = _CONTROL_CHARS_RE.sub("", text)
  775. if stripped == text:
  776. return text
  777. orig_idx = [i for i, c in enumerate(text) if not _CONTROL_CHARS_RE.match(c)]
  778. out, matches = list(text), []
  779. for m in _PREFIX_RE.finditer(stripped):
  780. start_orig = orig_idx[m.start(1)]
  781. end_orig = orig_idx[m.end(1) - 1] + 1
  782. span = text[start_orig:end_orig]
  783. # Self-matching fragment AND a span crossing a LINE boundary: do NOT join
  784. # (``ghp_<tok>\nbutton`` would mask ``button``; the prefix pass handles the
  785. # fragment). Non-newline controls (ESC, ZWSP) never legitimately sit
  786. # between a token and prose, so there the join proceeds regardless.
  787. if ("\n" in span or "\r" in span) and _PREFIX_RE.search(span):
  788. continue
  789. # Reject spans containing a non-token char (``sk_abc…\nTAVILY_API_KEY=``
  790. # matched across lines) and matches running into a ``KEY=`` name.
  791. if (all(c in _TOKEN_BODY_CHARS or _CONTROL_CHARS_RE.match(c) for c in span)
  792. and (end_orig >= len(text) or text[end_orig] != "=")):
  793. matches.append((start_orig, end_orig, mask_fn(m.group(1))))
  794. for start_orig, end_orig, replacement in reversed(matches):
  795. out[start_orig:end_orig] = list(replacement)
  796. return "".join(out)
  797. # mask_secret strips EVERY control char (incl. \n/\t, C1, DEL, zero-width) so a
  798. # masked secret never emits multiline or invisible bytes into display output.
  799. _DISPLAY_CONTROL_RE = re.compile(r"[\x00-\x1f\x7f\x80-\x9f\u200b-\u200f\u202a-\u202e\u2060-\u2064]")
  800. def mask_secret(value: str, *, head: int = 4, tail: int = 4, floor: int = 12,
  801. placeholder: str = "***", empty: str = "") -> str:
  802. """Mask a secret for display (``hermes config`` / ``status`` / ``dump``):
  803. ``sk-p...7890``; shorter than ``floor`` (after control-byte stripping) →
  804. ``placeholder``; falsy → ``empty``."""
  805. value = _DISPLAY_CONTROL_RE.sub("", value) if value else value
  806. if not value:
  807. return empty
  808. return placeholder if len(value) < floor else f"{value[:head]}...{value[-tail:]}"
  809. # Approval-prompt display integrity: the copy a human approves must show everything that will run.
  810. # Code points that render as nothing or as a line break become visible escapes: every control,
  811. # format, private-use, unassigned and surrogate code point (category C*, except tab/newline), line
  812. # and paragraph separators (Zl/Zp), and blank-rendering marks/letters in other categories (variation
  813. # selectors incl. U+E0100-E01EF, CGJ, Khmer/Mongolian selectors, Hangul fillers, Braille blank).
  814. # Ordinary non-ASCII text is untouched. Padding that pushes the tail off screen or past a preview
  815. # cut (space runs, whitespace-only lines) collapses to a marker; real indentation is kept.
  816. _APPROVAL_NON_ASCII_RE = re.compile(r"[^\x20-\x7e\t\n]+")
  817. _APPROVAL_BLANK_GLYPHS = frozenset(
  818. "\u034f\u115f\u1160\u17b4\u17b5\u180b\u180c\u180d\u180f\u2800\u3164\uffa0"
  819. + "".join(map(chr, range(0xFE00, 0xFE10))) + "".join(map(chr, range(0xE0100, 0xE01F0))))
  820. _APPROVAL_SPACE = r"[ \t\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]"
  821. # Mid-line runs (after visible text) of 21+, or line-leading runs past any sane indentation.
  822. _APPROVAL_PAD_RE = re.compile(rf"(?<=\S){_APPROVAL_SPACE}{{21,}}|(?m:^){_APPROVAL_SPACE}{{65,}}")
  823. # 3+ consecutive blank or whitespace-only lines; the match ends at a newline so the next line keeps its indent.
  824. _APPROVAL_BLANK_RE = re.compile(rf"\n(?:{_APPROVAL_SPACE}*\n){{3,}}")
  825. _APPROVAL_SAME_CHAR_RUN_RE = re.compile(r"(.)\1*", re.DOTALL)
  826. # Prose mode (questions shown to a human, never executed) surfaces only what can hide or reorder
  827. # text: controls, bidi embeddings/overrides/isolates, line/paragraph separators and padding. ZWJ,
  828. # ZWNJ and variation selectors stay, because emoji and scripts such as Persian need them.
  829. _PROSE_BIDI_CONTROLS = frozenset("\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
  830. def _escape_approval_run(run: str, prose: bool = False) -> str:
  831. """Escape the invisible code points of a non-ASCII run; 4+ repeats of one show once with their count."""
  832. def one(m: re.Match) -> str:
  833. ch, n = m.group(1), len(m.group())
  834. cat = unicodedata.category(ch)
  835. if prose:
  836. hidden = cat == "Cc" or cat in ("Zl", "Zp") or ch in _PROSE_BIDI_CONTROLS
  837. else:
  838. hidden = ch in _APPROVAL_BLANK_GLYPHS or cat[0] == "C" or cat in ("Zl", "Zp")
  839. if not hidden:
  840. return m.group()
  841. esc = ch.encode("unicode_escape").decode("ascii")
  842. return f"⟨{esc} ×{n}⟩" if n >= 4 else esc * n
  843. return _APPROVAL_SAME_CHAR_RUN_RE.sub(one, run)
  844. def sanitize_command_for_display(text: str | None, *, prose: bool = False) -> str:
  845. """Make hidden content in an approval-prompt string visible; clean text comes back unchanged.
  846. Display-only: the executed command, smart assessment and allowlist keys never see this copy.
  847. ``prose=True`` is the lighter pass for question text (see ``_PROSE_BIDI_CONTROLS``)."""
  848. if not text:
  849. return ""
  850. out = _APPROVAL_NON_ASCII_RE.sub(lambda m: _escape_approval_run(m.group(), prose), text)
  851. out = _APPROVAL_BLANK_RE.sub(lambda m: f"\n⟨+{m.group().count(chr(10)) - 1} blank lines⟩\n", out)
  852. return _APPROVAL_PAD_RE.sub(lambda m: f" ⟨+{len(m.group())} whitespace chars⟩ ", out)
  853. def redact_approval_display(text: str | None, *, force: bool = False, prose: bool = False) -> str:
  854. """Secret redaction plus :func:`sanitize_command_for_display`, for every approval surface."""
  855. return sanitize_command_for_display(redact_sensitive_text(str(text or ""), force=force), prose=prose)
  856. def _mask_token(token: str) -> str:
  857. """Mask a log token — 18-char floor, preserves 6 prefix / 4 suffix; empty → ``***``."""
  858. if not token:
  859. return "***"
  860. return mask_secret(token, head=6, tail=4, floor=18)
  861. # Output of an earlier pass: masking it again would erase the kind it keeps visible. Only a
  862. # value that IS a marker is skipped; ``***MyRealPass9`` or ``"«redacted:ghp_…» tail"`` still
  863. # carries cleartext. An unquoted ``[REDACTED PRIVATE KEY]`` reaches here cut at its space.
  864. _REDACTION_MARKER_RE = re.compile(r"\*{3}|«redacted(?::[^»\s]*|-secret)»|\[REDACTED(?: PRIVATE KEY\])?")
  865. def _is_redaction_marker(match: "re.Match[str]", value: str) -> bool:
  866. if value == "[REDACTED":
  867. return match.string.startswith(" PRIVATE KEY]", match.end("value"))
  868. return _REDACTION_MARKER_RE.fullmatch(value) is not None
  869. def _mask_flag_value(match: "re.Match[str]", mask=None) -> str:
  870. """Mask the value of a flag/connection-string key, keeping the flag and any quotes.
  871. ``mask`` is the caller's mask function: ``file_read`` passes the non-reusable sentinel,
  872. so a file read never hands back a head/tail value shaped like a usable key (#35519)."""
  873. value = match.group("value")
  874. stripped = value.strip("'\"")
  875. if not stripped or _is_redaction_marker(match, stripped.strip()):
  876. return match.group(0)
  877. # ``--password=$DB_PASS`` / ``--token=${TOKEN}`` name a variable; the reference is
  878. # not the credential and masking it would make the command unreplayable.
  879. if stripped[0] in "$<{%":
  880. return match.group(0)
  881. if stripped.isdigit():
  882. return match.group(0) # ``--token 60`` is a timeout, not a credential
  883. if match.re is _CLI_SECRET_FLAG_RE:
  884. if "=" not in match.group("sep") and _is_flag_placeholder(value):
  885. return match.group(0)
  886. if _is_flag_path_reference(match.group("flag"), stripped):
  887. return match.group(0)
  888. masked = (mask or _mask_token)(stripped)
  889. start = match.start("value") - match.start(0)
  890. end = match.end("value") - match.start(0)
  891. return match.group(0)[:start] + masked + match.group(0)[end:]
  892. # A space-separated flag is followed by whatever comes next, which in ``--help`` output and
  893. # prose is a placeholder or a word (``--token TOKEN``, ``[--password PASSWORD]``, ``use
  894. # --token to pass…``), the next flag (``psql --password -h db``) or a sub-option spec
  895. # (``docker build --secret id=npmrc``). None of those is the credential.
  896. # A metavar is upper case (``TOKEN``, ``DB_PASSWORD``), two short snake-case words
  897. # (``db_name``) or a one-hump camelCase type name (cobra's ``stringArray``); a prose word
  898. # is short and lower case. A passphrase is longer than that: a long lower-case word
  899. # (``correcthorsebatterystaple``), three or more words (``correct_horse_battery_staple``,
  900. # ``correctHorseBatteryStaple``) or anything carrying a digit or symbol is still masked.
  901. _FLAG_PLACEHOLDER_RE = re.compile(
  902. r"(?:[A-Z][A-Z_]*|[a-z]{1,8}|[a-z]{1,8}_[a-z]{1,8}|[a-z]{1,8}[A-Z][a-z]{1,8})[\]).,:;]*"
  903. r"|[-\[(<].*|[a-z][\w-]*=.*"
  904. r"|\S*(?:xxx|XXX|\.\.\.)" # a docs sample: ``--api-key sk-or-v1-xxx``
  905. )
  906. def _is_flag_placeholder(value: str) -> bool:
  907. """True for an unquoted space-separated flag value that is a word, not a credential."""
  908. return value[0] not in "'\"" and _FLAG_PLACEHOLDER_RE.fullmatch(value) is not None
  909. def _is_flag_path_reference(flag: str, value: str) -> bool:
  910. """``--credentials ~/.config/gcloud/creds.json`` names a file, same rule as the ENV passes:
  911. only password-class flags mask a path, and only while no segment looks opaque."""
  912. if _PASSWORD_KEY_RE.search(flag) or not value.startswith(("/", "~", "./", "../")):
  913. return False
  914. return not any(_OPAQUE_PATH_SEGMENT_RE.fullmatch(seg) for seg in value.split("/"))
  915. def _redact_cli_secrets(text: str, mask=None) -> str:
  916. """Mask credentials written as command-line flags and connection-string keys."""
  917. def sub(m: "re.Match[str]") -> str:
  918. return _mask_flag_value(m, mask)
  919. if "=" in text or " " in text:
  920. if _CLI_SECRET_FLAG_GATE.search(text):
  921. text = _CLI_SECRET_FLAG_RE.sub(sub, text)
  922. if _CLI_LOGIN_GATE.search(text):
  923. text = _CLI_LOGIN_P_RE.sub(sub, text)
  924. if _CONNSTR_GATE.search(text):
  925. text = _CONNSTR_SECRET_KEY_RE.sub(sub, text)
  926. return text
  927. def _redact_private_key_blocks(text: str) -> str:
  928. """Whole private-key blocks (PEM, base64 PEM, PGP, PuTTY, age). Runs before the
  929. flag/connection-string passes: those stop a value at whitespace, so masking
  930. ``AccountKey=-----BEGIN`` first would break the PEM header and leave the body visible."""
  931. if "-----BEGIN" in text and "-----END" in text: # no END, no match: skip the quadratic scan
  932. text = _PRIVATE_KEY_RE.sub("[REDACTED PRIVATE KEY]", text)
  933. if "LS0tLS1CRUdJTi" in text:
  934. text = _B64_PEM_RE.sub(
  935. lambda m: "[REDACTED PRIVATE KEY]" if _is_b64_pem_key(m.group(0)) else m.group(0),
  936. text,
  937. )
  938. if "END PGP PRIVATE KEY BLOCK" in text: # no END, no match: skip the quadratic scan
  939. text = _PGP_PRIVATE_KEY_RE.sub("[REDACTED PRIVATE KEY]", text)
  940. if "Private-Lines:" in text:
  941. text = _PUTTY_PRIVATE_LINES_RE.sub(
  942. lambda m: m.group(1) + "[REDACTED PRIVATE KEY]" + m.group("eol"), text
  943. )
  944. if "AGE-SECRET-KEY-1" in text:
  945. text = _AGE_SECRET_KEY_RE.sub("[REDACTED PRIVATE KEY]", text)
  946. return text
  947. def _is_python_repr_secret_key(key: str) -> bool:
  948. """Return True for exact secret keys or credential-suffixed key names."""
  949. folded = key.casefold()
  950. if folded in _PYTHON_REPR_SECRET_KEYS:
  951. return True
  952. if key.isupper() and key.endswith(_PYTHON_REPR_ENV_SUFFIXES):
  953. return True
  954. # Mixed/camel-case keys ending in a credential word (``UserPassword``,
  955. # ``sessionToken``, ``clientApiKey``) — the exact-set and uppercase-suffix
  956. # rules above miss these. Suffix-only matching keeps metadata names like
  957. # ``TOKEN_COUNT`` / ``PASSWORD_POLICY`` / ``SECRET_NAME`` untouched.
  958. # Class widened per OpenHands/software-agent-sdk#4508 (their dict-entry
  959. # redaction was uppercase-only and leaked mixed-case keys).
  960. if folded.endswith(_PYTHON_REPR_CREDENTIAL_SUFFIXES):
  961. return True
  962. # Names ending in ``key`` where a qualifier says which kind of key it is:
  963. # ``aws_secret_access_key``, ``SECRET_KEY``, ``privateKey``, ``signing_key``,
  964. # ``encryption_key``, ``master_key``, ``hmac_key``. A bare ``key``, ``public_key``,
  965. # ``hotkey`` and ``key_id`` stay untouched — they name cache keys, limits and ids.
  966. return _is_qualified_key_name(key)
  967. # A qualified key masks short values too, except a sample or switch that cannot be the
  968. # credential: ``{"encryptKey": "true"}``, ``master_key: default``, ``"<your key>"``. Only a
  969. # word-shaped bracket is a docs sample; ``<hunter2>`` / ``<Abc9!xyz>`` are still masked.
  970. _CONFIG_PLACEHOLDER_VALUE_RE = re.compile(
  971. r"<[a-z]+(?:[ _-][a-z]+)+>|<(?:key|token|secret|password|value)>|true|false|yes|no|null|none|default",
  972. re.IGNORECASE)
  973. _KEY_SECRET_QUALIFIERS = frozenset({
  974. "secret", "private", "signing", "encryption", "encrypt", "master", "hmac",
  975. "symmetric", "access",
  976. })
  977. def _key_name_words(key: str) -> list[str]:
  978. """Split a field name into lower-case words: ``aws_secret_access_key`` and
  979. ``awsSecretAccessKey`` both give ``[aws, secret, access, key]``."""
  980. spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", key)
  981. return [w for w in re.split(r"[^A-Za-z0-9]+", spaced.lower()) if w]
  982. def _is_qualified_key_name(key: str) -> bool:
  983. """True for a ``…key`` field name whose qualifier makes it a credential.
  984. ``access`` is in the qualifier set because AWS pairs an access-key id with the
  985. secret that signs with it, and the id alone still identifies an account; it is
  986. never a value safe to print in a chat approval card.
  987. """
  988. words = _key_name_words(key)
  989. if len(words) < 2 or words[-1] != "key":
  990. return False
  991. return any(w in _KEY_SECRET_QUALIFIERS for w in words[:-1])
  992. def is_secret_field_name(key: object) -> bool:
  993. """True when a mapping field named ``key`` holds a credential — the repr-field policy, for callers
  994. that mask structured config by field instead of by text."""
  995. return isinstance(key, str) and _is_python_repr_secret_key(key)
  996. def _redact_python_repr_fields(text: str) -> str:
  997. """Fully mask credential fields in Python mapping ``repr`` output."""
  998. def _sub(match: re.Match) -> str:
  999. key = match.group("key")
  1000. if not _is_python_repr_secret_key(key):
  1001. return match.group(0)
  1002. single_value = match.group("single_value")
  1003. if single_value is not None:
  1004. prefix = match.group("single_prefix") or ""
  1005. quote = "'"
  1006. value = single_value
  1007. else:
  1008. prefix = match.group("double_prefix") or ""
  1009. quote = '"'
  1010. value = match.group("double_value")
  1011. # Mapping repr can contain code-shaped fixture values too. Preserve
  1012. # programmatic env lookups just like the ENV/JSON/YAML passes do.
  1013. if _ENV_LOOKUP_VALUE_RE.match(value):
  1014. return match.group(0)
  1015. # An upstream pass (MCP probe header scrub, _mask_token) already masked this
  1016. # value; re-masking would erase the scheme word it deliberately kept
  1017. # (``'Authorization': 'Digest ***'`` → ``'***'``).
  1018. if "***" in value or value.startswith("«redacted:"):
  1019. return match.group(0)
  1020. # Do not retain head/tail characters here: escaped repr atoms can cross
  1021. # a slicing boundary and leave an unescaped quote behind. A full mask is
  1022. # parseable for both str and bytes values and leaks no opaque bytes.
  1023. return f"'{key}'{match.group('sep')}{prefix}{quote}***{quote}"
  1024. return _PYTHON_REPR_FIELD_RE.sub(_sub, text)
  1025. def _redact_python_diagnostic_repr_fields(text: str) -> str:
  1026. """Mask repr fields only on pytest/error lines in source-preserving output."""
  1027. lines = text.splitlines(keepends=True)
  1028. for index, line in enumerate(lines):
  1029. ending = ""
  1030. body_line = line
  1031. if line.endswith("\r\n"):
  1032. body_line, ending = line[:-2], "\r\n"
  1033. elif line.endswith("\n") or line.endswith("\r"):
  1034. body_line, ending = line[:-1], line[-1:]
  1035. match = _PYTEST_DIAGNOSTIC_LINE_RE.match(body_line)
  1036. if match is None:
  1037. match = _PYTHON_EXCEPTION_LINE_RE.match(body_line)
  1038. if match is not None:
  1039. lines[index] = (
  1040. match.group("prefix")
  1041. + _redact_python_repr_fields(match.group("body"))
  1042. + ending
  1043. )
  1044. return "".join(lines)
  1045. def _redact_query_string(query: str) -> str:
  1046. """Replace values of sensitive ``k=v&k=v`` params with ``***``; others pass through."""
  1047. if not query:
  1048. return query
  1049. return "&".join(
  1050. f"{key}=***" if sep and key.lower() in _SENSITIVE_QUERY_PARAMS else pair
  1051. for pair in query.split("&") for key, sep, _ in (pair.partition("="),)
  1052. )
  1053. def _canonical_url_param_name(name: str) -> str:
  1054. """Decode a URL parameter name (up to 3 unquote rounds) for case-insensitive matching."""
  1055. decoded = name
  1056. for _ in range(3):
  1057. next_value = unquote_plus(decoded)
  1058. if next_value == decoded:
  1059. break
  1060. decoded = next_value
  1061. folded = decoded.casefold()
  1062. # Preserve policy names that are canonically hyphenated (for example
  1063. # x-amz-signature) before accepting underscore-normalized aliases.
  1064. return folded if folded in _SENSITIVE_QUERY_PARAMS else folded.replace("-", "_")
  1065. def _redact_strict_url_credentials(text: str) -> str:
  1066. """Strict egress-boundary redaction of URL credentials (absolute, relative and
  1067. network references); preserves keys, separators, public params, hosts, paths."""
  1068. text = _STRICT_URL_PARAM_RE.sub(
  1069. lambda m: f"{m.group(1)}{m.group(2)}=***"
  1070. if _canonical_url_param_name(m.group(2)) in _SENSITIVE_QUERY_PARAMS else m.group(0), text)
  1071. return _STRICT_URL_USERINFO_RE.sub("//***:***@", text)
  1072. def redact_cdp_url(value: object) -> str:
  1073. """Mask secrets in a CDP/browser endpoint URL before it is logged.
  1074. Unlike ``redact_sensitive_text`` (which passes web-URL query params and
  1075. ``user:pass@`` through for OAuth callbacks / magic links), CDP discovery
  1076. tokens are pure credentials, so this opts INTO both URL redactors.
  1077. """
  1078. text = redact_sensitive_text("" if value is None else str(value))
  1079. if not text:
  1080. return text
  1081. text = _URL_WITH_QUERY_RE.sub(
  1082. lambda m: f"{m.group(1)}://{m.group(2)}{m.group(3)}?{_redact_query_string(m.group(4))}{m.group(5) or ''}",
  1083. text,
  1084. )
  1085. return _URL_USERINFO_RE.sub(lambda m: f"{m.group(1)}://{m.group(2)}:***@", text)
  1086. def _redact_form_body(text: str) -> str:
  1087. """Redact sensitive values when the ENTIRE text is a clean ``k=v&k=v`` body."""
  1088. if not text or "\n" in text or "&" not in text or not _FORM_BODY_RE.match(text.strip()):
  1089. return text
  1090. return _redact_query_string(text.strip())
  1091. def _mask_token_nonreusable(token: str) -> str:
  1092. """Redact a prefix-matched credential to a NON-REUSABLE sentinel: no head/tail
  1093. chars (an agent once wrote a truncated-looking mask back into a config file),
  1094. only the vendor prefix label so the credential KIND stays visible.
  1095. * cannot be mistaken for a usable-but-truncated key, so an agent that reads it from a config file and
  1096. writes it back does NOT corrupt the stored credential into a dead 13-char string (issue #35519); and *
  1097. still does not leak the secret material (no head/tail chars).
  1098. """
  1099. label = next((sub for sub in _PREFIX_SUBSTRINGS if token.startswith(sub)), "") if token else ""
  1100. return f"«redacted:{label}…»" if label else "«redacted-secret»"
  1101. def _assignment_sub(render, *, check_keyword: bool):
  1102. """re.sub callback: keep the match unless the key/value pair (groups[0], groups[-1]) needs redaction."""
  1103. def _sub(m):
  1104. groups = m.groups()
  1105. if not _should_redact_assignment(groups[0], groups[-1], check_keyword=check_keyword):
  1106. return m.group(0)
  1107. return render(groups)
  1108. return _sub
  1109. _JSON_FLAG_VALUES = frozenset({"true", "false", "yes", "no", "null", "none"})
  1110. _JSON_VALUE_SUFFIX_RE = re.compile(r"[_.\-]?(?:value|Value|VALUE)$")
  1111. def _json_affixed_key_secret(key: str, value: str) -> bool:
  1112. """Whether ``"key": "value"`` holds a credential under an affixed secret-word key (#147240)."""
  1113. if _JSON_KEY_NAMES_FULL_RE.fullmatch(key): # the closed-list pass already decided this field
  1114. return False
  1115. if not (_has_word_bounded_keyword(key, _YAML_CFG_WORD_RE) or _has_password_keyword(key)):
  1116. return False
  1117. if _DSN_PW_PLACEHOLDER_RE.fullmatch(value) or not _should_redact_assignment(key, value, check_keyword=True):
  1118. return False
  1119. # A key that does not END in the credential word is metadata about one: ``passwordChanged``,
  1120. # ``password_expires_at``, ``tokenUrl``, ``secret_id``, ``secret_arn``, ``PASSWORD_STORE_DIR``
  1121. # carry timestamps, URLs and ids that look opaque, so the value shape cannot decide (#149435
  1122. # review). A boolean answers ``has_password``, it does not hold one.
  1123. # ``api_key_value`` / ``passwordValue`` hold the credential itself: drop that one suffix first.
  1124. return _is_python_repr_secret_key(_JSON_VALUE_SUFFIX_RE.sub("", key)) and value.casefold() not in _JSON_FLAG_VALUES
  1125. def _redact_json_affixed_key(m: "re.Match[str]", mask) -> str:
  1126. if not _json_affixed_key_secret(m.group(2), m.group(3)):
  1127. return m.group(0)
  1128. return f'{m.group(1)}: "{mask(m.group(3))}"'
  1129. def _redact_json_affixed_key_escaped(m: "re.Match[str]", mask) -> str:
  1130. if not _json_affixed_key_secret(m.group(3), m.group(4)):
  1131. return m.group(0)
  1132. return f'{m.group(1)}: {m.group(2)}"{mask(m.group(4))}{m.group(2)}"'
  1133. def _redact_assignments(text: str, *, mask_nonreusable: bool = False) -> str:
  1134. """ENV / config / JSON / YAML assignment passes (skipped for code files). Passes
  1135. that would match ``token=``/``key=`` URL params skip ``://`` text (web-URL query
  1136. params are intentionally passed through, see redact_sensitive_text).
  1137. ``mask_nonreusable`` masks the assignments with the ``«redacted:…»`` sentinel that
  1138. ``file_read=True`` already uses for prefix-matched credentials. Without it an agent
  1139. that read a secret-bearing file would hold a head/tail mask shaped like a real but
  1140. truncated key and could write it back as a dead credential (#35519)."""
  1141. mask = _mask_token_nonreusable if mask_nonreusable else _mask_token
  1142. if "=" in text:
  1143. _redact_env = _assignment_sub(lambda g: f"{g[0]}={g[1]}{mask(g[2])}{g[1]}", check_keyword=True)
  1144. text = _ENV_ASSIGN_RE.sub(_redact_env, text)
  1145. if "://" not in text: # lowercase names would match URL params
  1146. # Skip URLs — the query string may contain ``token=``/``key=`` params that are intentionally
  1147. # passed through (see note near the bottom of this function; _redact_strict_url_credentials
  1148. # handles the opt-in case). The uppercase regex above is all-caps-only, so it never matches URL
  1149. # params; the lowercase one would (issue #77484).
  1150. text = _ENV_ASSIGN_LOWER_RE.sub(_redact_env, text)
  1151. # The keyword pre-gate is exact and matters: _CFG_DOTTED_RE backtracks
  1152. # quadratically on long unbroken [A-Za-z0-9_.\-] runs.
  1153. # Lowercase/dotted config keys (issue #16413). Skip URLs entirely — web-URL query params are
  1154. # intentionally passed through (see note near the bottom of this function); _DB_CONNSTR_RE still
  1155. # guards connection-string passwords. Extra gate: every _CFG_*_RE match requires a secret keyword in
  1156. # the key, so a text without any secret keyword cannot match — skipping is exact. This matters
  1157. # because _CFG_DOTTED_RE backtracks quadratically on long unbroken [A-Za-z0-9_.\-] runs (e.g.
  1158. # base64/hex blobs in compaction payloads); the linear keyword scan prevents that pathological path
  1159. # on secret-free text.
  1160. if "://" not in text and _CFG_SECRET_WORD_RE.search(text):
  1161. text = _CFG_DOTTED_RE.sub(_redact_env, text)
  1162. text = _CFG_ANCHORED_RE.sub(_redact_env, text)
  1163. if ":" in text and '"' in text:
  1164. text = _JSON_FIELD_RE.sub(
  1165. _assignment_sub(lambda g: f'{g[0]}: "{mask(g[1])}"', check_keyword=False), text)
  1166. if '\\"' in text:
  1167. text = _JSON_FIELD_ESCAPED_RE.sub(
  1168. _assignment_sub(lambda g: f'{g[0]}: {g[1]}"{mask(g[2])}{g[1]}"', check_keyword=False), text)
  1169. text = _JSON_ANY_KEY_FIELD_RE.sub(lambda m: _redact_json_affixed_key(m, mask), text)
  1170. if '\\"' in text:
  1171. text = _JSON_ANY_KEY_FIELD_ESCAPED_RE.sub(lambda m: _redact_json_affixed_key_escaped(m, mask), text)
  1172. # Python mapping repr fields ({'API_KEY': '…'}): single-quoted, so the JSON rule
  1173. # above never sees them — the traceback / pytest-introspection leak shape.
  1174. if ":" in text and "'" in text:
  1175. text = _redact_python_repr_fields(text)
  1176. # YAML after JSON; unwrap scalars before the shared assignment gate so
  1177. # quoted programmatic env lookups retain the same exemption as bare ones.
  1178. if ":" in text and "://" not in text:
  1179. def _redact_yaml(m):
  1180. key, sep, value = m.groups()
  1181. quote = value[0] if len(value) >= 2 and value[0] in "\"'" and value[-1] == value[0] else ""
  1182. inner = value[1:-1] if quote else value
  1183. # An empty or docs-placeholder scalar (``""``, ``"<your key>"``, ``"${API_KEY}"``) is not
  1184. # a credential; masking it hides what the config expects, as _redact_db already rules.
  1185. if quote and (not inner or _DSN_PW_PLACEHOLDER_RE.fullmatch(inner)
  1186. or (inner.startswith("{{") and inner.endswith("}}"))): # Helm/Jinja template
  1187. return m.group(0)
  1188. if not _should_redact_assignment(key, inner, check_keyword=True):
  1189. return m.group(0)
  1190. return f"{key}{sep}{quote}{mask(inner)}{quote}"
  1191. text = _YAML_ASSIGN_RE.sub(_redact_yaml, text)
  1192. return text
  1193. # A docs placeholder is a lower-case or ALL-CAPS name in angle brackets (``<password>``) or an
  1194. # upper-case env var reference (``${REDIS_PASSWORD}``); ``$ecretPassw0rd`` or ``<Zq8x…>`` is
  1195. # a real password that happens to start with the same character.
  1196. _DSN_PW_PLACEHOLDER_RE = re.compile(r"<(?:[a-z][a-z_ -]*|[A-Z][A-Z_ -]*)>|\$\{?[A-Z_][A-Z0-9_]*\}?|%[A-Z_][A-Z0-9_]*%")
  1197. # ``https://user:password@host`` in docs names the field instead of filling it in.
  1198. _USERINFO_PLACEHOLDER_WORDS = frozenset({"password", "passwd", "pass", "pw", "pwd", "secret", "token"})
  1199. def _is_userinfo_placeholder(password: str) -> bool:
  1200. """A template (``{pw}``, ``{{ pw }}``), docs placeholder or field name, never a literal secret."""
  1201. return ((password.startswith("{") and password.endswith("}"))
  1202. or _DSN_PW_PLACEHOLDER_RE.fullmatch(password) is not None
  1203. or password.casefold() in _USERINFO_PLACEHOLDER_WORDS)
  1204. def _redact_userinfo_passwords(text: str) -> str:
  1205. """Mask the password half of ``scheme://user:pass@host`` userinfo (#125664).
  1206. #34029's pass-through was about query-string round-trip tokens (magic links, OAuth callbacks,
  1207. pre-signed URLs live in the query, never in userinfo); the userinfo password is a credential —
  1208. the same reasoning #6396 used to mask the bare-token form. Keeps scheme, username and host
  1209. visible. Templates and docs samples (``{pw}``, ``${TOKEN}``, ``password``) stay as written."""
  1210. def _sub(m):
  1211. user, sep, password = m.group(2).partition(":")
  1212. # f-string / Jinja templates and docs samples (``user:{pw}@``, ``user:${TOKEN}@``,
  1213. # ``user:password@``) in prose and logs as much as in code files.
  1214. if not sep or not password or _is_userinfo_placeholder(password):
  1215. return m.group(0)
  1216. return f"{m.group(1)}{user}:***@"
  1217. return _URL_USERINFO_PAIR_RE.sub(_sub, text)
  1218. def _redact_url_credentials(text: str, code_file: bool) -> str:
  1219. """DB connection-string passwords, bare-token and ``user:pass`` URL userinfo (``://`` text only)."""
  1220. def _redact_db(m):
  1221. # code_file: a pure ``{...}`` password is an f-string template reference
  1222. # (f"postgresql://{user}:{pass}@{host}"), not a literal credential.
  1223. pw = m.group(2)
  1224. if code_file and pw.startswith("{") and pw.endswith("}"):
  1225. return m.group(0)
  1226. # Password-only URLs in docs: ``redis://:<password>@`` / ``redis://:${REDIS_PASSWORD}@``.
  1227. if m.group(1).endswith("://:") and _DSN_PW_PLACEHOLDER_RE.fullmatch(pw):
  1228. return m.group(0)
  1229. return f"{m.group(1)}***{m.group(3)}"
  1230. text = _DB_CONNSTR_RE.sub(_redact_db, text)
  1231. text = _URL_BARE_TOKEN_RE.sub(lambda m: f"{m.group(1)}{_mask_token(m.group(2))}{m.group(3)}", text)
  1232. return _redact_userinfo_passwords(text)
  1233. def _mask_network_path_userinfo(m) -> str:
  1234. """``//user:pass@host`` without a scheme (#125664 residual); placeholders stay as written."""
  1235. if _is_userinfo_placeholder(m.group(3)):
  1236. return m.group(0)
  1237. return f"{m.group(1)}{m.group(2)}:***@"
  1238. def _redact_phone(m):
  1239. phone = m.group(1)
  1240. keep = 2 if len(phone) <= 8 else 4
  1241. return phone[:keep] + "****" + phone[-keep:]
  1242. def redact_sensitive_text(text: str, *, force: bool = False, code_file: bool = False,
  1243. file_read: bool = False, secret_file: bool = False,
  1244. redact_url_credentials: bool = False) -> str:
  1245. """Apply all redaction patterns to a block of text.
  1246. Safe on any string. Enabled by default (``security.redact_secrets: false``
  1247. disables); ``force=True`` is for safety boundaries that must never return
  1248. raw secrets regardless.
  1249. ``redact_url_credentials=True``: also redact credential-named query params
  1250. and the entire URL userinfo (``***:***@``) — off by default because OAuth-callback /
  1251. magic-link / pre-signed URLs must survive ordinary tool flows unchanged.
  1252. ``code_file=True``: skip the ENV/JSON assignment passes for known source
  1253. code (``MAX_TOKENS=***``, ``"apiKey": "test"`` fixtures). ``file_read=True``
  1254. (implies code_file unless ``secret_file``): prefix-matched credentials become a
  1255. non-reusable sentinel (``«redacted:ghp_…»``) instead of a head/tail mask an agent
  1256. could write back into config.yaml as a dead credential.
  1257. Every regex sits behind a cheap substring gate that its pattern requires,
  1258. so the gates are never false-negative.
  1259. Set code_file=True to also skip the Python-repr mapping pass (``{'API_KEY': '…'}``
  1260. fixtures in source); pytest/exception diagnostic lines get a narrow pass in
  1261. redact_terminal_output instead.
  1262. Set file_read=True for file *content* returned to the agent (read_file / search_files / cat). The old
  1263. mask looked like a real-but-truncated key, so an agent reading it from config.yaml and writing it back
  1264. silently corrupted the stored credential into a dead 13-char value → 401 (issue #35519). The sentinel is
  1265. syntactically invalid as a token, so it can't be mistaken for a usable key or written back as one.
  1266. Set ``secret_file=True`` when the caller has already classified the SOURCE as secret-bearing
  1267. (``_is_secret_file_arg``: command reads on the terminal side, resolved tool paths on the
  1268. file side). It re-enables the ENV/JSON/YAML assignment passes that ``file_read`` would
  1269. otherwise skip, so an opaque prefix-less credential assigned to a credential-shaped key is
  1270. masked instead of passed through in cleartext (issue #110567). With ``file_read=True`` those
  1271. assignments are masked with the non-reusable sentinel, so the #35519 write-back hazard stays
  1272. closed. ``secret_file`` is authoritative: it wins over ``code_file``, so a caller cannot be
  1273. fail-open by setting both. Files that are not secret-bearing (any source file, a project's own
  1274. ``config.yaml``) keep the code_file behaviour: ``MAX_TOKENS: 100`` and ``"apiKey": "test"``
  1275. fixtures are untouched.
  1276. """
  1277. if text is None:
  1278. return None
  1279. text = text if isinstance(text, str) else str(text)
  1280. if not text:
  1281. return text
  1282. # Vault secrets are a hard model-egress boundary: scrubbed regardless of the redact_secrets preference.
  1283. text = redact_registered_vault_values(text)
  1284. if not (force or _redact_enabled()):
  1285. return text
  1286. # ``secret_file`` is authoritative: a caller that classified the source as secret-bearing must not
  1287. # be silently fail-open because another flag (code_file, or file_read implying it) was also set.
  1288. code_file = (code_file or file_read) and not secret_file
  1289. # Control/zero-width chars can split a token body so _PREFIX_RE alone misses it.
  1290. if _has_known_prefix_substring(text):
  1291. _prefix_sub = _mask_token_nonreusable if file_read else _mask_token
  1292. # Control/zero-width chars (\\n, \\r, ESC, U+200B, …) split a token body so _PREFIX_RE cannot match
  1293. # across them — a secret smuggled as ``sk-abc\\x1bdef…`` leaks verbatim (issue #77484). Mask such
  1294. # runs by first matching on a control-stripped copy, then re-masking the corresponding span in the
  1295. # original (the stripped copy and the original are aligned 1:1 for non-control chars).
  1296. text = _mask_control_split_tokens(text, _prefix_sub)
  1297. text = _PREFIX_SUB_RE.sub(lambda m: _mask_prefix_match(m, _prefix_sub), text)
  1298. if "." in text:
  1299. _zhipu_sub = _mask_token_nonreusable if file_read else _mask_token
  1300. text = _ZHIPU_API_KEY_RE.sub(lambda m: _zhipu_sub(m.group(1)), text)
  1301. if not code_file:
  1302. text = _redact_assignments(text, mask_nonreusable=file_read)
  1303. # Flags and connection-string keys carry a credential in command text, so they are not
  1304. # gated on code_file: the shape (``--password=`` / ``;AccountKey=``) is unambiguous, and
  1305. # the surfaces that leaked them (approval cards, chat status) run this pass anyway.
  1306. text = _redact_private_key_blocks(text)
  1307. text = _redact_cli_secrets(text, _mask_token_nonreusable if file_read else None)
  1308. if force and "=" in text:
  1309. # ``NAME=user:pass`` under an innocuous name (#125664): shape-only, so force-only — every
  1310. # force=True caller (persistence boundary, debug dumps, session export) is a safety boundary
  1311. # that must not return raw secrets; ordinary flows keep today's pass-through.
  1312. def _basic_pair(m):
  1313. password = m.group(4)
  1314. if password is None or not _BASIC_PAIR_NAME_RE.search(m.group(1)) \
  1315. or not _is_basic_pair_password(password):
  1316. return m.group(0)
  1317. # ``CHECKSUM=sha256:<hex>``: an algorithm:digest label, not a user:credential pair.
  1318. if m.group(3).lower() in _HASH_ALGO_NAMES:
  1319. return m.group(0)
  1320. # A template or docs placeholder, not a literal credential — the rule
  1321. # _redact_userinfo_passwords applies below.
  1322. if _is_userinfo_placeholder(password):
  1323. return m.group(0)
  1324. return f"{m.group(1)}={m.group(2)}{m.group(3)}:***{m.group(2)}"
  1325. text = _BASIC_PAIR_ASSIGN_RE.sub(_basic_pair, text)
  1326. if "uthorization" in text or "UTHORIZATION" in text: # cheapest gate over every casing
  1327. text = _AUTH_HEADER_RE.sub(lambda m: m.group(1) + (m.group(2) or "") + _mask_token(m.group(3)), text)
  1328. if ":" in text:
  1329. text = _SECRET_HEADER_RE.sub(lambda m: m.group(1) + _mask_token(m.group(2)), text)
  1330. text = _TELEGRAM_RE.sub(lambda m: f"{m.group(1) or ''}{m.group(2)}:***", text)
  1331. # Database connection string passwords. With code_file=True, a password group that is a pure ``{...}``
  1332. # brace expression is an f-string template reference (e.g. f"postgresql://{user}:{pass}@{host}"), not a
  1333. # literal credential — preserve it. Literal passwords are still redacted. The regex forbids whitespace
  1334. # in the password group, so a single-line template's group(2) is exactly the brace expression. See issue
  1335. # #33801.
  1336. if "://" in text:
  1337. text = _redact_url_credentials(text, code_file)
  1338. if force and "//" in text:
  1339. text = _NETWORK_PATH_USERINFO_RE.sub(_mask_network_path_userinfo, text)
  1340. # Keyword/value connection strings (``Host=…;Password=…``, libpq ``host=… password=…``):
  1341. # the forms the URL pass above never sees. Gate: a ``password=``/``pwd=`` key in any casing;
  1342. # the per-line companion-key check in _redact_kv_connection_strings keeps prose out.
  1343. if _KV_CONNSTR_GATE_RE.search(text):
  1344. text = _redact_kv_connection_strings(text, code_file)
  1345. if "eyJ" in text:
  1346. text = _JWT_RE.sub(lambda m: _mask_token(m.group(0)), text)
  1347. # A Webapp sign-in link signs a browser in as the user, so its credential is masked whatever the
  1348. # web-URL policy below.
  1349. if "hermes-session=" in text or "/webapp/launch#" in text:
  1350. text = _WEBAPP_LINK_CREDENTIAL_RE.sub(lambda m: m.group(1) + "***", text)
  1351. if redact_url_credentials: # opt-in; known credential shapes in URLs are caught above
  1352. # NOTE: Web-URL redaction (query params + userinfo + HTTP access-log request targets) is
  1353. # intentionally OFF. Many legitimate workflows pass opaque tokens through query strings — magic-link
  1354. # checkouts, OAuth callbacks the agent is meant to follow, pre-signed share URLs — and
  1355. # blanket-redacting param values by name breaks those skills mid-flow. DB connection-string
  1356. # passwords are still caught by _DB_CONNSTR_RE. The ONE userinfo case still redacted is the
  1357. # colon-less bare-token form ``scheme://TOKEN@host`` (#6396, handled by _URL_BARE_TOKEN_RE in the
  1358. # ``://`` block above): a bare credential in userinfo is never a round-trip workflow token (those
  1359. # live in the query string), so masking it can't break a skill. The ``user:pass@`` form is left to
  1360. # pass through per #34029.
  1361. text = _redact_strict_url_credentials(text)
  1362. if "&" in text and "=" in text:
  1363. text = _redact_form_body(text)
  1364. if "+" in text:
  1365. text = _SIGNAL_PHONE_RE.sub(_redact_phone, text)
  1366. return text
  1367. # Commands whose stdout is an env-var dump: terminal redaction runs the
  1368. # ENV-assignment pass (code_file=False) for these so opaque tokens with no vendor
  1369. # prefix are masked; everything else uses code_file=True (``MAX_TOKENS=100``).
  1370. # Commands whose stdout is an environment-variable dump (KEY=value lines), NOT source code.
  1371. # ``MY_SERVICE_TOKEN=abc123randomstring``) are still masked. For all other commands, code_file=True is used
  1372. # to avoid mangling legitimate source/config dumps (``MAX_TOKENS=100``, ``"apiKey": "x"`` fixtures,
  1373. # ``postgresql://{user}`` f-string templates). See issue #43025.
  1374. _ENV_DUMP_COMMANDS = frozenset({"env", "printenv", "set", "export", "declare"})
  1375. # Commands that read file contents to stdout, plus the filter readers (``grep``/``awk``/``sed``)
  1376. # the model reaches for on config files. A secret-bearing target (``.env`` per AGENTS.md,
  1377. # a shell rc/profile, Hermes' own ``config.yaml`` where ``hermes mcp add --env`` writes
  1378. # tokens) is a credential dump, so the ENV/YAML assignment pass must run. Arbitrary
  1379. # ``config.yaml`` / source files stay on the code_file path (``MAX_TOKENS: 100``).
  1380. _FILE_READ_COMMANDS = frozenset({
  1381. "cat", "head", "tail", "type", "bat", "less", "more", "nl",
  1382. "zcat", "tac", "view", "batcat", "grep", "awk", "sed",
  1383. })
  1384. _SHELL_RC_BASENAMES = frozenset({
  1385. ".bashrc", ".bash_profile", ".bash_login", ".profile",
  1386. ".zshrc", ".zprofile", ".zlogin", ".zshenv",
  1387. })
  1388. # Filter readers take a PATTERN/program as their first positional; only the operands after
  1389. # it are files, so ``grep .bashrc app.py`` must not gate on the pattern.
  1390. _PATTERN_FIRST_COMMANDS = frozenset({"grep", "awk", "sed"})
  1391. _HERMES_HOME_PREFIXES = ("$HERMES_HOME/", "${HERMES_HOME}/")
  1392. # ``$HOME/.hermes/config.yaml`` keeps the ``.hermes`` segment, so stripping the prefix is
  1393. # enough to gate it; ``~/`` already survives the ``$``-bearing-path bail-out.
  1394. _HOME_PREFIXES = ("$HOME/", "${HOME}/")
  1395. def _command_segments(command: str) -> list[str]:
  1396. """Pipeline/sequence segments, split only on unquoted ``| ; &`` so an
  1397. ``awk '{print $1; print $2}'`` program or ``grep 'foo|bar'`` pattern stays one
  1398. segment. Backslash is not an escape (Windows ``C:\\Users\\...``)."""
  1399. segments: list[str] = []
  1400. buf: list[str] = []
  1401. quote: str | None = None
  1402. for ch in command:
  1403. if quote:
  1404. buf.append(ch)
  1405. if ch == quote:
  1406. quote = None
  1407. continue
  1408. if ch in "'\"":
  1409. quote = ch
  1410. buf.append(ch)
  1411. continue
  1412. if ch in "|;&":
  1413. seg = "".join(buf).strip()
  1414. if seg:
  1415. segments.append(seg)
  1416. buf = []
  1417. continue
  1418. buf.append(ch)
  1419. seg = "".join(buf).strip()
  1420. if seg:
  1421. segments.append(seg)
  1422. return segments
  1423. def _is_under_hermes_home(path: str) -> bool:
  1424. """True when an absolute ``config.yaml`` path sits under the active Hermes home or root.
  1425. The default home's basename is an installation detail — ``.hermes`` on POSIX, ``hermes``
  1426. under ``AppData/Local`` on Windows — and a resolved path never spells ``$HERMES_HOME``,
  1427. so the literal-segment test in ``_is_secret_file_arg`` cannot see a native Windows path.
  1428. Compare against the resolved homes instead. Only reached for a ``config.yaml`` basename,
  1429. so the resolve cost stays off the per-token command scan.
  1430. """
  1431. from agent.file_safety import _hermes_dirs
  1432. try:
  1433. target = os.path.normcase(os.path.realpath(os.path.expanduser(path)))
  1434. except (OSError, ValueError):
  1435. return False
  1436. for home in _hermes_dirs():
  1437. try:
  1438. base = os.path.normcase(os.path.realpath(str(home)))
  1439. except (OSError, ValueError):
  1440. continue
  1441. if target == base or target.startswith(base + os.sep):
  1442. return True
  1443. return False
  1444. def _is_secret_file_arg(arg: str) -> bool:
  1445. """``.env``-style or shell rc basename anywhere; ``config.yaml`` only under a
  1446. ``.hermes`` directory, ``$HERMES_HOME``, or the resolved Hermes home (never arbitrary
  1447. YAML). The resolved-home arm is what covers native Windows, where the home directory
  1448. is ``%LOCALAPPDATA%\\hermes`` and carries no ``.hermes`` segment."""
  1449. path = arg.strip("\"'").replace("\\", "/")
  1450. hermes_home = False
  1451. for prefix in _HERMES_HOME_PREFIXES:
  1452. if path.startswith(prefix):
  1453. path = path[len(prefix):]
  1454. hermes_home = True
  1455. break
  1456. for prefix in _HOME_PREFIXES:
  1457. if path.startswith(prefix):
  1458. path = path[len(prefix):]
  1459. break
  1460. if "$" in path:
  1461. return False
  1462. parts = [part.lower() for part in path.split("/") if part]
  1463. if not parts:
  1464. return False
  1465. if parts[-1] in _ENV_FILE_BASENAMES or parts[-1] in _SHELL_RC_BASENAMES:
  1466. return True
  1467. # ``config.yaml`` plus the ``config.yaml.good.<stamp>`` / ``.corrupt.<stamp>`` copies Hermes
  1468. # writes under ``backups/config/`` — same contents, same secrets.
  1469. if parts[-1] != "config.yaml" and not parts[-1].startswith(("config.yaml.good.", "config.yaml.corrupt.")):
  1470. return False
  1471. return hermes_home or ".hermes" in parts[:-1] or _is_under_hermes_home(path)
  1472. def _command_reads_secret_file(command: str | None) -> bool:
  1473. """True if ``command`` reads a secret-bearing file (see ``_is_secret_file_arg``) to
  1474. stdout. Defense-in-depth, not a boundary: indirect reads (``sudo cat .env``, ``$(cat
  1475. .env)``, unresolved variable paths) are not detected, matching ``is_env_dump_command``."""
  1476. if not command or not isinstance(command, str):
  1477. return False
  1478. for seg in _command_segments(command):
  1479. tokens = seg.split() # not shlex: it mangles Windows paths (``C:\Users\...\.env``)
  1480. if not tokens:
  1481. continue
  1482. reader = tokens[0].rsplit("/", 1)[-1].lower()
  1483. if reader not in _FILE_READ_COMMANDS:
  1484. continue
  1485. positional = [arg for arg in tokens[1:] if not arg.startswith("-")]
  1486. if reader in _PATTERN_FIRST_COMMANDS:
  1487. positional = positional[1:]
  1488. if any(_is_secret_file_arg(arg) for arg in positional):
  1489. return True
  1490. return False
  1491. def is_env_dump_command(command: str | None) -> bool:
  1492. """True if any pipeline/sequence segment starts with an _ENV_DUMP_COMMANDS
  1493. token. Conservative: unrecognized → False (callers fall back to code_file=True)."""
  1494. if not command or not isinstance(command, str):
  1495. return False
  1496. for seg in _command_segments(command):
  1497. try:
  1498. tokens = shlex.split(seg)
  1499. except ValueError:
  1500. tokens = seg.split()
  1501. if tokens and tokens[0] in _ENV_DUMP_COMMANDS:
  1502. return True
  1503. return False
  1504. REDACTION_UNAVAILABLE = "[redaction-unavailable]"
  1505. # The opaque branch needs a 20-char floor (the floor the gateway/A2A sweeps always had): without it the
  1506. # English word "bearer" turns "the bearer of bad news" into "Bearer [redacted] bad news" on every chat
  1507. # reply. The bracket branch folds an already-masked residue ("Bearer [redacted-jwt]") to one marker.
  1508. _BEARER_RESIDUE_RE = re.compile(r"\bBearer\s+(?:\[[^\]]+\]|[A-Za-z0-9._~+/-]{20,}=*)", re.IGNORECASE)
  1509. def redact_for_egress(text: str) -> str:
  1510. """The one scrub for text leaving the process for a remote reader (chat platforms, A2A peers,
  1511. telemetry). ``redact_sensitive_text(force=True)`` — the only secret-pattern list — plus a bearer
  1512. sweep, because a ``Bearer <opaque>`` value with no vendor prefix carries no shape the prefix
  1513. matcher can key on. Fails CLOSED: if the redactor raises, the raw text is never returned."""
  1514. text = str(text or "")
  1515. try:
  1516. text = redact_sensitive_text(text, force=True)
  1517. except Exception:
  1518. return REDACTION_UNAVAILABLE
  1519. if "earer" in text:
  1520. text = _BEARER_RESIDUE_RE.sub("Bearer [redacted]", text)
  1521. return text
  1522. def redact_terminal_output(output: str, command: str | None = None, *, force: bool = False) -> str:
  1523. """Single redaction policy for ALL terminal-output surfaces: the ENV/YAML-assignment
  1524. pass runs only when ``command`` is an env dump or reads a secret-bearing file (``.env``,
  1525. shell rc, Hermes ``config.yaml``); otherwise code_file=True avoids false positives on
  1526. source/config dumps."""
  1527. if not output:
  1528. return output
  1529. code_file = not (is_env_dump_command(command) or _command_reads_secret_file(command))
  1530. redacted = redact_sensitive_text(output, force=force, code_file=code_file)
  1531. # Source-preserving output still gets the Python-repr pass on high-confidence
  1532. # diagnostic lines (pytest ``E `` introspection, final exception lines): that is
  1533. # where {'BRAVE_API_KEY': '…'} leaks, not in source dumps.
  1534. if code_file and (force or _redact_enabled()) and ":" in redacted and "'" in redacted:
  1535. redacted = _redact_python_diagnostic_repr_fields(redacted)
  1536. return redacted
  1537. # --- Prefix pre-screen: derived from _PREFIX_PATTERNS so a new prefix can't
  1538. # silently break the gate (every match contains its pattern's literal prefix).
  1539. def _extract_literal_prefix(pattern: str) -> str:
  1540. """Leading literal chars of a regex (up to the first metacharacter)."""
  1541. meta = "[(\\.?*+|{^$"
  1542. for i, ch in enumerate(pattern):
  1543. if ch in meta:
  1544. return pattern[:i]
  1545. return pattern
  1546. def _skip_char_class(pattern: str, i: int) -> int:
  1547. """Given ``pattern[i] == "["``, return the index just past the closing ``]``."""
  1548. i += 2 if pattern[i + 1:i + 2] == "]" else 1 # a leading "]" is literal
  1549. while i < len(pattern) and pattern[i] != "]":
  1550. i += 2 if pattern[i] == "\\" else 1
  1551. return i
  1552. def _unbounded_quantifier_follows(pattern: str, j: int) -> bool:
  1553. """True if an open-ended quantifier (``*``, ``+``, ``{m,}``) starts at ``pattern[j]``."""
  1554. if j >= len(pattern):
  1555. return False
  1556. if pattern[j] in "*+":
  1557. return True
  1558. if pattern[j] == "{":
  1559. k = pattern.find("}", j)
  1560. body = pattern[j + 1:k] if k != -1 else ""
  1561. return body[:-1].isdigit() and body.endswith(",") # {m,} is open-ended; {m} / {m,n} bounded
  1562. return False
  1563. def _pattern_structure(pattern: str) -> tuple[bool, bool]:
  1564. """One scan → ``(has_top_level_alternation, has_nested_unbounded_repeat)``.
  1565. Top-level ``|`` defeats the literal-prefix guarantee (in ``ab|.*`` the prefix
  1566. binds only the first branch; ``ab(?:x|y)`` is fine). An unbounded quantifier
  1567. on a group containing one (``(a+)+``, ``(a{2,})+``) is the canonical ReDoS
  1568. shape. Structural only; overlapping branches (``(a|aa)+``) are not detected.
  1569. """
  1570. top_level_alt = nested = False
  1571. contains_unbounded = [False] # per open group: does it contain an unbounded repeat?
  1572. i = 0
  1573. while i < len(pattern):
  1574. ch = pattern[i]
  1575. if ch == "\\":
  1576. i += 2
  1577. continue
  1578. if ch == "[":
  1579. i = _skip_char_class(pattern, i)
  1580. elif ch == "(":
  1581. contains_unbounded.append(False)
  1582. elif ch == ")":
  1583. inner = contains_unbounded.pop() if len(contains_unbounded) > 1 else False
  1584. if inner and _unbounded_quantifier_follows(pattern, i + 1):
  1585. nested = True
  1586. contains_unbounded[-1] = contains_unbounded[-1] or inner
  1587. elif ch == "|" and len(contains_unbounded) == 1:
  1588. top_level_alt = True
  1589. elif _unbounded_quantifier_follows(pattern, i):
  1590. contains_unbounded[-1] = True
  1591. if ch == "{":
  1592. i = pattern.find("}", i) # skip the {m,} body
  1593. i += 1
  1594. return top_level_alt, nested
  1595. def _has_top_level_alternation(pattern: str) -> bool:
  1596. return _pattern_structure(pattern)[0]
  1597. def _has_nested_unbounded_repeat(pattern: str) -> bool:
  1598. return _pattern_structure(pattern)[1]
  1599. _PREFIX_SUBSTRINGS = tuple(_extract_literal_prefix(p) for p in _PREFIX_PATTERNS)
  1600. def _has_known_prefix_substring(text: str) -> bool:
  1601. """Cheap pre-check before the expensive ``_PREFIX_RE``."""
  1602. return any(p in text for p in _PREFIX_SUBSTRINGS)
  1603. # --- Plugin-registered redaction patterns -----------------------------------
  1604. # ADDITIVE-ONLY: a plugin can extend what gets masked but cannot weaken a
  1605. # built-in, so it can only over-redact. Keyed by registration source so plugin
  1606. # unload has a clean seam to drop ONE plugin's patterns.
  1607. # There is deliberately no public removal API — additive-only stands; unload is a host-owned lifecycle
  1608. # concern. See #64229.
  1609. _PLUGIN_PREFIX_PATTERNS: dict = {}
  1610. _registry_lock = threading.Lock()
  1611. def _plugin_patterns() -> list:
  1612. """All plugin-registered patterns in registration order."""
  1613. return [p for patterns in _PLUGIN_PREFIX_PATTERNS.values() for p in patterns]
  1614. def _rebuild_prefix_matcher() -> None:
  1615. """Recompile the prefix alternation and pre-screen substrings; callers read the
  1616. module globals at call time, so the swap propagates immediately."""
  1617. global _PREFIX_RE, _PREFIX_SUB_RE, _PREFIX_SUBSTRINGS
  1618. combined = _PREFIX_PATTERNS + _plugin_patterns()
  1619. _PREFIX_RE = _compile_prefix_matcher(combined)
  1620. _PREFIX_SUB_RE = _compile_prefix_sub_matcher(combined)
  1621. _PREFIX_SUBSTRINGS = tuple(_extract_literal_prefix(p) for p in combined)
  1622. # Structural validators for register_redaction_patterns, in check order:
  1623. # (predicate -> reject when True, warning message with (source, pattern) args).
  1624. _PATTERN_REJECT_RULES = (
  1625. (_has_top_level_alternation,
  1626. "%s: skipping redaction pattern %r — top-level alternation escapes the literal-prefix "
  1627. "guarantee (in 'ab|.*' the prefix binds only the first branch); wrap alternation in "
  1628. "a group after the prefix, e.g. 'ab(?:x|y)'"),
  1629. (_has_nested_unbounded_repeat,
  1630. "%s: skipping redaction pattern %r — nested unbounded quantifiers (e.g. '(a+)+') can "
  1631. "backtrack catastrophically, and registered patterns run on every log line and tool output"),
  1632. (lambda pattern: len(_extract_literal_prefix(pattern)) < 2,
  1633. "%s: skipping redaction pattern %r — must start with at least 2 literal characters "
  1634. "(needed for the pre-screen substring gate)"),
  1635. )
  1636. def register_redaction_patterns(patterns, source: str = "plugin") -> int:
  1637. """Additively register credential-token regexes; returns the count accepted.
  1638. Invalid entries (non-compiling, top-level alternation, nested unbounded
  1639. quantifiers, < 2 literal prefix chars) and duplicates are warned/skipped,
  1640. never raised — a broken plugin must not break startup.
  1641. """
  1642. accepted = []
  1643. for pattern in patterns or []:
  1644. if not isinstance(pattern, str) or not pattern.strip():
  1645. logger.warning("%s: skipping empty/non-string redaction pattern", source)
  1646. continue
  1647. pattern = pattern.strip()
  1648. try:
  1649. re.compile(pattern)
  1650. except re.error as exc:
  1651. logger.warning("%s: skipping invalid redaction pattern %r (%s)", source, pattern, exc)
  1652. continue
  1653. rejected = next((message for reject, message in _PATTERN_REJECT_RULES if reject(pattern)), None)
  1654. if rejected:
  1655. logger.warning(rejected, source, pattern)
  1656. continue
  1657. if pattern in _PREFIX_PATTERNS or pattern in _plugin_patterns() or pattern in accepted:
  1658. logger.debug("%s: redaction pattern %r already registered", source, pattern)
  1659. continue
  1660. accepted.append(pattern)
  1661. if accepted:
  1662. with _registry_lock:
  1663. _PLUGIN_PREFIX_PATTERNS.setdefault(source, []).extend(accepted)
  1664. _rebuild_prefix_matcher()
  1665. logger.info("%s: registered %d redaction pattern(s)", source, len(accepted))
  1666. return len(accepted)
  1667. def _reset_plugin_redaction_patterns() -> None:
  1668. """Drop all plugin-registered patterns (tests/teardown only)."""
  1669. with _registry_lock:
  1670. _PLUGIN_PREFIX_PATTERNS.clear()
  1671. _rebuild_prefix_matcher()
  1672. class RedactingFormatter(logging.Formatter):
  1673. """Log formatter that redacts secrets from all log messages."""
  1674. def format(self, record: logging.LogRecord) -> str:
  1675. return redact_sensitive_text(super().format(record))