ansi_strip.py 4.2 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283
  1. """Strip ANSI escape sequences from subprocess output so they never reach the
  2. model's context (models otherwise copy them into file writes). Covers full
  3. ECMA-48: CSI, OSC (BEL/ST), DCS/SOS/PM/APC, nF, Fp/Fe/Fs and 8-bit C1 controls."""
  4. import re
  5. _ANSI_ESCAPE_RE = re.compile(
  6. r"\x1b"
  7. r"(?:"
  8. r"\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]" # CSI sequence
  9. r"|\][\s\S]*?(?:\x07|\x1b\\)" # OSC (BEL or ST terminator)
  10. r"|[PX^_][\s\S]*?(?:\x1b\\)" # DCS/SOS/PM/APC strings
  11. r"|[\x20-\x2f]+[\x30-\x7e]" # nF escape sequences
  12. r"|[\x30-\x7e]" # Fp/Fe/Fs single-byte
  13. r")"
  14. r"|\x9b[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]" # 8-bit CSI
  15. r"|\x9d[\s\S]*?(?:\x07|\x9c)" # 8-bit OSC
  16. r"|[\x80-\x9f]", # Other 8-bit C1 controls
  17. re.DOTALL,
  18. )
  19. # Fast-path checks: skip the full regex when no candidate bytes are present.
  20. _HAS_ESCAPE = re.compile(r"[\x1b\x80-\x9f]")
  21. _HAS_CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")
  22. _HAS_UNICODE_TAG = re.compile(r"[\U000E0000-\U000E007F]")
  23. # C0 controls (minus tab/newline/CR) plus DEL: they survive strip_ansi() (which only
  24. # removes *sequences*) but are dangerous echoed to a terminal (BEL, backspace, NUL).
  25. _CONTROL_CHARS_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
  26. # Unicode TAG chars (U+E0000–U+E007F) render as nothing but LLM tokenizers see them:
  27. # the "ASCII smuggling" injection channel. Emoji tag sequences (TR51: U+1F3F4 base +
  28. # tag spec + U+E007F CANCEL TAG, e.g. Scotland/Wales flags) are the only legit use.
  29. # Deprecated as language tags, these render as nothing in every terminal and chat UI but are perfectly
  30. # visible to an LLM tokenizer — the classic "ASCII smuggling" prompt-injection channel (hide
  31. # `\u{E0069}\u{E0067}\u{E006E}...` = invisible instructions inside otherwise benign tool output). Ported
  32. # from block/goose#10746. The ONLY legitimate modern use is emoji tag sequences (Unicode TR51): a U+1F3F4
  33. # black-flag base followed by tag spec characters and the U+E007F CANCEL TAG terminator (e.g. the flags of
  34. # Scotland/Wales/England). goose strips those too; we preserve them — same rationale as keeping ZWJ inside
  35. # emoji sequences.
  36. _UNICODE_TAG_SUB_RE = re.compile(
  37. r"(\U0001F3F4[\U000E0020-\U000E007E]+\U000E007F)" # valid emoji tag seq (kept)
  38. r"|[\U000E0000-\U000E007F]" # any other tag char (stripped)
  39. )
  40. def strip_ansi(text: str) -> str:
  41. """Remove ANSI escape sequences; clean text passes through unchanged (fast path)."""
  42. if not text or not _HAS_ESCAPE.search(text):
  43. return text
  44. return _ANSI_ESCAPE_RE.sub("", text)
  45. def sanitize_display_text(text: str) -> str:
  46. """Sanitize stored/untrusted text before echoing it to a terminal: strips ANSI
  47. sequences AND bare control chars, keeping only newlines/tabs (CRs become newlines
  48. so ``\\r``-overwrite spoofing can't hide content). Rich's ``Text()`` does NOT
  49. neutralize raw escape bytes, so a replayed ``/resume`` message must not be able
  50. to clear the screen, retitle the window, or restyle UI.
  51. Use this when re-rendering conversation history or other persisted text in a terminal UI (e.g. the
  52. ``/resume`` recap): a message that arrived with embedded escapes — pasted content, gateway-origin text,
  53. or model output echoing injected tool results — must not be able to clear the screen, retitle the
  54. window, move the cursor, or restyle adjacent UI when replayed. Mirrors openai/codex#31494
  55. (``sanitize_user_text``).
  56. """
  57. if not text or not _HAS_CONTROL.search(text):
  58. return text
  59. text = strip_ansi(text)
  60. if "\r" in text:
  61. text = text.replace("\r\n", "\n").replace("\r", "\n")
  62. return _CONTROL_CHARS_RE.sub("", text)
  63. def strip_unicode_tags(text: str) -> str:
  64. """Remove invisible Unicode TAG chars (a prompt-injection smuggling channel in
  65. untrusted tool output); valid emoji tag sequences are preserved.
  66. Returns the input unchanged (fast path) when no plane-14 tag characters are present. Ported from
  67. block/goose#10746.
  68. """
  69. if not text or not _HAS_UNICODE_TAG.search(text):
  70. return text
  71. return _UNICODE_TAG_SUB_RE.sub(lambda m: m.group(1) or "", text)