tts_text_normalize.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. """Deterministic cleanup turning assistant Markdown into a spoken script.
  2. Shared by explicit TTS calls, gateway auto-TTS, voice-mode streaming and the web
  3. dashboard. Non-ASCII characters are written as escapes on purpose so the file
  4. stays free of invisible/look-alike glyphs.
  5. """
  6. from __future__ import annotations
  7. import html
  8. import re
  9. # Sentinel appended to former heading lines so smooth_whitespace_for_tts folds the
  10. # heading into the sentence after it ("Weather, it will be sunny") instead of a bare
  11. # "Weather." label.
  12. _HEAD = "\x00"
  13. _MD_CODE_BLOCK_RE = re.compile(r"```[\s\S]*?```")
  14. _MD_LINK_RE = re.compile(r"\[([^\]]+)\]\((?:[^()]|\([^)]*\))*\)")
  15. _MD_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\((?:[^()]|\([^)]*\))*\)")
  16. _MD_INLINE_CODE_RE = re.compile(r"`([^`]+)`")
  17. _MD_BOLD_RE = re.compile(r"\*\*(.+?)\*\*", flags=re.DOTALL)
  18. _MD_UNDERSCORE_BOLD_RE = re.compile(r"__(.+?)__", flags=re.DOTALL)
  19. _MD_ITALIC_RE = re.compile(r"(?<!\*)\*(?!\*)(.+?)(?<!\*)\*(?!\*)", flags=re.DOTALL)
  20. _MD_UNDERSCORE_ITALIC_RE = re.compile(r"(?<!_)_(?!_)(.+?)(?<!_)_(?!_)", flags=re.DOTALL)
  21. _MD_STRIKE_RE = re.compile(r"~~(.+?)~~", flags=re.DOTALL)
  22. _MD_HEADING_LINE_RE = re.compile(r"^[ \t]{0,3}#{1,6}[ \t]+(.+?)[ \t]*#*[ \t]*$", flags=re.MULTILINE)
  23. _MD_BLOCKQUOTE_RE = re.compile(r"^\s*>\s?", flags=re.MULTILINE)
  24. _MD_LIST_ITEM_RE = re.compile(r"^\s*(?:[-*+]|\d+[.)])\s+", flags=re.MULTILINE)
  25. _MD_HR_RE = re.compile(r"^\s*[-*_]{3,}\s*$", flags=re.MULTILINE)
  26. _MD_TABLE_PIPE_RE = re.compile(r"\s*\|\s*")
  27. _URL_RE = re.compile(r"https?://\S+")
  28. # Local file links ("MEDIA:/Users/me/file.xlsx") are click targets on screen, not
  29. # speech: voices loop on the hyphenated slug ("eeeeee"). The token is silence; the
  30. # assistant's prose already says "the files are below". Trailing sentence
  31. # punctuation is left in place so "see MEDIA:/x.py. Then" keeps its full stop.
  32. _MEDIA_PATH_RE = re.compile(r"MEDIA:\S+?(?=[.,;:!?)\]]*(?:\s|$))")
  33. _DEGREE_UNITS = (("C", "Celsius"), ("F", "Fahrenheit"))
  34. # Unit suffix (regex, after a digit) -> spoken word; km/h variants before the bare "m".
  35. _UNIT_WORDS = (
  36. (r"km\s*/\s*h", "kilometres per hour"), (r"km/h", "kilometres per hour"),
  37. (r"mm", "millimetres"), (r"cm", "centimetres"), (r"m", "metres"))
  38. # Currency prefix (regex) -> spoken word; order matters (NZ$/A$/US$ before bare $).
  39. _CURRENCY_WORDS = (
  40. (r"NZ\$", "New Zealand dollars", re.IGNORECASE), (r"A\$", "Australian dollars", re.IGNORECASE),
  41. (r"US\$", "US dollars", re.IGNORECASE), ("€", "euros", 0), ("£", "pounds", 0), (r"\$", "dollars", 0),
  42. )
  43. # Broad emoji / pictograph cleanup: most voice providers read emojis as awkward labels.
  44. _EMOJI_RE = re.compile(
  45. "[\U0001F1E6-\U0001F1FF\U0001F300-\U0001F5FF\U0001F600-\U0001F64F\U0001F680-\U0001F6FF"
  46. "\U0001F700-\U0001F77F\U0001F780-\U0001F7FF\U0001F800-\U0001F8FF\U0001F900-\U0001F9FF"
  47. "\U0001FA00-\U0001FAFF☀-➿]+",
  48. flags=re.UNICODE)
  49. _VARIATION_SELECTOR_RE = re.compile("[︎️]")
  50. def strip_markdown_for_tts(text: str) -> str:
  51. """Strip Markdown/Telegram formatting while preserving readable words."""
  52. if not text:
  53. return ""
  54. text = html.unescape(str(text))
  55. text = _MD_CODE_BLOCK_RE.sub(" ", text)
  56. text = _MD_IMAGE_RE.sub(lambda m: f" {m.group(1)} " if m.group(1) else " ", text)
  57. text = _MD_LINK_RE.sub(r"\1", text)
  58. text = _URL_RE.sub("", text)
  59. text = _MEDIA_PATH_RE.sub(" ", text)
  60. text = _MD_INLINE_CODE_RE.sub(r"\1", text)
  61. text = _MD_BOLD_RE.sub(r"\1", text)
  62. text = _MD_UNDERSCORE_BOLD_RE.sub(r"\1", text)
  63. text = _MD_ITALIC_RE.sub(r"\1", text)
  64. text = _MD_UNDERSCORE_ITALIC_RE.sub(r"\1", text)
  65. text = _MD_STRIKE_RE.sub(r"\1", text)
  66. # Mark headings (do not just delete the marker): see _HEAD.
  67. text = _MD_HEADING_LINE_RE.sub(lambda m: m.group(1).rstrip() + _HEAD, text)
  68. text = _MD_BLOCKQUOTE_RE.sub("", text)
  69. text = _MD_LIST_ITEM_RE.sub("", text)
  70. text = _MD_HR_RE.sub("", text)
  71. # Leftover table pipes become pauses instead of a spoken "vertical bar".
  72. return _MD_TABLE_PIPE_RE.sub("; ", text)
  73. def _normalize_temperature_ranges(text: str) -> str:
  74. """``11-17°C`` -> ``11 to 17 degrees Celsius`` (en/em dash or hyphen; unicode minus normalized)."""
  75. number = r"([-+\u2212]?\d+(?:\.\d+)?)"
  76. for unit, word in _DEGREE_UNITS:
  77. text = re.sub(
  78. r"(?<!\w)" + number + r"\s*[\u2013\u2014-]\s*" + number + r"\s*°\s*" + unit + r"\b",
  79. lambda m, w=word: (
  80. f"{m.group(1).replace(chr(0x2212), '-')} to {m.group(2).replace(chr(0x2212), '-')} degrees {w}"
  81. ),
  82. text, flags=re.IGNORECASE)
  83. return text
  84. def normalize_symbols_for_tts(text: str) -> str:
  85. """Expand common symbols/shorthand into words a TTS engine reads well."""
  86. if not text:
  87. return ""
  88. text = re.sub("[   ]", " ", str(text)) # non-breaking / thin spaces
  89. text = text.replace("\u2212", "-").replace("…", "...") # minus sign, ellipsis
  90. text = _normalize_temperature_ranges(text)
  91. # Temperatures with a number first, then bare units ("measured in degrees C"),
  92. # then any remaining degree symbol (angles, stray cases).
  93. for unit, word in _DEGREE_UNITS:
  94. text = re.sub(
  95. r"(?<!\w)([-+]?\d+(?:\.\d+)?)\s*°\s*" + unit + r"\b", r"\1 degrees " + word, text, flags=re.IGNORECASE,
  96. )
  97. for unit, word in _DEGREE_UNITS:
  98. text = re.sub(r"°\s*" + unit + r"\b", "degrees " + word, text, flags=re.IGNORECASE)
  99. text = re.sub(r"(?<!\w)([-+]?\d+(?:\.\d+)?)\s*°", r"\1 degrees", text).replace("°", " degrees")
  100. # Common weather/travel units.
  101. for pattern, word in _UNIT_WORDS:
  102. text = re.sub(r"(?<=\d)\s*" + pattern + r"\b", " " + word, text, flags=re.IGNORECASE)
  103. # Numeric rates only ("5/month" -> "5 per month"). Requiring digit-then-letter
  104. # keeps "and/or", "N/A", "TCP/IP" and dates like "2026/06" intact.
  105. text = re.sub(r"(?<=\d)\s*/\s*(?=[A-Za-z])", " per ", text)
  106. # Money and percentages. The integer part must END in a digit so a trailing
  107. # comma ("A$50, ...") is not swallowed into the spoken amount. Prefixed
  108. # currencies run first so "$" doesn't eat "NZ$".
  109. for symbol, word, flags in _CURRENCY_WORDS:
  110. text = re.sub(symbol + r"\s*([\d,]*\d(?:\.\d+)?)", r"\1 " + word, text, flags=flags)
  111. text = re.sub(r"(?<=\d)\s*%", " percent", text)
  112. # Operators and separators that commonly leak from formatted answers.
  113. text = re.sub("[•◦▪▫]", " ", text.replace("&", " and ")) # bullet glyphs
  114. for symbol, word in (("→", " to "), ("⇒", " to "), ("≈", " about "), ("~", " about ")):
  115. text = text.replace(symbol, word)
  116. return _EMOJI_RE.sub("", _VARIATION_SELECTOR_RE.sub("", text))
  117. # --- Identifier-dense tokens (#119207) ---------------------------------------
  118. # Filenames with extensions, hashes, UUIDs, dense model/version IDs and paths
  119. # are read character-by-character ("peyton-sample-20260922.wav" becomes
  120. # "peyton dash sample dash two zero two six..."), making clean synthesized
  121. # audio sound corrupted. They become silence, never a hardcoded English
  122. # placeholder word (#86602: the reply may be in any language). Ordinary
  123. # prose, dates, ratios and short hyphenated words (COVID-19, well-known)
  124. # pass through untouched. Mirrored token-for-token by
  125. # apps/desktop/src/lib/speech-text.ts — keep both in lockstep and extend the
  126. # shared corpus (tests/fixtures/identifier_speech_corpus.json) first.
  127. _FILENAME_EXT_RE = re.compile(
  128. r"[\w.-]{0,60}\.(?:wav|ogg|mp3|flac|m4a|aac|py|pyc|ts|tsx|js|jsx|mjs|cjs|json|yaml|yml|toml|"
  129. r"md|mdx|txt|csv|xlsx|xls|pdf|png|jpg|jpeg|gif|webp|svg|log|sql|sh|bash|zsh|rs|go|java|rb|php|"
  130. r"html|css|lock|tar|gz|zip|db|sqlite|sqlite3|onnx|pt|bin|env|ini|conf|cfg|xml)\b",
  131. re.IGNORECASE)
  132. _UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
  133. _HASH_PREFIX_HEX_RE = re.compile(
  134. r"\b(?:sha(?:-?256|-?512|-?1|3)?|blake2[ab]?|md5|crc32?)[:\s]+[0-9a-fA-F]{7,64}", re.IGNORECASE)
  135. _HEX_RUN_RE = re.compile(r"(?<![0-9a-fA-F])[0-9a-fA-F]{7,64}(?![0-9a-fA-F])")
  136. _IDENTIFIER_TOKEN_RE = re.compile(r"[A-Za-z0-9_./~@-]+")
  137. _DATE_TOKEN_RE = re.compile(r"\d{4}[-/]\d{1,2}(?:[-/]\d{1,2})?")
  138. def _is_dense_identifier(token: str) -> bool:
  139. """True for a machine token a voice would spell out character by character."""
  140. if "@" in token or _DATE_TOKEN_RE.fullmatch(token):
  141. return False # email addresses, dates ("2026-09-28", "2026/06/02")
  142. if token.startswith(("~/", "./", "../", "/")):
  143. return True # filesystem paths
  144. if "/" in token and (_FILENAME_EXT_RE.search(token) or any(char.isdigit() for char in token)):
  145. return True # paths and dense model IDs ("meta-llama/Llama-3.3-70B-Instruct")
  146. if _FILENAME_EXT_RE.search(token) or _UUID_RE.search(token):
  147. return True
  148. # Hex-hash runs ("73688014f78", "e3b0c442...") — digit-free runs
  149. # ("defaced") are legitimate words and stay.
  150. if _HEX_RUN_RE.fullmatch(token) and re.search(r"\d", token):
  151. return True
  152. digits = any(char.isdigit() for char in token)
  153. if not digits:
  154. return False
  155. seps = sum(token.count(char) > 0 for char in "_./")
  156. hyphens = token.count("-")
  157. if seps >= 2 or hyphens >= 2:
  158. return True # v2.1.0-beta.3, Llama-3.3-70B, dated filename slugs
  159. return "/" in token or (hyphens >= 1 and seps >= 1)
  160. def prune_identifier_tokens_for_tts(text: str) -> str:
  161. """Silence identifier-dense tokens; keep ordinary prose verbatim."""
  162. if not text:
  163. return ""
  164. text = _HASH_PREFIX_HEX_RE.sub(" ", text)
  165. return _IDENTIFIER_TOKEN_RE.sub(
  166. lambda m: " " if _is_dense_identifier(m.group(0)) else m.group(0), text)
  167. def smooth_whitespace_for_tts(text: str) -> str:
  168. """Collapse visual formatting into calm spoken paragraphs. A _HEAD-marked heading folds into
  169. the next content line as a lead-in ("Weather, It will be sunny."); a heading with no content
  170. after it becomes its own sentence."""
  171. if not text:
  172. return ""
  173. raw_lines = text.splitlines()
  174. add_sentence_pauses = sum(1 for raw_line in raw_lines if raw_line.replace(_HEAD, "").strip()) > 1
  175. lines: list[str] = []
  176. pending_heading: str | None = None
  177. def flush_pending() -> None:
  178. nonlocal pending_heading
  179. if pending_heading is not None:
  180. lines.append(pending_heading.rstrip(".:;,") + ".")
  181. pending_heading = None
  182. for raw_line in raw_lines:
  183. is_heading = raw_line.rstrip().endswith(_HEAD)
  184. line = raw_line.replace(_HEAD, "").strip()
  185. if not line:
  186. # Hold a pending heading across blank lines so it still folds into the next content line.
  187. if pending_heading is None and lines and lines[-1] != "":
  188. lines.append("")
  189. continue
  190. if is_heading:
  191. flush_pending()
  192. pending_heading = line.rstrip(".:;,")
  193. continue
  194. if pending_heading is not None:
  195. line = f"{pending_heading.rstrip('.:;,')}, {line}"
  196. pending_heading = None
  197. if add_sentence_pauses:
  198. if line[-1] in ":;":
  199. # A trailing colon promises a continuation ("the list:"); the raw
  200. # token after it (file link, code fence) is already gone from
  201. # speech, so the voice would hang on the open pause. Close it.
  202. line = line.rstrip(":;").rstrip() + "."
  203. elif line[-1] not in ".!?":
  204. line += "."
  205. lines.append(line)
  206. flush_pending()
  207. text = "\n".join(lines)
  208. for pattern, repl in ((r"\n{3,}", "\n\n"), (r"[ \t]{2,}", " "), (r"\s+([,.;:!?])", r"\1"),
  209. # A sentence mark only gains a space when NOT glued to
  210. # a word char on both sides — "user@example.com" is a
  211. # single token, not "example. com".
  212. (r"(?<![A-Za-z0-9])[,.;:!?](?=[A-Za-z])", r"\g<0> "),
  213. (r"\.{4,}", "...")):
  214. text = re.sub(pattern, repl, text)
  215. # Single-line text ("Here is the list:") skips the per-line pause pass, so close a
  216. # final colon here too -- never a digit-preceded one ("final score 3:2" is a ratio).
  217. text = re.sub(r"([^0-9])(?:[:;]\s*)+$", r"\1.", text)
  218. return text.strip()
  219. # ``/reasoning show`` emits ``<think>...</think>`` in the final message: users want to
  220. # SEE reasoning, not hear it. An unterminated block (streaming cut-off) is also silenced.
  221. # Reasoning blocks: models with ``/reasoning show`` enabled emit ``<think>...</think>`` blocks in the final
  222. # assistant message. See #34213.
  223. _THINK_BLOCK_RE = re.compile(r"<think[\s>].*?</think>", flags=re.DOTALL | re.IGNORECASE)
  224. _THINK_BLOCK_OPEN_RE = re.compile(r"<think[\s>].*\Z", flags=re.DOTALL | re.IGNORECASE)
  225. # run_agent.py's turn-end file-mutation verifier footer (a ``⚠️ File-mutation verifier:``
  226. # header line plus indented ``•`` bullets) is a UI affordance, not speech.
  227. _VERIFIER_FOOTER_RE = re.compile(r"^\s*⚠️?\s*File-mutation verifier:.*(?:\n[ \t]+•.*)*", flags=re.MULTILINE)
  228. def strip_nonspoken_blocks(text: str) -> str:
  229. """Remove ``<think>`` reasoning blocks and the file-mutation verifier footer."""
  230. if not text:
  231. return ""
  232. for pattern in (_THINK_BLOCK_RE, _THINK_BLOCK_OPEN_RE, _VERIFIER_FOOTER_RE):
  233. text = pattern.sub(" ", text)
  234. return text
  235. def flatten_newlines_for_payload(text: str) -> str:
  236. """Collapse newlines into sentence breaks for single-line TTS payloads: some OpenAI-compatible
  237. backends (e.g. Kokoro) truncate at the first newline; smoothing already ends each line with
  238. punctuation, so this is safe.
  239. See #9004.
  240. """
  241. if not text:
  242. return ""
  243. for pattern, repl in ((r"\n{2,}", ". "), (r"(?<=[.!?;:,])\n", " "), (r"\n", ". "), (r"\.\s*\.", "."),
  244. (r"[ \t]{2,}", " ")):
  245. text = re.sub(pattern, repl, text)
  246. # Single-line text ("Here is the list:") skips the per-line pause pass, so close a
  247. # final colon here too -- never a digit-preceded one ("final score 3:2" is a ratio).
  248. text = re.sub(r"([^0-9])(?:[:;]\s*)+$", r"\1.", text)
  249. return text.strip()
  250. def prepare_spoken_text(text: str, max_chars: int | None = 4000) -> str:
  251. """Return a TTS-friendly script from assistant text (deterministic cleanup, not a rewrite).
  252. Pipeline: non-spoken blocks > Markdown > symbols/units > identifier-dense tokens >
  253. line formatting into sentence pauses > single line (for newline-sensitive providers),
  254. then ``max_chars``."""
  255. spoken = text
  256. for step in (strip_nonspoken_blocks, strip_markdown_for_tts, normalize_symbols_for_tts,
  257. prune_identifier_tokens_for_tts,
  258. smooth_whitespace_for_tts, flatten_newlines_for_payload):
  259. spoken = step(spoken)
  260. if max_chars is not None and max_chars > 0 and len(spoken) > max_chars:
  261. spoken = spoken[:max_chars].rstrip()
  262. return spoken
  263. def _strip_markdown_for_tts(text: str) -> str:
  264. """``prepare_spoken_text`` without a length cap (``tts_tool`` compatibility name)."""
  265. return prepare_spoken_text(text, max_chars=None)