helpers.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719
  1. """Shared helpers for gateway platform adapters: message dedup, markdown
  2. stripping, thread participation tracking, GFM table → bullets, mention-pattern
  3. compilation, and fence-aware markdown chunking."""
  4. import asyncio
  5. import contextlib
  6. import json
  7. import logging
  8. import re
  9. import threading
  10. import time
  11. from pathlib import Path
  12. from typing import Any, MutableMapping, Optional
  13. from gateway.platforms.event import MessageEvent
  14. from utils import atomic_json_write
  15. logger = logging.getLogger(__name__)
  16. class MessageDeduplicator:
  17. """TTL-based message deduplication cache (``if dedup.is_duplicate(msg_id): return``)."""
  18. def __init__(self, max_size: int = 2000, ttl_seconds: float = 300):
  19. self._seen: dict[str, float] = {}
  20. self._max_size = max_size
  21. self._ttl = ttl_seconds
  22. def is_duplicate(self, msg_id: str) -> bool:
  23. """Return True if *msg_id* was already seen within the TTL window."""
  24. if not msg_id:
  25. return False
  26. now = time.time()
  27. if msg_id in self._seen:
  28. if now - self._seen[msg_id] < self._ttl:
  29. return True
  30. del self._seen[msg_id] # expired: treat as new
  31. self._seen[msg_id] = now
  32. if len(self._seen) > self._max_size:
  33. cutoff = now - self._ttl
  34. self._seen = {k: v for k, v in self._seen.items() if v > cutoff}
  35. if len(self._seen) > self._max_size:
  36. # All entries still fresh: keep the newest so max_size holds under load.
  37. self._seen = dict(sorted(self._seen.items(), key=lambda item: item[1])[-self._max_size:])
  38. return False
  39. def contains(self, msg_id: str) -> bool:
  40. """Return whether *msg_id* is live in the cache without inserting it."""
  41. seen_at = self._seen.get(msg_id) if msg_id else None
  42. if seen_at is None:
  43. return False
  44. if time.time() - seen_at < self._ttl:
  45. return True
  46. del self._seen[msg_id]
  47. return False
  48. def discard(self, msg_id: str) -> None:
  49. """Release a claimed message ID after cancelled/failed handoff."""
  50. self._seen.pop(msg_id, None)
  51. def clear(self):
  52. self._seen.clear()
  53. def absorb(self, other: "MessageDeduplicator") -> None:
  54. """Adopt *other*'s still-live IDs (at their original seen times) into this cache."""
  55. cutoff = time.time() - self._ttl
  56. self._seen.update({k: v for k, v in other._seen.items() if v > cutoff and k not in self._seen})
  57. def carry_inbound_dedup(predecessor: Any, adapter: Any) -> None:
  58. """Seed a rebuilt adapter's dedup caches from the instance it replaces (call before connect).
  59. The runner's reconnect path builds a NEW adapter; without this a platform replaying a recent
  60. inbound ID after the reconnect (websocket resume, webhook retry, unacked poll batch) is
  61. admitted and answered a second time."""
  62. if predecessor is None:
  63. return
  64. for name, previous in vars(predecessor).items():
  65. current = getattr(adapter, name, None)
  66. if isinstance(previous, MessageDeduplicator) and isinstance(current, MessageDeduplicator) and current is not previous:
  67. current.absorb(previous)
  68. def hand_over_held_inbound(source: Any, target: Any) -> None:
  69. """Move inbound ``source`` is holding to ``target`` (#132829, #133399).
  70. The runner calls it at publish time (retired instance -> replacement) and after disposing a
  71. failed reconnect candidate (candidate -> retained predecessor): a candidate that fails connect
  72. can hold acked updates and salvage pending batches in disconnect(), which must not die with it."""
  73. adopt = getattr(target, "adopt_held_inbound", None)
  74. if source is not None and callable(adopt):
  75. adopt(source)
  76. # Worker-thread handoff used by the off-loop persist paths. A module attribute
  77. # so tests can replace THIS seam instead of patching ``asyncio.to_thread``
  78. # globally.
  79. _to_thread = asyncio.to_thread
  80. async def cancel_task(task: Optional[asyncio.Task]) -> None:
  81. """Cancel *task* and wait for it to unwind. ``None``/finished tasks are no-ops; awaiting the
  82. current task would deadlock, so a self-cancel only requests cancellation. Exceptions the task
  83. dies with are swallowed: at teardown nobody is left to handle them."""
  84. if task is None or task.done():
  85. return
  86. task.cancel()
  87. if task is not asyncio.current_task():
  88. with contextlib.suppress(asyncio.CancelledError, Exception):
  89. await task
  90. def bounded_put(store: MutableMapping[str, Any], key: str, value: Any, cap: int) -> None:
  91. """Insert into an insertion-ordered mapping with a hard size bound, evicting the oldest keys. A
  92. re-put moves the key to the newest position so live entries outlast stale ones."""
  93. store.pop(key, None)
  94. store[key] = value
  95. while len(store) > cap:
  96. del store[next(iter(store))]
  97. def typing_metadata_for_turn(adapter: Any, chat_id: Any, anchor: Any, metadata: Optional[dict]) -> Any:
  98. """A turn's typing/status metadata, fixed when the turn starts and reused for every refresh and
  99. clear. ``anchor`` is the turn's own message id: an adapter whose status thread depends on the turn
  100. (``status_metadata_for_turn``: a top-level Slack DM's status sits on its triggering message) takes
  101. it from here, not from per-chat state that a later inbound in the chat overwrites. Looked up on
  102. the class, so a test double's auto-attributes never stand in for the hook."""
  103. hook = getattr(type(adapter), "status_metadata_for_turn", None)
  104. if anchor is None or not callable(hook):
  105. return metadata
  106. return hook(adapter, str(chat_id), str(anchor), metadata)
  107. # Inline markdown-stripping rules, applied in order: bold, italic, bold/italic
  108. # underscore, code fence markers, inline code, headings. Links and the newline
  109. # squeeze run after these, in that order (see ``strip_markdown``).
  110. _STRIP_RULES = (
  111. (re.compile(r"\*\*(.+?)\*\*", re.DOTALL), r"\1"),
  112. (re.compile(r"\*(.+?)\*", re.DOTALL), r"\1"),
  113. (re.compile(r"\b__(?![\s_])(.+?)(?<![\s_])__\b", re.DOTALL), r"\1"),
  114. (re.compile(r"\b_(?![\s_])(.+?)(?<![\s_])_\b", re.DOTALL), r"\1"),
  115. (re.compile(r"```[a-zA-Z0-9_+-]*\n?"), ""),
  116. (re.compile(r"`(.+?)`"), r"\1"),
  117. (re.compile(r"^#{1,6}\s+", re.MULTILINE), ""),
  118. )
  119. _MD_LINK_RE = re.compile(r"\[([^\]]+)\]\(([^\)]+)\)")
  120. _HTTP_TARGET_RE = re.compile(r"https?://", re.IGNORECASE)
  121. _NEWLINE_SQUEEZE_RE = re.compile(r"\n{3,}")
  122. def _keep_link_target(match: re.Match[str]) -> str:
  123. r"""``[label](https://url)`` -> ``label\nurl``.
  124. The bare URL is the only thing a platform with its own data detection
  125. (iMessage) can turn back into a tap target, so dropping it makes the link
  126. unreachable rather than merely unformatted. Non-http targets (``mailto:``,
  127. relative paths) are not auto-linked, so they keep the label-only behaviour.
  128. """
  129. label, target = match.group(1).strip(), match.group(2).strip()
  130. if not _HTTP_TARGET_RE.match(target):
  131. return match.group(1)
  132. return target if label == target else f"{label}\n{target}"
  133. def strip_markdown(text: str, *, keep_link_targets: bool = False) -> str:
  134. r"""Strip markdown formatting for plain-text platforms (SMS, iMessage, etc.).
  135. ``keep_link_targets`` rewrites ``[label](https://url)`` as ``label\nurl``
  136. instead of discarding the URL; pass it on platforms that auto-link bare URLs.
  137. """
  138. for pattern, repl in _STRIP_RULES:
  139. text = pattern.sub(repl, text)
  140. text = _MD_LINK_RE.sub(_keep_link_target if keep_link_targets else r"\1", text)
  141. return _NEWLINE_SQUEEZE_RE.sub("\n\n", text).strip()
  142. class ThreadParticipationTracker:
  143. """Persistent set of threads the bot has participated in (``<platform>_threads.json``);
  144. ``thread_id in tracker`` checks membership, ``tracker.mark(thread_id)`` persists."""
  145. _MAX_TRACKED = 500
  146. def __init__(self, platform_name: str, max_tracked: int = 500):
  147. self._platform = platform_name
  148. self._max_tracked = max_tracked
  149. # ``mark_async`` runs the persist on a worker thread, which removes the
  150. # accidental serialization the event loop used to provide. Two locks,
  151. # never nested the other way round:
  152. # ``_lock`` guards ONLY the in-memory set (``_remember``,
  153. # ``__contains__``, ``clear`` and the snapshot in
  154. # ``_save``); held for microseconds, safe on the loop.
  155. # ``_io_lock`` worker-only; serializes snapshot+write in ``_save`` so
  156. # two persists cannot interleave and lose an entry.
  157. # ``_save`` must NOT hold ``_lock`` across ``os.replace``: the adapters'
  158. # ``thread_id in tracker`` gate runs on the loop thread and would stall
  159. # for the whole rename.
  160. self._lock = threading.Lock()
  161. self._io_lock = threading.Lock()
  162. self._threads: dict[str, None] = dict.fromkeys(str(t) for t in self._load())
  163. def _state_path(self) -> Path:
  164. from hermes_constants import get_hermes_home
  165. return get_hermes_home() / f"{self._platform}_threads.json"
  166. def _load(self) -> list[str]:
  167. try:
  168. data = json.loads(self._state_path().read_text(encoding="utf-8-sig"))
  169. except Exception:
  170. return []
  171. return [str(thread_id) for thread_id in data] if isinstance(data, list) else []
  172. def _save(self) -> None:
  173. with self._io_lock:
  174. with self._lock:
  175. thread_list = list(self._threads)
  176. if len(thread_list) > self._max_tracked:
  177. thread_list = thread_list[-self._max_tracked:]
  178. self._threads = dict.fromkeys(thread_list)
  179. atomic_json_write(self._state_path(), thread_list, indent=None)
  180. def _remember(self, thread_id: str) -> bool:
  181. """Record *thread_id* in memory; ``True`` when a persist is still owed.
  182. The in-memory half stays synchronous even for :meth:`mark_async` so that
  183. a ``thread_id in tracker`` check immediately after marking is correct --
  184. the mention-gating logic in the adapters depends on that.
  185. """
  186. with self._lock:
  187. if thread_id in self._threads:
  188. return False
  189. self._threads[thread_id] = None
  190. return True
  191. def mark(self, thread_id: str) -> None:
  192. """Mark *thread_id* as participated and persist.
  193. Blocking: ends in ``atomic_json_write`` -> ``os.replace``. Coroutines
  194. must use :meth:`mark_async` instead.
  195. """
  196. if self._remember(thread_id):
  197. self._save()
  198. async def mark_async(self, thread_id: str) -> None:
  199. """Off-loop form of :meth:`mark`.
  200. ``_save`` ends in ``atomic_json_write`` -> ``os.replace``, whose
  201. duration is unbounded under filesystem pressure. Every caller of this
  202. tracker sits on an inbound-message coroutine in the Discord and Matrix
  203. adapters, so the rename must not be paid inline on the event loop -- it
  204. stalls every other adapter's polling and every in-flight turn for as
  205. long as it runs.
  206. """
  207. if self._remember(thread_id):
  208. await _to_thread(self._save)
  209. def __contains__(self, thread_id: str) -> bool:
  210. with self._lock:
  211. return thread_id in self._threads
  212. def clear(self) -> None:
  213. with self._lock:
  214. self._threads.clear()
  215. async def send_chunks(chunks: list, send_one) -> Any:
  216. """Send ``chunks`` in order through ``send_one(chunk) -> SendResult``, stopping at the first failure.
  217. A failure after earlier chunks landed carries the ``partial_overflow`` contract that
  218. ``BasePlatformAdapter._is_partial_delivery`` reads, so no caller (send retry, plain-text
  219. fallback, cron standalone fallback) re-sends the head the recipient already has.
  220. """
  221. from gateway.platforms.base import SendResult
  222. result = SendResult(success=False, error="nothing to send")
  223. for delivered, chunk in enumerate(chunks):
  224. result = await send_one(chunk)
  225. if not result.success:
  226. if delivered:
  227. raw = dict(result.raw_response) if isinstance(result.raw_response, dict) else {}
  228. raw.update(partial_overflow=True, delivered_chunks=delivered, total_chunks=len(chunks))
  229. result.raw_response = raw
  230. break
  231. return result
  232. def redact_phone(phone: str) -> str:
  233. """Redact a phone number for logging, preserving country code and last 4."""
  234. if not phone:
  235. return "<none>"
  236. if len(phone) <= 8:
  237. return phone[:2] + "****" + phone[-2:] if len(phone) > 4 else "****"
  238. return phone[:4] + "****" + phone[-4:]
  239. # ─── GFM table → bullets. Discord calls convert_table_to_bullets(); Telegram imports the
  240. # primitives but keeps its own MarkdownV2-aware renderer.
  241. # Delimiter row: optional outer pipes, dash cells (optional alignment colons). Requires at
  242. # least one internal '|' so a lone '---' rule is NOT matched.
  243. TABLE_SEPARATOR_RE = re.compile(r'^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*){1,}\|?\s*$')
  244. def is_table_row(line: str) -> bool:
  245. """Return True if *line* could plausibly be a table data row."""
  246. return '|' in line.strip()
  247. def split_markdown_table_row(line: str) -> list[str]:
  248. """Split a GFM table row into stripped cells (delegates to agent.markdown_tables)."""
  249. from agent.markdown_tables import split_table_row
  250. return split_table_row(line)
  251. def _render_table_block(table_block: list[str]) -> str:
  252. """Render a GFM table as bold-heading + bullet groups (same alignment logic as Telegram's
  253. renderer: without a row-label column the full row is data and the heading bullet is skipped)."""
  254. headers = split_markdown_table_row(table_block[0]) if len(table_block) >= 3 else []
  255. if len(headers) < 2:
  256. return "\n".join(table_block)
  257. has_row_label_col = len(split_markdown_table_row(table_block[2])) == len(headers) + 1
  258. rendered_groups: list[str] = []
  259. for index, row in enumerate(table_block[2:], start=1):
  260. cells = split_markdown_table_row(row)
  261. if has_row_label_col:
  262. heading = cells[0] if cells and cells[0] else f"Row {index}"
  263. data_cells = cells[1:]
  264. else:
  265. heading = next((cell for cell in cells if cell), f"Row {index}")
  266. data_cells = cells
  267. data_cells = (data_cells + [""] * len(headers))[: len(headers)]
  268. bullets = [f"• {header}: {value}" for header, value in zip(headers, data_cells)
  269. if has_row_label_col or value != heading]
  270. rendered_groups.append("\n".join([f"**{heading}**", *bullets]))
  271. return "\n\n".join(rendered_groups)
  272. def convert_table_to_bullets(text: str) -> str:
  273. """Rewrite GFM pipe tables into bold-heading + bullet groups; fenced code is left alone."""
  274. if '|' not in text or '-' not in text:
  275. return text
  276. lines = text.split('\n')
  277. out: list[str] = []
  278. in_fence = False
  279. i = 0
  280. while i < len(lines):
  281. line = lines[i]
  282. is_fence_line = line.lstrip().startswith('```')
  283. in_fence ^= is_fence_line
  284. if not (in_fence or is_fence_line) and '|' in line and i + 1 < len(lines) \
  285. and TABLE_SEPARATOR_RE.match(lines[i + 1]):
  286. j = i + 2
  287. while j < len(lines) and is_table_row(lines[j]):
  288. j += 1
  289. out.append(_render_table_block(lines[i:j]))
  290. i = j
  291. else:
  292. out.append(line)
  293. i += 1
  294. return '\n'.join(out)
  295. def compile_mention_patterns(raw, *, log_prefix: str, platform_label: str | None = None,
  296. display_label: str | None = None, defaults: list[str] | None = None,
  297. logger_: logging.Logger | None = None) -> list[re.Pattern]:
  298. """Compile regex wake-word/mention patterns from config or env values.
  299. * **Config-style** (dingtalk, telegram): pass ``platform_label``. ``raw`` must be a
  300. list or string (else warn + ``[]``); non-string entries skipped; info log on load.
  301. * **Wakeword-style** (photon, bluebubbles): pass ``defaults``. ``raw`` may be None
  302. (defaults), a string (JSON list or comma/newline separated), a list, or a scalar.
  303. ``log_prefix`` is interpolated into every log line so per-adapter output stays
  304. byte-identical to the historical inline implementations.
  305. """
  306. log = logger_ or logger
  307. def _compile(patterns, warn_fmt, *warn_args):
  308. compiled: list[re.Pattern] = []
  309. for pattern in patterns:
  310. try:
  311. compiled.append(re.compile(pattern, re.IGNORECASE))
  312. except re.error as exc:
  313. log.warning(warn_fmt, log_prefix, *warn_args, pattern, exc)
  314. return compiled
  315. if platform_label is not None:
  316. display = display_label or platform_label
  317. if raw is None:
  318. return []
  319. patterns = [raw] if isinstance(raw, str) else raw
  320. if not isinstance(patterns, list):
  321. log.warning("[%s] %s mention_patterns must be a list or string; got %s",
  322. log_prefix, platform_label, type(patterns).__name__)
  323. return []
  324. compiled = _compile([p for p in patterns if isinstance(p, str) and p.strip()],
  325. "[%s] Invalid %s mention pattern %r: %s", display)
  326. if compiled:
  327. log.info("[%s] Loaded %d %s mention pattern(s)", log_prefix, len(compiled), display)
  328. return compiled
  329. if raw is None:
  330. patterns = list(defaults or [])
  331. elif isinstance(raw, str):
  332. text = raw.strip()
  333. try:
  334. loaded = json.loads(text) if text else []
  335. except Exception:
  336. loaded = None
  337. patterns = loaded if isinstance(loaded, list) else [
  338. part.strip() for line in text.splitlines() for part in line.split(",")]
  339. else:
  340. patterns = raw if isinstance(raw, list) else [raw]
  341. texts = [t for t in (str(p).strip() for p in patterns) if t]
  342. return _compile(texts, "[%s] Invalid mention pattern %r: %s")
  343. # ─── Fence-Aware Markdown Chunking ───────────────────────────────────────────
  344. # Shared core for gateway/stream_consumer.py (``prefer_paragraphs=False, balance_fences=True``),
  345. # yuanbao (``prefer_paragraphs=True, balance_fences=False``) and weixin (``greedy_pack_blocks``).
  346. def text_has_unclosed_fence(text: str) -> bool:
  347. """Return True when *text* ends inside an unclosed ``` code fence."""
  348. return sum(line.startswith('```') for line in text.split('\n')) % 2 == 1
  349. def text_ends_with_table_row(text: str) -> bool:
  350. """True when the last non-empty line starts and ends with ``|``."""
  351. return _is_pipe_row(text.rstrip().split('\n')[-1])
  352. def is_fence_atom(text: str) -> bool:
  353. """True when an atomic block is a code block (starts with ```)."""
  354. return text.lstrip().startswith('```')
  355. def _is_pipe_row(line: str) -> bool:
  356. stripped = line.strip()
  357. return stripped.startswith('|') and stripped.endswith('|')
  358. def is_table_atom(text: str) -> bool:
  359. """True when an atomic block is a table (first line is ``|...|``)."""
  360. return _is_pipe_row(text.split('\n')[0])
  361. _SENTENCE_END_NEWLINE_RE = re.compile(r'[。!?.!?]\n')
  362. def _cp_budget(text, budget, len_fn):
  363. """Code-point count of the longest prefix of *text* within *budget* ``len_fn`` units
  364. (callers guarantee ``len_fn(text) > budget``, so plain ``len`` needs no search)."""
  365. if len_fn is len:
  366. return budget
  367. from gateway.platforms.base import _custom_unit_to_cp # heavyweight; lazy
  368. return _custom_unit_to_cp(text, budget, len_fn)
  369. def split_at_paragraph_boundary(text, max_chars, len_fn=None):
  370. """Split at the nearest paragraph boundary within *max_chars*; return (head, tail).
  371. Priority: blank line → newline after sentence-ending punctuation (CJK/ASCII)
  372. → last newline → forced split at the window boundary. ``head + tail == text``
  373. always holds. *len_fn* measures in custom units (e.g. UTF-16 code units).
  374. """
  375. _len = len_fn or len
  376. if _len(text) <= max_chars:
  377. return text, ''
  378. window = text[:_cp_budget(text, max_chars, _len)]
  379. pos = window.rfind('\n\n')
  380. sentence_ends = [m.end() for m in _SENTENCE_END_NEWLINE_RE.finditer(window)]
  381. cut = pos + 2 if pos > 0 else (sentence_ends[-1] if sentence_ends else 0)
  382. if not cut:
  383. pos = window.rfind('\n')
  384. cut = pos + 1 if pos > 0 else len(window)
  385. return text[:cut], text[cut:]
  386. def split_markdown_atoms(text: str) -> list[str]:
  387. """Split markdown into indivisible atoms: fenced code blocks, tables
  388. (consecutive ``|...|`` lines) and paragraphs. Blank lines belong to no atom."""
  389. atoms: list[str] = []
  390. current_lines: list[str] = []
  391. in_fence = False
  392. def _flush_current() -> None:
  393. if current_lines:
  394. atom = '\n'.join(current_lines)
  395. if atom.strip():
  396. atoms.append(atom)
  397. current_lines.clear()
  398. for line in text.split('\n'):
  399. if in_fence:
  400. current_lines.append(line)
  401. if line.startswith('```'):
  402. in_fence = False
  403. _flush_current()
  404. elif line.startswith('```'):
  405. _flush_current()
  406. in_fence = True
  407. current_lines.append(line)
  408. elif line.strip() == '':
  409. _flush_current()
  410. else:
  411. # A table line and a non-table line never share an atom.
  412. if current_lines and _is_pipe_row(current_lines[-1]) != _is_pipe_row(line):
  413. _flush_current()
  414. current_lines.append(line)
  415. _flush_current()
  416. return atoms
  417. def infer_block_separator(prev_chunk: str, next_chunk: str) -> str:
  418. """``'\\n'`` when the boundary sits at a code fence or a continued table, else ``'\\n\\n'``."""
  419. prev_trimmed = prev_chunk.rstrip()
  420. next_trimmed = next_chunk.lstrip()
  421. if prev_trimmed.endswith('```') or next_trimmed.startswith('```'):
  422. return '\n'
  423. if text_ends_with_table_row(prev_chunk) and next_trimmed and _is_pipe_row(next_trimmed.split('\n')[0]):
  424. return '\n'
  425. return '\n\n'
  426. def merge_streaming_fences(chunks: list[str]) -> list[str]:
  427. """Rejoin chunks truncated mid-fence: while chunk *i* has an unclosed fence
  428. and a successor exists, merge the successor in via :func:`infer_block_separator`."""
  429. result: list[str] = []
  430. i = 0
  431. while i < len(chunks):
  432. current = chunks[i]
  433. while text_has_unclosed_fence(current) and i + 1 < len(chunks):
  434. current = current + infer_block_separator(current, chunks[i + 1]) + chunks[i + 1]
  435. i += 1
  436. result.append(current)
  437. i += 1
  438. return result
  439. def balance_fences_across_chunks(chunks: list[str]) -> list[str]:
  440. """Close orphaned ``` fences at each chunk boundary and reopen (with the
  441. original language tag) on the next, so every chunk is fence-balanced alone."""
  442. if len(chunks) <= 1:
  443. return chunks
  444. out: list[str] = []
  445. carry_lang = None
  446. for chunk in chunks:
  447. body = f"```{carry_lang}\n{chunk}" if carry_lang is not None else chunk
  448. in_code, lang = fence_state_after(chunk, carry_lang is not None, carry_lang or "")
  449. carry_lang = lang if in_code else None
  450. out.append(body + "\n```" if in_code else body)
  451. return out
  452. def fence_state_after(text: str, in_code: bool = False, lang: str = "") -> tuple[bool, str]:
  453. """Walk ``text`` line by line toggling on ``` lines; return the final (in_code, lang)."""
  454. for line in text.split("\n"):
  455. stripped = line.strip()
  456. if stripped.startswith("```"):
  457. tag = stripped[3:].split()
  458. in_code, lang = (False, "") if in_code else (True, tag[0] if tag else "")
  459. return in_code, lang
  460. def greedy_pack_blocks(blocks, max_length, len_fn=None, sep="\n\n", overflow=None):
  461. """Greedily pack *blocks* (joined with *sep*) into chunks of at most *max_length*; an
  462. oversized block goes through *overflow(block)* (-> list of chunks) if given, else as-is."""
  463. _len = len_fn or len
  464. packed: list[str] = []
  465. current = ""
  466. for block in blocks:
  467. candidate = block if not current else f"{current}{sep}{block}"
  468. if _len(candidate) <= max_length:
  469. current = candidate
  470. continue
  471. if current:
  472. packed.append(current)
  473. current = ""
  474. if _len(block) <= max_length:
  475. current = block
  476. elif overflow is not None:
  477. packed.extend(overflow(block))
  478. else:
  479. packed.append(block)
  480. if current:
  481. packed.append(current)
  482. return packed
  483. def split_text_fence_aware(text, limit, len_fn=None, *, prefer_paragraphs=True,
  484. balance_fences=False):
  485. """Split markdown into chunks of at most *limit*, respecting fences.
  486. ``prefer_paragraphs=True`` (yuanbao-derived): atoms (fences, tables, paragraphs) are
  487. greedily merged, oversized non-atomic chunks split at paragraph boundaries, small
  488. neighbours re-merged; a single atom larger than *limit* is emitted oversize, not broken.
  489. ``prefer_paragraphs=False`` (stream_consumer-derived): newline-preferred hard splitting
  490. with headroom reserved for fence markers. ``balance_fences=True`` closes/reopens fences
  491. at chunk boundaries so each chunk renders standalone.
  492. """
  493. if not text:
  494. return []
  495. if prefer_paragraphs:
  496. chunks = _chunk_markdown_paragraphs(text, limit, len_fn)
  497. else:
  498. chunks = _chunk_newline_preferred(text, limit, len_fn or len)
  499. return balance_fences_across_chunks(chunks) if balance_fences else chunks
  500. def _chunk_markdown_paragraphs(text, max_chars, len_fn=None):
  501. """Yuanbao-derived paragraph/atom chunking pipeline (see module docs)."""
  502. _len = len_fn or len
  503. if _len(text) <= max_chars:
  504. return [text]
  505. # Phase 2: greedy merge; oversized fence/table atoms stay indivisible.
  506. chunks: list[str] = []
  507. indivisible_set: set[int] = set()
  508. current_parts: list[str] = []
  509. current_len = 0
  510. for atom in split_markdown_atoms(text):
  511. atom_len = _len(atom)
  512. sep_len = 2 if current_parts else 0
  513. if current_len + sep_len + atom_len > max_chars and current_parts:
  514. chunks.append('\n\n'.join(current_parts))
  515. current_parts, current_len, sep_len = [], 0, 0
  516. if (not current_parts and atom_len > max_chars
  517. and (is_fence_atom(atom) or is_table_atom(atom))):
  518. indivisible_set.add(len(chunks))
  519. chunks.append(atom)
  520. continue
  521. current_parts.append(atom)
  522. current_len += sep_len + atom_len
  523. if current_parts:
  524. chunks.append('\n\n'.join(current_parts))
  525. # Phase 3: split still-oversized divisible chunks at paragraph boundaries.
  526. result: list[str] = []
  527. for idx, chunk in enumerate(chunks):
  528. if _len(chunk) <= max_chars or idx in indivisible_set or text_has_unclosed_fence(chunk):
  529. result.append(chunk)
  530. continue
  531. remaining = chunk
  532. while _len(remaining) > max_chars:
  533. head, remaining = split_at_paragraph_boundary(remaining, max_chars, len_fn=len_fn)
  534. if not head:
  535. head, remaining = remaining[:max_chars], remaining[max_chars:]
  536. if head:
  537. result.append(head)
  538. if remaining:
  539. result.append(remaining)
  540. # Phase 4: merge small chunks with neighbours.
  541. merged: list[str] = result[:1]
  542. for chunk in result[1:]:
  543. combined = merged[-1] + '\n\n' + chunk
  544. if _len(combined) <= max_chars:
  545. merged[-1] = combined
  546. else:
  547. merged.append(chunk)
  548. return [c for c in merged if c]
  549. def _chunk_newline_preferred(text, limit, len_fn):
  550. """Stream-consumer-derived newline-preferred splitting (no balancing)."""
  551. if len_fn(text) <= limit:
  552. return [text]
  553. # Reserve headroom for fence markers a balancing pass may add.
  554. split_limit = max(limit - 16, limit // 2, 1) if "```" in text else limit
  555. chunks: list[str] = []
  556. remaining = text
  557. while len_fn(remaining) > split_limit:
  558. budget = _cp_budget(remaining, split_limit, len_fn)
  559. split_at = remaining.rfind("\n", 0, budget)
  560. if split_at < budget // 2:
  561. split_at = budget
  562. chunks.append(remaining[:split_at])
  563. remaining = remaining[split_at:].lstrip("\n")
  564. if remaining:
  565. chunks.append(remaining)
  566. return chunks
  567. # ─── Discord channel obfuscation (Bot API change, mandatory Nov 16 2026) ───────
  568. # Channels a bot lacks VIEW_CHANNEL on are still dispatched over the Gateway with the
  569. # name replaced by "___hidden___", sensitive fields nulled and flag 1 << 17 set; over
  570. # HTTP they are omitted. https://discord.com/developers/docs/change-log (Aug 12, 2026)
  571. DISCORD_CHANNEL_OBFUSCATED_FLAG = 1 << 17
  572. DISCORD_OBFUSCATED_CHANNEL_NAME = "___hidden___"
  573. def is_discord_channel_obfuscated(channel) -> bool:
  574. """True when a discord.py channel object is an obfuscated placeholder.
  575. Skip these wherever guild channels are enumerated (channel directory, message
  576. backfill): history reads and sends always fail with a permission error. Flag
  577. first; the sentinel name is the fallback for discord.py builds that don't
  578. expose the flag (a visible channel literally named ``___hidden___`` is then
  579. also skipped — deliberate bias toward hiding). #90154
  580. """
  581. try:
  582. flag_value = channel.flags.value
  583. except AttributeError:
  584. flag_value = None
  585. if isinstance(flag_value, int) and flag_value & DISCORD_CHANNEL_OBFUSCATED_FLAG:
  586. return True
  587. return getattr(channel, "name", None) == DISCORD_OBFUSCATED_CHANNEL_NAME
  588. def platform_http_client(**kwargs: Any) -> Any:
  589. """``httpx.AsyncClient`` for a platform adapter: connect-time SSRF validation
  590. (``tools.url_safety.create_ssrf_safe_async_client``), every redirect hop re-checked
  591. (``gateway.platforms.base.ssrf_redirect_guard``, run before any caller ``response`` hooks) and the
  592. shared keepalive pool limits (``platform_httpx_limits()``; pass ``limits=None`` for httpx's
  593. defaults). Defaults ``timeout=30.0`` and ``follow_redirects=True``; other kwargs go to httpx."""
  594. from gateway.platforms._http_client_limits import platform_httpx_limits
  595. from gateway.platforms.base import ssrf_redirect_guard
  596. from tools.url_safety import create_ssrf_safe_async_client
  597. hooks = dict(kwargs.pop("event_hooks", None) or {})
  598. hooks["response"] = [ssrf_redirect_guard, *hooks.get("response", ())]
  599. kwargs.setdefault("timeout", 30.0)
  600. kwargs.setdefault("follow_redirects", True)
  601. if "limits" not in kwargs:
  602. kwargs["limits"] = platform_httpx_limits()
  603. elif kwargs["limits"] is None:
  604. del kwargs["limits"]
  605. return create_ssrf_safe_async_client(event_hooks=hooks, **kwargs)