scheduler_delivery.py 99 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953
  1. """Cron delivery: target resolution (origin/home/explicit/bot-chat/room), transcript mirroring and
  2. session seeding, live-adapter / relay / standalone send lanes, and ``_deliver_result``.
  3. Split out of ``cron.scheduler``. Import names from this module directly (``cron.scheduler`` only
  4. imports the few it calls itself). Origin-resident helpers and sibling split modules are reached
  5. late-bound (``_sched`` / module refs at the bottom) so monkeypatching the defining module works.
  6. """
  7. from __future__ import annotations
  8. import asyncio
  9. import concurrent.futures
  10. import contextlib
  11. import contextvars
  12. import logging
  13. import os
  14. import threading
  15. from dataclasses import dataclass
  16. from typing import Any, List, Optional
  17. # Log-record parity with the origin module.
  18. logger = logging.getLogger("cron.scheduler")
  19. # Validates user-supplied delivery platform names, preventing env-var enumeration via crafted names.
  20. # Plugin platforms are known through ``PlatformEntry.cron_deliver_env_var``; these rows have none:
  21. # ``webhook`` is core-only and ``wecom_callback`` (a ``wecom`` plugin sibling) delivers only to its
  22. # config.yaml ``home_channel``.
  23. _KNOWN_DELIVERY_PLATFORMS = frozenset({"webhook", "wecom_callback"})
  24. # Gateway platforms whose adapter declares ``supports_async_delivery = False`` (request/response
  25. # only, ``send()`` is a stub) — a cron report can never reach them, so they are never a
  26. # deliver=origin destination.
  27. _NON_PUSH_ORIGIN_PLATFORMS = frozenset({"api_server"})
  28. # Home-target platforms tried first, in this order, by ``deliver=origin`` with no origin (first
  29. # platform with a home chat wins) and ``deliver=all``; their env var is the plugin's
  30. # ``cron_deliver_env_var``. Other plugin platforms follow by name (a fresh process loads deferred
  31. # entries in name order, so this is the order they had when they trailed a static table).
  32. _HOME_TARGET_PRIORITY = (
  33. "matrix", "telegram", "discord", "slack", "signal", "mattermost", "sms", "email", "dingtalk",
  34. "feishu", "wecom", "weixin", "bluebubbles", "qqbot", "whatsapp", "whatsapp_cloud",
  35. )
  36. # Back-compat: primary env var -> previous name, consulted when the primary is unset.
  37. _LEGACY_HOME_TARGET_ENV_VARS = {"QQBOT_HOME_CHANNEL": "QQ_HOME_CHANNEL"}
  38. def _resolve_cron_surface_mode(pconfig, logical_platform_name: str) -> str:
  39. """Return ``"in_channel"`` or ``"thread"`` (default) for a platform config.
  40. Native: flat ``platforms.<p>.extra.cron_continuable_surface``. Relay-fronted:
  41. ``platforms.relay.extra.<logical>.cron_continuable_surface`` (same sub-block as the relay's
  42. Slack knobs); the sub-block wins and is scoped to its logical platform. Unlike
  43. _relay_slack_extra, a sub-block that omits the knob falls back to the flat key (deliberate —
  44. the flat key must keep working), so a flat value applies to EVERY platform the relay fronts.
  45. """
  46. with contextlib.suppress(Exception):
  47. extra = getattr(pconfig, "extra", None) or {}
  48. sub = extra.get(str(logical_platform_name or "").lower())
  49. raw = sub.get("cron_continuable_surface") if isinstance(sub, dict) else None
  50. if raw is None:
  51. raw = extra.get("cron_continuable_surface")
  52. if raw is not None and str(raw).strip().lower() == "in_channel":
  53. return "in_channel"
  54. return "thread"
  55. def _resolve_origin(job: dict) -> Optional[dict]:
  56. """Extract origin info from a job. Non-dict origins (provenance strings, hand-edited
  57. jobs.json) are treated as missing — otherwise every fire crashed on ``origin.get``.
  58. Without this guard, a job tagged with e.g. ``"combined-digest-replaces-x-and-y"`` crashed every fire
  59. attempt with ``'str' object has no attribute 'get'`` — ``mark_job_run`` recorded the failure, but the
  60. next tick re-loaded the same poisoned origin and crashed identically until the field was patched
  61. manually (#18722).
  62. """
  63. origin = job.get("origin")
  64. if isinstance(origin, dict) and origin.get("platform") and origin.get("chat_id"):
  65. # Jobs stamped before non-push origins stopped being captured (#69304): the api_server
  66. # adapter's send() is a stub, so honouring this origin fails every fire with
  67. # last_status=ok. Treat it as missing so deliver=origin takes the home-channel fallback.
  68. if str(origin["platform"]).lower() in _NON_PUSH_ORIGIN_PLATFORMS:
  69. return None
  70. return origin
  71. return None
  72. def _cron_mirror_delivery_enabled(job: dict, cfg: Optional[dict] = None) -> bool:
  73. """Whether a cron delivery is also mirrored into the target chat's session transcript.
  74. Default OFF. Precedence: per-job ``attach_to_session`` (bool) → global
  75. ``cron.mirror_delivery`` → False. CARVE-OUT: the ``in_channel`` surface seeds its session
  76. independently of this knob (the seed IS that feature) — this governs only the thread-surface
  77. mirror. ``mirror_to_session`` runs at a turn boundary, so it is alternation- and cache-safe.
  78. """
  79. per_job = job.get("attach_to_session")
  80. if isinstance(per_job, bool):
  81. return per_job
  82. try:
  83. if cfg is None:
  84. cfg = _sched.load_config() or {}
  85. return bool((cfg.get("cron", {}) or {}).get("mirror_delivery", False))
  86. except Exception:
  87. return False
  88. def _target_matches_origin(origin: dict, platform_name: str, chat_id: str,
  89. thread_id: Optional[str]) -> bool:
  90. """True when a delivery target is the job's own origin conversation. A pinned origin
  91. thread_id must match — a target without it is a different lane. Mirror eligibility for
  92. non-origin targets is decided by ``_target_mirror_eligible``."""
  93. if (
  94. not origin
  95. or str(origin.get("platform", "")).lower() != str(platform_name).lower()
  96. or str(origin.get("chat_id", "")) != str(chat_id)
  97. ):
  98. return False
  99. origin_thread = origin.get("thread_id")
  100. return origin_thread is None or str(origin_thread) == str(thread_id or "")
  101. # Provenance rank for the dedup OR-merge in _resolve_delivery_targets (higher = stronger mirror
  102. # claim). Broadcasts rank 0 so "origin,all"/"all,origin" keep the origin tag regardless of order.
  103. _MIRROR_PROVENANCE_RANK = {"origin": 3, "origin_fallback": 2, "home": 2, "explicit": 1}
  104. def _target_mirror_eligible(
  105. job: dict, target: dict, *, global_mirror: bool, origin_match: Optional[bool] = None) -> bool:
  106. """Whether a resolved delivery target may receive the transcript mirror. Origin targets:
  107. always. ``origin_fallback`` (deliver=origin with no captured origin → home channel, standing
  108. in for the primary conversation) and ``home`` (user-written bare-platform token, e.g.
  109. ``deliver: slack`` — deliberately addresses that platform's home channel): same flags as a
  110. true origin. ``explicit`` ``platform:chat_id``: ONLY with per-job ``attach_to_session: true``
  111. — the global flag must never write transcripts into arbitrary explicitly-addressed chats.
  112. Untagged broadcast expansions (``all``) are never eligible. ``origin_match`` may be
  113. precomputed."""
  114. if origin_match is None:
  115. origin = _resolve_origin(job) or {}
  116. origin_match = _target_matches_origin(
  117. origin, target.get("platform", ""), target.get("chat_id", ""), target.get("thread_id"))
  118. if origin_match:
  119. return True
  120. resolved_from = target.get("_resolved_from")
  121. if resolved_from in ("origin_fallback", "home"):
  122. # Same precedence as _cron_mirror_delivery_enabled (keep in sync): a per-job False must
  123. # beat a global True even for callers that don't pre-merge `global_mirror`.
  124. per_job = job.get("attach_to_session")
  125. return per_job if isinstance(per_job, bool) else bool(global_mirror)
  126. if resolved_from == "explicit":
  127. return job.get("attach_to_session") is True
  128. return False
  129. def _inchannel_seed_allowed(*, is_dm: bool, user_id: Optional[str]) -> bool:
  130. """Whether the flat in_channel seed may run. Group keys are user-isolated
  131. (``…:group:<chat_id>:<user_id>``): seeding without a real user_id creates an orphan session no
  132. reply resolves to — worse than no seed. DM keys omit user_id, so DMs are always seedable."""
  133. return bool(is_dm or user_id)
  134. def _redact_cron_payload(text: str, what: str) -> str:
  135. """Fail-closed secret redaction for anything a cron job emits outward.
  136. Every outward lane — chat message, session mirror, bot-chat turn — must apply the same policy,
  137. so the policy lives in one place. ``force=True`` because this is a safety boundary, not
  138. logging: the ``security.redact_secrets`` preference governs how much is scrubbed from the
  139. user's own logs and must not be able to turn scrubbing off on the way out to a chat (same
  140. reasoning as ``tools/delegation_live_log.py``). Empty input is returned as-is; any failure
  141. inside the redactor replaces the payload entirely rather than letting an unscanned value out.
  142. """
  143. if not text:
  144. return text
  145. try:
  146. from agent.redact import redact_sensitive_text
  147. return redact_sensitive_text(text, force=True)
  148. except Exception as e:
  149. logger.warning("Failed to redact secrets from cron %s: %s", what, e)
  150. return "[REDACTED - redaction failed]"
  151. def _cron_display_name(job: dict) -> str:
  152. """Job name/id as it appears in outward-facing text. The mirror sinks and the thread title
  153. splice the job *name* around the redacted payload, and the name is user-controlled config — a
  154. name embedding a credential would re-leak it next to the scrubbed body."""
  155. return _redact_cron_payload(job.get("name") or job.get("id", "cron"), "job name")
  156. def _cron_mirror_message(job: dict, text: str) -> str:
  157. return f"[Cron delivery: {_cron_display_name(job)}]\n{text}"
  158. def _maybe_mirror_cron_delivery(
  159. job: dict, platform_name: str, chat_id: str, mirror_text: str, thread_id: Optional[str] = None,
  160. user_id: Optional[str] = None, *, enabled: bool = False,
  161. ) -> None:
  162. """Best-effort mirror of a cron delivery into the origin chat's session. No-op unless
  163. ``enabled`` (caller resolves it, scoped to the origin target). Rides the same
  164. ``mirror_to_session`` path as ``send_message``, passing ``user_id`` so user-isolated group
  165. chats resolve to the scheduling member. All failures swallowed — a successful delivery must
  166. never be reported failed because the mirror broke."""
  167. if not enabled:
  168. return
  169. text = (mirror_text or "").strip()
  170. if not text:
  171. return
  172. try:
  173. from gateway.mirror import mirror_to_session
  174. # USER role + labelled prefix, NOT assistant: an assistant-role mirror lands
  175. # assistant→assistant and breaks strict alternation; consecutive user turns merge safely.
  176. # The brief is not the agent speaking; an assistant-role mirror lands as assistant→assistant after
  177. # the agent's last turn and breaks strict alternation (issue #2221, the exact failure #2313
  178. # removed). A user-role turn collapses safely via repair_message_sequence's consecutive-user merge
  179. # on every provider, and the prefix preserves the "this came from cron" context that the dropped
  180. # SQLite mirror metadata would otherwise lose on replay.
  181. ok = mirror_to_session(
  182. platform_name, str(chat_id), _cron_mirror_message(job, text),
  183. source_label="cron", thread_id=thread_id, user_id=user_id, role="user")
  184. if ok:
  185. logger.info(
  186. "Job '%s': mirrored delivery into %s:%s session transcript",
  187. job.get("id", "?"), platform_name, chat_id)
  188. else:
  189. logger.debug(
  190. "Job '%s': delivery mirror skipped for %s:%s "
  191. "(no matching gateway session — cold start)",
  192. job.get("id", "?"), platform_name, chat_id)
  193. except Exception as e:
  194. logger.debug(
  195. "Job '%s': delivery mirror failed for %s:%s: %s", job.get("id", "?"), platform_name,
  196. chat_id, e,
  197. )
  198. # chat_type slot a platform's adapter puts on a NON-DM in-thread reply. Discord (and the default)
  199. # key the shared "thread" lane; Slack, Matrix and Telegram (forum topics: ``_build_message_event``
  200. # types every supergroup "group") keep the parent channel/room's "group" — a seed on the wrong slot
  201. # is a row no reply ever resolves to (#111896, #112918).
  202. _THREAD_REPLY_CHAT_TYPE = {"slack": "group", "matrix": "group", "telegram": "group"}
  203. def _open_continuable_cron_thread(job: dict, adapter, chat_id: str, loop) -> Optional[str]:
  204. """Open a thread for a continuable cron job via ``adapter.create_handoff_thread``. Returns the
  205. thread_id, or ``None`` (no thread primitive / failed) = caller falls back to the DM mirror."""
  206. create_thread = getattr(adapter, "create_handoff_thread", None)
  207. if not callable(create_thread) or loop is None:
  208. return None
  209. thread_name = f"Hermes — {_cron_display_name(job)}"
  210. try:
  211. from agent.async_utils import safe_schedule_threadsafe
  212. coro = create_thread(str(chat_id), thread_name)
  213. future = safe_schedule_threadsafe(coro, loop) # type: ignore[arg-type]
  214. if future is None:
  215. return None
  216. new_thread_id = future.result(timeout=30)
  217. return str(new_thread_id) if new_thread_id else None
  218. except Exception as e:
  219. logger.debug(
  220. "Job '%s': create_handoff_thread failed on %s — falling back to "
  221. "DM-session mirror: %s",
  222. job.get("id", "?"), getattr(adapter, "name", "?"), e)
  223. return None
  224. def _seed_cron_session(
  225. job: dict, adapter, platform_name: str, chat_id: str, text: str, *, thread_id: Optional[str],
  226. chat_type: str, user_id: Optional[str], user_name: Optional[str] = None,
  227. chat_name: Optional[str], scope_id: Optional[str], discord_keys_on_thread: bool = False,
  228. ) -> bool:
  229. """Create the session row (so the mirror has a target) and mirror the brief as a USER turn.
  230. The seeded key must equal the reply's ``build_session_key``: chat_type, user_id, thread_id and
  231. scope_id (Slack team id) are all part of it, so callers pass exactly what the reply carries."""
  232. from gateway.config import Platform
  233. from gateway.session import SessionSource
  234. from gateway.mirror import mirror_to_session
  235. seeded_session_id: Optional[str] = None
  236. session_store = getattr(adapter, "_session_store", None)
  237. if session_store is not None:
  238. try:
  239. platform_enum = Platform(platform_name.lower())
  240. except (ValueError, KeyError):
  241. platform_enum = None
  242. if platform_enum is not None:
  243. # Discord keys in-thread messages with chat_id == thread_id; Slack/Telegram use the
  244. # parent channel.
  245. seed_chat_id = (
  246. str(thread_id)
  247. if discord_keys_on_thread and platform_enum == Platform.DISCORD
  248. else str(chat_id)
  249. )
  250. dest_source = SessionSource(
  251. platform=platform_enum, chat_id=seed_chat_id, chat_name=chat_name,
  252. chat_type=chat_type,
  253. user_id=user_id, user_name=user_name, thread_id=thread_id,
  254. scope_id=str(scope_id) if scope_id else None)
  255. # Create the row and pass its exact id to the mirror — origin-heuristic rediscovery
  256. # bails on populated chats.
  257. _entry = session_store.get_or_create_session(dest_source)
  258. seeded_session_id = getattr(_entry, "session_id", None)
  259. return mirror_to_session(
  260. platform_name, str(chat_id), _cron_mirror_message(job, text),
  261. source_label="cron", thread_id=thread_id, user_id=user_id, role="user",
  262. session_id=seeded_session_id,
  263. )
  264. def _seed_cron_thread_session(
  265. job: dict, adapter, platform_name: str, chat_id: str, thread_id: str, mirror_text: str,
  266. chat_name: Optional[str] = None, is_dm: bool = False, scope_id: Optional[str] = None,
  267. ) -> None:
  268. """Seed the freshly-opened cron thread's session with the brief (never raises), else the
  269. user's in-thread reply resolves to a transcript without it. Threads are participant-shared
  270. (no real user_id); a DM thread must seed ``chat_type="dm"`` — DM-thread replies route through
  271. the DM arm (``…:dm:<chat>:<thread>``), so a "thread"-typed seed is a row no DM reply hits.
  272. Non-DM threads seed the slot the platform's adapter puts on an in-thread reply
  273. (``_THREAD_REPLY_CHAT_TYPE``)."""
  274. text = (mirror_text or "").strip()
  275. if not text:
  276. return
  277. try:
  278. ok = _seed_cron_session(
  279. job, adapter, platform_name, chat_id, text,
  280. thread_id=str(thread_id),
  281. chat_type="dm" if is_dm else _THREAD_REPLY_CHAT_TYPE.get(platform_name.lower(), "thread"),
  282. user_id="system:cron", user_name="Cron", chat_name=chat_name, scope_id=scope_id,
  283. discord_keys_on_thread=True)
  284. if ok:
  285. logger.info(
  286. "Job '%s': opened continuable thread %s on %s:%s and seeded the brief",
  287. job.get("id", "?"), thread_id, platform_name, chat_id)
  288. else:
  289. logger.warning(
  290. "Job '%s': thread seed did NOT land on %s:%s thread=%s — an "
  291. "in-thread reply will not see this brief",
  292. job.get("id", "?"), platform_name, chat_id, thread_id)
  293. except Exception as e:
  294. # WARNING, not debug: a silent seed failure IS the continuation-amnesia bug.
  295. logger.warning(
  296. "Job '%s': seeding cron thread session failed for %s:%s:%s: %s",
  297. job.get("id", "?"), platform_name, chat_id, thread_id, e)
  298. def _seed_cron_channel_session(
  299. job: dict, adapter, platform_name: str, chat_id: str, mirror_text: str, *, is_dm: bool,
  300. user_id: Optional[str], chat_name: Optional[str] = None, scope_id: Optional[str] = None,
  301. ) -> bool:
  302. """Seed the FLAT (thread_id=None) session for an ``in_channel`` delivery; True on success.
  303. ``mirror_to_session`` only APPENDS to an existing session and the flat row is only created by
  304. an inbound human message, so create the row first or the brief is silently dropped. Group keys
  305. are user-isolated (``…:group:<chat_id>:<user_id>``): the seed MUST carry the origin's real
  306. user_id, not ``system:cron``; DM keys omit user_id. chat_type mirrors the inbound handler."""
  307. text = (mirror_text or "").strip()
  308. if not text:
  309. return False
  310. try:
  311. chat_type = "dm" if is_dm else "group"
  312. ok = _seed_cron_session(
  313. job, adapter, platform_name, chat_id, text,
  314. thread_id=None, # flat — the whole-channel/DM session
  315. chat_type=chat_type, user_id=str(user_id) if user_id else None,
  316. chat_name=chat_name, scope_id=scope_id,
  317. )
  318. if ok:
  319. logger.info(
  320. "Job '%s': seeded flat in_channel session on %s:%s (chat_type=%s)",
  321. job.get("id", "?"), platform_name, chat_id, chat_type)
  322. return bool(ok)
  323. except Exception as e:
  324. # WARNING, not debug: a silent seed failure IS the continuation-amnesia bug.
  325. logger.warning(
  326. "Job '%s': seeding in_channel session failed for %s:%s: %s",
  327. job.get("id", "?"), platform_name, chat_id, e)
  328. return False
  329. def _cron_job_origin_log_suffix(job: dict) -> str:
  330. """Secret-free provenance suffix (origin platform/chat/source-IP fields) for security warnings
  331. about a bad stored ``context_from`` reference, where no live request object exists."""
  332. origin = job.get("origin")
  333. if not isinstance(origin, dict):
  334. return ""
  335. fields = []
  336. for key in ("platform", "chat_id", "thread_id", "source_ip", "remote", "forwarded_for"):
  337. value = origin.get(key)
  338. if value is None:
  339. continue
  340. text = str(value).replace("\r", " ").replace("\n", " ").strip()
  341. if text:
  342. fields.append(f"origin_{key}={text[:200]!r}")
  343. return " " + " ".join(fields) if fields else ""
  344. def _plugin_cron_env_var(platform_name: str) -> str:
  345. """Cron home-channel env var registered by a plugin ``PlatformEntry.cron_deliver_env_var``."""
  346. with contextlib.suppress(Exception):
  347. from hermes_cli.plugins import discover_plugins
  348. discover_plugins() # idempotent
  349. from gateway.platform_registry import platform_registry
  350. entry = platform_registry.get(platform_name.lower())
  351. if entry and entry.cron_deliver_env_var:
  352. return entry.cron_deliver_env_var
  353. return ""
  354. def _is_known_delivery_platform(platform_name: str) -> bool:
  355. """Valid cron delivery platform: built-in, or plugin with a ``cron_deliver_env_var``."""
  356. name = platform_name.lower()
  357. return name in _KNOWN_DELIVERY_PLATFORMS or bool(_plugin_cron_env_var(name))
  358. def _resolve_home_env_var(platform_name: str) -> str:
  359. """Env var name for a platform's cron home channel (``PlatformEntry.cron_deliver_env_var``)."""
  360. return _plugin_cron_env_var(platform_name.lower())
  361. def _get_config_home_channel(platform_name: str):
  362. """Persisted ``HomeChannel`` from gateway config — the canonical store ``/sethome`` writes.
  363. The ``<PLATFORM>_HOME_CHANNEL`` env var is only a best-effort mirror; relay-fronted platforms
  364. may exist solely in config.yaml, so reading only the env mirror would drop their delivery."""
  365. try:
  366. from gateway.config import load_gateway_config, Platform
  367. return load_gateway_config().get_home_channel(Platform(platform_name.lower()))
  368. except Exception:
  369. logger.debug(
  370. "config home_channel lookup failed for platform %r", platform_name, exc_info=True)
  371. return None
  372. def _home_env_lookup(env_var: str, suffix: str = "", *, strip: bool = False) -> str:
  373. """Value of ``<env_var><suffix>``, falling back to the legacy name (same suffix) when unset.
  374. Reads via ``get_secret``, not ``os.getenv``: in a multiplex gateway the tick runs with the
  375. job-owning profile's secret scope (run_one_job sets it), so this resolves the OWNING profile's
  376. value, not the host environ. ``os.getenv`` only if the scope module is missing."""
  377. try:
  378. from agent.secret_scope import get_secret
  379. except Exception:
  380. get_secret = None # type: ignore
  381. def read(name: str) -> str:
  382. value = (get_secret(name, "") or "") if get_secret is not None else os.getenv(name, "")
  383. return value.strip() if strip else value
  384. value = read(env_var + suffix)
  385. legacy = _LEGACY_HOME_TARGET_ENV_VARS.get(env_var)
  386. if not value and legacy:
  387. value = read(legacy + suffix)
  388. return value
  389. def _env_home_target_chat_id(platform_name: str) -> str:
  390. """Home chat id from the env mirror only (no config).
  391. Reads through ``get_secret`` (not raw ``os.getenv``) so a profile-scoped secret scope wins in a
  392. multiplex gateway. ``DISCORD_HOME_CHANNEL`` lives in each profile's ``.env``; in a multiplex process the
  393. winning cron tick runs with the job-owning profile's scope installed (run_one_job sets it), so reading
  394. via ``get_secret`` resolves the OWNING profile's chat id rather than the host process's ``os.environ``
  395. (#83182, chat-id leg — the token leg was fixed earlier; chat id / thread id resolve through the same
  396. leak).
  397. """
  398. env_var = _resolve_home_env_var(platform_name)
  399. return _home_env_lookup(env_var) if env_var else ""
  400. def _get_home_target_chat_id(platform_name: str) -> str:
  401. """Home target chat id: env var (first, so operator overrides win) → legacy env var →
  402. config.yaml ``home_channel``."""
  403. value = _env_home_target_chat_id(platform_name)
  404. if value:
  405. return value
  406. home = _get_config_home_channel(platform_name)
  407. return str(home.chat_id) if home is not None and home.chat_id else ""
  408. def _get_home_target_thread_id(platform_name: str) -> Optional[str]:
  409. """Optional thread/topic id for a platform home target. Telegram: ``TELEGRAM_CRON_THREAD_ID``
  410. overrides ``TELEGRAM_HOME_CHANNEL_THREAD_ID`` — in topic mode a root-DM delivery lands in the
  411. system-only lobby where the user cannot reply.
  412. When topic mode is enabled, deliveries that land in the root DM (thread_id unset) end up in the
  413. system-only lobby where the user cannot reply — the gateway returns the lobby reminder and drops
  414. ``reply_to_message_id`` (#24409). Pointing cron at a dedicated topic via this env var lets replies work
  415. as expected without changing the lobby invariant.
  416. """
  417. if platform_name.lower() == "telegram":
  418. cron_thread = _home_env_lookup("TELEGRAM_CRON_THREAD_ID", strip=True)
  419. if cron_thread:
  420. return cron_thread
  421. env_var = _resolve_home_env_var(platform_name)
  422. value = _home_env_lookup(env_var, "_THREAD_ID", strip=True) if env_var else ""
  423. if value:
  424. return value
  425. # config.yaml fallback only when the chat id also came from config (an env-provided chat id
  426. # keeps its env-provided thread semantics).
  427. if not _env_home_target_chat_id(platform_name):
  428. home = _get_config_home_channel(platform_name)
  429. if home is not None and home.thread_id:
  430. return str(home.thread_id)
  431. return None
  432. def _iter_home_target_platforms():
  433. """Iterate plugin platform names that expose a home channel, ``_HOME_TARGET_PRIORITY`` first."""
  434. with contextlib.suppress(Exception):
  435. from hermes_cli.plugins import discover_plugins
  436. discover_plugins() # idempotent
  437. from gateway.platform_registry import platform_registry
  438. named = [e.name for e in platform_registry.plugin_entries() if e.cron_deliver_env_var]
  439. yield from (n for n in _HOME_TARGET_PRIORITY if n in named)
  440. yield from (n for n in named if n not in _HOME_TARGET_PRIORITY) # registry order, as before
  441. def _relay_fronted_delivery_platforms(connected: set) -> set:
  442. """Logical platforms deliverable through a connected relay. ``get_connected_platforms()`` only
  443. sees native platforms; fronted ones come from the same ``GATEWAY_RELAY_PLATFORMS`` stamp
  444. fire-time routing uses (validation symmetric with routing). No relay -> empty set."""
  445. if "relay" not in connected:
  446. return set()
  447. try:
  448. from gateway.relay import relay_fronted_platforms
  449. return relay_fronted_platforms()
  450. except Exception:
  451. logger.debug("relay fronted-platform lookup failed", exc_info=True)
  452. return set()
  453. def cron_delivery_targets() -> list[dict]:
  454. """Platforms a cron job can auto-deliver to (single source of truth for UIs): valid delivery
  455. platform AND gateway-configured; ``home_target_set`` flags whether the home channel exists.
  456. ``{"id", "name", "home_target_set", "home_env_var"}`` dicts in canonical order; callers
  457. prepend the implicit ``local`` option themselves."""
  458. targets: list[dict] = []
  459. try:
  460. from gateway.config import load_gateway_config
  461. connected = {p.value for p in load_gateway_config().get_connected_platforms()}
  462. connected |= _relay_fronted_delivery_platforms(connected)
  463. except Exception:
  464. logger.debug("cron_delivery_targets: gateway config unavailable", exc_info=True)
  465. connected = set()
  466. for name in _iter_home_target_platforms():
  467. if name not in connected or not _is_known_delivery_platform(name):
  468. continue
  469. targets.append({
  470. "id": name,
  471. "name": name.replace("_", " ").title(),
  472. "home_target_set": bool(_get_home_target_chat_id(name)),
  473. "home_env_var": _resolve_home_env_var(name) or None})
  474. # Bot Chat targets: one per local profile (machine-local; no gateway config or home channel).
  475. try:
  476. from hermes_cli.profiles import list_profile_names
  477. for profile_name in list_profile_names():
  478. targets.append({
  479. "id": f"{BOT_CHAT_PLATFORM}:{profile_name}",
  480. "name": f"Bot Chat ({profile_name})",
  481. "home_target_set": True,
  482. "home_env_var": None})
  483. except Exception:
  484. logger.debug("cron_delivery_targets: profile listing unavailable", exc_info=True)
  485. return targets
  486. def _origin_thread_is_stale(origin: dict) -> bool:
  487. """True when a Slack origin's thread is a stale creation-turn artifact. Thread-per-message
  488. Slack stamps each top-level message id as the session thread (a KEY, not a location); old jobs
  489. carry it as ``origin.thread_id``. Heuristic: if the origin chat IS the Slack home chat, the
  490. pinned thread is that artifact and delivery goes top-level (or to the home target's thread)."""
  491. if str(origin.get("platform") or "").lower() != "slack" or not origin.get("thread_id"):
  492. return False
  493. home_chat = _get_home_target_chat_id("slack")
  494. return bool(home_chat) and str(origin.get("chat_id")) == str(home_chat)
  495. def _origin_delivery_thread(origin: dict):
  496. """The thread a deliver=origin job should use, stale stamps dropped."""
  497. if _origin_thread_is_stale(origin):
  498. return _get_home_target_thread_id("slack") or None
  499. return origin.get("thread_id")
  500. def _home_target(platform_name: str, chat_id: str, resolved_from: Optional[str] = None) -> dict:
  501. """Target dict for a platform's configured home channel (+ optional mirror provenance)."""
  502. target = {
  503. "platform": platform_name,
  504. "chat_id": chat_id,
  505. "thread_id": _get_home_target_thread_id(platform_name)}
  506. if resolved_from:
  507. target["_resolved_from"] = resolved_from
  508. return target
  509. def _resolve_single_delivery_target(
  510. job: dict, deliver_value: str, *, from_broadcast: bool = False
  511. ) -> Optional[dict]:
  512. """Resolve one concrete auto-delivery target for a cron job.
  513. ``from_broadcast`` marks a bare-platform token that was produced by expanding a broadcast
  514. token (``all``) rather than written by the user; broadcast expansions carry no mirror
  515. provenance (fan-out is never continuable), while a user-written bare platform token is a
  516. deliberate home-channel address and gets the ``home`` tag."""
  517. origin = _resolve_origin(job)
  518. if deliver_value == "local":
  519. return None
  520. room_address = parse_room_deliver_token(deliver_value)
  521. if room_address is not None:
  522. return resolve_room_target(job, room_address)
  523. # Must precede the generic platform:chat_id split so the profile name isn't parsed as chat_id.
  524. bot_chat_profile = parse_bot_chat_deliver_token(deliver_value)
  525. if bot_chat_profile is not None:
  526. return _resolve_bot_chat_target(job, bot_chat_profile)
  527. if deliver_value == "origin":
  528. if origin:
  529. return {
  530. "platform": origin["platform"],
  531. "chat_id": str(origin["chat_id"]),
  532. "thread_id": _origin_delivery_thread(origin),
  533. "_resolved_from": "origin", # provenance for _target_mirror_eligible
  534. }
  535. # No origin (API/script job): fall back to a home channel instead of silently dropping.
  536. for platform_name in _iter_home_target_platforms():
  537. chat_id = _get_home_target_chat_id(platform_name)
  538. if chat_id:
  539. logger.info(
  540. "Job '%s' has deliver=origin but no origin; falling back to %s home channel",
  541. job.get("name", job.get("id", "?")), platform_name)
  542. # Stands in for the primary conversation (NOT a broadcast): mirror-eligible.
  543. return _home_target(platform_name, chat_id, "origin_fallback")
  544. return None
  545. if ":" in deliver_value:
  546. platform_name, rest = deliver_value.split(":", 1)
  547. platform_key = platform_name.lower()
  548. from tools.send_message_tool import prepare_send_message_platforms, resolve_send_target
  549. prepare_send_message_platforms()
  550. # pass_unresolved_references: no model in the loop to react; an unknown-to-directory target
  551. # must reach the adapter as written or the job's output is silently lost.
  552. chat_id, thread_id, resolution_error = resolve_send_target(
  553. platform_key, rest, pass_unresolved_references=True)
  554. if resolution_error:
  555. logger.warning("Invalid cron delivery target '%s': %s", deliver_value, resolution_error)
  556. return None
  557. if (
  558. thread_id is None
  559. and platform_key == "slack"
  560. and origin
  561. and str(origin.get("platform") or "").lower() == platform_key
  562. and str(origin.get("chat_id")) == str(chat_id)
  563. and origin.get("thread_id")
  564. and not _origin_thread_is_stale(origin)
  565. ):
  566. thread_id = origin.get("thread_id")
  567. return {
  568. "platform": platform_name,
  569. "chat_id": chat_id,
  570. "thread_id": thread_id,
  571. "_resolved_from": "explicit", # mirror-eligible only under attach_to_session opt-in
  572. }
  573. platform_name = deliver_value
  574. home_provenance = None if from_broadcast else "home"
  575. if origin and origin.get("platform") == platform_name:
  576. chat_id = _get_home_target_chat_id(platform_name)
  577. if chat_id:
  578. return _home_target(platform_name, chat_id, home_provenance)
  579. # No home configured: falls back to the origin chat. No tag needed — the
  580. # origin-match check in _target_mirror_eligible already covers this target.
  581. return {
  582. "platform": platform_name,
  583. "chat_id": str(origin["chat_id"]),
  584. "thread_id": origin.get("thread_id"),
  585. }
  586. if not _is_known_delivery_platform(platform_name):
  587. return None
  588. chat_id = _get_home_target_chat_id(platform_name)
  589. return _home_target(platform_name, chat_id, home_provenance) if chat_id else None
  590. def _get_standalone_send_timeout() -> int:
  591. """Wall-clock bound for one standalone-lane send (#115469).
  592. ``_send_to_platform``'s gateway-loop dispatch deliberately awaits its future with no
  593. timeout ("the adapter and outer _run_async bound the wait") — but on this lane the
  594. outer runner is a bare ``asyncio.run``, not ``model_tools._run_async``, so without a
  595. bound here a reconnecting transport pins the run (and the restart drain behind it)
  596. indefinitely. Mirrors the sibling lanes: live dispatch ``future.result(timeout=60)``,
  597. thread fallback ``result(timeout=30)``. ``cron.standalone_send_timeout_seconds``;
  598. default 60."""
  599. try:
  600. cfg = _sched.load_config()
  601. value = int(cfg.get("cron", {}).get("standalone_send_timeout_seconds", 60))
  602. return value if value > 0 else 60
  603. except Exception:
  604. return 60
  605. def bot_chat_message(job: dict, content: str) -> str:
  606. """The inbound Bot Chat turn text for one cron output.
  607. Outward lane: this text becomes an inbound turn in another profile's Bot Chat through the
  608. live owner's durable admission record, so it gets the same fail-closed scrub as the chat
  609. message and the session mirror; the admitted record carries the scrubbed copy, never the raw
  610. output.
  611. """
  612. content = _redact_cron_payload(content, "bot-chat payload")
  613. job_name = _redact_cron_payload(job.get("name", job.get("id", "?")), "job name")
  614. return (
  615. f'[Cronjob "{job_name}" output — '
  616. f"scheduled job, not the user. Review it, act on anything that needs action, and "
  617. f"summarize for the chat.]\n\n{content}"
  618. )
  619. def _deliver_to_bot_chat(job: dict, content: str, profile: str, *,
  620. for_failure: bool = False) -> Optional[str]:
  621. """Admit job output to the target profile's authority as a real inbound Bot Chat turn.
  622. None means the target's durable receipt is settled (or a failure notice was recorded as
  623. ``suppressed`` — a durable disposition, never a send — and flagged on the job); anything
  624. else is an explicit unverified status string so Optional[str] callers cannot misreport
  625. admission as delivery. ``profile`` is ``""`` for the job's own profile. There is no
  626. second-writer fallback: a target whose authority is not ready gets the payload parked under
  627. the same delivery id (``cron.bot_chat_legacy.defer``) and admitted by a later tick.
  628. """
  629. import hashlib
  630. import json
  631. import uuid
  632. from hermes_constants import get_hermes_home
  633. from hermes_cli.profiles import get_profile_dir
  634. from tools.bot_live_delivery import (
  635. deliver_to_live_owner, find_canonical_live_owner, read_delivery_result,
  636. )
  637. job_id = job.get("id", "?")
  638. profile_label = profile or "(own)"
  639. message = bot_chat_message(job, content)
  640. try:
  641. source_home = get_hermes_home().resolve()
  642. home = (get_profile_dir(profile) if profile else source_home).resolve()
  643. from gateway.warning_notifications import warning_notifications_enabled
  644. from hermes_cli.config_effective import load_user_config_effective
  645. suppress_notification = for_failure and not warning_notifications_enabled(
  646. BOT_CHAT_POLICY_PLATFORM, load_user_config_effective(home / "config.yaml"))
  647. # run_one_job/claim_fire attach the durable execution id before delivery. The
  648. # transient fallback supports direct helper callers, never deduping recurring
  649. # runs by their (potentially identical) output or previous last_run timestamp.
  650. run_id = job.get("execution_id")
  651. if not run_id:
  652. run_id = job.setdefault("_bot_chat_run_id", uuid.uuid4().hex)
  653. key = hashlib.sha256(json.dumps(
  654. [str(source_home), job_id, str(run_id), str(home)],
  655. ensure_ascii=False, separators=(",", ":"),
  656. ).encode("utf-8")).hexdigest()
  657. # Read BEFORE discovery: the previous owner may have exited after accepting.
  658. receipt = read_delivery_result(home, key)
  659. target = f"bot-chat:{profile_label}"
  660. if receipt is None and suppress_notification:
  661. # Suppression is a durable disposition, not a send: the failure notice is booked
  662. # against the job and never handed to the owner (the target profile hides
  663. # warning notifications).
  664. job.setdefault("_bot_chat_delivery_receipts", {})[target] = {
  665. "status": "suppressed", "delivery_id": key}
  666. job["_notification_all_targets_suppressed"] = True
  667. return None
  668. if receipt is None:
  669. try:
  670. owner = find_canonical_live_owner(home)
  671. except ValueError: # the target's gateway is down or starting: not a disposition
  672. from cron.bot_chat_legacy import defer
  673. defer(key, job, content, home, for_failure=for_failure)
  674. job.setdefault("_bot_chat_delivery_receipts", {})[target] = {"status": "queued", "delivery_id": key}
  675. return f"{target} queued (receipt {key}): target profile not running; admitted once it is"
  676. if owner is None:
  677. return f"bot-chat delivery to profile '{profile_label}' unverified: no canonical Bot Chat"
  678. # A failure notice rides as ``diagnostic`` so the target's reply stays muted.
  679. receipt = deliver_to_live_owner(home, owner, message, delivery_id=key,
  680. **({"notification_category": "diagnostic"} if for_failure else {}))
  681. if (receipt["message"] != message
  682. or receipt.get("notification_category", "result") != ("diagnostic" if for_failure else "result")):
  683. raise ValueError("delivery id already belongs to a different payload")
  684. status = receipt["status"]
  685. receipts = job.setdefault("_bot_chat_delivery_receipts", {})
  686. receipts[target] = {"status": status, "delivery_id": key}
  687. logger.info("Job '%s': Bot Chat %s receipt=%s status=%s",
  688. job_id, profile_label, key, status)
  689. if status == "settled":
  690. return None
  691. detail = ("completion unverified; do not resend" if status in ("queued", "claimed")
  692. else receipt.get("error") or receipt.get("reason") or "not completed")
  693. return f"{target} {status} (receipt {key}): {detail}"
  694. except Exception as exc:
  695. return f"bot-chat delivery to profile '{profile_label}' unverified: {exc}"
  696. def _normalize_deliver_value(deliver) -> str:
  697. """Normalize ``deliver`` to its canonical comma-separated string; ``"local"`` when falsy.
  698. Lists/tuples (MCP clients, hand-edited jobs.json) are flattened — ``str(["telegram"])`` would
  699. yield ``"['telegram']"`` and fail resolution silently."""
  700. if deliver is None or deliver == "":
  701. return "local"
  702. if isinstance(deliver, (list, tuple)):
  703. parts = [str(p).strip() for p in deliver if str(p).strip()]
  704. return ",".join(parts) if parts else "local"
  705. return str(deliver)
  706. # Routing tokens resolve at fire time (a job outlives platform wiring). ``all`` = platforms with a
  707. # configured home chat_id (_expand_routing_tokens); ``bot-chat`` is NOT in ``all`` (costs a turn).
  708. _ROUTING_TOKENS = frozenset({"all"})
  709. # Pseudo-platform: deliver output as a real inbound turn into a profile's "Bot Chat" (not a mirror).
  710. # ``bot-chat`` = own profile; ``bot-chat:<name>`` = named profile on THIS machine.
  711. BOT_CHAT_PLATFORM = "bot-chat"
  712. # Bot Chat is the TUI/Desktop transcript, so its warning policy is display.platforms.tui.
  713. BOT_CHAT_POLICY_PLATFORM = "tui"
  714. def parse_bot_chat_deliver_token(part: str) -> Optional[str]:
  715. """``bot-chat[:<name>]`` → ``""`` (own profile), the name, or ``None`` if not a bot-chat
  716. token. Token is case-insensitive; the name is normalized later by the profile layer."""
  717. raw = (part or "").strip()
  718. lowered = raw.lower()
  719. if lowered == BOT_CHAT_PLATFORM:
  720. return ""
  721. prefix = BOT_CHAT_PLATFORM + ":"
  722. if lowered.startswith(prefix):
  723. return raw[len(prefix):].strip()
  724. return None
  725. def _resolve_bot_chat_target(job: dict, profile_arg: str) -> Optional[dict]:
  726. """Resolve a bot-chat token to a delivery target. ``""`` = own profile (no ``-p`` needed);
  727. otherwise the profile must exist locally — cross-machine delivery is intentionally unsupported
  728. so same-named profiles on other gateways can never be targeted by accident."""
  729. if not profile_arg:
  730. return {"platform": BOT_CHAT_PLATFORM, "chat_id": "", "thread_id": None}
  731. try:
  732. from hermes_cli.profiles import normalize_profile_name, profile_exists
  733. canon = normalize_profile_name(profile_arg)
  734. if not profile_exists(canon):
  735. logger.warning(
  736. "Job '%s': bot-chat delivery profile '%s' not found on this "
  737. "machine — skipping target",
  738. job.get("id", "?"), profile_arg)
  739. return None
  740. return {"platform": BOT_CHAT_PLATFORM, "chat_id": canon, "thread_id": None}
  741. except Exception:
  742. logger.warning(
  743. "Job '%s': failed to resolve bot-chat profile '%s'", job.get("id", "?"), profile_arg,
  744. exc_info=True,
  745. )
  746. return None
  747. def _expand_routing_tokens(part: str) -> list[str]:
  748. """Expand ``all`` to every home-target platform with a configured chat_id; non-tokens pass
  749. through as a single-element list."""
  750. if part.lower() not in _ROUTING_TOKENS:
  751. return [part]
  752. return [p for p in _iter_home_target_platforms() if _get_home_target_chat_id(p)]
  753. def _delivery_lane_value(job: dict, *, for_failure: bool = False):
  754. """Raw deliver-lane value for a run outcome: the failure lane when ``for_failure`` and the job
  755. overrides it, else ``deliver``. Bookkeeping (outcome classification, unresolved-origin, incident
  756. 'alerted' marking) must read the SAME lane the notice was routed through (NS-788)."""
  757. if for_failure:
  758. failure_deliver = job.get("failure_deliver")
  759. if failure_deliver is not None and str(failure_deliver).strip():
  760. return failure_deliver
  761. return job.get("deliver", "local")
  762. def _resolve_delivery_targets(job: dict, *, for_failure: bool = False) -> list[dict]:
  763. """Resolve auto-delivery targets from comma-separated ``deliver``; ``all`` expands to every
  764. platform with a home channel and combines with explicit targets. Dedup by (platform, chat_id,
  765. thread_id). ``for_failure=True`` (failure summaries, interrupted-run notices, drift/preflight
  766. alerts) resolves from ``failure_deliver`` INSTEAD when the job carries one —
  767. ``failure_deliver: local`` is the structural opt-out; absent, failures follow ``deliver``."""
  768. deliver = _normalize_deliver_value(_delivery_lane_value(job, for_failure=for_failure))
  769. if deliver == "local":
  770. return []
  771. seen = {}
  772. targets = []
  773. for raw in deliver.split(","):
  774. raw = raw.strip()
  775. if not raw:
  776. continue
  777. from_broadcast = raw.lower() in _ROUTING_TOKENS
  778. for part in _expand_routing_tokens(raw):
  779. target = _resolve_single_delivery_target(job, part, from_broadcast=from_broadcast)
  780. if not target:
  781. continue
  782. key = (target["platform"].lower(), str(target["chat_id"]), target.get("thread_id"))
  783. kept = seen.get(key)
  784. if kept is None:
  785. seen[key] = target
  786. targets.append(target)
  787. elif (
  788. # Keep origin/origin_fallback/home provenance regardless of broadcast token order.
  789. _MIRROR_PROVENANCE_RANK.get(str(target.get("_resolved_from") or ""), 0)
  790. > _MIRROR_PROVENANCE_RANK.get(str(kept.get("_resolved_from") or ""), 0)
  791. ):
  792. kept["_resolved_from"] = target.get("_resolved_from")
  793. return targets
  794. def _resolve_delivery_target(job: dict) -> Optional[dict]:
  795. """Resolve the concrete auto-delivery target for a cron job, if any."""
  796. targets = _resolve_delivery_targets(job)
  797. return targets[0] if targets else None
  798. # Audio routing is centralized in gateway.platforms.base.should_send_media_as_audio().
  799. _VIDEO_EXTS = frozenset({'.mp4', '.mov', '.avi', '.mkv', '.webm', '.3gp'})
  800. _IMAGE_EXTS = frozenset({'.jpg', '.jpeg', '.png', '.webp', '.gif'})
  801. def _send_media_via_adapter(
  802. adapter, chat_id: str, media_files: list, metadata: dict | None, loop, job: dict, platform=None,
  803. ) -> list:
  804. """Send MEDIA files as native attachments (routed by extension, as in
  805. _process_message_background). Returns per-file error strings so a dropped attachment surfaces
  806. in run status, not just the gateway log."""
  807. from gateway.platforms.base import (
  808. BasePlatformAdapter, should_send_media_as_audio, validate_media_delivery_path)
  809. from agent.async_utils import safe_schedule_threadsafe
  810. job_ref = {"id": job.get("id", "?")}
  811. errors: list = []
  812. requested = [(str(p), v) for p, v in (media_files or [])]
  813. media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files)
  814. # Report paths the safety filter dropped (missing file, denied prefix, strict-mode miss).
  815. kept = {p for p, _ in media_files}
  816. for raw_path, _v in requested:
  817. try:
  818. dropped = validate_media_delivery_path(raw_path) not in kept
  819. except Exception:
  820. dropped = True
  821. if dropped:
  822. errors.append(f"attachment dropped by media path policy: {raw_path}")
  823. route_platform = platform if platform is not None else getattr(adapter, "platform", None)
  824. for media_path, _is_voice in media_files:
  825. try:
  826. ext = _sched.Path(media_path).suffix.lower()
  827. if should_send_media_as_audio(route_platform, ext, is_voice=_is_voice):
  828. method, path_kw = "send_voice", "audio_path"
  829. elif ext in _VIDEO_EXTS:
  830. method, path_kw = "send_video", "video_path"
  831. elif ext in _IMAGE_EXTS:
  832. method, path_kw = "send_image_file", "image_path"
  833. else:
  834. method, path_kw = "send_document", "file_path"
  835. # The voice sender decides bubble vs music-file from ``is_voice`` (Telegram
  836. # transcodes non-Opus only when it is set), matching the gateway dispatch
  837. # in BasePlatformAdapter._send_one.
  838. extra = {"is_voice": _is_voice} if method == "send_voice" else {}
  839. coro = getattr(adapter, method)(
  840. chat_id=chat_id, metadata=metadata, **{path_kw: media_path}, **extra)
  841. future = safe_schedule_threadsafe(coro, loop)
  842. if future is None:
  843. _note_target_error(
  844. job_ref, f"cannot send media {media_path}: gateway loop unavailable", errors)
  845. return errors
  846. try:
  847. # Large attachments can exceed 30s; configurable via _get_media_send_timeout().
  848. result = future.result(timeout=_script._get_media_send_timeout())
  849. except TimeoutError:
  850. future.cancel()
  851. raise
  852. raw = getattr(result, "raw_response", None)
  853. if result and getattr(result, "success", False) and isinstance(raw, dict) and raw.get("ambiguous"):
  854. # Sent but the ack was lost: possibly delivered, so never an error (a re-run
  855. # would duplicate it); _live_send_media records the target as unverified.
  856. job["_media_ack_lost"] = True
  857. elif result and not getattr(result, "success", True):
  858. _note_target_error(
  859. job_ref,
  860. f"media send failed for {media_path}: {getattr(result, 'error', 'unknown')}",
  861. errors,
  862. )
  863. except Exception as e:
  864. # TimeoutError etc. have an empty str(); fall back to the class name.
  865. _note_target_error(
  866. job_ref, f"failed to send media {media_path}: {str(e) or type(e).__name__}", errors)
  867. return errors
  868. def _result_field(send_result, key: str, default=None):
  869. """Read ``key`` from a SendResult-like object or the plain dict the silence filter returns."""
  870. if isinstance(send_result, dict):
  871. return send_result.get(key, default)
  872. return getattr(send_result, key, default)
  873. def _confirm_adapter_delivery(
  874. send_result, job_id: str = "?", unverified: Optional[list] = None) -> bool:
  875. """Return True only if ``send_result`` unambiguously confirms delivery. ``None`` or no
  876. ``success`` attr/key is NOT success (would log "delivered" while nothing was sent).
  877. ``delivered is False`` REJECTS even with truthy ``success`` (the silence-narration filter
  878. returns ``{"success": True, "delivered": False}``). No ``message_id``/``raw_response`` is still
  879. accepted (some adapters return a bare success) but logged at WARNING as UNVERIFIED.
  880. A live adapter that returns ``None`` (e.g. a swallowed exception, a busy platform, or a code path that
  881. returns early without producing a ``SendResult``) must NOT be treated as success — doing so causes the
  882. scheduler to log ``"delivered to <chat> via live adapter"`` while the gateway never actually sees the
  883. message (#47056).
  884. * No ``message_id`` and no ``raw_response`` means we have no positive evidence of a send. Telegram
  885. ``SendResult`` objects carry ``message_id``; the dict-filter shape does not. See #77763.
  886. """
  887. if send_result is None:
  888. return False
  889. if isinstance(send_result, dict):
  890. has_success = "success" in send_result
  891. else:
  892. has_success = hasattr(send_result, "success")
  893. if not has_success or not bool(_result_field(send_result, "success")):
  894. return False
  895. if _result_field(send_result, "delivered") is False:
  896. return False
  897. if (
  898. _result_field(send_result, "message_id") is None
  899. and not _result_field(send_result, "raw_response")
  900. ):
  901. logger.warning(
  902. "Job '%s': live adapter reported success with no delivery evidence "
  903. "(no message_id, no raw_response) — treating as delivered but "
  904. "UNVERIFIED",
  905. job_id)
  906. if unverified is not None:
  907. unverified.append(True)
  908. return True
  909. def _is_channel_dm_topic(runtime_adapter: Any, chat_id: Any, loop: Any, job_id: str) -> bool:
  910. """Is an ambiguous ``telegram:<positive_chat_id>:<numeric_thread_id>`` target a channel
  911. Direct-Messages topic (``direct_messages_topic_id``) rather than a private-chat forum topic
  912. (``message_thread_id``)? Shape cannot decide; signal is ``get_chat_info`` type == ``channel``.
  913. Fails SAFE to False (thread routing) without a probe or on any probe error/timeout.
  914. Callers gate this on the ambiguous shape first (``telegram:<positive_chat_id>:<numeric_thread_id>``) —
  915. that shape is identical for both cases, so shape alone cannot decide (this was the #52060 regression).
  916. Probe the live adapter's ``get_chat_info`` once and only return True when the chat is a channel.
  917. See #22773.
  918. """
  919. # Resolve on the CLASS, not the instance: a MagicMock instance auto-creates a truthy
  920. # ``get_chat_info``, so an instance-level probe would misclassify test doubles.
  921. get_chat_info = getattr(type(runtime_adapter), "get_chat_info", None)
  922. if not callable(get_chat_info):
  923. return False
  924. try:
  925. from agent.async_utils import safe_schedule_threadsafe
  926. coro = get_chat_info(runtime_adapter, str(chat_id))
  927. future = safe_schedule_threadsafe(coro, loop) # type: ignore[arg-type]
  928. if future is None:
  929. return False
  930. # Metadata-only call, so a shorter bound than the send waits is intentional.
  931. info = future.result(timeout=10)
  932. except Exception:
  933. logger.debug(
  934. "Job '%s': get_chat_info probe failed for chat=%s — "
  935. "defaulting to message_thread_id routing",
  936. job_id, chat_id, exc_info=True)
  937. return False
  938. is_channel = isinstance(info, dict) and str(info.get("type") or "").lower() == "channel"
  939. if is_channel:
  940. logger.info(
  941. "Job '%s': chat=%s is a channel — routing via direct_messages_topic_id",
  942. job_id, chat_id)
  943. return is_channel
  944. def _cron_delivery_notify_enabled(cfg: Optional[dict]) -> bool:
  945. """Resolve ``cron.delivery.notify`` (default True). Only an explicit ``False`` disables; a
  946. missing/malformed section keeps the default so a typo cannot silently mute briefs."""
  947. try:
  948. cron_cfg = (cfg or {}).get("cron")
  949. delivery_cfg = cron_cfg.get("delivery") if isinstance(cron_cfg, dict) else None
  950. return not isinstance(delivery_cfg, dict) or delivery_cfg.get("notify", True) is not False
  951. except Exception:
  952. return True
  953. def _record_delivery_verification(job: dict, unverified_targets: list) -> None:
  954. """Persist ``last_delivery_unverified``: list of ``platform:chat_id[:thread_id]`` targets acked with no
  955. evidence, or None, alongside queued Bot Chat receipts. Never raises (bookkeeping must not fail a
  956. delivery)."""
  957. new_value = list(unverified_targets) or None
  958. queued = {target: receipt for target, receipt in
  959. job.get("_bot_chat_delivery_receipts", {}).items()
  960. if receipt["status"] in ("queued", "claimed")} or None
  961. values = {key: value for key, value in {
  962. "last_delivery_unverified": new_value, "last_delivery_queued": queued,
  963. }.items() if (job.get(key) or None) != value}
  964. if not values:
  965. return
  966. job.update(values)
  967. try:
  968. from cron.jobs import update_job
  969. update_job(job["id"], values)
  970. except Exception as exc: # pragma: no cover - defensive
  971. logger.debug("Job '%s': could not record delivery verification: %s", job.get("id"), exc)
  972. @dataclass
  973. class _TargetDelivery:
  974. """Per-target delivery state shared by the live-adapter and standalone lanes."""
  975. job: dict
  976. platform: Any
  977. platform_name: str
  978. chat_id: str
  979. thread_id: Optional[str]
  980. transport: Any
  981. pconfig: Any
  982. runtime_adapter: Any
  983. target_adapters: Any
  984. config: Any
  985. loop: Any
  986. notify_delivery: bool
  987. origin: dict
  988. origin_target: bool
  989. origin_user_id: Optional[str]
  990. is_dm_target: bool
  991. mirror_text: str
  992. mirror_this_target: bool
  993. in_channel_surface: bool
  994. inchannel_continuable: bool
  995. opened_thread_id: Optional[str]
  996. live_adapter_ready: bool = False
  997. live_error: Optional[str] = None # the live lane's own rejection string, e.g. "send_path_degraded"
  998. @property
  999. def is_relay(self) -> bool:
  1000. return self.transport is not None and self.transport.is_relay
  1001. @property
  1002. def where(self) -> str:
  1003. # A topic-routed target without its thread id names the wrong lane in failure reports.
  1004. base = f"{self.platform_name}:{self.chat_id}"
  1005. return f"{base}:{self.thread_id}" if self.thread_id else base
  1006. def _note_target_error(job: dict, msg: str, errors: list) -> None:
  1007. """Log a per-target delivery failure as a WARNING and record it in ``errors``."""
  1008. logger.warning("Job '%s': %s", job["id"], msg)
  1009. errors.append(msg)
  1010. def _warn_live_lane_failure(job: dict, msg: str, is_relay: bool) -> None:
  1011. """Relay targets have no standalone fallback, so the log line must not promise one."""
  1012. if is_relay:
  1013. logger.warning("Job '%s': %s", job["id"], msg)
  1014. else:
  1015. logger.warning("Job '%s': %s, falling back to standalone", job["id"], msg)
  1016. def _resolve_target_transport(
  1017. job: dict, platform, platform_name: str, target: dict, adapters, config):
  1018. """Resolve ``(transport, pconfig, runtime_adapter, target_adapters)`` for one target, or
  1019. ``(None, error)`` when it cannot be served (relay-fronted with no live transport, or not
  1020. configured/enabled)."""
  1021. from gateway.delivery import DeliveryTransport, resolve_delivery_transport
  1022. target_adapters = adapters
  1023. transport = None
  1024. if isinstance(adapters, _preflight.SharedRouteAdapters):
  1025. # Credentialless satellite: the primary adapter serves THIS target only when an exact
  1026. # primary route maps it to this profile; a miss fails closed below.
  1027. # See #101113.
  1028. shared = adapters.get(platform, target)
  1029. target_adapters = {platform: shared} if shared is not None else {}
  1030. if shared is not None:
  1031. # The PRIMARY's route authorized this exact native adapter. The satellite's own
  1032. # ``platforms.<p>`` block describes a connector it never runs (no credential), so
  1033. # neither its absence nor ``enabled: false`` may veto the shared transport; only its
  1034. # non-credential settings (continuable surface, reply mode) are kept (#89302, #103701).
  1035. from dataclasses import replace
  1036. from gateway.config import PlatformConfig
  1037. own = config.platforms.get(platform)
  1038. transport = DeliveryTransport(
  1039. shared, replace(own, enabled=True) if own is not None else PlatformConfig(enabled=True),
  1040. platform)
  1041. if transport is None:
  1042. transport = resolve_delivery_transport(platform, config, target_adapters)
  1043. if transport is not None:
  1044. pconfig = transport.config
  1045. runtime_adapter = transport.adapter
  1046. else:
  1047. # Relay-fronted platforms have NO standalone fallback (the connector owns the credential),
  1048. # so surface that instead of the native configured/enabled gate, which misdiagnoses them.
  1049. from gateway.relay import relay_fronted_platforms
  1050. if platform_name in relay_fronted_platforms():
  1051. return None, (
  1052. f"platform '{platform_name}' is relay-fronted and has no "
  1053. "live gateway transport; start the gateway (its ticker "
  1054. "owns relay-fronted delivery and will fire the job on "
  1055. "schedule)"
  1056. )
  1057. pconfig = config.platforms.get(platform)
  1058. runtime_adapter = None
  1059. if transport is not None and (transport.is_relay or pconfig is None):
  1060. # Relay transport carries the RELAY adapter's config (enablement already checked): the
  1061. # logical platform is deliberately NOT natively enabled. A live NATIVE adapter with no
  1062. # ``platforms.<p>`` block is the same shape — the owning process already authorized the
  1063. # adapter; "no config" is not "disabled" (#89302).
  1064. if pconfig is None:
  1065. from gateway.config import PlatformConfig
  1066. pconfig = PlatformConfig(enabled=True)
  1067. elif not pconfig or not pconfig.enabled:
  1068. return None, f"platform '{platform_name}' not configured/enabled"
  1069. return (transport, pconfig, runtime_adapter, target_adapters), None
  1070. def _inchannel_surface_supported(runtime_adapter, platform_name: str) -> bool:
  1071. """D6 probe: can this adapter deliver a continuable in_channel brief on ``platform_name``?
  1072. Per-platform check first (one RelayAdapter fronts N platforms; the scalar attr only carries
  1073. the PRIMARY identity's bit); native adapters use the class attribute."""
  1074. per_platform_check = getattr(
  1075. runtime_adapter, "supports_inchannel_continuable_for_platform", None)
  1076. if callable(per_platform_check):
  1077. try:
  1078. return bool(per_platform_check(platform_name))
  1079. except Exception:
  1080. return False
  1081. return bool(getattr(runtime_adapter, "supports_inchannel_continuable", False))
  1082. def _live_route_metadata(t: _TargetDelivery) -> tuple[Optional[str], dict, dict]:
  1083. """Compute ``(route_thread_id, route_metadata, media_metadata)`` for a live send, ONCE so text
  1084. and media agree. ``telegram:<positive_chat_id>:<numeric_thread_id>`` is ambiguous (private
  1085. forum topic vs channel DM topic need OPPOSITE routing) — see ``_is_channel_dm_topic``.
  1086. ``thread_id`` rides in ``route_metadata`` to bypass the router's private-chat anchor rule."""
  1087. from gateway.config import Platform
  1088. from gateway.delivery import _looks_like_int, looks_like_telegram_private_chat_id
  1089. job = t.job
  1090. thread_id = t.thread_id
  1091. is_ambiguous_telegram_topic = (
  1092. t.platform == Platform.TELEGRAM
  1093. and thread_id is not None
  1094. and looks_like_telegram_private_chat_id(str(t.chat_id))
  1095. and _looks_like_int(str(thread_id))
  1096. )
  1097. if is_ambiguous_telegram_topic and _is_channel_dm_topic(
  1098. t.runtime_adapter, t.chat_id, t.loop, job["id"]):
  1099. # Channel DM topic: direct_messages_topic_id, no bare thread_id; media mirrors text.
  1100. # See #22773.
  1101. route_thread_id = None
  1102. route_metadata = {
  1103. "direct_messages_topic_id": str(thread_id), "job_id": job["id"],
  1104. "notify": t.notify_delivery,
  1105. }
  1106. media_metadata = {"direct_messages_topic_id": str(thread_id), "notify": t.notify_delivery}
  1107. else:
  1108. # Forum-style topic or non-topic target: message_thread_id.
  1109. # Put thread_id in *route_metadata* (not just the DeliveryTarget) deliberately — the
  1110. # DeliveryRouter's private-chat topic detection (gateway/delivery.py) demands a reply anchor when
  1111. # thread_id is absent from metadata; cron deliveries have no inbound reply anchor, so the metadata
  1112. # key bypasses that check and lets the adapter route via a plain message_thread_id. See #52060.
  1113. route_thread_id = str(thread_id) if thread_id is not None else None
  1114. route_metadata = {"job_id": job["id"], "notify": t.notify_delivery}
  1115. if route_thread_id:
  1116. route_metadata["thread_id"] = route_thread_id
  1117. media_metadata = {"notify": t.notify_delivery}
  1118. if thread_id:
  1119. media_metadata["thread_id"] = thread_id
  1120. # Relay egress discriminators (scope_id / user_id) from the persisted origin: the adapter's caches are cold
  1121. # after a restart. See cron/scheduler_delivery_origin.py.
  1122. _origin.stamp_origin_discriminators(t, route_metadata, media_metadata)
  1123. return route_thread_id, route_metadata, media_metadata
  1124. _LIVE_SEND_CONFIRM_TIMEOUT_SECS = 60
  1125. def _live_send_text(
  1126. t: _TargetDelivery, text_to_send: str, route_thread_id: Optional[str], route_metadata: dict, *,
  1127. target_errors: list, delivery_errors: list, unverified_targets: list,
  1128. ) -> tuple[bool, bool, Any]:
  1129. """Schedule the text send on the gateway loop; returns ``(adapter_ok, timed_out, message_id)``.
  1130. Re-raises a real send error so the caller falls through to standalone."""
  1131. from agent.async_utils import safe_schedule_threadsafe
  1132. from gateway.delivery import DeliveryRouter, DeliveryTarget, PartialDeliveryError
  1133. job = t.job
  1134. router = DeliveryRouter(t.config, t.target_adapters)
  1135. route_target = DeliveryTarget(
  1136. platform=t.platform, chat_id=str(t.chat_id), thread_id=route_thread_id, is_explicit=True)
  1137. # Thread routing goes via the target, not a bare metadata "thread_id": the router only applies
  1138. # its Telegram DM-topic detection when thread_id/message_thread_id are absent from metadata.
  1139. # Send through the already-authorized transport: re-resolving from the plain target_adapters
  1140. # dict cannot re-derive the SharedRouteAdapters satellite grant (the satellite owned
  1141. # platforms.<p> block is disabled), yields None, and drops the delivery (#115656).
  1142. # cancel() cannot tell "never started" from "in flight": a run_coroutine_threadsafe future stays
  1143. # PENDING until the coroutine finishes, so cancel() returns True mid-send AND kills it. The send
  1144. # records its own start under a lock; a timeout abandons it only if it never began.
  1145. dispatch_lock = threading.Lock()
  1146. dispatch = {"started": False, "abandoned": False}
  1147. async def _send_once():
  1148. with dispatch_lock:
  1149. if dispatch["abandoned"]:
  1150. return None
  1151. dispatch["started"] = True
  1152. return await router._deliver_to_platform(route_target, text_to_send, route_metadata, transport=t.transport)
  1153. future = safe_schedule_threadsafe(_send_once(), t.loop)
  1154. if future is None:
  1155. target_errors.append("live adapter event loop scheduling failed")
  1156. return False, False, None
  1157. try:
  1158. send_result = future.result(timeout=_LIVE_SEND_CONFIRM_TIMEOUT_SECS)
  1159. except TimeoutError:
  1160. # Slow confirmation != failure. Never started (loop wedged): nothing was sent, so fall through
  1161. # to standalone or it is silently dropped. Started: in flight (a paced multi-chunk send can
  1162. # legitimately outlast the wait) — leave it running; a standalone resend would DUPLICATE.
  1163. with dispatch_lock:
  1164. dispatch["abandoned"] = not dispatch["started"]
  1165. if dispatch["abandoned"]:
  1166. future.cancel()
  1167. msg = f"live adapter send to {t.where} timed out before the coroutine was dispatched"
  1168. logger.warning("Job '%s': %s, falling back to standalone", job["id"], msg)
  1169. target_errors.append(msg)
  1170. return False, False, None
  1171. logger.warning(
  1172. "Job '%s': live adapter send to %s:%s timed out "
  1173. "after 60s; already dispatched (in flight), "
  1174. "assuming delivered (skipping standalone fallback "
  1175. "to avoid duplicate)",
  1176. job["id"], t.platform_name, t.chat_id)
  1177. return True, True, None
  1178. except PartialDeliveryError as ex:
  1179. # The head of a split send is already on screen: a standalone resend would duplicate it.
  1180. raw = getattr(ex.result, "raw_response", None) or {}
  1181. _note_target_error(
  1182. job, f"live adapter send to {t.where} delivered {raw.get('delivered_chunks', '?')} of "
  1183. f"{raw.get('total_chunks', '?')} chunks, then failed: {ex}", delivery_errors)
  1184. return True, False, None
  1185. except Exception as ex:
  1186. # Real send error (not a slow confirmation): fall through to standalone. The router raises
  1187. # a failed SendResult's error string, so this is where send_path_degraded arrives.
  1188. t.live_error = str(ex)
  1189. target_errors.append(f"live adapter send failed: {ex}")
  1190. raise
  1191. # _deliver_to_platform returns a SendResult, or a plain dict {"success": True, "delivered":
  1192. # False, ...} when the silence-narration filter drops the message.
  1193. send_raw_response = _result_field(send_result, "raw_response")
  1194. delivered_message_id = _result_field(send_result, "message_id")
  1195. _evidence_gap: list = []
  1196. send_success = _confirm_adapter_delivery(send_result, job["id"], _evidence_gap)
  1197. if send_success and _evidence_gap:
  1198. unverified_targets.append(t.where)
  1199. if not send_success:
  1200. if send_result is None:
  1201. err, shape = "no response from adapter", "None"
  1202. elif isinstance(send_result, dict):
  1203. # A filtered drop carries no "error" — name the filter instead of reporting "unknown".
  1204. err = send_result.get("error") or send_result.get("filtered") or "unknown"
  1205. shape = "dict"
  1206. else:
  1207. err, shape = getattr(send_result, "error", None), type(send_result).__name__
  1208. msg = f"live adapter send to {t.where} returned unconfirmed result ({shape}, error={err})"
  1209. t.live_error = str(err) if err else None
  1210. _warn_live_lane_failure(job, msg, t.is_relay)
  1211. target_errors.append(msg)
  1212. return False, False, None
  1213. if send_raw_response and t.thread_id and send_raw_response.get("thread_fallback"):
  1214. requested_thread_id = send_raw_response.get("requested_thread_id") or t.thread_id
  1215. _note_target_error(
  1216. job,
  1217. f"configured thread_id {requested_thread_id} for "
  1218. f"{t.where} was not found; delivered without thread_id",
  1219. delivery_errors)
  1220. return True, False, delivered_message_id
  1221. def _live_send_media(
  1222. t: _TargetDelivery, media_metadata: dict, media_files: list, delivery_errors: list,
  1223. unverified_targets: list) -> None:
  1224. """Send extracted media as native attachments with the same routing as the text send."""
  1225. routed_media_metadata = dict(media_metadata or {})
  1226. if t.is_relay:
  1227. routed_media_metadata["_relay_logical_platform"] = t.platform.value
  1228. logical_home = t.config.get_home_channel(t.platform)
  1229. if logical_home is not None and logical_home.chat_id == t.chat_id:
  1230. if logical_home.user_id:
  1231. routed_media_metadata["user_id"] = logical_home.user_id
  1232. if logical_home.scope_id:
  1233. routed_media_metadata["scope_id"] = logical_home.scope_id
  1234. _media_errors = _send_media_via_adapter(
  1235. t.runtime_adapter, t.chat_id, media_files, routed_media_metadata or None, t.loop, t.job,
  1236. platform=t.platform,
  1237. )
  1238. if t.job.pop("_media_ack_lost", False) and t.where not in unverified_targets:
  1239. unverified_targets.append(t.where)
  1240. # Surface per-file failures into run status: text delivered but attachment lost is not ok.
  1241. for _me in _media_errors:
  1242. delivery_errors.append(f"{_me} (target {t.where})")
  1243. def _seed_live_delivery_sessions(t: _TargetDelivery, delivered_message_id) -> None:
  1244. """After a confirmed live send, seed continuation session(s) and run the generic mirror.
  1245. Thread seeding is deferred here so open-succeeds/deliver-fails never seeds an unseen brief."""
  1246. job = t.job
  1247. origin = t.origin
  1248. seed_kwargs = dict(
  1249. chat_name=origin.get("chat_name"), is_dm=t.is_dm_target, scope_id=origin.get("scope_id"))
  1250. thread_seeded = False
  1251. inchannel_seeded = False
  1252. if t.opened_thread_id:
  1253. _seed_cron_thread_session(
  1254. job, t.runtime_adapter, t.platform_name, t.chat_id, t.opened_thread_id, t.mirror_text,
  1255. **seed_kwargs,
  1256. )
  1257. thread_seeded = True
  1258. # in_channel: CREATE + seed the flat session (the mirror only APPENDS to an existing one). Same
  1259. # `inchannel_continuable` gate as the flatten in _deliver_result (must not drift). Origin
  1260. # seed without mirror opt-in; others only via _inchannel_seed_allowed (user-less seed = orphan).
  1261. if t.in_channel_surface and t.inchannel_continuable and not thread_seeded:
  1262. inchannel_seeded = _seed_cron_channel_session(
  1263. job, t.runtime_adapter, t.platform_name, t.chat_id, t.mirror_text,
  1264. user_id=t.origin_user_id, **seed_kwargs)
  1265. if not inchannel_seeded:
  1266. logger.warning(
  1267. "Job '%s': in_channel seed did NOT land on %s:%s "
  1268. "— a plain reply will not see this brief",
  1269. job["id"], t.platform_name, t.chat_id)
  1270. # Companion THREAD seed: a reply in the brief's own thread keys to (chat, thread=<ts>),
  1271. # which the flat seed never touches. Seed it too so BOTH reply surfaces continue the job.
  1272. if delivered_message_id:
  1273. _seed_cron_thread_session(
  1274. job, t.runtime_adapter, t.platform_name, t.chat_id, str(delivered_message_id),
  1275. t.mirror_text,
  1276. **seed_kwargs)
  1277. elif t.in_channel_surface and not t.inchannel_continuable:
  1278. logger.warning(
  1279. "Job '%s': in_channel delivery to %s:%s is not a "
  1280. "continuable target (origin=%s:%s thread=%s; not the "
  1281. "origin conversation, and not a mirror-eligible "
  1282. "fallback/opted-in target the seed can key) — seed "
  1283. "skipped; the plain mirror below may still apply",
  1284. job["id"], t.platform_name, t.chat_id,
  1285. origin.get("platform"), origin.get("chat_id"), origin.get("thread_id"))
  1286. _maybe_mirror_cron_delivery(
  1287. job, t.platform_name, t.chat_id, t.mirror_text, thread_id=t.thread_id,
  1288. user_id=t.origin_user_id,
  1289. enabled=t.mirror_this_target and not thread_seeded and not inchannel_seeded)
  1290. def _deliver_via_live_adapter(
  1291. t: _TargetDelivery, cleaned_text: str, media_files: list, *, target_errors: list,
  1292. delivery_errors: list, unverified_targets: list,
  1293. ) -> bool:
  1294. """Deliver one target via the live gateway adapter; True once delivered. ``target_errors`` =
  1295. this lane's soft failures (surfaced only if standalone also fails); ``delivery_errors`` =
  1296. partial failures (media, thread fallback) that surface even on success."""
  1297. job = t.job
  1298. route_thread_id, route_metadata, media_metadata = _live_route_metadata(t)
  1299. delivered = False
  1300. try:
  1301. # Send cleaned text (MEDIA tags stripped) through the gateway's DeliveryRouter so it gets
  1302. # the same platform routing as live messages (Telegram's three-mode topic routing).
  1303. text_to_send = cleaned_text.strip()
  1304. adapter_ok, timed_out, delivered_message_id = True, False, None
  1305. if not text_to_send and not media_files:
  1306. # Fail closed so the run reports the empty payload.
  1307. _note_target_error(
  1308. job, f"live adapter send skipped (empty text and no media) for {t.where}",
  1309. target_errors)
  1310. adapter_ok = False
  1311. elif text_to_send:
  1312. adapter_ok, timed_out, delivered_message_id = _live_send_text(
  1313. t, text_to_send, route_thread_id, route_metadata,
  1314. target_errors=target_errors, delivery_errors=delivery_errors,
  1315. unverified_targets=unverified_targets,
  1316. )
  1317. # Media rides the same DM-topic-aware routing as text. Skipped after a confirmation
  1318. # timeout (loop contended, text already assumed delivered) — record the drop instead.
  1319. # Send extracted media files as native attachments via the live adapter, using the same
  1320. # DM-topic-aware routing as the text send (#22773 — media previously used a bare thread_id and
  1321. # landed in the General lane for private DM topics). Skip on an in-flight confirmation timeout: the
  1322. # gateway loop is contended, so each media send would also block its 30s budget, and the text
  1323. # payload is already assumed delivered (#38922). Record the skipped attachments so the drop is
  1324. # visible rather than silently lost.
  1325. if adapter_ok and not timed_out and media_files:
  1326. _live_send_media(t, media_metadata, media_files, delivery_errors, unverified_targets)
  1327. elif timed_out and media_files:
  1328. _note_target_error(
  1329. job,
  1330. f"{len(media_files)} media attachment(s) not delivered to "
  1331. f"{t.where} (live adapter confirmation timed out)",
  1332. delivery_errors)
  1333. if adapter_ok:
  1334. # Log WHERE it went: a ghost delivery in the wrong lane is otherwise indistinguishable.
  1335. logger.info(
  1336. # Log WHERE it went, not just that it went: a ghost delivery that landed in the wrong lane
  1337. # (General topic instead of the routed thread) is indistinguishable from a real one without
  1338. # the routing identity (#77763).
  1339. "Job '%s': delivered to %s:%s via live adapter thread=%s message_id=%s",
  1340. job["id"], t.platform_name, t.chat_id,
  1341. route_thread_id if route_thread_id is not None else "-",
  1342. delivered_message_id if delivered_message_id is not None else "-")
  1343. delivered = True
  1344. _seed_live_delivery_sessions(t, delivered_message_id)
  1345. except Exception as e:
  1346. err_msg = f"live adapter delivery to {t.where} failed: {e}"
  1347. if not any(err_msg in err for err in target_errors):
  1348. target_errors.append(err_msg)
  1349. _warn_live_lane_failure(job, err_msg, t.is_relay)
  1350. return delivered
  1351. def _standalone_send(
  1352. t: _TargetDelivery, content: str, media_files: list) -> tuple[Any, Optional[str]]:
  1353. """Run the standalone sender for one target: ``(result, None)`` or ``(None, error)`` (already
  1354. logged — WARNING for a shutdown race, ERROR with traceback otherwise)."""
  1355. from tools.send_message_tool import _send_to_platform
  1356. job = t.job
  1357. shutdown_msg = f"delivery to {t.where} skipped — interpreter is shutting down"
  1358. send_timeout = _get_standalone_send_timeout()
  1359. async def _send():
  1360. # The bound lives inside the coroutine: the running-loop fallback below closes ``coro``
  1361. # unstarted, and a wait_for wrapper created out here would be left never awaited.
  1362. return await asyncio.wait_for(_send_to_platform(
  1363. t.platform, t.pconfig, t.chat_id, content, thread_id=t.thread_id,
  1364. media_files=media_files), timeout=send_timeout)
  1365. def _warned(msg: str) -> tuple[None, str]:
  1366. logger.warning("Job '%s': %s", job["id"], msg)
  1367. return None, msg
  1368. def _failed(e) -> tuple[None, str]:
  1369. msg = f"delivery to {t.where} failed: {e}"
  1370. logger.error("Job '%s': %s", job["id"], msg, exc_info=True)
  1371. return None, msg
  1372. # Interpreter finalizing (SIGTERM/restart/OOM): asyncio.run and a fresh ThreadPoolExecutor both
  1373. # raise "cannot schedule new futures after interpreter shutdown" — warn, not ERROR traceback.
  1374. if _sched._interpreter_shutting_down():
  1375. return _warned(shutdown_msg)
  1376. # The live lane failed closed on an empty payload; standalone senders don't (Telegram returns
  1377. # success=True for empty content WITHOUT an API call) — a phantom delivery would result.
  1378. if not content.strip() and not media_files:
  1379. return _warned(f"standalone send skipped (empty text and no media) for {t.where}")
  1380. coro = _send()
  1381. try:
  1382. return asyncio.run(coro), None
  1383. except TimeoutError:
  1384. # The send may still complete on the gateway loop (the dispatch shield keeps an in-flight
  1385. # send un-cancelled); the run is released instead of waiting on it unbounded (#115469).
  1386. msg = (f"standalone send to {t.where} timed out after {send_timeout}s "
  1387. "(the send may still be in flight)")
  1388. logger.error("Job '%s': %s", job["id"], msg)
  1389. return None, msg
  1390. except RuntimeError as run_err:
  1391. # asyncio.run() refuses inside a running loop; close the unstarted coro, retry in a thread.
  1392. coro.close()
  1393. if _sched._interpreter_shutting_down(run_err):
  1394. return _warned(shutdown_msg)
  1395. # The fallback can itself raise (SMTP, result timeout); catch it or remaining targets skip.
  1396. try:
  1397. pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
  1398. try:
  1399. # A fresh thread does NOT inherit the profile ContextVars (home override + secret
  1400. # scope); run in the active context or the sender reads the default bot token.
  1401. return pool.submit(contextvars.copy_context().run, asyncio.run, _send()).result(
  1402. timeout=30), None
  1403. finally:
  1404. pool.shutdown(wait=False)
  1405. except Exception as e:
  1406. if _sched._interpreter_shutting_down(e):
  1407. return _warned(shutdown_msg)
  1408. return _failed(e)
  1409. except Exception as e:
  1410. return _failed(e)
  1411. def _queue_for_live_reconnect(t: _TargetDelivery, content: str, media_files: list, delivery_errors: list) -> None:
  1412. """Hand a payload the live lane rejected as reconnect-only (``send_path_degraded``) and the
  1413. standalone lane then failed to send to the delivery ledger, as a failed reconnect-only row
  1414. owned by the adapter that rejected it: the post-reconnect sweep redelivers it (#125363). Only
  1415. reached after standalone failed, so nothing was sent and a replay cannot duplicate. The ledger
  1416. carries text only; dropped attachments are reported."""
  1417. try:
  1418. from gateway.delivery_ledger import (
  1419. compute_obligation_id, is_reconnect_only, ledger_enabled, mark_failed, record_obligation)
  1420. if not is_reconnect_only(t.live_error) or not ledger_enabled():
  1421. return
  1422. session_key = f"cron:{t.platform_name}:{t.chat_id}" + (f":{t.thread_id}" if t.thread_id else "")
  1423. obligation_id = compute_obligation_id(session_key, f"job:{t.job.get('id', '?')}", content)
  1424. record_obligation(
  1425. obligation_id=obligation_id, session_key=session_key, platform=t.platform_name,
  1426. chat_id=str(t.chat_id), thread_id=t.thread_id, content=content,
  1427. adapter_profile=getattr(getattr(t.transport, "adapter", None), "_owner_profile", None))
  1428. mark_failed(obligation_id, str(t.live_error))
  1429. except Exception:
  1430. logger.warning("Job '%s': could not queue %s for post-reconnect redelivery",
  1431. t.job.get("id"), t.where, exc_info=True)
  1432. return
  1433. note = f"queued text for {t.where} for redelivery once the live adapter reconnects"
  1434. if media_files:
  1435. note += f" ({len(media_files)} attachment(s) not queued)"
  1436. logger.warning("Job '%s': %s", t.job.get("id"), note)
  1437. delivery_errors.append(note)
  1438. def _deliver_standalone(
  1439. t: _TargetDelivery, content: str, media_files: list, target_errors: list, delivery_errors: list,
  1440. ) -> None:
  1441. """Standalone fallback for a target the live lane did not deliver."""
  1442. job = t.job
  1443. if t.is_relay:
  1444. # Relay owns the destination and credential; a native retry could duplicate — fail closed.
  1445. if not target_errors:
  1446. target_errors.append(f"relay delivery to {t.where} failed")
  1447. delivery_errors.extend(target_errors)
  1448. return
  1449. result, err = _standalone_send(t, content, media_files)
  1450. if err is None and result and result.get("error"):
  1451. # Not inside an except block — the error comes from the result dict, no traceback.
  1452. err = f"delivery error: {result['error']} (target {t.where})"
  1453. logger.error("Job '%s': %s", job["id"], err)
  1454. if err is not None:
  1455. target_errors.append(err)
  1456. delivery_errors.extend(target_errors)
  1457. # A satellite profile's worker has no platform token, so standalone cannot stand in for a
  1458. # live adapter that is only waiting to reconnect: keep the payload for that adapter.
  1459. _queue_for_live_reconnect(t, content, media_files, delivery_errors)
  1460. return
  1461. # Standalone senders report per-file attachment failures in ``warnings`` while returning
  1462. # success; surface them so a vanished attachment doesn't mark the run ok.
  1463. for _w in (result.get("warnings") if isinstance(result, dict) else None) or []:
  1464. msg = f"delivery warning: {_w} (target {t.where})"
  1465. logger.error("Job '%s': %s", job["id"], msg)
  1466. delivery_errors.append(msg)
  1467. logger.info("Job '%s': delivered to %s:%s", job["id"], t.platform_name, t.chat_id)
  1468. # Thread seeding only happens on the live lane, so no thread_seeded gate applies here.
  1469. _maybe_mirror_cron_delivery(
  1470. job, t.platform_name, t.chat_id, t.mirror_text, thread_id=t.thread_id,
  1471. user_id=t.origin_user_id,
  1472. enabled=t.mirror_this_target)
  1473. def _prepare_target_delivery(
  1474. job: dict, target: dict, *, adapters, loop, config, notify_delivery: bool, mirror_enabled: bool,
  1475. mirror_text: str, delivery_errors: list,
  1476. ) -> Optional[_TargetDelivery]:
  1477. """Per-target prologue of ``_deliver_result``: origin/mirror/in_channel gates, transport
  1478. resolution, continuable-thread open. None (error noted in ``delivery_errors``) if unservable."""
  1479. from gateway.config import Platform
  1480. platform_name = target["platform"]
  1481. chat_id = target["chat_id"]
  1482. thread_id = target.get("thread_id")
  1483. origin = _resolve_origin(job) or {}
  1484. origin_thread = origin.get("thread_id")
  1485. if origin_thread and not thread_id:
  1486. logger.warning(
  1487. "Job '%s': origin has thread_id=%s but delivery target lost it (deliver=%s, target=%s)",
  1488. job["id"], origin_thread, job.get("deliver", "local"), target)
  1489. elif thread_id:
  1490. logger.debug(
  1491. "Job '%s': delivering to %s:%s thread_id=%s",
  1492. job["id"], platform_name, chat_id, thread_id)
  1493. # Mirror: origin, origin-less home fallback, user-written home, or explicit-target opt-in.
  1494. origin_target = _target_matches_origin(origin, platform_name, chat_id, thread_id)
  1495. mirror_this_target = mirror_enabled and _target_mirror_eligible(
  1496. job, target, global_mirror=mirror_enabled, origin_match=origin_target)
  1497. # Resolved for ANY origin match (not just mirror-enabled): the in_channel seed needs it too.
  1498. origin_user_id = origin.get("user_id") if origin_target else None
  1499. # DM shape for BOTH the flatten gate and seed chat_type (Slack DM ids start with "D").
  1500. origin_chat_type = str(origin.get("chat_type") or "").lower()
  1501. is_dm_target = origin_chat_type == "dm" or (
  1502. not origin_chat_type and str(chat_id).startswith("D"))
  1503. # in_channel gate shared by thread-flatten and flat seed — they MUST match or brief and
  1504. # session land in different places. Origin qualifies unconditionally; others only when the
  1505. # seed can create a resolvable session (_inchannel_seed_allowed).
  1506. inchannel_continuable = origin_target or (
  1507. mirror_this_target and _inchannel_seed_allowed(is_dm=is_dm_target, user_id=origin_user_id))
  1508. # Plugin platform names create dynamic members via Platform._missing_().
  1509. try:
  1510. platform = Platform(platform_name.lower())
  1511. except (ValueError, KeyError):
  1512. _note_target_error(job, f"unknown platform '{platform_name}'", delivery_errors)
  1513. return None
  1514. resolved, resolve_err = _resolve_target_transport(
  1515. job, platform, platform_name, target, adapters, config)
  1516. if resolved is None:
  1517. _note_target_error(job, resolve_err, delivery_errors)
  1518. return None
  1519. transport, pconfig, runtime_adapter, target_adapters = resolved
  1520. # Live send needs a RUNNING loop, not just an adapter. Computed ONCE so the in_channel
  1521. # thread_id clear below stays in lockstep with the seed (standalone cannot seed flat).
  1522. live_adapter_ready = (
  1523. runtime_adapter is not None
  1524. and loop is not None
  1525. and getattr(loop, "is_running", lambda: False)()
  1526. )
  1527. # Continuable surface (D1/D2/D6) from platform config ``extra``; default "thread".
  1528. # ``in_channel`` delivers FLAT so a plain channel reply continues via the shared session
  1529. # ``(platform, chat_id, None)``. Unsupported adapters fail SAFE to thread.
  1530. in_channel_surface = _resolve_cron_surface_mode(pconfig, platform_name) == "in_channel"
  1531. if (
  1532. in_channel_surface
  1533. and runtime_adapter is not None
  1534. and not _inchannel_surface_supported(runtime_adapter, platform_name)
  1535. ):
  1536. logger.debug(
  1537. "Job '%s': cron_continuable_surface=in_channel not supported on %s, using thread",
  1538. job.get("id", "?"), platform_name)
  1539. in_channel_surface = False
  1540. if in_channel_surface and inchannel_continuable and live_adapter_ready:
  1541. # Force flat (D2): an inherited thread_id would never match the flat seed (None). Gated
  1542. # on `inchannel_continuable` (SAME gate as the seed) AND `live_adapter_ready` (fallback
  1543. # never seeds). Stay AFTER mirror_this_target/origin_user_id (need ORIGINAL thread_id).
  1544. thread_id = None
  1545. # Thread-preferred continuable cron: open a DEDICATED thread; its session is seeded after a
  1546. # successful send. DM-only platforms return None → mirror the origin DM. in_channel SKIPS
  1547. # this: it posts flat and _seed_cron_channel_session CREATES the session.
  1548. opened_thread_id: Optional[str] = None
  1549. if (
  1550. mirror_this_target
  1551. and not in_channel_surface
  1552. and runtime_adapter is not None
  1553. and loop is not None
  1554. and not thread_id # never override an explicit origin thread/topic
  1555. ):
  1556. opened_thread_id = _open_continuable_cron_thread(
  1557. job, runtime_adapter, chat_id, loop) or None
  1558. if opened_thread_id:
  1559. thread_id = opened_thread_id
  1560. return _TargetDelivery(
  1561. job=job, platform=platform, platform_name=platform_name, chat_id=chat_id,
  1562. thread_id=thread_id, transport=transport, pconfig=pconfig, runtime_adapter=runtime_adapter,
  1563. target_adapters=target_adapters, config=config, loop=loop, notify_delivery=notify_delivery,
  1564. origin=origin, origin_target=origin_target, origin_user_id=origin_user_id,
  1565. is_dm_target=is_dm_target, mirror_text=mirror_text, mirror_this_target=mirror_this_target,
  1566. in_channel_surface=in_channel_surface, inchannel_continuable=inchannel_continuable,
  1567. opened_thread_id=opened_thread_id, live_adapter_ready=live_adapter_ready)
  1568. def _unresolved_delivery_outcome(job: dict, for_failure: bool) -> Optional[str]:
  1569. """``_deliver_result`` outcome when no target resolved: None (not a failure) for ``local`` and
  1570. origin-less ``origin`` (CLI jobs never capture an origin — a spurious error every run), else
  1571. an error string."""
  1572. deliver_value = _normalize_deliver_value(_delivery_lane_value(job, for_failure=for_failure))
  1573. if deliver_value == "local":
  1574. return None
  1575. if deliver_value == "origin":
  1576. logger.info(
  1577. # deliver=origin with no resolvable origin and no configured home channels: treat as local
  1578. # rather than reporting an error. CLI-created jobs never capture a {platform, chat_id} origin,
  1579. # so failing here would make every CLI `deliver=origin` (or auto-detect) job emit a spurious "no
  1580. # delivery target resolved" error on every run (#43014). The output is still persisted in
  1581. # last_output for `cron list`/resume.
  1582. "Job '%s': deliver=origin but no origin or home channels — "
  1583. "skipping delivery (output saved in last_output)",
  1584. job.get("name", job.get("id", "?")))
  1585. return None
  1586. msg = f"no delivery target resolved for deliver={deliver_value}"
  1587. logger.warning("Job '%s': %s", job["id"], msg)
  1588. return msg
  1589. def _deliver_result(
  1590. job: dict, content: str, adapters=None, loop=None, *, for_failure: bool = False
  1591. ) -> Optional[str]:
  1592. """Deliver job output to the configured target(s). With ``adapters``/``loop`` (gateway
  1593. running) the live adapter is tried first (E2EE rooms can't use the standalone HTTP path), then
  1594. standalone fallback. ``for_failure=True`` routes failure-category notices through the job's
  1595. ``failure_deliver`` override when present (NS-788). Returns None on success, else an error."""
  1596. job.pop("_bot_chat_delivery_receipts", None)
  1597. job.pop("_notification_all_targets_suppressed", None)
  1598. targets = _resolve_delivery_targets(job, for_failure=for_failure)
  1599. if not targets:
  1600. _record_delivery_verification(job, [])
  1601. return _unresolved_delivery_outcome(job, for_failure)
  1602. # Restart-safe workers have no live gateway adapters: hand the send back through a durable
  1603. # queue so the current or replacement gateway performs it with relay/E2EE parity. The execution
  1604. # id is the idempotency key (the queue never retries an uncertain claimed send). Match on THIS
  1605. # job's own attempt: a worker's script may dispatch another job in-process (`hermes cron run`),
  1606. # and that nested delivery must not be keyed under the outer execution id.
  1607. external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER", "")
  1608. if (external_execution and adapters is None
  1609. and external_execution == str(job.get("execution_id") or "")
  1610. and any(target["platform"] not in (BOT_CHAT_PLATFORM, ROOM_PLATFORM) for target in targets)):
  1611. from cron.delivery_queue import enqueue_and_wait
  1612. _record_delivery_verification(job, [])
  1613. error = enqueue_and_wait(external_execution, job, content, for_failure=for_failure)
  1614. from cron.delivery_queue import get_status
  1615. delivery_status = get_status(external_execution)
  1616. if delivery_status and delivery_status["status"] == "suppressed":
  1617. job["_notification_all_targets_suppressed"] = True
  1618. from cron.jobs import get_job
  1619. refreshed = get_job(job["id"]) or {}
  1620. job["last_delivery_queued"] = refreshed.get("last_delivery_queued")
  1621. return error
  1622. from gateway.config import load_gateway_config
  1623. # Wrap with header/footer unless cron.wrap_response: false.
  1624. wrap_response = True
  1625. user_cfg = None
  1626. with contextlib.suppress(Exception):
  1627. user_cfg = _sched.load_config()
  1628. wrap_response = user_cfg.get("cron", {}).get("wrap_response", True)
  1629. # Mark live sends FINAL so the platform pushes them (Telegram "important" mode mutes otherwise).
  1630. notify_delivery = _cron_delivery_notify_enabled(user_cfg)
  1631. # Targets acked with NO evidence (bare SendResult(success=True) — Slack/Matrix/Mattermost);
  1632. # persisted as ``last_delivery_unverified`` so `hermes cron list` shows it.
  1633. unverified_targets: list = []
  1634. if wrap_response:
  1635. task_name = job.get("name", job["id"])
  1636. delivery_content = (
  1637. f"Cronjob Response: {task_name}\n"
  1638. f"(job_id: {job.get('id', '')})\n"
  1639. f"-------------\n\n"
  1640. f"{content}\n\n"
  1641. "To stop or manage this job, send me a new message "
  1642. f"(e.g. \"stop reminder {task_name}\")."
  1643. )
  1644. else:
  1645. delivery_content = content
  1646. from gateway.platforms.base import BasePlatformAdapter
  1647. # Bridge media-policy config into the env vars the path validator reads. The gateway does this
  1648. # at boot; standalone runs (`hermes cron run`) did not, silently dropping files. Idempotent.
  1649. from gateway.media_policy import apply_media_policy_env
  1650. apply_media_policy_env(user_cfg)
  1651. media_files, cleaned_delivery_content = BasePlatformAdapter.extract_media(delivery_content)
  1652. # Redact at this single chokepoint, BEFORE the live-adapter / standalone send lanes below.
  1653. # Shell-job stdout/stderr is already redacted where it is captured, but an LLM cron job's
  1654. # response text reaches delivery unscanned — so a job that surfaced a credential (echoed a
  1655. # failing curl with an API key, summarised a config file) sent it verbatim to the chat.
  1656. cleaned_delivery_content = _redact_cron_payload(cleaned_delivery_content, "delivery content")
  1657. requested_media = len(media_files)
  1658. media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files)
  1659. # Policy-dropped attachments will never be sent on ANY lane — record them in run status.
  1660. _policy_dropped = requested_media - len(media_files)
  1661. policy_drop_errors = [
  1662. f"{_policy_dropped} media attachment(s) dropped by media path "
  1663. "policy (missing file, denied prefix, or strict-mode miss); "
  1664. "see gateway.strict / media_delivery_allow_dirs in config.yaml"
  1665. ] if _policy_dropped > 0 else []
  1666. # Resolve the mirror gate ONCE (default off): successful deliveries are appended to the target
  1667. # chat's session transcript. Mirror the CLEAN, unwrapped output (not the header/footer).
  1668. try:
  1669. mirror_enabled = _cron_mirror_delivery_enabled(job, user_cfg)
  1670. except Exception:
  1671. mirror_enabled = False
  1672. # Independent of the mirror knob: continuable surfaces (in_channel) must seed even when
  1673. # attach_to_session=false and cron.mirror_delivery=false, else the seed gets "" and fails.
  1674. _, mirror_text = BasePlatformAdapter.extract_media(content)
  1675. # Derived from the raw `content`, so it does NOT inherit the redaction above. Without this,
  1676. # enabling the mirror writes an unredacted credential into the session transcript even though
  1677. # the chat message itself was clean — and a transcript outlives the message.
  1678. mirror_text = _redact_cron_payload((mirror_text or "").strip(), "mirror payload")
  1679. try:
  1680. config = load_gateway_config()
  1681. except Exception as e:
  1682. msg = f"failed to load gateway config: {e}"
  1683. logger.error("Job '%s': %s", job["id"], msg)
  1684. return msg
  1685. delivery_errors = []
  1686. suppressed_targets = 0 # local: `job` is snapshotted into durable deferred records mid-loop
  1687. for target in targets:
  1688. # A failure notice for a platform that hides warning notifications is a suppressed
  1689. # disposition, not a send; requested (non-failure) results are never gated.
  1690. from gateway.warning_notifications import warning_notifications_enabled
  1691. if (for_failure and target["platform"] != BOT_CHAT_PLATFORM
  1692. and not warning_notifications_enabled(target["platform"], user_cfg)):
  1693. suppressed_targets += 1
  1694. continue
  1695. # Bot Chat owns admission; never concurrently resume a live owner's transcript.
  1696. if target["platform"] == BOT_CHAT_PLATFORM:
  1697. bot_chat_error = _deliver_to_bot_chat(job, content, target["chat_id"], for_failure=for_failure)
  1698. suppressed_targets += job.pop("_notification_all_targets_suppressed", False)
  1699. if bot_chat_error:
  1700. receipt_target = f"bot-chat:{target['chat_id'] or '(own)'}"
  1701. receipt = job.get("_bot_chat_delivery_receipts", {}).get(receipt_target)
  1702. if not receipt or receipt["status"] not in ("queued", "claimed"):
  1703. delivery_errors.append(bot_chat_error)
  1704. if receipt and receipt["status"] == "ambiguous":
  1705. unverified_targets.append(bot_chat_error)
  1706. continue
  1707. if target["platform"] == ROOM_PLATFORM:
  1708. room_error = deliver_to_room(job, content, target["chat_id"], for_failure=for_failure)
  1709. if room_error:
  1710. delivery_errors.append(room_error)
  1711. continue
  1712. if target["platform"] == "local":
  1713. from gateway.session_cron import deliver_to_local_chat
  1714. from hermes_constants import get_hermes_home
  1715. local_error = deliver_to_local_chat(get_hermes_home(), target["chat_id"],
  1716. cleaned_delivery_content, diagnostic=for_failure)
  1717. if local_error:
  1718. delivery_errors.append(local_error)
  1719. continue
  1720. t = _prepare_target_delivery(
  1721. job, target, adapters=adapters, loop=loop, config=config,
  1722. notify_delivery=notify_delivery,
  1723. mirror_enabled=mirror_enabled, mirror_text=mirror_text, delivery_errors=delivery_errors)
  1724. if t is None:
  1725. continue
  1726. target_errors: list = []
  1727. delivered = t.live_adapter_ready and _deliver_via_live_adapter(
  1728. t, cleaned_delivery_content, media_files,
  1729. target_errors=target_errors, delivery_errors=delivery_errors,
  1730. unverified_targets=unverified_targets,
  1731. )
  1732. if not delivered:
  1733. _deliver_standalone(
  1734. t, cleaned_delivery_content, media_files, target_errors, delivery_errors)
  1735. # Filter-time drops apply to every target; report them once. A run whose every target was
  1736. # suppressed sent nothing, so there is no drop to report.
  1737. if suppressed_targets == len(targets):
  1738. job["_notification_all_targets_suppressed"] = True
  1739. else:
  1740. delivery_errors.extend(policy_drop_errors)
  1741. _record_delivery_verification(job, unverified_targets)
  1742. return "; ".join(delivery_errors) if delivery_errors else None
  1743. # Late-bound origin namespace (see module docstring). Imported LAST so this module is fully
  1744. # populated before ``scheduler`` re-exports from it.
  1745. from cron import scheduler as _sched
  1746. from cron.scheduler_delivery_room import (
  1747. ROOM_PLATFORM, deliver_to_room, parse_room_deliver_token, resolve_room_target)
  1748. from cron import scheduler_delivery_origin as _origin
  1749. from cron import scheduler_preflight as _preflight
  1750. from cron import scheduler_script as _script