bot_mode_dm_coordination.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. """``message_agent`` coordination: a fan-out to several teammates and cancelling one's own request.
  2. Fan-out: each target is resolved and admitted exactly like a single ``message_agent`` call, but its
  3. reply runner is not started on its own. One combined runner (``bot_mode_dm.py --fan-out <spec>``)
  4. runs those per-target runners side by side and prints ONE result once every one has settled or hit
  5. its bound, so the sender wakes once with a typed outcome per teammate instead of N separate turns.
  6. Cancel: the sender withdraws its own queued request, or stops the turn that request started, on the
  7. teammate's owner through the same ``prompt.cancel`` a viewer's Stop uses. Only the request's author
  8. may: the receipt records who sent it.
  9. The runner half (``run_fan_out``) is stdlib only: it runs as a background process whose stdout is
  10. the sender's completion notification, and each sub-runner boots Hermes itself.
  11. """
  12. from __future__ import annotations
  13. import contextlib
  14. import json
  15. import logging
  16. import os
  17. import re
  18. import shlex
  19. import subprocess
  20. import sys
  21. import tempfile
  22. import threading
  23. from pathlib import Path
  24. from typing import Any
  25. # Every sub-runner bounds its own wait (the live lane at 30 min, the relay waiter by its budget);
  26. # this only reaps one that outlives that, reported as pending rather than lost.
  27. FAN_OUT_WAIT_SECONDS = 32 * 60
  28. _HEX_ID_RE = re.compile(r"[0-9a-f]{32,64}")
  29. logger = logging.getLogger(__name__)
  30. def split_targets(target: str, sender: Any) -> list[str]:
  31. """``target`` as the recipients it names: one name, or a comma-separated list deduplicated by
  32. the destination each name reaches (``hermes`` and ``default`` are one bot; a registered peer
  33. and a local bot whose friendly name matches it are two). A friendly name that itself contains
  34. a comma ('QA, Release') still resolves whole."""
  35. from tools.bot_mode_dm import _resolve_local_name
  36. raw = str(target or "").strip().lstrip("@")
  37. if "," not in raw or _resolve_local_name(raw, list(sender.roster_homes), sender.root) is not None:
  38. return [raw] if raw else []
  39. named: dict[tuple[str, str], str] = {}
  40. for part in raw.split(","):
  41. name = part.strip().lstrip("@")
  42. if name:
  43. named.setdefault(_destination(name, sender), name)
  44. return list(named.values())
  45. def _destination(name: str, sender: Any) -> tuple[str, str]:
  46. """``(transport, recipient)`` for ``name``, resolved in ``_dispatch``'s order: a peer target
  47. first, then a connection-qualified relay name, then a local bot (folder id or friendly name)."""
  48. from tools.bot_mode_dm import _PEER_TARGET_RE, _resolve_local_name
  49. peer = _PEER_TARGET_RE.match(name)
  50. if peer or name.lower() in sender.peers:
  51. peer_name, peer_profile = peer.groups() if peer else (name.lower(), None)
  52. return "peer", f"{peer_name}/{peer_profile or ''}".lower()
  53. if "@" in name:
  54. return "relay", name.lower()
  55. local = _resolve_local_name(name, list(sender.roster_homes), sender.root)
  56. return ("local", local) if local is not None else ("unresolved", name.lower())
  57. def fan_out(targets: list[str], content: str, sender: Any, *, expects_reply: bool) -> str:
  58. """Dispatch ``content`` to every target; with a reply expected, start the one combined runner."""
  59. from tools.bot_mode_dm import _dispatch
  60. collected: list[dict] = []
  61. acks = []
  62. for target in targets:
  63. try:
  64. ack = json.loads(_dispatch(target, content, sender, wake=collected if expects_reply else "none"))
  65. except Exception as exc:
  66. # One target's setup failure stays that target's: requests already admitted still get
  67. # their combined waiter below.
  68. logger.debug("message_agent fan-out to %s failed", target, exc_info=True)
  69. ack = {"error": f"NOT sent: {exc}"}
  70. acks.append({"to": f"@{target}", **ack})
  71. sent = [ack for ack in acks if not ack.get("error")]
  72. if not collected:
  73. return json.dumps({"status": "queued" if sent else "failed", "fan_out": acks,
  74. "detail": ("Nothing to wait for. " if sent else "Nothing was sent. ")
  75. + "Do not resend to a teammate whose entry has no error."})
  76. started = _start_combined(collected, sender)
  77. if started.get("error"):
  78. # The local admissions stand (their owner runs them); only the joint wake is lost. A peer
  79. # entry was never sent: its runner owned the send.
  80. for ack in acks:
  81. if any(entry["delivery_id"] == ack.get("delivery_id") and entry["dm_file"] for entry in collected):
  82. ack.update(status="failed", error="NOT sent: the fan-out runner did not start")
  83. return json.dumps({"status": "queued", "fan_out": acks, "notification_error": started["error"],
  84. "detail": "The replies will NOT wake you. Do not resend to entries without an error."})
  85. detail = (f"One combined completion carries every reply (or why not) once all {len(collected)} settle "
  86. "or hit their bound — do NOT wait, poll or resend; finish your turn.")
  87. if started.get("reply_delivery") == "poll":
  88. detail = started["detail"]
  89. return json.dumps({"status": "queued", "fan_out": acks, "process_id": started.get("process_id"),
  90. "reply_delivery": started.get("reply_delivery"), "detail": detail})
  91. def _start_combined(collected: list[dict], sender: Any) -> dict:
  92. from tools.bot_mode_dm import REPLY_COMPLETION_CHARS, _dm_dir, _spawn_delivery, _unlink_dm_file
  93. fd, spec = tempfile.mkstemp(prefix="fanout-", suffix=".json", dir=_dm_dir(), text=True)
  94. with os.fdopen(fd, "w", encoding="utf-8") as stream:
  95. json.dump([{**{k: entry[k] for k in ("command", "label", "delivery_id")},
  96. "output": _runner_output(entry["command"])} for entry in collected], stream)
  97. argv = [sys.executable, str(Path(__file__).resolve().parent / "bot_mode_dm.py"), "--fan-out", spec]
  98. if sys.platform == "win32":
  99. argv = [part.replace("\\", "/") for part in argv] # Git Bash runs the tracked command (see _delivery_command)
  100. started = json.loads(_spawn_delivery(shlex.join(argv), f"{len(collected)} teammates", dm_file=spec,
  101. task_id=sender.task_id, agent=sender.agent,
  102. completion_chars=REPLY_COMPLETION_CHARS * len(collected)))
  103. if started.get("error"):
  104. for entry in collected:
  105. if entry["dm_file"]:
  106. _unlink_dm_file(entry["dm_file"])
  107. return started
  108. def _runner_output(command: str) -> str:
  109. """How a sub-runner reports: the live lane (``--run-delivery query-file``) prints one JSON
  110. payload; the relay waiter and the ``hermes peer dm`` transport print the reply as text."""
  111. from tools.bot_mode_dm import _runner_argv
  112. argv = shlex.split(command)
  113. marker = argv.index("--run-delivery") if "--run-delivery" in argv else len(argv)
  114. parsed = _runner_argv(argv[marker:])
  115. return "json" if parsed is not None and parsed[1] == "query-file" else "text"
  116. def run_fan_out(spec_path: str) -> int:
  117. """Runner entry: run each target's runner concurrently, print one JSON outcome list."""
  118. with open(spec_path, encoding="utf-8-sig") as stream:
  119. entries = json.load(stream)
  120. with contextlib.suppress(OSError):
  121. os.unlink(spec_path)
  122. results: list[dict] = [{} for _ in entries]
  123. def run(index: int, entry: dict) -> None:
  124. base = {"to": entry["label"], "delivery_id": entry["delivery_id"]}
  125. try:
  126. done = subprocess.run(shlex.split(entry["command"]), stdin=subprocess.DEVNULL, capture_output=True,
  127. text=True, encoding="utf-8", errors="replace", timeout=FAN_OUT_WAIT_SECONDS,
  128. check=False)
  129. except subprocess.TimeoutExpired:
  130. results[index] = {**base, "status": "pending", "detail": "No outcome within the bound. Do not resend."}
  131. return
  132. except (OSError, ValueError) as exc:
  133. results[index] = {**base, "status": "failed", "error": f"Its delivery runner could not start: {exc}"}
  134. return
  135. results[index] = {**base, **_outcome(entry.get("output", "text"), done.returncode, done.stdout, done.stderr)}
  136. threads = [threading.Thread(target=run, args=item, daemon=True) for item in enumerate(entries)]
  137. for thread in threads:
  138. thread.start()
  139. for thread in threads:
  140. thread.join()
  141. settled = sum(result.get("status") == "settled" for result in results)
  142. from tools.bot_mode_dm_outcome import combined_summary
  143. for result in results:
  144. result.pop("outcome", None) # the summary below says it once, by name
  145. print(json.dumps({"outcome": combined_summary(results), "replies": results, "settled": settled,
  146. "of": len(results)}, ensure_ascii=False))
  147. return 0 if settled == len(results) else 1
  148. def _outcome(output: str, returncode: int, stdout: str, stderr: str = "") -> dict:
  149. """A sub-runner's result as a typed outcome. Only a ``json`` runner's stdout is a delivery
  150. record; a ``text`` runner's successful stdout is the teammate's answer verbatim (even one that
  151. looks like JSON). A failing text runner may still have been accepted: every runner says "do not
  152. resend" exactly when the request is still in flight, which is pending, never failed."""
  153. text = stdout.strip()
  154. if output == "json" or returncode != 0:
  155. try:
  156. parsed = json.loads(text.splitlines()[-1]) if text else None
  157. except ValueError:
  158. parsed = None
  159. if isinstance(parsed, dict):
  160. outcome = {k: v for k, v in parsed.items() if k != "delivery_id"}
  161. outcome.setdefault("status", "failed" if parsed.get("error") else "settled")
  162. return outcome
  163. if returncode == 0:
  164. return {"status": "settled", "reply": text or "(empty reply)"}
  165. said = "\n".join(part for part in (text, stderr.strip()) if part)
  166. if "do not resend" in said.lower():
  167. return {"status": "pending", "detail": said}
  168. return {"status": "failed", "error": said or f"delivery runner exited {returncode}"}
  169. def cancel_own_delivery(delivery_id: str, raw_target: str, sender: Any) -> str:
  170. """Withdraw the caller's own queued request, or stop the turn it started, on a local teammate."""
  171. from tools.bot_live_delivery import _TERMINAL, _authority_rpc, read_delivery_result
  172. from tools.bot_mode_dm import _err, _resolve_local_name
  173. if not _HEX_ID_RE.fullmatch(delivery_id):
  174. return _err("cancel takes the delivery_id from your own message_agent ack. Do not retry.")
  175. named = raw_target.strip().lstrip("@")
  176. resolved = _resolve_local_name(named, list(sender.roster_homes), sender.root) if named else None
  177. homes = ([sender.roster_homes[resolved]] if resolved else
  178. [home for name, home in sender.roster_homes.items() if name != sender.me])
  179. found = next(((home, record) for home in homes if (record := read_delivery_result(home, delivery_id))), None)
  180. if found is None:
  181. return _err("No request with that delivery_id to a teammate on this install (requests to another "
  182. "machine cannot be cancelled). Do not retry.")
  183. home, record = found
  184. if (record.get("author") or {}).get("id") != f"bot:{sender.me}":
  185. return _err("Only the agent that sent a request can cancel it. Do not retry.")
  186. if record["status"] in _TERMINAL:
  187. return json.dumps({"status": record["status"], "delivery_id": delivery_id,
  188. "detail": "That request already finished; nothing to cancel."})
  189. if not record.get("admission_id"):
  190. return _err("That request has no admission to cancel (its outcome is unknown). Do not retry.")
  191. try:
  192. # The owner decides which admission is the request now, under the lock that admits its one
  193. # automatic retry: a retry not admitted yet is withdrawn instead of racing this cancel.
  194. answer = _authority_rpc(home, "bot_relay.cancel", {
  195. "id": delivery_id, "author_id": f"bot:{sender.me}",
  196. "profile": Path(home).name if Path(home).parent.name == "profiles" else "default"})
  197. except Exception as exc:
  198. # The owner may be down or refuse: the sender gets that as a tool error, never a crash.
  199. logger.debug("message_agent cancel of %s failed", delivery_id, exc_info=True)
  200. return _err(f"Cancel failed: {exc}")
  201. detail = {"cancelled": "Withdrawn before it ran (or before its automatic retry); it will not run.",
  202. "stopping": "Its turn was running and was told to stop; its delivery process reports how it ended.",
  203. }.get(answer.get("status"), "That request already finished; nothing to cancel.")
  204. status = answer.get("status") if answer.get("status") in {"cancelled", "stopping"} else answer.get("receipt_status")
  205. return json.dumps({"status": status or "terminal", "delivery_id": delivery_id, "detail": detail})