channel_directory.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475
  1. """Channel directory -- cached map of reachable channels/contacts per platform.
  2. Built on gateway startup, refreshed every 5 min, saved to ``<HERMES_HOME>/channel_directory.json``.
  3. send_message reads it for action="list" and to resolve friendly channel names to IDs.
  4. """
  5. import asyncio
  6. import contextlib
  7. import json
  8. import logging
  9. import time
  10. from datetime import datetime
  11. from pathlib import Path
  12. from typing import Any, Dict, Iterable, List, Optional, Tuple
  13. from hermes_cli.config import get_hermes_home
  14. from utils import atomic_json_write
  15. logger = logging.getLogger(__name__)
  16. # Paths resolve lazily: a multiplexed gateway serves several profile homes from one
  17. # process, so an import-time constant would pin every profile to whichever home imported
  18. # first. These globals are explicit overrides (tests patch them); None = current home.
  19. DIRECTORY_PATH: Optional[Path] = None
  20. # User-maintained friendly-name overlay {"<platform>": {"<chat_id>": "<friendly name>"}},
  21. # re-applied on every build AND load (hand-edits to the regenerated
  22. # channel_directory.json don't survive); also lets a chat be pre-named before first traffic.
  23. CHANNEL_ALIASES_PATH: Optional[Path] = None
  24. # Slack refresh failures recur on every timed rebuild (missing scope, revoked
  25. # token); warn once per (team, error detail) per interval, then DEBUG.
  26. _SLACK_DIRECTORY_WARNING_INTERVAL_SECONDS = 3600
  27. _slack_directory_warning_last: dict[tuple[str, str], float] = {}
  28. # Platforms whose historical session origins must never become send targets.
  29. _SKIP_SESSION_DISCOVERY = frozenset({"local", "api_server", "webhook"})
  30. _SLACK_RAW_ID_PREFIXES = ("C0", "D0", "G0")
  31. def _directory_path() -> Path:
  32. return DIRECTORY_PATH or get_hermes_home() / "channel_directory.json"
  33. def _aliases_path() -> Path:
  34. return CHANNEL_ALIASES_PATH or get_hermes_home() / "channel_aliases.json"
  35. def _read_json(path: Path) -> Any:
  36. # utf-8-sig: Windows editors may leave a BOM that plain utf-8 json.load rejects (pm-era fix).
  37. with open(path, encoding="utf-8-sig") as f:
  38. return json.load(f)
  39. def _load_json_dict(path: Path) -> dict[str, Any]:
  40. """Read a JSON object from *path*; {} when missing, unreadable, or not a dict."""
  41. if not path.exists():
  42. return {}
  43. try:
  44. data = _read_json(path)
  45. return data if isinstance(data, dict) else {}
  46. except Exception:
  47. return {}
  48. def _apply_channel_aliases(platforms: dict[str, Any]) -> None:
  49. """Overlay friendly names onto directory entries by chat_id, in place.
  50. An aliased id not yet discovered gets a placeholder entry so a freshly-created
  51. group is addressable by name before its first message.
  52. """
  53. for plat_name, id_map in _load_json_dict(_aliases_path()).items():
  54. if not isinstance(id_map, dict):
  55. continue
  56. entries = platforms.setdefault(plat_name, [])
  57. if not isinstance(entries, list):
  58. continue
  59. for chat_id, friendly in id_map.items():
  60. if not isinstance(friendly, str) or not friendly.strip():
  61. continue
  62. chat_id, friendly = str(chat_id), friendly.strip()
  63. matches = [e for e in entries if isinstance(e, dict) and e.get("id") == chat_id]
  64. for e in matches:
  65. e["name"] = friendly
  66. if not matches:
  67. entries.append({"id": chat_id, "name": friendly, "thread_id": None,
  68. "type": "group" if chat_id.endswith("@g.us") else "dm"})
  69. def _normalize_channel_query(value: str) -> str:
  70. return value.lstrip("#").strip().lower()
  71. def _channel_target_name(platform_name: str, channel: dict[str, Any]) -> str:
  72. """Human-facing target label for a channel entry."""
  73. name = channel["name"]
  74. if platform_name == "discord":
  75. return f"#{name}" if channel.get("guild") else name
  76. return f"{name} ({channel['type']})" if channel.get("type") else name
  77. def _session_entry_id(origin: dict[str, Any]) -> Optional[str]:
  78. chat_id = origin.get("chat_id")
  79. if not chat_id:
  80. return None
  81. return f"{chat_id}:{thread_id}" if (thread_id := origin.get("thread_id")) else str(chat_id)
  82. def _session_entry_name(origin: dict[str, Any]) -> str:
  83. base_name = origin.get("chat_name") or origin.get("user_name") or str(origin.get("chat_id"))
  84. if not (thread_id := origin.get("thread_id")):
  85. return base_name
  86. return f"{base_name} / {origin.get('chat_topic') or f'topic {thread_id}'}"
  87. def _report_slack_failure(team_id: str, error_code: Optional[str], detail: str) -> None:
  88. """missing_scope is expected (session-history fallback); anything else warns once per interval."""
  89. if error_code == "missing_scope":
  90. logger.debug("Channel directory: Slack team %s lacks channels:read; using session history only", team_id)
  91. return
  92. key = (str(team_id), str(detail))
  93. now = time.monotonic()
  94. last = _slack_directory_warning_last.get(key)
  95. if last is None or now - last >= _SLACK_DIRECTORY_WARNING_INTERVAL_SECONDS:
  96. _slack_directory_warning_last[key] = now
  97. logger.warning("Channel directory: failed to list Slack channels for team %s: %s", team_id, detail)
  98. else:
  99. logger.debug("Channel directory: suppressed repeated Slack channel list failure for team %s: %s", team_id, detail)
  100. # --- Build / refresh -------------------------------------------------------
  101. async def build_channel_directory(adapters: dict[Any, Any]) -> dict[str, Any]:
  102. """Build the directory from connected adapters + session data and persist it."""
  103. from gateway.config import Platform
  104. platforms: dict[str, list[dict[str, str]]] = {}
  105. for platform, adapter in adapters.items():
  106. try:
  107. list_channels = getattr(adapter, "list_channels", None)
  108. if callable(list_channels):
  109. platform_channels = await list_channels()
  110. if platform_channels is not None:
  111. platforms[platform.value] = _normalize_adapter_channels(platform_channels)
  112. continue
  113. if platform == Platform.DISCORD:
  114. platforms["discord"] = await asyncio.to_thread(_build_discord, adapter)
  115. elif platform == Platform.SLACK:
  116. platforms["slack"] = await _build_slack(adapter)
  117. except Exception as e:
  118. logger.warning("Channel directory: failed to build %s: %s", platform.value, e)
  119. # Platforms without channel enumeration get session-based discovery, but only when
  120. # connected in THIS gateway process: origins for disabled or decommissioned
  121. # platforms must not resurface as stale send targets.
  122. adapter_platform_names = {getattr(p, "value", str(p)) for p in adapters}
  123. async def _discover(plat_name: str) -> None:
  124. if plat_name in _SKIP_SESSION_DISCOVERY or plat_name in platforms or plat_name not in adapter_platform_names:
  125. return
  126. platforms[plat_name] = await asyncio.to_thread(_build_from_sessions, plat_name)
  127. for plat in Platform:
  128. await _discover(plat.value)
  129. # Plugin platforms are dynamic enum members missing from Platform.__members__. Only connected
  130. # adapters are discovered, so loaded entries suffice (never import an unused adapter here).
  131. with contextlib.suppress(Exception):
  132. from gateway.platform_registry import platform_registry
  133. for entry in platform_registry.loaded_entries():
  134. await _discover(entry.name)
  135. _apply_channel_aliases(platforms)
  136. directory = {"updated_at": datetime.now().isoformat(), "platforms": platforms}
  137. try:
  138. await asyncio.to_thread(atomic_json_write, _directory_path(), directory)
  139. except Exception as e:
  140. logger.warning("Channel directory: failed to write: %s", e)
  141. return directory
  142. def _build_discord(adapter) -> list[dict[str, str]]:
  143. """Enumerate text + forum channels the Discord bot can see, plus session DMs."""
  144. channels = []
  145. client = getattr(adapter, "_client", None)
  146. if not client:
  147. return channels
  148. try:
  149. import discord as _discord
  150. except ImportError:
  151. return channels
  152. from gateway.platforms.helpers import is_discord_channel_obfuscated
  153. for guild in client.guilds:
  154. # Forum channels (type 15): creating a message auto-spawns a thread post.
  155. forums = getattr(guild, "forum_channels", None) or []
  156. for chs, ch_type in ((guild.text_channels, "channel"), (forums, "forum")):
  157. for ch in chs:
  158. # Obfuscated placeholders (no VIEW_CHANNEL) can never be posted to. #90154
  159. if is_discord_channel_obfuscated(ch):
  160. continue
  161. channels.append({"id": str(ch.id), "name": ch.name, "guild": guild.name, "type": ch_type})
  162. # DM-capable users aren't reachable via guild enumeration; they come from sessions.
  163. channels.extend(_build_from_sessions("discord"))
  164. return channels
  165. def _slack_api_error_code(error: Exception) -> Optional[str]:
  166. """Slack Web API error code from SlackApiError-like exceptions."""
  167. with contextlib.suppress(Exception):
  168. value = error.response.get("error")
  169. return str(value) if value else None
  170. return None
  171. def _normalize_adapter_channels(raw_channels: Any) -> list[dict[str, Any]]:
  172. """Validate and dedupe entries returned by an adapter's ``list_channels()`` hook."""
  173. channels: list[dict[str, Any]] = []
  174. seen_ids = set()
  175. for raw in raw_channels if isinstance(raw_channels, list) else ():
  176. if not isinstance(raw, dict):
  177. continue
  178. channel_id = str(raw.get("id") or "").strip()
  179. name = str(raw.get("name") or channel_id).strip()
  180. if not channel_id or not name or channel_id in seen_ids:
  181. continue
  182. entry: dict[str, Any] = {"id": channel_id, "name": name, "type": str(raw.get("type") or "dm")}
  183. entry.update({key: str(raw[key]) for key in ("thread_id", "guild") if raw.get(key)})
  184. channels.append(entry)
  185. seen_ids.add(channel_id)
  186. return channels
  187. def _slack_base_id(entry_id: str) -> str:
  188. """Thread-qualified IDs (``C0xxx:ts``) are internal routing keys, not Slack API IDs."""
  189. return entry_id.split(":", 1)[0]
  190. def _slack_has_raw_name(entry: dict[str, Any]) -> bool:
  191. return entry.get("name", "").startswith(_SLACK_RAW_ID_PREFIXES)
  192. async def _slack_team_channels(team_id: str, client, seen_ids: set) -> list[dict[str, Any]]:
  193. """``users.conversations`` for one workspace (public + private member channels), paginated."""
  194. channels: list[dict[str, Any]] = []
  195. try:
  196. cursor: Optional[str] = None
  197. for _page in range(20): # safety cap on pagination
  198. response = await client.users_conversations(
  199. types="public_channel,private_channel", exclude_archived=True, limit=200, cursor=cursor,
  200. )
  201. if not response.get("ok"):
  202. error_code = response.get("error", "unknown")
  203. _report_slack_failure(team_id, error_code, f"users.conversations not ok: {error_code}")
  204. break
  205. for ch in response.get("channels", []):
  206. cid, name = ch.get("id"), ch.get("name")
  207. if not cid or not name or cid in seen_ids:
  208. continue
  209. seen_ids.add(cid)
  210. channels.append({"id": cid, "name": name, "type": "private" if ch.get("is_private") else "channel"})
  211. cursor = (response.get("response_metadata") or {}).get("next_cursor")
  212. if not cursor:
  213. break
  214. except Exception as e:
  215. _report_slack_failure(team_id, _slack_api_error_code(e), str(e))
  216. return channels
  217. async def _slack_resolve_raw_names(client, channels: list[dict[str, Any]]) -> None:
  218. """Name remaining raw-ID entries (DMs, channels outside bot scope) via
  219. conversations.info + users.info once per base conversation, concurrently."""
  220. unresolved_by_base: dict[str, list] = {}
  221. for entry in channels:
  222. if _slack_has_raw_name(entry):
  223. unresolved_by_base.setdefault(_slack_base_id(entry["id"]), []).append(entry)
  224. if not unresolved_by_base:
  225. return
  226. async def _resolve_base(base_id: str, entries: list) -> None:
  227. try:
  228. resp = await client.conversations_info(channel=base_id)
  229. if not resp.get("ok"):
  230. return
  231. ch_info = resp.get("channel", {})
  232. resolved_name = resolved_type = None
  233. if not ch_info.get("is_im"):
  234. resolved_name = ch_info.get("name") or ch_info.get("name_normalized")
  235. elif ch_info.get("user", ""):
  236. user_resp = await client.users_info(user=ch_info["user"])
  237. if user_resp.get("ok"):
  238. u = user_resp["user"]
  239. resolved_name = u.get("profile", {}).get("display_name") or u.get("real_name") or u.get("name")
  240. resolved_type = "dm"
  241. for entry in entries if resolved_name else ():
  242. entry["name"] = resolved_name
  243. if resolved_type:
  244. entry["type"] = resolved_type
  245. except Exception as e:
  246. logger.debug("Channel directory: failed to resolve %s: %s", base_id, e)
  247. await asyncio.gather(*[_resolve_base(bid, ents) for bid, ents in unresolved_by_base.items()])
  248. async def _build_slack(adapter) -> list[dict[str, Any]]:
  249. """List Slack channels the bot has joined across all workspaces, merged with
  250. session-history DMs. Missing channels:read falls back to session history quietly."""
  251. team_clients = getattr(adapter, "_team_clients", None) or {}
  252. if not team_clients:
  253. return await asyncio.to_thread(_build_from_sessions, "slack")
  254. channels: list[dict[str, Any]] = []
  255. seen_ids: set = set()
  256. for team_id, client in team_clients.items():
  257. channels.extend(await _slack_team_channels(team_id, client, seen_ids))
  258. # Merge session-history DM/group entries, naming raw-ID entries from the
  259. # API-discovered channels where the base conversation ID is known.
  260. api_name_lookup = {ch["id"]: ch["name"] for ch in channels}
  261. for entry in await asyncio.to_thread(_build_from_sessions, "slack"):
  262. eid = entry.get("id")
  263. if not isinstance(eid, str) or eid in seen_ids:
  264. continue
  265. if _slack_has_raw_name(entry) and _slack_base_id(eid) in api_name_lookup:
  266. entry["name"] = api_name_lookup[_slack_base_id(eid)]
  267. channels.append(entry)
  268. seen_ids.add(eid)
  269. await _slack_resolve_raw_names(next(iter(team_clients.values())), channels)
  270. return channels
  271. def _build_from_sessions(platform_name: str) -> list[dict[str, str]]:
  272. """Known channels/contacts from session origins: state.db first, sessions.json fallback (pre-migration).
  273. state.db is the primary source (#9006): gateway session rows persist origin_json.
  274. """
  275. return _build_from_sessions_db(platform_name) or _build_from_sessions_json(platform_name)
  276. def _entries_from_origins(platform_name: str, source: str, origins_fn) -> list[dict[str, Any]]:
  277. """Deduped entries for the (origin, chat_type) pairs from ``origins_fn()``; a mid-iteration
  278. failure keeps entries read so far."""
  279. entries: list[dict[str, Any]] = []
  280. try:
  281. seen_ids = set()
  282. for origin, chat_type in origins_fn():
  283. entry_id = _session_entry_id(origin)
  284. if not entry_id or entry_id in seen_ids:
  285. continue
  286. seen_ids.add(entry_id)
  287. entries.append({
  288. "id": entry_id, "name": _session_entry_name(origin),
  289. "type": chat_type, "thread_id": origin.get("thread_id"),
  290. })
  291. except Exception as e:
  292. logger.debug("Channel directory: %s for %s: %s", source, platform_name, e)
  293. return entries
  294. def _build_from_sessions_db(platform_name: str) -> list[dict[str, str]]:
  295. """Pull channels/contacts from state.db gateway session rows."""
  296. def _origins() -> Iterable[tuple[dict[str, Any], Any]]:
  297. from hermes_state_registry import acquire, release_or_close
  298. db = acquire()
  299. try:
  300. lister = getattr(db, "list_gateway_sessions", None)
  301. if not callable(lister):
  302. return
  303. rows = lister(platform=platform_name, active_only=False)
  304. finally:
  305. release_or_close(db)
  306. for row in rows:
  307. origin = None
  308. with contextlib.suppress(TypeError, ValueError):
  309. origin = json.loads(row["origin_json"]) if row.get("origin_json") else None
  310. if not isinstance(origin, dict) or not origin:
  311. origin = {"chat_id": row.get("chat_id"), "thread_id": row.get("thread_id"), "chat_name": row.get("display_name")}
  312. yield origin, row.get("chat_type") or "dm"
  313. return _entries_from_origins(platform_name, "state.db session read failed", _origins)
  314. def _build_from_sessions_json(platform_name: str) -> list[dict[str, str]]:
  315. """Legacy fallback: pull channels/contacts from sessions.json origin data."""
  316. sessions_path = get_hermes_home() / "sessions" / "sessions.json"
  317. if not sessions_path.exists():
  318. return []
  319. def _origins() -> Iterable[tuple[dict[str, Any], Any]]:
  320. for _key, session in _read_json(sessions_path).items():
  321. # Keys starting with "_" (e.g. the gateway's "_README") are metadata sentinels.
  322. if str(_key).startswith("_") or not isinstance(session, dict):
  323. continue
  324. origin = session.get("origin") or {}
  325. if origin.get("platform") == platform_name:
  326. yield origin, session.get("chat_type", "dm")
  327. return _entries_from_origins(platform_name, "failed to read sessions", _origins)
  328. # --- Read / resolve --------------------------------------------------------
  329. def load_directory() -> dict[str, Any]:
  330. """Load the cached directory from disk, with aliases re-applied on read."""
  331. directory_path = _directory_path()
  332. if directory_path.exists():
  333. with contextlib.suppress(Exception):
  334. data = _read_json(directory_path)
  335. # Aliases apply on read too, so new names take effect between timed rebuilds.
  336. _apply_channel_aliases(data.setdefault("platforms", {}))
  337. return data
  338. base = {"updated_at": None, "platforms": {}}
  339. _apply_channel_aliases(base["platforms"])
  340. return base
  341. def lookup_channel_type(platform_name: str, chat_id: str) -> Optional[str]:
  342. """Channel ``type`` string (e.g. ``"channel"``, ``"forum"``) for *chat_id*, or None if unknown."""
  343. channels = load_directory().get("platforms", {}).get(platform_name, [])
  344. return next((ch.get("type") for ch in channels if ch.get("id") == chat_id), None)
  345. def resolve_channel_name(platform_name: str, name: str) -> Optional[str]:
  346. """Resolve a friendly channel name (e.g. "bot-home", "#bot-home", "GuildName/bot-home",
  347. Slack "#engineering") to an ID; case-insensitive, first match wins."""
  348. channels = load_directory().get("platforms", {}).get(platform_name, [])
  349. if not channels:
  350. return None
  351. # 0. Exact ID match — case-sensitive, no normalization, so raw platform IDs (e.g. Slack
  352. # "C0B0QV5434G") work even when _parse_target_ref's format guard didn't recognize them.
  353. raw = name.strip()
  354. for ch in channels:
  355. if ch.get("id") == raw:
  356. return ch["id"]
  357. query = _normalize_channel_query(name)
  358. # 1. Exact name match, including the display labels shown by send_message(action="list")
  359. for ch in channels:
  360. if query in (_normalize_channel_query(ch["name"]), _normalize_channel_query(_channel_target_name(platform_name, ch))):
  361. return ch["id"]
  362. # 2. Guild-qualified match for Discord ("GuildName/channel")
  363. if "/" in query:
  364. guild_part, ch_part = query.rsplit("/", 1)
  365. for ch in channels:
  366. guild = ch.get("guild", "").strip().lower()
  367. if guild == guild_part and _normalize_channel_query(ch["name"]) == ch_part:
  368. return ch["id"]
  369. # 3. Partial prefix match (only if unambiguous)
  370. matches = [ch for ch in channels if _normalize_channel_query(ch["name"]).startswith(query)]
  371. return matches[0]["id"] if len(matches) == 1 else None
  372. def format_directory_for_display(platforms: Optional[dict[str, Any]] = None) -> str:
  373. """Format the channel directory as a human-readable list for the model.
  374. ``platforms`` overrides the on-disk directory (``hermes send --list`` merges in
  375. configured-but-undiscovered platforms); an empty channel list renders a "(no channels
  376. discovered yet)" hint because the platform is still a valid send target.
  377. """
  378. if platforms is None:
  379. platforms = load_directory().get("platforms", {})
  380. if not platforms:
  381. return "No messaging platforms connected or no channels discovered yet."
  382. lines = ["Available messaging targets:\n"]
  383. for plat_name, channels in sorted(platforms.items()):
  384. if not channels:
  385. lines.append(f"{plat_name.title()}:")
  386. lines.append(
  387. f" (no channels discovered yet — send directly with "
  388. f"{plat_name}:<chat_id>, or bare '{plat_name}' for the home channel)"
  389. )
  390. elif plat_name == "discord":
  391. # Group Discord channels by guild (sorted by name); DMs last, in discovery order.
  392. guilds: dict[str, list] = {}
  393. dms: list = []
  394. for ch in channels:
  395. (guilds.setdefault(ch["guild"], []) if ch.get("guild") else dms).append(ch)
  396. groups = [(f"Discord ({g}):", sorted(chs, key=lambda c: c["name"])) for g, chs in sorted(guilds.items())]
  397. if dms:
  398. groups.append(("Discord (DMs):", dms))
  399. for header, group in groups:
  400. lines.append(header)
  401. lines.extend(f" discord:{_channel_target_name(plat_name, ch)}" for ch in group)
  402. else:
  403. lines.append(f"{plat_name.title()}:")
  404. lines.extend(f" {plat_name}:{_channel_target_name(plat_name, ch)}" for ch in channels)
  405. lines.append("")
  406. lines.append('Use these as the "target" parameter when sending.')
  407. lines.append('Bare platform name (e.g. "telegram") sends to home channel.')
  408. return "\n".join(lines)