managed.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. from __future__ import annotations
  2. import json
  3. import logging
  4. import time
  5. from typing import Any, Callable, Dict, List, Optional, Tuple
  6. from tools.connectors.contract import Actor, SettleReason, TargetState, allowed
  7. from tools.connectors.gateway.config import ensure_guest_identity, operation_session_key
  8. from tools.connectors.gateway.errors import RateLimited, ToolGatewayError
  9. from tools.connectors.operation import ConnectionOperation, DetachedOperation, IllegalTransition, Target
  10. from tools.connectors.run import Kind, run_operation
  11. from tools.connectors.targets import HostedTarget, catalog_names, hosted_names, misrouted_to_hosted_error
  12. from tools.registry import tool_error
  13. logger = logging.getLogger(__name__)
  14. # The account route carries its own 180/min budget, so one read per pending target per second stays
  15. # inside it and still flips the card within a second of the user finishing at the vendor.
  16. WATCH_TICK_SECONDS = 1.0
  17. # A read never outlives the operation, never asks for less than one second, and never holds the
  18. # loop for more than ten: Continue must be able to return the tool while a gateway hangs.
  19. _MIN_READ_SECONDS = 1.0
  20. _MAX_READ_SECONDS = 10.0
  21. # The six-state account vocabulary -> the state that read ends the attempt in, and who caused it.
  22. # `pending` is not here: it is the attempt still running, and moves nothing.
  23. _ACCOUNT_OUTCOME: dict[str, tuple[TargetState, Actor]] = {
  24. "active": (TargetState.connected, Actor.backend_watcher),
  25. "failed": (TargetState.failed, Actor.backend_watcher),
  26. "revoked": (TargetState.failed, Actor.backend_watcher),
  27. "inactive": (TargetState.failed, Actor.backend_watcher),
  28. # The link's TTL ran out; the gateway reports it, the clock caused it.
  29. "expired": (TargetState.expired, Actor.clock),
  30. }
  31. NOTE = (
  32. "Settled once. connected → use the app now; skipped → the user chose Not now, do not connect it "
  33. "or route around it; not_connected → ask the user what to do, never re-mint on your own. "
  34. "A later request from the USER for that same app is not a re-ask — run it. "
  35. "If you named an account, tell the user its name and that they can rename it."
  36. )
  37. GUEST_SETUP_FAILED = "GUEST_SETUP_FAILED"
  38. # The connect request's ``initiator``: manage_connections, the Connectors page, the card's Try again.
  39. AGENT_TOOL = "agent_tool"
  40. DESKTOP_PAGE = "desktop_page"
  41. CARD_RETRY = "card_retry"
  42. def managed_client(*, creates_guest: bool, initiator: Optional[str] = AGENT_TOOL, call: Optional[Any] = None):
  43. """The real gateway client for a user action. ``creates_guest`` is the caller's declaration that
  44. it may create the guest identity when the user has none (a connect, an explicit list); a page
  45. read passes False and holds a token already. ``initiator`` names who asked, so every link the
  46. client mints is attributed."""
  47. from tools.connectors.gateway.client import ConnectorClient
  48. from tools.connectors.gateway.errors import ToolGatewayError
  49. if creates_guest and (setup_error := ensure_guest_identity()):
  50. raise ToolGatewayError(setup_error, code=GUEST_SETUP_FAILED, retryable=True)
  51. return ConnectorClient(initiator=initiator, call=call)
  52. def managed_kind(client: Any, action: str, force: bool) -> Kind:
  53. noted: set[str] = set()
  54. def noting(step: Callable[[ConnectionOperation], None]) -> Callable[[ConnectionOperation], None]:
  55. def run(operation: ConnectionOperation) -> None:
  56. step(operation)
  57. _note_connected(operation, noted)
  58. return run
  59. return Kind(prepare=noting(_prepare(client, action, force)),
  60. observe=noting(lambda operation: _observe(client, operation)), note=NOTE)
  61. def _note_connected(operation: ConnectionOperation, noted: set[str]) -> None:
  62. """Each app that connects counts once toward the connector sign-in offer."""
  63. fresh = {t.name for t in operation.targets if t.state == TargetState.connected} - noted
  64. if fresh:
  65. from hermes_cli.free_tier_offer import note_app_connected
  66. noted.update(fresh)
  67. note_app_connected()
  68. def _status_by_slug(client: Any) -> dict[str, dict[str, Any]]:
  69. """The toolkit list, by slug. Only the reconnect repair check reads it: it answers "is this app
  70. already connected" before any account exists for the watcher to read."""
  71. return {str(i.get("connector", "")).lower(): i for i in client.list_connectors() if isinstance(i, dict)}
  72. def mint(client: Any, operation: ConnectionOperation, names: list[str], *, reinitiate: bool, actor: Actor,
  73. connection_id: Optional[str] = None) -> None:
  74. """Mint links for ``names`` and apply the gateway's per-app answer to the operation. ``actor`` is
  75. the watcher on the first mint and the user on Try again. The operation id rides along so the
  76. vendor's done page can name it on the way back to the desktop. ``connection_id`` names the one
  77. existing account a repair restarts; the gateway then needs no alias to find it. A repair without
  78. one (Try again) reuses the account the named target already minted, so a rename in between
  79. cannot point the retry at another login."""
  80. from tools.connectors.gateway.client import return_to_args
  81. if not names:
  82. return
  83. # One aliased target per call, so the first named target is the request's.
  84. named = next((t for t in operation.targets if t.name in names and t.alias), None)
  85. if reinitiate and connection_id is None:
  86. repaired = next((t.repair_id for t in operation.targets if t.name in names and t.repair_id), None)
  87. connection_id = repaired or (named.connection_id if named is not None else None)
  88. alias = None if connection_id or named is None else named.alias
  89. response = client.connections(names, reinitiate=reinitiate, alias=alias, connection_id=connection_id,
  90. **return_to_args(op=operation.op_id))
  91. for entry in response.get("results", []):
  92. name = str(entry.get("connector") or "").lower()
  93. target = operation.target(name)
  94. if target is None:
  95. continue
  96. status = str(entry.get("status") or "")
  97. detail = str(entry.get("status_reason") or "")
  98. if status not in ("active", "initiated"):
  99. # A refusal's reason comes in its instruction ("not allowed by your organization's policy").
  100. detail = detail or str(entry.get("instruction") or "")
  101. connection_id = entry.get("connection_id")
  102. if status == "active":
  103. operation.transition(name, TargetState.initiated, actor)
  104. operation.transition(name, TargetState.connected, Actor.backend_watcher, connection_id=connection_id)
  105. elif status == "initiated":
  106. if not connection_id:
  107. logger.warning("connector %s: the mint named no account, so the watcher cannot read it; "
  108. "only the card or the deadline can end the row", name)
  109. operation.transition(
  110. name, TargetState.initiated, actor,
  111. connect_url=entry.get("connect_url"), connection_id=connection_id,
  112. detail=detail,
  113. )
  114. elif target.state == TargetState.failed:
  115. # Failed again: no state change to emit, but the old link is dead and the vendor's text is new.
  116. operation.refresh(name, connect_url=None, detail=detail)
  117. elif target.state == TargetState.expired:
  118. # The table has no expired → failed; the re-mint attempt is the user's, so step through initiated.
  119. operation.transition(name, TargetState.initiated, actor)
  120. operation.transition(name, TargetState.failed, Actor.backend_watcher, detail=detail)
  121. operation.refresh(name, connect_url=None, detail=detail)
  122. else:
  123. # `detail` is the vendor's text or empty; the state itself is never written into it (the card prints it).
  124. operation.transition(name, TargetState.failed, Actor.backend_watcher, detail=detail)
  125. def _status_for(client: Any, target: Target, *, timeout: float) -> Optional[dict[str, Any]]:
  126. """The one route the watcher reads: that target's own account row. ``None`` means "nothing to
  127. apply this tick" — no account to read, a rate-limit still in force, an account the gateway does
  128. not know yet (404 until the deadline), or a read that failed. A 429 is raised to the tick: its
  129. budget is the principal's, so it is not this one target's to wait out."""
  130. if not target.connection_id or time.time() < target.next_read_at:
  131. return None
  132. try:
  133. return client.account_status(target.connection_id, timeout=timeout)
  134. except RateLimited:
  135. raise
  136. except Exception as exc:
  137. logger.warning("connector account read failed for %s: %s", target.name, exc)
  138. return None
  139. def _apply_read(operation: ConnectionOperation, target: Target, status: str, reason: str) -> None:
  140. """Apply one account read to one target. The RPC thread can move the row while the read is in
  141. flight — a Skip resolves it, a Continue freezes the whole result — and the read then has no
  142. live row to move: it is dropped, not raised into the tool result (that would end the watch
  143. with the operation still open and no card to answer it). Any other refusal is a real
  144. contract violation."""
  145. outcome = _ACCOUNT_OUTCOME.get(status)
  146. if outcome is None:
  147. return
  148. to, actor = outcome
  149. try:
  150. if allowed(target.kind, target.state, to) is None:
  151. # No edge from pending: the read is itself the witness that the attempt started.
  152. operation.transition(target.name, TargetState.initiated, Actor.backend_watcher)
  153. operation.transition(target.name, to, actor, detail=reason or target.detail)
  154. except IllegalTransition:
  155. if not operation.settled and _live(target):
  156. raise
  157. logger.debug("connector %s: %s read dropped, the row is %s", target.name, status, target.state.value)
  158. def _live(target: Target) -> bool:
  159. """Only a live attempt (pending, initiated) can be advanced by a gateway read; a failed or
  160. expired link waits for the user, and a resolved row is done."""
  161. return target.state in (TargetState.pending, TargetState.initiated)
  162. def _park(operation: ConnectionOperation, until: float) -> None:
  163. """A 429 is per principal, not per account: every live target waits out the same Retry-After."""
  164. for target in operation.targets:
  165. if _live(target):
  166. target.next_read_at = until
  167. def _observe(client: Any, operation: ConnectionOperation) -> None:
  168. """One account read per live target per tick, sequential: this is the only thread reading them.
  169. A 429 ends the tick: the next read would spend the same refused budget."""
  170. for target in operation.targets:
  171. # A settled op is frozen; a row that is not live waits for the user or is done.
  172. if operation.settled or not _live(target):
  173. continue
  174. timeout = min(_MAX_READ_SECONDS, max(_MIN_READ_SECONDS, operation.remaining_seconds()))
  175. try:
  176. row = _status_for(client, target, timeout=timeout)
  177. except RateLimited as exc:
  178. _park(operation, time.time() + exc.retry_after)
  179. return
  180. if row is None:
  181. continue
  182. _apply_read(operation, target, str(row.get("status") or "").lower(), str(row.get("statusReason") or ""))
  183. def _mark_misrouted(operation: ConnectionOperation) -> None:
  184. unminted = [t for t in operation.targets
  185. if t.state == TargetState.failed and not t.connection_id]
  186. if not unminted:
  187. return
  188. catalog = catalog_names()
  189. candidates = [t for t in unminted if t.name in catalog]
  190. if not candidates:
  191. return
  192. hosted = hosted_names()
  193. if hosted is None:
  194. return
  195. misrouted = [t for t in candidates if t.name not in hosted]
  196. for target in misrouted:
  197. operation.refresh(target.name, connect_url=None, actor=Actor.backend_watcher,
  198. detail=misrouted_to_hosted_error(target.name))
  199. if misrouted and len(misrouted) == len(operation.targets):
  200. operation.settle(SettleReason.all_resolved)
  201. def _unknown_slug(name: str, known: set[str]) -> str:
  202. close = sorted(slug for slug in known if name in slug or slug in name)
  203. if close:
  204. return f"'{name}' is not a connector slug; use {' or '.join(close)}"
  205. return f"'{name}' is not a connector slug; manage_connections status lists them"
  206. def _mark_unknown(client: Any, operation: ConnectionOperation) -> None:
  207. """The gateway refuses a slug it does not list as a policy block. The model guesses a slug from the
  208. app's name (granola for granola_mcp), so the row names the listed slug instead."""
  209. unminted = [t for t in operation.targets if t.state == TargetState.failed and not t.connection_id]
  210. if operation.settled or not unminted:
  211. return
  212. catalog = catalog_names() # a local MCP name already carries its misrouted detail
  213. unminted = [t for t in unminted if t.name not in catalog]
  214. if not unminted:
  215. return
  216. try:
  217. known = {str(i.get("connector", "")).lower() for i in client.list_connectors()}
  218. except ToolGatewayError as exc:
  219. logger.debug("connector list for an unminted slug failed: %s", exc)
  220. return
  221. unknown = [t for t in unminted if t.name not in known]
  222. for target in unknown:
  223. operation.refresh(target.name, connect_url=None, actor=Actor.backend_watcher,
  224. detail=_unknown_slug(target.name, known))
  225. if unknown and len(unknown) == len(operation.targets):
  226. operation.settle(SettleReason.all_resolved)
  227. def _prepare(client: Any, action: str, force: bool) -> Callable[[ConnectionOperation], None]:
  228. def prepare(operation: ConnectionOperation) -> None:
  229. names = [t.name for t in operation.targets]
  230. if action == "connect":
  231. mint(client, operation, names, reinitiate=False, actor=Actor.backend_watcher)
  232. _mark_misrouted(operation)
  233. _mark_unknown(client, operation)
  234. return
  235. # A named account is checked on its own row, read before force so a label resolves to its
  236. # account either way: the connector-wide flag reports any account.
  237. named = _alias_status(operation.targets)
  238. if force:
  239. # The re-mint names a new account; the watcher reads that one, never the old row.
  240. account = next((s["connection_id"] for s in (named or {}).values() if s.get("connection_id")), None)
  241. mint(client, operation, names, reinitiate=True, actor=Actor.backend_watcher, connection_id=account)
  242. _mark_misrouted(operation)
  243. _mark_unknown(client, operation)
  244. return
  245. status = named or _status_by_slug(client)
  246. repair = []
  247. for name in names:
  248. target = operation.target(name)
  249. if target is not None and target.repair_id and name not in (named or {}):
  250. # A surface asked to repair one account by id: its Reconnect is the user's choice, and
  251. # the connector's other accounts say nothing about this one.
  252. repair.append(name)
  253. elif status.get(name, {}).get("connected"):
  254. operation.transition(name, TargetState.initiated, Actor.backend_watcher)
  255. operation.transition(name, TargetState.connected, Actor.backend_watcher)
  256. else:
  257. repair.append(name)
  258. # A named target is alone in its call, so at most one account id applies.
  259. account = next((status[n].get("connection_id") for n in repair if status.get(n, {}).get("connection_id")), None)
  260. mint(client, operation, repair, reinitiate=True, actor=Actor.backend_watcher, connection_id=account)
  261. _mark_misrouted(operation)
  262. _mark_unknown(client, operation)
  263. return prepare
  264. def portal_accounts() -> list[dict[str, Any]]:
  265. from tools.connectors.portal.client import PortalConnectorClient
  266. return PortalConnectorClient().list_accounts()
  267. def _alias_status(targets: list[Target]) -> Optional[dict[str, dict[str, Any]]]:
  268. """``{slug: {"connected": bool, "connection_id": str | None}}`` for an aliased target, read from
  269. that account's own row (``connection_id`` is None when no account has that name yet); ``None``
  270. when no target is aliased. A failed read raises: guessing "not connected" would send the user
  271. through a new login for an account that may be healthy."""
  272. aliased = [t for t in targets if t.alias]
  273. if not aliased:
  274. return None
  275. rows = portal_accounts()
  276. status: dict[str, dict[str, Any]] = {}
  277. for t in aliased:
  278. mine = [r for r in rows if r.get("connector") == t.name and not r.get("disabled")]
  279. row = next((r for r in mine if r.get("alias") == t.alias), None)
  280. if row is None:
  281. # An unnamed account is addressed by its label, as rename does; it stays unnamed.
  282. row = next((r for r in mine if not r.get("alias") and r.get("label") == t.alias), None)
  283. if row is not None:
  284. t.alias, t.repair_id = None, row.get("connectionId")
  285. status[t.name] = {"connected": bool(row) and row.get("status") == "active",
  286. "connection_id": row.get("connectionId") if row else None}
  287. return status
  288. def _targets(targets: list[HostedTarget], action: str) -> list[Target]:
  289. return [Target(t.name, "connector", action, alias=t.alias) for t in targets]
  290. def _no_card_result(client: Any, action: str, targets: list[HostedTarget], force: bool, session_id: str) -> str:
  291. operation = DetachedOperation(_targets(targets, action), session_key=session_id)
  292. _prepare(client, action, force)(operation)
  293. payload = operation.result(with_urls=True)
  294. payload["status"] = "initiated" if any(t.state == TargetState.initiated for t in operation.targets) else "settled"
  295. payload["note"] = (
  296. "Show each connect_url to the user; they open it in a browser to authorize. Ask them to tell you "
  297. "when they are done, then check with action 'status'. Do not call connect again for the same app."
  298. )
  299. if any(t.alias for t in targets):
  300. payload["note"] += " You named this account: tell the user its name and that they can rename it."
  301. return json.dumps(payload, ensure_ascii=False)
  302. def _status_result(client: Any, connectors: list[str]) -> str:
  303. items = client.list_connectors()
  304. from hermes_cli.free_tier_offer import note_connected_apps_seen
  305. note_connected_apps_seen(items)
  306. known = {str(i.get("connector", "")).lower() for i in items}
  307. unknown = {name: _unknown_slug(name, known) for name in connectors if name not in known}
  308. if connectors:
  309. wanted = set(connectors)
  310. items = [i for i in items if str(i.get("connector", "")).lower() in wanted]
  311. try:
  312. rows = portal_accounts()
  313. except ToolGatewayError as exc:
  314. # The account list is extra detail; the connector list alone still answers status.
  315. logger.debug("connector accounts for status failed: %s", exc)
  316. else:
  317. by_slug: dict[str, list[dict[str, Any]]] = {}
  318. for row in rows:
  319. entry = {"alias": row.get("alias"), "label": row.get("label"), "status": row.get("status"),
  320. "active": row.get("active")}
  321. if row.get("disabled"):
  322. entry["retired"] = True
  323. by_slug.setdefault(str(row.get("connector", "")).lower(), []).append(entry)
  324. items = [dict(i, accounts=by_slug.get(str(i.get("connector", "")).lower(), [])) for i in items]
  325. return json.dumps({"connectors": items, "hint": (
  326. "connected=false means calls to that connector will return CONNECTION_REQUIRED. "
  327. "Use action 'connect' to start an authorization. 'accounts' lists each account by its name "
  328. "(alias) and the vendor's label; address an unnamed account by its label. A retired account was "
  329. "replaced by a reconnect and is kept only for the record."), **({"unknown": unknown} if unknown else {})},
  330. ensure_ascii=False)
  331. def _display_result(client: Any, suggested: list[str], *, session_id: Optional[str], tool_call_id: Optional[str],
  332. connection_callback: Optional[Callable[[dict[str, Any]], Optional[str]]], call: Any) -> str:
  333. """The desktop card when one can draw; elsewhere the status list and where to connect."""
  334. from tools.connectors.display import OFF_DESKTOP_NOTE, run_display
  335. if connection_callback is None:
  336. return json.dumps({**json.loads(_status_result(client, [])), "note": OFF_DESKTOP_NOTE}, ensure_ascii=False)
  337. return run_display(client, suggested, session_key=operation_session_key(session_id), tool_call_id=tool_call_id,
  338. connection_callback=connection_callback, call=call)
  339. def _rename_result(target: HostedTarget) -> str:
  340. """Rename the one account of ``target.name`` called ``target.alias`` (or, when unnamed, labelled
  341. so) to ``target.to``. Synchronous: no operation and no card."""
  342. from tools.connectors.gateway.errors import GatewayUnavailable, IdempotencyConflict
  343. from tools.connectors.portal.client import PortalConnectorClient
  344. rows = [r for r in portal_accounts() if str(r.get("connector", "")).lower() == target.name]
  345. matches = [r for r in rows if r.get("alias") == target.alias] or [
  346. r for r in rows if not r.get("alias") and r.get("label") == target.alias]
  347. if len(matches) != 1:
  348. names = sorted({str(r.get("alias") or r.get("label")) for r in rows})
  349. found = f"Its accounts are: {', '.join(names)}." if names else f"The user has no {target.name} account."
  350. what = "No" if not matches else "More than one"
  351. return tool_error(f"{what} {target.name} account is called {target.alias!r}. {found}")
  352. try:
  353. account = PortalConnectorClient().rename_account(matches[0]["connectionId"], target.to)
  354. except IdempotencyConflict:
  355. return tool_error(f"The name {target.to!r} is already used by another {target.name} account. Pick another name.")
  356. except GatewayUnavailable as exc:
  357. if exc.code == "connection_not_found":
  358. return tool_error(f"That {target.name} account no longer exists. Use action status to list accounts.")
  359. return tool_error("This Nous Portal cannot rename connector accounts yet.")
  360. except ToolGatewayError as exc:
  361. if exc.status == 405:
  362. return tool_error("This Nous Portal cannot rename connector accounts yet.")
  363. raise
  364. return json.dumps({
  365. "renamed": {"connector": account["connector"], "alias": account.get("alias"), "label": account["label"]},
  366. "notice": f"Tell the user their {target.name} account {target.alias!r} is now called {target.to!r}.",
  367. }, ensure_ascii=False)
  368. def run_managed_action(
  369. action: str,
  370. targets: list[HostedTarget],
  371. args: dict[str, Any],
  372. *,
  373. client_factory: Optional[Callable[[], Any]] = None,
  374. session_id: Optional[str] = None,
  375. tool_call_id: Optional[str] = None,
  376. turn_id: Optional[str] = None,
  377. connection_callback: Optional[Callable[[dict[str, Any]], Optional[str]]] = None,
  378. connectors_available: Optional[Callable[[], bool]] = None,
  379. card_surface: bool = False,
  380. ) -> str:
  381. from tools.managed_gateway_identity import CallRef
  382. if connectors_available is not None and not connectors_available():
  383. return tool_error("Connectors are not available in this session.")
  384. try:
  385. if action == "rename":
  386. return _rename_result(targets[0])
  387. call = CallRef(turn_id=turn_id, tool_call_id=tool_call_id)
  388. client = client_factory() if client_factory else managed_client(creates_guest=True, call=call)
  389. if action == "display":
  390. return _display_result(client, [t.name for t in targets], session_id=session_id, tool_call_id=tool_call_id,
  391. connection_callback=connection_callback if card_surface else None, call=call)
  392. if action == "status":
  393. return _status_result(client, [t.name for t in targets])
  394. if not targets:
  395. return tool_error(
  396. f"'{action}' requires 'connectors': the connector slugs to authorize (e.g. [\"gmail\"]). "
  397. "Use action 'status' to list them."
  398. )
  399. force = bool(args.get("force", False))
  400. session_key = operation_session_key(session_id)
  401. if connection_callback is None:
  402. return _no_card_result(client, action, targets, force, session_key)
  403. return run_operation(
  404. _targets(targets, action),
  405. managed_kind(client, action, force),
  406. session_key=session_key, tool_call_id=tool_call_id, tick_seconds=WATCH_TICK_SECONDS,
  407. connection_callback=connection_callback, with_urls_in_result=False, call=call,
  408. )
  409. except Exception as exc:
  410. logger.warning("manage_connections %s failed: %s", action, exc)
  411. if getattr(exc, "code", None) == GUEST_SETUP_FAILED:
  412. return tool_error(str(exc))
  413. return tool_error(
  414. f"The connector gateway request failed: {exc}. "
  415. "If this persists, the user can manage connections in the Nous Portal."
  416. )