plugin_llm.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600
  1. """Plugin LLM facade — host-owned LLM access for trusted plugins (``ctx.llm``).
  2. ``complete`` / ``complete_structured`` (text + image inputs, JSON schema validation)
  3. and their async siblings. Provider/model/agent_id/profile are explicit keyword
  4. arguments mirroring the host config shape; the host owns routing, auth, timeouts
  5. and fallback, so the plugin never sees raw tokens or keys. Every override knob is
  6. gated by the per-plugin ``plugins.entries.<id>.llm.allow_*_override`` trust flags
  7. (fail-closed: a missing block means "no overrides"). Backed by
  8. :func:`agent.auxiliary_client.call_llm`.
  9. """
  10. from __future__ import annotations
  11. import base64
  12. import json
  13. import logging
  14. import re
  15. from dataclasses import dataclass, field
  16. from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Union
  17. logger = logging.getLogger(__name__)
  18. @dataclass
  19. class PluginLlmTextInput:
  20. """Text block in a structured input list."""
  21. text: str
  22. type: str = "text"
  23. @dataclass
  24. class PluginLlmImageInput:
  25. """Image block: ``data`` (raw bytes) or ``url`` (http(s)/data: URL).
  26. ``mime_type`` is required for non-PNG bytes to render across providers."""
  27. data: Optional[bytes] = None
  28. url: Optional[str] = None
  29. mime_type: str = "image/png"
  30. file_name: str = ""
  31. type: str = "image"
  32. PluginLlmInput = PluginLlmTextInput | PluginLlmImageInput | dict[str, Any]
  33. """One structured input block: a dataclass above or a plain dict of the same shape."""
  34. @dataclass
  35. class PluginLlmUsage:
  36. """Token + cost usage; every field optional. ``cost_usd`` is the host's best estimate."""
  37. input_tokens: int = 0
  38. output_tokens: int = 0
  39. total_tokens: int = 0
  40. cache_read_tokens: int = 0
  41. cache_write_tokens: int = 0
  42. cost_usd: Optional[float] = None
  43. @dataclass
  44. class PluginLlmCompleteResult:
  45. """Result of :meth:`PluginLlm.complete`."""
  46. text: str
  47. provider: str
  48. model: str
  49. agent_id: str
  50. usage: PluginLlmUsage = field(default_factory=PluginLlmUsage)
  51. audit: dict[str, Any] = field(default_factory=dict)
  52. @dataclass
  53. class PluginLlmStructuredResult:
  54. """Result of :meth:`PluginLlm.complete_structured`.
  55. ``parsed`` is set only when JSON output was requested AND the response was
  56. valid JSON; ``content_type`` is then ``"json"``, otherwise ``"text"``."""
  57. text: str
  58. provider: str
  59. model: str
  60. agent_id: str
  61. usage: PluginLlmUsage = field(default_factory=PluginLlmUsage)
  62. parsed: Optional[Any] = None
  63. content_type: str = "text"
  64. audit: dict[str, Any] = field(default_factory=dict)
  65. @dataclass(frozen=True)
  66. class _TrustPolicy:
  67. """Resolved trust gate for one plugin's LLM access."""
  68. plugin_id: str
  69. allow_provider_override: bool = False
  70. allowed_providers: Optional[frozenset] = None # None = no allowlist
  71. allow_any_provider: bool = False # True when allowed_providers == ["*"]
  72. allow_model_override: bool = False
  73. allowed_models: Optional[frozenset] = None # None = no allowlist
  74. allow_any_model: bool = False # True when allowed_models == ["*"]
  75. allow_agent_id_override: bool = False
  76. allow_profile_override: bool = False
  77. # Lets ``complete(task=...)`` borrow the host's *built-in* aux task slots.
  78. # Slots the plugin registered itself are always allowed. Fail-closed.
  79. allow_task_override: bool = False
  80. # The ``allow_*_override`` config keys; each is a same-named ``_TrustPolicy`` field.
  81. _OVERRIDE_FLAGS = ("allow_provider_override", "allow_model_override", "allow_agent_id_override",
  82. "allow_profile_override", "allow_task_override")
  83. def _coerce_allowlist(raw: Any) -> tuple[Optional[frozenset], bool]:
  84. """YAML list → ``(frozenset_or_None, allow_any)``. A ``"*"`` entry sets
  85. ``allow_any``; missing / non-list → ``(None, False)`` = no allowlist."""
  86. if not isinstance(raw, list):
  87. return None, False
  88. normalized = [item.strip().lower() for item in raw if isinstance(item, str)]
  89. return frozenset(item for item in normalized if item and item != "*"), "*" in normalized
  90. def _resolve_trust_policy(plugin_id: str) -> _TrustPolicy:
  91. """Read ``plugins.entries.<plugin_id>.llm`` from config.yaml (missing → fully
  92. restrictive). Resolved per call so config edits apply without a restart."""
  93. if not plugin_id:
  94. return _TrustPolicy(plugin_id="")
  95. try:
  96. from hermes_cli.config import load_config_readonly
  97. llm_cfg: Any = (load_config_readonly() or {}).get("plugins")
  98. except Exception: # pragma: no cover — config IO failure
  99. llm_cfg = None
  100. for key in ("entries", plugin_id, "llm"):
  101. llm_cfg = llm_cfg.get(key) if isinstance(llm_cfg, dict) else None
  102. if not isinstance(llm_cfg, dict):
  103. return _TrustPolicy(plugin_id=plugin_id)
  104. allowed_models, allow_any_model = _coerce_allowlist(llm_cfg.get("allowed_models"))
  105. allowed_providers, allow_any_provider = _coerce_allowlist(llm_cfg.get("allowed_providers"))
  106. return _TrustPolicy(
  107. plugin_id=plugin_id, allowed_providers=allowed_providers, allow_any_provider=allow_any_provider,
  108. allowed_models=allowed_models, allow_any_model=allow_any_model,
  109. **{name: bool(llm_cfg.get(name, False)) for name in _OVERRIDE_FLAGS},
  110. )
  111. class PluginLlmTrustError(PermissionError):
  112. """Raised when a plugin attempts an LLM override without trust."""
  113. def _denied(plugin_id: str, what: str, flag: str) -> PluginLlmTrustError:
  114. return PluginLlmTrustError(
  115. f"Plugin {plugin_id!r} cannot {what} (set plugins.entries.{plugin_id}.llm.{flag} to true to allow).")
  116. def _gate_ref_override(policy: _TrustPolicy, kind: str, requested: str) -> str:
  117. """Gate a ``provider`` / ``model`` override: trust flag, then optional allowlist."""
  118. if not getattr(policy, f"allow_{kind}_override"):
  119. raise _denied(policy.plugin_id, f"override the {kind}", f"allow_{kind}_override")
  120. allowed = getattr(policy, f"allowed_{kind}s")
  121. if not getattr(policy, f"allow_any_{kind}") and allowed is not None and requested.strip().lower() not in allowed:
  122. raise PluginLlmTrustError(f"Plugin {policy.plugin_id!r} {kind} override {requested!r} is not in "
  123. f"plugins.entries.{policy.plugin_id}.llm.allowed_{kind}s.")
  124. return requested.strip()
  125. # Overrides gated by a bare trust flag (no allowlist): ``kind`` -> denial wording.
  126. _FLAG_ONLY_OVERRIDES = {
  127. "agent_id": "run completions against a non-default agent id", "profile": "override the auth profile",
  128. }
  129. def _check_overrides(
  130. policy: _TrustPolicy, *, requested_provider: Optional[str], requested_model: Optional[str],
  131. requested_agent_id: Optional[str], requested_profile: Optional[str],
  132. ) -> tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
  133. """Gate each override independently, in the order provider, model, agent_id,
  134. profile. Returns ``(provider, model, agent_id, profile)`` (agent_id unstripped)."""
  135. final_provider = _gate_ref_override(policy, "provider", requested_provider) if requested_provider else None
  136. final_model = _gate_ref_override(policy, "model", requested_model) if requested_model else None
  137. for kind, requested in (("agent_id", requested_agent_id), ("profile", requested_profile)):
  138. if requested and not getattr(policy, f"allow_{kind}_override"):
  139. raise _denied(policy.plugin_id, _FLAG_ONLY_OVERRIDES[kind], f"allow_{kind}_override")
  140. return final_provider, final_model, requested_agent_id, requested_profile.strip() if requested_profile else None
  141. def _resolve_task_ownership(plugin_id: str) -> tuple[frozenset, frozenset]:
  142. """``(owned_keys, builtin_keys)`` for the task trust gate.
  143. Imports are lazy (circular import at plugin discovery); an unreadable registry
  144. yields empty sets, failing the gate closed. Ownership matches on the canonical id
  145. ``ctx.llm`` is bound to (``manifest.key or manifest.name``), which is what
  146. ``register_auxiliary_task`` stores as the entry's ``plugin``."""
  147. owned: set = set()
  148. builtin: set = set()
  149. try:
  150. from hermes_cli.plugins import get_plugin_auxiliary_tasks
  151. owned = {e.get("key") for e in get_plugin_auxiliary_tasks()
  152. if e.get("plugin") == plugin_id and isinstance(e.get("key"), str) and e.get("key")}
  153. except Exception: # pragma: no cover — registry unavailable
  154. pass
  155. try:
  156. from hermes_cli.main_provider_setup import _AUX_TASKS
  157. builtin = {k for k, _name, _desc in _AUX_TASKS}
  158. except Exception: # pragma: no cover — main import failure
  159. pass
  160. return frozenset(owned), frozenset(builtin)
  161. def _check_task(policy: _TrustPolicy, *, plugin_id: str, requested_task: Optional[str]) -> Optional[str]:
  162. """Validate a plugin's requested auxiliary ``task`` key.
  163. unset / ``""`` / ``"auto"`` → ``None`` (main-model path); a key the plugin
  164. registered itself → allowed; a built-in key → only with ``allow_task_override``;
  165. anything else raises + logs. Never silently downgraded to ``auto``: that would
  166. mask the misconfiguration and could route to a main model the user steered
  167. elsewhere on purpose.
  168. A foreign/unknown key raises :class:`PluginLlmTrustError` and logs a warning naming the offending plugin
  169. and key. See #64174, #64182.
  170. """
  171. task = (requested_task or "").strip()
  172. if not task or task.lower() == "auto":
  173. return None
  174. owned, builtin = _resolve_task_ownership(plugin_id)
  175. if task in owned or (task in builtin and policy.allow_task_override):
  176. return task
  177. if task in builtin:
  178. logger.warning("plugin_llm task routing denied: plugin %r requested built-in "
  179. "auxiliary task %r without plugins.entries.%s.llm.allow_task_override", plugin_id, task, plugin_id)
  180. raise _denied(plugin_id, f"route through the built-in auxiliary task {task!r}", "allow_task_override")
  181. logger.warning("plugin_llm task routing denied: plugin %r requested auxiliary task %r it did not register",
  182. plugin_id, task)
  183. raise PluginLlmTrustError(
  184. f"Plugin {plugin_id!r} cannot route through auxiliary task {task!r} — a "
  185. f"plugin may only pass a task key it registered itself via "
  186. f"ctx.register_auxiliary_task() (or a built-in key when plugins.entries."
  187. f"{plugin_id}.llm.allow_task_override is true)."
  188. )
  189. def _normalize_input_block(block: PluginLlmInput) -> dict[str, Any]:
  190. """Coerce a structured input block to a plain dict. Unknown shapes raise ``ValueError``."""
  191. if isinstance(block, PluginLlmTextInput):
  192. return {"type": "text", "text": block.text}
  193. if isinstance(block, PluginLlmImageInput):
  194. d: dict[str, Any] = {"type": "image", "mime_type": block.mime_type, "file_name": block.file_name}
  195. if block.data is not None:
  196. d["data"] = block.data
  197. if block.url:
  198. d["url"] = block.url
  199. return d
  200. if not isinstance(block, dict):
  201. raise ValueError(f"Unsupported input block: {type(block).__name__}")
  202. kind = block.get("type")
  203. if kind == "text":
  204. if not isinstance(block.get("text"), str):
  205. raise ValueError("text input block requires 'text' string")
  206. return {"type": "text", "text": block["text"]}
  207. if kind == "image":
  208. if "data" not in block and not block.get("url"):
  209. raise ValueError("image input block requires 'data' bytes or 'url'")
  210. return {"type": "image", "data": block.get("data"), "url": block.get("url"),
  211. "mime_type": block.get("mime_type") or "image/png", "file_name": block.get("file_name") or ""}
  212. raise ValueError(f"Unknown input block type: {kind!r}")
  213. def _image_part(norm: dict[str, Any]) -> dict[str, Any]:
  214. """Normalized image block → OpenAI ``image_url`` part (data: URL for bytes)."""
  215. url = norm.get("url")
  216. if not url:
  217. data = norm.get("data") or b""
  218. if not isinstance(data, (bytes, bytearray)):
  219. raise ValueError("image input 'data' must be bytes")
  220. url = f"data:{norm.get('mime_type') or 'image/png'};base64,{base64.b64encode(data).decode('ascii')}"
  221. return {"type": "image_url", "image_url": {"url": url}}
  222. def _build_structured_messages(
  223. *, instructions: str, inputs: Sequence[PluginLlmInput], json_mode: bool,
  224. json_schema: Optional[Any], schema_name: Optional[str], system_prompt: Optional[str],
  225. ) -> list[dict[str, Any]]:
  226. """OpenAI-style messages for a structured call: optional system message (prompt +
  227. JSON-only directive), then a user message whose first text part is the
  228. instructions (+ schema name / JSON schema) followed by the input blocks."""
  229. messages: list[dict[str, Any]] = []
  230. sys_parts: list[str] = [system_prompt.strip()] if system_prompt else []
  231. if json_mode or json_schema is not None:
  232. sys_parts.append("Respond with a single JSON object that matches the requested shape. "
  233. "Do not include prose or markdown fences.")
  234. if sys_parts:
  235. messages.append({"role": "system", "content": "\n\n".join(sys_parts)})
  236. header = instructions.strip()
  237. if schema_name:
  238. header = f"{header}\n\nSchema name: {schema_name}"
  239. if json_schema is not None:
  240. try:
  241. schema_text = json.dumps(json_schema, ensure_ascii=False, sort_keys=True)
  242. except (TypeError, ValueError):
  243. schema_text = str(json_schema)
  244. header = f"{header}\n\nJSON schema:\n{schema_text}"
  245. user_parts: list[dict[str, Any]] = [{"type": "text", "text": header}]
  246. for block in inputs:
  247. norm = _normalize_input_block(block) # always "text" or "image"
  248. user_parts.append({"type": "text", "text": norm["text"]} if norm["type"] == "text" else _image_part(norm))
  249. messages.append({"role": "user", "content": user_parts})
  250. return messages
  251. _FENCE_RE = re.compile(r"```(?:json)?\s*(.+?)```", re.DOTALL | re.IGNORECASE)
  252. def _strip_code_fences(text: str) -> str:
  253. """The first fenced code block's body, or the stripped text when unfenced."""
  254. match = _FENCE_RE.search(text)
  255. return match.group(1).strip() if match else text.strip()
  256. def _parse_structured_text(*, text: str, json_mode: bool, json_schema: Optional[Any]) -> tuple[Optional[Any], str]:
  257. """``(parsed, content_type)``: ``"json"`` when parsing (and schema validation, if
  258. given) succeeded, ``"text"`` otherwise. Schema violations raise ``ValueError``;
  259. a missing ``jsonschema`` package skips validation with a debug log."""
  260. if not (json_mode or json_schema is not None) or not text:
  261. return None, "text"
  262. try:
  263. parsed = json.loads(_strip_code_fences(text))
  264. except (json.JSONDecodeError, ValueError):
  265. return None, "text"
  266. if json_schema is not None:
  267. try:
  268. import jsonschema # type: ignore[import-untyped]
  269. jsonschema.validate(parsed, json_schema)
  270. except ImportError:
  271. logger.debug("jsonschema unavailable; skipping schema validation")
  272. except jsonschema.ValidationError as exc: # type: ignore[attr-defined]
  273. raise ValueError(f"Plugin LLM structured output did not match schema: {exc.message}") from exc
  274. return parsed, "json"
  275. def _extract_usage(response: Any) -> PluginLlmUsage:
  276. """Token usage from an OpenAI-shaped response, tolerating provider naming
  277. (``prompt_tokens``/``completion_tokens`` vs ``input_tokens``/``output_tokens``,
  278. ``cache_read_input_tokens`` vs ``cache_read_tokens``)."""
  279. raw = getattr(response, "usage", None)
  280. if raw is None:
  281. return PluginLlmUsage()
  282. def _g(*names: str) -> int:
  283. for name in names:
  284. v = getattr(raw, name, None)
  285. if v is None and isinstance(raw, dict):
  286. v = raw.get(name)
  287. try:
  288. if v is not None and int(v):
  289. return int(v)
  290. except (TypeError, ValueError):
  291. pass
  292. return 0
  293. inp, out = _g("prompt_tokens", "input_tokens"), _g("completion_tokens", "output_tokens")
  294. return PluginLlmUsage(
  295. input_tokens=inp, output_tokens=out, total_tokens=_g("total_tokens") or (inp + out),
  296. cache_read_tokens=_g("cache_read_input_tokens", "cache_read_tokens"),
  297. cache_write_tokens=_g("cache_creation_input_tokens", "cache_write_tokens"),
  298. )
  299. def _extract_text(response: Any) -> str:
  300. """Assistant text of an OpenAI-shaped response (string or text-part list content)."""
  301. try:
  302. content = getattr(response.choices[0].message, "content", None)
  303. if isinstance(content, str):
  304. return content
  305. if isinstance(content, list):
  306. texts = ((part.get("text") if part.get("type") == "text" else None)
  307. if isinstance(part, dict) else getattr(part, "text", None) for part in content)
  308. return "".join(t for t in texts if isinstance(t, str))
  309. except (AttributeError, IndexError, TypeError):
  310. pass
  311. return ""
  312. def _main_config_value(reader: str, default: str) -> str:
  313. """Read the current main provider/model via ``agent.auxiliary_client``."""
  314. try:
  315. import agent.auxiliary_client as ac
  316. return (getattr(ac, reader)() or "").strip() or default
  317. except Exception: # pragma: no cover — defensive
  318. return default
  319. def _resolve_attribution(*, provider_override: Optional[str], model_override: Optional[str], response: Any,
  320. route_info: Optional[dict[str, str]] = None) -> tuple[str, str]:
  321. """``(provider, model)`` to record on the result.
  322. Provider: route selected by ``auxiliary_client`` > explicit override > current
  323. main provider > ``"auto"``. Model: ``response.model`` (the canonical id that
  324. actually ran) > route > override > current main model > ``"default"``."""
  325. route_info = route_info or {}
  326. provider = route_info.get("provider") or provider_override or _main_config_value("_read_main_provider", "auto")
  327. response_model = getattr(response, "model", None)
  328. if isinstance(response_model, str) and response_model.strip():
  329. return provider, response_model.strip()
  330. return provider, route_info.get("model") or model_override or _main_config_value("_read_main_model", "default")
  331. def _json_response_format(*, json_mode: bool, json_schema: Optional[Any]) -> Optional[dict[str, Any]]:
  332. """``extra_body.response_format``; falls back to ``json_object`` without a
  333. schema so schema-blind providers still get a hint."""
  334. if json_schema is not None:
  335. schema = {"name": "plugin_structured_output", "schema": json_schema, "strict": False}
  336. return {"response_format": {"type": "json_schema", "json_schema": schema}}
  337. if json_mode:
  338. return {"response_format": {"type": "json_object"}}
  339. return None
  340. def _structured_spec(
  341. name: str, instructions: str, input: Sequence[PluginLlmInput], system_prompt: Optional[str],
  342. json_mode: bool, json_schema: Optional[Any], schema_name: Optional[str],
  343. ) -> dict[str, Any]:
  344. """Argument check for the structured methods (runs before the trust gate)."""
  345. if not instructions or not instructions.strip():
  346. raise ValueError(f"{name} requires non-empty instructions")
  347. if not input:
  348. raise ValueError(f"{name} requires at least one input block")
  349. return dict(instructions=instructions, inputs=list(input), system_prompt=system_prompt,
  350. json_mode=json_mode, json_schema=json_schema, schema_name=schema_name)
  351. class PluginLlm:
  352. """Host-owned LLM access for one trusted plugin.
  353. Constructed by :class:`hermes_cli.plugins.PluginContext` and exposed as
  354. ``ctx.llm``; the constructor binds plugin identity for trust enforcement, so
  355. plugins should not instantiate it directly. Every public method is ``_gate``
  356. (trust checks → call kwargs) → ``_invoke_*`` (host ``call_llm`` or injected
  357. caller) → ``_finish`` (result + audit log)."""
  358. def __init__(
  359. self, *, plugin_id: str, policy_loader: Optional[Callable[[str], _TrustPolicy]] = None,
  360. sync_caller: Optional[Callable[..., Any]] = None,
  361. async_caller: Optional[Callable[..., Awaitable[Any]]] = None,
  362. ) -> None:
  363. self._plugin_id = plugin_id
  364. self._policy_loader = policy_loader or _resolve_trust_policy
  365. self._sync_caller = sync_caller
  366. self._async_caller = async_caller
  367. def complete(
  368. self, messages: list[dict[str, Any]], *, provider: Optional[str] = None, model: Optional[str] = None,
  369. temperature: Optional[float] = None, max_tokens: Optional[int] = None,
  370. timeout: Optional[float] = None, agent_id: Optional[str] = None, profile: Optional[str] = None,
  371. purpose: Optional[str] = None, task: Optional[str] = None,
  372. ) -> PluginLlmCompleteResult:
  373. """Run a host-owned chat completion against the user's active model.
  374. ``provider``/``model``/``agent_id``/``profile`` are each gated by
  375. ``plugins.entries.<id>.llm.allow_*_override``. ``task`` routes through a
  376. plugin-registered auxiliary slot (see :func:`_check_task`)."""
  377. agent, kw = self._gate(provider, model, agent_id, profile, task, messages, temperature, max_tokens, timeout)
  378. return self._finish("complete", agent, kw, self._invoke_sync(kw), purpose)
  379. def complete_structured(
  380. self, *, instructions: str, input: Sequence[PluginLlmInput], json_schema: Optional[Any] = None,
  381. json_mode: bool = False, schema_name: Optional[str] = None, system_prompt: Optional[str] = None,
  382. provider: Optional[str] = None, model: Optional[str] = None, temperature: Optional[float] = None,
  383. max_tokens: Optional[int] = None, timeout: Optional[float] = None, agent_id: Optional[str] = None,
  384. profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None,
  385. ) -> PluginLlmStructuredResult:
  386. """Run a bounded host-owned structured completion.
  387. ``input`` accepts text and image blocks. With ``json_mode=True`` or a
  388. ``json_schema`` the response is parsed (and validated when the optional
  389. ``jsonschema`` package is installed) into ``result.parsed``."""
  390. spec = _structured_spec("complete_structured", instructions, input, system_prompt, json_mode, json_schema, schema_name)
  391. agent, kw = self._gate(provider, model, agent_id, profile, task, None, temperature, max_tokens, timeout, spec)
  392. return self._finish("complete_structured", agent, kw, self._invoke_sync(kw), purpose, spec)
  393. async def acomplete(
  394. self, messages: list[dict[str, Any]], *, provider: Optional[str] = None, model: Optional[str] = None,
  395. temperature: Optional[float] = None, max_tokens: Optional[int] = None,
  396. timeout: Optional[float] = None, agent_id: Optional[str] = None, profile: Optional[str] = None,
  397. purpose: Optional[str] = None, task: Optional[str] = None,
  398. ) -> PluginLlmCompleteResult:
  399. """Async sibling of :meth:`complete`."""
  400. agent, kw = self._gate(provider, model, agent_id, profile, task, messages, temperature, max_tokens, timeout)
  401. return self._finish("acomplete", agent, kw, await self._invoke_async(kw), purpose)
  402. async def acomplete_structured(
  403. self, *, instructions: str, input: Sequence[PluginLlmInput], json_schema: Optional[Any] = None,
  404. json_mode: bool = False, schema_name: Optional[str] = None, system_prompt: Optional[str] = None,
  405. provider: Optional[str] = None, model: Optional[str] = None, temperature: Optional[float] = None,
  406. max_tokens: Optional[int] = None, timeout: Optional[float] = None, agent_id: Optional[str] = None,
  407. profile: Optional[str] = None, purpose: Optional[str] = None, task: Optional[str] = None,
  408. ) -> PluginLlmStructuredResult:
  409. """Async sibling of :meth:`complete_structured`."""
  410. spec = _structured_spec("acomplete_structured", instructions, input, system_prompt, json_mode, json_schema, schema_name)
  411. agent, kw = self._gate(provider, model, agent_id, profile, task, None, temperature, max_tokens, timeout, spec)
  412. return self._finish("acomplete_structured", agent, kw, await self._invoke_async(kw), purpose, spec)
  413. def _gate(
  414. self, provider: Optional[str], model: Optional[str], agent_id: Optional[str], profile: Optional[str],
  415. task: Optional[str], messages: Optional[list[dict[str, Any]]], temperature: Optional[float],
  416. max_tokens: Optional[int], timeout: Optional[float], spec: Optional[dict[str, Any]] = None,
  417. ) -> tuple[Optional[str], dict[str, Any]]:
  418. """Trust gate (task first, then overrides), then — for a structured ``spec`` —
  419. build messages/response_format (input-shape errors surface only after trust
  420. passes). Returns the effective agent id and the call kwargs, in the documented
  421. order: messages, provider_override, model_override, profile_override,
  422. temperature, max_tokens, timeout, extra_body, task."""
  423. policy = self._policy_loader(self._plugin_id)
  424. eff_task = _check_task(policy, plugin_id=self._plugin_id, requested_task=task)
  425. eff_provider, eff_model, eff_agent, eff_profile = _check_overrides(
  426. policy, requested_provider=provider, requested_model=model, requested_agent_id=agent_id,
  427. requested_profile=profile)
  428. extra_body = None
  429. if spec is not None:
  430. messages = _build_structured_messages(**spec)
  431. extra_body = _json_response_format(json_mode=spec["json_mode"], json_schema=spec["json_schema"])
  432. return eff_agent, dict(messages=messages, provider_override=eff_provider, model_override=eff_model,
  433. profile_override=eff_profile, temperature=temperature, max_tokens=max_tokens,
  434. timeout=timeout, extra_body=extra_body, task=eff_task)
  435. def _finish(
  436. self, name: str, agent_id: Optional[str], kw: dict[str, Any], invoked: tuple[str, str, Any],
  437. purpose: Optional[str], spec: Optional[dict[str, Any]] = None,
  438. ) -> Any:
  439. """Build the result object + audit dict and emit the INFO audit line."""
  440. real_provider, real_model, response = invoked
  441. text = _extract_text(response)
  442. usage = _extract_usage(response)
  443. eff_task = kw["task"] or ""
  444. audit: dict[str, Any] = {"plugin_id": self._plugin_id, "purpose": purpose or "", "profile": kw["profile_override"] or ""}
  445. fields: dict[str, Any] = dict(text=text, provider=real_provider, model=real_model, agent_id=agent_id or "default", usage=usage)
  446. fmt = f"plugin_llm.{name} plugin=%s provider=%s model=%s task=%s purpose=%s "
  447. log_args = [self._plugin_id, real_provider, real_model, eff_task, purpose or ""]
  448. cls: Any = PluginLlmCompleteResult
  449. if spec is not None:
  450. parsed, content_type = _parse_structured_text(text=text, json_mode=spec["json_mode"], json_schema=spec["json_schema"])
  451. audit["schema_name"] = spec["schema_name"] or ""
  452. fields.update(parsed=parsed, content_type=content_type)
  453. fmt += "content_type=%s "
  454. log_args.append(content_type)
  455. cls = PluginLlmStructuredResult
  456. audit["task"] = eff_task
  457. logger.info(fmt + "tokens=%d", *log_args, usage.total_tokens)
  458. return cls(**fields, audit=audit)
  459. @staticmethod
  460. def _host_kwargs(kw: dict[str, Any]) -> tuple[dict[str, Any], Optional[dict[str, str]]]:
  461. """Call kwargs → ``call_llm`` kwargs. The auth profile rides in
  462. ``extra_body.metadata.auth_profile``; ``route_info`` is only requested when
  463. routing through a task slot."""
  464. merged_extra = dict(kw["extra_body"] or {})
  465. if kw["profile_override"]:
  466. merged_extra.setdefault("metadata", {})["auth_profile"] = kw["profile_override"]
  467. route_info: Optional[dict[str, str]] = {} if kw["task"] else None
  468. return dict(task=kw["task"], provider=kw["provider_override"], model=kw["model_override"],
  469. messages=kw["messages"], temperature=kw["temperature"], max_tokens=kw["max_tokens"],
  470. timeout=kw["timeout"], extra_body=merged_extra or None, route_info=route_info), route_info
  471. @staticmethod
  472. def _attributed(kw: dict[str, Any], response: Any, route_info: Optional[dict[str, str]]) -> tuple[str, str, Any]:
  473. return (*_resolve_attribution(provider_override=kw["provider_override"], model_override=kw["model_override"],
  474. response=response, route_info=route_info), response)
  475. def _invoke_sync(self, kw: dict[str, Any]) -> tuple[str, str, Any]:
  476. """Host ``call_llm`` (lazy import: circular deps at plugin discovery) →
  477. ``(provider, model, response)``. An injected ``sync_caller`` replaces the
  478. whole path and receives the call kwargs."""
  479. if self._sync_caller is not None:
  480. return self._sync_caller(**kw)
  481. from agent.auxiliary_client import call_llm
  482. call_kw, route_info = self._host_kwargs(kw)
  483. return self._attributed(kw, call_llm(**call_kw), route_info)
  484. async def _invoke_async(self, kw: dict[str, Any]) -> tuple[str, str, Any]:
  485. """Async sibling of :meth:`_invoke_sync` (``async_call_llm`` / ``async_caller``)."""
  486. if self._async_caller is not None:
  487. return await self._async_caller(**kw)
  488. from agent.auxiliary_client import async_call_llm
  489. call_kw, route_info = self._host_kwargs(kw)
  490. return self._attributed(kw, await async_call_llm(**call_kw), route_info)
  491. def make_plugin_llm_for_test(*, plugin_id: str, policy: _TrustPolicy, sync_caller: Optional[Callable[..., Any]] = None,
  492. async_caller: Optional[Callable[..., Awaitable[Any]]] = None) -> PluginLlm:
  493. """:class:`PluginLlm` with an injected policy and caller (no config.yaml, no
  494. provider). Not part of the public plugin API."""
  495. return PluginLlm(plugin_id=plugin_id, policy_loader=lambda _pid: policy, sync_caller=sync_caller, async_caller=async_caller)
  496. __all__ = [
  497. "PluginLlm",
  498. "PluginLlmCompleteResult",
  499. "PluginLlmImageInput",
  500. "PluginLlmInput",
  501. "PluginLlmStructuredResult",
  502. "PluginLlmTextInput",
  503. "PluginLlmTrustError",
  504. "PluginLlmUsage",
  505. "make_plugin_llm_for_test",
  506. ]