files.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325
  1. """File inputs and outputs of connector tools.
  2. The model passes a local path wherever a tool takes a file. Before execute, each path is checked
  3. with the media-delivery rule (the one that guards ``MEDIA:`` attachments), staged through the tool
  4. gateway, and PUT to the presigned storage URL; the gateway's descriptor replaces the path. Files a
  5. tool returns are downloaded at once into the profile's connector downloads folder, and the model
  6. sees ``{name, mimetype, path}``. Storage URLs and keys never reach the model or the logs.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import mimetypes
  11. import os
  12. import re
  13. import tempfile
  14. from dataclasses import replace as dataclass_replace
  15. from datetime import UTC, datetime
  16. from pathlib import Path
  17. from typing import Any, Iterator, Mapping, Optional, Sequence
  18. from tools.connectors.gateway.errors import ToolGatewayError
  19. from tools.connectors.gateway.merge import PlannedCall
  20. from tools.connectors.gateway.names import parse_connector_name, vendor_slug_candidates
  21. from tools.connectors.gateway.wire import CONNECTOR_FILE_MAX_BYTES, execute_error_family
  22. logger = logging.getLogger(__name__)
  23. __all__ = [
  24. "DOWNLOAD_MAX_BYTES",
  25. "FileInputError",
  26. "model_parameters",
  27. "save_outputs",
  28. "stage_inputs",
  29. ]
  30. DOWNLOAD_MAX_BYTES = 50 * 1024 * 1024
  31. FILE_INPUT_NOTE = "Pass a local file path."
  32. # The argument-path segment that stands for every element of an array.
  33. _ARRAY_ITEM = "*"
  34. _MAX_SCHEMA_DEPTH = 32
  35. # Terminal backends whose files this machine can read, as media delivery reads them: the host, and
  36. # Docker through its mounts. The others keep the model's files on another machine.
  37. _HOST_READABLE_BACKENDS = frozenset({"local", "docker"})
  38. _UNSAFE_NAME_CHARS = re.compile(r"[\x00-\x1f\x7f<>:\"|?*\\/]")
  39. _MAX_NAME_CHARS = 150
  40. Path_ = tuple[str, ...]
  41. # Composed tool name -> argument paths that are file inputs; an empty tuple means "none".
  42. _FILE_INPUTS: dict[str, tuple[Path_, ...]] = {}
  43. class _Staged(dict):
  44. """A descriptor this dispatch staged; a second file-input path over the same value leaves it alone."""
  45. class FileInputError(Exception):
  46. """A file input refused before the call reached execute; nothing ran."""
  47. def __init__(self, code: str, message: str, *, hint: Optional[str] = None) -> None:
  48. super().__init__(message)
  49. self.code = code
  50. self.hint = hint
  51. def result(self) -> dict[str, Any]:
  52. """The call's error entry, in the shape the gateway's execute errors take."""
  53. error: dict[str, Any] = {"code": self.code, "message": str(self), "family": execute_error_family(self.code)}
  54. if self.hint:
  55. error["hint"] = self.hint
  56. return {"data": None, "error": error}
  57. # ---------------------------------------------------------------------------
  58. # Schemas: what the model sees, and where the file inputs are
  59. # ---------------------------------------------------------------------------
  60. def model_parameters(name: str, input_schema: Any) -> dict[str, Any]:
  61. """The tool's parameters as the model sees them (file inputs become path strings); records the
  62. file-input argument paths for ``name`` so execute knows what to stage."""
  63. if not isinstance(input_schema, dict):
  64. return {}
  65. _FILE_INPUTS[name] = tuple(dict.fromkeys(_file_paths(input_schema, input_schema, (), 0)))
  66. return _as_model_schema(input_schema)
  67. def _as_model_schema(node: Any) -> Any:
  68. if isinstance(node, list):
  69. return [_as_model_schema(item) for item in node]
  70. if not isinstance(node, dict):
  71. return node
  72. if node.get("file_uploadable") is True:
  73. description = f"{node.get('description') or ''} {FILE_INPUT_NOTE}".strip()
  74. return {"type": "string", "description": description}
  75. return {key: _as_model_schema(value) for key, value in node.items()}
  76. def _file_paths(node: Any, root: dict[str, Any], path: Path_, depth: int) -> Iterator[Path_]:
  77. """Argument paths of every ``file_uploadable`` subschema: properties, items, combinators, local refs."""
  78. from tools.schema_sanitizer import resolve_local_schema_ref
  79. if not isinstance(node, dict) or depth > _MAX_SCHEMA_DEPTH:
  80. return
  81. node = resolve_local_schema_ref(node, root)
  82. if node.get("file_uploadable") is True:
  83. yield path
  84. return
  85. for branch in ("anyOf", "oneOf", "allOf"):
  86. for option in node.get(branch) or ():
  87. yield from _file_paths(option, root, path, depth + 1)
  88. properties = node.get("properties")
  89. if isinstance(properties, dict):
  90. for key, child in properties.items():
  91. yield from _file_paths(child, root, (*path, key), depth + 1)
  92. yield from _file_paths(node.get("items"), root, (*path, _ARRAY_ITEM), depth + 1)
  93. def _known_file_inputs(client: Any, plan: PlannedCall) -> tuple[Path_, ...]:
  94. """The plan's file-input paths: recorded by search or describe, else fetched once."""
  95. if plan.name in _FILE_INPUTS:
  96. return _FILE_INPUTS[plan.name]
  97. parsed = parse_connector_name(plan.name)
  98. if parsed is None:
  99. return ()
  100. candidates = vendor_slug_candidates(parsed.connector, parsed.tool)
  101. try:
  102. schemas = (client.schemas(list(candidates)) or {}).get("schemas") or {}
  103. except ToolGatewayError as exc:
  104. logger.warning("connector %s: schema fetch for file inputs failed (%s); executing without staging",
  105. plan.name, exc.code)
  106. return ()
  107. schema = next((schemas[slug] for slug in candidates if isinstance(schemas.get(slug), dict)), {})
  108. model_parameters(plan.name, schema.get("input_schema") or {})
  109. return _FILE_INPUTS[plan.name]
  110. # ---------------------------------------------------------------------------
  111. # Execute: stage every path at a file input
  112. # ---------------------------------------------------------------------------
  113. def stage_inputs(client: Any, plan: PlannedCall, *, task_id: Optional[str]) -> PlannedCall:
  114. """``plan`` with each file-input path replaced by a staged descriptor for ``plan.tool``.
  115. Raises ``FileInputError`` when a path is refused or a stage fails; nothing is executed then."""
  116. if not _has_text(plan.arguments):
  117. return plan
  118. paths = _known_file_inputs(client, plan)
  119. if not paths:
  120. return plan
  121. arguments = plan.arguments
  122. for path in paths:
  123. arguments = _substitute(arguments, path, lambda raw: _stage_one(client, plan, raw, task_id))
  124. return dataclass_replace(plan, arguments=arguments)
  125. def _has_text(value: Any) -> bool:
  126. """A file input needs a string; a call with none has nothing to stage."""
  127. if isinstance(value, str):
  128. return True
  129. if isinstance(value, Mapping):
  130. return any(_has_text(item) for item in value.values())
  131. if isinstance(value, (list, tuple)):
  132. return any(_has_text(item) for item in value)
  133. return False
  134. def _substitute(value: Any, path: Sequence[str], stage) -> Any:
  135. """A copy of ``value`` with the leaf at ``path`` staged; missing or non-path leaves stay as they are."""
  136. if not path:
  137. if isinstance(value, str):
  138. return stage(value)
  139. if isinstance(value, Mapping) and not isinstance(value, _Staged) and "s3key" in value:
  140. raise FileInputError("FILE_NOT_ALLOWED", "File inputs take a local file path, not a storage descriptor.",
  141. hint="Pass the path of the local file to send.")
  142. return value
  143. head, rest = path[0], path[1:]
  144. if head == _ARRAY_ITEM:
  145. return [_substitute(item, rest, stage) for item in value] if isinstance(value, list) else value
  146. if isinstance(value, Mapping) and head in value:
  147. return {**value, head: _substitute(value[head], rest, stage)}
  148. return value
  149. def _stage_one(client: Any, plan: PlannedCall, raw: str, task_id: Optional[str]) -> dict[str, str]:
  150. path = _readable_file(raw, task_id)
  151. data = _read_bytes(raw, path)
  152. name = Path(path).name[-255:]
  153. mimetype = mimetypes.guess_type(name)[0] or "application/octet-stream"
  154. try:
  155. staged = client.stage_file(toolkit=plan.connector.lower(), tool=plan.tool, filename=name,
  156. mimetype=mimetype, size=len(data))
  157. except ToolGatewayError as exc:
  158. if exc.code == "TOOL_NOT_ALLOWED":
  159. raise FileInputError("TOOL_NOT_ALLOWED", str(exc)) from exc
  160. raise FileInputError("FILE_UPLOAD_FAILED", f"The tool gateway could not stage {name}: {exc}") from exc
  161. try:
  162. client.put_file(staged["upload_url"], staged["headers"], data)
  163. except ToolGatewayError as exc:
  164. raise FileInputError("FILE_UPLOAD_FAILED", f"Uploading {name} to storage failed: {exc}") from exc
  165. return _Staged(staged["descriptor"])
  166. def _readable_file(raw: str, task_id: Optional[str]) -> str:
  167. """The host path of a file the model may send, by the media-delivery rule; else ``FileInputError``."""
  168. from gateway.platforms.base import validate_media_delivery_path
  169. from gateway.session_context import get_session_env
  170. candidate = _backend_path(raw, task_id)
  171. allowed = validate_media_delivery_path(candidate, session_key=get_session_env("HERMES_SESSION_KEY", ""))
  172. if allowed:
  173. return allowed
  174. if not Path(candidate).is_file():
  175. raise FileInputError("FILE_NOT_FOUND", f"{raw} is not a regular file on this machine.",
  176. hint="Pass the path of an existing file.")
  177. raise FileInputError(
  178. "FILE_NOT_ALLOWED", f"{raw} may not leave this machine: the file-sending policy refuses it.",
  179. hint="Files under the Hermes downloads and cache folders, or a folder listed in "
  180. "gateway.media_delivery_allow_dirs, can be sent.")
  181. def _backend_path(raw: str, task_id: Optional[str]) -> str:
  182. """``raw`` as an absolute path in the terminal backend's namespace, resolved the way the file
  183. tools resolve it. Backends whose files live on another machine are refused, as ``MEDIA:`` is."""
  184. from tools.file_tools_paths import _resolve_path_for_task, _terminal_env_type_for_task
  185. task = task_id or "default"
  186. backend = _terminal_env_type_for_task(task)
  187. if backend not in _HOST_READABLE_BACKENDS:
  188. raise FileInputError(
  189. "FILE_NOT_ALLOWED", f"Files in the {backend} terminal backend are not on this machine, "
  190. "so they cannot be sent as a connector file input.",
  191. hint="Use a file on this machine, or switch to the local or Docker terminal backend.")
  192. return str(_resolve_path_for_task(raw, task))
  193. def _read_bytes(raw: str, path: str) -> bytes:
  194. try:
  195. size = os.path.getsize(path)
  196. if size > CONNECTOR_FILE_MAX_BYTES:
  197. raise FileInputError("FILE_TOO_LARGE",
  198. f"{raw} is {size // (1024 * 1024)} MiB; connector file inputs are limited to "
  199. f"{CONNECTOR_FILE_MAX_BYTES // (1024 * 1024)} MiB.")
  200. data = Path(path).read_bytes()
  201. except OSError as exc:
  202. raise FileInputError("FILE_NOT_FOUND", f"{raw} could not be read: {exc.strerror or exc}.") from exc
  203. if not data:
  204. raise FileInputError("FILE_NOT_FOUND", f"{raw} is empty.")
  205. return data
  206. # ---------------------------------------------------------------------------
  207. # Results: save every returned file
  208. # ---------------------------------------------------------------------------
  209. def save_outputs(client: Any, result: Any) -> Any:
  210. """A successful result with each ``{name, mimetype, url}`` replaced by ``{name, mimetype, path}``,
  211. or ``{name, mimetype, error}`` when the download failed. Errors and replays pass through."""
  212. if not isinstance(result, Mapping) or result.get("error") is not None or result.get("data") is None:
  213. return result
  214. return {**result, "data": _saved(client, result["data"])}
  215. def _saved(client: Any, value: Any) -> Any:
  216. if isinstance(value, list):
  217. return [_saved(client, item) for item in value]
  218. if not isinstance(value, Mapping):
  219. return value
  220. if _is_file_output(value):
  221. return _download(client, value)
  222. return {key: _saved(client, item) for key, item in value.items()}
  223. def _is_file_output(value: Mapping[str, Any]) -> bool:
  224. return all(isinstance(value.get(key), str) for key in ("name", "mimetype", "url")) and \
  225. value["url"].startswith(("https://", "http://"))
  226. def _download(client: Any, file: Mapping[str, Any]) -> dict[str, Any]:
  227. kept = {key: item for key, item in file.items() if key not in ("url", "expiresAt")}
  228. try:
  229. kept["path"] = _fetch_to_downloads(client, file["url"], file["name"])
  230. except (ToolGatewayError, OSError) as exc:
  231. logger.warning("connector file download of %s failed: %s", _safe_name(file["name"]), type(exc).__name__)
  232. kept["error"] = f"download failed: {exc}"
  233. return kept
  234. def _fetch_to_downloads(client: Any, url: str, name: str) -> str:
  235. """Stream ``url`` into a temp file, then link it under a fresh name: a cut download never leaves
  236. a partial file under the final name, and an existing file is never replaced."""
  237. from hermes_constants import CONNECTOR_DOWNLOADS_SUBDIR, get_hermes_home, mkdir_under_hermes_home
  238. folder = mkdir_under_hermes_home(get_hermes_home() / CONNECTOR_DOWNLOADS_SUBDIR)
  239. handle = tempfile.NamedTemporaryFile(dir=folder, prefix=".partial-", delete=False)
  240. try:
  241. with handle:
  242. client.get_file(url, handle.write, max_bytes=DOWNLOAD_MAX_BYTES)
  243. target = _reserve_name(folder, _safe_name(name))
  244. os.replace(handle.name, target)
  245. return str(target)
  246. finally:
  247. Path(handle.name).unlink(missing_ok=True)
  248. def _reserve_name(folder: Path, name: str) -> Path:
  249. """Create an empty file under a fresh ``<UTC stamp>-<name>``; the download then replaces it."""
  250. stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
  251. stem, suffix = os.path.splitext(name)
  252. for attempt in range(1000):
  253. target = folder / (f"{stamp}-{name}" if attempt == 0 else f"{stamp}-{stem}-{attempt}{suffix}")
  254. try:
  255. target.open("xb").close()
  256. except FileExistsError:
  257. continue
  258. return target
  259. raise FileExistsError(f"no free name for {name}")
  260. def _safe_name(name: str) -> str:
  261. """A basename safe on every OS: no separators, control characters, or ``.``/``..``."""
  262. base = re.split(r"[\\/]", name)[-1]
  263. cleaned = _UNSAFE_NAME_CHARS.sub("_", base).strip().strip(".")
  264. return cleaned[-_MAX_NAME_CHARS:] or "file"