media_repair.py 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143
  1. """Repair model-mangled ``computer_use`` screenshot paths in final responses.
  2. Some models rewrite the Windows path ``computer_use`` reported into a POSIX-looking
  3. one (``C:\\Users\\Alice\\...`` -> ``/Users/Alice/...``) inside an explicit ``MEDIA:``
  4. directive, so delivery-path validation rejects it and drops the attachment.
  5. Deliberately narrow: only rewrites paths in a response that *already* carries a
  6. ``MEDIA:`` directive whose ``computer_use_<uuid>`` basename exactly matches a
  7. canonical path returned this turn. Never auto-attaches; validation still runs.
  8. """
  9. from __future__ import annotations
  10. import json
  11. import logging
  12. import re
  13. from typing import Any, Dict, Iterator, List
  14. logger = logging.getLogger(__name__)
  15. # Absolute-path prefix for canonical capture paths: Windows drive letter, POSIX
  16. # root, or UNC share. Shared string so the summary regex stays in sync.
  17. _ABS_PATH_PREFIX_PATTERN = r"(?:[A-Za-z]:[/\\]|/|\\\\)"
  18. _ABS_PATH_PREFIX_RE = re.compile(r"^" + _ABS_PATH_PREFIX_PATTERN)
  19. _CAPTURE_BASENAME_PATTERN = r"computer_use_[0-9a-f]{32}\.(?:png|jpe?g)"
  20. _COMPUTER_USE_CAPTURE_BASENAME_RE = re.compile(r"^" + _CAPTURE_BASENAME_PATTERN + r"$", re.IGNORECASE)
  21. _COMPUTER_USE_CAPTURE_SUMMARY_RE = re.compile(
  22. r"\(shareable screenshot saved to "
  23. r"(?P<path>" + _ABS_PATH_PREFIX_PATTERN + r"[^\r\n]*?" + _CAPTURE_BASENAME_PATTERN + r")\)",
  24. re.IGNORECASE,
  25. )
  26. def tool_name_by_call_id(messages: list[dict[str, Any]]) -> dict[str, str]:
  27. """Map assistant tool-call ids to tool names for the given messages."""
  28. mapping: dict[str, str] = {}
  29. for msg in messages:
  30. if msg.get("role") != "assistant":
  31. continue
  32. for call in msg.get("tool_calls") or []:
  33. call_id = call.get("id") or call.get("call_id")
  34. name = str((call.get("function") or {}).get("name") or call.get("name") or "")
  35. if call_id and name:
  36. mapping[str(call_id)] = name
  37. return mapping
  38. def _computer_use_capture_basename(path: Any) -> str:
  39. """Canonical (lowercased) capture basename for either separator style, or ''."""
  40. basename = re.split(r"[/\\]", str(path or "").strip().strip("`\"'"))[-1]
  41. return basename.lower() if _COMPUTER_USE_CAPTURE_BASENAME_RE.fullmatch(basename) else ""
  42. def _iter_computer_use_capture_paths(content: Any) -> Iterator[str]:
  43. """Yield persisted screenshot paths from computer_use result content (JSON, a
  44. multimodal list, or text; the latter two keep the canonical path in the summary
  45. line since the envelope's ``meta`` is not stored in the tool message)."""
  46. if isinstance(content, str):
  47. stripped = content.strip()
  48. if not stripped.startswith(("{", "[")):
  49. for match in _COMPUTER_USE_CAPTURE_SUMMARY_RE.finditer(content):
  50. yield match.group("path").strip()
  51. return
  52. # Parse JSON, never regex-scan it: JSON escaping doubles backslashes, so a raw-text
  53. # hit would yield a path that exists nowhere. Fail closed on truncated JSON.
  54. try:
  55. payload = json.loads(stripped)
  56. except Exception:
  57. return
  58. if isinstance(payload, (dict, list)):
  59. yield from _iter_computer_use_capture_paths(payload)
  60. elif isinstance(content, list):
  61. for part in content:
  62. yield from _iter_computer_use_capture_paths(part)
  63. elif isinstance(content, dict):
  64. meta = content.get("meta")
  65. for holder in (content, meta if isinstance(meta, dict) else {}):
  66. if isinstance(holder.get("screenshot_path"), str):
  67. yield holder["screenshot_path"]
  68. # Producer shapes (computer_use ``_capture_response``): content/text = multimodal
  69. # parts; text_summary/summary = the "(shareable screenshot saved to ...)" line.
  70. for field in ("content", "text", "text_summary", "summary"):
  71. nested = content.get(field)
  72. if isinstance(nested, (str, dict, list)):
  73. yield from _iter_computer_use_capture_paths(nested)
  74. def _current_turn_messages(messages: list[dict[str, Any]], history_offset: int) -> list[dict[str, Any]]:
  75. if not history_offset or len(messages) >= history_offset:
  76. return messages[history_offset:]
  77. # Compression can invalidate the slice boundary: recover the turn from its last user
  78. # message, fail closed if none remains. Narrower than run.py's scan-everything fallback
  79. # for auto-attach — that decides whether to ATTACH; this only rewrites emitted paths.
  80. for index in range(len(messages) - 1, -1, -1):
  81. if messages[index].get("role") == "user":
  82. return messages[index:]
  83. return []
  84. def repair_explicit_computer_use_media_paths(response: str, messages: list[dict[str, Any]], history_offset: int = 0) -> str:
  85. """Recover model-mangled paths in explicit ``MEDIA:`` directives whose basename
  86. matches (case-insensitively) a canonical screenshot path from this turn.
  87. Fail-open: the repair is cosmetic, so any error returns the response unchanged."""
  88. try:
  89. return _repair_explicit_computer_use_media_paths_inner(response, messages, history_offset)
  90. except Exception:
  91. logger.debug("computer_use media path repair failed", exc_info=True)
  92. return response
  93. def _canonical_capture_paths(turn_messages: list[dict[str, Any]]) -> dict[str, str]:
  94. """``{lowercase basename: absolute canonical path}`` from this turn's computer_use results."""
  95. call_id_names = tool_name_by_call_id(turn_messages)
  96. canonical: dict[str, str] = {}
  97. for msg in turn_messages:
  98. if msg.get("role") not in {"tool", "function"}:
  99. continue
  100. call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "")
  101. if str(msg.get("name") or msg.get("tool_name") or call_id_names.get(call_id) or "") != "computer_use":
  102. continue
  103. for path in _iter_computer_use_capture_paths(msg.get("content")):
  104. basename = _computer_use_capture_basename(path)
  105. if basename and _ABS_PATH_PREFIX_RE.match(path):
  106. canonical[basename] = path
  107. return canonical
  108. def _repair_explicit_computer_use_media_paths_inner(response: str, messages: list[dict[str, Any]], history_offset: int = 0) -> str:
  109. if "MEDIA:" not in response:
  110. return response
  111. canonical_by_basename = _canonical_capture_paths(_current_turn_messages(messages, history_offset))
  112. if not canonical_by_basename:
  113. return response
  114. # Lazy: keeps the import cheap for standalone cron processes that may never hit MEDIA:.
  115. from gateway.platforms.base import BasePlatformAdapter
  116. repaired = response
  117. for emitted_path, _is_voice in BasePlatformAdapter.extract_media(response)[0]:
  118. canonical = canonical_by_basename.get(_computer_use_capture_basename(emitted_path))
  119. if canonical and emitted_path != canonical:
  120. repaired = repaired.replace(emitted_path, canonical)
  121. return repaired