workspace_media.py 2.2 KB

123456789101112131415161718192021222324252627282930313233343536
  1. """Carry trusted VM screenshots through the kernel's native vision pipeline."""
  2. import base64
  3. import json
  4. def computer_result(name, value):
  5. if not name.startswith('computer_') or not isinstance(value, dict) or value.get('isError'):
  6. return json.dumps(value, ensure_ascii=False)
  7. content = value.get('content')
  8. images = [part for part in content if isinstance(part, dict) and part.get('type') == 'image'] if isinstance(content, list) else []
  9. if not images:
  10. return json.dumps(value, ensure_ascii=False)
  11. # The shared Box bridge produces exactly one PNG per capture. Invalid media
  12. # must be an explicit error, never a huge base64 text result/spillover file.
  13. try:
  14. if len(images) != 1 or images[0].get('mimeType') != 'image/png':
  15. raise ValueError('unsupported screenshot envelope')
  16. data = images[0].get('data')
  17. if not isinstance(data, str) or len(data) > 32 * 1024 * 1024:
  18. raise ValueError('invalid screenshot size')
  19. raw = base64.b64decode(data, validate=True)
  20. if not raw.startswith(b'\x89PNG\r\n\x1a\n'):
  21. raise ValueError('invalid screenshot bytes')
  22. from agent.image_eviction_policy import image_dimensions_from_bytes
  23. from tools.computer_use.backend import CaptureResult
  24. from tools.computer_use.tool import _capture_response
  25. dimensions = image_dimensions_from_bytes(raw)
  26. if not dimensions or not all(0 < n <= 16384 for n in dimensions):
  27. raise ValueError('invalid screenshot dimensions')
  28. except (ValueError, TypeError):
  29. return json.dumps({'isError': True, 'error': 'VM screenshot payload is invalid; no other computer was used.'})
  30. note = '\n'.join(part['text'] for part in content if isinstance(part, dict) and part.get('type') == 'text' and isinstance(part.get('text'), str))
  31. # Reuse Hermes native/aux-vision routing, provider limits and private image
  32. # cache. This only renders the returned pixels; it never captures the Host.
  33. return _capture_response(CaptureResult(mode='vision', width=dimensions[0], height=dimensions[1],
  34. png_b64=data, png_bytes_len=len(raw), image_mime_type='image/png', note=note))