{"repo":"NousResearch/hermes-agent","public_id":"39242","display_label":"Computer-use vision capture API fix","summary":"A bug was identified where computer_use vision capture called a nonexistent 'screenshot' tool, returning a 0x0 result. Multiple fixes replace this with get_window_state-based capture to align with the cua-driver API.","subsystem":"agent","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":51078},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-22T23:37:19+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":39242,"pr_number":51078,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":39242,"title":"computer_use(capture, mode='vision') calls non-existent cua-driver 'screenshot' tool and returns 0x0 result","url":null,"state":"closed"},{"kind":"pr","number":39262,"title":"fix(computer_use): use get_window_state for CUA vision capture (#39242)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"f2e37549c","number":0}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":39342,"title":"fix(computer_use): use get_window_state for vision/som captures (#39242)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"760fd9513e7d7350a672c76369ba632ca1ed1448","number":0},{"kind":"pr","value":"#51078","number":51078}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":40111,"title":"fix(computer-use): capture via window state","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#39342","number":39342}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":51078,"title":"fix(computer-use): vision capture returns an image on cua-driver >=0.5.x","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tools/computer_use/cua_backend.py"],"fix_hunks":[{"file":"tools/computer_use/cua_backend.py","hunks":["@@ -723,6 +723,28 @@ def supports_capability(self, capability: str, tool: Optional[str] = None) -> bo\n             return capability in self._capabilities.get(tool, set())\n         return any(capability in caps for caps in self._capabilities.values())\n \n+    def _has_tool(self, name: str) -> bool:\n+        \"\"\"Return True when ``tools/list`` advertised a tool by this name.\n+\n+        Used to route capture(): cua-driver dropped the standalone\n+        ``screenshot`` tool and folded full-window PNG capture into\n+        ``get_window_state`` (whose own description notes it \"Also captures\n+        a PNG screenshot of the specified window\"). Older drivers that still\n+        expose ``screenshot`` keep using it; newer ones fall through to\n+        ``get_window_state``.\n+\n+        Returns False when discovery hasn't populated the map yet — callers\n+        treat that as \"unknown\" and probe defensively rather than trusting it.\n+        \"\"\"\n+        return name in self._capabilities\n+\n+    @property\n+    def capabilities_discovered(self) -> bool:\n+        \"\"\"True once ``tools/list`` populated the per-tool map. When False,\n+        ``_has_tool`` answers are not trustworthy (discovery failed or the\n+        session hasn't started) and capture() should probe defensively.\"\"\"\n+        return bool(self._capabilities)\n+\n     @property\n     def capability_version(self) -> str:\n         \"\"\"Driver-advertised capability vocabulary version (empty string","@@ -825,6 +847,45 @@ def _extract_tool_result(mcp_result: Any) -> Dict[str, Any]:\n     }\n \n \n+def _image_from_tool_result(out: Dict[str, Any]) -> tuple[Optional[str], Optional[str]]:\n+    \"\"\"Pull a (png_b64, mime_type) pair out of a flattened tool result.\n+\n+    cua-driver delivers window screenshots in two shapes depending on tool +\n+    transport:\n+\n+      * As an MCP ``image`` content part — surfaced by ``_extract_tool_result``\n+        in ``out[\"images\"]`` with a parallel ``image_mime_types`` entry. This\n+        is what ``get_window_state`` emits over the stdio MCP transport.\n+      * As a base64 field inside ``structuredContent`` —\n+        ``screenshot_png_b64`` (+ ``screenshot_mime_type``). This is what\n+        ``get_window_state`` returns when its structured payload carries the\n+        image instead of a content part (newer driver builds; also the shape\n+        seen via the ``cua-driver call`` CLI surface).\n+\n+    Checking both makes capture() robust to either delivery shape, so the\n+    image never silently drops just because the driver moved it between the\n+    content list and structuredContent. Returns ``(None, None)`` when neither\n+    location carries an image.\n+    \"\"\"\n+    images = out.get(\"images\") or []\n+    if images and images[0]:\n+        mimes = out.get(\"image_mime_types\") or []\n+        mime = mimes[0] if mimes and mimes[0] else None\n+        return images[0], mime\n+\n+    structured = out.get(\"structuredContent\") or {}\n+    b64 = structured.get(\"screenshot_png_b64\") or structured.get(\"png_b64\")\n+    if b64:\n+        mime = (\n+            structured.get(\"screenshot_mime_type\")\n+            or structured.get(\"mime_type\")\n+            or None\n+        )\n+        return b64, mime\n+\n+    return None, None\n+\n+\n # ---------------------------------------------------------------------------\n # The backend itself\n # ---------------------------------------------------------------------------","@@ -1003,25 +1064,61 @@ def capture(self, mode: str = \"som\", app: Optional[str] = None) -> CaptureResult\n         window_title = \"\"\n \n         if mode == \"vision\":\n-            # screenshot tool: just the PNG, no AX walk.\n-            sc_out = self._session.call_tool(\n-                \"screenshot\",\n-                {\n-                    \"window_id\": self._active_window_id,\n-                    \"format\": \"jpeg\",\n-                    \"quality\": 85,\n-                    \"session\": self._session_id,\n-                },\n+            # Plain screenshot, no AX walk. cua-driver dropped the standalone\n+            # `screenshot` tool (≥0.5.x) and folded full-window PNG capture\n+            # into `get_window_state`. Route accordingly:\n+            #   * Driver advertises `screenshot` (older builds) → use it; it's\n+            #     the cheapest path (no AX tree walked server-side).\n+            #   * Otherwise (current drivers) → call `get_window_state` but\n+            #     DISCARD the AX tree/elements, returning only the PNG. Vision\n+            #     mode's whole contract is \"just the pixels, no element noise\",\n+            #     so we drop everything but the image.\n+            # When capability discovery hasn't run (empty map), we don't trust\n+            # a negative `_has_tool` answer — we still try `screenshot` first\n+            # and fall back if the driver rejects it, so the path self-heals on\n+            # any driver version.\n+            use_screenshot = (\n+                self._session._has_tool(\"screenshot\")\n+                or not self._session.capabilities_discovered\n             )\n-            if sc_out[\"images\"]:\n-                png_b64 = sc_out[\"images\"][0]\n-                # Pick up the explicit mimeType cua-driver attaches to image\n-                # parts (Surface 7). Empty string means the driver didn't\n-                # carry one — callers will fall back to magic-byte sniffing.\n-                mimes = sc_out.get(\"image_mime_types\") or []\n-                image_mime_type = mimes[0] if mimes and mimes[0] else None\n+            sc_out: Optional[Dict[str, Any]] = None\n+            if use_screenshot:\n+                sc_out = self._session.call_tool(\n+                    \"screenshot\",\n+                    {\n+                        \"window_id\": self._active_window_id,\n+                        \"format\": \"jpeg\",\n+                        \"quality\": 85,\n+                        \"session\": self._session_id,\n+                    },\n+                )\n+                png_b64, image_mime_type = _image_from_tool_result(sc_out)\n+                if not png_b64:\n+                    # Driver had no usable `screenshot` (e.g. \"Unknown tool:\n+                    # screenshot\" on ≥0.5.x, or an empty image part). Fall\n+                    # through to the get_window_state path below.\n+                    sc_out = None\n+\n+            if sc_out is None:\n+                gws_out = self._session.call_tool(\n+                    \"get_window_state\",\n+                    {\n+                        \"pid\": self._active_pid,\n+                        \"window_id\": self._active_window_id,\n+                        \"session\": self._session_id,\n+                    },\n+                )\n+                png_b64, image_mime_type = _image_from_tool_result(gws_out)\n+                # Still grab the window title — it's cheap and useful in the\n+                # vision response — but deliberately leave `elements` empty so\n+                # vision stays free of AX-tree noise.\n+                text = gws_out[\"data\"] if isinstance(gws_out[\"data\"], str) else \"\"\n+                _, tree = _split_tree_text(text)\n+                wt = re.search(r'AXWindow\\s+\"([^\"]+)\"', tree)\n+                if wt:\n+                    window_title = wt.group(1)\n         else:\n-            # get_window_state: AX tree + optional screenshot.\n+            # get_window_state: AX tree + screenshot.\n             gws_out = self._session.call_tool(\n                 \"get_window_state\",\n                 {","@@ -1058,10 +1155,10 @@ def capture(self, mode: str = \"som\", app: Optional[str] = None) -> CaptureResult\n                 if e.element_token\n             }\n \n-            if gws_out[\"images\"]:\n-                png_b64 = gws_out[\"images\"][0]\n-                mimes = gws_out.get(\"image_mime_types\") or []\n-                image_mime_type = mimes[0] if mimes and mimes[0] else None\n+            # Image may arrive as an MCP image part or inside\n+            # structuredContent (screenshot_png_b64) depending on the driver\n+            # build — _image_from_tool_result handles both.\n+            png_b64, image_mime_type = _image_from_tool_result(gws_out)\n \n             # Extract window title from the AX tree first AXWindow line.\n             wt = re.search(r'AXWindow\\s+\"([^\"]+)\"', tree)\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}