{"repo":"NousResearch/hermes-agent","public_id":"24170","display_label":"computer_use toolset bug fixes and corrections","summary":"A bug report identifying five issues in the computer_use toolset on macOS, along with multiple fix PRs addressing tool naming, drag action, app filter behavior, element label parsing, app context preservation, and cua-driver session reconnection.","subsystem":"mcp","counts":{"members":8,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":3,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":30032},"resolution":{"state":"resolved","by":"pr-30051"},"freshness":{"state":"verified","last_verified_at":"2026-05-22T00:15:36+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.25},"fix_edges":[{"issue_number":24170,"pr_number":30032,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":24170,"pr_number":30046,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":24170,"pr_number":30051,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":24170,"title":"[Bug] computer_use toolset: 5 bugs found during hands-on testing (macOS 26.4.1, cua-driver v0.1.6)","url":null,"state":"closed"},{"kind":"pr","number":24181,"title":"fix(computer_use): correct type_text MCP tool name and implement drag action","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#30032","number":30032}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":24242,"title":"fix(computer_use): preserve app context for capture_after; fix element label parsing (#24170 bugs 2 & 5)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#30046","number":30046},{"kind":"pr","value":"#24170","number":24170}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":24324,"title":"fix(computer-use): surface app=… filter no-match instead of silently using frontmost (#24170 bug 1)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#30051","number":30051},{"kind":"pr","value":"#24170","number":24170}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":30032,"title":"fix(computer_use): correct type_text MCP tool name and implement drag action (#24170 bugs 3 & 4)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_computer_use.py","tools/computer_use/cua_backend.py","tools/computer_use/tool.py"]},{"kind":"pr","number":30046,"title":"fix(computer_use): preserve app context for capture_after; fix element label parsing (#24170 bugs 2 & 5)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_computer_use.py","tools/computer_use/cua_backend.py","tools/computer_use/tool.py"]},{"kind":"pr","number":30051,"title":"fix(computer-use): surface app=… filter no-match instead of silently using frontmost (#24170 bug 1)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_computer_use.py","tools/computer_use/cua_backend.py"],"fix_hunks":[{"file":"tests/tools/test_computer_use.py","hunks":["@@ -925,3 +925,136 @@ def wait(self, seconds=1.0):\n         # No app context — should pass None so cua-driver picks the frontmost window\n         assert len(captured_app_args) == 1\n         assert captured_app_args[0] is None\n+\n+# ---------------------------------------------------------------------------\n+# Regression tests for bug 1 from issue #24170:\n+#   capture(app=...) and focus_app(app=...) must surface when the filter\n+#   matches nothing instead of silently picking the frontmost window.\n+# ---------------------------------------------------------------------------\n+\n+def _make_cua_backend_with_windows(windows: List[Dict[str, Any]]):\n+    \"\"\"Construct a CuaDriverBackend with a mocked MCP session that returns\n+    the supplied list_windows payload.\"\"\"\n+    from tools.computer_use.cua_backend import CuaDriverBackend\n+\n+    backend = CuaDriverBackend()\n+    backend._session = MagicMock()\n+    backend._session.call_tool.return_value = {\n+        \"data\": \"\",\n+        \"images\": [],\n+        \"structuredContent\": {\"windows\": windows},\n+        \"isError\": False,\n+    }\n+    return backend\n+\n+\n+class TestCaptureAppFilterNoMatch:\n+    \"\"\"capture(app=X) must not silently fall back to the frontmost window\n+    when X matches nothing — on a non-English macOS, list_windows returns\n+    localized app names (e.g. \"計算機\"), so an English `app=\"Calculator\"`\n+    legitimately matches nothing and the caller needs to retry with the\n+    localized name. The old code silently captured the frontmost window\n+    (e.g. a menu-bar utility), giving the agent wrong UI elements.\n+    \"\"\"\n+\n+    def test_app_filter_no_match_returns_empty_capture_with_diagnostic(self):\n+        # Simulates a localized macOS where Calculator's app_name is \"計算機\".\n+        windows = [\n+            {\"app_name\": \"Fuwari\", \"pid\": 100, \"window_id\": 1,\n+             \"is_on_screen\": True, \"title\": \"menu bar\", \"z_index\": 0},\n+            {\"app_name\": \"計算機\", \"pid\": 200, \"window_id\": 2,\n+             \"is_on_screen\": True, \"title\": \"Calculator\", \"z_index\": 1},\n+        ]\n+        backend = _make_cua_backend_with_windows(windows)\n+\n+        cap = backend.capture(mode=\"som\", app=\"Calculator\")\n+\n+        # No window matched; capture must NOT pick the frontmost (Fuwari).\n+        assert cap.app == \"\", (\n+            f\"app= filter no-match should not silently target a window; got {cap.app!r}\"\n+        )\n+        assert cap.elements == []\n+        assert \"Calculator\" in cap.window_title\n+        assert \"list_apps\" in cap.window_title\n+        # _active_pid must remain unset so a subsequent click doesn't hit Fuwari.\n+        assert backend._active_pid is None\n+        assert backend._active_window_id is None\n+\n+    def test_app_filter_match_still_works(self):\n+        windows = [\n+            {\"app_name\": \"Fuwari\", \"pid\": 100, \"window_id\": 1,\n+             \"is_on_screen\": True, \"title\": \"menu bar\", \"z_index\": 0},\n+            {\"app_name\": \"計算機\", \"pid\": 200, \"window_id\": 2,\n+             \"is_on_screen\": True, \"title\": \"Calculator\", \"z_index\": 1},\n+        ]\n+        backend = _make_cua_backend_with_windows(windows)\n+        # get_window_state for the matched window\n+        backend._session.call_tool.side_effect = [\n+            {\"data\": \"\", \"images\": [], \"isError\": False,\n+             \"structuredContent\": {\"windows\": windows}},\n+            {\"data\": '✅ 計算機 — 0 elements\\n', \"images\": [], \"isError\": False,\n+             \"structuredContent\": None},\n+        ]\n+\n+        cap = backend.capture(mode=\"ax\", app=\"計算機\")\n+\n+        assert backend._active_pid == 200\n+        assert backend._active_window_id == 2\n+\n+    def test_no_app_filter_still_picks_frontmost(self):\n+        \"\"\"When no app= is given, capture continues to pick the frontmost\n+        window — the no-match early-return must not fire on the empty case.\"\"\"\n+        windows = [\n+            {\"app_name\": \"Fuwari\", \"pid\": 100, \"window_id\": 1,\n+             \"is_on_screen\": True, \"title\": \"menu bar\", \"z_index\": 0},\n+        ]\n+        backend = _make_cua_backend_with_windows(windows)\n+        backend._session.call_tool.side_effect = [\n+            {\"data\": \"\", \"images\": [], \"isError\": False,\n+             \"structuredContent\": {\"windows\": windows}},\n+            {\"data\": '✅ Fuwari — 0 elements\\n', \"images\": [], \"isError\": False,\n+             \"structuredContent\": None},\n+        ]\n+\n+        cap = backend.capture(mode=\"ax\", app=None)\n+\n+        assert backend._active_pid == 100\n+\n+\n+class TestFocusAppFilterNoMatch:\n+    \"\"\"focus_app(app=X) must return ok=False when X matches nothing —\n+    not silently target the frontmost window and report ok=True with a\n+    misleading 'Targeted Fuwari' message.\n+    \"\"\"\n+\n+    def test_focus_app_no_match_returns_not_ok(self):\n+        windows = [\n+            {\"app_name\": \"Fuwari\", \"pid\": 100, \"window_id\": 1,\n+             \"is_on_screen\": True, \"title\": \"menu bar\", \"z_index\": 0},\n+            {\"app_name\": \"計算機\", \"pid\": 200, \"window_id\": 2,\n+             \"is_on_screen\": True, \"title\": \"Calculator\", \"z_index\": 1},\n+        ]\n+        backend = _make_cua_backend_with_windows(windows)\n+\n+        res = backend.focus_app(\"Calculator\")\n+\n+        assert res.ok is False\n+        assert res.action == \"focus_app\"\n+        assert \"Calculator\" in res.message\n+        # _active_pid must remain unset so a subsequent click doesn't hit Fuwari.\n+        assert backend._active_pid is None\n+\n+    def test_focus_app_match_still_works(self):\n+        windows = [\n+            {\"app_name\": \"Fuwari\", \"pid\": 100, \"window_id\": 1,\n+             \"is_on_screen\": True, \"title\": \"menu bar\", \"z_index\": 0},\n+            {\"app_name\": \"計算機\", \"pid\": 200, \"window_id\": 2,\n+             \"is_on_screen\": True, \"title\": \"Calculator\", \"z_index\": 1},\n+        ]\n+        backend = _make_cua_backend_with_windows(windows)\n+\n+        res = backend.focus_app(\"計算機\")\n+\n+        assert res.ok is True\n+        assert backend._active_pid == 200\n+        assert backend._active_window_id == 2"]},{"file":"tools/computer_use/cua_backend.py","hunks":["@@ -393,11 +393,27 @@ def capture(self, mode: str = \"som\", app: Optional[str] = None) -> CaptureResult\n                                  elements=[], app=\"\", window_title=\"\", png_bytes_len=0)\n \n         # Filter by app name (case-insensitive substring) if requested.\n+        # When the filter matches nothing, surface that explicitly instead of\n+        # silently capturing the frontmost window — on macOS the `app_name`\n+        # returned by list_windows is the localized name (e.g. \"計算機\"), so\n+        # `app=\"Calculator\"` legitimately matches no windows on a non-English\n+        # system and the caller needs to retry with the localized name.\n         if app:\n             app_lower = app.lower()\n             filtered = [w for w in windows if app_lower in w[\"app_name\"].lower()]\n-            if filtered:\n-                windows = filtered\n+            if not filtered:\n+                return CaptureResult(\n+                    mode=mode, width=0, height=0, png_b64=None,\n+                    elements=[], app=\"\",\n+                    window_title=(\n+                        f\"<no on-screen window matched app={app!r}; \"\n+                        f\"call list_apps to see available app names \"\n+                        f\"(macOS reports localized names, e.g. '計算機' \"\n+                        f\"instead of 'Calculator')>\"\n+                    ),\n+                    png_bytes_len=0,\n+                )\n+            windows = filtered\n \n         # Pick first on-screen window (sorted by z_index / z-order above).\n         target = next((w for w in windows if not w[\"off_screen\"]), windows[0])","@@ -658,7 +674,11 @@ def focus_app(self, app: str, raise_window: bool = False) -> ActionResult:\n \n         app_lower = app.lower()\n         matched = [w for w in windows if app_lower in w[\"app_name\"].lower()]\n-        target = matched[0] if matched else (windows[0] if windows else None)\n+        # Don't silently fall back to the frontmost window when the filter\n+        # matches nothing — that hides the real failure (often a localized\n+        # macOS app name mismatch, e.g. caller passed \"Calculator\" but\n+        # list_windows returns \"計算機\").\n+        target = matched[0] if matched else None\n         if target:\n             self._active_pid = target[\"pid\"]\n             self._active_window_id = target[\"window_id\"]\n"]}]},{"kind":"pr","number":30133,"title":"fix(computer_use): reconnect cua-driver stdio session between turns","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"365437e4aaf3756d1d921dad6012eb93d4303465","number":0},{"kind":"pr","value":"#40570","number":40570}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}