{"repo":"NousResearch/hermes-agent","public_id":"22865","display_label":"Computer-use AX tree context size and routing fixes","summary":"These items address two computer-use issues: capping the accessibility element array to prevent context overflow on large AX trees (e.g. Electron/Obsidian), and routing SOM/vision captures through the auxiliary vision path.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":2,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":30145,"merged_at":"2026-05-22T02:07:33+00:00"},"resolution":{"state":"resolved","by":"pr-30145"},"freshness":{"state":"verified","last_verified_at":"2026-05-22T02:07:33+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":22865,"pr_number":30126,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-05-22T00:38:20+00:00"},{"issue_number":22865,"pr_number":30145,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-22T02:07:33+00:00"},{"issue_number":24015,"pr_number":30126,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-22T00:38:20+00:00"},{"issue_number":29407,"pr_number":30126,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-22T00:38:20+00:00"}],"members":[{"kind":"issue","number":22865,"title":"computer_use capture can blow context on Electron/Obsidian AX trees","url":null,"state":"closed"},{"kind":"pr","number":22891,"title":"fix(computer-use): cap AX `elements` array to prevent context blowup (#22865)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#30145","number":30145},{"kind":"pr","value":"#30126","number":30126},{"kind":"pr","value":"#22865","number":22865}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":30126,"title":"fix(computer_use): route SOM/vision captures via auxiliary.vision (#24015)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_computer_use_capture_routing.py","tests/tools/test_computer_use_vision_routing.py","tools/computer_use/tool.py","tools/computer_use/vision_routing.py"],"fix_hunks":[{"file":"tests/tools/test_computer_use_capture_routing.py","hunks":["@@ -0,0 +1,431 @@\n+\"\"\"End-to-end regression for #24015 — capture routing via auxiliary.vision.\n+\n+When ``computer_use(action='capture', mode='som'|'vision')`` returns a\n+screenshot, ``_capture_response`` previously always returned a\n+``_multimodal`` envelope. For non-vision main models, or when the user\n+explicitly configured ``auxiliary.vision`` in ``config.yaml``, that\n+envelope tripped HTTP 404 / 400 at the provider boundary even though a\n+perfectly good vision backend was sitting in config waiting to be used.\n+\n+This file exercises the integrated ``_capture_response`` flow with\n+deterministic stubs for:\n+\n+* ``should_route_capture_to_aux_vision`` (the policy decision)\n+* ``_run_async`` (sync->async bridge)\n+* ``vision_analyze_tool`` (the aux LLM call)\n+* ``hermes_constants.get_hermes_dir`` (cache path)\n+\n+…so the full code path is covered without a live cua-driver, a real\n+auxiliary client, or network access.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import base64\n+import json\n+import os\n+from pathlib import Path\n+from typing import Any\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fixtures / helpers\n+# ---------------------------------------------------------------------------\n+\n+# 1×1 PNG (transparent) — minimal bytes that decode cleanly.\n+_PNG_B64 = (\n+    \"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42m\"\n+    \"NkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\"\n+)\n+\n+# 1×1 JPEG — used to verify mime detection works for either stream type.\n+_JPEG_B64 = (\n+    \"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB\"\n+    \"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/\"\n+)\n+\n+\n[redacted:email]\n+def tmp_cache_dir(tmp_path):\n+    \"\"\"Override get_hermes_dir so cache writes land under tmp_path.\"\"\"\n+    cache_dir = tmp_path / \"cache_vision\"\n+    cache_dir.mkdir()\n+\n+    def _fake_get(*_args, **_kw):\n+        return cache_dir\n+\n+    with patch(\"hermes_constants.get_hermes_dir\", _fake_get):\n+        yield cache_dir\n+\n+\n+def _make_capture(\n+    *,\n+    png_b64: str = _PNG_B64,\n+    mode: str = \"som\",\n+    elements=None,\n+    app: str = \"Safari\",\n+    window_title: str = \"GitHub – Issue #24015\",\n+    width: int = 1280,\n+    height: int = 800,\n+):\n+    from tools.computer_use.backend import CaptureResult, UIElement\n+\n+    elements = list(elements or [\n+        UIElement(index=0, role=\"AXButton\", label=\"Sign in\",\n+                  bounds=(10, 20, 80, 30)),\n+        UIElement(index=1, role=\"AXTextField\", label=\"username\",\n+                  bounds=(10, 60, 200, 24)),\n+    ])\n+    raw = base64.b64decode(png_b64, validate=False)\n+    return CaptureResult(\n+        mode=mode,\n+        width=width,\n+        height=height,\n+        png_b64=png_b64,\n+        elements=elements,\n+        app=app,\n+        window_title=window_title,\n+        png_bytes_len=len(raw),\n+    )\n+\n+\n+def _stub_aux_analysis(text: str):\n+    \"\"\"Return a fake vision_analyze_tool coroutine result (JSON envelope).\"\"\"\n+    return json.dumps({\"success\": True, \"analysis\": text})\n+\n+\n+# ---------------------------------------------------------------------------\n+# _capture_response: routing OFF (current/native behaviour)\n+# ---------------------------------------------------------------------------\n+\n+class TestCaptureResponseDefaultPath:\n+    \"\"\"When routing helper says 'native', the existing multimodal envelope wins.\"\"\"\n+\n+    def test_som_capture_returns_multimodal_envelope_when_native(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(png_b64=_PNG_B64, mode=\"som\")\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=False):\n+            resp = cu_tool._capture_response(cap)\n+\n+        assert isinstance(resp, dict)\n+        assert resp.get(\"_multimodal\") is True\n+        # Image part must use image/png MIME for a PNG payload.\n+        image_part = next(\n+            p for p in resp[\"content\"] if p.get(\"type\") == \"image_url\"\n+        )\n+        url = image_part[\"image_url\"][\"url\"]\n+        assert url.startswith(\"data:image/png;base64,\")\n+        assert \"vision_analysis\" not in resp\n+\n+    def test_jpeg_capture_returns_image_jpeg_mime_when_native(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(png_b64=_JPEG_B64, mode=\"som\")\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=False):\n+            resp = cu_tool._capture_response(cap)\n+\n+        url = next(p for p in resp[\"content\"] if p.get(\"type\") == \"image_url\")\n+        assert url[\"image_url\"][\"url\"].startswith(\"data:image/jpeg;base64,\")\n+\n+    def test_ax_only_capture_returns_text_regardless_of_routing(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"ax\", png_b64=\"\")\n+        # ax mode never has a PNG so neither path matters; assert pure text.\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True) as routing:\n+            resp = cu_tool._capture_response(cap)\n+\n+        # ax never even consults the routing helper — short-circuited above\n+        # the image branch.\n+        routing.assert_not_called()\n+        assert isinstance(resp, str)\n+        body = json.loads(resp)\n+        assert body[\"mode\"] == \"ax\"\n+\n+\n+# ---------------------------------------------------------------------------\n+# _capture_response: routing ON (the #24015 fix)\n+# ---------------------------------------------------------------------------\n+\n+class TestCaptureResponseRoutedToAuxVision:\n+    \"\"\"When routing helper says 'aux', the PNG is pre-analysed and a text\n+    response is returned with no image_url parts at all.\"\"\"\n+\n+    def test_som_capture_returns_text_with_vision_analysis(\n+        self, tmp_cache_dir,\n+    ):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"som\")\n+\n+        captured_calls = {}\n+\n+        def _fake_run_async(coro):\n+            captured_calls[\"called\"] = True\n+            return _stub_aux_analysis(\n+                \"A Safari window showing a GitHub issue page with a 'Sign \"\n+                \"in' button and a 'username' text field.\"\n+            )\n+\n+        # vision_analyze_tool is async; force a sync MagicMock so we can\n+        # assert positional args without dealing with awaitables.\n+        fake_vat = MagicMock(return_value=\"<coro>\")\n+\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True), \\\n+             patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+             patch(\"tools.vision_tools.vision_analyze_tool\",\n+                   new_callable=lambda: fake_vat):\n+            resp = cu_tool._capture_response(cap)\n+\n+        # Must be a JSON string, NOT a multimodal envelope. This is exactly\n+        # the contract that prevents #24015's HTTP 404 from firing on the\n+        # next agent turn.\n+        assert isinstance(resp, str)\n+        body = json.loads(resp)\n+        assert body[\"mode\"] == \"som\"\n+        assert body[\"app\"] == \"Safari\"\n+        assert \"Sign in\" in body[\"vision_analysis\"]\n+        assert body[\"vision_analysis_routed_via\"] == \"auxiliary.vision\"\n+        # The original AX-only metadata (window title, element index, app)\n+        # is preserved alongside the new vision analysis so the agent loses\n+        # no context vs the multimodal path.\n+        assert body[\"window_title\"] == \"GitHub – Issue #24015\"\n+        assert len(body[\"elements\"]) == 2\n+\n+        assert captured_calls.get(\"called\") is True\n+        # vision_analyze_tool was invoked with a path under the patched cache\n+        # and a non-empty prompt.\n+        args, _kwargs = fake_vat.call_args\n+        path_arg, prompt_arg = args[0], args[1]\n+        assert str(tmp_cache_dir) in path_arg\n+        assert \"macOS application screenshot\" in prompt_arg\n+        # AX summary is included so the aux model can ground its description\n+        # against the same set-of-mark index the agent will see.\n+        assert \"Sign in\" in prompt_arg\n+\n+    def test_temp_screenshot_file_is_cleaned_up_after_routing(\n+        self, tmp_cache_dir,\n+    ):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"som\")\n+        # We capture the path the aux call sees so we can assert it's gone\n+        # after _capture_response returns.\n+        observed_path = {}\n+\n+        def _fake_run_async(_coro):\n+            return _stub_aux_analysis(\"description goes here\")\n+\n+        def _fake_vat(image_path, _prompt):\n+            observed_path[\"path\"] = image_path\n+            # File must exist while aux is being arranged.\n+            assert os.path.exists(image_path)\n+            return \"<coro>\"\n+\n+        fake_vat = MagicMock(side_effect=_fake_vat)\n+\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True), \\\n+             patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+             patch(\"tools.vision_tools.vision_analyze_tool\",\n+                   new_callable=lambda: fake_vat):\n+            cu_tool._capture_response(cap)\n+\n+        # File must be unlinked after _capture_response returns.\n+        assert observed_path[\"path\"]\n+        assert not os.path.exists(observed_path[\"path\"])\n+\n+    def test_temp_file_cleaned_up_even_when_aux_call_raises(\n+        self, tmp_cache_dir,\n+    ):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"som\")\n+        observed_path = {}\n+\n+        def _fake_vat(image_path, _prompt):\n+            observed_path[\"path\"] = image_path\n+            return \"<coro>\"\n+\n+        def _fake_run_async(_coro):\n+            raise RuntimeError(\"aux LLM down\")\n+\n+        fake_vat = MagicMock(side_effect=_fake_vat)\n+\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True), \\\n+             patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+             patch(\"tools.vision_tools.vision_analyze_tool\",\n+                   new_callable=lambda: fake_vat):\n+            resp = cu_tool._capture_response(cap)\n+\n+        # Aux failure → fall back to multimodal envelope (so the user still\n+        # gets *something* useful even if vision is broken).\n+        assert isinstance(resp, dict)\n+        assert resp.get(\"_multimodal\") is True\n+        # Temp file must still be cleaned up.\n+        assert observed_path[\"path\"]\n+        assert not os.path.exists(observed_path[\"path\"])\n+\n+    def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"som\")\n+\n+        def _fake_run_async(_coro):\n+            return _stub_aux_analysis(\"\")\n+\n+        fake_vat = MagicMock(return_value=\"<coro>\")\n+\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True), \\\n+             patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+             patch(\"tools.vision_tools.vision_analyze_tool\",\n+                   new_callable=lambda: fake_vat):\n+            resp = cu_tool._capture_response(cap)\n+\n+        # Empty analysis is treated as failure — we'd rather show pixels\n+        # than embed an empty 'vision_analysis' string into the result.\n+        assert isinstance(resp, dict)\n+        assert resp.get(\"_multimodal\") is True\n+\n+    def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"som\")\n+\n+        def _fake_run_async(_coro):\n+            return 1234  # not a string at all\n+\n+        fake_vat = MagicMock(return_value=\"<coro>\")\n+\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True), \\\n+             patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+             patch(\"tools.vision_tools.vision_analyze_tool\",\n+                   new_callable=lambda: fake_vat):\n+            resp = cu_tool._capture_response(cap)\n+\n+        assert isinstance(resp, dict)\n+        assert resp.get(\"_multimodal\") is True\n+\n+\n+# ---------------------------------------------------------------------------\n+# _should_route_through_aux_vision: end-to-end with real config plumbing\n+# ---------------------------------------------------------------------------\n+\n+class TestRoutingDecisionWiring:\n+    \"\"\"Verify _should_route_through_aux_vision wires the right config + helper.\"\"\"\n+\n+    def test_explicit_aux_vision_in_config_routes_to_aux(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cfg = {\n+            \"model\": {\"default\": \"tencent/hy3-preview\", \"provider\": \"openrouter\"},\n+            \"auxiliary\": {\n+                \"vision\": {\n+                    \"provider\": \"openrouter\",\n+                    \"model\": \"google/gemini-2.5-flash\",\n+                }\n+            },\n+        }\n+        with patch(\"agent.auxiliary_client._read_main_provider\",\n+                   return_value=\"openrouter\"), \\\n+             patch(\"agent.auxiliary_client._read_main_model\",\n+                   return_value=\"tencent/hy3-preview\"), \\\n+             patch(\"hermes_cli.config.load_config\", return_value=cfg):\n+            assert cu_tool._should_route_through_aux_vision() is True\n+\n+    def test_no_explicit_aux_and_vision_capable_main_keeps_multimodal(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cfg = {\n+            \"model\": {\"default\": \"claude-opus-4-5\", \"provider\": \"anthropic\"},\n+        }\n+        with patch(\"agent.auxiliary_client._read_main_provider\",\n+                   return_value=\"anthropic\"), \\\n+             patch(\"agent.auxiliary_client._read_main_model\",\n+                   return_value=\"claude-opus-4-5\"), \\\n+             patch(\"hermes_cli.config.load_config\", return_value=cfg), \\\n+             patch(\"tools.computer_use.vision_routing._lookup_supports_vision\",\n+                   return_value=True), \\\n+             patch(\"tools.computer_use.vision_routing.\"\n+                   \"_provider_accepts_multimodal_tool_result\",\n+                   return_value=True):\n+            assert cu_tool._should_route_through_aux_vision() is False\n+\n+    def test_config_load_failure_disables_routing_safely(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        with patch(\"hermes_cli.config.load_config\",\n+                   side_effect=RuntimeError(\"config.yaml unreadable\")):\n+            # No exception should bubble up — fail open by returning False\n+            # so the legacy multimodal envelope continues to work.\n+            assert cu_tool._should_route_through_aux_vision() is False\n+\n+    def test_helper_decision_exception_is_swallowed(self):\n+        from tools.computer_use import tool as cu_tool\n+        from tools.computer_use import vision_routing as vr_mod\n+\n+        with patch(\"agent.auxiliary_client._read_main_provider\",\n+                   return_value=\"openrouter\"), \\\n+             patch(\"agent.auxiliary_client._read_main_model\",\n+                   return_value=\"x\"), \\\n+             patch(\"hermes_cli.config.load_config\", return_value={}), \\\n+             patch.object(vr_mod, \"should_route_capture_to_aux_vision\",\n+                          side_effect=ValueError(\"policy bug\")):\n+            assert cu_tool._should_route_through_aux_vision() is False\n+\n+\n+# ---------------------------------------------------------------------------\n+# Bug reproduction marker — proves the fix is needed.\n+# ---------------------------------------------------------------------------\n+\n+class TestBugReproductionAnchor:\n+    \"\"\"Without the fix, this test would assert the wrong thing.\n+\n+    On upstream/main HEAD prior to this branch, _capture_response returns a\n+    multimodal envelope unconditionally — so when a non-vision main model\n+    is configured, the captured PNG is delivered to the main provider as\n+    image_url content and the request is rejected with HTTP 404. We don't\n+    have a live provider here, but we can pin the contract: with routing\n+    enabled the response MUST be a JSON string with no image_url parts.\n+    \"\"\"\n+\n+    def test_non_vision_main_model_never_returns_image_url_when_routed(\n+        self, tmp_cache_dir,\n+    ):\n+        from tools.computer_use import tool as cu_tool\n+\n+        cap = _make_capture(mode=\"som\")\n+\n+        def _fake_run_async(_coro):\n+            return _stub_aux_analysis(\n+                \"Screenshot showing a GitHub.com window with a sign-in \"\n+                \"form.\"\n+            )\n+\n+        fake_vat = MagicMock(return_value=\"<coro>\")\n+\n+        with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+                          return_value=True), \\\n+             patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+             patch(\"tools.vision_tools.vision_analyze_tool\",\n+                   new_callable=lambda: fake_vat):\n+            resp = cu_tool._capture_response(cap)\n+\n+        # Must be a string (text-only result).\n+        assert isinstance(resp, str)\n+        # Must NOT contain a base64 image URL anywhere — that's what tripped\n+        # 'No endpoints found that support image input' on the reporter's\n+        # main provider in #24015.\n+        assert \"data:image\" not in resp\n+        assert \"image_url\" not in resp"]},{"file":"tests/tools/test_computer_use_vision_routing.py","hunks":["@@ -0,0 +1,260 @@\n+\"\"\"Unit tests for tools.computer_use.vision_routing.\n+\n+Cover the small ``should_route_capture_to_aux_vision`` policy helper that\n+decides whether a captured screenshot from ``computer_use(action='capture')``\n+should be returned as a multimodal envelope (main model handles vision\n+natively) or pre-analysed via the ``auxiliary.vision`` pipeline so the\n+main model only sees text.\n+\n+The companion end-to-end regression for #24015 lives in\n+``tests/tools/test_computer_use_capture_routing.py``; this file pins the\n+unit contract of the helper in isolation so behaviour does not regress\n+silently if the surrounding ``computer_use`` plumbing is refactored.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+from unittest.mock import patch\n+\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# _explicit_aux_vision_override\n+# ---------------------------------------------------------------------------\n+\n+class TestExplicitAuxVisionOverride:\n+    \"\"\"Mirror agent.image_routing — config detection must agree across paths.\"\"\"\n+\n+    def test_returns_false_for_none_cfg(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        assert _explicit_aux_vision_override(None) is False\n+\n+    def test_returns_false_for_non_dict_cfg(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        assert _explicit_aux_vision_override(\"not-a-dict\") is False\n+        assert _explicit_aux_vision_override([]) is False\n+\n+    def test_returns_false_when_auxiliary_block_missing(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        assert _explicit_aux_vision_override({}) is False\n+        assert _explicit_aux_vision_override({\"model\": {\"default\": \"x\"}}) is False\n+\n+    def test_returns_false_when_vision_block_missing(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"compression\": {\"provider\": \"openai\"}}}\n+        assert _explicit_aux_vision_override(cfg) is False\n+\n+    def test_returns_false_for_blank_provider_no_model_no_base_url(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"\", \"model\": \"\", \"base_url\": \"\"}}}\n+        assert _explicit_aux_vision_override(cfg) is False\n+\n+    def test_returns_false_for_provider_auto(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"auto\"}}}\n+        assert _explicit_aux_vision_override(cfg) is False\n+\n+    def test_returns_false_for_provider_AUTO_uppercase(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"  AUTO  \"}}}\n+        assert _explicit_aux_vision_override(cfg) is False\n+\n+    def test_returns_true_for_explicit_provider(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"openrouter\"}}}\n+        assert _explicit_aux_vision_override(cfg) is True\n+\n+    def test_returns_true_for_explicit_model_only(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": {\"model\": \"google/gemini-2.5-flash\"}}}\n+        assert _explicit_aux_vision_override(cfg) is True\n+\n+    def test_returns_true_for_explicit_base_url_only(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": {\"base_url\": \"http://localhost:1234/v1\"}}}\n+        assert _explicit_aux_vision_override(cfg) is True\n+\n+    def test_returns_true_for_provider_auto_plus_explicit_model(self):\n+        \"\"\"``provider: auto`` + an explicit model still counts as override.\"\"\"\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\n+            \"auxiliary\": {\n+                \"vision\": {\"provider\": \"auto\", \"model\": \"claude-3-haiku\"},\n+            }\n+        }\n+        assert _explicit_aux_vision_override(cfg) is True\n+\n+    def test_handles_non_dict_vision_block(self):\n+        from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+        cfg = {\"auxiliary\": {\"vision\": \"not-a-dict\"}}\n+        assert _explicit_aux_vision_override(cfg) is False\n+\n+\n+# ---------------------------------------------------------------------------\n+# should_route_capture_to_aux_vision\n+# ---------------------------------------------------------------------------\n+\n+class TestRouteDecision:\n+    \"\"\"End-to-end policy: explicit override > tool-result support > vision caps.\"\"\"\n+\n+    def test_explicit_override_routes_to_aux_even_for_vision_main(self):\n+        \"\"\"Issue #24015 core repro: explicit aux config must win.\n+\n+        Even if the main model fully supports vision (Anthropic / Claude),\n+        an explicit ``auxiliary.vision`` block means the user wants their\n+        configured backend used. Don't silently bypass it.\n+        \"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        cfg = {\n+            \"auxiliary\": {\n+                \"vision\": {\n+                    \"provider\": \"openrouter\",\n+                    \"model\": \"google/gemini-2.5-flash\",\n+                }\n+            }\n+        }\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=True):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"anthropic\", \"claude-opus-4-5\", cfg\n+            ) is True\n+\n+    def test_non_vision_main_model_routes_to_aux(self):\n+        \"\"\"The reported #24015 scenario: tencent/hy3-preview has no vision.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        cfg = {\"model\": {\"default\": \"tencent/hy3-preview\", \"provider\": \"openrouter\"}}\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=False), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=True):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"openrouter\", \"tencent/hy3-preview\", cfg\n+            ) is True\n+\n+    def test_vision_main_model_no_override_keeps_multimodal(self):\n+        \"\"\"Default path: vision-capable main model + no aux override → native.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=True):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"anthropic\", \"claude-opus-4-5\", None\n+            ) is False\n+\n+    def test_provider_rejects_multimodal_tool_results_routes_to_aux(self):\n+        \"\"\"Some providers' tool-result messages won't carry images at all.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=False):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"some-aggregator\", \"some-vision-model\", {}\n+            ) is True\n+\n+    def test_unknown_provider_capabilities_fail_closed(self):\n+        \"\"\"When tool-result lookup returns None, route to aux (safe default).\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=None):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"exotic-provider\", \"exotic-model\", {}\n+            ) is True\n+\n+    def test_unknown_vision_capability_fails_closed(self):\n+        \"\"\"When models.dev has no entry, prefer aux over a likely 404.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=None), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=True):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"openrouter\", \"novel/never-seen-model\", {}\n+            ) is True\n+\n+    def test_explicit_override_wins_over_unknown_caps(self):\n+        \"\"\"Explicit aux config wins regardless of unknown caps elsewhere.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"openrouter\"}}}\n+        with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=None), \\\n+             patch.object(vision_routing,\n+                          \"_provider_accepts_multimodal_tool_result\",\n+                          return_value=None):\n+            assert vision_routing.should_route_capture_to_aux_vision(\n+                \"openrouter\", \"tencent/hy3-preview\", cfg\n+            ) is True\n+\n+\n+# ---------------------------------------------------------------------------\n+# Internal lookups — defensive paths\n+# ---------------------------------------------------------------------------\n+\n+class TestLookupHelpers:\n+    def test_lookup_supports_vision_returns_none_for_blank_provider(self):\n+        from tools.computer_use.vision_routing import _lookup_supports_vision\n+        assert _lookup_supports_vision(\"\", \"claude\") is None\n+\n+    def test_lookup_supports_vision_returns_none_for_blank_model(self):\n+        from tools.computer_use.vision_routing import _lookup_supports_vision\n+        assert _lookup_supports_vision(\"anthropic\", \"\") is None\n+\n+    def test_lookup_supports_vision_handles_lookup_exception(self):\n+        \"\"\"Underlying caps lookup may raise; helper must swallow + return None.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        def _boom(_provider, _model):\n+            raise RuntimeError(\"models.dev unreachable\")\n+\n+        with patch(\"agent.models_dev.get_model_capabilities\", side_effect=_boom):\n+            assert vision_routing._lookup_supports_vision(\"anthropic\", \"claude\") is None\n+\n+    def test_lookup_supports_vision_returns_none_when_caps_missing(self):\n+        from tools.computer_use import vision_routing\n+\n+        with patch(\"agent.models_dev.get_model_capabilities\", return_value=None):\n+            assert vision_routing._lookup_supports_vision(\"anthropic\", \"claude\") is None\n+\n+    def test_provider_accepts_multimodal_tool_result_returns_none_for_blank_provider(self):\n+        from tools.computer_use.vision_routing import (\n+            _provider_accepts_multimodal_tool_result,\n+        )\n+        assert _provider_accepts_multimodal_tool_result(\"\", \"claude\") is None\n+\n+\n+# ---------------------------------------------------------------------------\n+# Module surface\n+# ---------------------------------------------------------------------------\n+\n+class TestModuleSurface:\n+    \"\"\"Pin the public surface so dependents stay in lockstep.\"\"\"\n+\n+    def test_should_route_capture_to_aux_vision_is_exported(self):\n+        from tools.computer_use import vision_routing\n+\n+        assert \"should_route_capture_to_aux_vision\" in vision_routing.__all__\n+        assert callable(vision_routing.should_route_capture_to_aux_vision)\n+\n+    @pytest.mark.parametrize(\"name\", [\n+        \"_explicit_aux_vision_override\",\n+        \"_lookup_supports_vision\",\n+        \"_provider_accepts_multimodal_tool_result\",\n+    ])\n+    def test_internal_helpers_are_addressable(self, name):\n+        \"\"\"Internal helpers stay importable so tests can monkeypatch them.\"\"\"\n+        from tools.computer_use import vision_routing\n+\n+        assert hasattr(vision_routing, name)\n+        assert callable(getattr(vision_routing, name))"]},{"file":"tools/computer_use/tool.py","hunks":["@@ -429,6 +429,21 @@ def _capture_response(cap: CaptureResult) -> Any:\n     summary = \"\\n\".join(summary_lines)\n \n     if cap.png_b64 and cap.mode != \"ax\":\n+        # Decide whether to hand the screenshot to the auxiliary.vision\n+        # pipeline (text-only result) or keep the multimodal envelope (main\n+        # model handles vision natively). Issue #24015: previously the\n+        # multimodal envelope was returned unconditionally, so non-vision\n+        # main models tripped HTTP 404 / 400 at the provider boundary even\n+        # when auxiliary.vision was explicitly configured to handle this.\n+        if _should_route_through_aux_vision():\n+            routed = _route_capture_through_aux_vision(cap, summary)\n+            if routed is not None:\n+                return routed\n+            # Aux routing was requested but failed (no vision client, aux\n+            # call raised, etc.). Fall through to the multimodal envelope —\n+            # better to surface a tool-result error from the main model\n+            # than to silently drop the screenshot entirely.\n+\n         # Detect actual image format from base64 magic bytes so the MIME type\n         # matches what the data contains (cua-driver may return JPEG or PNG).\n         # JPEG: base64 starts with /9j/   PNG: starts with iVBOR","@@ -457,6 +472,140 @@ def _capture_response(cap: CaptureResult) -> Any:\n     })\n \n \n+# ---------------------------------------------------------------------------\n+# auxiliary.vision routing for captured screenshots (#24015)\n+# ---------------------------------------------------------------------------\n+\n+def _should_route_through_aux_vision() -> bool:\n+    \"\"\"Return True when ``_capture_response`` should hand the PNG to aux vision.\n+\n+    Reads the active main provider/model and the loaded config and asks the\n+    routing helper. Any failure (config import, runtime override missing,\n+    etc.) returns False so the existing multimodal envelope continues to be\n+    returned — fail open on the routing decision so a broken config can\n+    never silently drop the screenshot for vision-capable main models.\n+    \"\"\"\n+    try:\n+        from agent.auxiliary_client import _read_main_model, _read_main_provider\n+        from hermes_cli.config import load_config\n+        from tools.computer_use.vision_routing import (\n+            should_route_capture_to_aux_vision,\n+        )\n+    except Exception as exc:  # pragma: no cover - defensive\n+        logger.debug(\"computer_use: aux-vision routing import failed: %s\", exc)\n+        return False\n+    try:\n+        provider = _read_main_provider()\n+        model = _read_main_model()\n+        cfg = load_config()\n+    except Exception as exc:  # pragma: no cover - defensive\n+        logger.debug(\"computer_use: aux-vision routing config read failed: %s\", exc)\n+        return False\n+    try:\n+        return bool(should_route_capture_to_aux_vision(provider, model, cfg))\n+    except Exception as exc:  # pragma: no cover - defensive\n+        logger.debug(\"computer_use: aux-vision routing decision failed: %s\", exc)\n+        return False\n+\n+\n+def _route_capture_through_aux_vision(\n+    cap: CaptureResult,\n+    summary: str,\n+) -> Optional[str]:\n+    \"\"\"Pre-analyse the captured PNG via ``vision_analyze`` and return a text result.\n+\n+    The captured base64 PNG is materialised to ``$HERMES_HOME/cache/vision/``\n+    and handed to ``vision_analyze_tool`` with a generic describe prompt.\n+    The resulting text description is merged into the existing AX/SOM\n+    summary so the main model receives a single text payload that mentions\n+    every interactable element AND a description of what the screenshot\n+    looked like.\n+\n+    Returns:\n+      A JSON-encoded text response on success.\n+      ``None`` on failure (caller falls back to the multimodal envelope).\n+    \"\"\"\n+    if not cap.png_b64:\n+        return None\n+    try:\n+        import base64 as _base64\n+        import os as _os\n+        import uuid as _uuid\n+\n+        from hermes_constants import get_hermes_dir\n+        from model_tools import _run_async\n+        from tools.vision_tools import vision_analyze_tool\n+    except Exception as exc:  # pragma: no cover - defensive\n+        logger.debug(\"computer_use: aux-vision import failed: %s\", exc)\n+        return None\n+\n+    temp_image_path = None\n+    try:\n+        try:\n+            raw = _base64.b64decode(cap.png_b64, validate=False)\n+        except Exception as exc:\n+            logger.debug(\"computer_use: failed to decode capture base64: %s\", exc)\n+            return None\n+\n+        # Pick an extension that matches the on-disk bytes so vision_analyze's\n+        # MIME sniffing returns the right content-type.\n+        ext = \".jpg\" if cap.png_b64[:8].startswith(\"/9j/\") else \".png\"\n+        cache_dir = get_hermes_dir(\"cache/vision\", \"temp_vision_images\")\n+        temp_image_path = cache_dir / f\"computer_use_{_uuid.uuid4().hex}{ext}\"\n+        temp_image_path.write_bytes(raw)\n+\n+        prompt = (\n+            \"Describe what is visible in this macOS application screenshot in \"\n+            \"concise but specific terms. Mention the app name and window \"\n+            \"title if visible, the overall layout, any labelled buttons, \"\n+            \"menus or text fields, and any prominent text content the user \"\n+            \"would need to know about. Do not invent details that are not \"\n+            \"actually visible.\\n\\n\"\n+            f\"AX/SOM index for cross-reference:\\n{summary}\"\n+        )\n+\n+        result_json = _run_async(\n+            vision_analyze_tool(str(temp_image_path), prompt)\n+        )\n+    except Exception as exc:\n+        logger.warning(\n+            \"computer_use: auxiliary.vision pre-analysis failed (%s); \"\n+            \"falling back to native multimodal envelope\",\n+            exc,\n+        )\n+        return None\n+    finally:\n+        if temp_image_path is not None:\n+            try:\n+                _os.unlink(str(temp_image_path))\n+            except Exception:\n+                pass\n+\n+    analysis_text = \"\"\n+    if isinstance(result_json, str):\n+        try:\n+            parsed = json.loads(result_json)\n+            if isinstance(parsed, dict):\n+                analysis_text = str(parsed.get(\"analysis\") or \"\").strip()\n+        except (TypeError, json.JSONDecodeError):\n+            analysis_text = result_json.strip()\n+\n+    if not analysis_text:\n+        return None\n+\n+    return json.dumps({\n+        \"mode\": cap.mode,\n+        \"width\": cap.width,\n+        \"height\": cap.height,\n+        \"app\": cap.app,\n+        \"window_title\": cap.window_title,\n+        \"elements\": [_element_to_dict(e) for e in cap.elements],\n+        \"summary\": summary,\n+        \"vision_analysis\": analysis_text,\n+        \"vision_analysis_routed_via\": \"auxiliary.vision\",\n+    })\n+\n+\n def _maybe_follow_capture(\n     backend: ComputerUseBackend, res: ActionResult, do_capture: bool,\n ) -> Any:"]},{"file":"tools/computer_use/vision_routing.py","hunks":["@@ -0,0 +1,152 @@\n+\"\"\"Vision-routing decisions for ``computer_use`` capture results.\n+\n+Background\n+----------\n+``computer_use(action='capture', mode='som'|'vision')`` returns a\n+``_multimodal`` envelope containing the captured screenshot. That envelope\n+is delivered back to the **active session model** as the tool result. When\n+the active main model has no vision capability (e.g. text-only or\n+text+code-only models), or when the active provider rejects multimodal\n+content inside tool-result messages, the screenshot trips a 404 / 400 at\n+the provider boundary and the agent loop reports a hard tool failure.\n+\n+Issue #24015 reports this regression for the ``cua-driver`` backend:\n+configuring ``auxiliary.vision`` (a dedicated vision-capable model) in\n+``config.yaml`` was silently ignored — the screenshot was still routed at\n+the *main* model and failed with HTTP 404 ``No endpoints found that\n+support image input`` even though a perfectly good vision backend was\n+sitting in config waiting to be used.\n+\n+This module centralises the small policy decision: should a captured\n+screenshot be returned as multimodal content (main model handles vision\n+natively) or pre-analysed via the auxiliary vision pipeline so the main\n+model only ever sees text?\n+\n+Behaviour (mirrors ``vision_analyze`` for consistency)\n+------------------------------------------------------\n+* If the user explicitly configured ``auxiliary.vision`` (any of\n+  ``provider``, ``model``, or ``base_url`` non-empty / not ``\"auto\"``),\n+  the screenshot is routed through the aux vision pipeline. Users who\n+  pay for a dedicated vision model usually want it used.\n+* Otherwise, if the active main model+provider can carry an image inside\n+  a tool-result message AND the model reports ``supports_vision=True``\n+  in models.dev metadata, return ``False`` (use the multimodal path).\n+* In every other case (non-vision main model, provider that does not\n+  accept multimodal tool results, lookup failure), route through aux\n+  vision so the main model receives a text description it can act on.\n+\n+The decision intentionally fails *closed* (i.e. towards aux routing) when\n+metadata is missing or ambiguous: returning a screenshot to a model that\n+cannot read it is a hard tool failure, while routing it through aux costs\n+one extra LLM call and yields a usable description.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import logging\n+from typing import Any, Dict, Optional\n+\n+logger = logging.getLogger(__name__)\n+\n+\n+def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:\n+    \"\"\"True when ``auxiliary.vision`` carries a non-default user override.\n+\n+    Mirrors ``agent.image_routing._explicit_aux_vision_override`` so the\n+    capture path and the user-attached-image path agree on what counts as\n+    an explicit user request for the aux vision pipeline. ``provider:\n+    \"auto\"``, blank values, or a missing block all count as *not*\n+    explicit.\n+    \"\"\"\n+    if not isinstance(cfg, dict):\n+        return False\n+    aux = cfg.get(\"auxiliary\") or {}\n+    if not isinstance(aux, dict):\n+        return False\n+    vision = aux.get(\"vision\") or {}\n+    if not isinstance(vision, dict):\n+        return False\n+\n+    provider = str(vision.get(\"provider\") or \"\").strip().lower()\n+    model = str(vision.get(\"model\") or \"\").strip()\n+    base_url = str(vision.get(\"base_url\") or \"\").strip()\n+\n+    if provider in (\"\", \"auto\") and not model and not base_url:\n+        return False\n+    return True\n+\n+\n+def _lookup_supports_vision(provider: str, model: str) -> Optional[bool]:\n+    \"\"\"Return models.dev ``supports_vision`` for *(provider, model)* or None.\"\"\"\n+    if not provider or not model:\n+        return None\n+    try:\n+        from agent.models_dev import get_model_capabilities\n+        caps = get_model_capabilities(provider, model)\n+    except Exception as exc:  # pragma: no cover - defensive\n+        logger.debug(\n+            \"computer_use vision_routing: caps lookup failed for %s:%s — %s\",\n+            provider, model, exc,\n+        )\n+        return None\n+    if caps is None:\n+        return None\n+    return bool(getattr(caps, \"supports_vision\", False))\n+\n+\n+def _provider_accepts_multimodal_tool_result(provider: str, model: str) -> Optional[bool]:\n+    \"\"\"Return whether *provider*+*model* carries images inside tool-result messages.\n+\n+    Reuses ``tools.vision_tools._supports_media_in_tool_results`` so the\n+    capture-routing decision stays in lockstep with the\n+    ``vision_analyze`` native fast path. Returns None on import failure\n+    so callers fall back to aux routing rather than guessing.\n+    \"\"\"\n+    if not provider:\n+        return None\n+    try:\n+        from tools.vision_tools import _supports_media_in_tool_results\n+    except Exception as exc:  # pragma: no cover - defensive\n+        logger.debug(\n+            \"computer_use vision_routing: tool-result support lookup failed: %s\",\n+            exc,\n+        )\n+        return None\n+    return bool(_supports_media_in_tool_results(provider, model))\n+\n+\n+def should_route_capture_to_aux_vision(\n+    provider: str,\n+    model: str,\n+    cfg: Optional[Dict[str, Any]],\n+) -> bool:\n+    \"\"\"Return True iff the captured screenshot should be pre-analysed via aux vision.\n+\n+    Args:\n+      provider: active inference provider id (e.g. ``\"openrouter\"``,\n+        ``\"anthropic\"``, ``\"openai-codex\"``). Lower-case canonical id.\n+      model:    active main model slug as it would be sent to the provider.\n+      cfg:      loaded ``config.yaml`` dict (or None).\n+\n+    Returns:\n+      ``True`` when the caller should hand the screenshot to the aux vision\n+      pipeline (and surface a text-only tool result). ``False`` when the\n+      caller should keep the existing multimodal envelope (main model\n+      handles vision natively).\n+    \"\"\"\n+    if _explicit_aux_vision_override(cfg):\n+        return True\n+\n+    accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)\n+    if accepts_tool_image is None or accepts_tool_image is False:\n+        return True\n+\n+    supports_vision = _lookup_supports_vision(provider, model)\n+    if supports_vision is True:\n+        return False\n+    return True\n+\n+\n+__all__ = [\n+    \"should_route_capture_to_aux_vision\",\n+]\n"]}]},{"kind":"pr","number":30145,"title":"fix(computer-use): cap AX elements array to prevent context blowup (#22865)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_computer_use.py","tools/computer_use/schema.py","tools/computer_use/tool.py"],"fix_hunks":[{"file":"tests/tools/test_computer_use.py","hunks":["@@ -76,6 +76,27 @@ def test_capture_mode_enum_has_som_vision_ax(self):\n         modes = set(COMPUTER_USE_SCHEMA[\"parameters\"][\"properties\"][\"mode\"][\"enum\"])\n         assert modes == {\"som\", \"vision\", \"ax\"}\n \n+    def test_schema_exposes_max_elements_cap_for_capture(self):\n+        from tools.computer_use.schema import COMPUTER_USE_SCHEMA\n+        props = COMPUTER_USE_SCHEMA[\"parameters\"][\"properties\"]\n+        assert \"max_elements\" in props\n+        assert props[\"max_elements\"][\"type\"] == \"integer\"\n+        assert props[\"max_elements\"].get(\"minimum\", 1) >= 1\n+\n+    def test_schema_max_elements_documents_default_and_upper_bound(self):\n+        \"\"\"Schema description must agree with the runtime. The original PR\n+        text said \"Default 100\" without a corresponding `default` field, and\n+        had no upper bound — both Copilot findings.\n+        \"\"\"\n+        from tools.computer_use.schema import COMPUTER_USE_SCHEMA\n+        from tools.computer_use.tool import (\n+            _DEFAULT_MAX_ELEMENTS,\n+            _MAX_ALLOWED_MAX_ELEMENTS,\n+        )\n+        prop = COMPUTER_USE_SCHEMA[\"parameters\"][\"properties\"][\"max_elements\"]\n+        assert prop.get(\"default\") == _DEFAULT_MAX_ELEMENTS\n+        assert prop.get(\"maximum\") == _MAX_ALLOWED_MAX_ELEMENTS\n+\n \n class TestRegistration:\n     def test_tool_registers_with_registry(self):","@@ -337,6 +358,193 @@ def focus_app(self, app, raise_window=False): ...\n         assert \"AXButton\" in text_part[\"text\"]\n         assert \"AXTextField\" in text_part[\"text\"]\n \n+    def _ax_backend_with(self, count: int):\n+        \"\"\"Construct a fake backend that yields ``count`` AX elements.\"\"\"\n+        from tools.computer_use.backend import CaptureResult, UIElement\n+\n+        elements = [\n+            UIElement(index=i + 1, role=\"AXButton\", label=f\"el-{i}\", bounds=(0, 0, 1, 1))\n+            for i in range(count)\n+        ]\n+\n+        class FakeBackend:\n+            def start(self): pass\n+            def stop(self): pass\n+            def is_available(self): return True\n+            def capture(self, mode=\"som\", app=None):\n+                return CaptureResult(\n+                    mode=mode, width=800, height=600,\n+                    png_b64=\"\",\n+                    elements=list(elements),\n+                    app=\"Obsidian\",\n+                )\n+            def click(self, **kw): ...\n+            def drag(self, **kw): ...\n+            def scroll(self, **kw): ...\n+            def type_text(self, text): ...\n+            def key(self, keys): ...\n+            def list_apps(self): return []\n+            def focus_app(self, app, raise_window=False): ...\n+\n+        return FakeBackend()\n+\n+    def test_capture_ax_caps_elements_at_default_for_dense_trees(self):\n+        \"\"\"Regression for #22865: an Electron-style 600-element AX tree must\n+        not emit the entire array verbatim into the tool result.\n+        \"\"\"\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_backend = self._ax_backend_with(600)\n+        cu_tool.reset_backend_for_tests()\n+        with patch.object(cu_tool, \"_get_backend\", return_value=fake_backend):\n+            out = cu_tool.handle_computer_use({\"action\": \"capture\", \"mode\": \"ax\"})\n+\n+        parsed = json.loads(out)\n+        assert parsed[\"mode\"] == \"ax\"\n+        assert parsed[\"total_elements\"] == 600\n+        assert len(parsed[\"elements\"]) == cu_tool._DEFAULT_MAX_ELEMENTS\n+        assert parsed[\"truncated_elements\"] == 600 - cu_tool._DEFAULT_MAX_ELEMENTS\n+        # Truncation must be visible in the human summary so the model knows\n+        # the JSON view is partial and can re-issue with a tighter scope.\n+        assert \"truncated to\" in parsed[\"summary\"]\n+\n+    def test_capture_ax_honors_explicit_max_elements_override(self):\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_backend = self._ax_backend_with(600)\n+        cu_tool.reset_backend_for_tests()\n+        with patch.object(cu_tool, \"_get_backend\", return_value=fake_backend):\n+            out = cu_tool.handle_computer_use(\n+                {\"action\": \"capture\", \"mode\": \"ax\", \"max_elements\": 250}\n+            )\n+\n+        parsed = json.loads(out)\n+        assert len(parsed[\"elements\"]) == 250\n+        assert parsed[\"truncated_elements\"] == 350\n+\n+    def test_capture_ax_below_cap_is_unchanged(self):\n+        \"\"\"Backwards-compat: small captures keep the full elements array and\n+        do not surface a `truncated_elements` field.\n+        \"\"\"\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_backend = self._ax_backend_with(5)\n+        cu_tool.reset_backend_for_tests()\n+        with patch.object(cu_tool, \"_get_backend\", return_value=fake_backend):\n+            out = cu_tool.handle_computer_use({\"action\": \"capture\", \"mode\": \"ax\"})\n+\n+        parsed = json.loads(out)\n+        assert len(parsed[\"elements\"]) == 5\n+        assert parsed[\"total_elements\"] == 5\n+        assert \"truncated_elements\" not in parsed\n+        assert \"truncated to\" not in parsed[\"summary\"]\n+\n+    def test_capture_ax_invalid_max_elements_falls_back_to_default(self):\n+        \"\"\"Malformed `max_elements` (string, negative, zero) must not silently\n+        disable the cap and re-introduce the original unbounded behavior.\n+        \"\"\"\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_backend = self._ax_backend_with(600)\n+        cu_tool.reset_backend_for_tests()\n+        for bad in (\"not-a-number\", 0, -10):\n+            with patch.object(cu_tool, \"_get_backend\", return_value=fake_backend):\n+                out = cu_tool.handle_computer_use(\n+                    {\"action\": \"capture\", \"mode\": \"ax\", \"max_elements\": bad}\n+                )\n+            parsed = json.loads(out)\n+            assert len(parsed[\"elements\"]) == cu_tool._DEFAULT_MAX_ELEMENTS, (\n+                f\"bad max_elements={bad!r} disabled the cap\"\n+            )\n+\n+    def test_capture_ax_clamps_oversized_max_elements_to_hard_cap(self):\n+        \"\"\"A caller passing a very large `max_elements` must not be able to\n+        disable the safeguard. The cap is clamped to a hard upper bound so\n+        the context-blow-up protection cannot be bypassed by argument.\n+        \"\"\"\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_backend = self._ax_backend_with(5000)\n+        cu_tool.reset_backend_for_tests()\n+        with patch.object(cu_tool, \"_get_backend\", return_value=fake_backend):\n+            out = cu_tool.handle_computer_use(\n+                {\"action\": \"capture\", \"mode\": \"ax\", \"max_elements\": 10_000}\n+            )\n+        parsed = json.loads(out)\n+        assert len(parsed[\"elements\"]) == cu_tool._MAX_ALLOWED_MAX_ELEMENTS\n+        assert parsed[\"total_elements\"] == 5000\n+        assert parsed[\"truncated_elements\"] == 5000 - cu_tool._MAX_ALLOWED_MAX_ELEMENTS\n+\n+    def test_capture_ax_summary_indices_match_returned_elements(self):\n+        \"\"\"When `max_elements` is below the human-summary's own line cap, the\n+        summary must not index elements that aren't in the returned array.\n+        Otherwise the model sees `#15` in the summary and finds no matching\n+        entry in `elements`.\n+        \"\"\"\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_backend = self._ax_backend_with(600)\n+        cu_tool.reset_backend_for_tests()\n+        with patch.object(cu_tool, \"_get_backend\", return_value=fake_backend):\n+            out = cu_tool.handle_computer_use(\n+                {\"action\": \"capture\", \"mode\": \"ax\", \"max_elements\": 5}\n+            )\n+        parsed = json.loads(out)\n+        returned_indices = {e[\"index\"] for e in parsed[\"elements\"]}\n+        summary_lines = parsed[\"summary\"].splitlines()\n+        indexed_lines = [ln for ln in summary_lines if ln.lstrip().startswith(\"#\")]\n+        for ln in indexed_lines:\n+            idx_token = [redacted:secret-assignment]\"#\")\n+            idx = int(idx_token)\n+            assert idx in returned_indices, (\n+                f\"summary references #{idx} but it is absent from elements payload \"\n+                f\"(returned: {sorted(returned_indices)})\"\n+            )\n+\n+    def test_capture_multimodal_summary_omits_truncation_note(self):\n+        \"\"\"The som/vision multimodal envelope returns a screenshot, not an\n+        `elements` array — so a \"response truncated to N of M elements\"\n+        claim in the summary would be inaccurate.\n+        \"\"\"\n+        from tools.computer_use.backend import CaptureResult, UIElement\n+        from tools.computer_use import tool as cu_tool\n+\n+        fake_png = \"iVBORw0KGgo=\"\n+        elements = [\n+            UIElement(index=i + 1, role=\"AXButton\", label=f\"el-{i}\", bounds=(0, 0, 1, 1))\n+            for i in range(600)\n+        ]\n+\n+        class FakeBackend:\n+            def start(self): pass\n+            def stop(self): pass\n+            def is_available(self): return True\n+            def capture(self, mode=\"som\", app=None):\n+                return CaptureResult(\n+                    mode=mode, width=800, height=600,\n+                    png_b64=fake_png, elements=list(elements),\n+                    app=\"Obsidian\",\n+                )\n+            def click(self, **kw): ...\n+            def drag(self, **kw): ...\n+            def scroll(self, **kw): ...\n+            def type_text(self, text): ...\n+            def key(self, keys): ...\n+            def list_apps(self): return []\n+            def focus_app(self, app, raise_window=False): ...\n+\n+        cu_tool.reset_backend_for_tests()\n+        with patch.object(cu_tool, \"_get_backend\", return_value=FakeBackend()):\n+            out = cu_tool.handle_computer_use({\"action\": \"capture\", \"mode\": \"som\"})\n+\n+        assert isinstance(out, dict) and out[\"_multimodal\"] is True\n+        text_part = next(p for p in out[\"content\"] if p.get(\"type\") == \"text\")\n+        assert \"truncated to\" not in text_part[\"text\"], (\n+            \"multimodal response carries an image, not an elements array; \"\n+            \"the truncation note describes a payload field that isn't present\"\n+        )\n+        assert \"truncated to\" not in out[\"text_summary\"]\n+\n \n # ---------------------------------------------------------------------------\n # Anthropic adapter: multimodal tool-result conversion"]},{"file":"tools/computer_use/schema.py","hunks":["@@ -75,6 +75,28 @@\n                     \"frontmost app's window or the whole screen.\"\n                 ),\n             },\n+            \"max_elements\": {\n+                \"type\": \"integer\",\n+                \"description\": (\n+                    \"Optional cap on the AX `elements` array returned by \"\n+                    \"`action='capture'`. Default 100, hard maximum 1000. \"\n+                    \"Dense UIs (Electron apps such as Obsidian or VS Code, \"\n+                    \"JetBrains IDEs) can publish 500+ AX nodes — capping \"\n+                    \"prevents a single capture from blowing session \"\n+                    \"context. When the cap trims the response, \"\n+                    \"`total_elements` and `truncated_elements` are \"\n+                    \"surfaced in the result so you can re-call with \"\n+                    \"`app=` to narrow scope or raise `max_elements` when \"\n+                    \"the full tree is required. Has no effect on \"\n+                    \"`mode='som'` / `mode='vision'` when a screenshot is \"\n+                    \"included in the response; only the rare image-\"\n+                    \"missing fallback returns an `elements` array and is \"\n+                    \"subject to the cap.\"\n+                ),\n+                \"default\": 100,\n+                \"minimum\": 1,\n+                \"maximum\": 1000,\n+            },\n             # ── click / drag / scroll targeting ────────────────────\n             \"element\": {\n                 \"type\": \"integer\","]},{"file":"tools/computer_use/tool.py","hunks":["@@ -317,7 +317,7 @@ def _dispatch(backend: ComputerUseBackend, action: str, args: Dict[str, Any]) ->\n         if mode not in {\"som\", \"vision\", \"ax\"}:\n             return json.dumps({\"error\": f\"bad mode {mode!r}; use som|vision|ax\"})\n         cap = backend.capture(mode=mode, app=args.get(\"app\"))\n-        return _capture_response(cap)\n+        return _capture_response(cap, max_elements=_coerce_max_elements(args.get(\"max_elements\")))\n \n     if action == \"wait\":\n         seconds = float(args.get(\"seconds\", 1.0))","@@ -416,16 +416,62 @@ def _text_response(res: ActionResult) -> str:\n     return json.dumps(payload)\n \n \n-def _capture_response(cap: CaptureResult) -> Any:\n-    element_index = _format_elements(cap.elements)\n+# Default cap for the AX `elements` array returned by capture. Dense UIs\n+# (Electron apps, Obsidian, JetBrains IDEs) can publish 500+ AX nodes, which\n+# can exhaust session context after a single capture. The model-facing\n+# `max_elements` argument lets callers raise this when they need the full tree.\n+_DEFAULT_MAX_ELEMENTS = 100\n+# Hard upper bound on caller-supplied `max_elements`. Without this, a tool\n+# call passing a very large integer would silently disable the safeguard and\n+# reintroduce the original unbounded behavior.\n+_MAX_ALLOWED_MAX_ELEMENTS = 1000\n+\n+\n+def _coerce_max_elements(value: Any) -> int:\n+    \"\"\"Validate the caller-supplied ``max_elements``.\n+\n+    Falls back to :data:`_DEFAULT_MAX_ELEMENTS` for missing / non-integer /\n+    sub-1 inputs so the cap can never be silently disabled by a malformed\n+    tool-call argument. Clamps oversized values to\n+    :data:`_MAX_ALLOWED_MAX_ELEMENTS` so a caller cannot bypass the\n+    safeguard by passing a very large integer.\n+    \"\"\"\n+    if value is None:\n+        return _DEFAULT_MAX_ELEMENTS\n+    try:\n+        n = int(value)\n+    except (TypeError, ValueError):\n+        return _DEFAULT_MAX_ELEMENTS\n+    if n < 1:\n+        return _DEFAULT_MAX_ELEMENTS\n+    if n > _MAX_ALLOWED_MAX_ELEMENTS:\n+        return _MAX_ALLOWED_MAX_ELEMENTS\n+    return n\n+\n+\n+def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEMENTS) -> Any:\n+    total_elements = len(cap.elements)\n+    visible_elements = cap.elements[:max_elements]\n+    truncated_elements = max(0, total_elements - len(visible_elements))\n+\n+    # Index only what's actually surfaced in the response — otherwise the\n+    # human-readable summary references element indices the model cannot\n+    # find in the JSON `elements` array (e.g. max_elements=10 vs the default\n+    # 40-line index window).\n+    element_index = _format_elements(visible_elements)\n     summary_lines = [\n         f\"capture mode={cap.mode} {cap.width}x{cap.height}\"\n         + (f\" app={cap.app}\" if cap.app else \"\")\n         + (f\" window={cap.window_title!r}\" if cap.window_title else \"\"),\n-        f\"{len(cap.elements)} interactable element(s):\",\n+        f\"{total_elements} interactable element(s):\",\n     ]\n     if element_index:\n         summary_lines.extend(element_index)\n+    # Multimodal and AX paths both reference `summary`; build it once up-front\n+    # so the aux-vision routing branch (which fires before either path is\n+    # selected) has a valid value to hand to _route_capture_through_aux_vision.\n+    # The AX path appends the \"truncated to N of M\" note to summary_lines\n+    # below and rebuilds; the multimodal path keeps this version untouched.\n     summary = \"\\n\".join(summary_lines)\n \n     if cap.png_b64 and cap.mode != \"ax\":","@@ -449,6 +495,9 @@ def _capture_response(cap: CaptureResult) -> Any:\n         # JPEG: base64 starts with /9j/   PNG: starts with iVBOR\n         _b64_prefix = cap.png_b64[:8]\n         _mime = \"image/jpeg\" if _b64_prefix.startswith(\"/9j/\") else \"image/png\"\n+        # The multimodal response carries the screenshot, not the AX\n+        # elements array, so a \"response truncated to N of M elements\"\n+        # note would be inaccurate — skip it on this branch.\n         return {\n             \"_multimodal\": True,\n             \"content\": [","@@ -458,18 +507,29 @@ def _capture_response(cap: CaptureResult) -> Any:\n             ],\n             \"text_summary\": summary,\n             \"meta\": {\"mode\": cap.mode, \"width\": cap.width, \"height\": cap.height,\n-                     \"elements\": len(cap.elements), \"png_bytes\": cap.png_bytes_len},\n+                     \"elements\": total_elements, \"png_bytes\": cap.png_bytes_len},\n         }\n-    # AX-only (or image missing): text path.\n-    return json.dumps({\n+    # AX-only (or image-missing fallback): text path actually carries the\n+    # `elements` array, so the truncation note applies here.\n+    if truncated_elements:\n+        summary_lines.append(\n+            f\"  (response truncated to {len(visible_elements)} of {total_elements} elements; \"\n+            f\"raise max_elements or pass app= to narrow)\"\n+        )\n+    summary = \"\\n\".join(summary_lines)\n+    payload: Dict[str, Any] = {\n         \"mode\": cap.mode,\n         \"width\": cap.width,\n         \"height\": cap.height,\n         \"app\": cap.app,\n         \"window_title\": cap.window_title,\n-        \"elements\": [_element_to_dict(e) for e in cap.elements],\n+        \"elements\": [_element_to_dict(e) for e in visible_elements],\n+        \"total_elements\": total_elements,\n         \"summary\": summary,\n-    })\n+    }\n+    if truncated_elements:\n+        payload[\"truncated_elements\"] = truncated_elements\n+    return json.dumps(payload)\n \n \n # ---------------------------------------------------------------------------\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}