{"repo":"NousResearch/hermes-agent","public_id":"14693","display_label":"Auxiliary vision routing config not honored","summary":"Multiple issues and fixes around auxiliary.vision configuration being ignored, overridden by hardcoded defaults, or causing errors (400/401) when the main model lacks vision capability. Covers provider resolution, config.yaml precedence, and routing logic across several providers including Gemini and DeepSeek.","subsystem":"agent","counts":{"members":33,"open_issues":2,"open_issues_distinct":2,"open_prs":1,"closed_issues":12,"merged_prs":4,"attempt_prs":15},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":30126},"resolution":{"state":"partially_resolved","by":"pr-30126,pr-31452,pr-57651,pr-97075"},"freshness":{"state":"verified","last_verified_at":"2026-08-28T11:58:00+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.052575757575757574},"fix_edges":[{"issue_number":14693,"pr_number":57651,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":22865,"pr_number":30126,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":24015,"pr_number":30126,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":29407,"pr_number":30126,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":31179,"pr_number":31452,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":53749,"pr_number":57651,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":57948,"pr_number":97075,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":14693,"title":"Tool-layer env vars defeat config.yaml auxiliary routing for vision and extraction models","url":null,"state":"open"},{"kind":"issue","number":24015,"title":"cua-driver ignores auxiliary.vision config, uses main session model for image analysis","url":null,"state":"closed"},{"kind":"issue","number":24842,"title":"Bug: vision_analyze and browser_vision hardcoded to Gemma, ignore auxiliary.vision config","url":null,"state":"closed"},{"kind":"issue","number":29407,"title":"[Feature] computer_use: route screenshots through auxiliary.vision when main model lacks vision","url":null,"state":"closed"},{"kind":"issue","number":31179,"title":"vision_analyze and browser_vision ignore auxiliary.vision — route images to main model","url":null,"state":"closed"},{"kind":"issue","number":32160,"title":"Feature Request: Automatic vision fallback for non-vision primary models","url":null,"state":"closed"},{"kind":"issue","number":33389,"title":"auxiliary.vision.provider: gemini explicit configuration is not honored — falls through to main provider or fails","url":null,"state":"closed"},{"kind":"issue","number":36070,"title":"Gemini vision provider fails — `image_url` not accepted by gemini-2.0-flash[Bug]:","url":null,"state":"closed"},{"kind":"issue","number":43951,"title":"vision_analyze tool should be auto-disabled when main model lacks vision capability","url":null,"state":"open"},{"kind":"issue","number":48269,"title":"Hermes Desktop: vision_analyze ignores config.yaml — Electron main process has hardcoded env defaults","url":null,"state":"closed"},{"kind":"issue","number":50712,"title":"Vision analysis 401 for DeepSeek users — missing provider in PROVIDERS_WITHOUT_VISION","url":null,"state":"closed"},{"kind":"issue","number":53749,"title":"vision_analyze tool reads aux model from env var, not config.yaml auxiliary.vision.model","url":null,"state":"closed"},{"kind":"issue","number":57948,"title":"bug: vision_analyze first call returns 400 when main model lacks vision support","url":null,"state":"closed"},{"kind":"issue","number":58581,"title":"vision_analyze auxiliary fallback not working with DeepSeek primary model","url":null,"state":"closed"},{"kind":"pr","number":15009,"title":"fix(tools): respect config.yaml auxiliary routing in vision/browser model helpers","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":24070,"title":"Fix/computer use aux vision routing","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#30126","number":30126},{"kind":"pr","value":"#24015","number":24015},{"kind":"pr","value":"#29407","number":29407}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":24872,"title":"fix: read auxiliary.vision.model from config.yaml in vision/browser tools","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"149641485c7f5bcb33f6acf11544f0a816d8b054","number":0}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":24875,"title":"fix(vision): respect auxiliary vision routing","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#24842","number":24842},{"kind":"pr","value":"#24872","number":24872}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":30126,"title":"fix(computer_use): route SOM/vision captures via auxiliary.vision (#24015)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_computer_use_capture_routing.py","tests/tools/test_computer_use_vision_routing.py","tools/computer_use/tool.py","tools/computer_use/vision_routing.py"],"fix_hunks":[{"file":"tests/tools/test_computer_use_capture_routing.py","hunks":["@@ -0,0 +1,431 @@\n+\"\"\"End-to-end regression for #24015 — capture routing via auxiliary.vision.\n+\n+When ``computer_use(action='capture', mode='som'|'vision')`` returns a\n+screenshot, ``_capture_response`` previously always returned a\n+``_multimodal`` envelope. For non-vision main models, or when the user\n+explicitly configured ``auxiliary.vision`` in ``config.yaml``, that\n+envelope tripped HTTP 404 / 400 at the provider boundary even though a\n+perfectly good vision backend was sitting in config waiting to be used.\n+\n+This file exercises the integrated ``_capture_response`` flow with\n+deterministic stubs for:\n+\n+* ``should_route_capture_to_aux_vision`` (the policy decision)\n+* ``_run_async`` (sync->async bridge)\n+* ``vision_analyze_tool`` (the aux LLM call)\n+* ``hermes_constants.get_hermes_dir`` (cache path)\n+\n+…so the full code path is covered without a live cua-driver, a real\n+auxiliary client, or network access.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import base64\n+import json\n+import os\n+from pathlib import Path\n+from typing import Any\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fixtures / helpers\n+# ---------------------------------------------------------------------------\n+\n+# 1×1 PNG (transparent) — minimal bytes that decode cleanly.\n+_PNG_B64 = (\n+ \"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42m\"\n+ \"NkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\"\n+)\n+\n+# 1×1 JPEG — used to verify mime detection works for either stream type.\n+_JPEG_B64 = (\n+ \"/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEB\"\n+ \"AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/\"\n+)\n+\n+\n[redacted:email]\n+def tmp_cache_dir(tmp_path):\n+ \"\"\"Override get_hermes_dir so cache writes land under tmp_path.\"\"\"\n+ cache_dir = tmp_path / \"cache_vision\"\n+ cache_dir.mkdir()\n+\n+ def _fake_get(*_args, **_kw):\n+ return cache_dir\n+\n+ with patch(\"hermes_constants.get_hermes_dir\", _fake_get):\n+ yield cache_dir\n+\n+\n+def _make_capture(\n+ *,\n+ png_b64: str = _PNG_B64,\n+ mode: str = \"som\",\n+ elements=None,\n+ app: str = \"Safari\",\n+ window_title: str = \"GitHub – Issue #24015\",\n+ width: int = 1280,\n+ height: int = 800,\n+):\n+ from tools.computer_use.backend import CaptureResult, UIElement\n+\n+ elements = list(elements or [\n+ UIElement(index=0, role=\"AXButton\", label=\"Sign in\",\n+ bounds=(10, 20, 80, 30)),\n+ UIElement(index=1, role=\"AXTextField\", label=\"username\",\n+ bounds=(10, 60, 200, 24)),\n+ ])\n+ raw = base64.b64decode(png_b64, validate=False)\n+ return CaptureResult(\n+ mode=mode,\n+ width=width,\n+ height=height,\n+ png_b64=png_b64,\n+ elements=elements,\n+ app=app,\n+ window_title=window_title,\n+ png_bytes_len=len(raw),\n+ )\n+\n+\n+def _stub_aux_analysis(text: str):\n+ \"\"\"Return a fake vision_analyze_tool coroutine result (JSON envelope).\"\"\"\n+ return json.dumps({\"success\": True, \"analysis\": text})\n+\n+\n+# ---------------------------------------------------------------------------\n+# _capture_response: routing OFF (current/native behaviour)\n+# ---------------------------------------------------------------------------\n+\n+class TestCaptureResponseDefaultPath:\n+ \"\"\"When routing helper says 'native', the existing multimodal envelope wins.\"\"\"\n+\n+ def test_som_capture_returns_multimodal_envelope_when_native(self):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(png_b64=_PNG_B64, mode=\"som\")\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=False):\n+ resp = cu_tool._capture_response(cap)\n+\n+ assert isinstance(resp, dict)\n+ assert resp.get(\"_multimodal\") is True\n+ # Image part must use image/png MIME for a PNG payload.\n+ image_part = next(\n+ p for p in resp[\"content\"] if p.get(\"type\") == \"image_url\"\n+ )\n+ url = image_part[\"image_url\"][\"url\"]\n+ assert url.startswith(\"data:image/png;base64,\")\n+ assert \"vision_analysis\" not in resp\n+\n+ def test_jpeg_capture_returns_image_jpeg_mime_when_native(self):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(png_b64=_JPEG_B64, mode=\"som\")\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=False):\n+ resp = cu_tool._capture_response(cap)\n+\n+ url = next(p for p in resp[\"content\"] if p.get(\"type\") == \"image_url\")\n+ assert url[\"image_url\"][\"url\"].startswith(\"data:image/jpeg;base64,\")\n+\n+ def test_ax_only_capture_returns_text_regardless_of_routing(self):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"ax\", png_b64=\"\")\n+ # ax mode never has a PNG so neither path matters; assert pure text.\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True) as routing:\n+ resp = cu_tool._capture_response(cap)\n+\n+ # ax never even consults the routing helper — short-circuited above\n+ # the image branch.\n+ routing.assert_not_called()\n+ assert isinstance(resp, str)\n+ body = json.loads(resp)\n+ assert body[\"mode\"] == \"ax\"\n+\n+\n+# ---------------------------------------------------------------------------\n+# _capture_response: routing ON (the #24015 fix)\n+# ---------------------------------------------------------------------------\n+\n+class TestCaptureResponseRoutedToAuxVision:\n+ \"\"\"When routing helper says 'aux', the PNG is pre-analysed and a text\n+ response is returned with no image_url parts at all.\"\"\"\n+\n+ def test_som_capture_returns_text_with_vision_analysis(\n+ self, tmp_cache_dir,\n+ ):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"som\")\n+\n+ captured_calls = {}\n+\n+ def _fake_run_async(coro):\n+ captured_calls[\"called\"] = True\n+ return _stub_aux_analysis(\n+ \"A Safari window showing a GitHub issue page with a 'Sign \"\n+ \"in' button and a 'username' text field.\"\n+ )\n+\n+ # vision_analyze_tool is async; force a sync MagicMock so we can\n+ # assert positional args without dealing with awaitables.\n+ fake_vat = MagicMock(return_value=\"\")\n+\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True), \\\n+ patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+ patch(\"tools.vision_tools.vision_analyze_tool\",\n+ new_callable=lambda: fake_vat):\n+ resp = cu_tool._capture_response(cap)\n+\n+ # Must be a JSON string, NOT a multimodal envelope. This is exactly\n+ # the contract that prevents #24015's HTTP 404 from firing on the\n+ # next agent turn.\n+ assert isinstance(resp, str)\n+ body = json.loads(resp)\n+ assert body[\"mode\"] == \"som\"\n+ assert body[\"app\"] == \"Safari\"\n+ assert \"Sign in\" in body[\"vision_analysis\"]\n+ assert body[\"vision_analysis_routed_via\"] == \"auxiliary.vision\"\n+ # The original AX-only metadata (window title, element index, app)\n+ # is preserved alongside the new vision analysis so the agent loses\n+ # no context vs the multimodal path.\n+ assert body[\"window_title\"] == \"GitHub – Issue #24015\"\n+ assert len(body[\"elements\"]) == 2\n+\n+ assert captured_calls.get(\"called\") is True\n+ # vision_analyze_tool was invoked with a path under the patched cache\n+ # and a non-empty prompt.\n+ args, _kwargs = fake_vat.call_args\n+ path_arg, prompt_arg = args[0], args[1]\n+ assert str(tmp_cache_dir) in path_arg\n+ assert \"macOS application screenshot\" in prompt_arg\n+ # AX summary is included so the aux model can ground its description\n+ # against the same set-of-mark index the agent will see.\n+ assert \"Sign in\" in prompt_arg\n+\n+ def test_temp_screenshot_file_is_cleaned_up_after_routing(\n+ self, tmp_cache_dir,\n+ ):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"som\")\n+ # We capture the path the aux call sees so we can assert it's gone\n+ # after _capture_response returns.\n+ observed_path = {}\n+\n+ def _fake_run_async(_coro):\n+ return _stub_aux_analysis(\"description goes here\")\n+\n+ def _fake_vat(image_path, _prompt):\n+ observed_path[\"path\"] = image_path\n+ # File must exist while aux is being arranged.\n+ assert os.path.exists(image_path)\n+ return \"\"\n+\n+ fake_vat = MagicMock(side_effect=_fake_vat)\n+\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True), \\\n+ patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+ patch(\"tools.vision_tools.vision_analyze_tool\",\n+ new_callable=lambda: fake_vat):\n+ cu_tool._capture_response(cap)\n+\n+ # File must be unlinked after _capture_response returns.\n+ assert observed_path[\"path\"]\n+ assert not os.path.exists(observed_path[\"path\"])\n+\n+ def test_temp_file_cleaned_up_even_when_aux_call_raises(\n+ self, tmp_cache_dir,\n+ ):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"som\")\n+ observed_path = {}\n+\n+ def _fake_vat(image_path, _prompt):\n+ observed_path[\"path\"] = image_path\n+ return \"\"\n+\n+ def _fake_run_async(_coro):\n+ raise RuntimeError(\"aux LLM down\")\n+\n+ fake_vat = MagicMock(side_effect=_fake_vat)\n+\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True), \\\n+ patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+ patch(\"tools.vision_tools.vision_analyze_tool\",\n+ new_callable=lambda: fake_vat):\n+ resp = cu_tool._capture_response(cap)\n+\n+ # Aux failure → fall back to multimodal envelope (so the user still\n+ # gets *something* useful even if vision is broken).\n+ assert isinstance(resp, dict)\n+ assert resp.get(\"_multimodal\") is True\n+ # Temp file must still be cleaned up.\n+ assert observed_path[\"path\"]\n+ assert not os.path.exists(observed_path[\"path\"])\n+\n+ def test_empty_aux_analysis_falls_back_to_multimodal(self, tmp_cache_dir):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"som\")\n+\n+ def _fake_run_async(_coro):\n+ return _stub_aux_analysis(\"\")\n+\n+ fake_vat = MagicMock(return_value=\"\")\n+\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True), \\\n+ patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+ patch(\"tools.vision_tools.vision_analyze_tool\",\n+ new_callable=lambda: fake_vat):\n+ resp = cu_tool._capture_response(cap)\n+\n+ # Empty analysis is treated as failure — we'd rather show pixels\n+ # than embed an empty 'vision_analysis' string into the result.\n+ assert isinstance(resp, dict)\n+ assert resp.get(\"_multimodal\") is True\n+\n+ def test_invalid_aux_response_falls_back_to_multimodal(self, tmp_cache_dir):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"som\")\n+\n+ def _fake_run_async(_coro):\n+ return 1234 # not a string at all\n+\n+ fake_vat = MagicMock(return_value=\"\")\n+\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True), \\\n+ patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+ patch(\"tools.vision_tools.vision_analyze_tool\",\n+ new_callable=lambda: fake_vat):\n+ resp = cu_tool._capture_response(cap)\n+\n+ assert isinstance(resp, dict)\n+ assert resp.get(\"_multimodal\") is True\n+\n+\n+# ---------------------------------------------------------------------------\n+# _should_route_through_aux_vision: end-to-end with real config plumbing\n+# ---------------------------------------------------------------------------\n+\n+class TestRoutingDecisionWiring:\n+ \"\"\"Verify _should_route_through_aux_vision wires the right config + helper.\"\"\"\n+\n+ def test_explicit_aux_vision_in_config_routes_to_aux(self):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cfg = {\n+ \"model\": {\"default\": \"tencent/hy3-preview\", \"provider\": \"openrouter\"},\n+ \"auxiliary\": {\n+ \"vision\": {\n+ \"provider\": \"openrouter\",\n+ \"model\": \"google/gemini-2.5-flash\",\n+ }\n+ },\n+ }\n+ with patch(\"agent.auxiliary_client._read_main_provider\",\n+ return_value=\"openrouter\"), \\\n+ patch(\"agent.auxiliary_client._read_main_model\",\n+ return_value=\"tencent/hy3-preview\"), \\\n+ patch(\"hermes_cli.config.load_config\", return_value=cfg):\n+ assert cu_tool._should_route_through_aux_vision() is True\n+\n+ def test_no_explicit_aux_and_vision_capable_main_keeps_multimodal(self):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cfg = {\n+ \"model\": {\"default\": \"claude-opus-4-5\", \"provider\": \"anthropic\"},\n+ }\n+ with patch(\"agent.auxiliary_client._read_main_provider\",\n+ return_value=\"anthropic\"), \\\n+ patch(\"agent.auxiliary_client._read_main_model\",\n+ return_value=\"claude-opus-4-5\"), \\\n+ patch(\"hermes_cli.config.load_config\", return_value=cfg), \\\n+ patch(\"tools.computer_use.vision_routing._lookup_supports_vision\",\n+ return_value=True), \\\n+ patch(\"tools.computer_use.vision_routing.\"\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=True):\n+ assert cu_tool._should_route_through_aux_vision() is False\n+\n+ def test_config_load_failure_disables_routing_safely(self):\n+ from tools.computer_use import tool as cu_tool\n+\n+ with patch(\"hermes_cli.config.load_config\",\n+ side_effect=RuntimeError(\"config.yaml unreadable\")):\n+ # No exception should bubble up — fail open by returning False\n+ # so the legacy multimodal envelope continues to work.\n+ assert cu_tool._should_route_through_aux_vision() is False\n+\n+ def test_helper_decision_exception_is_swallowed(self):\n+ from tools.computer_use import tool as cu_tool\n+ from tools.computer_use import vision_routing as vr_mod\n+\n+ with patch(\"agent.auxiliary_client._read_main_provider\",\n+ return_value=\"openrouter\"), \\\n+ patch(\"agent.auxiliary_client._read_main_model\",\n+ return_value=\"x\"), \\\n+ patch(\"hermes_cli.config.load_config\", return_value={}), \\\n+ patch.object(vr_mod, \"should_route_capture_to_aux_vision\",\n+ side_effect=ValueError(\"policy bug\")):\n+ assert cu_tool._should_route_through_aux_vision() is False\n+\n+\n+# ---------------------------------------------------------------------------\n+# Bug reproduction marker — proves the fix is needed.\n+# ---------------------------------------------------------------------------\n+\n+class TestBugReproductionAnchor:\n+ \"\"\"Without the fix, this test would assert the wrong thing.\n+\n+ On upstream/main HEAD prior to this branch, _capture_response returns a\n+ multimodal envelope unconditionally — so when a non-vision main model\n+ is configured, the captured PNG is delivered to the main provider as\n+ image_url content and the request is rejected with HTTP 404. We don't\n+ have a live provider here, but we can pin the contract: with routing\n+ enabled the response MUST be a JSON string with no image_url parts.\n+ \"\"\"\n+\n+ def test_non_vision_main_model_never_returns_image_url_when_routed(\n+ self, tmp_cache_dir,\n+ ):\n+ from tools.computer_use import tool as cu_tool\n+\n+ cap = _make_capture(mode=\"som\")\n+\n+ def _fake_run_async(_coro):\n+ return _stub_aux_analysis(\n+ \"Screenshot showing a GitHub.com window with a sign-in \"\n+ \"form.\"\n+ )\n+\n+ fake_vat = MagicMock(return_value=\"\")\n+\n+ with patch.object(cu_tool, \"_should_route_through_aux_vision\",\n+ return_value=True), \\\n+ patch(\"model_tools._run_async\", side_effect=_fake_run_async), \\\n+ patch(\"tools.vision_tools.vision_analyze_tool\",\n+ new_callable=lambda: fake_vat):\n+ resp = cu_tool._capture_response(cap)\n+\n+ # Must be a string (text-only result).\n+ assert isinstance(resp, str)\n+ # Must NOT contain a base64 image URL anywhere — that's what tripped\n+ # 'No endpoints found that support image input' on the reporter's\n+ # main provider in #24015.\n+ assert \"data:image\" not in resp\n+ assert \"image_url\" not in resp"]},{"file":"tests/tools/test_computer_use_vision_routing.py","hunks":["@@ -0,0 +1,260 @@\n+\"\"\"Unit tests for tools.computer_use.vision_routing.\n+\n+Cover the small ``should_route_capture_to_aux_vision`` policy helper that\n+decides whether a captured screenshot from ``computer_use(action='capture')``\n+should be returned as a multimodal envelope (main model handles vision\n+natively) or pre-analysed via the ``auxiliary.vision`` pipeline so the\n+main model only sees text.\n+\n+The companion end-to-end regression for #24015 lives in\n+``tests/tools/test_computer_use_capture_routing.py``; this file pins the\n+unit contract of the helper in isolation so behaviour does not regress\n+silently if the surrounding ``computer_use`` plumbing is refactored.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+from unittest.mock import patch\n+\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# _explicit_aux_vision_override\n+# ---------------------------------------------------------------------------\n+\n+class TestExplicitAuxVisionOverride:\n+ \"\"\"Mirror agent.image_routing — config detection must agree across paths.\"\"\"\n+\n+ def test_returns_false_for_none_cfg(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ assert _explicit_aux_vision_override(None) is False\n+\n+ def test_returns_false_for_non_dict_cfg(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ assert _explicit_aux_vision_override(\"not-a-dict\") is False\n+ assert _explicit_aux_vision_override([]) is False\n+\n+ def test_returns_false_when_auxiliary_block_missing(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ assert _explicit_aux_vision_override({}) is False\n+ assert _explicit_aux_vision_override({\"model\": {\"default\": \"x\"}}) is False\n+\n+ def test_returns_false_when_vision_block_missing(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"compression\": {\"provider\": \"openai\"}}}\n+ assert _explicit_aux_vision_override(cfg) is False\n+\n+ def test_returns_false_for_blank_provider_no_model_no_base_url(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"\", \"model\": \"\", \"base_url\": \"\"}}}\n+ assert _explicit_aux_vision_override(cfg) is False\n+\n+ def test_returns_false_for_provider_auto(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"auto\"}}}\n+ assert _explicit_aux_vision_override(cfg) is False\n+\n+ def test_returns_false_for_provider_AUTO_uppercase(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": {\"provider\": \" AUTO \"}}}\n+ assert _explicit_aux_vision_override(cfg) is False\n+\n+ def test_returns_true_for_explicit_provider(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"openrouter\"}}}\n+ assert _explicit_aux_vision_override(cfg) is True\n+\n+ def test_returns_true_for_explicit_model_only(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": {\"model\": \"google/gemini-2.5-flash\"}}}\n+ assert _explicit_aux_vision_override(cfg) is True\n+\n+ def test_returns_true_for_explicit_base_url_only(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": {\"base_url\": \"http://localhost:1234/v1\"}}}\n+ assert _explicit_aux_vision_override(cfg) is True\n+\n+ def test_returns_true_for_provider_auto_plus_explicit_model(self):\n+ \"\"\"``provider: auto`` + an explicit model still counts as override.\"\"\"\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\n+ \"auxiliary\": {\n+ \"vision\": {\"provider\": \"auto\", \"model\": \"claude-3-haiku\"},\n+ }\n+ }\n+ assert _explicit_aux_vision_override(cfg) is True\n+\n+ def test_handles_non_dict_vision_block(self):\n+ from tools.computer_use.vision_routing import _explicit_aux_vision_override\n+ cfg = {\"auxiliary\": {\"vision\": \"not-a-dict\"}}\n+ assert _explicit_aux_vision_override(cfg) is False\n+\n+\n+# ---------------------------------------------------------------------------\n+# should_route_capture_to_aux_vision\n+# ---------------------------------------------------------------------------\n+\n+class TestRouteDecision:\n+ \"\"\"End-to-end policy: explicit override > tool-result support > vision caps.\"\"\"\n+\n+ def test_explicit_override_routes_to_aux_even_for_vision_main(self):\n+ \"\"\"Issue #24015 core repro: explicit aux config must win.\n+\n+ Even if the main model fully supports vision (Anthropic / Claude),\n+ an explicit ``auxiliary.vision`` block means the user wants their\n+ configured backend used. Don't silently bypass it.\n+ \"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ cfg = {\n+ \"auxiliary\": {\n+ \"vision\": {\n+ \"provider\": \"openrouter\",\n+ \"model\": \"google/gemini-2.5-flash\",\n+ }\n+ }\n+ }\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=True):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"anthropic\", \"claude-opus-4-5\", cfg\n+ ) is True\n+\n+ def test_non_vision_main_model_routes_to_aux(self):\n+ \"\"\"The reported #24015 scenario: tencent/hy3-preview has no vision.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ cfg = {\"model\": {\"default\": \"tencent/hy3-preview\", \"provider\": \"openrouter\"}}\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=False), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=True):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"openrouter\", \"tencent/hy3-preview\", cfg\n+ ) is True\n+\n+ def test_vision_main_model_no_override_keeps_multimodal(self):\n+ \"\"\"Default path: vision-capable main model + no aux override → native.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=True):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"anthropic\", \"claude-opus-4-5\", None\n+ ) is False\n+\n+ def test_provider_rejects_multimodal_tool_results_routes_to_aux(self):\n+ \"\"\"Some providers' tool-result messages won't carry images at all.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=False):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"some-aggregator\", \"some-vision-model\", {}\n+ ) is True\n+\n+ def test_unknown_provider_capabilities_fail_closed(self):\n+ \"\"\"When tool-result lookup returns None, route to aux (safe default).\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=True), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=None):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"exotic-provider\", \"exotic-model\", {}\n+ ) is True\n+\n+ def test_unknown_vision_capability_fails_closed(self):\n+ \"\"\"When models.dev has no entry, prefer aux over a likely 404.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=None), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=True):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"openrouter\", \"novel/never-seen-model\", {}\n+ ) is True\n+\n+ def test_explicit_override_wins_over_unknown_caps(self):\n+ \"\"\"Explicit aux config wins regardless of unknown caps elsewhere.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ cfg = {\"auxiliary\": {\"vision\": {\"provider\": \"openrouter\"}}}\n+ with patch.object(vision_routing, \"_lookup_supports_vision\", return_value=None), \\\n+ patch.object(vision_routing,\n+ \"_provider_accepts_multimodal_tool_result\",\n+ return_value=None):\n+ assert vision_routing.should_route_capture_to_aux_vision(\n+ \"openrouter\", \"tencent/hy3-preview\", cfg\n+ ) is True\n+\n+\n+# ---------------------------------------------------------------------------\n+# Internal lookups — defensive paths\n+# ---------------------------------------------------------------------------\n+\n+class TestLookupHelpers:\n+ def test_lookup_supports_vision_returns_none_for_blank_provider(self):\n+ from tools.computer_use.vision_routing import _lookup_supports_vision\n+ assert _lookup_supports_vision(\"\", \"claude\") is None\n+\n+ def test_lookup_supports_vision_returns_none_for_blank_model(self):\n+ from tools.computer_use.vision_routing import _lookup_supports_vision\n+ assert _lookup_supports_vision(\"anthropic\", \"\") is None\n+\n+ def test_lookup_supports_vision_handles_lookup_exception(self):\n+ \"\"\"Underlying caps lookup may raise; helper must swallow + return None.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ def _boom(_provider, _model):\n+ raise RuntimeError(\"models.dev unreachable\")\n+\n+ with patch(\"agent.models_dev.get_model_capabilities\", side_effect=_boom):\n+ assert vision_routing._lookup_supports_vision(\"anthropic\", \"claude\") is None\n+\n+ def test_lookup_supports_vision_returns_none_when_caps_missing(self):\n+ from tools.computer_use import vision_routing\n+\n+ with patch(\"agent.models_dev.get_model_capabilities\", return_value=None):\n+ assert vision_routing._lookup_supports_vision(\"anthropic\", \"claude\") is None\n+\n+ def test_provider_accepts_multimodal_tool_result_returns_none_for_blank_provider(self):\n+ from tools.computer_use.vision_routing import (\n+ _provider_accepts_multimodal_tool_result,\n+ )\n+ assert _provider_accepts_multimodal_tool_result(\"\", \"claude\") is None\n+\n+\n+# ---------------------------------------------------------------------------\n+# Module surface\n+# ---------------------------------------------------------------------------\n+\n+class TestModuleSurface:\n+ \"\"\"Pin the public surface so dependents stay in lockstep.\"\"\"\n+\n+ def test_should_route_capture_to_aux_vision_is_exported(self):\n+ from tools.computer_use import vision_routing\n+\n+ assert \"should_route_capture_to_aux_vision\" in vision_routing.__all__\n+ assert callable(vision_routing.should_route_capture_to_aux_vision)\n+\n+ @pytest.mark.parametrize(\"name\", [\n+ \"_explicit_aux_vision_override\",\n+ \"_lookup_supports_vision\",\n+ \"_provider_accepts_multimodal_tool_result\",\n+ ])\n+ def test_internal_helpers_are_addressable(self, name):\n+ \"\"\"Internal helpers stay importable so tests can monkeypatch them.\"\"\"\n+ from tools.computer_use import vision_routing\n+\n+ assert hasattr(vision_routing, name)\n+ assert callable(getattr(vision_routing, name))"]},{"file":"tools/computer_use/tool.py","hunks":["@@ -429,6 +429,21 @@ def _capture_response(cap: CaptureResult) -> Any:\n summary = \"\\n\".join(summary_lines)\n \n if cap.png_b64 and cap.mode != \"ax\":\n+ # Decide whether to hand the screenshot to the auxiliary.vision\n+ # pipeline (text-only result) or keep the multimodal envelope (main\n+ # model handles vision natively). Issue #24015: previously the\n+ # multimodal envelope was returned unconditionally, so non-vision\n+ # main models tripped HTTP 404 / 400 at the provider boundary even\n+ # when auxiliary.vision was explicitly configured to handle this.\n+ if _should_route_through_aux_vision():\n+ routed = _route_capture_through_aux_vision(cap, summary)\n+ if routed is not None:\n+ return routed\n+ # Aux routing was requested but failed (no vision client, aux\n+ # call raised, etc.). Fall through to the multimodal envelope —\n+ # better to surface a tool-result error from the main model\n+ # than to silently drop the screenshot entirely.\n+\n # Detect actual image format from base64 magic bytes so the MIME type\n # matches what the data contains (cua-driver may return JPEG or PNG).\n # JPEG: base64 starts with /9j/ PNG: starts with iVBOR","@@ -457,6 +472,140 @@ def _capture_response(cap: CaptureResult) -> Any:\n })\n \n \n+# ---------------------------------------------------------------------------\n+# auxiliary.vision routing for captured screenshots (#24015)\n+# ---------------------------------------------------------------------------\n+\n+def _should_route_through_aux_vision() -> bool:\n+ \"\"\"Return True when ``_capture_response`` should hand the PNG to aux vision.\n+\n+ Reads the active main provider/model and the loaded config and asks the\n+ routing helper. Any failure (config import, runtime override missing,\n+ etc.) returns False so the existing multimodal envelope continues to be\n+ returned — fail open on the routing decision so a broken config can\n+ never silently drop the screenshot for vision-capable main models.\n+ \"\"\"\n+ try:\n+ from agent.auxiliary_client import _read_main_model, _read_main_provider\n+ from hermes_cli.config import load_config\n+ from tools.computer_use.vision_routing import (\n+ should_route_capture_to_aux_vision,\n+ )\n+ except Exception as exc: # pragma: no cover - defensive\n+ logger.debug(\"computer_use: aux-vision routing import failed: %s\", exc)\n+ return False\n+ try:\n+ provider = _read_main_provider()\n+ model = _read_main_model()\n+ cfg = load_config()\n+ except Exception as exc: # pragma: no cover - defensive\n+ logger.debug(\"computer_use: aux-vision routing config read failed: %s\", exc)\n+ return False\n+ try:\n+ return bool(should_route_capture_to_aux_vision(provider, model, cfg))\n+ except Exception as exc: # pragma: no cover - defensive\n+ logger.debug(\"computer_use: aux-vision routing decision failed: %s\", exc)\n+ return False\n+\n+\n+def _route_capture_through_aux_vision(\n+ cap: CaptureResult,\n+ summary: str,\n+) -> Optional[str]:\n+ \"\"\"Pre-analyse the captured PNG via ``vision_analyze`` and return a text result.\n+\n+ The captured base64 PNG is materialised to ``$HERMES_HOME/cache/vision/``\n+ and handed to ``vision_analyze_tool`` with a generic describe prompt.\n+ The resulting text description is merged into the existing AX/SOM\n+ summary so the main model receives a single text payload that mentions\n+ every interactable element AND a description of what the screenshot\n+ looked like.\n+\n+ Returns:\n+ A JSON-encoded text response on success.\n+ ``None`` on failure (caller falls back to the multimodal envelope).\n+ \"\"\"\n+ if not cap.png_b64:\n+ return None\n+ try:\n+ import base64 as _base64\n+ import os as _os\n+ import uuid as _uuid\n+\n+ from hermes_constants import get_hermes_dir\n+ from model_tools import _run_async\n+ from tools.vision_tools import vision_analyze_tool\n+ except Exception as exc: # pragma: no cover - defensive\n+ logger.debug(\"computer_use: aux-vision import failed: %s\", exc)\n+ return None\n+\n+ temp_image_path = None\n+ try:\n+ try:\n+ raw = _base64.b64decode(cap.png_b64, validate=False)\n+ except Exception as exc:\n+ logger.debug(\"computer_use: failed to decode capture base64: %s\", exc)\n+ return None\n+\n+ # Pick an extension that matches the on-disk bytes so vision_analyze's\n+ # MIME sniffing returns the right content-type.\n+ ext = \".jpg\" if cap.png_b64[:8].startswith(\"/9j/\") else \".png\"\n+ cache_dir = get_hermes_dir(\"cache/vision\", \"temp_vision_images\")\n+ temp_image_path = cache_dir / f\"computer_use_{_uuid.uuid4().hex}{ext}\"\n+ temp_image_path.write_bytes(raw)\n+\n+ prompt = (\n+ \"Describe what is visible in this macOS application screenshot in \"\n+ \"concise but specific terms. Mention the app name and window \"\n+ \"title if visible, the overall layout, any labelled buttons, \"\n+ \"menus or text fields, and any prominent text content the user \"\n+ \"would need to know about. Do not invent details that are not \"\n+ \"actually visible.\\n\\n\"\n+ f\"AX/SOM index for cross-reference:\\n{summary}\"\n+ )\n+\n+ result_json = _run_async(\n+ vision_analyze_tool(str(temp_image_path), prompt)\n+ )\n+ except Exception as exc:\n+ logger.warning(\n+ \"computer_use: auxiliary.vision pre-analysis failed (%s); \"\n+ \"falling back to native multimodal envelope\",\n+ exc,\n+ )\n+ return None\n+ finally:\n+ if temp_image_path is not None:\n+ try:\n+ _os.unlink(str(temp_image_path))\n+ except Exception:\n+ pass\n+\n+ analysis_text = \"\"\n+ if isinstance(result_json, str):\n+ try:\n+ parsed = json.loads(result_json)\n+ if isinstance(parsed, dict):\n+ analysis_text = str(parsed.get(\"analysis\") or \"\").strip()\n+ except (TypeError, json.JSONDecodeError):\n+ analysis_text = result_json.strip()\n+\n+ if not analysis_text:\n+ return None\n+\n+ return json.dumps({\n+ \"mode\": cap.mode,\n+ \"width\": cap.width,\n+ \"height\": cap.height,\n+ \"app\": cap.app,\n+ \"window_title\": cap.window_title,\n+ \"elements\": [_element_to_dict(e) for e in cap.elements],\n+ \"summary\": summary,\n+ \"vision_analysis\": analysis_text,\n+ \"vision_analysis_routed_via\": \"auxiliary.vision\",\n+ })\n+\n+\n def _maybe_follow_capture(\n backend: ComputerUseBackend, res: ActionResult, do_capture: bool,\n ) -> Any:"]},{"file":"tools/computer_use/vision_routing.py","hunks":["@@ -0,0 +1,152 @@\n+\"\"\"Vision-routing decisions for ``computer_use`` capture results.\n+\n+Background\n+----------\n+``computer_use(action='capture', mode='som'|'vision')`` returns a\n+``_multimodal`` envelope containing the captured screenshot. That envelope\n+is delivered back to the **active session model** as the tool result. When\n+the active main model has no vision capability (e.g. text-only or\n+text+code-only models), or when the active provider rejects multimodal\n+content inside tool-result messages, the screenshot trips a 404 / 400 at\n+the provider boundary and the agent loop reports a hard tool failure.\n+\n+Issue #24015 reports this regression for the ``cua-driver`` backend:\n+configuring ``auxiliary.vision`` (a dedicated vision-capable model) in\n+``config.yaml`` was silently ignored — the screenshot was still routed at\n+the *main* model and failed with HTTP 404 ``No endpoints found that\n+support image input`` even though a perfectly good vision backend was\n+sitting in config waiting to be used.\n+\n+This module centralises the small policy decision: should a captured\n+screenshot be returned as multimodal content (main model handles vision\n+natively) or pre-analysed via the auxiliary vision pipeline so the main\n+model only ever sees text?\n+\n+Behaviour (mirrors ``vision_analyze`` for consistency)\n+------------------------------------------------------\n+* If the user explicitly configured ``auxiliary.vision`` (any of\n+ ``provider``, ``model``, or ``base_url`` non-empty / not ``\"auto\"``),\n+ the screenshot is routed through the aux vision pipeline. Users who\n+ pay for a dedicated vision model usually want it used.\n+* Otherwise, if the active main model+provider can carry an image inside\n+ a tool-result message AND the model reports ``supports_vision=True``\n+ in models.dev metadata, return ``False`` (use the multimodal path).\n+* In every other case (non-vision main model, provider that does not\n+ accept multimodal tool results, lookup failure), route through aux\n+ vision so the main model receives a text description it can act on.\n+\n+The decision intentionally fails *closed* (i.e. towards aux routing) when\n+metadata is missing or ambiguous: returning a screenshot to a model that\n+cannot read it is a hard tool failure, while routing it through aux costs\n+one extra LLM call and yields a usable description.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import logging\n+from typing import Any, Dict, Optional\n+\n+logger = logging.getLogger(__name__)\n+\n+\n+def _explicit_aux_vision_override(cfg: Optional[Dict[str, Any]]) -> bool:\n+ \"\"\"True when ``auxiliary.vision`` carries a non-default user override.\n+\n+ Mirrors ``agent.image_routing._explicit_aux_vision_override`` so the\n+ capture path and the user-attached-image path agree on what counts as\n+ an explicit user request for the aux vision pipeline. ``provider:\n+ \"auto\"``, blank values, or a missing block all count as *not*\n+ explicit.\n+ \"\"\"\n+ if not isinstance(cfg, dict):\n+ return False\n+ aux = cfg.get(\"auxiliary\") or {}\n+ if not isinstance(aux, dict):\n+ return False\n+ vision = aux.get(\"vision\") or {}\n+ if not isinstance(vision, dict):\n+ return False\n+\n+ provider = str(vision.get(\"provider\") or \"\").strip().lower()\n+ model = str(vision.get(\"model\") or \"\").strip()\n+ base_url = str(vision.get(\"base_url\") or \"\").strip()\n+\n+ if provider in (\"\", \"auto\") and not model and not base_url:\n+ return False\n+ return True\n+\n+\n+def _lookup_supports_vision(provider: str, model: str) -> Optional[bool]:\n+ \"\"\"Return models.dev ``supports_vision`` for *(provider, model)* or None.\"\"\"\n+ if not provider or not model:\n+ return None\n+ try:\n+ from agent.models_dev import get_model_capabilities\n+ caps = get_model_capabilities(provider, model)\n+ except Exception as exc: # pragma: no cover - defensive\n+ logger.debug(\n+ \"computer_use vision_routing: caps lookup failed for %s:%s — %s\",\n+ provider, model, exc,\n+ )\n+ return None\n+ if caps is None:\n+ return None\n+ return bool(getattr(caps, \"supports_vision\", False))\n+\n+\n+def _provider_accepts_multimodal_tool_result(provider: str, model: str) -> Optional[bool]:\n+ \"\"\"Return whether *provider*+*model* carries images inside tool-result messages.\n+\n+ Reuses ``tools.vision_tools._supports_media_in_tool_results`` so the\n+ capture-routing decision stays in lockstep with the\n+ ``vision_analyze`` native fast path. Returns None on import failure\n+ so callers fall back to aux routing rather than guessing.\n+ \"\"\"\n+ if not provider:\n+ return None\n+ try:\n+ from tools.vision_tools import _supports_media_in_tool_results\n+ except Exception as exc: # pragma: no cover - defensive\n+ logger.debug(\n+ \"computer_use vision_routing: tool-result support lookup failed: %s\",\n+ exc,\n+ )\n+ return None\n+ return bool(_supports_media_in_tool_results(provider, model))\n+\n+\n+def should_route_capture_to_aux_vision(\n+ provider: str,\n+ model: str,\n+ cfg: Optional[Dict[str, Any]],\n+) -> bool:\n+ \"\"\"Return True iff the captured screenshot should be pre-analysed via aux vision.\n+\n+ Args:\n+ provider: active inference provider id (e.g. ``\"openrouter\"``,\n+ ``\"anthropic\"``, ``\"openai-codex\"``). Lower-case canonical id.\n+ model: active main model slug as it would be sent to the provider.\n+ cfg: loaded ``config.yaml`` dict (or None).\n+\n+ Returns:\n+ ``True`` when the caller should hand the screenshot to the aux vision\n+ pipeline (and surface a text-only tool result). ``False`` when the\n+ caller should keep the existing multimodal envelope (main model\n+ handles vision natively).\n+ \"\"\"\n+ if _explicit_aux_vision_override(cfg):\n+ return True\n+\n+ accepts_tool_image = _provider_accepts_multimodal_tool_result(provider, model)\n+ if accepts_tool_image is None or accepts_tool_image is False:\n+ return True\n+\n+ supports_vision = _lookup_supports_vision(provider, model)\n+ if supports_vision is True:\n+ return False\n+ return True\n+\n+\n+__all__ = [\n+ \"should_route_capture_to_aux_vision\",\n+]\n"]}]},{"kind":"pr","number":31228,"title":"fix(vision): respect auxiliary.vision config so images stop silently routing to text-only main models","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3d66787a04d29beaec16a723e4c2c2bb4dda2cee","number":0},{"kind":"pr","value":"#31452","number":31452},{"kind":"pr","value":"#31452","number":31452},{"kind":"pr","value":"#31179","number":31179},{"kind":"pr","value":"#31179","number":31179},{"kind":"pr","value":"#31452","number":31452}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":31452,"title":"fix(vision): route auxiliary.vision.provider=openai to api.openai.com, skip text-only main","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/auxiliary_client.py","tests/agent/test_vision_routing_31179.py","tools/browser_tool.py","tools/vision_tools.py"],"fix_hunks":[{"file":"agent/auxiliary_client.py","hunks":["@@ -3730,6 +3730,37 @@ def get_async_text_auxiliary_client(task: str = \"\", *, main_runtime: Optional[Di\n )\n \n \n+def _main_model_supports_vision(provider: str, model: Optional[str]) -> bool:\n+ \"\"\"Return True when ``provider``/``model`` is known to accept image input.\n+\n+ Used by the vision auto-detect chain to skip the user's main provider\n+ when it's known to be text-only (e.g. DeepSeek, gpt-oss without vision).\n+ Without this guard, ``resolve_vision_provider_client(provider=\"auto\")``\n+ would happily return the main-provider client and any subsequent image\n+ payload would surface as a cryptic provider-side error\n+ (``unknown variant `image_url`, expected `text```, #31179).\n+\n+ Returns True when capability lookup is unknown — preserves the historical\n+ behaviour of attempting the call, so providers we haven't catalogued yet\n+ don't silently regress to text-only.\n+ \"\"\"\n+ try:\n+ from agent.image_routing import _lookup_supports_vision\n+ from hermes_cli.config import load_config\n+ except ImportError:\n+ return True\n+ try:\n+ supports = _lookup_supports_vision(provider, model, load_config())\n+ except Exception: # pragma: no cover - defensive\n+ return True\n+ if supports is None:\n+ # No capability data — keep current behaviour and let the call attempt\n+ # happen rather than silently skipping. This avoids false-positive\n+ # skips for new/custom providers.\n+ return True\n+ return bool(supports)\n+\n+\n def _normalize_vision_provider(provider: Optional[str]) -> str:\n return _normalize_aux_provider(provider)\n ","@@ -3870,6 +3901,23 @@ def _finalize(resolved_provider: str, sync_client: Any, default_model: Optional[\n \"vision support) — falling through to aggregator chain\",\n main_provider,\n )\n+ elif not _main_model_supports_vision(main_provider, vision_model):\n+ # The main model is known to be text-only (e.g. DeepSeek V4,\n+ # gpt-oss-120b without vision). Building a client and sending\n+ # an image would produce a cryptic provider-side error like\n+ # ``unknown variant `image_url`, expected `text``` (#31179).\n+ # Fall through to the aggregator chain instead.\n+ #\n+ # Only log the provider name (not the model) — mirrors the\n+ # sibling _PROVIDERS_WITHOUT_VISION branch above, and avoids\n+ # CodeQL py/clear-text-logging-sensitive-data heuristic false\n+ # positives on multi-value interpolations.\n+ logger.debug(\n+ \"Vision auto-detect: skipping main provider %s \"\n+ \"(reports no vision capability) — falling through to \"\n+ \"aggregator chain\",\n+ main_provider,\n+ )\n else:\n rpc_client, rpc_model = resolve_provider_client(\n main_provider, vision_model,","@@ -4281,6 +4329,23 @@ def _get_cached_client(\n return client, model or default_model\n \n \n+# Aliases that target direct REST APIs not modeled as first-class providers\n+# in PROVIDER_REGISTRY. Used for ``auxiliary..provider`` so users can\n+# write the obvious name and have it resolve to a working ``custom`` endpoint\n+# without needing to know our internal provider IDs.\n+#\n+# Why these specifically: PROVIDER_REGISTRY has ``openai-codex`` (OAuth) and\n+# ``custom`` (manual base_url + OPENAI_API_KEY) but no plain ``openai`` for\n+# direct API-key access. Users predictably type ``provider: openai`` and\n+# expect it to use OPENAI_API_KEY against api.openai.com. Previously this\n+# silently fell back to the user's main provider, sending OpenAI model names\n+# to e.g. DeepSeek and producing cryptic ``unknown variant 'image_url'``\n+# errors (issue #31179).\n+_AUX_DIRECT_API_BASE_URLS: Dict[str, str] = {\n+ \"openai\": \"https://api.openai.com/v1\",\n+}\n+\n+\n def _resolve_task_provider_model(\n task: str = None,\n provider: str = None,","@@ -4317,6 +4382,25 @@ def _resolve_task_provider_model(\n resolved_model = model or cfg_model\n resolved_api_mode = cfg_api_mode\n \n+ # Convenience aliases for direct API-key endpoints that aren't first-class\n+ # providers (e.g. ``provider: openai`` → custom + api.openai.com/v1).\n+ # Applied to both explicit args and config-derived values. When the user\n+ # has already supplied a base_url we keep their endpoint but still rewrite\n+ # the provider to ``custom`` so resolution doesn't hit the\n+ # PROVIDER_REGISTRY-only path (which has no ``openai`` entry).\n+ def _expand_direct_api_alias(prov: Optional[str], existing_base: Optional[str]) -> Tuple[Optional[str], Optional[str]]:\n+ if not prov:\n+ return prov, existing_base\n+ target_base = _AUX_DIRECT_API_BASE_URLS.get(prov.strip().lower())\n+ if target_base is None:\n+ return prov, existing_base\n+ return \"custom\", existing_base or target_base\n+\n+ if provider:\n+ provider, base_url = _expand_direct_api_alias(provider, base_url)\n+ if cfg_provider:\n+ cfg_provider, cfg_base_url = _expand_direct_api_alias(cfg_provider, cfg_base_url)\n+\n if base_url:\n return \"custom\", resolved_model, base_url, api_key, resolved_api_mode\n if provider:"]},{"file":"tests/agent/test_vision_routing_31179.py","hunks":["@@ -0,0 +1,297 @@\n+\"\"\"Regression tests for issue #31179.\n+\n+Before the fix:\n+ - ``auxiliary.vision.provider: openai`` silently failed to resolve because\n+ ``openai`` is not a first-class provider in PROVIDER_REGISTRY (only\n+ ``openai-codex`` for OAuth and ``custom`` for OPENAI_BASE_URL).\n+ - The vision branch of ``call_llm`` then silently fell back to ``auto``\n+ which happily picked the user's main provider (e.g. DeepSeek), sending\n+ image content to a text-only endpoint and producing cryptic\n+ ``unknown variant 'image_url', expected 'text'`` errors.\n+ - ``check_vision_requirements`` used the explicit-only path, so\n+ ``vision_analyze`` disappeared from the tool list while ``browser_vision``\n+ stayed (its check_fn only validated the browser).\n+\n+The three fixes covered here:\n+ 1. ``provider: openai`` in auxiliary task config resolves to\n+ ``custom`` + ``https://api.openai.com/v1``.\n+ 2. The vision auto-detect chain skips the user's main provider when it\n+ reports ``supports_vision=False`` instead of routing image content to\n+ a text-only endpoint.\n+ 3. ``check_vision_requirements`` mirrors the runtime fallback chain so\n+ ``vision_analyze`` shows up whenever the auto chain can serve vision,\n+ and ``browser_vision`` gates on vision availability as well.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import os\n+import shutil\n+import sys\n+import tempfile\n+\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# Test infrastructure\n+# ---------------------------------------------------------------------------\n+\n+\n[redacted:email]\n+def isolated_home(monkeypatch):\n+ \"\"\"Temp HERMES_HOME with config + clean credential env vars.\"\"\"\n+ test_home = tempfile.mkdtemp(prefix=\"hermes_test_31179_\")\n+ hermes_home = os.path.join(test_home, \".hermes\")\n+ os.makedirs(hermes_home)\n+ monkeypatch.setenv(\"HERMES_HOME\", hermes_home)\n+\n+ # Strip all credential-shaped env vars so each scenario starts hermetic.\n+ for k in list(os.environ.keys()):\n+ if k.endswith(\"_API_KEY\") or k.endswith(\"_TOKEN\"):\n+ monkeypatch.delenv(k, raising=False)\n+\n+ yield hermes_home\n+ shutil.rmtree(test_home, ignore_errors=True)\n+\n+\n+def _write_config(home: str, text: str) -> None:\n+ with open(os.path.join(home, \"config.yaml\"), \"w\") as fp:\n+ fp.write(text)\n+\n+\n+def _fresh_modules():\n+ \"\"\"Drop cached hermes modules so each test reloads against current env.\"\"\"\n+ for mod in list(sys.modules.keys()):\n+ if mod.startswith((\"agent.auxiliary_client\", \"agent.image_routing\",\n+ \"tools.vision_tools\", \"tools.browser_tool\",\n+ \"hermes_cli.config\")):\n+ del sys.modules[mod]\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fix 1: provider=openai → custom + api.openai.com/v1\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestOpenAiAliasForAuxiliary:\n+ \"\"\"``auxiliary..provider: openai`` should produce a working client.\"\"\"\n+\n+ def test_provider_openai_routes_to_openai_dot_com(self, isolated_home, monkeypatch):\n+ _write_config(isolated_home, \"\"\"\n+auxiliary:\n+ vision:\n+ provider: openai\n+ model: gpt-4o-mini\n+\"\"\")\n+ monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ from agent.auxiliary_client import _resolve_task_provider_model\n+ provider, model, base_url, _key, _mode = _resolve_task_provider_model(\"vision\")\n+ assert provider == \"custom\"\n+ assert model == \"gpt-4o-mini\"\n+ assert base_url == \"https://api.openai.com/v1\"\n+\n+ def test_provider_openai_with_explicit_base_url_preserves_user_endpoint(\n+ self, isolated_home, monkeypatch\n+ ):\n+ \"\"\"User-supplied base_url wins; alias still normalizes provider name\n+ to ``custom`` so resolution doesn't hit the unknown-provider path.\"\"\"\n+ _write_config(isolated_home, \"\"\"\n+auxiliary:\n+ vision:\n+ provider: openai\n+ model: gpt-4o-mini\n+ base_url: https://my-proxy.example.com/v1\n+\"\"\")\n+ monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ from agent.auxiliary_client import _resolve_task_provider_model\n+ provider, _model, base_url, _key, _mode = _resolve_task_provider_model(\"vision\")\n+ assert provider == \"custom\"\n+ assert base_url == \"https://my-proxy.example.com/v1\"\n+\n+ def test_provider_openai_resolves_to_working_client(self, isolated_home, monkeypatch):\n+ \"\"\"End-to-end: the resolved client points at api.openai.com.\"\"\"\n+ _write_config(isolated_home, \"\"\"\n+auxiliary:\n+ vision:\n+ provider: openai\n+ model: gpt-4o-mini\n+\"\"\")\n+ monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ from agent.auxiliary_client import resolve_vision_provider_client\n+ from urllib.parse import urlparse\n+ provider, client, model = resolve_vision_provider_client()\n+ assert client is not None, \"openai alias should produce a usable client\"\n+ # Exact hostname comparison (not substring) — defends against URLs\n+ # like ``api.openai.com.evil.example`` and keeps CodeQL happy.\n+ host = urlparse(str(getattr(client, \"base_url\", \"\"))).hostname or \"\"\n+ assert host == \"api.openai.com\", f\"expected api.openai.com host, got {host!r}\"\n+ assert model == \"gpt-4o-mini\"\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fix 2: auto chain skips text-only main providers\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestTextOnlyMainSkippedForVision:\n+ \"\"\"Vision auto-detect must not return a text-only main-provider client.\"\"\"\n+\n+ def test_text_only_main_skipped_when_no_aggregator(self, isolated_home, monkeypatch):\n+ \"\"\"DeepSeek main + no aggregator credentials → no client built.\n+\n+ Pre-fix this silently returned the deepseek client with model\n+ substitution, producing ``unknown variant 'image_url'`` at call time.\n+ \"\"\"\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: deepseek\n+ default: deepseek-v4-pro\n+\"\"\")\n+ monkeypatch.setenv(\"DEEPSEEK_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ from agent.auxiliary_client import resolve_vision_provider_client\n+ provider, client, _model = resolve_vision_provider_client(provider=\"auto\")\n+ assert client is None, (\n+ f\"Vision auto-detect must skip text-only main {provider!r} when \"\n+ \"no vision-capable aggregator is available, not return a client \"\n+ \"that will fail at API time\"\n+ )\n+\n+ def test_vision_capable_main_used(self, isolated_home, monkeypatch):\n+ \"\"\"Vision-capable main provider should be returned by auto chain.\"\"\"\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: anthropic\n+ default: claude-sonnet-4-6\n+\"\"\")\n+ monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"sk-ant-test\")\n+ _fresh_modules()\n+\n+ from agent.auxiliary_client import resolve_vision_provider_client\n+ provider, client, _model = resolve_vision_provider_client(provider=\"auto\")\n+ assert client is not None\n+ assert provider == \"anthropic\"\n+\n+ def test_unknown_capability_does_not_block(self, isolated_home, monkeypatch):\n+ \"\"\"When models.dev has no entry, fall back to permissive (attempt the call).\n+\n+ This keeps new/custom providers working — only providers we have\n+ cataloged as text-only are skipped.\n+ \"\"\"\n+ _fresh_modules()\n+ from agent.auxiliary_client import _main_model_supports_vision\n+ # Bogus provider/model — capability lookup returns None → permissive.\n+ assert _main_model_supports_vision(\"nonexistent-provider\", \"nonexistent-model\") is True\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fix 3: check_vision_requirements + check_browser_vision_requirements parity\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestVisionToolGating:\n+ \"\"\"Tool visibility must match runtime capability.\"\"\"\n+\n+ def test_check_vision_succeeds_for_aliased_openai(self, isolated_home, monkeypatch):\n+ \"\"\"The user's exact reported scenario: provider=openai unhides\n+ vision_analyze instead of silently dropping it.\"\"\"\n+ _write_config(isolated_home, \"\"\"\n+auxiliary:\n+ vision:\n+ provider: openai\n+ model: gpt-4o-mini\n+\"\"\")\n+ monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ from tools.vision_tools import check_vision_requirements\n+ assert check_vision_requirements() is True\n+\n+ def test_check_vision_falls_back_to_auto(self, isolated_home, monkeypatch):\n+ \"\"\"Bad explicit provider doesn't hide the tool when auto fallback works.\n+\n+ Mirrors call_llm's runtime fallback chain.\n+ \"\"\"\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: openrouter\n+ default: anthropic/claude-sonnet-4\n+auxiliary:\n+ vision:\n+ provider: not-a-real-provider\n+\"\"\")\n+ monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"sk-or-test\")\n+ _fresh_modules()\n+\n+ from tools.vision_tools import check_vision_requirements\n+ assert check_vision_requirements() is True\n+\n+ def test_check_vision_false_with_text_only_main_and_no_aggregator(\n+ self, isolated_home, monkeypatch\n+ ):\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: deepseek\n+ default: deepseek-v4-pro\n+\"\"\")\n+ monkeypatch.setenv(\"DEEPSEEK_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ from tools.vision_tools import check_vision_requirements\n+ assert check_vision_requirements() is False\n+\n+ def test_browser_vision_requires_both_browser_and_vision(self, isolated_home, monkeypatch):\n+ \"\"\"``browser_vision`` must not be advertised when vision is unavailable.\"\"\"\n+ from unittest.mock import patch\n+\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: deepseek\n+ default: deepseek-v4-pro\n+\"\"\")\n+ monkeypatch.setenv(\"DEEPSEEK_API_KEY\", \"sk-test\")\n+ _fresh_modules()\n+\n+ import tools.browser_tool\n+ # Force the browser side to True so we exercise the vision-gating part.\n+ with patch.object(tools.browser_tool, \"check_browser_requirements\", return_value=True):\n+ assert tools.browser_tool.check_browser_vision_requirements() is False\n+\n+ def test_browser_vision_false_when_browser_missing(self, isolated_home, monkeypatch):\n+ from unittest.mock import patch\n+\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: openrouter\n+ default: anthropic/claude-sonnet-4\n+\"\"\")\n+ monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"sk-or-test\")\n+ _fresh_modules()\n+\n+ import tools.browser_tool\n+ with patch.object(tools.browser_tool, \"check_browser_requirements\", return_value=False):\n+ # Vision available but browser missing → still False.\n+ assert tools.browser_tool.check_browser_vision_requirements() is False\n+\n+ def test_browser_vision_true_when_both_available(self, isolated_home, monkeypatch):\n+ from unittest.mock import patch\n+\n+ _write_config(isolated_home, \"\"\"\n+model:\n+ provider: openrouter\n+ default: anthropic/claude-sonnet-4\n+\"\"\")\n+ monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"sk-or-test\")\n+ _fresh_modules()\n+\n+ import tools.browser_tool\n+ with patch.object(tools.browser_tool, \"check_browser_requirements\", return_value=True):\n+ assert tools.browser_tool.check_browser_vision_requirements() is True"]},{"file":"tools/browser_tool.py","hunks":["@@ -3652,6 +3652,24 @@ def check_browser_requirements() -> bool:\n return True\n \n \n+def check_browser_vision_requirements() -> bool:\n+ \"\"\"Whether ``browser_vision`` should be advertised to the model.\n+\n+ Requires BOTH a working browser (``check_browser_requirements``) AND a\n+ resolvable vision backend. Without the vision check, the tool stays in\n+ the model's tool list even when no vision provider is configured, then\n+ fails at call time with a cryptic provider-side error like\n+ ``unknown variant `image_url`, expected `text``` (issue #31179).\n+ \"\"\"\n+ if not check_browser_requirements():\n+ return False\n+ try:\n+ from tools.vision_tools import check_vision_requirements\n+ except ImportError:\n+ return False\n+ return check_vision_requirements()\n+\n+\n # ============================================================================\n # Module Test\n # ============================================================================","@@ -3786,7 +3804,7 @@ def check_browser_requirements() -> bool:\n toolset=\"browser\",\n schema=_BROWSER_SCHEMA_MAP[\"browser_vision\"],\n handler=lambda args, **kw: browser_vision(question=args.get(\"question\", \"\"), annotate=args.get(\"annotate\", False), task_id=kw.get(\"task_id\")),\n- check_fn=check_browser_requirements,\n+ check_fn=check_browser_vision_requirements,\n emoji=\"👁️\",\n )\n registry.register("]},{"file":"tools/vision_tools.py","hunks":["@@ -914,11 +914,26 @@ async def vision_analyze_tool(\n \n \n def check_vision_requirements() -> bool:\n- \"\"\"Check if the configured runtime vision path can resolve a client.\"\"\"\n+ \"\"\"Check if the configured runtime vision path can resolve a client.\n+\n+ Mirrors the fallback chain that ``call_llm(task=\"vision\")`` actually uses\n+ at runtime: first the explicit ``auxiliary.vision.provider`` (if any),\n+ and if that fails, the auto chain (main provider → openrouter → nous).\n+ Without the auto-fallback step the tool would disappear from the model's\n+ tool list whenever the explicit provider name was unresolvable, even\n+ when the auto chain would have served the request (issue #31179).\n+ \"\"\"\n try:\n from agent.auxiliary_client import resolve_vision_provider_client\n-\n+ except ImportError:\n+ return False\n+ try:\n _provider, client, _model = resolve_vision_provider_client()\n+ if client is not None:\n+ return True\n+ # Same fallback to \"auto\" that call_llm performs when the configured\n+ # provider can't be resolved.\n+ _provider, client, _model = resolve_vision_provider_client(provider=\"auto\")\n return client is not None\n except Exception:\n return False\n"]}]},{"kind":"pr","number":33427,"title":"fix(vision): honor explicit gemini provider in resolve_vision_provider_client","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a8841e2a6877f7b5e7b4b6ab0122ef2fde229fe2","number":0},{"kind":"pr","value":"#33389","number":33389}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":39825,"title":"fix(gemini): translate Cloud Code image parts","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"7130d60861a9243301514bff611a9381830d59d8","number":0},{"kind":"commit","value":"a8841e2a6877f7b5e7b4b6ab0122ef2fde229fe2","number":0},{"kind":"pr","value":"#36070","number":36070}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":39830,"title":"fix(tools): preserve auxiliary config model precedence","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#57651","number":57651}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":41420,"title":"fix: route vision to Gemini OpenAI-compatible endpoint","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a8841e2a6877f7b5e7b4b6ab0122ef2fde229fe2","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":44029,"title":"feat(agent): auto-disable vision toolset for non-vision models","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#43951","number":43951}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":50725,"title":"fix(auxiliary): add deepseek to _PROVIDERS_WITHOUT_VISION to prevent 401 on vision requests (#50712)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3d66787a0","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":53754,"title":"fix(vision): read auxiliary model from config.yaml before env var","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#57651","number":57651},{"kind":"pr","value":"#50685","number":50685}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":57651,"title":"fix(vision): forward custom-endpoint credentials + read aux model from config.yaml","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/auxiliary_client.py","scripts/release.py","tests/agent/test_auxiliary_main_first.py","tests/tools/test_vision_tools.py","tools/vision_tools.py"],"fix_hunks":[{"file":"agent/auxiliary_client.py","hunks":["@@ -4940,9 +4940,35 @@ def _finalize(resolved_provider: str, sync_client: Any, default_model: Optional[\n main_provider,\n )\n else:\n+ # Custom endpoints (``custom`` / ``custom:``) carry no\n+ # built-in base_url/api_key — resolve_provider_client(\"custom\")\n+ # would return None (\"no endpoint credentials found\") and the\n+ # whole chain would fall through to the aggregators, breaking\n+ # vision for every user on a custom provider that has no\n+ # separate ``auxiliary.vision`` block. Recover the live main\n+ # endpoint that ``set_runtime_main()`` recorded for this turn so\n+ # Step 1 can build a working client.\n+ rpc_base_url = None\n+ rpc_api_key = None\n+ rpc_api_mode = resolved_api_mode\n+ if main_provider == \"custom\" or main_provider.startswith(\"custom:\"):\n+ if _RUNTIME_MAIN_BASE_URL:\n+ rpc_base_url = _RUNTIME_MAIN_BASE_URL\n+ rpc_api_key = [redacted:secret-assignment] or None\n+ rpc_api_mode = resolved_api_mode or _RUNTIME_MAIN_API_MODE or None\n+ else:\n+ # No live runtime recorded (non-gateway caller): fall\n+ # back to resolving the configured custom endpoint.\n+ custom_base, custom_key, custom_mode = _resolve_custom_runtime()\n+ if custom_base:\n+ rpc_base_url = custom_base\n+ rpc_api_key = [redacted:secret-assignment]\n+ rpc_api_mode = resolved_api_mode or custom_mode or None\n rpc_client, rpc_model = resolve_provider_client(\n main_provider, vision_model,\n- api_mode=resolved_api_mode,\n+ api_mode=rpc_api_mode,\n+ explicit_base_url=rpc_base_url,\n+ explicit_api_key=[redacted:secret-assignment]\n is_vision=True)\n if rpc_client is not None:\n logger.info("]},{"file":"scripts/release.py","hunks":["@@ -1388,6 +1388,7 @@\n \"[redacted:email]\": \"holynn-q\",\n \"[redacted:email]\": \"jacdevos\",\n \"[redacted:email]\": \"liuhao1024\",\n+ \"[redacted:email]\": \"suninrain086\", # PR #57651 salvage of #50685 (vision custom-endpoint creds)\n \"[redacted:email]\": \"mohamedorigami-jpg\", # PR #32117 (cron storage root anchor; #32091)\n \"[redacted:email]\": \"sherman-yang\", # PR #32788 (cron per-job MCP merge; #23997)\n \"[redacted:email]\": \"rbrtbn\","]},{"file":"tests/agent/test_auxiliary_main_first.py","hunks":["@@ -543,6 +543,124 @@ def test_explicit_provider_override_still_wins(self):\n mock_strict.assert_called_once_with(\"nous\", None)\n \n \n+# ── Vision — custom provider endpoint credential passthrough ────────────────\n+\n+\n+class TestResolveVisionCustomProvider:\n+ \"\"\"Custom-endpoint mains must forward base_url/api_key to Step 1.\n+\n+ Regression: a ``custom:`` main provider resolves to the bare\n+ runtime provider id ``\"custom\"``. ``resolve_provider_client(\"custom\")``\n+ has no built-in endpoint, so without forwarding the live base_url/api_key\n+ it returns ``(None, None)`` and vision falls through to OpenRouter / Nous,\n+ which an offline / aggregator-less user has never configured — breaking\n+ vision entirely with ``No LLM provider configured for task=vision\n+ provider=auto``. The fix recovers the live endpoint that\n+ ``set_runtime_main()`` recorded for the turn.\n+ \"\"\"\n+\n+ def test_custom_main_forwards_runtime_endpoint(self, monkeypatch):\n+ \"\"\"custom main with recorded runtime endpoint → Step 1 builds a client.\"\"\"\n+ import agent.auxiliary_client as aux\n+\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_BASE_URL\", \"https://my.endpoint.example/v1\")\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_API_KEY\", \"sk-runtime-key\")\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_API_MODE\", \"anthropic_messages\")\n+\n+ with patch(\n+ \"agent.auxiliary_client._read_main_provider\", return_value=\"custom\",\n+ ), patch(\n+ \"agent.auxiliary_client._read_main_model\", return_value=\"claude-opus-4-8\",\n+ ), patch(\n+ \"agent.auxiliary_client._resolve_task_provider_model\",\n+ return_value=(\"auto\", None, None, None, None),\n+ ), patch(\n+ \"agent.auxiliary_client.resolve_provider_client\"\n+ ) as mock_resolve:\n+ mock_client = MagicMock()\n+ mock_resolve.return_value = (mock_client, \"claude-opus-4-8\")\n+\n+ from agent.auxiliary_client import resolve_vision_provider_client\n+\n+ provider, client, model = resolve_vision_provider_client()\n+\n+ assert provider == \"custom\"\n+ assert client is mock_client\n+ assert model == \"claude-opus-4-8\"\n+ # The endpoint credentials recorded for the turn MUST be forwarded,\n+ # otherwise resolve_provider_client(\"custom\") returns (None, None).\n+ kwargs = mock_resolve.call_args.kwargs\n+ assert kwargs.get(\"explicit_base_url\") == \"https://my.endpoint.example/v1\"\n+ assert kwargs.get(\"explicit_api_key\") == \"sk-runtime-key\"\n+ assert kwargs.get(\"is_vision\") is True\n+\n+ def test_custom_prefixed_main_forwards_runtime_endpoint(self, monkeypatch):\n+ \"\"\"A ``custom:`` provider id also forwards the runtime endpoint.\"\"\"\n+ import agent.auxiliary_client as aux\n+\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_BASE_URL\", \"https://named.example/v1\")\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_API_KEY\", \"sk-named\")\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_API_MODE\", \"\")\n+\n+ with patch(\n+ \"agent.auxiliary_client._read_main_provider\",\n+ return_value=\"custom:copilot-gateway\",\n+ ), patch(\n+ \"agent.auxiliary_client._read_main_model\", return_value=\"claude-opus-4-8\",\n+ ), patch(\n+ \"agent.auxiliary_client._resolve_task_provider_model\",\n+ return_value=(\"auto\", None, None, None, None),\n+ ), patch(\n+ \"agent.auxiliary_client.resolve_provider_client\"\n+ ) as mock_resolve:\n+ mock_client = MagicMock()\n+ mock_resolve.return_value = (mock_client, \"claude-opus-4-8\")\n+\n+ from agent.auxiliary_client import resolve_vision_provider_client\n+\n+ provider, client, model = resolve_vision_provider_client()\n+\n+ assert provider == \"custom:copilot-gateway\"\n+ assert client is mock_client\n+ kwargs = mock_resolve.call_args.kwargs\n+ assert kwargs.get(\"explicit_base_url\") == \"https://named.example/v1\"\n+ assert kwargs.get(\"explicit_api_key\") == \"sk-named\"\n+ assert kwargs.get(\"is_vision\") is True\n+\n+ def test_custom_main_no_runtime_falls_back_to_configured_endpoint(self, monkeypatch):\n+ \"\"\"No recorded runtime endpoint → resolve the configured custom endpoint.\"\"\"\n+ import agent.auxiliary_client as aux\n+\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_BASE_URL\", \"\")\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_API_KEY\", \"\")\n+ monkeypatch.setattr(aux, \"_RUNTIME_MAIN_API_MODE\", \"\")\n+\n+ with patch(\n+ \"agent.auxiliary_client._read_main_provider\", return_value=\"custom\",\n+ ), patch(\n+ \"agent.auxiliary_client._read_main_model\", return_value=\"claude-opus-4-8\",\n+ ), patch(\n+ \"agent.auxiliary_client._resolve_task_provider_model\",\n+ return_value=(\"auto\", None, None, None, None),\n+ ), patch(\n+ \"agent.auxiliary_client._resolve_custom_runtime\",\n+ return_value=(\"https://configured.example/v1\", \"sk-configured\", \"chat_completions\"),\n+ ), patch(\n+ \"agent.auxiliary_client.resolve_provider_client\"\n+ ) as mock_resolve:\n+ mock_client = MagicMock()\n+ mock_resolve.return_value = (mock_client, \"claude-opus-4-8\")\n+\n+ from agent.auxiliary_client import resolve_vision_provider_client\n+\n+ provider, client, model = resolve_vision_provider_client()\n+\n+ assert client is mock_client\n+ kwargs = mock_resolve.call_args.kwargs\n+ assert kwargs.get(\"explicit_base_url\") == \"https://configured.example/v1\"\n+ assert kwargs.get(\"explicit_api_key\") == \"sk-configured\"\n+\n+\n # ── Constant cleanup ────────────────────────────────────────────────────────\n \n "]},{"file":"tests/tools/test_vision_tools.py","hunks":["@@ -261,6 +261,56 @@ async def test_falls_back_to_default_model(self):\n # (the centralized call_llm router picks the default)\n assert model is None\n \n+ @pytest.mark.asyncio\n+ async def test_config_yaml_model_takes_priority_over_env(self):\n+ \"\"\"config.yaml auxiliary.vision.model should be preferred over env var.\"\"\"\n+ with (\n+ patch(\n+ \"tools.vision_tools.vision_analyze_tool\", new_callable=AsyncMock\n+ ) as mock_tool,\n+ patch(\n+ \"tools.vision_tools._should_use_native_vision_fast_path\",\n+ return_value=False,\n+ ),\n+ patch(\n+ \"hermes_cli.config.load_config\",\n+ return_value={\"auxiliary\": {\"vision\": {\"model\": \"qwen3.7-plus\"}}},\n+ ),\n+ patch.dict(os.environ, {\"AUXILIARY_VISION_MODEL\": \"env-model\"}),\n+ ):\n+ mock_tool.return_value = json.dumps({\"result\": \"ok\"})\n+ await _handle_vision_analyze(\n+ {\"image_url\": \"https://example.com/img.png\", \"question\": \"test\"}\n+ )\n+ call_args = mock_tool.call_args\n+ model = call_args[0][2] # third positional arg\n+ assert model == \"qwen3.7-plus\"\n+\n+ @pytest.mark.asyncio\n+ async def test_env_var_used_when_config_missing_model(self):\n+ \"\"\"Env var should be used when config.yaml has no auxiliary.vision.model.\"\"\"\n+ with (\n+ patch(\n+ \"tools.vision_tools.vision_analyze_tool\", new_callable=AsyncMock\n+ ) as mock_tool,\n+ patch(\n+ \"tools.vision_tools._should_use_native_vision_fast_path\",\n+ return_value=False,\n+ ),\n+ patch(\n+ \"hermes_cli.config.load_config\",\n+ return_value={\"auxiliary\": {\"vision\": {}}},\n+ ),\n+ patch.dict(os.environ, {\"AUXILIARY_VISION_MODEL\": \"fallback-model\"}),\n+ ):\n+ mock_tool.return_value = json.dumps({\"result\": \"ok\"})\n+ await _handle_vision_analyze(\n+ {\"image_url\": \"https://example.com/img.png\", \"question\": \"test\"}\n+ )\n+ call_args = mock_tool.call_args\n+ model = call_args[0][2]\n+ assert model == \"fallback-model\"\n+\n def test_empty_args_graceful(self):\n \"\"\"Missing keys should default to empty strings, not raise.\"\"\"\n with patch("]},{"file":"tools/vision_tools.py","hunks":["@@ -1356,7 +1356,18 @@ async def _handle_vision_analyze(args: Dict[str, Any], **kw: Any) -> str:\n \"Fully describe and explain everything about this image, then answer the \"\n f\"following question:\\n\\n{question}\"\n )\n- model = os.getenv(\"AUXILIARY_VISION_MODEL\", \"\").strip() or None\n+ # Prefer config.yaml auxiliary.vision.model; env var is a legacy override.\n+ model = None\n+ try:\n+ from hermes_cli.config import cfg_get, load_config\n+ _cfg = load_config()\n+ _vmodel = cfg_get(_cfg, \"auxiliary\", \"vision\", \"model\")\n+ if _vmodel:\n+ model = str(_vmodel).strip() or None\n+ except Exception:\n+ pass\n+ if not model:\n+ model = os.getenv(\"AUXILIARY_VISION_MODEL\", \"\").strip() or None\n return await vision_analyze_tool(image_url, full_prompt, model)\n \n ","@@ -1718,7 +1729,19 @@ def _handle_video_analyze(args: Dict[str, Any], **kw: Any) -> Awaitable[str]:\n \"including visual content, motion, audio cues, text overlays, and scene \"\n f\"transitions. Then answer the following question:\\n\\n{question}\"\n )\n- model = os.getenv(\"AUXILIARY_VIDEO_MODEL\", \"\").strip() or os.getenv(\"AUXILIARY_VISION_MODEL\", \"\").strip() or None\n+ # Prefer config.yaml auxiliary.video.model (falling back to vision);\n+ # env vars are a legacy override.\n+ model = None\n+ try:\n+ from hermes_cli.config import cfg_get, load_config\n+ _cfg = load_config()\n+ _vmodel = cfg_get(_cfg, \"auxiliary\", \"video\", \"model\") or cfg_get(_cfg, \"auxiliary\", \"vision\", \"model\")\n+ if _vmodel:\n+ model = str(_vmodel).strip() or None\n+ except Exception:\n+ pass\n+ if not model:\n+ model = os.getenv(\"AUXILIARY_VIDEO_MODEL\", \"\").strip() or os.getenv(\"AUXILIARY_VISION_MODEL\", \"\").strip() or None\n return video_analyze_tool(video_url, full_prompt, model)\n \n \n"]}]},{"kind":"pr","number":58120,"title":"fix(vision): recover from generic image content rejection","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#97075","number":97075},{"kind":"pr","value":"#57948","number":57948}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":58543,"title":"fix(vision): skip main provider with unknown vision capability when auxiliary.vision is configured","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#58600","number":58600}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":58600,"title":"fix(vision): skip main provider with unknown vision capability when auxiliary.vision is configured (#58581)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#58543","number":58543},{"kind":"pr","value":"#58581","number":58581},{"kind":"pr","value":"#57948","number":57948},{"kind":"pr","value":"#58543","number":58543}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":97075,"title":"fix(agent): generic 'Unexpected item type in content' 400s now strip images and retry (#57948, salvage #58120)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/conversation_loop.py","agent/message_sanitization.py","run_agent.py","tests/run_agent/test_image_rejection_fallback.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}