{"repo":"NousResearch/hermes-agent","public_id":"31179","display_label":"Vision tools not respecting auxiliary.vision provider configuration","summary":"These three items all concern the auxiliary.vision config and how image requests get routed, with titles describing images going to the main model (including text-only ones) instead of the configured vision provider. Two are fix-style pull requests and one reads as an issue report; the titles alone don't confirm outcomes or exact causes.","subsystem":"agent","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":31452,"merged_at":"2026-05-24T22:01:29+00:00"},"resolution":{"state":"resolved","by":"pr-31452"},"freshness":{"state":"verified","last_verified_at":"2026-05-24T22:01:29+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":31179,"pr_number":31452,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-24T22:01:29+00:00"}],"members":[{"kind":"issue","number":31179,"title":"vision_analyze and browser_vision ignore auxiliary.vision — route images to main model","url":null,"state":"closed"},{"kind":"pr","number":31228,"title":"fix(vision): respect auxiliary.vision config so images stop silently routing to text-only main models","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3d66787a04d29beaec16a723e4c2c2bb4dda2cee","number":0},{"kind":"pr","value":"#31452","number":31452},{"kind":"pr","value":"#31179","number":31179}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":31452,"title":"fix(vision): route auxiliary.vision.provider=openai to api.openai.com, skip text-only main","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/auxiliary_client.py","tests/agent/test_vision_routing_31179.py","tools/browser_tool.py","tools/vision_tools.py"],"fix_hunks":[{"file":"agent/auxiliary_client.py","hunks":["@@ -3730,6 +3730,37 @@ def get_async_text_auxiliary_client(task: str = \"\", *, main_runtime: Optional[Di\n )\n \n \n+def _main_model_supports_vision(provider: str, model: Optional[str]) -> bool:\n+    \"\"\"Return True when ``provider``/``model`` is known to accept image input.\n+\n+    Used by the vision auto-detect chain to skip the user's main provider\n+    when it's known to be text-only (e.g. DeepSeek, gpt-oss without vision).\n+    Without this guard, ``resolve_vision_provider_client(provider=\"auto\")``\n+    would happily return the main-provider client and any subsequent image\n+    payload would surface as a cryptic provider-side error\n+    (``unknown variant `image_url`, expected `text```, #31179).\n+\n+    Returns True when capability lookup is unknown — preserves the historical\n+    behaviour of attempting the call, so providers we haven't catalogued yet\n+    don't silently regress to text-only.\n+    \"\"\"\n+    try:\n+        from agent.image_routing import _lookup_supports_vision\n+        from hermes_cli.config import load_config\n+    except ImportError:\n+        return True\n+    try:\n+        supports = _lookup_supports_vision(provider, model, load_config())\n+    except Exception:  # pragma: no cover - defensive\n+        return True\n+    if supports is None:\n+        # No capability data — keep current behaviour and let the call attempt\n+        # happen rather than silently skipping. This avoids false-positive\n+        # skips for new/custom providers.\n+        return True\n+    return bool(supports)\n+\n+\n def _normalize_vision_provider(provider: Optional[str]) -> str:\n     return _normalize_aux_provider(provider)\n ","@@ -3870,6 +3901,23 @@ def _finalize(resolved_provider: str, sync_client: Any, default_model: Optional[\n                     \"vision support) — falling through to aggregator chain\",\n                     main_provider,\n                 )\n+            elif not _main_model_supports_vision(main_provider, vision_model):\n+                # The main model is known to be text-only (e.g. DeepSeek V4,\n+                # gpt-oss-120b without vision). Building a client and sending\n+                # an image would produce a cryptic provider-side error like\n+                # ``unknown variant `image_url`, expected `text``` (#31179).\n+                # Fall through to the aggregator chain instead.\n+                #\n+                # Only log the provider name (not the model) — mirrors the\n+                # sibling _PROVIDERS_WITHOUT_VISION branch above, and avoids\n+                # CodeQL py/clear-text-logging-sensitive-data heuristic false\n+                # positives on multi-value interpolations.\n+                logger.debug(\n+                    \"Vision auto-detect: skipping main provider %s \"\n+                    \"(reports no vision capability) — falling through to \"\n+                    \"aggregator chain\",\n+                    main_provider,\n+                )\n             else:\n                 rpc_client, rpc_model = resolve_provider_client(\n                     main_provider, vision_model,","@@ -4281,6 +4329,23 @@ def _get_cached_client(\n     return client, model or default_model\n \n \n+# Aliases that target direct REST APIs not modeled as first-class providers\n+# in PROVIDER_REGISTRY. Used for ``auxiliary.<task>.provider`` so users can\n+# write the obvious name and have it resolve to a working ``custom`` endpoint\n+# without needing to know our internal provider IDs.\n+#\n+# Why these specifically: PROVIDER_REGISTRY has ``openai-codex`` (OAuth) and\n+# ``custom`` (manual base_url + OPENAI_API_KEY) but no plain ``openai`` for\n+# direct API-key access. Users predictably type ``provider: openai`` and\n+# expect it to use OPENAI_API_KEY against api.openai.com. Previously this\n+# silently fell back to the user's main provider, sending OpenAI model names\n+# to e.g. DeepSeek and producing cryptic ``unknown variant 'image_url'``\n+# errors (issue #31179).\n+_AUX_DIRECT_API_BASE_URLS: Dict[str, str] = {\n+    \"openai\": \"https://api.openai.com/v1\",\n+}\n+\n+\n def _resolve_task_provider_model(\n     task: str = None,\n     provider: str = None,","@@ -4317,6 +4382,25 @@ def _resolve_task_provider_model(\n     resolved_model = model or cfg_model\n     resolved_api_mode = cfg_api_mode\n \n+    # Convenience aliases for direct API-key endpoints that aren't first-class\n+    # providers (e.g. ``provider: openai`` → custom + api.openai.com/v1).\n+    # Applied to both explicit args and config-derived values. When the user\n+    # has already supplied a base_url we keep their endpoint but still rewrite\n+    # the provider to ``custom`` so resolution doesn't hit the\n+    # PROVIDER_REGISTRY-only path (which has no ``openai`` entry).\n+    def _expand_direct_api_alias(prov: Optional[str], existing_base: Optional[str]) -> Tuple[Optional[str], Optional[str]]:\n+        if not prov:\n+            return prov, existing_base\n+        target_base = _AUX_DIRECT_API_BASE_URLS.get(prov.strip().lower())\n+        if target_base is None:\n+            return prov, existing_base\n+        return \"custom\", existing_base or target_base\n+\n+    if provider:\n+        provider, base_url = _expand_direct_api_alias(provider, base_url)\n+    if cfg_provider:\n+        cfg_provider, cfg_base_url = _expand_direct_api_alias(cfg_provider, cfg_base_url)\n+\n     if base_url:\n         return \"custom\", resolved_model, base_url, api_key, resolved_api_mode\n     if provider:"]},{"file":"tests/agent/test_vision_routing_31179.py","hunks":["@@ -0,0 +1,297 @@\n+\"\"\"Regression tests for issue #31179.\n+\n+Before the fix:\n+  - ``auxiliary.vision.provider: openai`` silently failed to resolve because\n+    ``openai`` is not a first-class provider in PROVIDER_REGISTRY (only\n+    ``openai-codex`` for OAuth and ``custom`` for OPENAI_BASE_URL).\n+  - The vision branch of ``call_llm`` then silently fell back to ``auto``\n+    which happily picked the user's main provider (e.g. DeepSeek), sending\n+    image content to a text-only endpoint and producing cryptic\n+    ``unknown variant 'image_url', expected 'text'`` errors.\n+  - ``check_vision_requirements`` used the explicit-only path, so\n+    ``vision_analyze`` disappeared from the tool list while ``browser_vision``\n+    stayed (its check_fn only validated the browser).\n+\n+The three fixes covered here:\n+  1. ``provider: openai`` in auxiliary task config resolves to\n+     ``custom`` + ``https://api.openai.com/v1``.\n+  2. The vision auto-detect chain skips the user's main provider when it\n+     reports ``supports_vision=False`` instead of routing image content to\n+     a text-only endpoint.\n+  3. ``check_vision_requirements`` mirrors the runtime fallback chain so\n+     ``vision_analyze`` shows up whenever the auto chain can serve vision,\n+     and ``browser_vision`` gates on vision availability as well.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import os\n+import shutil\n+import sys\n+import tempfile\n+\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# Test infrastructure\n+# ---------------------------------------------------------------------------\n+\n+\n[redacted:email]\n+def isolated_home(monkeypatch):\n+    \"\"\"Temp HERMES_HOME with config + clean credential env vars.\"\"\"\n+    test_home = tempfile.mkdtemp(prefix=\"hermes_test_31179_\")\n+    hermes_home = os.path.join(test_home, \".hermes\")\n+    os.makedirs(hermes_home)\n+    monkeypatch.setenv(\"HERMES_HOME\", hermes_home)\n+\n+    # Strip all credential-shaped env vars so each scenario starts hermetic.\n+    for k in list(os.environ.keys()):\n+        if k.endswith(\"_API_KEY\") or k.endswith(\"_TOKEN\"):\n+            monkeypatch.delenv(k, raising=False)\n+\n+    yield hermes_home\n+    shutil.rmtree(test_home, ignore_errors=True)\n+\n+\n+def _write_config(home: str, text: str) -> None:\n+    with open(os.path.join(home, \"config.yaml\"), \"w\") as fp:\n+        fp.write(text)\n+\n+\n+def _fresh_modules():\n+    \"\"\"Drop cached hermes modules so each test reloads against current env.\"\"\"\n+    for mod in list(sys.modules.keys()):\n+        if mod.startswith((\"agent.auxiliary_client\", \"agent.image_routing\",\n+                           \"tools.vision_tools\", \"tools.browser_tool\",\n+                           \"hermes_cli.config\")):\n+            del sys.modules[mod]\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fix 1: provider=openai → custom + api.openai.com/v1\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestOpenAiAliasForAuxiliary:\n+    \"\"\"``auxiliary.<task>.provider: openai`` should produce a working client.\"\"\"\n+\n+    def test_provider_openai_routes_to_openai_dot_com(self, isolated_home, monkeypatch):\n+        _write_config(isolated_home, \"\"\"\n+auxiliary:\n+  vision:\n+    provider: openai\n+    model: gpt-4o-mini\n+\"\"\")\n+        monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        from agent.auxiliary_client import _resolve_task_provider_model\n+        provider, model, base_url, _key, _mode = _resolve_task_provider_model(\"vision\")\n+        assert provider == \"custom\"\n+        assert model == \"gpt-4o-mini\"\n+        assert base_url == \"https://api.openai.com/v1\"\n+\n+    def test_provider_openai_with_explicit_base_url_preserves_user_endpoint(\n+        self, isolated_home, monkeypatch\n+    ):\n+        \"\"\"User-supplied base_url wins; alias still normalizes provider name\n+        to ``custom`` so resolution doesn't hit the unknown-provider path.\"\"\"\n+        _write_config(isolated_home, \"\"\"\n+auxiliary:\n+  vision:\n+    provider: openai\n+    model: gpt-4o-mini\n+    base_url: https://my-proxy.example.com/v1\n+\"\"\")\n+        monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        from agent.auxiliary_client import _resolve_task_provider_model\n+        provider, _model, base_url, _key, _mode = _resolve_task_provider_model(\"vision\")\n+        assert provider == \"custom\"\n+        assert base_url == \"https://my-proxy.example.com/v1\"\n+\n+    def test_provider_openai_resolves_to_working_client(self, isolated_home, monkeypatch):\n+        \"\"\"End-to-end: the resolved client points at api.openai.com.\"\"\"\n+        _write_config(isolated_home, \"\"\"\n+auxiliary:\n+  vision:\n+    provider: openai\n+    model: gpt-4o-mini\n+\"\"\")\n+        monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        from agent.auxiliary_client import resolve_vision_provider_client\n+        from urllib.parse import urlparse\n+        provider, client, model = resolve_vision_provider_client()\n+        assert client is not None, \"openai alias should produce a usable client\"\n+        # Exact hostname comparison (not substring) — defends against URLs\n+        # like ``api.openai.com.evil.example`` and keeps CodeQL happy.\n+        host = urlparse(str(getattr(client, \"base_url\", \"\"))).hostname or \"\"\n+        assert host == \"api.openai.com\", f\"expected api.openai.com host, got {host!r}\"\n+        assert model == \"gpt-4o-mini\"\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fix 2: auto chain skips text-only main providers\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestTextOnlyMainSkippedForVision:\n+    \"\"\"Vision auto-detect must not return a text-only main-provider client.\"\"\"\n+\n+    def test_text_only_main_skipped_when_no_aggregator(self, isolated_home, monkeypatch):\n+        \"\"\"DeepSeek main + no aggregator credentials → no client built.\n+\n+        Pre-fix this silently returned the deepseek client with model\n+        substitution, producing ``unknown variant 'image_url'`` at call time.\n+        \"\"\"\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: deepseek\n+  default: deepseek-v4-pro\n+\"\"\")\n+        monkeypatch.setenv(\"DEEPSEEK_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        from agent.auxiliary_client import resolve_vision_provider_client\n+        provider, client, _model = resolve_vision_provider_client(provider=\"auto\")\n+        assert client is None, (\n+            f\"Vision auto-detect must skip text-only main {provider!r} when \"\n+            \"no vision-capable aggregator is available, not return a client \"\n+            \"that will fail at API time\"\n+        )\n+\n+    def test_vision_capable_main_used(self, isolated_home, monkeypatch):\n+        \"\"\"Vision-capable main provider should be returned by auto chain.\"\"\"\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: anthropic\n+  default: claude-sonnet-4-6\n+\"\"\")\n+        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"sk-ant-test\")\n+        _fresh_modules()\n+\n+        from agent.auxiliary_client import resolve_vision_provider_client\n+        provider, client, _model = resolve_vision_provider_client(provider=\"auto\")\n+        assert client is not None\n+        assert provider == \"anthropic\"\n+\n+    def test_unknown_capability_does_not_block(self, isolated_home, monkeypatch):\n+        \"\"\"When models.dev has no entry, fall back to permissive (attempt the call).\n+\n+        This keeps new/custom providers working — only providers we have\n+        cataloged as text-only are skipped.\n+        \"\"\"\n+        _fresh_modules()\n+        from agent.auxiliary_client import _main_model_supports_vision\n+        # Bogus provider/model — capability lookup returns None → permissive.\n+        assert _main_model_supports_vision(\"nonexistent-provider\", \"nonexistent-model\") is True\n+\n+\n+# ---------------------------------------------------------------------------\n+# Fix 3: check_vision_requirements + check_browser_vision_requirements parity\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestVisionToolGating:\n+    \"\"\"Tool visibility must match runtime capability.\"\"\"\n+\n+    def test_check_vision_succeeds_for_aliased_openai(self, isolated_home, monkeypatch):\n+        \"\"\"The user's exact reported scenario: provider=openai unhides\n+        vision_analyze instead of silently dropping it.\"\"\"\n+        _write_config(isolated_home, \"\"\"\n+auxiliary:\n+  vision:\n+    provider: openai\n+    model: gpt-4o-mini\n+\"\"\")\n+        monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        from tools.vision_tools import check_vision_requirements\n+        assert check_vision_requirements() is True\n+\n+    def test_check_vision_falls_back_to_auto(self, isolated_home, monkeypatch):\n+        \"\"\"Bad explicit provider doesn't hide the tool when auto fallback works.\n+\n+        Mirrors call_llm's runtime fallback chain.\n+        \"\"\"\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: openrouter\n+  default: anthropic/claude-sonnet-4\n+auxiliary:\n+  vision:\n+    provider: not-a-real-provider\n+\"\"\")\n+        monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"sk-or-test\")\n+        _fresh_modules()\n+\n+        from tools.vision_tools import check_vision_requirements\n+        assert check_vision_requirements() is True\n+\n+    def test_check_vision_false_with_text_only_main_and_no_aggregator(\n+        self, isolated_home, monkeypatch\n+    ):\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: deepseek\n+  default: deepseek-v4-pro\n+\"\"\")\n+        monkeypatch.setenv(\"DEEPSEEK_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        from tools.vision_tools import check_vision_requirements\n+        assert check_vision_requirements() is False\n+\n+    def test_browser_vision_requires_both_browser_and_vision(self, isolated_home, monkeypatch):\n+        \"\"\"``browser_vision`` must not be advertised when vision is unavailable.\"\"\"\n+        from unittest.mock import patch\n+\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: deepseek\n+  default: deepseek-v4-pro\n+\"\"\")\n+        monkeypatch.setenv(\"DEEPSEEK_API_KEY\", \"sk-test\")\n+        _fresh_modules()\n+\n+        import tools.browser_tool\n+        # Force the browser side to True so we exercise the vision-gating part.\n+        with patch.object(tools.browser_tool, \"check_browser_requirements\", return_value=True):\n+            assert tools.browser_tool.check_browser_vision_requirements() is False\n+\n+    def test_browser_vision_false_when_browser_missing(self, isolated_home, monkeypatch):\n+        from unittest.mock import patch\n+\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: openrouter\n+  default: anthropic/claude-sonnet-4\n+\"\"\")\n+        monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"sk-or-test\")\n+        _fresh_modules()\n+\n+        import tools.browser_tool\n+        with patch.object(tools.browser_tool, \"check_browser_requirements\", return_value=False):\n+            # Vision available but browser missing → still False.\n+            assert tools.browser_tool.check_browser_vision_requirements() is False\n+\n+    def test_browser_vision_true_when_both_available(self, isolated_home, monkeypatch):\n+        from unittest.mock import patch\n+\n+        _write_config(isolated_home, \"\"\"\n+model:\n+  provider: openrouter\n+  default: anthropic/claude-sonnet-4\n+\"\"\")\n+        monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"sk-or-test\")\n+        _fresh_modules()\n+\n+        import tools.browser_tool\n+        with patch.object(tools.browser_tool, \"check_browser_requirements\", return_value=True):\n+            assert tools.browser_tool.check_browser_vision_requirements() is True"]},{"file":"tools/browser_tool.py","hunks":["@@ -3652,6 +3652,24 @@ def check_browser_requirements() -> bool:\n     return True\n \n \n+def check_browser_vision_requirements() -> bool:\n+    \"\"\"Whether ``browser_vision`` should be advertised to the model.\n+\n+    Requires BOTH a working browser (``check_browser_requirements``) AND a\n+    resolvable vision backend. Without the vision check, the tool stays in\n+    the model's tool list even when no vision provider is configured, then\n+    fails at call time with a cryptic provider-side error like\n+    ``unknown variant `image_url`, expected `text``` (issue #31179).\n+    \"\"\"\n+    if not check_browser_requirements():\n+        return False\n+    try:\n+        from tools.vision_tools import check_vision_requirements\n+    except ImportError:\n+        return False\n+    return check_vision_requirements()\n+\n+\n # ============================================================================\n # Module Test\n # ============================================================================","@@ -3786,7 +3804,7 @@ def check_browser_requirements() -> bool:\n     toolset=\"browser\",\n     schema=_BROWSER_SCHEMA_MAP[\"browser_vision\"],\n     handler=lambda args, **kw: browser_vision(question=args.get(\"question\", \"\"), annotate=args.get(\"annotate\", False), task_id=kw.get(\"task_id\")),\n-    check_fn=check_browser_requirements,\n+    check_fn=check_browser_vision_requirements,\n     emoji=\"👁️\",\n )\n registry.register("]},{"file":"tools/vision_tools.py","hunks":["@@ -914,11 +914,26 @@ async def vision_analyze_tool(\n \n \n def check_vision_requirements() -> bool:\n-    \"\"\"Check if the configured runtime vision path can resolve a client.\"\"\"\n+    \"\"\"Check if the configured runtime vision path can resolve a client.\n+\n+    Mirrors the fallback chain that ``call_llm(task=\"vision\")`` actually uses\n+    at runtime: first the explicit ``auxiliary.vision.provider`` (if any),\n+    and if that fails, the auto chain (main provider → openrouter → nous).\n+    Without the auto-fallback step the tool would disappear from the model's\n+    tool list whenever the explicit provider name was unresolvable, even\n+    when the auto chain would have served the request (issue #31179).\n+    \"\"\"\n     try:\n         from agent.auxiliary_client import resolve_vision_provider_client\n-\n+    except ImportError:\n+        return False\n+    try:\n         _provider, client, _model = resolve_vision_provider_client()\n+        if client is not None:\n+            return True\n+        # Same fallback to \"auto\" that call_llm performs when the configured\n+        # provider can't be resolved.\n+        _provider, client, _model = resolve_vision_provider_client(provider=\"auto\")\n         return client is not None\n     except Exception:\n         return False\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}