{"repo":"NousResearch/hermes-agent","public_id":"5366","display_label":"Vision routing, image handling, and bug fixes","summary":"These items cover multimodal/vision routing logic, image source resolution, provider selection order, and auto-resizing of oversized images. One entry reports a vision tool failure, likely related to the same routing and handling work.","subsystem":"agent","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":6041,"merged_at":"2026-04-08T08:21:54+00:00"},"resolution":{"state":"resolved","by":"pr-6041"},"freshness":{"state":"stale","last_verified_at":"2026-04-08T08:21:54+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":5366,"pr_number":6041,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-08T08:21:54+00:00"}],"members":[{"kind":"issue","number":5366,"title":"[Bug]: Vision tool failed","url":null,"state":"closed"},{"kind":"pr","number":5376,"title":"fix(vision): apply non-aggregator provider fix to vision routing (follow-up to #5091)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#6041","number":6041}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":6041,"title":"fix(vision): try main provider first in vision auto-detection","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/auxiliary_client.py","tests/agent/test_auxiliary_client.py"],"fix_hunks":[{"file":"agent/auxiliary_client.py","hunks":["@@ -1377,9 +1377,6 @@ def get_async_text_auxiliary_client(task: str = \"\"):\n _VISION_AUTO_PROVIDER_ORDER = (\n     \"openrouter\",\n     \"nous\",\n-    \"openai-codex\",\n-    \"anthropic\",\n-    \"custom\",\n )\n \n ","@@ -1435,17 +1432,20 @@ def _preferred_main_vision_provider() -> Optional[str]:\n def get_available_vision_backends() -> List[str]:\n     \"\"\"Return the currently available vision backends in auto-selection order.\n \n-    This is the single source of truth for setup, tool gating, and runtime\n-    auto-routing of vision tasks. The selected main provider is preferred when\n-    it is also a known-good vision backend; otherwise Hermes falls back through\n-    the standard conservative order.\n+    Order: OpenRouter → Nous → active provider.  This is the single source\n+    of truth for setup, tool gating, and runtime auto-routing of vision tasks.\n     \"\"\"\n-    ordered = list(_VISION_AUTO_PROVIDER_ORDER)\n-    preferred = _preferred_main_vision_provider()\n-    if preferred in ordered:\n-        ordered.remove(preferred)\n-        ordered.insert(0, preferred)\n-    return [provider for provider in ordered if _strict_vision_backend_available(provider)]\n+    available = [p for p in _VISION_AUTO_PROVIDER_ORDER\n+                 if _strict_vision_backend_available(p)]\n+    # Also check the user's active provider (may be DeepSeek, Alibaba, named\n+    # custom, etc.) — resolve_provider_client handles all provider types.\n+    main_provider = _read_main_provider()\n+    if (main_provider and main_provider not in (\"auto\", \"\")\n+            and main_provider not in available):\n+        client, _ = resolve_provider_client(main_provider, _read_main_model())\n+        if client is not None:\n+            available.append(main_provider)\n+    return available\n \n \n def resolve_vision_provider_client(","@@ -1490,16 +1490,30 @@ def _finalize(resolved_provider: str, sync_client: Any, default_model: Optional[\n         return \"custom\", client, final_model\n \n     if requested == \"auto\":\n-        ordered = list(_VISION_AUTO_PROVIDER_ORDER)\n-        preferred = _preferred_main_vision_provider()\n-        if preferred in ordered:\n-            ordered.remove(preferred)\n-            ordered.insert(0, preferred)\n-\n-        for candidate in ordered:\n+        # Vision auto-detection order:\n+        #   1. OpenRouter  (known vision-capable default model)\n+        #   2. Nous Portal (known vision-capable default model)\n+        #   3. Active provider + model (user's main chat config)\n+        #   4. Stop\n+        for candidate in _VISION_AUTO_PROVIDER_ORDER:\n             sync_client, default_model = _resolve_strict_vision_backend(candidate)\n             if sync_client is not None:\n                 return _finalize(candidate, sync_client, default_model)\n+\n+        # Fall back to the user's active provider + model.\n+        main_provider = _read_main_provider()\n+        main_model = _read_main_model()\n+        if main_provider and main_provider not in (\"auto\", \"\"):\n+            sync_client, resolved_model = resolve_provider_client(\n+                main_provider, main_model)\n+            if sync_client is not None:\n+                logger.info(\n+                    \"Vision auto-detect: using active provider %s (%s)\",\n+                    main_provider, resolved_model or main_model,\n+                )\n+                return _finalize(\n+                    main_provider, sync_client, resolved_model or main_model)\n+\n         logger.debug(\"Auxiliary vision client: none available\")\n         return None, None, None\n "]},{"file":"tests/agent/test_auxiliary_client.py","hunks":["@@ -624,12 +624,15 @@ def test_vision_returns_none_without_any_credentials(self):\n         assert client is None\n         assert model is None\n \n-    def test_vision_auto_includes_anthropic_when_configured(self, monkeypatch):\n-        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"sk-ant-api03-key\")\n+    def test_vision_auto_includes_active_provider_when_configured(self, monkeypatch):\n+        \"\"\"Active provider appears in available backends when credentials exist.\"\"\"\n+        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"***\")\n         with (\n             patch(\"agent.auxiliary_client._read_nous_auth\", return_value=None),\n+            patch(\"agent.auxiliary_client._read_main_provider\", return_value=\"anthropic\"),\n+            patch(\"agent.auxiliary_client._read_main_model\", return_value=\"claude-sonnet-4\"),\n             patch(\"agent.anthropic_adapter.build_anthropic_client\", return_value=MagicMock()),\n-            patch(\"agent.anthropic_adapter.resolve_anthropic_token\", return_value=\"sk-ant-api03-key\"),\n+            patch(\"agent.anthropic_adapter.resolve_anthropic_token\", return_value=\"***\"),\n         ):\n             backends = get_available_vision_backends()\n ","@@ -702,88 +705,50 @@ def test_resolve_provider_client_copilot_uses_runtime_credentials(self, monkeypa\n         assert call_kwargs[\"base_url\"] == \"https://api.githubcopilot.com\"\n         assert call_kwargs[\"default_headers\"][\"Editor-Version\"]\n \n-    def test_vision_auto_uses_anthropic_when_no_higher_priority_backend(self, monkeypatch):\n-        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"sk-ant-api03-key\")\n+    def test_vision_auto_uses_active_provider_as_fallback(self, monkeypatch):\n+        \"\"\"When no OpenRouter/Nous available, vision auto falls back to active provider.\"\"\"\n+        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"***\")\n         with (\n             patch(\"agent.auxiliary_client._read_nous_auth\", return_value=None),\n+            patch(\"agent.auxiliary_client._read_main_provider\", return_value=\"anthropic\"),\n+            patch(\"agent.auxiliary_client._read_main_model\", return_value=\"claude-sonnet-4\"),\n             patch(\"agent.anthropic_adapter.build_anthropic_client\", return_value=MagicMock()),\n-            patch(\"agent.anthropic_adapter.resolve_anthropic_token\", return_value=\"sk-ant-api03-key\"),\n+            patch(\"agent.anthropic_adapter.resolve_anthropic_token\", return_value=\"***\"),\n         ):\n             client, model = get_vision_auxiliary_client()\n \n         assert client is not None\n         assert client.__class__.__name__ == \"AnthropicAuxiliaryClient\"\n-        assert model == \"claude-haiku-4-5-20251001\"\n \n-    def test_selected_anthropic_provider_is_preferred_for_vision_auto(self, monkeypatch):\n+    def test_vision_auto_prefers_openrouter_over_active_provider(self, monkeypatch):\n+        \"\"\"OpenRouter is tried before the active provider in vision auto.\"\"\"\n         monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"or-key\")\n-        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"sk-ant-api03-key\")\n-\n-        def fake_load_config():\n-            return {\"model\": {\"provider\": \"anthropic\", \"default\": \"claude-sonnet-4-6\"}}\n+        monkeypatch.setenv(\"ANTHROPIC_API_KEY\", \"***\")\n \n         with (\n             patch(\"agent.auxiliary_client._read_nous_auth\", return_value=None),\n-            patch(\"agent.anthropic_adapter.build_anthropic_client\", return_value=MagicMock()),\n-            patch(\"agent.anthropic_adapter.resolve_anthropic_token\", return_value=\"sk-ant-api03-key\"),\n+            patch(\"agent.auxiliary_client._read_main_provider\", return_value=\"anthropic\"),\n+            patch(\"agent.auxiliary_client._read_main_model\", return_value=\"claude-sonnet-4\"),\n             patch(\"agent.auxiliary_client.OpenAI\") as mock_openai,\n-            patch(\"hermes_cli.config.load_config\", fake_load_config),\n-        ):\n-            client, model = get_vision_auxiliary_client()\n-\n-        assert client is not None\n-        assert client.__class__.__name__ == \"AnthropicAuxiliaryClient\"\n-        assert model == \"claude-haiku-4-5-20251001\"\n-\n-    def test_selected_codex_provider_short_circuits_vision_auto(self, monkeypatch):\n-        def fake_load_config():\n-            return {\"model\": {\"provider\": \"openai-codex\", \"default\": \"gpt-5.2-codex\"}}\n-\n-        codex_client = MagicMock()\n-        with (\n-            patch(\"hermes_cli.config.load_config\", fake_load_config),\n-            patch(\"agent.auxiliary_client._try_codex\", return_value=(codex_client, \"gpt-5.2-codex\")) as mock_codex,\n-            patch(\"agent.auxiliary_client._try_openrouter\") as mock_openrouter,\n-            patch(\"agent.auxiliary_client._try_nous\") as mock_nous,\n-            patch(\"agent.auxiliary_client._try_anthropic\") as mock_anthropic,\n-            patch(\"agent.auxiliary_client._try_custom_endpoint\") as mock_custom,\n         ):\n             provider, client, model = resolve_vision_provider_client()\n \n-        assert provider == \"openai-codex\"\n-        assert client is codex_client\n-        assert model == \"gpt-5.2-codex\"\n-        mock_codex.assert_called_once()\n-        mock_openrouter.assert_not_called()\n-        mock_nous.assert_not_called()\n-        mock_anthropic.assert_not_called()\n-        mock_custom.assert_not_called()\n-\n-    def test_vision_auto_includes_codex(self, codex_auth_dir):\n-        \"\"\"Codex supports vision (gpt-5.3-codex), so auto mode should use it.\"\"\"\n-        with patch(\"agent.auxiliary_client._read_nous_auth\", return_value=None), \\\n-             patch(\"agent.auxiliary_client.OpenAI\"):\n-            client, model = get_vision_auxiliary_client()\n-        from agent.auxiliary_client import CodexAuxiliaryClient\n-        assert isinstance(client, CodexAuxiliaryClient)\n-        assert model == \"gpt-5.2-codex\"\n-\n-    def test_vision_auto_falls_back_to_custom_endpoint(self, monkeypatch):\n-        \"\"\"Custom endpoint is used as fallback in vision auto mode.\n+        # OpenRouter should win over anthropic active provider\n+        assert provider == \"openrouter\"\n \n-        Many local models (Qwen-VL, LLaVA, etc.) support vision.\n-        When no OpenRouter/Nous/Codex is available, try the custom endpoint.\n-        \"\"\"\n+    def test_vision_auto_uses_named_custom_as_active_provider(self, monkeypatch):\n+        \"\"\"Named custom provider works as active provider fallback in vision auto.\"\"\"\n         monkeypatch.delenv(\"OPENROUTER_API_KEY\", raising=False)\n         monkeypatch.delenv(\"ANTHROPIC_API_KEY\", raising=False)\n         with patch(\"agent.auxiliary_client._read_nous_auth\", return_value=None), \\\n              patch(\"agent.auxiliary_client._select_pool_entry\", return_value=(False, None)), \\\n-             patch(\"agent.auxiliary_client._read_codex_access_token\", return_value=None), \\\n-             patch(\"agent.auxiliary_client._resolve_custom_runtime\",\n-                   return_value=(\"http://localhost:1234/v1\", \"local-key\")), \\\n-             patch(\"agent.auxiliary_client.OpenAI\") as mock_openai:\n-            client, model = get_vision_auxiliary_client()\n-        assert client is not None  # Custom endpoint picked up as fallback\n+             patch(\"agent.auxiliary_client._read_main_provider\", return_value=\"custom:local\"), \\\n+             patch(\"agent.auxiliary_client._read_main_model\", return_value=\"my-local-model\"), \\\n+             patch(\"agent.auxiliary_client.resolve_provider_client\",\n+                   return_value=(MagicMock(), \"my-local-model\")) as mock_resolve:\n+            provider, client, model = resolve_vision_provider_client()\n+        assert client is not None\n+        assert provider == \"custom:local\"\n \n     def test_vision_direct_endpoint_override(self, monkeypatch):\n         monkeypatch.setenv(\"OPENROUTER_API_KEY\", \"or-key\")","@@ -846,7 +811,14 @@ def test_vision_forced_main_returns_none_without_creds(self, monkeypatch):\n         monkeypatch.setenv(\"AUXILIARY_VISION_PROVIDER\", \"main\")\n         monkeypatch.delenv(\"OPENAI_BASE_URL\", raising=False)\n         monkeypatch.delenv(\"OPENAI_API_KEY\", raising=False)\n+        # Clear client cache to avoid stale entries from previous tests\n+        from agent.auxiliary_client import _client_cache\n+        _client_cache.clear()\n         with patch(\"agent.auxiliary_client._read_nous_auth\", return_value=None), \\\n+             patch(\"agent.auxiliary_client._read_main_provider\", return_value=\"\"), \\\n+             patch(\"agent.auxiliary_client._read_main_model\", return_value=\"\"), \\\n+             patch(\"agent.auxiliary_client._select_pool_entry\", return_value=(False, None)), \\\n+             patch(\"agent.auxiliary_client._resolve_custom_runtime\", return_value=(None, None)), \\\n              patch(\"agent.auxiliary_client._read_codex_access_token\", return_value=None), \\\n              patch(\"agent.auxiliary_client._resolve_api_key_provider\", return_value=(None, None)):\n             client, model = get_vision_auxiliary_client()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}