{"repo":"NousResearch/hermes-agent","public_id":"16678","display_label":"credential pool not propagated on provider/model switches","summary":"Multiple bugs and fixes related to credential pools failing to load, attach, or propagate when switching providers or models via /model overrides, switch_model, or fallback paths. Issues span the gateway, agent, and desktop model picker.","subsystem":"agent","counts":{"members":9,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":3,"merged_prs":1,"attempt_prs":5},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":53591},"resolution":{"state":"partially_resolved","by":"pr-53591"},"freshness":{"state":"verified","last_verified_at":"2026-06-27T11:39:27+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.2422222222222222},"fix_edges":[{"issue_number":16678,"pr_number":53591,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":25273,"pr_number":53591,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":52727,"pr_number":53591,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":16678,"title":"[Bug] Credential pool not loaded when /model switches provider in gateway — causes fallback instead of rotation on 429","url":null,"state":"closed"},{"kind":"issue","number":25273,"title":"[Bug] /model switch does not load credential pool for new provider","url":null,"state":"closed"},{"kind":"issue","number":52727,"title":"Desktop model picker: credential pool exhaustion blocks provider switching after any 401","url":null,"state":"closed"},{"kind":"pr","number":16701,"title":"fix(gateway): propagate credential_pool through /model session overrides (#16678)","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":25277,"title":"fix: load credential pool in switch_model for new provider (#25273)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53591","number":53591}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":52763,"title":"fix(agent): reload credential pool on switch_model provider change (#52727)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53591","number":53591},{"kind":"pr","value":"#49799","number":49799}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":52785,"title":"fix(agent): reload credential pool on switch_model provider change","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53591","number":53591}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":53591,"title":"fix(auth): reload credential pool on provider switch (/model + fallback)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/agent_runtime_helpers.py","agent/chat_completion_helpers.py","tests/run_agent/test_fallback_credential_isolation.py","tests/run_agent/test_switch_model_pool_reload_52727.py"],"fix_hunks":[{"file":"agent/agent_runtime_helpers.py","hunks":["@@ -1499,6 +1499,10 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo\n     # _client_kwargs is a dict — snapshot a shallow copy so mutating the\n     # live dict doesn't poison the rollback target.\n     _snapshot[\"_client_kwargs\"] = dict(getattr(agent, \"_client_kwargs\", {}) or {})\n+    # Snapshot the credential pool reference so a failed client rebuild can\n+    # restore the original pool (issue #52727: pool reload is part of this\n+    # switch and must be reversible on rollback).\n+    _snapshot[\"_credential_pool\"] = getattr(agent, \"_credential_pool\", _MISSING)\n \n     try:\n         # Clear the per-config context_length override so the new model's","@@ -1523,6 +1527,27 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo\n         if api_key:\n             [redacted:secret-assignment] = api_key\n \n+        # ── Reload credential pool for the new provider (issue #52727) ──\n+        # Without this, ``recover_with_credential_pool`` sees a\n+        # ``pool.provider != agent.provider`` mismatch and short-circuits,\n+        # leaving the new provider with no rotation/recovery on 401/429 and\n+        # burning the original pool's entries. Only reload when the provider\n+        # actually changed (or the pool was missing) — re-selecting the same\n+        # provider must not churn the pool reference. A reload failure is\n+        # logged + swallowed: the switch itself must still complete.\n+        old_norm = (old_provider or \"\").strip().lower()\n+        new_norm = (new_provider or \"\").strip().lower()\n+        if old_norm != new_norm or getattr(agent, \"_credential_pool\", None) is None:\n+            try:\n+                from agent.credential_pool import load_pool\n+                agent._credential_pool = [redacted:secret-assignment]\n+            except Exception as _pool_exc:  # noqa: BLE001\n+                logger.warning(\n+                    \"switch_model: credential pool reload failed for %s (%s); \"\n+                    \"continuing without pool rotation this turn\",\n+                    new_provider, _pool_exc,\n+                )\n+\n         # ── Build new client ──\n         if api_mode == \"anthropic_messages\":\n             from agent.anthropic_adapter import ("]},{"file":"agent/chat_completion_helpers.py","hunks":["@@ -1210,14 +1210,16 @@ def try_activate_fallback(agent, reason: \"FailoverReason | None\" = None) -> bool\n             agent._transport_cache.clear()\n         agent._fallback_activated = True\n \n-        # Clear the credential pool when the fallback provider doesn't match\n-        # the pool's provider.  The pool was seeded for the primary provider;\n-        # leaving it attached means downstream recovery (rate_limit / billing /\n-        # auth) calls ``_swap_credential`` with a primary entry which overwrites\n-        # the agent's ``base_url`` back to the primary's endpoint — every\n-        # fallback request then 404s against the wrong host.  See #33163.\n+        # Rebind the credential pool to the fallback provider when the provider\n+        # changes.  Keeping the primary pool attached would make downstream\n+        # recovery (rate_limit / billing / auth) mutate the wrong credential\n+        # set and can overwrite the fallback's base_url back to the primary\n+        # endpoint.  See #33163.\n+        #\n         # When the fallback shares the pool's provider (e.g. both openrouter\n-        # entries with different routing) the pool is preserved.\n+        # entries with different routing) the pool is preserved.  When the\n+        # providers differ, load the fallback provider's own pool if one exists\n+        # so provider-specific rotation continues to work after the switch.\n         _existing_pool = getattr(agent, \"_credential_pool\", None)\n         if _existing_pool is not None:\n             _pool_provider = (getattr(_existing_pool, \"provider\", \"\") or \"\").strip().lower()","@@ -1228,6 +1230,22 @@ def try_activate_fallback(agent, reason: \"FailoverReason | None\" = None) -> bool\n                     fb_provider, fb_model, _pool_provider,\n                 )\n                 agent._credential_pool = None\n+        if getattr(agent, \"_credential_pool\", None) is None:\n+            try:\n+                from agent.credential_pool import load_pool\n+\n+                fallback_pool = load_pool(fb_provider)\n+                if fallback_pool and fallback_pool.has_credentials():\n+                    agent._credential_pool = [redacted:secret-assignment]\n+                    logger.info(\n+                        \"Fallback to %s/%s: attached fallback credential pool\",\n+                        fb_provider, fb_model,\n+                    )\n+            except Exception as exc:\n+                logger.debug(\n+                    \"Fallback to %s/%s: could not attach credential pool: %s\",\n+                    fb_provider, fb_model, exc,\n+                )\n \n         # Honor per-provider / per-model request_timeout_seconds for the\n         # fallback target (same knob the primary client uses).  None = use"]},{"file":"tests/run_agent/test_fallback_credential_isolation.py","hunks":["@@ -11,8 +11,8 @@\n fallback calls, contaminating primary state with fallback-provider errors.\n \"\"\"\n \n-import sys\n-from unittest.mock import MagicMock\n+from types import SimpleNamespace\n+from unittest.mock import MagicMock, patch\n \n \n ","@@ -81,10 +81,8 @@ class TestFallbackCredentialIsolation:\n \n     def test_fallback_clears_primary_pool(self):\n         \"\"\"When switching from openai-codex to openrouter, the codex pool is cleared.\"\"\"\n-        # Import the real method\n-        sys.path.insert(0, \"/mnt/g/knowledge/project/hermes-agent\")\n-        # We test the isolation logic directly, not the full _try_activate_fallback\n-        # which has many dependencies. Instead we verify the pool-clearing guard.\n+        # We test the isolation logic directly here as a minimal guard; the\n+        # integration-style test below calls the real fallback activator.\n \n         agent = _make_agent(provider=\"openai-codex\", base_url=\"https://chatgpt.com/backend-api/codex\")\n         agent._fallback_activated = True","@@ -122,6 +120,53 @@ def test_fallback_keeps_matching_pool(self):\n             \"Pool should be preserved when fallback provider matches pool provider\"\n         )\n \n+    def test_fallback_attaches_matching_pool_after_clear(self):\n+        \"\"\"Provider-switch fallback should attach the fallback provider's pool.\"\"\"\n+        from agent.chat_completion_helpers import try_activate_fallback\n+\n+        agent = _make_agent(\n+            provider=\"ollama-cloud\",\n+            model=\"glm-5.2\",\n+            base_url=\"https://ollama.com/v1\",\n+            api_mode=\"chat_completions\",\n+        )\n+        agent._fallback_chain = [{\"provider\": \"openai-codex\", \"model\": \"gpt-5.5\"}]\n+        agent._credential_pool = [redacted:secret-assignment]\"ollama-cloud\")\n+        agent._buffer_status = MagicMock()\n+        agent._is_azure_openai_url.return_value = False\n+        agent._is_direct_openai_url.return_value = False\n+        agent._provider_model_requires_responses_api.return_value = False\n+        agent._anthropic_prompt_cache_policy.return_value = (False, False)\n+        agent._ensure_lmstudio_runtime_loaded = MagicMock()\n+        agent._replace_primary_openai_client = MagicMock()\n+        agent.context_compressor = None\n+\n+        fallback_client = SimpleNamespace(\n+            api_key=\"[redacted:secret-assignment]\",\n+            base_url=\"https://chatgpt.com/backend-api/codex\",\n+            _custom_headers={},\n+        )\n+        fallback_pool = _make_pool(\"openai-codex\")\n+\n+        with patch(\n+            \"agent.auxiliary_client.resolve_provider_client\",\n+            return_value=(fallback_client, \"gpt-5.5\"),\n+        ) as resolve_provider_client, patch(\n+            \"agent.credential_pool.load_pool\",\n+            return_value=fallback_pool,\n+        ) as load_pool:\n+            assert try_activate_fallback(agent) is True\n+\n+        resolve_provider_client.assert_called_once()\n+        load_pool.assert_called_once_with(\"openai-codex\")\n+        assert agent.provider == \"openai-codex\"\n+        assert agent.model == \"gpt-5.5\"\n+        assert agent.base_url == \"https://chatgpt.com/backend-api/codex\"\n+        assert agent.api_mode == \"codex_responses\"\n+        assert agent._credential_pool is fallback_pool\n+        assert agent._credential_pool.provider == \"openai-codex\"\n+        assert agent._transport_cache == {}\n+\n \n # ── Test: _recover_with_credential_pool rejects mismatched pool ──────\n "]},{"file":"tests/run_agent/test_switch_model_pool_reload_52727.py","hunks":["@@ -0,0 +1,218 @@\n+\"\"\"Regression tests for #52727: switch_model() must reload the credential pool\n+when the provider changes.\n+\n+When the desktop model picker swaps providers mid-session, ``switch_model``\n+mutates ``agent.model``/``agent.provider``/``agent.base_url``/``agent.api_key``\n+but never refreshes ``agent._credential_pool``. The pool stays bound to the\n+ORIGINAL provider. ``recover_with_credential_pool`` then sees a\n+``pool.provider != agent.provider`` mismatch and skips rotation entirely —\n+the 401 burns the whole retry cycle with no recovery, and the original\n+provider's pool entry gets marked ``STATUS_EXHAUSTED`` and persisted in\n+``auth.json`` (issue #52727).\n+\n+The fix reloads the pool via ``load_pool(new_provider)`` inside ``switch_model``\n+whenever the provider changes (or the pool is missing). This keeps the\n+defensive mismatch guard in ``recover_with_credential_pool`` intact while\n+making it impossible for a legitimate same-call switch to trip the guard.\n+\"\"\"\n+\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+from agent.agent_runtime_helpers import switch_model\n+\n+\n+# ---------------------------------------------------------------------------\n+# helpers\n+# ---------------------------------------------------------------------------\n+\n+\n+def _make_agent(current_provider, current_model, current_pool):\n+    \"\"\"Bare agent object with the minimum attributes switch_model touches.\n+\n+    Uses ``MagicMock`` so ``_anthropic_prompt_cache_policy(...)`` and\n+    ``_ensure_lmstudio_runtime_loaded()`` work without real implementations.\n+    \"\"\"\n+    agent = MagicMock(name=f\"Agent[{current_provider}]\")\n+    agent.provider = current_provider\n+    agent.model = current_model\n+    agent.base_url = f\"https://{current_provider}.example/v1\"\n+    agent.api_key = f\"{current_provider}-key\"\n+    agent.api_mode = \"chat_completions\"\n+    agent.client = MagicMock(name=\"Client\")\n+    agent._client_kwargs = {\n+        \"api_key\": \"***\",\n+        \"base_url\": f\"https://{current_provider}.example/v1\",\n+    }\n+    agent._anthropic_client = None\n+    agent._anthropic_api_key = \"\"\n+    agent._anthropic_base_url = None\n+    agent._is_anthropic_oauth = False\n+    agent._config_context_length = None\n+    agent._transport_cache = {}\n+    agent._cached_system_prompt = \"cached-system-prompt\"\n+    agent.context_compressor = None\n+    agent._use_prompt_caching = False\n+    agent._use_native_cache_layout = False\n+    agent._primary_runtime = {}\n+    agent._fallback_activated = False\n+    agent._fallback_index = 0\n+    agent._fallback_chain = []\n+    agent._fallback_model = None\n+    agent._credential_pool = [redacted:secret-assignment]\n+    # Real-ish instance methods that switch_model calls\n+    agent._anthropic_prompt_cache_policy = MagicMock(return_value=(False, False))\n+    agent._ensure_lmstudio_runtime_loaded = MagicMock()\n+    return agent\n+\n+\n+def _make_pool(provider):\n+    pool = MagicMock(name=f\"Pool[{provider}]\")\n+    pool.provider = provider\n+    return pool\n+\n+\n+# ---------------------------------------------------------------------------\n+# the fix\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestSwitchModelReloadsCredentialPool:\n+    \"\"\"Issue #52727: switch_model must refresh _credential_pool on provider change.\"\"\"\n+\n+    def test_switch_to_different_provider_reloads_pool(self):\n+        \"\"\"opencode-go -> groq must replace the agent's pool with a groq pool.\"\"\"\n+        old_pool = _make_pool(\"opencode-go\")\n+        new_pool = _make_pool(\"groq\")\n+        agent = _make_agent(\"opencode-go\", \"qwen-coder\", old_pool)\n+\n+        with patch(\n+            \"agent.credential_pool.load_pool\",\n+            return_value=new_pool,\n+        ) as load_pool_mock:\n+            switch_model(\n+                agent,\n+                new_model=\"llama-3.3-70b\",\n+                new_provider=\"groq\",\n+                api_key=\"[redacted:secret-assignment]\",\n+                base_url=\"https://api.groq.com/openai/v1\",\n+                api_mode=\"chat_completions\",\n+            )\n+\n+        # The agent's pool must now point at the NEW provider's pool.\n+        assert agent._credential_pool is new_pool, (\n+            f\"agent._credential_pool was not reloaded on provider switch \"\n+            f\"(still references {old_pool.provider})\"\n+        )\n+        assert agent._credential_pool.provider == \"groq\"\n+        assert agent._credential_pool is not old_pool\n+        # load_pool MUST have been called with the new provider.\n+        load_pool_mock.assert_called_once_with(\"groq\")\n+\n+    def test_switch_to_same_provider_does_not_reload_pool(self):\n+        \"\"\"Re-selecting the current provider must NOT churn the pool reference.\"\"\"\n+        existing_pool = _make_pool(\"opencode-go\")\n+        agent = _make_agent(\"opencode-go\", \"qwen-coder\", existing_pool)\n+\n+        load_pool_mock = MagicMock(name=\"load_pool\")\n+\n+        with patch(\"agent.credential_pool.load_pool\", load_pool_mock):\n+            switch_model(\n+                agent,\n+                new_model=\"qwen-coder\",\n+                new_provider=\"opencode-go\",  # SAME provider\n+                api_key=\"[redacted:secret-assignment]\",\n+                base_url=\"https://opencode.example/v1\",\n+                api_mode=\"chat_completions\",\n+            )\n+\n+        # Pool must remain the same object — no churn for same-provider switch.\n+        assert agent._credential_pool is existing_pool\n+        load_pool_mock.assert_not_called()\n+\n+    def test_switch_creates_pool_when_agent_had_none(self):\n+        \"\"\"An agent without a pool that switches providers must acquire one.\"\"\"\n+        new_pool = _make_pool(\"groq\")\n+        agent = _make_agent(\"opencode-go\", \"qwen-coder\", None)\n+\n+        with patch(\"agent.credential_pool.load_pool\", return_value=new_pool):\n+            switch_model(\n+                agent,\n+                new_model=\"llama-3.3-70b\",\n+                new_provider=\"groq\",\n+                api_key=\"[redacted:secret-assignment]\",\n+                base_url=\"https://api.groq.com/openai/v1\",\n+                api_mode=\"chat_completions\",\n+            )\n+\n+        assert agent._credential_pool is new_pool\n+        assert agent._credential_pool.provider == \"groq\"\n+\n+    def test_recover_pool_mismatch_guard_no_longer_trips_after_switch(self):\n+        \"\"\"End-to-end: after a provider switch, recover_with_credential_pool\n+        must not skip rotation due to a provider mismatch.\n+\n+        Before the fix: pool.provider=='opencode-go', agent.provider=='groq'\n+        → mismatch guard fires → recovery skipped → 401 burns the cycle.\n+        After the fix: switch_model reloaded the pool to groq, so the guard\n+        is a no-op and recovery proceeds.\n+        \"\"\"\n+        from agent.agent_runtime_helpers import recover_with_credential_pool\n+        from agent.error_classifier import FailoverReason\n+\n+        old_pool = _make_pool(\"opencode-go\")\n+        new_pool = _make_pool(\"groq\")\n+        new_pool.mark_exhausted_and_rotate.return_value = None\n+        agent = _make_agent(\"opencode-go\", \"qwen-coder\", old_pool)\n+\n+        with patch(\"agent.credential_pool.load_pool\", return_value=new_pool):\n+            switch_model(\n+                agent,\n+                new_model=\"llama-3.3-70b\",\n+                new_provider=\"groq\",\n+                api_key=\"[redacted:secret-assignment]\",\n+                base_url=\"https://api.groq.com/openai/v1\",\n+                api_mode=\"chat_completions\",\n+            )\n+\n+        # After the switch, the pool's provider matches the agent's provider.\n+        # A 429 on groq should now reach pool.mark_exhausted_and_rotate.\n+        recover_with_credential_pool(\n+            agent,\n+            status_code=429,\n+            has_retried_429=False,\n+            classified_reason=FailoverReason.rate_limit,\n+        )\n+\n+        # The guard would have returned (False, has_retried_429) early\n+        # without touching the pool. After the fix, the pool is consulted.\n+        assert new_pool.current.called, (\n+            \"pool.current() was never called — mismatch guard short-circuited\"\n+        )\n+\n+    def test_pool_reload_failure_does_not_block_switch(self):\n+        \"\"\"If load_pool raises (e.g. corrupt auth.json), switch_model must\n+        still complete — the pool will simply be missing for this turn, and\n+        the next turn can re-attempt. Crashing the whole switch is worse\n+        than a transient pool gap.\"\"\"\n+        agent = _make_agent(\"opencode-go\", \"qwen-coder\", _make_pool(\"opencode-go\"))\n+\n+        with patch(\n+            \"agent.credential_pool.load_pool\",\n+            side_effect=RuntimeError(\"simulated corrupt auth.json\"),\n+        ):\n+            # Should NOT raise — pool reload failure is logged+swallowed.\n+            switch_model(\n+                agent,\n+                new_model=\"llama-3.3-70b\",\n+                new_provider=\"groq\",\n+                api_key=\"[redacted:secret-assignment]\",\n+                base_url=\"https://api.groq.com/openai/v1\",\n+                api_mode=\"chat_completions\",\n+            )\n+\n+        # The switch itself completed (provider/model updated) even though\n+        # the pool reload failed.\n+        assert agent.provider == \"groq\"\n+        assert agent.model == \"llama-3.3-70b\"\n\\ No newline at end of file\n"]}]},{"kind":"pr","number":58439,"title":"fix(gateway): attach credential_pool to session /model overrides","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}