{"repo":"NousResearch/hermes-agent","public_id":"33088","display_label":"Credential pool isolation during provider fallback","summary":"The members concern credential pools being shared or scoped incorrectly when an agent falls back to another provider. One reports that a fallback provider's 429 could exhaust the primary provider's pool, and the others, including a duplicated title, propose changes to keep pools isolated or scoped to the active provider.","subsystem":"agent","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":33233,"merged_at":"2026-05-27T17:45:26+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-05-27T17:45:26+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":33088,"pr_number":33233,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-05-27T17:45:26+00:00"},{"issue_number":33163,"pr_number":33233,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-27T17:45:26+00:00"}],"members":[{"kind":"issue","number":33088,"title":"Fallback provider 429 can exhaust primary provider credential pool","url":null,"state":"closed"},{"kind":"pr","number":33217,"title":"fix(agent): isolate credential pool on provider fallback","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"2e181602a","number":0},{"kind":"pr","value":"#33233","number":33233},{"kind":"pr","value":"#33163","number":33163}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":33233,"title":"fix(agent): isolate credential pool on provider fallback","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/agent_runtime_helpers.py","agent/chat_completion_helpers.py","tests/run_agent/test_fallback_credential_isolation.py"],"fix_hunks":[{"file":"agent/agent_runtime_helpers.py","hunks":["@@ -560,6 +560,24 @@ def recover_with_credential_pool(\n     if pool is None:\n         return False, has_retried_429\n \n+    # Defensive guard: if a fallback provider is active and its provider name\n+    # doesn't match the pool's provider, the pool belongs to the PRIMARY\n+    # provider.  Mutating it based on fallback errors would corrupt the\n+    # primary's credential state (see #33088) and, via _swap_credential,\n+    # overwrite the agent's base_url back to the primary's endpoint — every\n+    # subsequent request then goes to the wrong host and 404s (see #33163).\n+    # The pool should only act when the agent is still on the same provider\n+    # that seeded the pool.\n+    current_provider = (getattr(agent, \"provider\", \"\") or \"\").strip().lower()\n+    pool_provider = (getattr(pool, \"provider\", \"\") or \"\").strip().lower()\n+    if current_provider and pool_provider and current_provider != pool_provider:\n+        _ra().logger.warning(\n+            \"Credential pool provider mismatch: pool=%s, agent=%s — \"\n+            \"skipping pool mutation to avoid cross-provider contamination\",\n+            pool_provider, current_provider,\n+        )\n+        return False, has_retried_429\n+\n     effective_reason = classified_reason\n     if effective_reason is None:\n         if status_code == 402:"]},{"file":"agent/chat_completion_helpers.py","hunks":["@@ -1042,6 +1042,25 @@ def try_activate_fallback(agent, reason: \"FailoverReason | None\" = None) -> bool\n             agent._transport_cache.clear()\n         agent._fallback_activated = True\n \n+        # Clear the credential pool when the fallback provider doesn't match\n+        # the pool's provider.  The pool was seeded for the primary provider;\n+        # leaving it attached means downstream recovery (rate_limit / billing /\n+        # auth) calls ``_swap_credential`` with a primary entry which overwrites\n+        # the agent's ``base_url`` back to the primary's endpoint — every\n+        # fallback request then 404s against the wrong host.  See #33163.\n+        # When the fallback shares the pool's provider (e.g. both openrouter\n+        # entries with different routing) the pool is preserved.\n+        _existing_pool = getattr(agent, \"_credential_pool\", None)\n+        if _existing_pool is not None:\n+            _pool_provider = (getattr(_existing_pool, \"provider\", \"\") or \"\").strip().lower()\n+            if _pool_provider and _pool_provider != fb_provider:\n+                logger.info(\n+                    \"Fallback to %s/%s: clearing primary credential pool \"\n+                    \"(pool_provider=%s) to prevent cross-provider contamination\",\n+                    fb_provider, fb_model, _pool_provider,\n+                )\n+                agent._credential_pool = None\n+\n         # Honor per-provider / per-model request_timeout_seconds for the\n         # fallback target (same knob the primary client uses).  None = use\n         # SDK default."]},{"file":"tests/run_agent/test_fallback_credential_isolation.py","hunks":["@@ -0,0 +1,223 @@\n+\"\"\"Tests for fallback credential pool isolation.\n+\n+Verifies that fallback activation isolates the credential pool from the\n+primary provider, preventing two bugs:\n+\n+1. GH #33163: fallback retains primary's base_url → requests go to wrong endpoint\n+2. GH #33088: fallback provider's 429 exhausts primary credential pool\n+\n+Both bugs share the same root cause: _recover_with_credential_pool and\n+_swap_credential continue operating on the PRIMARY's credential pool during\n+fallback calls, contaminating primary state with fallback-provider errors.\n+\"\"\"\n+\n+import logging\n+import sys\n+import types\n+from dataclasses import dataclass, replace\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+\n+# ── Helpers ──────────────────────────────────────────────────────────\n+\n+def _make_pool(provider, n_entries=1):\n+    \"\"\"Create a mock credential pool with N entries.\"\"\"\n+    pool = MagicMock()\n+    pool.provider = provider\n+    pool.has_credentials.return_value = n_entries > 0\n+    pool.has_available.return_value = n_entries > 0\n+    entry = MagicMock()\n+    entry.id = f\"{provider}-entry-0\"\n+    entry.runtime_api_key = f\"key-{provider}\"\n+    entry.runtime_base_url = f\"https://{provider}.example.com/v1\"\n+    entry.access_token = f\"token-{provider}\"\n+    entry.base_url = f\"https://{provider}.example.com/v1\"\n+    pool.current.return_value = entry\n+    pool.mark_exhausted_and_rotate.return_value = entry\n+    return pool\n+\n+\n+def _make_agent(provider=\"openai-codex\", model=\"gpt-5.5\",\n+                base_url=\"https://chatgpt.com/backend-api/codex\",\n+                api_mode=\"codex_responses\"):\n+    \"\"\"Create a minimal AIAgent-like object with just the fields we need.\"\"\"\n+    agent = MagicMock()\n+    agent.provider = provider\n+    agent.model = model\n+    agent.base_url = base_url\n+    agent.api_mode = api_mode\n+    agent.api_key = \"[redacted:secret-assignment]\"\n+    agent._fallback_activated = False\n+    agent._fallback_index = 0\n+    agent._fallback_chain = []\n+    agent._primary_runtime = {\n+        \"provider\": provider,\n+        \"model\": model,\n+        \"base_url\": base_url,\n+        \"api_mode\": api_mode,\n+        \"api_key\": \"[redacted:secret-assignment]\",\n+        \"client_kwargs\": {\n+            \"api_key\": \"[redacted:secret-assignment]\",\n+            \"base_url\": base_url,\n+        },\n+        \"use_prompt_caching\": False,\n+        \"use_native_cache_layout\": False,\n+        \"anthropic_api_key\": \"\",\n+        \"anthropic_base_url\": \"\",\n+    }\n+    agent._config_context_length = None\n+    agent._credential_pool = [redacted:secret-assignment]\n+    agent._rate_limited_until = 0\n+    agent._transport_cache = {}\n+    agent._client_kwargs = {\n+        \"api_key\": \"[redacted:secret-assignment]\",\n+        \"base_url\": base_url,\n+    }\n+    return agent\n+\n+\n+# ── Test: _try_activate_fallback clears mismatched pool ──────────────\n+\n+class TestFallbackCredentialIsolation:\n+    \"\"\"Test that _try_activate_fallback isolates the credential pool.\"\"\"\n+\n+    def test_fallback_clears_primary_pool(self):\n+        \"\"\"When switching from openai-codex to openrouter, the codex pool is cleared.\"\"\"\n+        # Import the real method\n+        sys.path.insert(0, \"/mnt/g/knowledge/project/hermes-agent\")\n+        # We test the isolation logic directly, not the full _try_activate_fallback\n+        # which has many dependencies. Instead we verify the pool-clearing guard.\n+\n+        agent = _make_agent(provider=\"openai-codex\", base_url=\"https://chatgpt.com/backend-api/codex\")\n+        agent._fallback_activated = True\n+        agent._credential_pool = _make_pool(\"openai-codex\")\n+\n+        # Simulate: after fallback activation, provider is now openrouter\n+        fb_provider = \"openrouter\"\n+        fb_model = \"openrouter/auto\"\n+\n+        # The isolation code from _try_activate_fallback:\n+        pool = getattr(agent, \"_credential_pool\", None)\n+        if pool is not None:\n+            pool_provider = getattr(pool, \"provider\", \"\") or \"\"\n+            if pool_provider.lower() != fb_provider:\n+                agent._credential_pool = None\n+\n+        assert agent._credential_pool is None, (\n+            \"Pool should be cleared when fallback provider differs from pool provider\"\n+        )\n+\n+    def test_fallback_keeps_matching_pool(self):\n+        \"\"\"When fallback provider matches pool provider, pool is preserved.\"\"\"\n+        agent = _make_agent(provider=\"openrouter\", base_url=\"https://openrouter.ai/api/v1\")\n+        agent._credential_pool = _make_pool(\"openrouter\")\n+\n+        fb_provider = \"openrouter\"\n+\n+        pool = getattr(agent, \"_credential_pool\", None)\n+        if pool is not None:\n+            pool_provider = getattr(pool, \"provider\", \"\") or \"\"\n+            if pool_provider.lower() != fb_provider:\n+                agent._credential_pool = None\n+\n+        assert agent._credential_pool is not None, (\n+            \"Pool should be preserved when fallback provider matches pool provider\"\n+        )\n+\n+\n+# ── Test: _recover_with_credential_pool rejects mismatched pool ──────\n+\n+class TestRecoveryProviderGuard:\n+    \"\"\"Test that _recover_with_credential_pool skips mismatched pools.\"\"\"\n+\n+    def test_recovery_skips_mismatched_pool(self):\n+        \"\"\"_recover_with_credential_pool should not mutate a pool belonging\n+        to a different provider than the active agent provider.\"\"\"\n+        agent = _make_agent(provider=\"openrouter\")\n+        # Pool still belongs to primary (openai-codex) — mismatch\n+        agent._credential_pool = _make_pool(\"openai-codex\")\n+\n+        current_provider = (getattr(agent, \"provider\", \"\") or \"\").strip().lower()\n+        pool_provider = getattr(agent._credential_pool, \"provider\", \"\") or \"\"\n+\n+        # The guard logic:\n+        should_skip = (current_provider and pool_provider and\n+                       current_provider != pool_provider)\n+\n+        assert should_skip is True, (\n+            f\"Provider mismatch: agent={current_provider}, pool={pool_provider} — should skip\"\n+        )\n+\n+    def test_recovery_allows_matching_pool(self):\n+        \"\"\"When pool and agent provider match, recovery proceeds normally.\"\"\"\n+        agent = _make_agent(provider=\"openrouter\")\n+        agent._credential_pool = _make_pool(\"openrouter\")\n+\n+        current_provider = (getattr(agent, \"provider\", \"\") or \"\").strip().lower()\n+        pool_provider = getattr(agent._credential_pool, \"provider\", \"\") or \"\"\n+\n+        should_skip = (current_provider and pool_provider and\n+                       current_provider != pool_provider)\n+\n+        assert should_skip is False, (\n+            \"Same provider — should allow recovery\"\n+        )\n+\n+    def test_recovery_429_from_zai_does_not_exhaust_codex_pool(self):\n+        \"\"\"Regression test for GH #33088: zai 429 should NOT exhaust\n+        openai-codex credential pool.\"\"\"\n+        agent = _make_agent(provider=\"zai\", base_url=\"https://api.z.com/v1\")\n+        # Stale codex pool from primary\n+        codex_pool = _make_pool(\"openai-codex\")\n+        agent._credential_pool = [redacted:secret-assignment]\n+\n+        # The guard should prevent mark_exhausted_and_rotate from being called\n+        current_provider = \"zai\"\n+        pool_provider = \"openai-codex\"\n+        should_skip = current_provider != pool_provider\n+\n+        assert should_skip is True\n+        codex_pool.mark_exhausted_and_rotate.assert_not_called()\n+\n+\n+# ── Test: base_url not overwritten after fallback ────────────────────\n+\n+class TestBaseUrlLeak:\n+    \"\"\"Regression tests for GH #33163: base_url leaks from primary.\"\"\"\n+\n+    def test_client_kwargs_base_url_preserved_after_pool_clear(self):\n+        \"\"\"After fallback activation clears the pool, _client_kwargs should\n+        still have the fallback base_url, not the primary's.\"\"\"\n+        agent = _make_agent(\n+            provider=\"openai-codex\",\n+            base_url=\"https://chatgpt.com/backend-api/codex\"\n+        )\n+\n+        # Simulate what _try_activate_fallback does:\n+        fb_base_url = \"https://openrouter.ai/api/v1/\"\n+        agent.provider = \"openrouter\"\n+        agent.base_url = fb_base_url\n+        agent._client_kwargs = {\n+            \"api_key\": \"[redacted:secret-assignment]\",\n+            \"base_url\": fb_base_url,\n+        }\n+\n+        # Clear mismatched pool\n+        agent._credential_pool = None\n+\n+        assert agent._client_kwargs[\"base_url\"] == fb_base_url, (\n+            f\"base_url should be {fb_base_url}, not primary's URL\"\n+        )\n+\n+    def test_swap_credential_does_not_restore_primary_url(self):\n+        \"\"\"_swap_credential should not be called when pool is None,\n+        preventing it from overwriting base_url back to primary's.\"\"\"\n+        agent = _make_agent(provider=\"openrouter\", base_url=\"https://openrouter.ai/api/v1/\")\n+        agent._credential_pool = None  # Cleared by fallback isolation\n+\n+        # If pool is None, _recover_with_credential_pool returns early\n+        # and _swap_credential is never called\n+        pool = agent._credential_pool\n+        assert pool is None, \"Pool should be None — _swap_credential won't be reached\"\n"]}]},{"kind":"pr","number":33258,"title":"fix(agent): keep credential pools scoped to active provider","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"2e181602a","number":0},{"kind":"pr","value":"#33233","number":33233},{"kind":"pr","value":"#33217","number":33217},{"kind":"pr","value":"#33088","number":33088},{"kind":"pr","value":"#33163","number":33163}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}