Credential pool provider isolation fixes and misc patches

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #56903 fix(agent): provider-match credential pool on primary restore (salvage #56392)

  merged to main

  files: agent/agent_runtime_helpers.py, tests/run_agent/test_primary_runtime_restore.py

  Show the diff

  • agent/agent_runtime_helpers.py

    @@ -1184,11 +1184,38 @@ def restore_primary_runtime(agent) -> bool:
             if pool is not None and pool.has_available():
                 entry = pool.select()
                 if entry is not None:
    +                entry_provider = str(getattr(entry, "provider", "") or "").strip().lower()
    +                primary_provider = str(rt.get("provider") or "").strip().lower()
    +                entry_matches_primary = entry_provider == primary_provider
    +                # Custom endpoints all carry the generic ``custom`` provider on
    +                # the agent while the pool entry is keyed ``custom:<name>`` (see
    +                # CUSTOM_POOL_PREFIX). Resolve the primary's base_url to its
    +                # ``custom:<name>`` key via the canonical helper and compare
    +                # against the entry's key — this mirrors the sibling guard in
    +                # ``recover_with_credential_pool`` (see above) and correctly
    +                # disambiguates multiple custom providers that share one gateway
    +                # base_url. Fixes #56885.
    +                from agent.credential_pool import CUSTOM_POOL_PREFIX
    +                if (
    +                    primary_provider == "custom"
    +                    and entry_provider.startswith(CUSTOM_POOL_PREFIX)
    +                ):
    +                    entry_matches_primary = False
    +                    try:
    +                        from agent.credential_pool import get_custom_provider_pool_key
    +                        primary_base_url = str(rt.get("base_url") or "").strip()
    +                        primary_key = (
    +                            get_custom_provider_pool_key(primary_base_url) or ""
    +                        ).strip().lower()
    +                        entry_matches_primary = bool(primary_key) and primary_key == entry_provider
    +                    except Exception:
    +                        entry_matches_primary = False
    +
                     entry_key = (
                         getattr(entry, "runtime_api_key", None)
                         or getattr(entry, "access_token", "")
                     )
    -                if entry_key:
    +                if entry_key and entry_matches_primary:
                         # ``_swap_credential`` rebuilds the OpenAI/Anthropic client,
                         # reapplies base-url-scoped headers, and carries the
                         # accumulated base_url / OAuth-detection fixes (#33163).
    @@ -1198,6 +1225,14 @@ def restore_primary_runtime(agent) -> bool:
                             getattr(entry, "id", "?"),
                             getattr(entry, "label", "?"),
                         )
    +                elif entry_key:
    +                    logger.info(
    +                        "Restore skipped pool entry %s (%s): provider %s does not match primary provider %s",
    +                        getattr(entry, "id", "?"),
    +                        getattr(entry, "label", "?"),
    +                        entry_provider or "?",
    +                        primary_provider or "?",
    +                    )
     
             # ── Reset fallback chain for the new turn ──
             agent._fallback_activated = False

  • tests/run_agent/test_primary_runtime_restore.py

    @@ -201,6 +201,179 @@ def test_restores_prompt_caching_flag(self):
     
             assert agent._use_prompt_caching == original_caching
     
    +    def test_restore_skips_cross_provider_pool_entry(self):
    +        """Restore must not swap in a fallback provider credential for the primary runtime."""
    +
    +        class _Entry:
    +            provider = "openrouter"
    +            id = "fallback-entry"
    +            label = "fallback"
    +            runtime_api_key = "[redacted:secret-assignment]"
    +            runtime_base_url = "https://openrouter.ai/api/v1"
    +            access_token = "[redacted:secret-assignment]"
    +
    +        class _Pool:
    +            provider = "openrouter"
    +
    +            def has_available(self):
    +                return True
    +
    +            def select(self):
    +                return _Entry()
    +
    +        agent = _make_agent(
    +            provider="custom",
    +            base_url="https://primary.example.com/v1",
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        original_base_url = agent.base_url
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback()
    +        agent._credential_pool = [redacted:secret-assignment]
    +        agent._swap_credential = [redacted:secret-assignment]
    +
    +        with patch("run_agent.OpenAI", return_value=MagicMock()):
    +            result = agent._restore_primary_runtime()
    +
    +        assert result is True
    +        assert agent.provider == "custom"
    +        assert agent.base_url == original_base_url
    +        agent._swap_credential.assert_not_called()
    +
    +    def test_restore_keeps_primary_base_url_when_fallback_pool_attached(self):
    +        """Issue #56885: plain-provider primary must not inherit a fallback
    +        provider's base_url via the restore-path pool reselect.
    +
    +        Repro: primary is openai-api/gpt-5.5, a transient failure falls back to
    +        deepseek and attaches deepseek's credential pool. On the next turn the
    +        restore reselect must NOT swap in the deepseek entry — otherwise the
    +        request goes out as model=gpt-5.5 to base_url=api.deepseek.com → 404.
    +        """
    +
    +        class _DeepseekEntry:
    +            provider = "deepseek"
    +            id = "dsk-1"
    +            label = "deepseek-key"
    +            runtime_api_key = "[redacted:secret-assignment]"
    +            runtime_base_url = "https://api.deepseek.com/v1"
    +            base_url = "https://api.deepseek.com/v1"
    +            access_token = "[redacted:secret-assignment]"
    +
    +        class _DeepseekPool:
    +            provider = "deepseek"
    +
    +            def has_available(self):
    +                return True
    +
    +            def select(self):
    +                return _DeepseekEntry()
    +
    +        agent = _make_agent(
    +            provider="openai-api",
    +            base_url="https://api.openai.com/v1",
    +            fallback_model={"provider": "deepseek", "model": "deepseek-v4-flash"},
    +        )
    +        primary_base_url = agent.base_url
    +        primary_provider = agent.provider
    +        mock_client = _mock_resolve(base_url="https://api.deepseek.com/v1")
    +        with patch(
    +            "agent.auxiliary_client.resolve_provider_client",
    +            return_value=(mock_client, None),
    +        ):
    +            agent._try_activate_fallback()
    +        # Fallback attached deepseek's pool; simulate it surviving into the next turn.
    +        agent._credential_pool = [redacted:secret-assignment]
    +        agent._swap_credential = [redacted:secret-assignment]
    +
    +        with patch("run_agent.OpenAI", return_value=MagicMock()):
    +            result = agent._restore_primary_runtime()
    +
    +        assert result is True
    +        assert agent.provider == primary_provider
    +        assert agent.base_url == primary_base_url
    +        assert "deepseek" not in str(agent.base_url)
    +        agent._swap_credential.assert_not_called()
    +
    +    def test_restore_swaps_matching_custom_pool_entry(self):
    +        """Custom primary + custom:<name> entry whose base_url resolves to the
    +        SAME custom key must swap (legitimate same-endpoint rotation)."""
    +
    +        class _Entry:
    +            provider = "custom:myllm"
    +            id = "custom-entry"
    +            label = "myllm"
    +            runtime_api_key = "[redacted:secret-assignment]"
    +            runtime_base_url = "https://my-llm.example.com/v1"
    +            access_token = "[redacted:secret-assignment]"
    +
    +        class _Pool:
    +            provider = "custom:myllm"
    +
    +            def has_available(self):
    +                return True
    +
    +            def select(self):
    +                return _Entry()
    +
    +        agent = _make_agent(provider="custom", base_url="https://my-llm.example.com/v1")
    +        agent._fallback_activated = True
    +        agent._credential_pool = [redacted:secret-assignment]
    +        agent._swap_credential = [redacted:secret-assignment]
    +
    +        with (
    +            patch(
    +                "agent.credential_pool.get_custom_provider_pool_key",
    +                return_value="custom:myllm",
    +            ),
    +            patch("run_agent.OpenAI", return_value=MagicMock()),
    +        ):
    +            result = agent._restore_primary_runtime()
    +
    +        assert result is True
    +        agent._swap_credential.assert_called_once()
    +
    +    def test_restore_skips_cross_endpoint_custom_pool_entry(self):
    +        """Custom primary + custom:<name> entry whose base_url resolves to a
    +        DIFFERENT custom key must skip — two named custom providers sharing a
    +        gateway must not cross-contaminate."""
    +
    +        class _Entry:
    +            provider = "custom:otherllm"
    +            id = "other-entry"
    +            label = "otherllm"
    +            runtime_api_key = "[redacted:secret-assignment]"
    +            runtime_base_url = "https://my-llm.example.com/v1"
    +            access_token = "[redacted:secret-assignment]"
    +
    +        class _Pool:
    +            provider = "custom:otherllm"
    +
    +            def has_available(self):
    +                return True
    +
    +            def select(self):
    +                return _Entry()
    +
    +        agent = _make_agent(provider="custom", base_url="https://my-llm.example.com/v1")
    +        agent._fallback_activated = True
    +        original_base_url = agent.base_url
    +        agent._credential_pool = [redacted:secret-assignment]
    +        agent._swap_credential = [redacted:secret-assignment]
    +
    +        with (
    +            patch(
    +                "agent.credential_pool.get_custom_provider_pool_key",
    +                return_value="custom:myllm",  # primary resolves to a DIFFERENT key
    +            ),
    +            patch("run_agent.OpenAI", return_value=MagicMock()),
    +        ):
    +            result = agent._restore_primary_runtime()
    +
    +        assert result is True
    +        assert agent.base_url == original_base_url
    +        agent._swap_credential.assert_not_called()
    +
         def test_restore_survives_exception(self):
             """If client rebuild fails, the method returns False gracefully."""
             agent = _make_agent()

• #63048 fix(auth): enforce credential pool provider boundaries

  merged to main

  files: agent/agent_init.py, agent/credential_pool.py, hermes_cli/runtime_provider.py, tests/agent/test_credential_pool_provider_boundary.py

  Show the diff

  • agent/agent_init.py

    @@ -412,13 +412,25 @@ def init_agent(
         agent.skip_context_files = skip_context_files
         agent.load_soul_identity = load_soul_identity
         agent.pass_session_id = pass_session_id
    -    agent._credential_pool = [redacted:secret-assignment]
         agent.log_prefix_chars = log_prefix_chars
         agent.log_prefix = f"{log_prefix} " if log_prefix else ""
         # Store effective base URL for feature detection (prompt caching, reasoning, etc.)
         agent.base_url = base_url or ""
         provider_name = provider.strip().lower() if isinstance(provider, str) and provider.strip() else None
         agent.provider = provider_name or ""
    +    if credential_pool is not None:
    +        try:
    +            from agent.credential_pool import credential_pool_matches_provider
    +
    +            if not credential_pool_matches_provider(
    +                credential_pool,
    +                agent.provider,
    +                base_url=agent.base_url,
    +            ):
    +                credential_pool = None
    +        except Exception:
    +            credential_pool = None
    +    agent._credential_pool = [redacted:secret-assignment]
         agent.acp_command = acp_command or command
         agent.acp_args = list(acp_args or args or [])
         if api_mode in {"chat_completions", "codex_responses", "anthropic_messages", "bedrock_converse", "codex_app_server"}:

  • agent/credential_pool.py

    @@ -445,6 +445,44 @@ def get_pool_strategy(provider: str) -> str:
         return STRATEGY_FILL_FIRST
     
     
    +def credential_pool_matches_provider(
    +    pool_or_provider: Any,
    +    provider: Optional[str],
    +    *,
    +    base_url: Optional[str] = None,
    +) -> bool:
    +    """Return whether a pool belongs to the requested runtime provider.
    +
    +    Named custom endpoints intentionally use two identities: the live agent is
    +    ``custom`` while its pool is keyed ``custom:<name>``. Accept that pair only
    +    when the runtime base URL resolves to the exact same custom pool key.
    +    Empty string identities fail closed. Legacy pool adapters without a
    +    ``provider`` attribute remain compatible; production pools are scoped.
    +    """
    +    raw_pool_provider = getattr(pool_or_provider, "provider", None)
    +    if raw_pool_provider is None:
    +        if isinstance(pool_or_provider, str):
    +            raw_pool_provider = pool_or_provider
    +        else:
    +            # Backward compatibility for lightweight/unscoped pool adapters.
    +            # Production CredentialPool instances always carry ``provider``;
    +            # old plugins and tests may expose only select()/has_credentials().
    +            return True
    +    pool_provider = str(raw_pool_provider or "").strip().lower()
    +    provider_norm = str(provider or "").strip().lower()
    +    if not pool_provider or not provider_norm:
    +        return False
    +    if pool_provider == provider_norm:
    +        return True
    +    if provider_norm != "custom" or not pool_provider.startswith(CUSTOM_POOL_PREFIX):
    +        return False
    +    try:
    +        matched_pool = get_custom_provider_pool_key(base_url or "")
    +    except Exception:
    +        return False
    +    return str(matched_pool or "").strip().lower() == pool_provider
    +
    +
     DEFAULT_MAX_CONCURRENT_PER_CREDENTIAL = 1
     
     

  • hermes_cli/runtime_provider.py

    @@ -11,7 +11,13 @@
     logger = logging.getLogger(__name__)
     
     from hermes_cli import auth as auth_mod
    -from agent.credential_pool import CredentialPool, PooledCredential, get_custom_provider_pool_key, load_pool
    +from agent.credential_pool import (
    +    CredentialPool,
    +    PooledCredential,
    +    credential_pool_matches_provider,
    +    get_custom_provider_pool_key,
    +    load_pool,
    +)
     from agent.secret_scope import get_secret as _get_secret
     from hermes_cli.auth import (
         AuthError,
    @@ -1731,7 +1737,19 @@ def resolve_runtime_provider(
                     if not pool_api_key or not _agent_key_is_usable(nous_state, min_ttl):
                         logger.debug("Nous pool entry agent_key still unavailable, falling through to runtime resolution")
                         pool_api_key = ""
    -        if entry is not None and pool_api_key:
    +        if (
    +            entry is not None
    +            and pool_api_key
    +            and credential_pool_matches_provider(
    +                pool,
    +                provider,
    +                base_url=(
    +                    getattr(entry, "runtime_base_url", None)
    +                    or getattr(entry, "base_url", None)
    +                    or ""
    +                ),
    +            )
    +        ):
                 return _resolve_runtime_from_pool_entry(
                     provider=provider,
                     entry=entry,

  • tests/agent/test_credential_pool_provider_boundary.py

    @@ -0,0 +1,64 @@
    +"""Credential pools must never cross provider or custom-endpoint boundaries."""
    +
    +from types import SimpleNamespace
    +from unittest.mock import patch
    +
    +from agent.credential_pool import credential_pool_matches_provider
    +from hermes_cli import runtime_provider as rp
    +
    +
    +def test_provider_match_requires_exact_non_custom_identity():
    +    assert credential_pool_matches_provider("deepseek", "deepseek")
    +    assert not credential_pool_matches_provider("openai-codex", "deepseek")
    +    assert not credential_pool_matches_provider("", "deepseek")
    +
    +
    +def test_custom_pool_match_is_scoped_by_endpoint():
    +    with patch(
    +        "agent.credential_pool.get_custom_provider_pool_key",
    +        return_value="custom:lab",
    +    ):
    +        assert credential_pool_matches_provider(
    +            "custom:lab", "custom", base_url="https://lab.example/v1"
    +        )
    +        assert not credential_pool_matches_provider(
    +            "custom:other", "custom", base_url="https://lab.example/v1"
    +        )
    +
    +
    +def test_runtime_ignores_pool_loaded_for_different_provider(monkeypatch):
    +    entry = SimpleNamespace(
    +        provider="openai-codex",
    +        access_token="[redacted:secret-assignment]",
    +        runtime_api_key="[redacted:secret-assignment]",
    +        runtime_base_url="https://chatgpt.com/backend-api/codex",
    +        base_url="https://chatgpt.com/backend-api/codex",
    +    )
    +    pool = SimpleNamespace(
    +        provider="openai-codex",
    +        has_credentials=[redacted:secret-assignment] True,
    +        select=lambda: entry,
    +    )
    +    monkeypatch.setattr(rp, "load_pool", lambda _provider: pool)
    +    monkeypatch.setattr(rp, "resolve_provider", lambda *_a, **_kw: "deepseek")
    +    monkeypatch.setattr(
    +        rp,
    +        "_get_model_config",
    +        lambda: {"provider": "deepseek", "default": "deepseek-chat"},
    +    )
    +    monkeypatch.setattr(
    +        rp,
    +        "resolve_api_key_provider_credentials",
    +        lambda _provider: {
    +            "provider": "deepseek",
    +            "api_key": "deepseek-key",
    +            "base_url": "https://api.deepseek.com/v1",
    +            "source": "env",
    +        },
    +    )
    +
    +    resolved = rp.resolve_runtime_provider(requested="deepseek")
    +
    +    assert resolved["provider"] == "deepseek"
    +    assert resolved["api_key"] == "deepseek-key"
    +    assert resolved["base_url"] == "https://api.deepseek.com/v1"
    \ No newline at end of file