auxiliary tasks failing to inherit custom provider credentials

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #35259 fix(auxiliary): route custom-provider aux tasks to the user endpoint (salvage #34783)

  merged to main

  files: agent/auxiliary_client.py, agent/conversation_loop.py, tests/agent/test_set_runtime_main_custom_provider.py

  Show the diff

  • agent/auxiliary_client.py

    @@ -1680,26 +1680,48 @@ def _read_main_provider() -> str:
     # per turn — no lock needed. Cleared by ``clear_runtime_main()``.
     _RUNTIME_MAIN_PROVIDER: str = ""
     _RUNTIME_MAIN_MODEL: str = ""
    +_RUNTIME_MAIN_BASE_URL: str = ""
    +_RUNTIME_MAIN_API_KEY: str = ""
    +_RUNTIME_MAIN_API_MODE: str = ""
     
     
    -def set_runtime_main(provider: str, model: str) -> None:
    -    """Record the live runtime provider/model for the current AIAgent.
    +def set_runtime_main(
    +    provider: str,
    +    model: str,
    +    *,
    +    base_url: str = "",
    +    api_key: str = "",
    +    api_mode: str = "",
    +) -> None:
    +    """Record the live runtime provider/model/credentials for the current AIAgent.
     
         Called by ``run_agent.AIAgent._sync_runtime_main_for_aux_routing`` (or
         equivalent setter) at the top of each turn so that
         ``_read_main_provider`` / ``_read_main_model`` reflect CLI/gateway
         overrides instead of the stale config.yaml default.
    +
    +    For ``custom:`` providers, ``base_url`` and ``api_key`` must also be
    +    recorded so that ``_resolve_auto`` can construct a valid client in
    +    Step 1 instead of falling through to the aggregator chain.
         """
         global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL
    +    global _RUNTIME_MAIN_BASE_URL, _RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE
         _RUNTIME_MAIN_PROVIDER = (provider or "").strip().lower()
         _RUNTIME_MAIN_MODEL = (model or "").strip()
    +    _RUNTIME_MAIN_BASE_URL = (base_url or "").strip()
    +    _RUNTIME_MAIN_API_KEY = [redacted:secret-assignment] if isinstance(api_key, str) else ""
    +    _RUNTIME_MAIN_API_MODE = (api_mode or "").strip()
     
     
     def clear_runtime_main() -> None:
         """Clear the runtime override (e.g. on session end)."""
         global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL
    +    global _RUNTIME_MAIN_BASE_URL, _RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE
         _RUNTIME_MAIN_PROVIDER = ""
         _RUNTIME_MAIN_MODEL = ""
    +    _RUNTIME_MAIN_BASE_URL = ""
    +    _RUNTIME_MAIN_API_KEY = ""
    +    _RUNTIME_MAIN_API_MODE = ""
     
     
     def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[str]]:
    @@ -2980,6 +3002,18 @@ def _resolve_auto(main_runtime: Optional[Dict[str, Any]] = None) -> Tuple[Option
         runtime_api_key = [redacted:secret-assignment]"api_key", "")
         runtime_api_mode = str(runtime.get("api_mode") or "")
     
    +    # Fall back to process-local globals when main_runtime dict was not
    +    # provided or was incomplete.  ``set_runtime_main()`` now records
    +    # base_url/api_key/api_mode alongside provider/model, so custom:
    +    # providers get the full credential surface in Step 1 of the
    +    # auto-detect chain.
    +    if not runtime_base_url and _RUNTIME_MAIN_BASE_URL:
    +        runtime_base_url = _RUNTIME_MAIN_BASE_URL
    +    if not runtime_api_key and _RUNTIME_MAIN_API_KEY:
    +        runtime_api_key = [redacted:secret-assignment]
    +    if not runtime_api_mode and _RUNTIME_MAIN_API_MODE:
    +        runtime_api_mode = _RUNTIME_MAIN_API_MODE
    +
         # ── Warn once if OPENAI_BASE_URL is set but config.yaml uses a named
         #    provider (not 'custom').  This catches the common "env poisoning"
         #    scenario where a user switches providers via `hermes model` but the

  • agent/conversation_loop.py

    @@ -392,6 +392,9 @@ def run_conversation(
             set_runtime_main(
                 getattr(agent, "provider", "") or "",
                 getattr(agent, "model", "") or "",
    +            base_url=getattr(agent, "base_url", "") or "",
    +            api_key=[redacted:secret-assignment] "api_key", "") or "",
    +            api_mode=getattr(agent, "api_mode", "") or "",
             )
         except Exception:
             pass

  • tests/agent/test_set_runtime_main_custom_provider.py

    @@ -0,0 +1,226 @@
    +"""Regression test: set_runtime_main() must pass base_url/api_key/api_mode
    +so that _resolve_auto() can route custom: providers in Step 1.
    +
    +Fixes https://github.com/NousResearch/hermes-agent/issues/34777
    +"""
    +import pytest
    +from unittest.mock import patch, MagicMock
    +
    +
    +def _get_globals(mod):
    +    """Read runtime globals without triggering redaction."""
    +    return {
    +        "provider": mod._RUNTIME_MAIN_PROVIDER,
    +        "model": mod._RUNTIME_MAIN_MODEL,
    +        "base_url": mod._RUNTIME_MAIN_BASE_URL,
    +        "cred": mod._RUNTIME_MAIN_API_KEY,  # renamed to avoid redaction
    +        "api_mode": mod._RUNTIME_MAIN_API_MODE,
    +    }
    +
    +
    +class TestSetRuntimeMainCustomProvider:
    +    """set_runtime_main must propagate base_url/api_key/api_mode for custom providers."""
    +
    +    def test_globals_stored(self):
    +        """set_runtime_main stores all five fields in process-local globals."""
    +        import agent.auxiliary_client as mod
    +
    +        mod.clear_runtime_main()
    +        try:
    +            mod.set_runtime_main(
    +                "custom:my-router",
    +                "glm-5.1",
    +                base_url="https://my-server.example.com/v1",
    +                api_key="[redacted:secret-assignment]",
    +                api_mode="chat_completions",
    +            )
    +            g = _get_globals(mod)
    +            assert g["provider"] == "custom:my-router"
    +            assert g["model"] == "glm-5.1"
    +            assert g["base_url"] == "https://my-server.example.com/v1"
    +            assert g["cred"] == "sk-test-key"
    +            assert g["api_mode"] == "chat_completions"
    +        finally:
    +            mod.clear_runtime_main()
    +
    +    def test_clear_resets_all_globals(self):
    +        """clear_runtime_main resets all five globals to empty."""
    +        import agent.auxiliary_client as mod
    +
    +        mod.set_runtime_main(
    +            "custom:x", "m",
    +            base_url="https://x.example.com",
    +            api_key="[redacted:secret-assignment]",
    +            api_mode="chat_completions",
    +        )
    +        mod.clear_runtime_main()
    +        g = _get_globals(mod)
    +        for v in g.values():
    +            assert v == "", f"Expected empty, got {v!r}"
    +
    +    def test_resolve_auto_uses_globals_for_custom_provider(self):
    +        """_resolve_auto reads base_url/api_key from globals when main_runtime is None."""
    +        import agent.auxiliary_client as mod
    +
    +        mod.clear_runtime_main()
    +        try:
    +            mod.set_runtime_main(
    +                "custom:test-router",
    +                "test-model",
    +                base_url="https://custom-endpoint.example.com/v1",
    +                api_key="[redacted:secret-assignment]",
    +            )
    +
    +            with patch.object(mod, "resolve_provider_client") as mock_resolve:
    +                mock_resolve.return_value = (MagicMock(), "test-model")
    +                client, resolved = mod._resolve_auto(main_runtime=None)
    +
    +                mock_resolve.assert_called_once()
    +                call_args = mock_resolve.call_args
    +                assert call_args[0][0] == "custom"
    +                assert call_args[1]["explicit_base_url"] == "https://custom-endpoint.example.com/v1"
    +                assert call_args[1]["explicit_api_key"] == "sk-test-123"
    +        finally:
    +            mod.clear_runtime_main()
    +
    +    def test_explicit_main_runtime_takes_precedence(self):
    +        """When main_runtime dict has values, globals are NOT used."""
    +        import agent.auxiliary_client as mod
    +
    +        mod.clear_runtime_main()
    +        try:
    +            mod.set_runtime_main(
    +                "custom:router-a",
    +                "model-a",
    +                base_url="https://from-global.example.com",
    +                api_key="[redacted:secret-assignment]",
    +            )
    +
    +            with patch.object(mod, "resolve_provider_client") as mock_resolve:
    +                mock_resolve.return_value = (MagicMock(), "model-b")
    +                main_rt = {
    +                    "provider": "custom:router-b",
    +                    "model": "model-b",
    +                    "base_url": "https://from-dict.example.com",
    +                    "api_key": "sk-dict",
    +                }
    +                mod._resolve_auto(main_runtime=main_rt)
    +
    +                call_args = mock_resolve.call_args[1]
    +                assert call_args["explicit_base_url"] == "https://from-dict.example.com"
    +                assert call_args["explicit_api_key"] == "sk-dict"
    +        finally:
    +            mod.clear_runtime_main()
    +
    +    def test_backward_compatible_defaults(self):
    +        """Calling set_runtime_main with only positional args still works."""
    +        import agent.auxiliary_client as mod
    +
    +        mod.clear_runtime_main()
    +        try:
    +            mod.set_runtime_main("openrouter", "gpt-4o")
    +            g = _get_globals(mod)
    +            assert g["provider"] == "openrouter"
    +            assert g["model"] == "gpt-4o"
    +            assert g["base_url"] == ""
    +            assert g["cred"] == ""
    +            assert g["api_mode"] == ""
    +        finally:
    +            mod.clear_runtime_main()
    +
    +
    +class TestResolveAutoCustomEndToEnd:
    +    """End-to-end routing assertions — build a *real* client (no mock on
    +    resolve_provider_client) and verify the auxiliary auto-detect chain lands
    +    on the user's custom endpoint instead of falling through to the aggregator
    +    chain.  These guard the actual user-visible symptom in #34777 (aux tasks
    +    silently routed to a fallback provider) rather than just the wiring.
    +    """
    +
    +    @staticmethod
    +    def _client_base_url(client):
    +        for chain in (("base_url",), ("_client", "base_url")):
    +            obj = client
    +            try:
    +                for attr in chain:
    +                    obj = getattr(obj, attr)
    +                return str(obj)
    +            except AttributeError:
    +                continue
    +        return None
    +
    +    def test_config_less_custom_endpoint_routes_via_global(self, tmp_path, monkeypatch):
    +        """custom:<name> with NO config entry: the live base_url carried by
    +        set_runtime_main() must build a real client at that endpoint — not
    +        fall through to Step 2 (the regression in #34777)."""
    +        import agent.auxiliary_client as mod
    +
    +        # Hermetic: no aggregator creds, no stale OPENAI_BASE_URL.
    +        for var in ("OPENROUTER_API_KEY", "NOUS_API_KEY", "OPENAI_API_KEY",
    +                    "OPENAI_BASE_URL"):
    +            monkeypatch.delenv(var, raising=False)
    +        hermes_home = tmp_path / ".hermes"
    +        hermes_home.mkdir()
    +        (hermes_home / "config.yaml").write_text(
    +            "model:\n"
    +            "  default: glm-5.1\n"
    +            "  provider: 'custom:ephemeral'\n"
    +            "  base_url: ''\n"
    +        )
    +        monkeypatch.setenv("HERMES_HOME", str(hermes_home))
    +
    +        mod.clear_runtime_main()
    +        try:
    +            mod.set_runtime_main(
    +                "custom:ephemeral",
    +                "glm-5.1",
    +                base_url="https://ephemeral.live/v1",
    +                api_key="[redacted:secret-assignment]",
    +            )
    +            client, resolved = mod.resolve_provider_client("auto", None)
    +            assert client is not None, (
    +                "config-less custom endpoint fell through to Step 2 — "
    +                "the #34777 bug is back"
    +            )
    +            assert resolved == "glm-5.1"
    +            base = self._client_base_url(client)
    +            assert base and base.rstrip("/") == "https://ephemeral.live/v1"
    +        finally:
    +            mod.clear_runtime_main()
    +
    +    def test_named_custom_with_config_entry_still_routes(self, tmp_path, monkeypatch):
    +        """Regression guard: custom:<name> WITH a custom_providers entry must
    +        still resolve to that entry's endpoint.  An earlier competing fix
    +        collapsed the provider to bare ``custom`` before resolution, which
    +        broke the named-custom branch and returned None here."""
    +        import agent.auxiliary_client as mod
    +
    +        for var in ("OPENROUTER_API_KEY", "NOUS_API_KEY", "OPENAI_API_KEY",
    +                    "OPENAI_BASE_URL"):
    +            monkeypatch.delenv(var, raising=False)
    +        hermes_home = tmp_path / ".hermes"
    +        hermes_home.mkdir()
    +        (hermes_home / "config.yaml").write_text(
    +            "model:\n"
    +            "  default: glm-5.1\n"
    +            "  provider: 'custom:openclaw'\n"
    +            "  base_url: ''\n"
    +            "custom_providers:\n"
    +            "  - name: openclaw\n"
    +            "    base_url: 'https://withcfg.example/v1'\n"
    +            "    model: glm-5.1\n"
    +            "    api_key: [redacted:secret-assignment]"
    +        )
    +        monkeypatch.setenv("HERMES_HOME", str(hermes_home))
    +
    +        # No live base_url carried — resolution must come from config alone,
    +        # via the named-custom branch in resolve_provider_client.
    +        mod.clear_runtime_main()
    +        try:
    +            mod.set_runtime_main("custom:openclaw", "glm-5.1")
    +            client, resolved = mod.resolve_provider_client("auto", None)
    +            assert client is not None
    +            base = self._client_base_url(client)
    +            assert base and base.rstrip("/") == "https://withcfg.example/v1"
    +        finally:
    +            mod.clear_runtime_main()

• #59301 fix(auxiliary): reuse main_runtime credentials for named custom providers

  merged to main

  files: agent/auxiliary_client.py, tests/agent/test_auxiliary_named_custom_providers.py

  Show the diff

  • agent/auxiliary_client.py

    @@ -4307,6 +4307,8 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "",
     
         # ── Custom endpoint (OPENAI_BASE_URL + OPENAI_API_KEY) ───────────
         if provider == "custom":
    +        custom_base = ""
    +        custom_key = ""
             if explicit_base_url:
                 custom_base = _to_openai_base_url(explicit_base_url).strip()
                 custom_key = (
    @@ -4321,6 +4323,19 @@ def _wrap_if_needed(client_obj, final_model_str: str, base_url_str: str = "",
                         "but base_url is empty"
                     )
                     return None, None
    +        elif main_runtime:
    +            # When main_runtime carries a concrete base_url + api_key for a
    +            # named custom provider (custom:<name>), use it directly instead
    +            # of re-resolving from the bare "custom" provider name.
    +            # Re-resolution loses the provider name and falls back to
    +            # OpenRouter or a wrong API-key provider — the main agent already
    +            # solved this, we just need to reuse its answer. (#45472)
    +            _main_base = str(main_runtime.get("base_url") or "").strip().rstrip("/")
    +            _main_key = str(main_runtime.get("api_key") or "").strip()
    +            if _main_base and _main_key:
    +                custom_base = _main_base
    +                custom_key = _main_key
    +        if custom_base and custom_key:
                 final_model = _normalize_resolved_model(
                     model or (main_runtime.get("model") if main_runtime else None) or "gpt-4o-mini",
                     provider,

  • tests/agent/test_auxiliary_named_custom_providers.py

    @@ -491,3 +491,73 @@ def test_explicit_overrides_applied_on_api_key_branch(self, tmp_path, monkeypatc
             assert isinstance(client, OpenAI)
             assert "override.example.com" in str(client.base_url)
             assert client.api_key == "override-key"
    +
    +
    +class TestResolveProviderClientMainRuntimeCustom:
    +    """When the main agent uses a named custom provider (custom:<name>),
    +    resolve_provider_client('custom', ..., main_runtime=...) must reuse the
    +    main_runtime's base_url + api_key instead of re-resolving from the bare
    +    'custom' provider name.  Re-resolution loses the provider name and falls
    +    back to OpenRouter or a wrong API-key provider. (#45472)"""
    +
    +    def test_custom_provider_main_runtime_used_directly(self, tmp_path, monkeypatch):
    +        """main_runtime with base_url + api_key for a named custom provider
    +        is used directly, bypassing the _try_custom_endpoint / API-key
    +        fallback chain."""
    +        from agent.auxiliary_client import resolve_provider_client
    +        main_runtime = {
    +            "provider": "custom",
    +            "base_url": "https://my-gateway.example.com/v1",
    +            "api_key": "***",
    +            "model": "glm-5.1",
    +        }
    +        client, model = resolve_provider_client(
    +            "custom",
    +            model="explicit-glm-5.1",
    +            main_runtime=main_runtime,
    +        )
    +        assert client is not None
    +        assert model == "explicit-glm-5.1"
    +        assert "my-gateway.example.com" in str(client.base_url)
    +        assert client.api_key == "***"
    +
    +    def test_custom_provider_main_runtime_no_credentials_falls_through(self, tmp_path, monkeypatch):
    +        """When main_runtime has no base_url or no api_key, the existing
    +        _try_custom_endpoint / _resolve_api_key_provider fallback chain is
    +        still tried."""
    +        # Ensure no env-provided credentials interfere
    +        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
    +        monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
    +        monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
    +        monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
    +
    +        from agent.auxiliary_client import resolve_provider_client
    +        # main_runtime with key but no base_url → must fall through
    +        client, model = resolve_provider_client(
    +            "custom",
    +            main_runtime={"api_key": "k", "base_url": ""},
    +        )
    +        # Should fall through to _try_custom_endpoint → return None,None
    +        # because no OPENAI_BASE_URL is set and no custom endpoint is configured
    +        assert client is None
    +
    +    def test_custom_provider_main_runtime_respects_explicit_base_url(self, tmp_path):
    +        """explicit_base_url still wins over main_runtime — the caller's
    +        explicit argument is the strongest signal."""
    +        from agent.auxiliary_client import resolve_provider_client
    +        main_runtime = {
    +            "base_url": "https://main-runtime.example.com/v1",
    +            "api_key": "sk-main",
    +            "model": "ignored-model",
    +        }
    +        client, model = resolve_provider_client(
    +            "custom",
    +            model="explicit-model",
    +            explicit_base_url="https://explicit.example.com/v1",
    +            explicit_api_key="[redacted:secret-assignment]",
    +            main_runtime=main_runtime,
    +        )
    +        assert client is not None
    +        assert model == "explicit-model"
    +        assert "explicit.example.com" in str(client.base_url)
    +        assert client.api_key == "sk-explicit"