{"repo":"NousResearch/hermes-agent","public_id":"33333","display_label":"Custom provider credentials lost in background/cron tasks","summary":"Multiple fixes and a bug report addressing that custom provider settings (base_url, api_key, api_mode) are dropped or not passed through when auxiliary tasks run in cron or background contexts without a runtime override.","subsystem":"agent","counts":{"members":6,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":35259},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-05-30T09:39:00+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.3333333333333333},"fix_edges":[{"issue_number":33333,"pr_number":35259,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":34777,"pr_number":35259,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":33333,"title":"Bug: `auxiliary._resolve_auto()` drops `model.base_url` and `model.api_key` for `custom` provider when no runtime override exists (cron/background tasks)","url":null,"state":"closed"},{"kind":"pr","number":33339,"title":"fix(agent): read custom model.base_url/api_key in _resolve_auto cron path","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"24a5ad37c","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":33347,"title":"fix(aux): fall back to config base_url/api_key for custom provider in cron/background tasks","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#35259","number":35259},{"kind":"pr","value":"#34783","number":34783},{"kind":"pr","value":"#33333","number":33333},{"kind":"pr","value":"#35259","number":35259}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":33765,"title":"fix(provider): custom provider respects api_mode and works in cron/bg tasks (#33600, #33333)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#33604","number":33604},{"kind":"pr","value":"#33637","number":33637},{"kind":"pr","value":"#33347","number":33347},{"kind":"pr","value":"#33339","number":33339}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":34783,"title":"fix(auxiliary): pass base_url/api_key/api_mode through set_runtime_main for custom providers","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#33347","number":33347},{"kind":"pr","value":"#34777","number":34777},{"kind":"pr","value":"#33333","number":33333},{"kind":"pr","value":"#17846","number":17846}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":35259,"title":"fix(auxiliary): route custom-provider aux tasks to the user endpoint (salvage #34783)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/auxiliary_client.py","agent/conversation_loop.py","tests/agent/test_set_runtime_main_custom_provider.py"],"fix_hunks":[{"file":"agent/auxiliary_client.py","hunks":["@@ -1680,26 +1680,48 @@ def _read_main_provider() -> str:\n # per turn — no lock needed. Cleared by ``clear_runtime_main()``.\n _RUNTIME_MAIN_PROVIDER: str = \"\"\n _RUNTIME_MAIN_MODEL: str = \"\"\n+_RUNTIME_MAIN_BASE_URL: str = \"\"\n+_RUNTIME_MAIN_API_KEY: str = \"\"\n+_RUNTIME_MAIN_API_MODE: str = \"\"\n \n \n-def set_runtime_main(provider: str, model: str) -> None:\n-    \"\"\"Record the live runtime provider/model for the current AIAgent.\n+def set_runtime_main(\n+    provider: str,\n+    model: str,\n+    *,\n+    base_url: str = \"\",\n+    api_key: str = \"\",\n+    api_mode: str = \"\",\n+) -> None:\n+    \"\"\"Record the live runtime provider/model/credentials for the current AIAgent.\n \n     Called by ``run_agent.AIAgent._sync_runtime_main_for_aux_routing`` (or\n     equivalent setter) at the top of each turn so that\n     ``_read_main_provider`` / ``_read_main_model`` reflect CLI/gateway\n     overrides instead of the stale config.yaml default.\n+\n+    For ``custom:`` providers, ``base_url`` and ``api_key`` must also be\n+    recorded so that ``_resolve_auto`` can construct a valid client in\n+    Step 1 instead of falling through to the aggregator chain.\n     \"\"\"\n     global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL\n+    global _RUNTIME_MAIN_BASE_URL, _RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE\n     _RUNTIME_MAIN_PROVIDER = (provider or \"\").strip().lower()\n     _RUNTIME_MAIN_MODEL = (model or \"\").strip()\n+    _RUNTIME_MAIN_BASE_URL = (base_url or \"\").strip()\n+    _RUNTIME_MAIN_API_KEY = [redacted:secret-assignment] if isinstance(api_key, str) else \"\"\n+    _RUNTIME_MAIN_API_MODE = (api_mode or \"\").strip()\n \n \n def clear_runtime_main() -> None:\n     \"\"\"Clear the runtime override (e.g. on session end).\"\"\"\n     global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL\n+    global _RUNTIME_MAIN_BASE_URL, _RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE\n     _RUNTIME_MAIN_PROVIDER = \"\"\n     _RUNTIME_MAIN_MODEL = \"\"\n+    _RUNTIME_MAIN_BASE_URL = \"\"\n+    _RUNTIME_MAIN_API_KEY = \"\"\n+    _RUNTIME_MAIN_API_MODE = \"\"\n \n \n def _resolve_custom_runtime() -> Tuple[Optional[str], Optional[str], Optional[str]]:","@@ -2980,6 +3002,18 @@ def _resolve_auto(main_runtime: Optional[Dict[str, Any]] = None) -> Tuple[Option\n     runtime_api_key = [redacted:secret-assignment]\"api_key\", \"\")\n     runtime_api_mode = str(runtime.get(\"api_mode\") or \"\")\n \n+    # Fall back to process-local globals when main_runtime dict was not\n+    # provided or was incomplete.  ``set_runtime_main()`` now records\n+    # base_url/api_key/api_mode alongside provider/model, so custom:\n+    # providers get the full credential surface in Step 1 of the\n+    # auto-detect chain.\n+    if not runtime_base_url and _RUNTIME_MAIN_BASE_URL:\n+        runtime_base_url = _RUNTIME_MAIN_BASE_URL\n+    if not runtime_api_key and _RUNTIME_MAIN_API_KEY:\n+        runtime_api_key = [redacted:secret-assignment]\n+    if not runtime_api_mode and _RUNTIME_MAIN_API_MODE:\n+        runtime_api_mode = _RUNTIME_MAIN_API_MODE\n+\n     # ── Warn once if OPENAI_BASE_URL is set but config.yaml uses a named\n     #    provider (not 'custom').  This catches the common \"env poisoning\"\n     #    scenario where a user switches providers via `hermes model` but the"]},{"file":"agent/conversation_loop.py","hunks":["@@ -392,6 +392,9 @@ def run_conversation(\n         set_runtime_main(\n             getattr(agent, \"provider\", \"\") or \"\",\n             getattr(agent, \"model\", \"\") or \"\",\n+            base_url=getattr(agent, \"base_url\", \"\") or \"\",\n+            api_key=[redacted:secret-assignment] \"api_key\", \"\") or \"\",\n+            api_mode=getattr(agent, \"api_mode\", \"\") or \"\",\n         )\n     except Exception:\n         pass"]},{"file":"tests/agent/test_set_runtime_main_custom_provider.py","hunks":["@@ -0,0 +1,226 @@\n+\"\"\"Regression test: set_runtime_main() must pass base_url/api_key/api_mode\n+so that _resolve_auto() can route custom: providers in Step 1.\n+\n+Fixes https://github.com/NousResearch/hermes-agent/issues/34777\n+\"\"\"\n+import pytest\n+from unittest.mock import patch, MagicMock\n+\n+\n+def _get_globals(mod):\n+    \"\"\"Read runtime globals without triggering redaction.\"\"\"\n+    return {\n+        \"provider\": mod._RUNTIME_MAIN_PROVIDER,\n+        \"model\": mod._RUNTIME_MAIN_MODEL,\n+        \"base_url\": mod._RUNTIME_MAIN_BASE_URL,\n+        \"cred\": mod._RUNTIME_MAIN_API_KEY,  # renamed to avoid redaction\n+        \"api_mode\": mod._RUNTIME_MAIN_API_MODE,\n+    }\n+\n+\n+class TestSetRuntimeMainCustomProvider:\n+    \"\"\"set_runtime_main must propagate base_url/api_key/api_mode for custom providers.\"\"\"\n+\n+    def test_globals_stored(self):\n+        \"\"\"set_runtime_main stores all five fields in process-local globals.\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        mod.clear_runtime_main()\n+        try:\n+            mod.set_runtime_main(\n+                \"custom:my-router\",\n+                \"glm-5.1\",\n+                base_url=\"https://my-server.example.com/v1\",\n+                api_key=\"[redacted:secret-assignment]\",\n+                api_mode=\"chat_completions\",\n+            )\n+            g = _get_globals(mod)\n+            assert g[\"provider\"] == \"custom:my-router\"\n+            assert g[\"model\"] == \"glm-5.1\"\n+            assert g[\"base_url\"] == \"https://my-server.example.com/v1\"\n+            assert g[\"cred\"] == \"sk-test-key\"\n+            assert g[\"api_mode\"] == \"chat_completions\"\n+        finally:\n+            mod.clear_runtime_main()\n+\n+    def test_clear_resets_all_globals(self):\n+        \"\"\"clear_runtime_main resets all five globals to empty.\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        mod.set_runtime_main(\n+            \"custom:x\", \"m\",\n+            base_url=\"https://x.example.com\",\n+            api_key=\"[redacted:secret-assignment]\",\n+            api_mode=\"chat_completions\",\n+        )\n+        mod.clear_runtime_main()\n+        g = _get_globals(mod)\n+        for v in g.values():\n+            assert v == \"\", f\"Expected empty, got {v!r}\"\n+\n+    def test_resolve_auto_uses_globals_for_custom_provider(self):\n+        \"\"\"_resolve_auto reads base_url/api_key from globals when main_runtime is None.\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        mod.clear_runtime_main()\n+        try:\n+            mod.set_runtime_main(\n+                \"custom:test-router\",\n+                \"test-model\",\n+                base_url=\"https://custom-endpoint.example.com/v1\",\n+                api_key=\"[redacted:secret-assignment]\",\n+            )\n+\n+            with patch.object(mod, \"resolve_provider_client\") as mock_resolve:\n+                mock_resolve.return_value = (MagicMock(), \"test-model\")\n+                client, resolved = mod._resolve_auto(main_runtime=None)\n+\n+                mock_resolve.assert_called_once()\n+                call_args = mock_resolve.call_args\n+                assert call_args[0][0] == \"custom\"\n+                assert call_args[1][\"explicit_base_url\"] == \"https://custom-endpoint.example.com/v1\"\n+                assert call_args[1][\"explicit_api_key\"] == \"sk-test-123\"\n+        finally:\n+            mod.clear_runtime_main()\n+\n+    def test_explicit_main_runtime_takes_precedence(self):\n+        \"\"\"When main_runtime dict has values, globals are NOT used.\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        mod.clear_runtime_main()\n+        try:\n+            mod.set_runtime_main(\n+                \"custom:router-a\",\n+                \"model-a\",\n+                base_url=\"https://from-global.example.com\",\n+                api_key=\"[redacted:secret-assignment]\",\n+            )\n+\n+            with patch.object(mod, \"resolve_provider_client\") as mock_resolve:\n+                mock_resolve.return_value = (MagicMock(), \"model-b\")\n+                main_rt = {\n+                    \"provider\": \"custom:router-b\",\n+                    \"model\": \"model-b\",\n+                    \"base_url\": \"https://from-dict.example.com\",\n+                    \"api_key\": \"sk-dict\",\n+                }\n+                mod._resolve_auto(main_runtime=main_rt)\n+\n+                call_args = mock_resolve.call_args[1]\n+                assert call_args[\"explicit_base_url\"] == \"https://from-dict.example.com\"\n+                assert call_args[\"explicit_api_key\"] == \"sk-dict\"\n+        finally:\n+            mod.clear_runtime_main()\n+\n+    def test_backward_compatible_defaults(self):\n+        \"\"\"Calling set_runtime_main with only positional args still works.\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        mod.clear_runtime_main()\n+        try:\n+            mod.set_runtime_main(\"openrouter\", \"gpt-4o\")\n+            g = _get_globals(mod)\n+            assert g[\"provider\"] == \"openrouter\"\n+            assert g[\"model\"] == \"gpt-4o\"\n+            assert g[\"base_url\"] == \"\"\n+            assert g[\"cred\"] == \"\"\n+            assert g[\"api_mode\"] == \"\"\n+        finally:\n+            mod.clear_runtime_main()\n+\n+\n+class TestResolveAutoCustomEndToEnd:\n+    \"\"\"End-to-end routing assertions — build a *real* client (no mock on\n+    resolve_provider_client) and verify the auxiliary auto-detect chain lands\n+    on the user's custom endpoint instead of falling through to the aggregator\n+    chain.  These guard the actual user-visible symptom in #34777 (aux tasks\n+    silently routed to a fallback provider) rather than just the wiring.\n+    \"\"\"\n+\n+    @staticmethod\n+    def _client_base_url(client):\n+        for chain in ((\"base_url\",), (\"_client\", \"base_url\")):\n+            obj = client\n+            try:\n+                for attr in chain:\n+                    obj = getattr(obj, attr)\n+                return str(obj)\n+            except AttributeError:\n+                continue\n+        return None\n+\n+    def test_config_less_custom_endpoint_routes_via_global(self, tmp_path, monkeypatch):\n+        \"\"\"custom:<name> with NO config entry: the live base_url carried by\n+        set_runtime_main() must build a real client at that endpoint — not\n+        fall through to Step 2 (the regression in #34777).\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        # Hermetic: no aggregator creds, no stale OPENAI_BASE_URL.\n+        for var in (\"OPENROUTER_API_KEY\", \"NOUS_API_KEY\", \"OPENAI_API_KEY\",\n+                    \"OPENAI_BASE_URL\"):\n+            monkeypatch.delenv(var, raising=False)\n+        hermes_home = tmp_path / \".hermes\"\n+        hermes_home.mkdir()\n+        (hermes_home / \"config.yaml\").write_text(\n+            \"model:\\n\"\n+            \"  default: glm-5.1\\n\"\n+            \"  provider: 'custom:ephemeral'\\n\"\n+            \"  base_url: ''\\n\"\n+        )\n+        monkeypatch.setenv(\"HERMES_HOME\", str(hermes_home))\n+\n+        mod.clear_runtime_main()\n+        try:\n+            mod.set_runtime_main(\n+                \"custom:ephemeral\",\n+                \"glm-5.1\",\n+                base_url=\"https://ephemeral.live/v1\",\n+                api_key=\"[redacted:secret-assignment]\",\n+            )\n+            client, resolved = mod.resolve_provider_client(\"auto\", None)\n+            assert client is not None, (\n+                \"config-less custom endpoint fell through to Step 2 — \"\n+                \"the #34777 bug is back\"\n+            )\n+            assert resolved == \"glm-5.1\"\n+            base = self._client_base_url(client)\n+            assert base and base.rstrip(\"/\") == \"https://ephemeral.live/v1\"\n+        finally:\n+            mod.clear_runtime_main()\n+\n+    def test_named_custom_with_config_entry_still_routes(self, tmp_path, monkeypatch):\n+        \"\"\"Regression guard: custom:<name> WITH a custom_providers entry must\n+        still resolve to that entry's endpoint.  An earlier competing fix\n+        collapsed the provider to bare ``custom`` before resolution, which\n+        broke the named-custom branch and returned None here.\"\"\"\n+        import agent.auxiliary_client as mod\n+\n+        for var in (\"OPENROUTER_API_KEY\", \"NOUS_API_KEY\", \"OPENAI_API_KEY\",\n+                    \"OPENAI_BASE_URL\"):\n+            monkeypatch.delenv(var, raising=False)\n+        hermes_home = tmp_path / \".hermes\"\n+        hermes_home.mkdir()\n+        (hermes_home / \"config.yaml\").write_text(\n+            \"model:\\n\"\n+            \"  default: glm-5.1\\n\"\n+            \"  provider: 'custom:openclaw'\\n\"\n+            \"  base_url: ''\\n\"\n+            \"custom_providers:\\n\"\n+            \"  - name: openclaw\\n\"\n+            \"    base_url: 'https://withcfg.example/v1'\\n\"\n+            \"    model: glm-5.1\\n\"\n+            \"    api_key: [redacted:secret-assignment]\"\n+        )\n+        monkeypatch.setenv(\"HERMES_HOME\", str(hermes_home))\n+\n+        # No live base_url carried — resolution must come from config alone,\n+        # via the named-custom branch in resolve_provider_client.\n+        mod.clear_runtime_main()\n+        try:\n+            mod.set_runtime_main(\"custom:openclaw\", \"glm-5.1\")\n+            client, resolved = mod.resolve_provider_client(\"auto\", None)\n+            assert client is not None\n+            base = self._client_base_url(client)\n+            assert base and base.rstrip(\"/\") == \"https://withcfg.example/v1\"\n+        finally:\n+            mod.clear_runtime_main()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}