Fallback provider propagation bugs and fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #15134 fix(agent): consolidated fallback/retry correctness — SSL retry, cooldown, config context, auth fallback parity

  merged to main

  files: agent/error_classifier.py, cron/scheduler.py, gateway/run.py, run_agent.py, scripts/release.py, tests/agent/test_error_classifier.py, tests/gateway/test_auth_fallback.py, tests/run_agent/test_primary_runtime_restore.py, tests/run_agent/test_switch_model_fallback_prune.py

  Show the diff

  • agent/error_classifier.py

    @@ -343,6 +343,11 @@ def classify_api_error(
         """
         status_code = _extract_status_code(error)
         error_type = type(error).__name__
    +    # Copilot/GitHub Models RateLimitError may not set .status_code; force 429
    +    # so downstream rate-limit handling (classifier reason, pool rotation,
    +    # fallback gating) fires correctly instead of misclassifying as generic.
    +    if status_code is None and error_type == "RateLimitError":
    +        status_code = 429
         body = _extract_error_body(error)
         error_code = _extract_error_code(body)
     

  • cron/scheduler.py

    @@ -895,6 +895,7 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
                 resolve_runtime_provider,
                 format_runtime_provider_error,
             )
    +        from hermes_cli.auth import AuthError
             try:
                 runtime_kwargs = {
                     "requested": job.get("provider") or os.getenv("HERMES_INFERENCE_PROVIDER"),
    @@ -902,6 +903,28 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
                 if job.get("base_url"):
                     runtime_kwargs["explicit_base_url"] = job.get("base_url")
                 runtime = resolve_runtime_provider(**runtime_kwargs)
    +        except AuthError as auth_exc:
    +            # Primary provider auth failed — try fallback chain before giving up.
    +            logger.warning("Job '%s': primary auth failed (%s), trying fallback", job_id, auth_exc)
    +            fb = _cfg.get("fallback_providers") or _cfg.get("fallback_model")
    +            fb_list = (fb if isinstance(fb, list) else [fb]) if fb else []
    +            runtime = None
    +            for entry in fb_list:
    +                if not isinstance(entry, dict):
    +                    continue
    +                try:
    +                    fb_kwargs = {"requested": entry.get("provider")}
    +                    if entry.get("base_url"):
    +                        fb_kwargs["explicit_base_url"] = entry["base_url"]
    +                    if entry.get("api_key"):
    +                        fb_kwargs["explicit_api_key"] = entry["api_key"]
    +                    runtime = resolve_runtime_provider(**fb_kwargs)
    +                    logger.info("Job '%s': fallback resolved to %s", job_id, runtime.get("provider"))
    +                    break
    +                except Exception as fb_exc:
    +                    logger.debug("Job '%s': fallback %s failed: %s", job_id, entry.get("provider"), fb_exc)
    +            if runtime is None:
    +                raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc
             except Exception as exc:
                 message = format_runtime_provider_error(exc)
                 raise RuntimeError(message) from exc

  • gateway/run.py

    @@ -350,16 +350,30 @@ def _expand_whatsapp_auth_aliases(identifier: str) -> set:
     
     
     def _resolve_runtime_agent_kwargs() -> dict:
    -    """Resolve provider credentials for gateway-created AIAgent instances."""
    +    """Resolve provider credentials for gateway-created AIAgent instances.
    +
    +    If the primary provider fails with an authentication error, attempt to
    +    resolve credentials using the fallback provider chain from config.yaml
    +    before giving up.
    +    """
         from hermes_cli.runtime_provider import (
             resolve_runtime_provider,
             format_runtime_provider_error,
         )
    +    from hermes_cli.auth import AuthError
     
         try:
             runtime = resolve_runtime_provider(
                 requested=os.getenv("HERMES_INFERENCE_PROVIDER"),
             )
    +    except AuthError as auth_exc:
    +        # Primary provider auth failed (expired token, revoked key, etc.).
    +        # Try the fallback provider chain before raising.
    +        logger.warning("Primary provider auth failed: %s — trying fallback", auth_exc)
    +        fb_config = _try_resolve_fallback_provider()
    +        if fb_config is not None:
    +            return fb_config
    +        raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc
         except Exception as exc:
             raise RuntimeError(format_runtime_provider_error(exc)) from exc
     
    @@ -374,6 +388,48 @@ def _resolve_runtime_agent_kwargs() -> dict:
         }
     
     
    +def _try_resolve_fallback_provider() -> dict | None:
    +    """Attempt to resolve credentials from the fallback_model/fallback_providers config."""
    +    from hermes_cli.runtime_provider import resolve_runtime_provider
    +    try:
    +        import yaml as _y
    +        cfg_path = _hermes_home / "config.yaml"
    +        if not cfg_path.exists():
    +            return None
    +        with open(cfg_path, encoding="utf-8") as _f:
    +            cfg = _y.safe_load(_f) or {}
    +        fb = cfg.get("fallback_providers") or cfg.get("fallback_model")
    +        if not fb:
    +            return None
    +        # Normalize to list
    +        fb_list = fb if isinstance(fb, list) else [fb]
    +        for entry in fb_list:
    +            if not isinstance(entry, dict):
    +                continue
    +            try:
    +                runtime = resolve_runtime_provider(
    +                    requested=entry.get("provider"),
    +                    explicit_base_url=entry.get("base_url"),
    +                    explicit_api_key=[redacted:secret-assignment]"api_key"),
    +                )
    +                logger.info("Fallback provider resolved: %s", runtime.get("provider"))
    +                return {
    +                    "api_key": runtime.get("api_key"),
    +                    "base_url": runtime.get("base_url"),
    +                    "provider": runtime.get("provider"),
    +                    "api_mode": runtime.get("api_mode"),
    +                    "command": runtime.get("command"),
    +                    "args": list(runtime.get("args") or []),
    +                    "credential_pool": runtime.get("credential_pool"),
    +                }
    +            except Exception as fb_exc:
    +                logger.debug("Fallback entry %s failed: %s", entry.get("provider"), fb_exc)
    +                continue
    +    except Exception:
    +        pass
    +    return None
    +
    +
     def _build_media_placeholder(event) -> str:
         """Build a text placeholder for media-only events so they aren't dropped.
     

  • run_agent.py

    @@ -31,6 +31,7 @@
     import os
     import random
     import re
    +import ssl
     import sys
     import tempfile
     import time
    @@ -2118,12 +2119,14 @@ def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mod
             # ("switched to anthropic, tui keeps trying openrouter").
             old_norm = (old_provider or "").strip().lower()
             new_norm = (new_provider or "").strip().lower()
    +        fallback_chain = list(getattr(self, "_fallback_chain", []) or [])
             if old_norm and new_norm and old_norm != new_norm:
    -            self._fallback_chain = [
    -                entry for entry in self._fallback_chain
    +            fallback_chain = [
    +                entry for entry in fallback_chain
                     if (entry.get("provider") or "").strip().lower() not in {old_norm, new_norm}
                 ]
    -            self._fallback_model = self._fallback_chain[0] if self._fallback_chain else None
    +        self._fallback_chain = fallback_chain
    +        self._fallback_model = fallback_chain[0] if fallback_chain else None
     
             logging.info(
                 "Model switched in-place: %s (%s) -> %s (%s)",
    @@ -6491,7 +6494,7 @@ def _call():
     
         # ── Provider fallback ──────────────────────────────────────────────────
     
    -    def _try_activate_fallback(self) -> bool:
    +    def _try_activate_fallback(self, reason: "FailoverReason | None" = None) -> bool:
             """Switch to the next fallback model/provider in the chain.
     
             Called when the current model is failing after retries.  Swaps the
    @@ -6503,6 +6506,15 @@ def _try_activate_fallback(self) -> bool:
             auth resolution and client construction — no duplicated provider→key
             mappings.
             """
    +        if reason in (FailoverReason.rate_limit, FailoverReason.billing):
    +            # Only start cooldown when leaving the primary provider.  If we're
    +            # already on a fallback and chain-switching, the primary wasn't the
    +            # source of the 429 so the cooldown should not be reset/extended.
    +            fallback_already_active = bool(getattr(self, "_fallback_activated", False))
    +            current_provider = (getattr(self, "provider", "") or "").strip().lower()
    +            primary_provider = ((self._primary_runtime or {}).get("provider") or "").strip().lower()
    +            if (not fallback_already_active) or (primary_provider and current_provider == primary_provider):
    +                self._rate_limited_until = time.monotonic() + 60
             if self._fallback_index >= len(self._fallback_chain):
                 return False
     
    @@ -6639,11 +6651,15 @@ def _try_activate_fallback(self) -> bool:
                 # Without this, compression decisions use the primary model's
                 # context window (e.g. 200K) instead of the fallback's (e.g. 32K),
                 # causing oversized sessions to overflow the fallback.
    +            # Also pass _config_context_length so the explicit config override
    +            # (model.context_length in config.yaml) is respected — without this,
    +            # the fallback activation drops to 128K even when config says 204800.
                 if hasattr(self, 'context_compressor') and self.context_compressor:
                     from agent.model_metadata import get_model_context_length
                     fb_context_length = get_model_context_length(
                         self.model, base_url=self.base_url,
                         api_key=[redacted:secret-assignment] provider=self.provider,
    +                    config_context_length=getattr(self, "_config_context_length", None),
                     )
                     self.context_compressor.update_model(
                         model=self.model,
    @@ -6682,6 +6698,9 @@ def _restore_primary_runtime(self) -> bool:
             if not self._fallback_activated:
                 return False
     
    +        if getattr(self, "_rate_limited_until", 0) > time.monotonic():
    +            return False  # primary still in rate-limit cooldown, stay on fallback
    +
             rt = self._primary_runtime
             try:
                 # ── Core runtime state ──
    @@ -10658,7 +10677,7 @@ def _stop_spinner():
                             )
                             if not pool_may_recover:
                                 self._emit_status("⚠️ Rate limited — switching to fallback provider...")
    -                            if self._try_activate_fallback():
    +                            if self._try_activate_fallback(reason=classified.reason):
                                     retry_count = 0
                                     compression_attempts = 0
                                     primary_recovery_attempted = False
    @@ -10923,6 +10942,14 @@ def _stop_spinner():
                             and not isinstance(
                                 api_error, (UnicodeEncodeError, json.JSONDecodeError)
                             )
    +                        # ssl.SSLError (and its subclass SSLCertVerificationError)
    +                        # inherits from OSError *and* ValueError via Python MRO,
    +                        # so the isinstance(ValueError) check above would
    +                        # misclassify a TLS transport failure as a local
    +                        # programming bug and abort without retrying.  Exclude
    +                        # ssl.SSLError explicitly so the error classifier's
    +                        # retryable=True mapping takes effect instead.
    +                        and not isinstance(api_error, ssl.SSLError)
                         )
                         is_client_error = (
                             is_local_validation_error

  • scripts/release.py

    @@ -187,6 +187,11 @@
         "[redacted:email]": "alexzhu0",
         "[redacted:email]": "AntAISecurityLab",
         "[redacted:email]": "briandevans",
    +    "[redacted:email]": "Bartok9",
    +    "[redacted:email]": "CruxExperts",
    +    "[redacted:email]": "LeonSGP43",
    +    "[redacted:email]": "vlwkaos",
    +    "[redacted:email]": "Tranquil-Flow",
         "[redacted:email]": "nummanali",
         "[redacted:email]": "whitehatjr1001",
         "[redacted:email]": "0xNyk",

  • tests/agent/test_error_classifier.py

    @@ -1094,3 +1094,37 @@ def test_real_ssl_error_type_classifies_as_timeout(self):
             result = classify_api_error(e)
             assert result.reason == FailoverReason.timeout
             assert result.retryable is True
    +
    +# ── Test: RateLimitError without status_code (Copilot/GitHub Models) ──────────
    +
    +class TestRateLimitErrorWithoutStatusCode:
    +    """Regression tests for the Copilot/GitHub Models edge case where the
    +    OpenAI SDK raises RateLimitError but does not populate .status_code."""
    +
    +    def _make_rate_limit_error(self, status_code=None):
    +        """Create an exception whose class name is 'RateLimitError' with
    +        an optionally missing status_code, mirroring the OpenAI SDK shape."""
    +        cls = type("RateLimitError", (Exception,), {})
    +        e = cls("You have exceeded your rate limit.")
    +        e.status_code = status_code  # None simulates the Copilot case
    +        return e
    +
    +    def test_rate_limit_error_without_status_code_classified_as_rate_limit(self):
    +        """RateLimitError with status_code=None must classify as rate_limit."""
    +        e = self._make_rate_limit_error(status_code=None)
    +        result = classify_api_error(e, provider="copilot", model="gpt-4o")
    +        assert result.reason == FailoverReason.rate_limit
    +
    +    def test_rate_limit_error_with_status_code_429_classified_as_rate_limit(self):
    +        """RateLimitError that does set status_code=429 still classifies correctly."""
    +        e = self._make_rate_limit_error(status_code=429)
    +        result = classify_api_error(e, provider="copilot", model="gpt-4o")
    +        assert result.reason == FailoverReason.rate_limit
    +
    +    def test_other_error_without_status_code_not_forced_to_rate_limit(self):
    +        """A non-RateLimitError with missing status_code must NOT be forced to 429."""
    +        cls = type("APIError", (Exception,), {})
    +        e = cls("something went wrong")
    +        e.status_code = None
    +        result = classify_api_error(e, provider="copilot", model="gpt-4o")
    +        assert result.reason != FailoverReason.rate_limit

  • tests/gateway/test_auth_fallback.py

    @@ -0,0 +1,73 @@
    +"""Test that AuthError triggers fallback provider resolution (#7230)."""
    +
    +import os
    +from unittest.mock import patch, MagicMock
    +
    +import pytest
    +
    +
    +class TestResolveRuntimeAgentKwargsAuthFallback:
    +    """_resolve_runtime_agent_kwargs should try fallback on AuthError."""
    +
    +    def test_auth_error_tries_fallback(self, tmp_path, monkeypatch):
    +        """When primary provider raises AuthError, fallback is attempted."""
    +        from hermes_cli.auth import AuthError
    +
    +        # Create a config with fallback
    +        config_path = tmp_path / "config.yaml"
    +        config_path.write_text(
    +            "model:\n  provider: openai-codex\n"
    +            "fallback_model:\n  provider: openrouter\n"
    +            "  model: meta-llama/llama-4-maverick\n"
    +        )
    +
    +        monkeypatch.setattr("gateway.run._hermes_home", tmp_path)
    +
    +        call_count = {"n": 0}
    +
    +        def _mock_resolve(**kwargs):
    +            call_count["n"] += 1
    +            requested = kwargs.get("requested", "")
    +            if requested and "codex" in str(requested).lower():
    +                raise AuthError("Codex token refresh failed with status 401")
    +            return {
    +                "api_key": "fallback-key",
    +                "base_url": "https://openrouter.ai/api/v1",
    +                "provider": "openrouter",
    +                "api_mode": "openai_chat",
    +                "command": None,
    +                "args": None,
    +                "credential_pool": None,
    +            }
    +
    +        monkeypatch.setenv("HERMES_INFERENCE_PROVIDER", "openai-codex")
    +
    +        with patch(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            side_effect=_mock_resolve,
    +        ):
    +            from gateway.run import _resolve_runtime_agent_kwargs
    +            result = _resolve_runtime_agent_kwargs()
    +
    +        assert result["provider"] == "openrouter"
    +        assert result["api_key"] == "fallback-key"
    +        # Should have been called at least twice (primary + fallback)
    +        assert call_count["n"] >= 2
    +
    +    def test_auth_error_no_fallback_raises(self, tmp_path, monkeypatch):
    +        """When primary fails and no fallback configured, RuntimeError is raised."""
    +        from hermes_cli.auth import AuthError
    +
    +        config_path = tmp_path / "config.yaml"
    +        config_path.write_text("model:\n  provider: openai-codex\n")
    +
    +        monkeypatch.setattr("gateway.run._hermes_home", tmp_path)
    +        monkeypatch.setenv("HERMES_INFERENCE_PROVIDER", "openai-codex")
    +
    +        with patch(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            side_effect=AuthError("token expired"),
    +        ):
    +            from gateway.run import _resolve_runtime_agent_kwargs
    +            with pytest.raises(RuntimeError):
    +                _resolve_runtime_agent_kwargs()

  • tests/run_agent/test_primary_runtime_restore.py

    @@ -446,3 +446,85 @@ def test_full_cycle_fallback_then_restore(self):
             assert agent._fallback_index == 0
             assert agent.provider == "custom"
             assert agent.base_url == "https://my-llm.example.com/v1"
    +
    +
    +# =============================================================================
    +# Rate-limit cooldown gate
    +# =============================================================================
    +
    +class TestRateLimitCooldown:
    +    """Verify _restore_primary_runtime() respects the 60s rate-limit cooldown."""
    +
    +    def test_restore_blocked_during_cooldown(self):
    +        """While _rate_limited_until is in the future, restore returns False."""
    +        agent = _make_agent(
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback()
    +
    +        assert agent._fallback_activated is True
    +
    +        # Manually set cooldown well into the future
    +        agent._rate_limited_until = time.monotonic() + 60
    +
    +        result = agent._restore_primary_runtime()
    +        assert result is False
    +        assert agent._fallback_activated is True  # still on fallback
    +
    +    def test_restore_allowed_after_cooldown_expires(self):
    +        """Once the cooldown window passes, restore proceeds normally."""
    +        agent = _make_agent(
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback()
    +
    +        assert agent._fallback_activated is True
    +
    +        # Cooldown already expired
    +        agent._rate_limited_until = time.monotonic() - 1
    +
    +        with patch("run_agent.OpenAI", return_value=MagicMock()):
    +            result = agent._restore_primary_runtime()
    +
    +        assert result is True
    +        assert agent._fallback_activated is False
    +
    +    def test_cooldown_set_on_rate_limit_reason(self):
    +        """_try_activate_fallback with rate_limit reason sets _rate_limited_until."""
    +        from run_agent import FailoverReason
    +        agent = _make_agent(
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        before = time.monotonic()
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback(reason=FailoverReason.rate_limit)
    +
    +        assert hasattr(agent, "_rate_limited_until")
    +        assert agent._rate_limited_until > before + 50  # ~60s from now
    +
    +    def test_cooldown_not_set_when_already_on_fallback(self):
    +        """Chain-switching while already on fallback must not reset cooldown."""
    +        from run_agent import FailoverReason
    +        agent = _make_agent(
    +            fallback_model=[
    +                {"provider": "openrouter", "model": "model-a"},
    +                {"provider": "anthropic", "model": "model-b"},
    +            ],
    +        )
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            # First call: leaving primary → cooldown should be set
    +            agent._try_activate_fallback(reason=FailoverReason.rate_limit)
    +            first_cooldown = getattr(agent, "_rate_limited_until", 0)
    +
    +            # Second call: already on fallback (provider != primary) → cooldown must not advance
    +            agent._try_activate_fallback(reason=FailoverReason.rate_limit)
    +            second_cooldown = getattr(agent, "_rate_limited_until", 0)
    +
    +        # second call should not have extended the cooldown
    +        assert second_cooldown == first_cooldown

  • tests/run_agent/test_switch_model_fallback_prune.py

    @@ -78,6 +78,17 @@ def test_switch_with_empty_chain_stays_empty():
         assert agent._fallback_model is None
     
     
    +def test_switch_initializes_missing_fallback_attrs():
    +    agent = _make_agent([])
    +    del agent._fallback_chain
    +    del agent._fallback_model
    +
    +    _switch_to_anthropic(agent)
    +
    +    assert agent._fallback_chain == []
    +    assert agent._fallback_model is None
    +
    +
     def test_switch_within_same_provider_preserves_chain():
         chain = [{"provider": "openrouter", "model": "x-ai/grok-4"}]
         agent = _make_agent(chain)

• #42625 fix(tui): wire fallback provider chain into desktop GUI / TUI / dashboard chat (salvage #18310)

  merged to main

  files: scripts/release.py, tests/test_tui_gateway_server.py, tui_gateway/server.py

  Show the diff

  • scripts/release.py

    @@ -58,6 +58,7 @@
         "[redacted:email]": "RyTsYdUp",
         "[redacted:email]": "ProgramCaiCai",
         "[redacted:email]": "bmoore210",
    +    "[redacted:email]": "psionic73",
         "[redacted:email]": "manishbyatroy",
         "[redacted:email]": "TheGardenGallery",
         "[redacted:email]": "randomsnowflake",

  • tests/test_tui_gateway_server.py

    @@ -902,6 +902,115 @@ def fake_detect(model, current_provider):
         )
     
     
    +def test_load_fallback_model_merges_chain_providers_first(monkeypatch):
    +    # Parity with HermesCLI / gateway: fallback_providers stays first and keeps
    +    # its order, with any distinct legacy fallback_model entry merged in after
    +    # (deduped on provider/model/base_url).
    +    fallback_chain = [
    +        {"provider": "openrouter", "model": "openai/gpt-5.5"},
    +        {"provider": "anthropic", "model": "claude-sonnet-4-6"},
    +    ]
    +    monkeypatch.setattr(
    +        server,
    +        "_load_cfg",
    +        lambda: {
    +            "fallback_model": {"provider": "legacy", "model": "legacy-model"},
    +            "fallback_providers": fallback_chain,
    +        },
    +    )
    +
    +    assert server._load_fallback_model() == [
    +        {"provider": "openrouter", "model": "openai/gpt-5.5"},
    +        {"provider": "anthropic", "model": "claude-sonnet-4-6"},
    +        {"provider": "legacy", "model": "legacy-model"},
    +    ]
    +
    +
    +def test_make_agent_passes_configured_fallback_chain(monkeypatch):
    +    captured = {}
    +    fallback_chain = [
    +        {"provider": "openrouter", "model": "openai/gpt-5.5"},
    +    ]
    +
    +    def fake_agent(**kwargs):
    +        captured.update(kwargs)
    +        return types.SimpleNamespace(model=kwargs.get("model"))
    +
    +    monkeypatch.delenv("HERMES_MODEL", raising=False)
    +    monkeypatch.delenv("HERMES_INFERENCE_MODEL", raising=False)
    +    monkeypatch.delenv("HERMES_TUI_PROVIDER", raising=False)
    +    monkeypatch.setattr(
    +        server,
    +        "_load_cfg",
    +        lambda: {
    +            "model": {"default": "gpt-5.5", "provider": "openai-codex"},
    +            "fallback_providers": fallback_chain,
    +        },
    +    )
    +    monkeypatch.setattr(
    +        "hermes_cli.runtime_provider.resolve_runtime_provider",
    +        lambda requested=None, target_model=None: {
    +            "provider": "openai-codex",
    +            "base_url": "https://chatgpt.com/backend-api/codex",
    +            "api_key": "token",
    +            "api_mode": "codex_responses",
    +            "credential_pool": None,
    +        },
    +    )
    +    monkeypatch.setattr("run_agent.AIAgent", fake_agent)
    +    monkeypatch.setattr(server, "_load_enabled_toolsets", lambda: ["file"])
    +    monkeypatch.setattr(server, "_get_db", lambda: None)
    +
    +    agent = server._make_agent("sid", "session-key")
    +
    +    assert agent.model == "gpt-5.5"
    +    assert captured["fallback_model"] == fallback_chain
    +    assert captured["platform"] == "tui"
    +
    +
    +def test_background_agent_kwargs_preserves_full_fallback_chain(monkeypatch):
    +    chain = [
    +        {"provider": "openrouter", "model": "openai/gpt-5.5"},
    +        {"provider": "anthropic", "model": "claude-sonnet-4-6"},
    +    ]
    +    agent = types.SimpleNamespace(
    +        model="gpt-5.5",
    +        provider="openai-codex",
    +        _fallback_chain=chain,
    +    )
    +    monkeypatch.setattr(server, "_load_cfg", lambda: {"max_turns": 25})
    +    monkeypatch.setattr(server, "_load_enabled_toolsets", lambda: ["file"])
    +    monkeypatch.setattr(server, "_get_db", lambda: None)
    +
    +    kwargs = server._background_agent_kwargs(agent, "task-id")
    +
    +    assert kwargs["fallback_model"] == chain
    +
    +
    +def test_background_agent_kwargs_preserves_empty_fallback_chain(monkeypatch):
    +    agent = types.SimpleNamespace(
    +        model="gpt-5.5",
    +        provider="anthropic",
    +        _fallback_chain=[],
    +    )
    +    monkeypatch.setattr(
    +        server,
    +        "_load_cfg",
    +        lambda: {
    +            "max_turns": 25,
    +            "fallback_providers": [
    +                {"provider": "openrouter", "model": "openai/gpt-5.5"},
    +            ],
    +        },
    +    )
    +    monkeypatch.setattr(server, "_load_enabled_toolsets", lambda: ["file"])
    +    monkeypatch.setattr(server, "_get_db", lambda: None)
    +
    +    kwargs = server._background_agent_kwargs(agent, "task-id")
    +
    +    assert kwargs["fallback_model"] == []
    +
    +
     def test_startup_runtime_resolves_short_alias_without_network(monkeypatch):
         monkeypatch.setenv("HERMES_MODEL", "sonnet")
         monkeypatch.delenv("HERMES_TUI_PROVIDER", raising=False)

  • tui_gateway/server.py

    @@ -2585,6 +2585,29 @@ def _parse_tui_skills_env() -> list[str]:
         return skills
     
     
    +def _load_fallback_model():
    +    """Return the configured fallback chain for TUI-created agents.
    +
    +    Delegates to the shared ``get_fallback_chain`` helper so the TUI path
    +    stays in parity with ``HermesCLI.__init__`` and ``gateway/run.py``:
    +    ``fallback_providers`` is the primary source of truth and keeps its
    +    order, with legacy ``fallback_model`` entries merged in afterwards
    +    (deduped on provider/model/base_url).
    +    """
    +    from hermes_cli.fallback_config import get_fallback_chain
    +
    +    return get_fallback_chain(_load_cfg())
    +
    +
    +def _agent_fallback_model(agent):
    +    """Return an agent's fallback chain without rehydrating deliberately empty chains."""
    +    if hasattr(agent, "_fallback_chain"):
    +        return getattr(agent, "_fallback_chain") or []
    +    if hasattr(agent, "_fallback_model"):
    +        return getattr(agent, "_fallback_model", None)
    +    return _load_fallback_model()
    +
    +
     def _background_agent_kwargs(agent, task_id: str) -> dict:
         cfg = _load_cfg()
     
    @@ -2619,7 +2642,7 @@ def _background_agent_kwargs(agent, task_id: str) -> dict:
             "request_overrides": dict(getattr(agent, "request_overrides", {}) or {}),
             "platform": "tui",
             "session_db": _get_db(),
    -        "fallback_model": getattr(agent, "_fallback_model", None),
    +        "fallback_model": _agent_fallback_model(agent),
         }
     
     
    @@ -2878,6 +2901,7 @@ def _make_agent(
             pass_session_id=is_truthy_value(os.environ.get("HERMES_TUI_PASS_SESSION_ID")),
             skip_context_files=is_truthy_value(os.environ.get("HERMES_IGNORE_RULES")),
             skip_memory=is_truthy_value(os.environ.get("HERMES_IGNORE_RULES")),
    +        fallback_model=_load_fallback_model(),
             **_agent_cbs(sid),
         )
     

• #52797 fix(gateway): add init-time provider fallback to TUI/gateway _make_agent (#47627)

  merged to main

  files: tests/test_tui_gateway_server.py, tui_gateway/server.py

  Show the diff

  • tests/test_tui_gateway_server.py

    @@ -8173,3 +8173,143 @@ def test_persist_model_switch_clears_stale_base_url(tmp_path, monkeypatch):
         assert saved["model"]["provider"] == "anthropic"
         # Stale custom base_url must be cleared (null coalesces to absent on read).
         assert not saved["model"].get("base_url"), saved["model"].get("base_url")
    +
    +
    +# ---------------------------------------------------------------------------
    +# _resolve_runtime_with_fallback — init-time provider fallback
    +# ---------------------------------------------------------------------------
    +
    +class TestResolveRuntimeWithFallback:
    +    """Tests for _resolve_runtime_with_fallback(): init-time provider
    +    fallback when the primary provider raises AuthError."""
    +
    +    def test_primary_success_returns_runtime(self, monkeypatch):
    +        """When primary resolve succeeds, return its result directly."""
    +        expected = {"provider": "openai", "api_key": "tok"}
    +        monkeypatch.setattr(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            lambda **kw: expected,
    +        )
    +        result = server._resolve_runtime_with_fallback({"requested": "openai"})
    +        assert result == expected
    +
    +    def test_auth_error_tries_fallback_chain(self, monkeypatch):
    +        """On AuthError from primary, walk fallback_providers chain."""
    +        from hermes_cli.auth import AuthError
    +
    +        fallback_runtime = {"provider": "deepseek", "api_key": "fb-tok"}
    +
    +        def fake_resolve(**kwargs):
    +            if kwargs.get("requested") == "openai-codex":
    +                raise AuthError("No Codex credentials stored")
    +            return fallback_runtime
    +
    +        monkeypatch.setattr(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            fake_resolve,
    +        )
    +        monkeypatch.setattr(
    +            server,
    +            "_load_fallback_model",
    +            lambda: [{"provider": "deepseek", "model": "deepseek-v4-pro"}],
    +        )
    +        result = server._resolve_runtime_with_fallback(
    +            {"requested": "openai-codex"},
    +        )
    +        assert result == fallback_runtime
    +
    +    def test_auth_error_all_fallbacks_fail_raises(self, monkeypatch):
    +        """When all fallbacks also fail, re-raise the original AuthError."""
    +        from hermes_cli.auth import AuthError
    +
    +        def fake_resolve(**kwargs):
    +            raise AuthError("No credentials for " + str(kwargs.get("requested")))
    +
    +        monkeypatch.setattr(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            fake_resolve,
    +        )
    +        monkeypatch.setattr(
    +            server,
    +            "_load_fallback_model",
    +            lambda: [{"provider": "deepseek", "model": "deepseek-v4-pro"}],
    +        )
    +        import pytest
    +
    +        with pytest.raises(AuthError, match="No credentials for openai-codex"):
    +            server._resolve_runtime_with_fallback(
    +                {"requested": "openai-codex"},
    +            )
    +
    +    def test_auth_error_skips_non_dict_entries(self, monkeypatch):
    +        """Fallback chain entries that are not dicts are skipped."""
    +        from hermes_cli.auth import AuthError
    +
    +        fallback_runtime = {"provider": "anthropic", "api_key": "ant-tok"}
    +
    +        def fake_resolve(**kwargs):
    +            if kwargs.get("requested") == "openai-codex":
    +                raise AuthError("No Codex credentials stored")
    +            return fallback_runtime
    +
    +        monkeypatch.setattr(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            fake_resolve,
    +        )
    +        monkeypatch.setattr(
    +            server,
    +            "_load_fallback_model",
    +            lambda: [
    +                "invalid-string-entry",
    +                {"provider": "anthropic", "model": "claude-sonnet-4-6"},
    +            ],
    +        )
    +        result = server._resolve_runtime_with_fallback(
    +            {"requested": "openai-codex"},
    +        )
    +        assert result == fallback_runtime
    +
    +    def test_make_agent_uses_fallback_on_auth_error(self, monkeypatch):
    +        """Integration: _make_agent falls back to configured fallback
    +        provider when the primary provider raises AuthError."""
    +        import types
    +
    +        from hermes_cli.auth import AuthError
    +
    +        captured = {}
    +        fallback_runtime = {"provider": "deepseek", "api_key": "fb-tok"}
    +
    +        def fake_resolve(**kwargs):
    +            if kwargs.get("requested") == "openai-codex":
    +                raise AuthError("No Codex credentials stored")
    +            return fallback_runtime
    +
    +        def fake_agent(**kwargs):
    +            captured.update(kwargs)
    +            return types.SimpleNamespace(model=kwargs.get("model"))
    +
    +        monkeypatch.delenv("HERMES_MODEL", raising=False)
    +        monkeypatch.delenv("HERMES_INFERENCE_MODEL", raising=False)
    +        monkeypatch.delenv("HERMES_TUI_PROVIDER", raising=False)
    +        monkeypatch.setattr(
    +            server,
    +            "_load_cfg",
    +            lambda: {
    +                "model": {"default": "gpt-5.5", "provider": "openai-codex"},
    +                "fallback_providers": [
    +                    {"provider": "deepseek", "model": "deepseek-v4-pro"},
    +                ],
    +            },
    +        )
    +        monkeypatch.setattr(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            fake_resolve,
    +        )
    +        monkeypatch.setattr("run_agent.AIAgent", fake_agent)
    +        monkeypatch.setattr(server, "_load_enabled_toolsets", lambda: ["file"])
    +        monkeypatch.setattr(server, "_get_db", lambda: None)
    +
    +        agent = server._make_agent("sid", "session-key")
    +
    +        assert agent.model == "gpt-5.5"
    +        assert captured["provider"] == "deepseek"

  • tui_gateway/server.py

    @@ -4081,7 +4081,6 @@ def _wait_then_refresh() -> None:
                 info = _session_info(agent, session)
             # Emit outside the lock — write_json must not block under _sessions_lock.
             _emit("session.info", sid, info)
    -
         threading.Thread(
             target=_wait_then_refresh,
             name=f"tui-mcp-late-refresh-{sid}",
    @@ -4089,6 +4088,50 @@ def _wait_then_refresh() -> None:
         ).start()
     
     
    +def _resolve_runtime_with_fallback(
    +    resolve_kwargs: dict | None = None,
    +) -> dict:
    +    """Resolve runtime provider with init-time fallback on auth failure.
    +
    +    Mirrors the fallback pattern in ``cron/scheduler.py`` and
    +    ``hermes_cli/cli_agent_setup_mixin.py``: when the primary provider
    +    raises ``AuthError``, walk the configured ``fallback_providers`` /
    +    ``fallback_model`` chain before giving up.
    +    """
    +    from hermes_cli.auth import AuthError
    +    from hermes_cli.runtime_provider import resolve_runtime_provider
    +
    +    kwargs = resolve_kwargs or {}
    +    try:
    +        return resolve_runtime_provider(**kwargs)
    +    except AuthError as primary_exc:
    +        fb_chain = _load_fallback_model() or []
    +        for entry in fb_chain:
    +            if not isinstance(entry, dict):
    +                continue
    +            fb_provider = (entry.get("provider") or "").strip()
    +            if not fb_provider:
    +                continue
    +            try:
    +                fb_kwargs: dict = {"requested": fb_provider}
    +                if entry.get("base_url"):
    +                    fb_kwargs["explicit_base_url"] = entry["base_url"]
    +                if entry.get("api_key"):
    +                    fb_kwargs["explicit_api_key"] = entry["api_key"]
    +                runtime = resolve_runtime_provider(**fb_kwargs)
    +                import logging
    +
    +                logging.getLogger(__name__).warning(
    +                    "Primary auth failed (%s), falling back to %s",
    +                    primary_exc,
    +                    fb_provider,
    +                )
    +                return runtime
    +            except Exception:
    +                continue
    +        raise
    +
    +
     def _make_agent(
         sid: str,
         key: str,
    @@ -4100,7 +4143,6 @@ def _make_agent(
         service_tier_override: str | None = None,
     ):
         from run_agent import AIAgent
    -    from hermes_cli.runtime_provider import resolve_runtime_provider
     
         # MCP tool discovery runs in a background daemon thread at startup so a
         # dead server can't freeze the shell.  The agent snapshots its tool list
    @@ -4169,11 +4211,9 @@ def _make_agent(
                     # Failing identity recovery, still hand the base_url to the
                     # direct-alias branch so pool/env credentials resolve for it.
                     resolve_kwargs["explicit_base_url"] = override_base_url
    -        runtime = resolve_runtime_provider(
    -            requested=requested_provider,
    -            target_model=model or None,
    -            **resolve_kwargs,
    -        )
    +        resolve_kwargs["requested"] = requested_provider
    +        resolve_kwargs["target_model"] = model or None
    +        runtime = _resolve_runtime_with_fallback(resolve_kwargs)
             # The switch already resolved concrete credentials/endpoint; honor them
             # so a custom/named endpoint survives the rebuild even if global
             # resolution would pick a different one.
    @@ -4189,10 +4229,10 @@ def _make_agent(
                 model = model_override
             if provider_override:
                 requested_provider = provider_override
    -        runtime = resolve_runtime_provider(
    -            requested=requested_provider,
    -            target_model=model or None,
    -        )
    +        runtime = _resolve_runtime_with_fallback({
    +            "requested": requested_provider,
    +            "target_model": model or None,
    +        })
         _pr = _load_provider_routing()
         return AIAgent(
             model=model,

• #65656 fix(auth): preserve fallback routes and OAuth state (salvage #65264)

  merged to main

  files: agent/credential_pool.py, cron/scheduler.py, hermes_cli/auth.py, tests/agent/test_credential_pool_oauth_writethrough.py, tests/cron/test_scheduler.py, tests/hermes_cli/test_auth_profile_fallback.py, tests/test_tui_gateway_server.py, tui_gateway/server.py

• #65682 fix(auth): honor per-entry key_env when resolving fallback providers (salvage #43861)

  merged to main

  files: cron/scheduler.py, gateway/run.py, hermes_cli/cli_agent_setup_mixin.py, hermes_cli/fallback_config.py, tests/hermes_cli/test_fallback_config.py, tests/test_tui_gateway_server.py, tui_gateway/server.py