Fallback model visibility and user notification improvements

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #61340 fix(agent): surface fallback switch notice on successful fallback (#35419)

  merged to main

  files: agent/chat_completion_helpers.py, agent/conversation_loop.py, run_agent.py, scripts/release.py, tests/run_agent/test_retry_status_buffer.py

  Show the diff

  • agent/chat_completion_helpers.py

    @@ -1399,6 +1399,7 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
                 fb_api_mode = "bedrock_converse"
     
             old_model = agent.model
    +        old_provider = agent.provider
     
             # Clear the per-config context_length override so the fallback
             # model's actual context window is resolved instead of inheriting
    @@ -1544,6 +1545,16 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool
                 f"🔄 Primary model failed — switching to fallback: "
                 f"{fb_model} via {fb_provider}"
             )
    +        # The buffered line above is dropped on successful recovery, but a
    +        # provider/model switch is a durable state change operators must see
    +        # even when the fallback succeeds.  Record a one-shot notice that the
    +        # success path surfaces exactly once via _emit_pending_fallback_notice
    +        # (see run_agent.py); it is discarded on terminal failure since the
    +        # buffered line is flushed instead.  See fallback-observability fix.
    +        agent._pending_fallback_notice = (
    +            f"🔄 Switched to fallback model: {old_model} via {old_provider} "
    +            f"→ {fb_model} via {fb_provider}"
    +        )
             logger.info(
                 "Fallback activated: %s → %s (%s)",
                 old_model, fb_model, fb_provider,

  • agent/conversation_loop.py

    @@ -5062,8 +5062,11 @@ def _perform_api_call(next_api_kwargs):
                     # Reset retry counter/signature on successful content
                     agent._empty_content_retries = 0
                     agent._thinking_prefill_retries = 0
    -                # Successful content reached — drop any buffered retry
    -                # status from earlier failed attempts in this turn.
    +                # Successful content reached — surface the one-shot fallback
    +                # switch notice (if a fallback activated this turn) before
    +                # dropping the noisy retry buffer, so a provider/model switch
    +                # stays visible even when the fallback succeeds.
    +                agent._emit_pending_fallback_notice()
                     agent._clear_status_buffer()
     
                     from agent.agent_runtime_helpers import (

  • run_agent.py

    @@ -981,6 +981,29 @@ def _clear_status_buffer(self) -> None:
             except Exception:
                 pass
     
    +    def _emit_pending_fallback_notice(self) -> None:
    +        """Surface the one-shot fallback-switch notice on successful recovery.
    +
    +        A provider/model switch is a durable state change operators must see,
    +        unlike transient retry chatter that ``_clear_status_buffer`` drops.
    +        ``try_activate_fallback`` records the switch in
    +        ``self._pending_fallback_notice``; this emits it exactly once via
    +        ``_emit_status`` and then clears it, so a successful fallback still
    +        produces one visible notice.  On terminal failure the buffered switch
    +        line is flushed instead (and this notice discarded) — see
    +        ``_flush_status_buffer`` — so the user always sees the switch once.
    +        """
    +        try:
    +            notice = getattr(self, "_pending_fallback_notice", None)
    +            if notice:
    +                # Clear before emitting so a (swallowed) callback error can't
    +                # leave the notice set for a stale re-emit on a later turn.
    +                self._pending_fallback_notice = None
    +                self._emit_status(notice)
    +        except Exception:
    +            # Never break the conversation loop on a notice hiccup.
    +            pass
    +
         def _flush_status_buffer(self) -> None:
             """Emit buffered retry messages — call on terminal failure.
     
    @@ -988,6 +1011,10 @@ def _flush_status_buffer(self) -> None:
             was tried before the turn gave up.
             """
             try:
    +            # The buffered trace already carries the fallback switch line, so
    +            # drop any one-shot fallback notice to avoid a stale duplicate
    +            # leaking into a later successful turn.
    +            self._pending_fallback_notice = None
                 buf = getattr(self, "_retry_status_buffer", None)
                 if not buf:
                     return

  • scripts/release.py

    @@ -57,6 +57,7 @@
         "[redacted:email]": "AndreasHiltner",  # PR #56854 salvage (gateway: route multiplex profile responses through the profile's own adapter — 53-site _adapter_for_source sweep)
         "[redacted:email]": "allenliang2022",  # PR #56932 test coverage folded into #56909 salvage (408 → retryable timeout)
         "[redacted:email]": "ManniBr",  # PR #57417 partial salvage (gateway: fail-closed adapter resolution for unregistered secondary profiles)
    +    "[redacted:email]": "Umi4Life",  # PR #47377 salvage (agent: emit one-shot fallback switch notice on successful fallback so gateway users see model/provider change; #35419)
         "[redacted:email]": "austinlaw076",  # PR #57563 partial salvage (auth: lazy per-profile Anthropic OAuth file; gateway: whatsapp_cloud/line added to port-binding platform set)
         "[redacted:email]": "liuhao1024",  # PR #56993 salvage (gateway: process-level HERMES_HOME for pid/lock/status identity files; #56986)
         "[redacted:email]": "gauravsaxena1997",  # PR #59868 partial salvage (agent: guard response.text against httpx.ResponseNotRead in _summarize_api_error; #59769)

  • tests/run_agent/test_retry_status_buffer.py

    @@ -135,6 +135,86 @@ def test_mixed_kinds_replay_through_correct_channels():
         assert warns == ["warn-1"]
     
     
    +def test_pending_fallback_notice_emitted_once_on_success():
    +    """On successful recovery the one-shot fallback notice is surfaced even
    +    though the noisy retry buffer is dropped."""
    +    agent = _make_bare_agent()
    +    emitted = []
    +    agent._emit_status = lambda msg: emitted.append(msg)
    +
    +    # Simulate try_activate_fallback: buffer the noisy switch line AND record
    +    # the durable one-shot notice.
    +    agent._buffer_status("🔄 Primary model failed — switching to fallback: m2 via p2")
    +    agent._pending_fallback_notice = "🔄 Switched to fallback model: m1 via p1 → m2 via p2"
    +
    +    # Success path order: emit pending notice, then drop the buffer.
    +    agent._emit_pending_fallback_notice()
    +    agent._clear_status_buffer()
    +
    +    # The durable notice was shown exactly once; the buffered retry noise was
    +    # silently dropped.
    +    assert emitted == ["🔄 Switched to fallback model: m1 via p1 → m2 via p2"]
    +    assert agent._retry_status_buffer == []
    +    # Notice is cleared so it cannot re-emit on a later turn.
    +    assert agent._pending_fallback_notice is None
    +
    +    # A second success path with no new fallback emits nothing.
    +    agent._emit_pending_fallback_notice()
    +    assert emitted == ["🔄 Switched to fallback model: m1 via p1 → m2 via p2"]
    +
    +
    +def test_pending_fallback_notice_noop_when_unset():
    +    """No fallback this turn → no notice emitted on the success path."""
    +    agent = _make_bare_agent()
    +    emitted = []
    +    agent._emit_status = lambda msg: emitted.append(msg)
    +
    +    # No _pending_fallback_notice attribute set at all.
    +    agent._emit_pending_fallback_notice()
    +    assert emitted == []
    +
    +
    +def test_flush_discards_pending_fallback_notice():
    +    """On terminal failure the flushed buffer already carries the switch line,
    +    so the one-shot notice is discarded to avoid a stale duplicate later."""
    +    agent = _make_bare_agent()
    +    emitted = []
    +    agent._emit_status = lambda msg: emitted.append(msg)
    +
    +    agent._buffer_status("🔄 Primary model failed — switching to fallback: m2 via p2")
    +    agent._pending_fallback_notice = "🔄 Switched to fallback model: m1 via p1 → m2 via p2"
    +
    +    # Terminal failure flushes the buffered trace...
    +    agent._flush_status_buffer()
    +    assert emitted == ["🔄 Primary model failed — switching to fallback: m2 via p2"]
    +    # ...and discards the pending notice so it won't re-emit on a later turn.
    +    assert agent._pending_fallback_notice is None
    +
    +    emitted.clear()
    +    agent._emit_pending_fallback_notice()
    +    assert emitted == []
    +
    +
    +def test_pending_fallback_notice_survives_emit_callback_error():
    +    """A failing status callback must not leave the notice set for a stale
    +    re-emit, and must not raise."""
    +    agent = _make_bare_agent()
    +    seen = []
    +
    +    def boom(msg):
    +        seen.append(msg)
    +        raise RuntimeError("simulated callback failure")
    +
    +    agent._emit_status = boom
    +    agent._pending_fallback_notice = "🔄 Switched to fallback model: m1 via p1 → m2 via p2"
    +
    +    # Should not raise.
    +    agent._emit_pending_fallback_notice()
    +    # Attempt was made and the notice is cleared regardless.
    +    assert seen == ["🔄 Switched to fallback model: m1 via p1 → m2 via p2"]
    +    assert agent._pending_fallback_notice is None
    +
    +
     def test_flush_swallows_callback_exceptions():
         agent = _make_bare_agent()
         seen = []

• #115925 feat(gateway): footer can show the model that really answered behind a routing proxy or fallback (#54864)

  merged to main

  files: agent/agent_runtime_helpers.py, agent/served_model.py, agent/turn_finalizer.py, gateway/run_turn.py, gateway/runtime_footer.py, hermes_cli/config_defaults.py, tests/agent/test_served_model.py, tests/gateway/test_runtime_footer.py, website/docs/user-guide/configuration.md