Agent SSL errors retry classification fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #15134 fix(agent): consolidated fallback/retry correctness — SSL retry, cooldown, config context, auth fallback parity

  merged to main

  files: agent/error_classifier.py, cron/scheduler.py, gateway/run.py, run_agent.py, scripts/release.py, tests/agent/test_error_classifier.py, tests/gateway/test_auth_fallback.py, tests/run_agent/test_primary_runtime_restore.py, tests/run_agent/test_switch_model_fallback_prune.py

  Show the diff

  • agent/error_classifier.py

    @@ -343,6 +343,11 @@ def classify_api_error(
         """
         status_code = _extract_status_code(error)
         error_type = type(error).__name__
    +    # Copilot/GitHub Models RateLimitError may not set .status_code; force 429
    +    # so downstream rate-limit handling (classifier reason, pool rotation,
    +    # fallback gating) fires correctly instead of misclassifying as generic.
    +    if status_code is None and error_type == "RateLimitError":
    +        status_code = 429
         body = _extract_error_body(error)
         error_code = _extract_error_code(body)
     

  • cron/scheduler.py

    @@ -895,6 +895,7 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
                 resolve_runtime_provider,
                 format_runtime_provider_error,
             )
    +        from hermes_cli.auth import AuthError
             try:
                 runtime_kwargs = {
                     "requested": job.get("provider") or os.getenv("HERMES_INFERENCE_PROVIDER"),
    @@ -902,6 +903,28 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
                 if job.get("base_url"):
                     runtime_kwargs["explicit_base_url"] = job.get("base_url")
                 runtime = resolve_runtime_provider(**runtime_kwargs)
    +        except AuthError as auth_exc:
    +            # Primary provider auth failed — try fallback chain before giving up.
    +            logger.warning("Job '%s': primary auth failed (%s), trying fallback", job_id, auth_exc)
    +            fb = _cfg.get("fallback_providers") or _cfg.get("fallback_model")
    +            fb_list = (fb if isinstance(fb, list) else [fb]) if fb else []
    +            runtime = None
    +            for entry in fb_list:
    +                if not isinstance(entry, dict):
    +                    continue
    +                try:
    +                    fb_kwargs = {"requested": entry.get("provider")}
    +                    if entry.get("base_url"):
    +                        fb_kwargs["explicit_base_url"] = entry["base_url"]
    +                    if entry.get("api_key"):
    +                        fb_kwargs["explicit_api_key"] = entry["api_key"]
    +                    runtime = resolve_runtime_provider(**fb_kwargs)
    +                    logger.info("Job '%s': fallback resolved to %s", job_id, runtime.get("provider"))
    +                    break
    +                except Exception as fb_exc:
    +                    logger.debug("Job '%s': fallback %s failed: %s", job_id, entry.get("provider"), fb_exc)
    +            if runtime is None:
    +                raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc
             except Exception as exc:
                 message = format_runtime_provider_error(exc)
                 raise RuntimeError(message) from exc

  • gateway/run.py

    @@ -350,16 +350,30 @@ def _expand_whatsapp_auth_aliases(identifier: str) -> set:
     
     
     def _resolve_runtime_agent_kwargs() -> dict:
    -    """Resolve provider credentials for gateway-created AIAgent instances."""
    +    """Resolve provider credentials for gateway-created AIAgent instances.
    +
    +    If the primary provider fails with an authentication error, attempt to
    +    resolve credentials using the fallback provider chain from config.yaml
    +    before giving up.
    +    """
         from hermes_cli.runtime_provider import (
             resolve_runtime_provider,
             format_runtime_provider_error,
         )
    +    from hermes_cli.auth import AuthError
     
         try:
             runtime = resolve_runtime_provider(
                 requested=os.getenv("HERMES_INFERENCE_PROVIDER"),
             )
    +    except AuthError as auth_exc:
    +        # Primary provider auth failed (expired token, revoked key, etc.).
    +        # Try the fallback provider chain before raising.
    +        logger.warning("Primary provider auth failed: %s — trying fallback", auth_exc)
    +        fb_config = _try_resolve_fallback_provider()
    +        if fb_config is not None:
    +            return fb_config
    +        raise RuntimeError(format_runtime_provider_error(auth_exc)) from auth_exc
         except Exception as exc:
             raise RuntimeError(format_runtime_provider_error(exc)) from exc
     
    @@ -374,6 +388,48 @@ def _resolve_runtime_agent_kwargs() -> dict:
         }
     
     
    +def _try_resolve_fallback_provider() -> dict | None:
    +    """Attempt to resolve credentials from the fallback_model/fallback_providers config."""
    +    from hermes_cli.runtime_provider import resolve_runtime_provider
    +    try:
    +        import yaml as _y
    +        cfg_path = _hermes_home / "config.yaml"
    +        if not cfg_path.exists():
    +            return None
    +        with open(cfg_path, encoding="utf-8") as _f:
    +            cfg = _y.safe_load(_f) or {}
    +        fb = cfg.get("fallback_providers") or cfg.get("fallback_model")
    +        if not fb:
    +            return None
    +        # Normalize to list
    +        fb_list = fb if isinstance(fb, list) else [fb]
    +        for entry in fb_list:
    +            if not isinstance(entry, dict):
    +                continue
    +            try:
    +                runtime = resolve_runtime_provider(
    +                    requested=entry.get("provider"),
    +                    explicit_base_url=entry.get("base_url"),
    +                    explicit_api_key=[redacted:secret-assignment]"api_key"),
    +                )
    +                logger.info("Fallback provider resolved: %s", runtime.get("provider"))
    +                return {
    +                    "api_key": runtime.get("api_key"),
    +                    "base_url": runtime.get("base_url"),
    +                    "provider": runtime.get("provider"),
    +                    "api_mode": runtime.get("api_mode"),
    +                    "command": runtime.get("command"),
    +                    "args": list(runtime.get("args") or []),
    +                    "credential_pool": runtime.get("credential_pool"),
    +                }
    +            except Exception as fb_exc:
    +                logger.debug("Fallback entry %s failed: %s", entry.get("provider"), fb_exc)
    +                continue
    +    except Exception:
    +        pass
    +    return None
    +
    +
     def _build_media_placeholder(event) -> str:
         """Build a text placeholder for media-only events so they aren't dropped.
     

  • run_agent.py

    @@ -31,6 +31,7 @@
     import os
     import random
     import re
    +import ssl
     import sys
     import tempfile
     import time
    @@ -2118,12 +2119,14 @@ def switch_model(self, new_model, new_provider, api_key='', base_url='', api_mod
             # ("switched to anthropic, tui keeps trying openrouter").
             old_norm = (old_provider or "").strip().lower()
             new_norm = (new_provider or "").strip().lower()
    +        fallback_chain = list(getattr(self, "_fallback_chain", []) or [])
             if old_norm and new_norm and old_norm != new_norm:
    -            self._fallback_chain = [
    -                entry for entry in self._fallback_chain
    +            fallback_chain = [
    +                entry for entry in fallback_chain
                     if (entry.get("provider") or "").strip().lower() not in {old_norm, new_norm}
                 ]
    -            self._fallback_model = self._fallback_chain[0] if self._fallback_chain else None
    +        self._fallback_chain = fallback_chain
    +        self._fallback_model = fallback_chain[0] if fallback_chain else None
     
             logging.info(
                 "Model switched in-place: %s (%s) -> %s (%s)",
    @@ -6491,7 +6494,7 @@ def _call():
     
         # ── Provider fallback ──────────────────────────────────────────────────
     
    -    def _try_activate_fallback(self) -> bool:
    +    def _try_activate_fallback(self, reason: "FailoverReason | None" = None) -> bool:
             """Switch to the next fallback model/provider in the chain.
     
             Called when the current model is failing after retries.  Swaps the
    @@ -6503,6 +6506,15 @@ def _try_activate_fallback(self) -> bool:
             auth resolution and client construction — no duplicated provider→key
             mappings.
             """
    +        if reason in (FailoverReason.rate_limit, FailoverReason.billing):
    +            # Only start cooldown when leaving the primary provider.  If we're
    +            # already on a fallback and chain-switching, the primary wasn't the
    +            # source of the 429 so the cooldown should not be reset/extended.
    +            fallback_already_active = bool(getattr(self, "_fallback_activated", False))
    +            current_provider = (getattr(self, "provider", "") or "").strip().lower()
    +            primary_provider = ((self._primary_runtime or {}).get("provider") or "").strip().lower()
    +            if (not fallback_already_active) or (primary_provider and current_provider == primary_provider):
    +                self._rate_limited_until = time.monotonic() + 60
             if self._fallback_index >= len(self._fallback_chain):
                 return False
     
    @@ -6639,11 +6651,15 @@ def _try_activate_fallback(self) -> bool:
                 # Without this, compression decisions use the primary model's
                 # context window (e.g. 200K) instead of the fallback's (e.g. 32K),
                 # causing oversized sessions to overflow the fallback.
    +            # Also pass _config_context_length so the explicit config override
    +            # (model.context_length in config.yaml) is respected — without this,
    +            # the fallback activation drops to 128K even when config says 204800.
                 if hasattr(self, 'context_compressor') and self.context_compressor:
                     from agent.model_metadata import get_model_context_length
                     fb_context_length = get_model_context_length(
                         self.model, base_url=self.base_url,
                         api_key=[redacted:secret-assignment] provider=self.provider,
    +                    config_context_length=getattr(self, "_config_context_length", None),
                     )
                     self.context_compressor.update_model(
                         model=self.model,
    @@ -6682,6 +6698,9 @@ def _restore_primary_runtime(self) -> bool:
             if not self._fallback_activated:
                 return False
     
    +        if getattr(self, "_rate_limited_until", 0) > time.monotonic():
    +            return False  # primary still in rate-limit cooldown, stay on fallback
    +
             rt = self._primary_runtime
             try:
                 # ── Core runtime state ──
    @@ -10658,7 +10677,7 @@ def _stop_spinner():
                             )
                             if not pool_may_recover:
                                 self._emit_status("⚠️ Rate limited — switching to fallback provider...")
    -                            if self._try_activate_fallback():
    +                            if self._try_activate_fallback(reason=classified.reason):
                                     retry_count = 0
                                     compression_attempts = 0
                                     primary_recovery_attempted = False
    @@ -10923,6 +10942,14 @@ def _stop_spinner():
                             and not isinstance(
                                 api_error, (UnicodeEncodeError, json.JSONDecodeError)
                             )
    +                        # ssl.SSLError (and its subclass SSLCertVerificationError)
    +                        # inherits from OSError *and* ValueError via Python MRO,
    +                        # so the isinstance(ValueError) check above would
    +                        # misclassify a TLS transport failure as a local
    +                        # programming bug and abort without retrying.  Exclude
    +                        # ssl.SSLError explicitly so the error classifier's
    +                        # retryable=True mapping takes effect instead.
    +                        and not isinstance(api_error, ssl.SSLError)
                         )
                         is_client_error = (
                             is_local_validation_error

  • scripts/release.py

    @@ -187,6 +187,11 @@
         "[redacted:email]": "alexzhu0",
         "[redacted:email]": "AntAISecurityLab",
         "[redacted:email]": "briandevans",
    +    "[redacted:email]": "Bartok9",
    +    "[redacted:email]": "CruxExperts",
    +    "[redacted:email]": "LeonSGP43",
    +    "[redacted:email]": "vlwkaos",
    +    "[redacted:email]": "Tranquil-Flow",
         "[redacted:email]": "nummanali",
         "[redacted:email]": "whitehatjr1001",
         "[redacted:email]": "0xNyk",

  • tests/agent/test_error_classifier.py

    @@ -1094,3 +1094,37 @@ def test_real_ssl_error_type_classifies_as_timeout(self):
             result = classify_api_error(e)
             assert result.reason == FailoverReason.timeout
             assert result.retryable is True
    +
    +# ── Test: RateLimitError without status_code (Copilot/GitHub Models) ──────────
    +
    +class TestRateLimitErrorWithoutStatusCode:
    +    """Regression tests for the Copilot/GitHub Models edge case where the
    +    OpenAI SDK raises RateLimitError but does not populate .status_code."""
    +
    +    def _make_rate_limit_error(self, status_code=None):
    +        """Create an exception whose class name is 'RateLimitError' with
    +        an optionally missing status_code, mirroring the OpenAI SDK shape."""
    +        cls = type("RateLimitError", (Exception,), {})
    +        e = cls("You have exceeded your rate limit.")
    +        e.status_code = status_code  # None simulates the Copilot case
    +        return e
    +
    +    def test_rate_limit_error_without_status_code_classified_as_rate_limit(self):
    +        """RateLimitError with status_code=None must classify as rate_limit."""
    +        e = self._make_rate_limit_error(status_code=None)
    +        result = classify_api_error(e, provider="copilot", model="gpt-4o")
    +        assert result.reason == FailoverReason.rate_limit
    +
    +    def test_rate_limit_error_with_status_code_429_classified_as_rate_limit(self):
    +        """RateLimitError that does set status_code=429 still classifies correctly."""
    +        e = self._make_rate_limit_error(status_code=429)
    +        result = classify_api_error(e, provider="copilot", model="gpt-4o")
    +        assert result.reason == FailoverReason.rate_limit
    +
    +    def test_other_error_without_status_code_not_forced_to_rate_limit(self):
    +        """A non-RateLimitError with missing status_code must NOT be forced to 429."""
    +        cls = type("APIError", (Exception,), {})
    +        e = cls("something went wrong")
    +        e.status_code = None
    +        result = classify_api_error(e, provider="copilot", model="gpt-4o")
    +        assert result.reason != FailoverReason.rate_limit

  • tests/gateway/test_auth_fallback.py

    @@ -0,0 +1,73 @@
    +"""Test that AuthError triggers fallback provider resolution (#7230)."""
    +
    +import os
    +from unittest.mock import patch, MagicMock
    +
    +import pytest
    +
    +
    +class TestResolveRuntimeAgentKwargsAuthFallback:
    +    """_resolve_runtime_agent_kwargs should try fallback on AuthError."""
    +
    +    def test_auth_error_tries_fallback(self, tmp_path, monkeypatch):
    +        """When primary provider raises AuthError, fallback is attempted."""
    +        from hermes_cli.auth import AuthError
    +
    +        # Create a config with fallback
    +        config_path = tmp_path / "config.yaml"
    +        config_path.write_text(
    +            "model:\n  provider: openai-codex\n"
    +            "fallback_model:\n  provider: openrouter\n"
    +            "  model: meta-llama/llama-4-maverick\n"
    +        )
    +
    +        monkeypatch.setattr("gateway.run._hermes_home", tmp_path)
    +
    +        call_count = {"n": 0}
    +
    +        def _mock_resolve(**kwargs):
    +            call_count["n"] += 1
    +            requested = kwargs.get("requested", "")
    +            if requested and "codex" in str(requested).lower():
    +                raise AuthError("Codex token refresh failed with status 401")
    +            return {
    +                "api_key": "fallback-key",
    +                "base_url": "https://openrouter.ai/api/v1",
    +                "provider": "openrouter",
    +                "api_mode": "openai_chat",
    +                "command": None,
    +                "args": None,
    +                "credential_pool": None,
    +            }
    +
    +        monkeypatch.setenv("HERMES_INFERENCE_PROVIDER", "openai-codex")
    +
    +        with patch(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            side_effect=_mock_resolve,
    +        ):
    +            from gateway.run import _resolve_runtime_agent_kwargs
    +            result = _resolve_runtime_agent_kwargs()
    +
    +        assert result["provider"] == "openrouter"
    +        assert result["api_key"] == "fallback-key"
    +        # Should have been called at least twice (primary + fallback)
    +        assert call_count["n"] >= 2
    +
    +    def test_auth_error_no_fallback_raises(self, tmp_path, monkeypatch):
    +        """When primary fails and no fallback configured, RuntimeError is raised."""
    +        from hermes_cli.auth import AuthError
    +
    +        config_path = tmp_path / "config.yaml"
    +        config_path.write_text("model:\n  provider: openai-codex\n")
    +
    +        monkeypatch.setattr("gateway.run._hermes_home", tmp_path)
    +        monkeypatch.setenv("HERMES_INFERENCE_PROVIDER", "openai-codex")
    +
    +        with patch(
    +            "hermes_cli.runtime_provider.resolve_runtime_provider",
    +            side_effect=AuthError("token expired"),
    +        ):
    +            from gateway.run import _resolve_runtime_agent_kwargs
    +            with pytest.raises(RuntimeError):
    +                _resolve_runtime_agent_kwargs()

  • tests/run_agent/test_primary_runtime_restore.py

    @@ -446,3 +446,85 @@ def test_full_cycle_fallback_then_restore(self):
             assert agent._fallback_index == 0
             assert agent.provider == "custom"
             assert agent.base_url == "https://my-llm.example.com/v1"
    +
    +
    +# =============================================================================
    +# Rate-limit cooldown gate
    +# =============================================================================
    +
    +class TestRateLimitCooldown:
    +    """Verify _restore_primary_runtime() respects the 60s rate-limit cooldown."""
    +
    +    def test_restore_blocked_during_cooldown(self):
    +        """While _rate_limited_until is in the future, restore returns False."""
    +        agent = _make_agent(
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback()
    +
    +        assert agent._fallback_activated is True
    +
    +        # Manually set cooldown well into the future
    +        agent._rate_limited_until = time.monotonic() + 60
    +
    +        result = agent._restore_primary_runtime()
    +        assert result is False
    +        assert agent._fallback_activated is True  # still on fallback
    +
    +    def test_restore_allowed_after_cooldown_expires(self):
    +        """Once the cooldown window passes, restore proceeds normally."""
    +        agent = _make_agent(
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback()
    +
    +        assert agent._fallback_activated is True
    +
    +        # Cooldown already expired
    +        agent._rate_limited_until = time.monotonic() - 1
    +
    +        with patch("run_agent.OpenAI", return_value=MagicMock()):
    +            result = agent._restore_primary_runtime()
    +
    +        assert result is True
    +        assert agent._fallback_activated is False
    +
    +    def test_cooldown_set_on_rate_limit_reason(self):
    +        """_try_activate_fallback with rate_limit reason sets _rate_limited_until."""
    +        from run_agent import FailoverReason
    +        agent = _make_agent(
    +            fallback_model={"provider": "openrouter", "model": "anthropic/claude-sonnet-4"},
    +        )
    +        before = time.monotonic()
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            agent._try_activate_fallback(reason=FailoverReason.rate_limit)
    +
    +        assert hasattr(agent, "_rate_limited_until")
    +        assert agent._rate_limited_until > before + 50  # ~60s from now
    +
    +    def test_cooldown_not_set_when_already_on_fallback(self):
    +        """Chain-switching while already on fallback must not reset cooldown."""
    +        from run_agent import FailoverReason
    +        agent = _make_agent(
    +            fallback_model=[
    +                {"provider": "openrouter", "model": "model-a"},
    +                {"provider": "anthropic", "model": "model-b"},
    +            ],
    +        )
    +        mock_client = _mock_resolve()
    +        with patch("agent.auxiliary_client.resolve_provider_client", return_value=(mock_client, None)):
    +            # First call: leaving primary → cooldown should be set
    +            agent._try_activate_fallback(reason=FailoverReason.rate_limit)
    +            first_cooldown = getattr(agent, "_rate_limited_until", 0)
    +
    +            # Second call: already on fallback (provider != primary) → cooldown must not advance
    +            agent._try_activate_fallback(reason=FailoverReason.rate_limit)
    +            second_cooldown = getattr(agent, "_rate_limited_until", 0)
    +
    +        # second call should not have extended the cooldown
    +        assert second_cooldown == first_cooldown

  • tests/run_agent/test_switch_model_fallback_prune.py

    @@ -78,6 +78,17 @@ def test_switch_with_empty_chain_stays_empty():
         assert agent._fallback_model is None
     
     
    +def test_switch_initializes_missing_fallback_attrs():
    +    agent = _make_agent([])
    +    del agent._fallback_chain
    +    del agent._fallback_model
    +
    +    _switch_to_anthropic(agent)
    +
    +    assert agent._fallback_chain == []
    +    assert agent._fallback_model is None
    +
    +
     def test_switch_within_same_provider_preserves_chain():
         chain = [{"provider": "openrouter", "model": "x-ai/grok-4"}]
         agent = _make_agent(chain)