Misclassification of 'overloaded' errors as rate_limit

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #52890 fix(agent): classify message-only 'overloaded' as server overload (salvage of #14261 by @ms-alan)

  merged to main

  files: agent/error_classifier.py, tests/agent/test_curator.py, tests/agent/test_error_classifier.py

  Show the diff

  • agent/error_classifier.py

    @@ -1214,6 +1214,20 @@ def _classify_by_message(
                 should_fallback=True,
             )
     
    +    # Overloaded / server-busy patterns — must come BEFORE the rate_limit and
    +    # billing checks so that a message-only "overloaded" (no 503/529 status,
    +    # e.g. some Anthropic-compatible proxies) classifies as a transient
    +    # overload (backoff + retry) instead of falling through to `unknown` or
    +    # incorrectly triggering credential rotation.
    +    if any(p in error_msg for p in (
    +        "overloaded", "temporarily overloaded",
    +        "service is temporarily overloaded",
    +    )):
    +        return result_fn(
    +            FailoverReason.overloaded,
    +            retryable=True,
    +        )
    +
         # Billing patterns
         if any(p in error_msg for p in _BILLING_PATTERNS):
             return result_fn(

  • tests/agent/test_curator.py

    @@ -7,6 +7,7 @@
     from __future__ import annotations
     
     import importlib
    +import threading
     from datetime import datetime, timedelta, timezone
     from pathlib import Path
     
    @@ -37,7 +38,21 @@ def curator_env(tmp_path, monkeypatch):
         # directly). Tests opt in with _enable_prune_builtins(...).
         monkeypatch.setattr(usage, "_prune_builtins_enabled", lambda: False)
     
    -    return {"home": home, "curator": curator, "usage": usage}
    +    yield {"home": home, "curator": curator, "usage": usage}
    +
    +    # Teardown: a curator review launched with synchronous=False spawns a
    +    # daemon "curator-review" thread that calls save_state() when it finishes.
    +    # save_state() resolves the state path from HERMES_HOME at write time, so a
    +    # straggler thread that outlives this test would write into whatever home
    +    # the *next* test has configured (or the default ~/.hermes once monkeypatch
    +    # restores the env) — corrupting an unrelated test's state file. This race
    +    # is invisible on a fast machine but flakes under CI load. Join any such
    +    # thread here, while HERMES_HOME is still pinned to this test's tmp home
    +    # (curator_env depends on monkeypatch, so this teardown runs before the
    +    # monkeypatch env is restored). See the salvage of #14261 CI flake.
    +    for t in threading.enumerate():
    +        if t.name == "curator-review" and t.is_alive():
    +            t.join(timeout=10.0)
     
     
     def _write_skill(skills_dir: Path, name: str):

  • tests/agent/test_error_classifier.py

    @@ -347,6 +347,18 @@ def test_529_anthropic_overloaded(self):
             result = classify_api_error(e)
             assert result.reason == FailoverReason.overloaded
     
    +    def test_message_only_overloaded_without_status_is_overloaded(self):
    +        """Some Anthropic-compatible proxies surface 'overloaded' in the
    +        message with no 503/529 status_code. It must classify as overloaded
    +        (transient backoff+retry), not unknown / credential rotation. (#14261)"""
    +        e = MockAPIError(
    +            "Anthropic API error: Overloaded - the service is temporarily overloaded"
    +        )  # no status_code
    +        result = classify_api_error(e, provider="anthropic")
    +        assert result.reason == FailoverReason.overloaded
    +        assert result.retryable is True
    +        assert result.should_rotate_credential is False
    +
         # ── 5xx that are actually request-validation errors ──
         # Some OpenAI-compatible gateways (e.g. codex.nekos.me) return
         # request-validation failures with a 5xx status. These are

• #53578 fix(agent): classify 429 'overloaded' bodies as overloaded, not rate_limit (#14038)

  merged to main

  files: agent/error_classifier.py, tests/agent/test_error_classifier.py

  Show the diff

  • agent/error_classifier.py

    @@ -133,6 +133,31 @@ def is_auth(self) -> bool:
         "servicequotaexceededexception",
     ]
     
    +# Patterns that indicate provider-side overload, NOT a per-credential rate
    +# limit or billing problem.  The credential is valid — the server is just
    +# busy — so the correct recovery is "back off and retry the same key", never
    +# "rotate the credential" (rotating exhausts the pool while the endpoint is
    +# still busy; a single-key user has nothing to rotate to).  Some providers
    +# (notably Z.AI / Zhipu) reuse HTTP 429 for server-wide overload, so the 429
    +# status path matches the body against this list before falling through to
    +# the rate_limit default.  Phrases are kept narrow and overload-flavoured so a
    +# normal rate-limit message ("you have been rate-limited") doesn't hit this
    +# bucket. (#14038, #15297)
    +_OVERLOADED_PATTERNS = [
    +    "overloaded",
    +    "temporarily overloaded",
    +    "service is temporarily overloaded",
    +    "service may be temporarily overloaded",
    +    "server is overloaded",
    +    "server overloaded",
    +    "service overloaded",
    +    "service is overloaded",
    +    "upstream overloaded",
    +    "currently overloaded",
    +    "at capacity",
    +    "over capacity",
    +]
    +
     # Usage-limit patterns that need disambiguation (could be billing OR rate_limit)
     _USAGE_LIMIT_PATTERNS = [
         "usage limit",
    @@ -863,7 +888,19 @@ def _classify_by_status(
             )
     
         if status_code == 429:
    -        # Already checked long_context_tier above; this is a normal rate limit
    +        # Already checked long_context_tier above. Some providers (notably
    +        # Z.AI / Zhipu) reuse HTTP 429 for server-wide overload — same status
    +        # code as a true per-credential rate limit, but the credential is
    +        # valid and the correct recovery is "back off and retry the same key",
    +        # NOT "rotate the credential" (which exhausts the pool while the
    +        # endpoint is still busy, and does nothing for a single-key user).
    +        # Disambiguate on the error body so an overload 429 takes the
    +        # transient-overload path instead of burning the pool. (#14038)
    +        if any(p in error_msg for p in _OVERLOADED_PATTERNS):
    +            return result_fn(
    +                FailoverReason.overloaded,
    +                retryable=True,
    +            )
             return result_fn(
                 FailoverReason.rate_limit,
                 retryable=True,
    @@ -1219,10 +1256,7 @@ def _classify_by_message(
         # e.g. some Anthropic-compatible proxies) classifies as a transient
         # overload (backoff + retry) instead of falling through to `unknown` or
         # incorrectly triggering credential rotation.
    -    if any(p in error_msg for p in (
    -        "overloaded", "temporarily overloaded",
    -        "service is temporarily overloaded",
    -    )):
    +    if any(p in error_msg for p in _OVERLOADED_PATTERNS):
             return result_fn(
                 FailoverReason.overloaded,
                 retryable=True,

  • tests/agent/test_error_classifier.py

    @@ -359,6 +359,30 @@ def test_message_only_overloaded_without_status_is_overloaded(self):
             assert result.retryable is True
             assert result.should_rotate_credential is False
     
    +    def test_429_with_overloaded_body_is_overloaded_not_rate_limit(self):
    +        """Z.AI / Zhipu reuse HTTP 429 for server-wide overload. The credential
    +        is valid — the server is just busy — so it must classify as overloaded
    +        (back off + retry the same key), NOT rate_limit (which would rotate and
    +        exhaust the pool, doing nothing for a single-key user). (#14038)"""
    +        e = MockAPIError(
    +            "The service may be temporarily overloaded, please try again later",
    +            status_code=429,
    +        )
    +        result = classify_api_error(e, provider="zai")
    +        assert result.reason == FailoverReason.overloaded
    +        assert result.retryable is True
    +        assert result.should_rotate_credential is False
    +
    +    def test_429_normal_rate_limit_still_rotates(self):
    +        """Guard: a genuine 429 rate limit (no overload language) must still
    +        classify as rate_limit and rotate the credential. (#14038)"""
    +        e = MockAPIError(
    +            "Rate limit exceeded: too many requests", status_code=429
    +        )
    +        result = classify_api_error(e, provider="zai")
    +        assert result.reason == FailoverReason.rate_limit
    +        assert result.should_rotate_credential is True
    +
         # ── 5xx that are actually request-validation errors ──
         # Some OpenAI-compatible gateways (e.g. codex.nekos.me) return
         # request-validation failures with a 5xx status. These are