{"repo":"NousResearch/hermes-agent","public_id":"14038","display_label":"Misclassification of 'overloaded' errors as rate_limit","summary":"Multiple issues and fix attempts address a bug where 429 responses with 'overloaded' message bodies are incorrectly classified as rate_limit errors instead of server overload, triggering the wrong recovery path and exhausting credential pools.","subsystem":"agent","counts":{"members":8,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":2,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":52890},"resolution":{"state":"partially_resolved","by":"pr-53578"},"freshness":{"state":"verified","last_verified_at":"2026-06-27T11:16:54+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.22714285714285712},"fix_edges":[{"issue_number":14038,"pr_number":52890,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":14038,"pr_number":53578,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":14038,"title":"\"overloaded\" server errors classified as rate_limit, exhausting credential pool","url":null,"state":"closed"},{"kind":"issue","number":15297,"title":"Error classifier: 429 'temporarily overloaded' misclassified as rate_limit — triggers wrong recovery path","url":null,"state":"closed"},{"kind":"pr","number":14055,"title":"fix(error_classifier): classify 'overloaded' message as FailoverReason.overloaded","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52890","number":52890},{"kind":"pr","value":"#14038","number":14038},{"kind":"pr","value":"#53578","number":53578}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":14261,"title":"fix(agent): classify 'overloaded' messages as server overload, not rate_limit","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52890","number":52890},{"kind":"pr","value":"#14038","number":14038},{"kind":"pr","value":"#53578","number":53578}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":14788,"title":"fix(agent): classify overloaded errors as server-side overload, not rate_limit","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"38e7bd8a08a9df450b7d8661778ecc47f25323c2","number":0},{"kind":"pr","value":"#14038","number":14038}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":15414,"title":"fix(error-classifier): route 429 'overloaded' messages to FailoverReason.overloaded (#15297)","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":52890,"title":"fix(agent): classify message-only 'overloaded' as server overload (salvage of #14261 by @ms-alan)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/error_classifier.py","tests/agent/test_curator.py","tests/agent/test_error_classifier.py"],"fix_hunks":[{"file":"agent/error_classifier.py","hunks":["@@ -1214,6 +1214,20 @@ def _classify_by_message(\n             should_fallback=True,\n         )\n \n+    # Overloaded / server-busy patterns — must come BEFORE the rate_limit and\n+    # billing checks so that a message-only \"overloaded\" (no 503/529 status,\n+    # e.g. some Anthropic-compatible proxies) classifies as a transient\n+    # overload (backoff + retry) instead of falling through to `unknown` or\n+    # incorrectly triggering credential rotation.\n+    if any(p in error_msg for p in (\n+        \"overloaded\", \"temporarily overloaded\",\n+        \"service is temporarily overloaded\",\n+    )):\n+        return result_fn(\n+            FailoverReason.overloaded,\n+            retryable=True,\n+        )\n+\n     # Billing patterns\n     if any(p in error_msg for p in _BILLING_PATTERNS):\n         return result_fn("]},{"file":"tests/agent/test_curator.py","hunks":["@@ -7,6 +7,7 @@\n from __future__ import annotations\n \n import importlib\n+import threading\n from datetime import datetime, timedelta, timezone\n from pathlib import Path\n ","@@ -37,7 +38,21 @@ def curator_env(tmp_path, monkeypatch):\n     # directly). Tests opt in with _enable_prune_builtins(...).\n     monkeypatch.setattr(usage, \"_prune_builtins_enabled\", lambda: False)\n \n-    return {\"home\": home, \"curator\": curator, \"usage\": usage}\n+    yield {\"home\": home, \"curator\": curator, \"usage\": usage}\n+\n+    # Teardown: a curator review launched with synchronous=False spawns a\n+    # daemon \"curator-review\" thread that calls save_state() when it finishes.\n+    # save_state() resolves the state path from HERMES_HOME at write time, so a\n+    # straggler thread that outlives this test would write into whatever home\n+    # the *next* test has configured (or the default ~/.hermes once monkeypatch\n+    # restores the env) — corrupting an unrelated test's state file. This race\n+    # is invisible on a fast machine but flakes under CI load. Join any such\n+    # thread here, while HERMES_HOME is still pinned to this test's tmp home\n+    # (curator_env depends on monkeypatch, so this teardown runs before the\n+    # monkeypatch env is restored). See the salvage of #14261 CI flake.\n+    for t in threading.enumerate():\n+        if t.name == \"curator-review\" and t.is_alive():\n+            t.join(timeout=10.0)\n \n \n def _write_skill(skills_dir: Path, name: str):"]},{"file":"tests/agent/test_error_classifier.py","hunks":["@@ -347,6 +347,18 @@ def test_529_anthropic_overloaded(self):\n         result = classify_api_error(e)\n         assert result.reason == FailoverReason.overloaded\n \n+    def test_message_only_overloaded_without_status_is_overloaded(self):\n+        \"\"\"Some Anthropic-compatible proxies surface 'overloaded' in the\n+        message with no 503/529 status_code. It must classify as overloaded\n+        (transient backoff+retry), not unknown / credential rotation. (#14261)\"\"\"\n+        e = MockAPIError(\n+            \"Anthropic API error: Overloaded - the service is temporarily overloaded\"\n+        )  # no status_code\n+        result = classify_api_error(e, provider=\"anthropic\")\n+        assert result.reason == FailoverReason.overloaded\n+        assert result.retryable is True\n+        assert result.should_rotate_credential is False\n+\n     # ── 5xx that are actually request-validation errors ──\n     # Some OpenAI-compatible gateways (e.g. codex.nekos.me) return\n     # request-validation failures with a 5xx status. These are\n"]}]},{"kind":"pr","number":53578,"title":"fix(agent): classify 429 'overloaded' bodies as overloaded, not rate_limit (#14038)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/error_classifier.py","tests/agent/test_error_classifier.py"],"fix_hunks":[{"file":"agent/error_classifier.py","hunks":["@@ -133,6 +133,31 @@ def is_auth(self) -> bool:\n     \"servicequotaexceededexception\",\n ]\n \n+# Patterns that indicate provider-side overload, NOT a per-credential rate\n+# limit or billing problem.  The credential is valid — the server is just\n+# busy — so the correct recovery is \"back off and retry the same key\", never\n+# \"rotate the credential\" (rotating exhausts the pool while the endpoint is\n+# still busy; a single-key user has nothing to rotate to).  Some providers\n+# (notably Z.AI / Zhipu) reuse HTTP 429 for server-wide overload, so the 429\n+# status path matches the body against this list before falling through to\n+# the rate_limit default.  Phrases are kept narrow and overload-flavoured so a\n+# normal rate-limit message (\"you have been rate-limited\") doesn't hit this\n+# bucket. (#14038, #15297)\n+_OVERLOADED_PATTERNS = [\n+    \"overloaded\",\n+    \"temporarily overloaded\",\n+    \"service is temporarily overloaded\",\n+    \"service may be temporarily overloaded\",\n+    \"server is overloaded\",\n+    \"server overloaded\",\n+    \"service overloaded\",\n+    \"service is overloaded\",\n+    \"upstream overloaded\",\n+    \"currently overloaded\",\n+    \"at capacity\",\n+    \"over capacity\",\n+]\n+\n # Usage-limit patterns that need disambiguation (could be billing OR rate_limit)\n _USAGE_LIMIT_PATTERNS = [\n     \"usage limit\",","@@ -863,7 +888,19 @@ def _classify_by_status(\n         )\n \n     if status_code == 429:\n-        # Already checked long_context_tier above; this is a normal rate limit\n+        # Already checked long_context_tier above. Some providers (notably\n+        # Z.AI / Zhipu) reuse HTTP 429 for server-wide overload — same status\n+        # code as a true per-credential rate limit, but the credential is\n+        # valid and the correct recovery is \"back off and retry the same key\",\n+        # NOT \"rotate the credential\" (which exhausts the pool while the\n+        # endpoint is still busy, and does nothing for a single-key user).\n+        # Disambiguate on the error body so an overload 429 takes the\n+        # transient-overload path instead of burning the pool. (#14038)\n+        if any(p in error_msg for p in _OVERLOADED_PATTERNS):\n+            return result_fn(\n+                FailoverReason.overloaded,\n+                retryable=True,\n+            )\n         return result_fn(\n             FailoverReason.rate_limit,\n             retryable=True,","@@ -1219,10 +1256,7 @@ def _classify_by_message(\n     # e.g. some Anthropic-compatible proxies) classifies as a transient\n     # overload (backoff + retry) instead of falling through to `unknown` or\n     # incorrectly triggering credential rotation.\n-    if any(p in error_msg for p in (\n-        \"overloaded\", \"temporarily overloaded\",\n-        \"service is temporarily overloaded\",\n-    )):\n+    if any(p in error_msg for p in _OVERLOADED_PATTERNS):\n         return result_fn(\n             FailoverReason.overloaded,\n             retryable=True,"]},{"file":"tests/agent/test_error_classifier.py","hunks":["@@ -359,6 +359,30 @@ def test_message_only_overloaded_without_status_is_overloaded(self):\n         assert result.retryable is True\n         assert result.should_rotate_credential is False\n \n+    def test_429_with_overloaded_body_is_overloaded_not_rate_limit(self):\n+        \"\"\"Z.AI / Zhipu reuse HTTP 429 for server-wide overload. The credential\n+        is valid — the server is just busy — so it must classify as overloaded\n+        (back off + retry the same key), NOT rate_limit (which would rotate and\n+        exhaust the pool, doing nothing for a single-key user). (#14038)\"\"\"\n+        e = MockAPIError(\n+            \"The service may be temporarily overloaded, please try again later\",\n+            status_code=429,\n+        )\n+        result = classify_api_error(e, provider=\"zai\")\n+        assert result.reason == FailoverReason.overloaded\n+        assert result.retryable is True\n+        assert result.should_rotate_credential is False\n+\n+    def test_429_normal_rate_limit_still_rotates(self):\n+        \"\"\"Guard: a genuine 429 rate limit (no overload language) must still\n+        classify as rate_limit and rotate the credential. (#14038)\"\"\"\n+        e = MockAPIError(\n+            \"Rate limit exceeded: too many requests\", status_code=429\n+        )\n+        result = classify_api_error(e, provider=\"zai\")\n+        assert result.reason == FailoverReason.rate_limit\n+        assert result.should_rotate_credential is True\n+\n     # ── 5xx that are actually request-validation errors ──\n     # Some OpenAI-compatible gateways (e.g. codex.nekos.me) return\n     # request-validation failures with a 5xx status. These are\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}