{"repo":"NousResearch/hermes-agent","public_id":"47685","display_label":"Z.ai 429/1305 errors triggered by prompt content","summary":"A cluster of bug reports and fixes centered on Z.ai (GLM/Coding Plan) returning 429 errors with code 1305 when requests contain the phrase 'Hermes Agent' or similar strings, apparently due to a content filter or WAF. Multiple fix attempts address prompt sanitization, identity redaction, and backoff behavior across different code paths.","subsystem":"agent","counts":{"members":12,"open_issues":1,"open_issues_distinct":1,"open_prs":4,"closed_issues":4,"merged_prs":1,"attempt_prs":6},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":52333},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-25T07:17:18+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.1690909090909091},"fix_edges":[{"issue_number":47685,"pr_number":52333,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":47685,"title":"Z.AI GLM-5.2 Coding Plan returns 429/code 1305 when system prompt contains exact phrase \"Hermes Agent\"","url":null,"state":"closed"},{"kind":"issue","number":50663,"title":"[Bug]: z.ai limits hermes agent during \"peak hours\"","url":null,"state":"closed"},{"kind":"issue","number":51276,"title":"GLM-5.x models return 429 from Zhipu z.ai while identical curl requests succeed","url":null,"state":"closed"},{"kind":"issue","number":53002,"title":"[Bug]: Z.ai 429/code 1305 persists on chat/completions path — #47685 fix only covers anthropic adapter","url":null,"state":"open"},{"kind":"issue","number":60118,"title":"[Bug]: z.ai (GLM Coding Plan) rejects system prompts containing \"Hermes Agent\" — surfaces as a fake HTTP 429 / code 1305","url":null,"state":"closed"},{"kind":"pr","number":48124,"title":"fix(zai): sanitize blocked Hermes prompt phrase","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52333","number":52333},{"kind":"pr","value":"#47205","number":47205}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":51792,"title":"fix: use opencode identity for zai clients. address #50663","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":52333,"title":"fix(zai): adaptive backoff for Coding Plan GLM-5.2 overload (1305) + named custom-provider extra_body","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/agent_init.py","agent/conversation_loop.py","agent/retry_utils.py","scripts/release.py","tests/agent/test_custom_provider_extra_body.py","tests/test_retry_utils.py"],"fix_hunks":[{"file":"agent/agent_init.py","hunks":["@@ -106,7 +106,12 @@ def _custom_provider_extra_body_for_agent(\n     base_url: str,\n     custom_providers: List[Dict[str, Any]],\n ) -> Optional[Dict[str, Any]]:\n-    if (provider or \"\").strip().lower() != \"custom\":\n+    provider_norm = (provider or \"\").strip().lower()\n+    if provider_norm == \"custom\":\n+        provider_key_filter = \"\"\n+    elif provider_norm.startswith(\"custom:\"):\n+        provider_key_filter = provider_norm.split(\":\", 1)[1].strip()\n+    else:\n         return None\n \n     target_url = _normalized_custom_base_url(base_url)","@@ -117,6 +122,13 @@ def _custom_provider_extra_body_for_agent(\n     for entry in custom_providers or []:\n         if not isinstance(entry, dict):\n             continue\n+        if provider_key_filter:\n+            entry_keys = {\n+                str(entry.get(\"provider_key\", \"\") or \"\").strip().lower(),\n+                str(entry.get(\"name\", \"\") or \"\").strip().lower(),\n+            }\n+            if provider_key_filter not in entry_keys:\n+                continue\n         if _normalized_custom_base_url(entry.get(\"base_url\")) != target_url:\n             continue\n         extra_body = entry.get(\"extra_body\")"]},{"file":"agent/conversation_loop.py","hunks":["@@ -55,7 +55,7 @@\n )\n from agent.process_bootstrap import _install_safe_stdio\n from agent.prompt_caching import apply_anthropic_cache_control\n-from agent.retry_utils import jittered_backoff\n+from agent.retry_utils import adaptive_rate_limit_backoff, jittered_backoff\n from agent.trajectory import has_incomplete_scratchpad\n from agent.usage_pricing import estimate_usage_cost, normalize_usage\n from hermes_constants import PARTIAL_STREAM_STUB_ID","@@ -3537,16 +3537,38 @@ def _perform_api_call(next_api_kwargs):\n                             except (TypeError, ValueError):\n                                 pass\n                 wait_time = _retry_after if _retry_after else jittered_backoff(retry_count, base_delay=2.0, max_delay=60.0)\n+                _backoff_policy = None\n+                if is_rate_limited and not _retry_after:\n+                    wait_time, _backoff_policy = adaptive_rate_limit_backoff(\n+                        retry_count,\n+                        base_url=str(_base),\n+                        model=_model,\n+                        error=api_error,\n+                        default_wait=wait_time,\n+                    )\n                 if is_rate_limited:\n-                    agent._buffer_status(f\"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries})...\")\n+                    _policy_note = \"\"\n+                    if _backoff_policy == \"zai_coding_overload_long\":\n+                        _policy_note = \" (Z.AI Coding overload adaptive long backoff)\"\n+                    elif _backoff_policy == \"zai_coding_overload_short\":\n+                        _policy_note = \" (Z.AI Coding overload short retry)\"\n+                    _rate_limit_status = f\"⏱️ Rate limited. Waiting {wait_time:.1f}s (attempt {retry_count + 1}/{max_retries}){_policy_note}...\"\n+                    # Normal retries are buffered to avoid noisy transient chatter. Long\n+                    # Z.AI Coding waits are different: they can last minutes, so surface\n+                    # progress immediately instead of making the TUI look frozen.\n+                    if _backoff_policy == \"zai_coding_overload_long\":\n+                        agent._emit_status(_rate_limit_status)\n+                    else:\n+                        agent._buffer_status(_rate_limit_status)\n                 else:\n                     agent._buffer_status(f\"⏳ Retrying in {wait_time:.1f}s (attempt {retry_count}/{max_retries})...\")\n                 logger.warning(\n-                    \"Retrying API call in %ss (attempt %s/%s) %s error=%s\",\n+                    \"Retrying API call in %ss (attempt %s/%s) %s policy=%s error=%s\",\n                     wait_time,\n                     retry_count,\n                     max_retries,\n                     agent._client_log_context(),\n+                    _backoff_policy or \"default\",\n                     api_error,\n                 )\n                 # Sleep in small increments so we can respond to interrupts quickly"]},{"file":"agent/retry_utils.py","hunks":["@@ -8,6 +8,7 @@\n import random\n import threading\n import time\n+from typing import Any\n \n # Monotonic counter for jitter seed uniqueness within the same process.\n # Protected by a lock to avoid race conditions in concurrent retry paths","@@ -15,6 +16,14 @@\n _jitter_counter = 0\n _jitter_lock = threading.Lock()\n \n+# Z.AI Coding Plan's GLM-5.2 endpoint often returns HTTP 429 code 1305\n+# (\"The service may be temporarily overloaded...\") for otherwise valid\n+# Hermes requests. Short retries tend to hammer the same overloaded window;\n+# after a few normal retries, progressively widen the wait window. Keep the\n+# cap interactive-friendly: a simple TUI message should fail visibly in minutes,\n+# not sit silent for 20+ minutes.\n+_ZAI_CODING_OVERLOAD_LONG_BACKOFF = (30.0, 60.0, 90.0, 120.0)\n+\n \n def jittered_backoff(\n     attempt: int,","@@ -55,3 +64,66 @@ def jittered_backoff(\n     jitter = rng.uniform(0, jitter_ratio * delay)\n \n     return delay + jitter\n+\n+\n+def _error_text(error: Any) -> str:\n+    \"\"\"Best-effort flattened provider error text for retry classification.\"\"\"\n+    parts = [\n+        error,\n+        getattr(error, \"message\", None),\n+        getattr(error, \"body\", None),\n+        getattr(error, \"response\", None),\n+    ]\n+    return \" \".join(str(part) for part in parts if part is not None).lower()\n+\n+\n+def is_zai_coding_overload_error(*, base_url: str | None, model: str | None, error: Any) -> bool:\n+    \"\"\"Return True for Z.AI Coding Plan transient overload 429s.\n+\n+    The coding-plan endpoint reports overload as HTTP 429 with body code 1305\n+    and message \"The service may be temporarily overloaded...\". Treat only\n+    that narrow shape specially so ordinary quota/billing 429s still fail fast\n+    through the existing classifier.\n+    \"\"\"\n+    base = (base_url or \"\").lower()\n+    model_name = (model or \"\").lower()\n+    status = getattr(error, \"status_code\", None)\n+    text = _error_text(error)\n+    return (\n+        status == 429\n+        and \"api.z.ai/api/coding/paas/v4\" in base\n+        and \"glm-5.2\" in model_name\n+        and (\"1305\" in text or \"temporarily overloaded\" in text)\n+    )\n+\n+\n+def adaptive_rate_limit_backoff(\n+    attempt: int,\n+    *,\n+    base_url: str | None,\n+    model: str | None,\n+    error: Any,\n+    default_wait: float,\n+    short_attempts: int = 3,\n+) -> tuple[float, str | None]:\n+    \"\"\"Provider-aware rate-limit backoff.\n+\n+    For most providers this returns ``default_wait`` unchanged. For Z.AI\n+    Coding Plan GLM-5.2 overloads, keep the first ``short_attempts`` retries on\n+    the normal short exponential schedule, then switch to progressively longer\n+    waits (30s → 60s → 90s → 120s, capped) plus light jitter.\n+\n+    ``attempt`` is 1-based, matching the retry loop's logged attempt number.\n+    Returns ``(wait_seconds, reason_label)`` where ``reason_label`` is suitable\n+    for status/log decoration when a provider-specific policy fired.\n+    \"\"\"\n+    if not is_zai_coding_overload_error(base_url=base_url, model=model, error=error):\n+        return default_wait, None\n+    if attempt <= short_attempts:\n+        return default_wait, \"zai_coding_overload_short\"\n+\n+    idx = min(attempt - short_attempts - 1, len(_ZAI_CODING_OVERLOAD_LONG_BACKOFF) - 1)\n+    base_delay = _ZAI_CODING_OVERLOAD_LONG_BACKOFF[idx]\n+    # A smaller jitter ratio keeps long waits readable while still avoiding\n+    # synchronized retry storms across concurrent Hermes sessions.\n+    return jittered_backoff(1, base_delay=base_delay, max_delay=base_delay, jitter_ratio=0.2), \"zai_coding_overload_long\""]},{"file":"scripts/release.py","hunks":["@@ -45,6 +45,7 @@\n \n # Auto-extracted from noreply emails + manual overrides\n AUTHOR_MAP = {\n+    \"[redacted:email]\": \"benbenlijie\",  # PR #47205 salvage (named custom-provider extra_body + Z.AI Coding overload adaptive backoff; #50663)\n     \"[redacted:email]\": \"Danamove\",  # PR #46726 salvage (kill venv-resident pythonw gateway before recreating venv on Windows; #47036/#47557/#47910)\n     \"[redacted:email]\": \"wgu9\",  # PR #51468 salvage (WSL/no-systemd orphan gateway tracking, #51325)\n     \"[redacted:email]\": \"uperLu\",  # PR #50958 salvage (rename plugins/cron → plugins/cron_providers; #50872)"]},{"file":"tests/agent/test_custom_provider_extra_body.py","hunks":["@@ -91,3 +91,34 @@ def test_custom_provider_extra_body_ignores_other_custom_models():\n     )\n \n     assert agent.request_overrides == {}\n+\n+\n+def test_named_custom_provider_extra_body_matches_provider_key():\n+    agent = SimpleNamespace(\n+        provider=\"custom:zai-coding-plan\",\n+        model=\"glm-5.2\",\n+        base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+        request_overrides={},\n+    )\n+\n+    _merge_custom_provider_extra_body(\n+        agent,\n+        [\n+            {\n+                \"provider_key\": \"other-provider\",\n+                \"name\": \"Other Provider\",\n+                \"base_url\": \"https://api.z.ai/api/coding/paas/v4\",\n+                \"model\": \"glm-5.2\",\n+                \"extra_body\": {\"enable_thinking\": True},\n+            },\n+            {\n+                \"provider_key\": \"zai-coding-plan\",\n+                \"name\": \"Z.AI Coding Plan\",\n+                \"base_url\": \"https://api.z.ai/api/coding/paas/v4/\",\n+                \"model\": \"glm-5.2\",\n+                \"extra_body\": {\"enable_thinking\": False},\n+            },\n+        ],\n+    )\n+\n+    assert agent.request_overrides == {\"extra_body\": {\"enable_thinking\": False}}"]},{"file":"tests/test_retry_utils.py","hunks":["@@ -3,7 +3,9 @@\n import threading\n \n import agent.retry_utils as retry_utils\n-from agent.retry_utils import jittered_backoff\n+from types import SimpleNamespace\n+\n+from agent.retry_utils import adaptive_rate_limit_backoff, is_zai_coding_overload_error, jittered_backoff\n \n \n def test_backoff_is_exponential():","@@ -115,3 +117,96 @@ def _call():\n \n     assert len(recorded_seeds) == 2\n     assert len(set(recorded_seeds)) == 2, f\"Expected unique seeds, got {recorded_seeds}\"\n+\n+\n+def _zai_overload_error():\n+    return SimpleNamespace(\n+        status_code=429,\n+        body={\n+            \"error\": {\n+                \"code\": \"1305\",\n+                \"message\": \"The service may be temporarily overloaded, please try again later\",\n+            }\n+        },\n+    )\n+\n+\n+def test_zai_coding_overload_classifier_is_narrow():\n+    err = _zai_overload_error()\n+    assert is_zai_coding_overload_error(\n+        base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+        model=\"glm-5.2\",\n+        error=err,\n+    )\n+\n+    assert not is_zai_coding_overload_error(\n+        base_url=\"https://api.z.ai/api/paas/v4\",\n+        model=\"glm-5.2\",\n+        error=err,\n+    )\n+    assert not is_zai_coding_overload_error(\n+        base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+        model=\"glm-5.1\",\n+        error=err,\n+    )\n+    assert not is_zai_coding_overload_error(\n+        base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+        model=\"glm-5.2\",\n+        error=SimpleNamespace(status_code=429, body={\"error\": {\"code\": \"1113\", \"message\": \"Insufficient balance\"}}),\n+    )\n+\n+\n+def test_zai_coding_overload_backoff_keeps_first_retries_short(monkeypatch):\n+    monkeypatch.setattr(retry_utils, \"jittered_backoff\", lambda *a, **kw: kw[\"base_delay\"])\n+    err = _zai_overload_error()\n+\n+    wait, policy = adaptive_rate_limit_backoff(\n+        1,\n+        base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+        model=\"glm-5.2\",\n+        error=err,\n+        default_wait=2.5,\n+    )\n+    assert wait == 2.5\n+    assert policy == \"zai_coding_overload_short\"\n+\n+    wait, policy = adaptive_rate_limit_backoff(\n+        3,\n+        base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+        model=\"glm-5.2\",\n+        error=err,\n+        default_wait=9.0,\n+    )\n+    assert wait == 9.0\n+    assert policy == \"zai_coding_overload_short\"\n+\n+\n+def test_zai_coding_overload_backoff_grows_after_short_retries(monkeypatch):\n+    monkeypatch.setattr(retry_utils, \"jittered_backoff\", lambda *a, **kw: kw[\"base_delay\"])\n+    err = _zai_overload_error()\n+\n+    waits = []\n+    for attempt in range(4, 10):\n+        wait, policy = adaptive_rate_limit_backoff(\n+            attempt,\n+            base_url=\"https://api.z.ai/api/coding/paas/v4\",\n+            model=\"glm-5.2\",\n+            error=err,\n+            default_wait=10.0,\n+        )\n+        waits.append(wait)\n+        assert policy == \"zai_coding_overload_long\"\n+\n+    assert waits == [30.0, 60.0, 90.0, 120.0, 120.0, 120.0]\n+\n+\n+def test_non_zai_backoff_returns_default_wait():\n+    wait, policy = adaptive_rate_limit_backoff(\n+        10,\n+        base_url=\"https://openrouter.ai/api/v1\",\n+        model=\"glm-5.2\",\n+        error=_zai_overload_error(),\n+        default_wait=12.0,\n+    )\n+    assert wait == 12.0\n+    assert policy is None\n"]}]},{"kind":"pr","number":53006,"title":"fix(agent): sanitize \"Hermes Agent\" prompt block on Z.ai endpoints (#47685 follow-up)","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":56823,"title":"Fix: avoid bare skill_view() call syntax in plain-text prompt guidance (z.ai 429/1305)","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#56847","number":56847}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":59958,"title":"fix(agent): reword help guidance to avoid Z.AI WAF prompt-block false 429","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":60140,"title":"fix: redact brand string for z.ai provider to avoid content-filter 429 (#60118)","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#59975","number":59975},{"kind":"pr","value":"#59975","number":59975},{"kind":"pr","value":"#53006","number":53006},{"kind":"pr","value":"#56823","number":56823},{"kind":"pr","value":"#60118","number":60118},{"kind":"pr","value":"#53002","number":53002}],"category":"duplicate_of","implication":"duplicate"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}