{"repo":"NousResearch/hermes-agent","public_id":"32790","display_label":"Codex 429 quota errors misclassified as auth failures","summary":"Multiple fixes addressing a bug where upstream provider 429 rate-limit or quota exhaustion responses were incorrectly surfaced as authentication or credential failures. Work spans gateway error classification, OAuth error parsing, and rate-limit state preservation.","subsystem":"gateway","counts":{"members":7,"open_issues":1,"open_issues_distinct":1,"open_prs":1,"closed_issues":1,"merged_prs":1,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":33168,"merged_at":"2026-05-27T10:13:16+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-05-27T10:13:16+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.25523809523809526},"fix_edges":[{"issue_number":32790,"pr_number":33168,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-05-27T10:13:16+00:00"}],"members":[{"kind":"issue","number":32790,"title":"Hermes mislabels upstream 429 quota exhaustion as missing Codex credentials","url":null,"state":"closed"},{"kind":"issue","number":60846,"title":"bug(gateway): provider 429 rate-limit surfaces as \"Provider authentication failed\" when credentials are valid","url":null,"state":"open"},{"kind":"pr","number":33168,"title":"fix(gateway): classify Codex 429 quota as rate-limit, not missing credentials","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","hermes_cli/auth.py","tests/gateway/test_verbose_command.py","tests/hermes_cli/test_auth_codex_provider.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -1078,14 +1078,19 @@ def _resolve_runtime_agent_kwargs() -> dict:\n         resolve_runtime_provider,\n         format_runtime_provider_error,\n     )\n-    from hermes_cli.auth import AuthError\n+    from hermes_cli.auth import AuthError, is_rate_limited_auth_error\n \n     try:\n         runtime = resolve_runtime_provider()\n     except AuthError as auth_exc:\n-        # Primary provider auth failed (expired token, revoked key, etc.).\n-        # Try the fallback provider chain before raising.\n-        logger.warning(\"Primary provider auth failed: %s — trying fallback\", auth_exc)\n+        # Distinguish a transient rate-limit/quota cap (credentials are fine,\n+        # re-auth cannot help) from a genuine auth failure (expired/revoked\n+        # token). Both fall through to the fallback chain, but the log message\n+        # must not mislabel a quota exhaustion as an auth failure (#32790).\n+        if is_rate_limited_auth_error(auth_exc):\n+            logger.warning(\"Primary provider rate-limited (429): %s — trying fallback\", auth_exc)\n+        else:\n+            logger.warning(\"Primary provider auth failed: %s — trying fallback\", auth_exc)\n         fb_config = _try_resolve_fallback_provider()\n         if fb_config is not None:\n             return fb_config","@@ -1131,9 +1136,13 @@ def _try_resolve_fallback_provider() -> dict | None:\n                     explicit_base_url=entry.get(\"base_url\"),\n                     explicit_api_key=[redacted:secret-assignment]\n                 )\n+                # Log the literal `provider` key from config, not the resolved\n+                # runtime category — an Ollama fallback resolves through the\n+                # OpenAI-compatible path and would otherwise be logged as\n+                # \"openrouter\", contradicting the operator's config (#32790).\n                 logger.info(\n                     \"Fallback provider resolved: %s model=%s\",\n-                    runtime.get(\"provider\"),\n+                    entry.get(\"provider\") or runtime.get(\"provider\"),\n                     entry.get(\"model\"),\n                 )\n                 return {"]},{"file":"hermes_cli/auth.py","hunks":["@@ -729,6 +729,12 @@ def _resolve_zai_base_url(api_key: str, default_url: str, env_override: str) ->\n # Error Types\n # =============================================================================\n \n+# Error code marking upstream rate-limit / usage-quota exhaustion (HTTP 429).\n+# Such failures are transient and re-authenticating cannot resolve them, so\n+# they must be kept distinct from missing/expired-credential errors.\n+CODEX_RATE_LIMITED_CODE = \"codex_rate_limited\"\n+\n+\n class AuthError(RuntimeError):\n     \"\"\"Structured auth error with UX mapping hints.\"\"\"\n ","@@ -746,11 +752,52 @@ def __init__(\n         self.relogin_required = relogin_required\n \n \n+def is_rate_limited_auth_error(error: Exception) -> bool:\n+    \"\"\"True when an :class:`AuthError` represents upstream rate-limiting / quota\n+    exhaustion rather than missing or invalid credentials.\n+\n+    These failures are transient — re-authenticating cannot resolve them — so\n+    callers should surface a \"retry later\" notice and prefer a fallback chain\n+    instead of prompting the operator to run ``hermes auth``.\n+    \"\"\"\n+    return (\n+        isinstance(error, AuthError)\n+        and not error.relogin_required\n+        and error.code == CODEX_RATE_LIMITED_CODE\n+    )\n+\n+\n+def _parse_retry_after_seconds(headers: Any) -> Optional[int]:\n+    \"\"\"Best-effort parse of a ``Retry-After`` header into whole seconds.\n+\n+    Supports the delta-seconds form (e.g. ``\"120\"``). HTTP-date forms and\n+    missing/unparseable values return ``None`` rather than guessing.\n+    \"\"\"\n+    if headers is None:\n+        return None\n+    try:\n+        raw = headers.get(\"retry-after\")\n+    except Exception:\n+        return None\n+    if raw is None:\n+        return None\n+    try:\n+        seconds = int(str(raw).strip())\n+    except (TypeError, ValueError):\n+        return None\n+    return seconds if seconds >= 0 else None\n+\n+\n def format_auth_error(error: Exception) -> str:\n     \"\"\"Map auth failures to concise user-facing guidance.\"\"\"\n     if not isinstance(error, AuthError):\n         return str(error)\n \n+    # Rate-limit / quota errors are not credential problems — never append the\n+    # \"re-authenticate\" remediation, which would mislead the operator.\n+    if is_rate_limited_auth_error(error):\n+        return str(error)\n+\n     if error.relogin_required:\n         return f\"{error} Run `hermes model` to re-authenticate.\"\n ","@@ -3308,6 +3355,30 @@ def refresh_codex_oauth_pure(\n             },\n         )\n \n+    if response.status_code == 429:\n+        # Upstream rate-limit / usage-quota exhaustion on the token endpoint.\n+        # The stored refresh token is still valid here — re-authenticating\n+        # cannot lift a quota cap. Classify distinctly from auth failures so\n+        # callers surface a \"retry later\" notice instead of a misleading\n+        # \"run hermes auth\" prompt (see issue #32790).\n+        retry_after = _parse_retry_after_seconds(getattr(response, \"headers\", None))\n+        if retry_after is not None:\n+            message = (\n+                f\"Codex provider quota exhausted (429); retry after {retry_after}s. \"\n+                \"Credentials are still valid.\"\n+            )\n+        else:\n+            message = (\n+                \"Codex provider quota exhausted (429). Credentials are still valid; \"\n+                \"retry after the usage limit resets.\"\n+            )\n+        raise AuthError(\n+            message,\n+            provider=\"openai-codex\",\n+            code=CODEX_RATE_LIMITED_CODE,\n+            relogin_required=False,\n+        )\n+\n     if response.status_code != 200:\n         code = \"codex_refresh_failed\"\n         message = f\"Codex token refresh failed with status {response.status_code}.\""]},{"file":"tests/gateway/test_verbose_command.py","hunks":["@@ -128,8 +128,14 @@ async def test_cycles_through_all_modes(self, tmp_path, monkeypatch):\n                 f\"Expected {mode}, got {actual}\"\n \n     @pytest.mark.asyncio\n-    async def test_defaults_to_all_when_no_tool_progress_set(self, tmp_path, monkeypatch):\n-        \"\"\"When tool_progress is not in config, defaults to platform default then cycles.\"\"\"\n+    async def test_defaults_to_platform_default_when_no_tool_progress_set(self, tmp_path, monkeypatch):\n+        \"\"\"When tool_progress is not in config, starts from platform default then cycles.\n+\n+        Telegram's tier-1 preset overrides ``tool_progress`` to ``\"off\"`` so the\n+        platform stays final-answer-first by default on mobile inboxes.  The\n+        first ``/verbose`` invocation therefore cycles ``off → new``, not\n+        ``all → ...``.\n+        \"\"\"\n         hermes_home = tmp_path / \"hermes\"\n         hermes_home.mkdir()\n         config_path = hermes_home / \"config.yaml\"","@@ -143,17 +149,18 @@ async def test_defaults_to_all_when_no_tool_progress_set(self, tmp_path, monkeyp\n         runner = _make_runner()\n         result = await runner._handle_verbose_command(_make_event())\n \n-        # Telegram platform default is \"new\" → cycles to \"all\"\n-        assert \"ALL\" in result\n+        # Telegram platform default is \"off\" → cycles to \"new\"\n+        assert \"NEW\" in result\n         saved = yaml.safe_load(config_path.read_text(encoding=\"utf-8\"))\n-        assert saved[\"display\"][\"platforms\"][\"telegram\"][\"tool_progress\"] == \"all\"\n+        assert saved[\"display\"][\"platforms\"][\"telegram\"][\"tool_progress\"] == \"new\"\n \n     @pytest.mark.asyncio\n     async def test_per_platform_isolation(self, tmp_path, monkeypatch):\n         \"\"\"Cycling /verbose on Telegram doesn't change Slack's setting.\n \n         Without a global tool_progress, each platform uses its built-in\n-        default: Telegram = 'new' (overridden high tier), Slack = 'off' (quiet Slack default).\n+        default — Telegram = 'off' (tier-1 inbox override), Slack = 'off'\n+        (quiet Slack default). Both cycle to 'new' on first /verbose.\n         \"\"\"\n         hermes_home = tmp_path / \"hermes\"\n         hermes_home.mkdir()","@@ -178,8 +185,8 @@ async def test_per_platform_isolation(self, tmp_path, monkeypatch):\n \n         saved = yaml.safe_load(config_path.read_text(encoding=\"utf-8\"))\n         platforms = saved[\"display\"][\"platforms\"]\n-        # Telegram: new -> all (platform default = new)\n-        assert platforms[\"telegram\"][\"tool_progress\"] == \"all\"\n+        # Telegram: off -> new (platform default = off, tier-1 inbox override)\n+        assert platforms[\"telegram\"][\"tool_progress\"] == \"new\"\n         # Slack: off -> new (first /verbose cycle from quiet default)\n         assert platforms[\"slack\"][\"tool_progress\"] == \"new\"\n "]},{"file":"tests/hermes_cli/test_auth_codex_provider.py","hunks":["@@ -263,9 +263,10 @@ def test_resolve_returns_hermes_auth_store_source(tmp_path, monkeypatch):\n \n \n class _StubHTTPResponse:\n-    def __init__(self, status_code: int, payload):\n+    def __init__(self, status_code: int, payload, headers=None):\n         self.status_code = status_code\n         self._payload = payload\n+        self.headers = headers or {}\n         self.text = json.dumps(payload) if isinstance(payload, (dict, list)) else str(payload)\n \n     def json(self):","@@ -382,6 +383,74 @@ def test_refresh_falls_back_to_generic_message_on_unparseable_body(monkeypatch):\n     assert \"status 401\" in str(err)\n \n \n+def test_refresh_429_classified_as_quota_not_auth_failure(monkeypatch):\n+    \"\"\"429 from the token endpoint is a usage-quota cap, not an auth failure.\n+\n+    Regression test for #32790: must NOT force relogin and must carry the\n+    dedicated rate-limit code so callers surface a \"retry later\" notice rather\n+    than a misleading \"run hermes auth\".\n+    \"\"\"\n+    from hermes_cli.auth import (\n+        CODEX_RATE_LIMITED_CODE,\n+        format_auth_error,\n+        is_rate_limited_auth_error,\n+    )\n+\n+    response = _StubHTTPResponse(\n+        429,\n+        {\"error\": {\"message\": \"You hit your usage limit.\", \"code\": \"usage_limit_reached\"}},\n+        headers={\"retry-after\": \"120\"},\n+    )\n+    _patch_httpx(monkeypatch, response)\n+\n+    with pytest.raises(AuthError) as exc_info:\n+        refresh_codex_oauth_pure(\"a-tok\", \"r-tok\")\n+\n+    err = exc_info.value\n+    assert err.code == CODEX_RATE_LIMITED_CODE\n+    assert err.relogin_required is False\n+    assert is_rate_limited_auth_error(err) is True\n+    assert \"retry after 120s\" in str(err)\n+    # User-facing copy must not tell the operator to re-authenticate.\n+    rendered = format_auth_error(err)\n+    assert \"re-authenticate\" not in rendered\n+    assert \"hermes auth\" not in rendered\n+\n+\n+def test_refresh_429_without_retry_after_header(monkeypatch):\n+    \"\"\"429 without a Retry-After header still classifies as quota, no relogin.\"\"\"\n+    from hermes_cli.auth import CODEX_RATE_LIMITED_CODE\n+\n+    response = _StubHTTPResponse(429, {\"error\": \"rate_limited\"})\n+    _patch_httpx(monkeypatch, response)\n+\n+    with pytest.raises(AuthError) as exc_info:\n+        refresh_codex_oauth_pure(\"a-tok\", \"r-tok\")\n+\n+    err = exc_info.value\n+    assert err.code == CODEX_RATE_LIMITED_CODE\n+    assert err.relogin_required is False\n+    assert \"quota exhausted\" in str(err).lower()\n+\n+\n+def test_is_rate_limited_auth_error_distinguishes_credential_errors():\n+    \"\"\"Missing/expired credentials must NOT be treated as rate-limit errors.\"\"\"\n+    from hermes_cli.auth import CODEX_RATE_LIMITED_CODE, is_rate_limited_auth_error\n+\n+    rate_limited = AuthError(\n+        \"quota\", provider=\"openai-codex\", code=CODEX_RATE_LIMITED_CODE, relogin_required=False\n+    )\n+    missing_creds = AuthError(\n+        \"No Codex credentials stored.\",\n+        provider=\"openai-codex\",\n+        code=\"codex_auth_missing\",\n+        relogin_required=True,\n+    )\n+    assert is_rate_limited_auth_error(rate_limited) is True\n+    assert is_rate_limited_auth_error(missing_creds) is False\n+    assert is_rate_limited_auth_error(ValueError(\"nope\")) is False\n+\n+\n def test_login_openai_codex_force_new_login_skips_existing_reuse_prompt(monkeypatch):\n     called = {\"device_login\": 0}\n \n"]}]},{"kind":"pr","number":33779,"title":"fix(gateway): separate 429 quota errors from auth failures (#32790)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#33168","number":33168},{"kind":"pr","value":"#32790","number":32790},{"kind":"pr","value":"#33780","number":33780}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":33780,"title":"fix(gateway): separate 429 quota errors from auth failures (#32790)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#33168","number":33168},{"kind":"pr","value":"#32790","number":32790},{"kind":"pr","value":"#32881","number":32881}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":46643,"title":"fix(gateway): consolidate Codex quota error handling","url":null,"state":"closed","close_reason":{"category":"volume_quota","implication":"active"},"merged":false},{"kind":"pr","number":60856,"title":"Fix gateway 429 provider error classification","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}