{"repo":"NousResearch/hermes-agent","public_id":"31273","display_label":"HTTP 402 billing errors treated as retryable; fallback and retry controls","summary":"Most members concern HTTP 402 (payment required) responses being retried like transient errors, with reported excess token spend, and propose aborting the retry loop on 402. One member adds a config option to disable eager rate-limit fallback, which is related but distinct.","subsystem":"agent","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":31443,"merged_at":"2026-05-24T22:14:13+00:00"},"resolution":{"state":"resolved","by":"pr-31443"},"freshness":{"state":"verified","last_verified_at":"2026-05-24T22:14:13+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":31273,"pr_number":31443,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-24T22:14:13+00:00"}],"members":[{"kind":"issue","number":31273,"title":"HTTP 402 (payment required) incorrectly retried as transient error — causes runaway token spend","url":null,"state":"closed"},{"kind":"pr","number":27858,"title":"feat: add config option to disable eager rate-limit fallback","url":null,"state":"closed","close_reason":{"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":31303,"title":"fix(agent): abort retry loop on 402 billing instead of burning api_max_retries","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":31443,"title":"fix(agent): abort on HTTP 402 after pool rotation and fallback fail","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/conversation_loop.py","tests/run_agent/test_31273_402_not_retried.py"],"fix_hunks":[{"file":"agent/conversation_loop.py","hunks":["@@ -2772,6 +2772,21 @@ def _stop_spinner():\n                     # retryable=True mapping takes effect instead.\n                     and not isinstance(api_error, ssl.SSLError)\n                 )\n+                # ``FailoverReason.billing`` (HTTP 402) is NOT in this\n+                # exclusion set.  By the time we reach this block:\n+                #   • credential-pool rotation (line ~2031) has already\n+                #     fired for billing and either ``continue``d or\n+                #     returned (False, ...) — pool is exhausted or absent.\n+                #   • the eager-fallback branch above (line ~2422) also\n+                #     fires on billing and ``continue``s if a fallback\n+                #     provider is configured.\n+                # Falling through to here means BOTH recovery paths\n+                # gave up.  Treating 402 as retryable from this point\n+                # just burns more paid requests against a depleted\n+                # balance with no recovery mechanism left — see #31273\n+                # (real-world: ~$40 in 48h on a 24/7 gateway).  Aborting\n+                # mirrors how 401/403 (also ``should_fallback=True``)\n+                # already behave once their recovery paths have failed.\n                 is_client_error = (\n                     is_local_validation_error\n                     or (","@@ -2779,7 +2794,6 @@ def _stop_spinner():\n                         and not classified.should_compress\n                         and classified.reason not in {\n                             FailoverReason.rate_limit,\n-                            FailoverReason.billing,\n                             FailoverReason.overloaded,\n                             FailoverReason.context_overflow,\n                             FailoverReason.payload_too_large,"]},{"file":"tests/run_agent/test_31273_402_not_retried.py","hunks":["@@ -0,0 +1,147 @@\n+\"\"\"Regression guard for #31273: HTTP 402 (billing exhaustion) must abort\n+after credential-pool rotation and provider fallback have failed.\n+\n+Before the fix, ``FailoverReason.billing`` was in the exclusion set that\n+prevents the loop's ``is_client_error`` branch from firing.  When a user\n+ran a pay-per-token provider (OpenRouter, etc.) with no credential pool\n+and no fallback configured, a single 402 cascaded into\n+``agent.api_max_retries`` paid requests against an exhausted balance.\n+Real-world impact: ~$40 burned in 48h on a 24/7 gateway routing Telegram\n++ Discord traffic.\n+\n+The fix removes ``FailoverReason.billing`` from the exclusion set.  By\n+the time control reaches the ``is_client_error`` check:\n+  * credential-pool rotation has already run (and either ``continue``d\n+    on rotation, or returned False because the pool is exhausted/absent).\n+  * the eager-fallback branch for billing has also run (and either\n+    ``continue``d on fallback activation, or fell through because no\n+    fallback is configured).\n+Falling through to the retry-backoff path from here just burns paid\n+requests with no recovery mechanism left.  Aborting mirrors how 401/403\n+(also ``should_fallback=True``) already behave once their recovery paths\n+have failed.\n+\"\"\"\n+from __future__ import annotations\n+\n+\n+class TestBillingTriggersClientErrorAbort:\n+    \"\"\"Mirror the ``is_client_error`` predicate shape used in\n+    ``agent/conversation_loop.py`` and verify ``FailoverReason.billing``\n+    now resolves to True (i.e. aborts the loop).\n+    \"\"\"\n+\n+    def _mirror_is_client_error(\n+        self,\n+        *,\n+        classified_retryable: bool,\n+        classified_reason,\n+        classified_should_compress: bool = False,\n+        is_local_validation_error: bool = False,\n+        is_context_length_error: bool = False,\n+    ) -> bool:\n+        \"\"\"Exact shape of conversation_loop.py's is_client_error check.\n+\n+        Kept in lock-step with the source.  If you change one, change\n+        both — or, better, refactor the predicate into a shared helper\n+        and have both sites import it.\n+        \"\"\"\n+        from agent.error_classifier import FailoverReason\n+\n+        return (\n+            is_local_validation_error\n+            or (\n+                not classified_retryable\n+                and not classified_should_compress\n+                and classified_reason not in {\n+                    FailoverReason.rate_limit,\n+                    FailoverReason.overloaded,\n+                    FailoverReason.context_overflow,\n+                    FailoverReason.payload_too_large,\n+                    FailoverReason.long_context_tier,\n+                    FailoverReason.thinking_signature,\n+                }\n+            )\n+        ) and not is_context_length_error\n+\n+    def test_billing_now_aborts_the_loop(self):\n+        \"\"\"402 with no fallback / no pool entry → ``is_client_error`` True.\"\"\"\n+        from agent.error_classifier import FailoverReason\n+\n+        # This is what classify_api_error() returns for a plain 402:\n+        #   reason=billing, retryable=False, should_compress=False\n+        assert self._mirror_is_client_error(\n+            classified_retryable=False,\n+            classified_reason=FailoverReason.billing,\n+        ), (\n+            \"FailoverReason.billing must trigger is_client_error abort after \"\n+            \"credential-pool rotation and provider fallback have failed — see #31273.\"\n+        )\n+\n+    def test_rate_limit_still_retries(self):\n+        \"\"\"Sanity check: rate_limit must still fall through to backoff retry.\"\"\"\n+        from agent.error_classifier import FailoverReason\n+\n+        # 429 / transient 402 / rate-limited usage: must NOT abort,\n+        # because Retry-After backoff and pool rotation are the right\n+        # recovery paths.\n+        assert not self._mirror_is_client_error(\n+            classified_retryable=True,\n+            classified_reason=FailoverReason.rate_limit,\n+        )\n+\n+    def test_local_validation_error_still_aborts(self):\n+        \"\"\"Sanity check: bare ValueError/TypeError still abort.\"\"\"\n+        from agent.error_classifier import FailoverReason\n+\n+        assert self._mirror_is_client_error(\n+            classified_retryable=True,\n+            classified_reason=FailoverReason.unknown,\n+            is_local_validation_error=True,\n+        )\n+\n+    def test_context_overflow_still_falls_through_to_compression(self):\n+        \"\"\"Sanity check: context-overflow must NOT be classified as\n+        client error — compression is the recovery path.\"\"\"\n+        from agent.error_classifier import FailoverReason\n+\n+        assert not self._mirror_is_client_error(\n+            classified_retryable=True,\n+            classified_reason=FailoverReason.context_overflow,\n+            classified_should_compress=True,\n+        )\n+\n+\n+class TestSourceStillHasBillingExclusionRemoved:\n+    \"\"\"Belt-and-suspenders: the production source must actually omit\n+    ``FailoverReason.billing`` from the ``is_client_error`` exclusion\n+    set.  Protects against an accidental re-introduction.\n+    \"\"\"\n+\n+    def test_conversation_loop_omits_billing_from_client_error_exclusion(self):\n+        import inspect\n+        from agent import conversation_loop\n+\n+        src = inspect.getsource(conversation_loop)\n+\n+        # Locate the is_client_error block and inspect its exclusion set.\n+        marker = \"is_client_error = (\"\n+        assert marker in src, (\n+            \"agent/conversation_loop.py must define is_client_error — \"\n+            \"the bug-fix anchor for #31273 has moved or been renamed.\"\n+        )\n+        idx = src.index(marker)\n+        # Window large enough to span the full predicate (~30 lines).\n+        window = src[idx:idx + 2000]\n+\n+        assert \"FailoverReason.rate_limit\" in window, (\n+            \"is_client_error exclusion set has changed shape — re-verify \"\n+            \"that FailoverReason.billing is still NOT in it (#31273).\"\n+        )\n+        assert \"FailoverReason.billing\" not in window, (\n+            \"FailoverReason.billing must NOT appear in the is_client_error \"\n+            \"exclusion set — see #31273.  Billing (HTTP 402) is non-retryable \"\n+            \"by the time control reaches this block: credential-pool rotation \"\n+            \"and provider fallback have both already had their chance to \"\n+            \"continue the loop.  Re-adding it causes runaway token spend on \"\n+            \"depleted balances.\"\n+        )\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}