{"repo":"NousResearch/hermes-agent","public_id":"32421","display_label":"Content-filter stream stall fallback routing fixes","summary":"A bug report and related fixes addressing cases where content-filter triggers during streaming do not activate fallback providers or chains. Changes target both the minimax integration and agent routing logic.","subsystem":"agent","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":54064,"merged_at":"2026-06-28T08:15:22+00:00"},"resolution":{"state":"resolved","by":"pr-54064"},"freshness":{"state":"verified","last_verified_at":"2026-06-28T08:15:22+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":32421,"pr_number":54064,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-28T08:15:22+00:00"}],"members":[{"kind":"issue","number":32421,"title":"[Bug]: Content-filter triggered stream stall (output new_sensitive) does not trigger fallback_providers","url":null,"state":"closed"},{"kind":"pr","number":32479,"title":"fix(agent): trigger fallback when content filter terminates stream mid-delivery (#32421)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4f16950e9","number":0},{"kind":"pr","value":"#54064","number":54064},{"kind":"pr","value":"#37851","number":37851},{"kind":"pr","value":"#33845","number":33845}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":33845,"title":"fix(minimax): activate fallback after content-filter stream stalls","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4f16950e9","number":0},{"kind":"pr","value":"#54064","number":54064},{"kind":"pr","value":"#37851","number":37851},{"kind":"pr","value":"#32479","number":32479}],"category":"partial_fix","implication":"needs_llm"},"merged":false},{"kind":"pr","number":54064,"title":"fix(agent): route content-filter stream stalls to fallback chain (#32421)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/chat_completion_helpers.py","agent/conversation_loop.py","agent/error_classifier.py","agent/turn_retry_state.py","infographic/content-filter-fallback/infographic.png","scripts/release.py","tests/agent/test_turn_retry_state.py","tests/run_agent/test_18028_content_policy_blocked.py","tests/run_agent/test_partial_stream_finish_reason.py"],"fix_hunks":[{"file":"agent/chat_completion_helpers.py","hunks":["@@ -2788,7 +2788,30 @@ def _call():\n                 role=\"assistant\", content=_partial_text, tool_calls=None,\n                 reasoning_content=None,\n             )\n-            return SimpleNamespace(\n+            # Detect provider output-layer content filtering (e.g. MiniMax\n+            # \"output new_sensitive (1027)\", Azure/OpenAI content_filter,\n+            # Anthropic safety refusal).  The raw error is about to be\n+            # swallowed into a finish_reason=length stub, so classify it HERE\n+            # while we still have it and stamp the stub.  Retrying such a\n+            # content-deterministic filter on the same primary just re-hits\n+            # the filter — the conversation loop reads this tag and activates\n+            # the fallback chain instead of burning continuation retries.\n+            # error_classifier is the single source of truth for \"what counts\n+            # as a content filter\" (#32421).\n+            _content_filter_terminated = False\n+            try:\n+                from agent.error_classifier import classify_api_error, FailoverReason\n+                _cls = classify_api_error(\n+                    result[\"error\"],\n+                    provider=str(getattr(agent, \"provider\", \"\") or \"\"),\n+                    model=str(getattr(agent, \"model\", \"\") or \"\"),\n+                )\n+                _content_filter_terminated = (\n+                    _cls.reason == FailoverReason.content_policy_blocked\n+                )\n+            except Exception:\n+                _content_filter_terminated = False\n+            _stub = SimpleNamespace(\n                 id=PARTIAL_STREAM_STUB_ID,\n                 model=getattr(agent, \"model\", \"unknown\"),\n                 choices=[SimpleNamespace(","@@ -2797,6 +2820,9 @@ def _call():\n                 usage=None,\n                 _dropped_tool_names=_partial_names or None,\n             )\n+            if _content_filter_terminated:\n+                _stub._content_filter_terminated = True\n+            return _stub\n         raise result[\"error\"]\n     return result[\"response\"]\n "]},{"file":"agent/conversation_loop.py","hunks":["@@ -1699,6 +1699,56 @@ def _perform_api_call(next_api_kwargs):\n \n                     if agent.api_mode in {\"chat_completions\", \"bedrock_converse\", \"anthropic_messages\"}:\n                         assistant_message = _trunc_msg\n+                        # ── Content-filter stream stall → fallback (#32421) ──\n+                        # When the provider's output-layer safety filter (e.g.\n+                        # MiniMax \"output new_sensitive (1027)\", Azure\n+                        # content_filter) kills the stream mid-delivery, the\n+                        # raw error was classified at the swallow point and the\n+                        # stub tagged ``_content_filter_terminated``.  This\n+                        # filter is content-deterministic — continuation\n+                        # retries against the SAME primary just re-hit it and\n+                        # burn paid attempts (the loop used to give up with\n+                        # \"Response remained truncated after 3 continuation\n+                        # attempts\" and never consult the fallback chain).\n+                        # Escalate to the configured fallback BEFORE retrying.\n+                        _cf_terminated = getattr(\n+                            response, \"_content_filter_terminated\", False\n+                        )\n+                        if (\n+                            _cf_terminated\n+                            and agent._fallback_index < len(agent._fallback_chain)\n+                        ):\n+                            agent._vprint(\n+                                f\"{agent.log_prefix}🛡️  Content filter terminated \"\n+                                f\"stream — activating fallback provider...\",\n+                                force=True,\n+                            )\n+                            agent._emit_status(\n+                                \"Content filter terminated stream; switching to fallback...\"\n+                            )\n+                            if agent._try_activate_fallback():\n+                                # Roll the partial content (if any was already\n+                                # appended in a prior continuation pass) back to\n+                                # the last clean turn so the fallback provider\n+                                # gets a coherent continuation point.\n+                                if truncated_response_parts:\n+                                    messages = agent._get_messages_up_to_last_assistant(messages)\n+                                agent._session_messages = messages\n+                                length_continue_retries = 0\n+                                truncated_response_parts = []\n+                                retry_count = 0\n+                                compression_attempts = 0\n+                                _retry.primary_recovery_attempted = False\n+                                _retry.restart_with_rebuilt_messages = True\n+                                break\n+                            # No fallback available — fall through to normal\n+                            # continuation (best-effort, may loop).\n+                            agent._vprint(\n+                                f\"{agent.log_prefix}⚠️  No fallback provider \"\n+                                f\"configured — retrying with same provider \"\n+                                f\"(may re-hit filter)...\",\n+                                force=True,\n+                            )\n                         if assistant_message is not None and not _trunc_has_tool_calls:\n                             length_continue_retries += 1\n                             interim_msg = agent._build_assistant_message(assistant_message, finish_reason)","@@ -3774,6 +3824,17 @@ def _perform_api_call(next_api_kwargs):\n             _retry.restart_with_compressed_messages = False\n             continue\n \n+        if _retry.restart_with_rebuilt_messages:\n+            # A content-filter stream stall (#32421) was escalated to the\n+            # fallback chain and the partial content rolled back.  Re-issue\n+            # the API call against the now-active fallback provider.  Refund\n+            # the budget/count for the stalled attempt so the fallback gets a\n+            # fair turn.\n+            api_call_count -= 1\n+            agent.iteration_budget.refund()\n+            _retry.restart_with_rebuilt_messages = False\n+            continue\n+\n         if _retry.restart_with_length_continuation:\n             # Progressively boost the output token budget on each retry.\n             # Retry 1 → 2× base, retry 2 → 3× base, capped at 32 768."]},{"file":"agent/error_classifier.py","hunks":["@@ -355,6 +355,14 @@ def is_auth(self) -> bool:\n     # echo back; the underscore form is provider-specific enough.\n     \"content_filter\",\n     \"responsibleaipolicyviolation\",\n+    # MiniMax output-layer safety filter. The error string is surfaced\n+    # verbatim by MiniMax SDK / OpenAI-compatible endpoints, usually in the\n+    # form \"output new_sensitive (1027)\" when the model's *output* (often a\n+    # large tool-call argument block) trips the upstream safety filter and\n+    # the SSE stream is truncated mid-flight. ``new_sensitive`` is the\n+    # filter name and is narrow enough that billing / format / auth error\n+    # strings will not collide. See #32421.\n+    \"new_sensitive\",\n ]\n \n # Auth patterns (non-status-code signals)"]},{"file":"agent/turn_retry_state.py","hunks":["@@ -67,6 +67,11 @@ class TurnRetryState:\n     # ── Restart signals (read by the outer loop after the attempt) ───────\n     restart_with_compressed_messages: bool = False\n     restart_with_length_continuation: bool = False\n+    # Set when a content-filter stream stall (e.g. MiniMax \"new_sensitive\")\n+    # has been escalated to the fallback chain: the partial-stream content\n+    # was rolled back off ``messages`` and the loop should re-issue the API\n+    # call against the newly-activated provider (#32421).\n+    restart_with_rebuilt_messages: bool = False\n \n     def __iter__(self):\n         # Convenience for debugging / tests: iterate (name, value) pairs."]},{"file":"scripts/release.py","hunks":["@@ -64,6 +64,7 @@\n     \"8180647+herbalizer404@users.noreply.github.com\": \"herbalizer404\",  # PR #49076 + #51835 salvage (auxiliary compression fallback: 403/session-usage payment errors + honor fallback chain when aux provider auth unavailable)\n     \"pyxl-dev@users.noreply.github.com\": \"pyxl-dev\",  # PR #52230 salvage (include rate-limit in auxiliary capacity-error fallback gate; #52228)\n     \"[redacted:email]\": \"yashiels\",  # PR #53284 salvage (discord markdown table-to-bullet conversion; #21168)\n+    \"15205536+595650661@users.noreply.github.com\": \"595650661\",  # PR #37851 salvage (classify MiniMax new_sensitive content filter → content_policy_blocked; #32421)\n     \"[redacted:email]\": \"benbenlijie\",  # PR #47205 salvage (named custom-provider extra_body + Z.AI Coding overload adaptive backoff; #50663)\n     \"[redacted:email]\": \"Danamove\",  # PR #46726 salvage (kill venv-resident pythonw gateway before recreating venv on Windows; #47036/#47557/#47910)\n     \"145739220+wgu9@users.noreply.github.com\": \"wgu9\",  # PR #51468 salvage (WSL/no-systemd orphan gateway tracking, #51325)"]},{"file":"tests/agent/test_turn_retry_state.py","hunks":["@@ -30,6 +30,7 @@\n     \"auth_failover_attempted\",\n     \"restart_with_compressed_messages\",\n     \"restart_with_length_continuation\",\n+    \"restart_with_rebuilt_messages\",\n }\n \n "]},{"file":"tests/run_agent/test_18028_content_policy_blocked.py","hunks":["@@ -41,6 +41,27 @@ def test_openai_codex_cybersecurity_no_status(self):\n         assert result.should_compress is False\n         assert result.should_rotate_credential is False\n \n+    def test_minimax_output_safety_filter(self):\n+        \"\"\"#32421 — MiniMax output-layer safety filter (e.g. ``output\n+        new_sensitive (1027)``) trips mid-stream when the model emits a\n+        large tool-call argument block. Must classify as\n+        ``content_policy_blocked`` so the loop aborts the 3x retry burn and\n+        routes to a configured fallback model.\n+        \"\"\"\n+        from agent.error_classifier import classify_api_error, FailoverReason\n+\n+        e = Exception(\n+            \"Stream stalled mid tool-call: output new_sensitive (1027) \"\n+            \"[MiniMax-M2.7] — request was rejected by upstream safety \"\n+            \"filter, see provider response for details.\"\n+        )\n+        result = classify_api_error(e, provider=\"MiniMax\", model=\"MiniMax-M2.7\")\n+        assert result.reason == FailoverReason.content_policy_blocked\n+        assert result.retryable is False\n+        assert result.should_fallback is True\n+        assert result.should_compress is False\n+        assert result.should_rotate_credential is False\n+\n \n class TestContentPolicyTriggersClientErrorAbort:\n     \"\"\"Mirror the ``is_client_error`` predicate in"]},{"file":"tests/run_agent/test_partial_stream_finish_reason.py","hunks":["@@ -362,3 +362,194 @@ def test_partial_stream_stub_does_not_exit_loop_immediately(self, loop_agent):\n         # And the final response stitches both halves together.\n         assert \"first half of\" in result[\"final_response\"]\n         assert \"forty-two\" in result[\"final_response\"]\n+\n+\n+class TestContentFilterStallActivatesFallback:\n+    \"\"\"Regression for #32421: a provider output-layer content safety filter\n+    (e.g. MiniMax ``output new_sensitive (1027)``) terminates a streaming\n+    response mid-delivery.  The raw error is swallowed into a\n+    finish_reason=length partial-stream stub, so before the fix the loop\n+    burned 3 continuation retries against the SAME primary (re-hitting the\n+    content-deterministic filter every time) and gave up with\n+    ``\"Response remained truncated after 3 continuation attempts\"`` — the\n+    configured fallback chain was never consulted.\n+\n+    The fix has three layers:\n+      1. error_classifier classifies ``new_sensitive`` as\n+         ``content_policy_blocked``.\n+      2. interruptible_streaming_api_call runs the swallowed error through\n+         that classifier and stamps the stub ``_content_filter_terminated``.\n+      3. the conversation loop reads the tag and activates fallback BEFORE\n+         burning any continuation retries.\n+    \"\"\"\n+\n+    @patch(\"run_agent.AIAgent._create_request_openai_client\")\n+    @patch(\"run_agent.AIAgent._close_request_openai_client\")\n+    def test_streaming_call_tags_content_filter_stub(\n+        self, _mock_close, mock_create, monkeypatch,\n+    ):\n+        \"\"\"Layer 2: the real streaming path stamps _content_filter_terminated\n+        when the swallowed error matches a content-filter pattern.\"\"\"\n+\n+        def _minimax_stall():\n+            yield _make_stream_chunk(content=\"Writing the file: \")\n+            yield _make_stream_chunk(tool_calls=[\n+                _make_tool_call_delta(index=0, tc_id=\"call_1\", name=\"write_file\"),\n+            ])\n+            yield _make_stream_chunk(tool_calls=[\n+                _make_tool_call_delta(index=0, arguments='{\"path\": \"/tmp/x\", '),\n+            ])\n+            raise RuntimeError(\"output new_sensitive (1027) [MiniMax-M2.7]\")\n+\n+        mock_client = MagicMock()\n+        mock_client.chat.completions.create.side_effect = (\n+            lambda *a, **kw: _minimax_stall()\n+        )\n+        mock_create.return_value = mock_client\n+\n+        agent = _make_agent()\n+        agent._fire_stream_delta = lambda text: None\n+        agent._current_streamed_assistant_text = \"Writing the file: \"\n+\n+        monkeypatch.setenv(\"HERMES_STREAM_RETRIES\", \"0\")\n+        response = agent._interruptible_streaming_api_call({})\n+\n+        assert response.id == PARTIAL_STREAM_STUB_ID\n+        assert getattr(response, \"_content_filter_terminated\", False) is True, (\n+            \"MiniMax new_sensitive stream stall must tag the stub so the loop \"\n+            \"can route to fallback (#32421).\"\n+        )\n+\n+    @patch(\"run_agent.AIAgent._create_request_openai_client\")\n+    @patch(\"run_agent.AIAgent._close_request_openai_client\")\n+    def test_plain_network_stall_not_tagged(\n+        self, _mock_close, mock_create, monkeypatch,\n+    ):\n+        \"\"\"A plain network stall (no content-filter signature) must NOT be\n+        tagged — it should still use the normal continuation path, not\n+        switch providers.\"\"\"\n+\n+        def _network_stall():\n+            yield _make_stream_chunk(content=\"Writing the file: \")\n+            raise RuntimeError(\"connection reset by peer\")\n+\n+        mock_client = MagicMock()\n+        mock_client.chat.completions.create.side_effect = (\n+            lambda *a, **kw: _network_stall()\n+        )\n+        mock_create.return_value = mock_client\n+\n+        agent = _make_agent()\n+        agent._fire_stream_delta = lambda text: None\n+        agent._current_streamed_assistant_text = \"Writing the file: \"\n+\n+        monkeypatch.setenv(\"HERMES_STREAM_RETRIES\", \"0\")\n+        response = agent._interruptible_streaming_api_call({})\n+\n+        assert response.id == PARTIAL_STREAM_STUB_ID\n+        assert getattr(response, \"_content_filter_terminated\", False) is False, (\n+            \"A plain network stall must not be misclassified as a content \"\n+            \"filter — that would needlessly switch providers.\"\n+        )\n+\n+    def test_tagged_stub_activates_fallback_first_pass(self, loop_agent):\n+        \"\"\"Layer 3: a tagged stub activates fallback on the FIRST pass, with\n+        zero continuation retries burned, and the fallback provider then\n+        completes the turn.\"\"\"\n+        from tests.run_agent.test_run_agent import _mock_assistant_msg, _mock_response\n+\n+        def _filter_stub():\n+            return SimpleNamespace(\n+                id=PARTIAL_STREAM_STUB_ID,\n+                model=\"minimax/MiniMax-M2.7\",\n+                choices=[SimpleNamespace(\n+                    index=0,\n+                    message=_mock_assistant_msg(content=\"Writing the file...\"),\n+                    finish_reason=FINISH_REASON_LENGTH,\n+                )],\n+                usage=None,\n+                _dropped_tool_names=[\"write_file\"],\n+                _content_filter_terminated=True,\n+            )\n+\n+        recovery = _mock_response(\n+            content=\"Done on the fallback provider.\", finish_reason=\"stop\",\n+        )\n+        loop_agent.client.chat.completions.create.side_effect = [\n+            _filter_stub(), recovery,\n+        ]\n+        loop_agent._fallback_chain = [\n+            {\"provider\": \"openrouter\", \"model\": \"anthropic/claude-sonnet-4.7\"},\n+        ]\n+        loop_agent._fallback_index = 0\n+        fb_calls = {\"n\": 0}\n+\n+        def _fake_activate(reason=None):\n+            fb_calls[\"n\"] += 1\n+            loop_agent._fallback_index = len(loop_agent._fallback_chain)\n+            return True\n+\n+        with (\n+            patch.object(loop_agent, \"_persist_session\"),\n+            patch.object(loop_agent, \"_save_trajectory\"),\n+            patch.object(loop_agent, \"_cleanup_task_resources\"),\n+            patch.object(loop_agent, \"_try_activate_fallback\",\n+                         side_effect=_fake_activate),\n+        ):\n+            result = loop_agent.run_conversation(\"write me a long file\")\n+\n+        assert fb_calls[\"n\"] == 1, (\n+            \"Content-filter-tagged stub must activate fallback exactly once, \"\n+            \"on the first pass — not after exhausting continuation retries.\"\n+        )\n+        assert result[\"final_response\"] == \"Done on the fallback provider.\"\n+        assert result[\"completed\"] is True\n+\n+    def test_tagged_stub_no_fallback_falls_through(self, loop_agent):\n+        \"\"\"When no fallback chain is configured, a tagged stub falls through\n+        to the normal continuation path (best-effort) rather than crashing.\"\"\"\n+        from tests.run_agent.test_run_agent import _mock_assistant_msg, _mock_response\n+\n+        def _filter_stub():\n+            return SimpleNamespace(\n+                id=PARTIAL_STREAM_STUB_ID,\n+                model=\"minimax/MiniMax-M2.7\",\n+                choices=[SimpleNamespace(\n+                    index=0,\n+                    message=_mock_assistant_msg(content=\"partial \"),\n+                    finish_reason=FINISH_REASON_LENGTH,\n+                )],\n+                usage=None,\n+                _dropped_tool_names=[\"write_file\"],\n+                _content_filter_terminated=True,\n+            )\n+\n+        recovery = _mock_response(content=\"recovered text\", finish_reason=\"stop\")\n+        loop_agent.client.chat.completions.create.side_effect = [\n+            _filter_stub(), recovery,\n+        ]\n+        # No fallback chain configured.\n+        loop_agent._fallback_chain = []\n+        loop_agent._fallback_index = 0\n+        fb_calls = {\"n\": 0}\n+\n+        def _fake_activate(reason=None):\n+            fb_calls[\"n\"] += 1\n+            return False\n+\n+        with (\n+            patch.object(loop_agent, \"_persist_session\"),\n+            patch.object(loop_agent, \"_save_trajectory\"),\n+            patch.object(loop_agent, \"_cleanup_task_resources\"),\n+            patch.object(loop_agent, \"_try_activate_fallback\",\n+                         side_effect=_fake_activate),\n+        ):\n+            result = loop_agent.run_conversation(\"write me a long file\")\n+\n+        # Fallback was not attempted (empty chain gates it out); the loop\n+        # continued normally and produced a response.\n+        assert fb_calls[\"n\"] == 0, (\n+            \"With an empty fallback chain, the loop must not even call \"\n+            \"_try_activate_fallback — it should fall through to continuation.\"\n+        )\n+        assert result[\"completed\"] is True\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}