{"repo":"NousResearch/hermes-agent","public_id":"27480","display_label":"Stale-stream watchdog rebuilds wrong client for non-OpenAI providers","summary":"These issues and fixes concern the stale/retry stream watchdog incorrectly invoking the OpenAI client rebuild path when the active provider is Anthropic, Bedrock, or MiniMax, causing hangs, dropped API keys, and silent recovery failures. The work spans detection, routing, and rebuilding the correct provider client during stream cleanup and retry.","subsystem":"agent","counts":{"members":15,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":6,"merged_prs":1,"attempt_prs":8},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":53926},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-28T02:37:34+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.1554285714285714},"fix_edges":[{"issue_number":28161,"pr_number":53926,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":36693,"pr_number":53926,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":44006,"pr_number":53926,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":51844,"pr_number":53926,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":27480,"title":"MiniMax: stream stall + rebuild failure leaves worker hung (provider=minimax, api_key dropped during _replace_primary_openai_client)","url":null,"state":"closed"},{"kind":"issue","number":28161,"title":"Anthropic streaming: stale/retry paths call _replace_primary_openai_client, causing 15-min hang on stuck streams","url":null,"state":"closed"},{"kind":"issue","number":31128,"title":"Stale-stream handler tries to rebuild OpenAI client when provider is Anthropic","url":null,"state":"closed"},{"kind":"issue","number":36693,"title":"Bedrock provider: stale-stream client rebuild logs misleading \"OPENAI_API_KEY missing\" warning","url":null,"state":"closed"},{"kind":"issue","number":44006,"title":"Stream-retry rebuilds OpenAI client without api_key, drops the retry","url":null,"state":"closed"},{"kind":"issue","number":51844,"title":"Stale-stream recovery calls the OpenAI client rebuild for native/OAuth Anthropic → `api_key must be set` (recovery silently fails)","url":null,"state":"closed"},{"kind":"pr","number":28240,"title":"fix(streaming): rebuild Anthropic client on stream cleanup instead of OpenAI client","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a0b9663c7","number":0},{"kind":"pr","value":"#53926","number":53926},{"kind":"pr","value":"#28161","number":28161},{"kind":"pr","value":"#28240","number":28240}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":36714,"title":"fix(agent): stale-stream watchdog rebuilds wrong client for Bedrock/Anthropic","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a0b9663c7","number":0},{"kind":"pr","value":"#53926","number":53926},{"kind":"pr","value":"#28161","number":28161},{"kind":"pr","value":"#28240","number":28240}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":36719,"title":"fix(run_agent): skip OpenAI client rebuild for anthropic_messages api_mode","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a0b9663c7","number":0},{"kind":"pr","value":"#53926","number":53926},{"kind":"pr","value":"#28161","number":28161},{"kind":"pr","value":"#28240","number":28240}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":36832,"title":"fix: skip OpenAI rebuild for anthropic clients","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#36719","number":36719},{"kind":"pr","value":"#36714","number":36714},{"kind":"pr","value":"#36693","number":36693},{"kind":"pr","value":"#28240","number":28240},{"kind":"pr","value":"#33855","number":33855}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":44076,"title":"fix(stream): rebuild Anthropic client on stream retry, not the OpenAI client","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a0b9663c7","number":0},{"kind":"pr","value":"#53926","number":53926},{"kind":"pr","value":"#28161","number":28161},{"kind":"pr","value":"#28240","number":28240}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":44148,"title":"fix(agent): skip OpenAI client rebuild for Anthropic-mode providers (#44006)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#36719","number":36719},{"kind":"pr","value":"#44006","number":44006}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":51851,"title":"fix(agent): dispatch stale-stream pool cleanup to correct client for Anthropic/Bedrock","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a0b9663c7","number":0},{"kind":"pr","value":"#53926","number":53926},{"kind":"pr","value":"#28161","number":28161},{"kind":"pr","value":"#28240","number":28240}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":51856,"title":"fix(agent): dispatch stale-stream pool cleanup by api_mode (#51844)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a0b9663c7","number":0},{"kind":"pr","value":"#53926","number":53926},{"kind":"pr","value":"#28161","number":28161},{"kind":"pr","value":"#28240","number":28240}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":53926,"title":"fix(streaming): route Anthropic stream cleanup to the Anthropic client, not OpenAI rebuild (#28161)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/chat_completion_helpers.py","tests/run_agent/test_28161_anthropic_stream_pool_cleanup.py","tests/run_agent/test_streaming.py"],"fix_hunks":[{"file":"agent/chat_completion_helpers.py","hunks":["@@ -2459,12 +2459,19 @@ def _call():\n                             diag=request_client_holder.get(\"diag\"),\n                         )\n                         _close_request_client_once(\"stream_mid_tool_retry_cleanup\")\n-                        try:\n-                            agent._replace_primary_openai_client(\n-                                reason=\"stream_mid_tool_retry_pool_cleanup\"\n-                            )\n-                        except Exception:\n-                            pass\n+                        if agent.api_mode == \"anthropic_messages\":\n+                            try:\n+                                agent._anthropic_client.close()\n+                                agent._rebuild_anthropic_client()\n+                            except Exception:\n+                                pass\n+                        else:\n+                            try:\n+                                agent._replace_primary_openai_client(\n+                                    reason=\"stream_mid_tool_retry_pool_cleanup\"\n+                                )\n+                            except Exception:\n+                                pass\n                         continue\n \n                     # SSE error events from proxies (e.g. OpenRouter sends","@@ -2512,12 +2519,19 @@ def _call():\n                             _close_request_client_once(\"stream_retry_cleanup\")\n                             # Also rebuild the primary client to purge\n                             # any dead connections from the pool.\n-                            try:\n-                                agent._replace_primary_openai_client(\n-                                    reason=\"stream_retry_pool_cleanup\"\n-                                )\n-                            except Exception:\n-                                pass\n+                            if agent.api_mode == \"anthropic_messages\":\n+                                try:\n+                                    agent._anthropic_client.close()\n+                                    agent._rebuild_anthropic_client()\n+                                except Exception:\n+                                    pass\n+                            else:\n+                                try:\n+                                    agent._replace_primary_openai_client(\n+                                        reason=\"stream_retry_pool_cleanup\"\n+                                    )\n+                                except Exception:\n+                                    pass\n                             continue\n                         # Retries exhausted. Log the final failure with\n                         # full diagnostic detail (chain, headers,","@@ -2688,10 +2702,17 @@ def _call():\n                 pass\n             # Rebuild the primary client too — its connection pool\n             # may hold dead sockets from the same provider outage.\n-            try:\n-                agent._replace_primary_openai_client(reason=\"stale_stream_pool_cleanup\")\n-            except Exception:\n-                pass\n+            if agent.api_mode == \"anthropic_messages\":\n+                try:\n+                    agent._anthropic_client.close()\n+                    agent._rebuild_anthropic_client()\n+                except Exception:\n+                    pass\n+            else:\n+                try:\n+                    agent._replace_primary_openai_client(reason=\"stale_stream_pool_cleanup\")\n+                except Exception:\n+                    pass\n             # Reset the timer so we don't kill repeatedly while\n             # the inner thread processes the closure.\n             last_chunk_time[\"t\"] = time.time()"]},{"file":"tests/run_agent/test_28161_anthropic_stream_pool_cleanup.py","hunks":["@@ -0,0 +1,150 @@\n+\"\"\"Anthropic stream cleanup must call _anthropic_client.close() + _rebuild_anthropic_client(),\n+not _replace_primary_openai_client(), to avoid 15-minute hangs on Anthropic-native configs.\n+\n+Three cleanup sites in chat_completion_helpers.interruptible_streaming_api_call() were\n+calling _replace_primary_openai_client() unconditionally.  For api_mode=anthropic_messages\n+this silently fails (no OPENAI_API_KEY) and leaves the in-flight httpx stream unclosed,\n+blocking the worker thread until the 900s httpx read-timeout fires.\n+\n+Tests cover:\n+- stream_retry_pool_cleanup  (connection error on fresh stream, L1836)\n+- stale_stream_pool_cleanup  (outer poll loop detects stale stream, L1987)\n+\n+Fixes #28161\n+\"\"\"\n+import threading\n+from types import SimpleNamespace\n+from unittest.mock import MagicMock, patch\n+\n+import httpx\n+import pytest\n+\n+\n+# ---------------------------------------------------------------------------\n+# Helpers\n+# ---------------------------------------------------------------------------\n+\n+\n+def _make_anthropic_agent(**kwargs):\n+    from run_agent import AIAgent\n+\n+    defaults = dict(\n+        api_key=\"[redacted:secret-assignment]\",\n+        base_url=\"https://example.com/v1\",\n+        model=\"claude-opus-4-7\",\n+        quiet_mode=True,\n+        skip_context_files=True,\n+        skip_memory=True,\n+    )\n+    defaults.update(kwargs)\n+    agent = AIAgent(**defaults)\n+    agent.api_mode = \"anthropic_messages\"\n+    agent._anthropic_client = MagicMock()\n+    agent._anthropic_api_key = \"[redacted:secret-assignment]\"\n+    return agent\n+\n+\n+def _good_stream_cm():\n+    \"\"\"Context manager whose stream yields no events and returns a valid message.\"\"\"\n+    cm = MagicMock()\n+    stream = MagicMock()\n+    stream.__iter__ = MagicMock(return_value=iter([]))\n+    msg = MagicMock()\n+    msg.content = []\n+    msg.stop_reason = \"end_turn\"\n+    msg.usage = SimpleNamespace(input_tokens=10, output_tokens=5)\n+    stream.get_final_message = MagicMock(return_value=msg)\n+    cm.__enter__ = MagicMock(return_value=stream)\n+    cm.__exit__ = MagicMock(return_value=False)\n+    return cm\n+\n+\n+def _failing_stream_cm():\n+    \"\"\"Context manager whose __enter__ raises ConnectError immediately.\"\"\"\n+    cm = MagicMock()\n+    cm.__enter__ = MagicMock(\n+        side_effect=httpx.ConnectError(\"connection reset by peer\")\n+    )\n+    return cm\n+\n+\n+# ---------------------------------------------------------------------------\n+# Tests\n+# ---------------------------------------------------------------------------\n+\n+\n+class TestAnthropicStreamPoolCleanup:\n+    \"\"\"_replace_primary_openai_client must not be called for api_mode=anthropic_messages.\"\"\"\n+\n+    @pytest.mark.filterwarnings(\n+        \"ignore::pytest.PytestUnhandledThreadExceptionWarning\"\n+    )\n+    def test_stream_retry_calls_anthropic_rebuild_not_openai(self):\n+        \"\"\"Connection error during stream retry → close+rebuild Anthropic client, not OpenAI.\"\"\"\n+        agent = _make_anthropic_agent()\n+\n+        attempt_count = [0]\n+\n+        def _stream_side_effect(*args, **kwargs):\n+            attempt_count[0] += 1\n+            if attempt_count[0] == 1:\n+                return _failing_stream_cm()\n+            return _good_stream_cm()\n+\n+        agent._anthropic_client.messages.stream.side_effect = _stream_side_effect\n+\n+        with patch.object(agent, \"_rebuild_anthropic_client\") as mock_rebuild:\n+            with patch.object(\n+                agent, \"_replace_primary_openai_client\"\n+            ) as mock_replace:\n+                agent._interruptible_streaming_api_call({})\n+\n+        mock_replace.assert_not_called()\n+        mock_rebuild.assert_called_once()\n+        agent._anthropic_client.close.assert_called_once()\n+\n+    @pytest.mark.filterwarnings(\n+        \"ignore::pytest.PytestUnhandledThreadExceptionWarning\"\n+    )\n+    def test_stale_stream_calls_anthropic_rebuild_not_openai(self, monkeypatch):\n+        \"\"\"Stale-stream outer-poll detector → close+rebuild Anthropic client, not OpenAI.\"\"\"\n+        monkeypatch.setenv(\"HERMES_STREAM_STALE_TIMEOUT\", \"0.1\")\n+\n+        agent = _make_anthropic_agent()\n+        unblock = threading.Event()\n+        attempt_count = [0]\n+\n+        def _stream_side_effect(*args, **kwargs):\n+            attempt_count[0] += 1\n+            if attempt_count[0] == 1:\n+                # First attempt: stream that yields nothing (triggers stale detector),\n+                # then raises ConnectError once _anthropic_client.close() unblocks it.\n+                cm = MagicMock()\n+                stream = MagicMock()\n+\n+                def _blocking_gen():\n+                    unblock.wait(timeout=5.0)\n+                    raise httpx.ConnectError(\"connection dropped after close()\")\n+                    yield  # make this a generator so next() triggers the wait\n+\n+                stream.__iter__ = MagicMock(return_value=_blocking_gen())\n+                cm.__enter__ = MagicMock(return_value=stream)\n+                cm.__exit__ = MagicMock(return_value=False)\n+                return cm\n+            # Second attempt: succeed\n+            return _good_stream_cm()\n+\n+        agent._anthropic_client.messages.stream.side_effect = _stream_side_effect\n+        # close() on the mock Anthropic client unblocks the inner thread.\n+        agent._anthropic_client.close.side_effect = unblock.set\n+\n+        with patch.object(agent, \"_rebuild_anthropic_client\") as mock_rebuild:\n+            with patch.object(\n+                agent, \"_replace_primary_openai_client\"\n+            ) as mock_replace:\n+                agent._interruptible_streaming_api_call({})\n+\n+        mock_replace.assert_not_called()\n+        # close() and rebuild called at least once by the stale detector.\n+        agent._anthropic_client.close.assert_called()\n+        assert mock_rebuild.call_count >= 1"]},{"file":"tests/run_agent/test_streaming.py","hunks":["@@ -986,11 +986,17 @@ def test_anthropic_stream_refreshes_activity_on_every_event(self):\n \n         assert touch_calls.count(\"receiving stream response\") == len(events)\n \n+    @patch(\"run_agent.AIAgent._rebuild_anthropic_client\")\n     @patch(\"run_agent.AIAgent._replace_primary_openai_client\")\n     def test_anthropic_stream_parser_valueerror_retries_before_delivery(\n-        self, mock_replace, monkeypatch,\n+        self, mock_replace, mock_rebuild, monkeypatch,\n     ):\n-        \"\"\"Malformed Anthropic event-stream frames retry instead of surfacing HTTP None.\"\"\"\n+        \"\"\"Malformed Anthropic event-stream frames retry instead of surfacing HTTP None.\n+\n+        On the Anthropic-native path the stream-retry cleanup must close + rebuild the\n+        Anthropic client, NOT the OpenAI primary client (which would fail with\n+        Missing-credentials and leave the wedged stream open). See #28161.\n+        \"\"\"\n         from run_agent import AIAgent\n \n         agent = AIAgent(","@@ -1035,7 +1041,11 @@ def __iter__(self):\n \n         assert response is final_message\n         assert agent._anthropic_client.messages.stream.call_count == 2\n-        assert mock_replace.call_count == 1\n+        # Anthropic-native cleanup: close + rebuild the Anthropic client, never\n+        # the OpenAI primary client.\n+        assert mock_replace.call_count == 0\n+        assert mock_rebuild.call_count == 1\n+        assert agent._anthropic_client.close.call_count == 1\n \n     @patch(\"run_agent.AIAgent._replace_primary_openai_client\")\n     def test_generic_anthropic_valueerror_still_propagates_without_stream_retry(\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}