{"repo":"NousResearch/hermes-agent","public_id":"16788","display_label":"MCP circuit breaker wedging and reconnection fixes","summary":"These issues and pull requests address scenarios where the MCP circuit breaker can permanently wedge the gateway when a transport dies, covering fixes for reconnection resilience, tool deregistration/re-registration, and recovery behavior across stdio and HTTP transports.","subsystem":"mcp","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":53599},"resolution":{"state":"resolved","by":"pr-53599"},"freshness":{"state":"verified","last_verified_at":"2026-06-27T11:39:55+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":16788,"pr_number":53599,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":16788,"title":"MCP circuit breaker open state can permanently wedge gateway when stdio subprocess dies","url":null,"state":"closed"},{"kind":"pr","number":17016,"title":"fix: MCP circuit breaker recovery and HTTP keepalive (#16788, #17003)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":21006,"title":"fix(mcp): reconnect dead stdio before breaker probe","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"88c02469cc41b28a8c498d63ee40482288e7f057","number":0},{"kind":"pr","value":"#53599","number":53599},{"kind":"pr","value":"#16788","number":16788}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":49255,"title":"fix(mcp): deregister tools when reconnection retries are exhausted","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"88c02469cc41b28a8c498d63ee40482288e7f057","number":0},{"kind":"pr","value":"#53599","number":53599},{"kind":"pr","value":"#16788","number":16788}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":53599,"title":"fix(mcp): never permanently wedge the circuit breaker on a dead transport","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_mcp_circuit_breaker.py","tools/mcp_tool.py"],"fix_hunks":[{"file":"tests/tools/test_mcp_circuit_breaker.py","hunks":["@@ -179,6 +179,107 @@ def _fake_monotonic():\n         _cleanup(mcp_tool, \"srv\")\n \n \n+def test_half_open_probe_on_dead_session_requests_reconnect(monkeypatch, tmp_path):\n+    \"\"\"A half-open probe against a server with no live session must request\n+    a transport reconnect and return a clean error — NOT write into a dead\n+    pipe or permanently re-arm the breaker.\n+\n+    This is the #16788 wedge: a dead stdio subprocess leaves ``session=None``\n+    (the run loop parked after exhausting retries). The old handler bumped\n+    the breaker every cooldown forever; the fix signals ``_reconnect_event``\n+    so the parked task revives and rebuilds the transport.\n+    \"\"\"\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+\n+    from tools import mcp_tool\n+    from tools.mcp_tool import _make_tool_handler\n+\n+    server = _install_stub_server(mcp_tool, \"srv\", None)\n+    # Simulate a dead/parked transport: no live session.\n+    server.session = None\n+    # Drive _signal_reconnect down its direct .set() path (no live loop).\n+    monkeypatch.setattr(mcp_tool, \"_mcp_loop\", None)\n+\n+    try:\n+        mcp_tool._server_error_counts[\"srv\"] = mcp_tool._CIRCUIT_BREAKER_THRESHOLD\n+        fake_now = [1000.0]\n+\n+        def _fake_monotonic():\n+            return fake_now[0]\n+\n+        monkeypatch.setattr(mcp_tool.time, \"monotonic\", _fake_monotonic)\n+        mcp_tool._server_breaker_opened_at[\"srv\"] = fake_now[0]\n+        cooldown = getattr(mcp_tool, \"_CIRCUIT_BREAKER_COOLDOWN_SEC\", 60.0)\n+\n+        # Advance past cooldown → next call is a half-open probe.\n+        fake_now[0] += cooldown + 1.0\n+\n+        handler = _make_tool_handler(\"srv\", \"tool1\", 10.0)\n+        result = handler({})\n+        parsed = json.loads(result)\n+\n+        # Clean \"reconnecting\" error, and a reconnect was actually signalled.\n+        assert \"reconnect\" in parsed.get(\"error\", \"\").lower(), parsed\n+        server._reconnect_event.set.assert_called_once()\n+    finally:\n+        _cleanup(mcp_tool, \"srv\")\n+\n+\n+def test_half_open_dead_session_recovers_after_reconnect(monkeypatch, tmp_path):\n+    \"\"\"Once the transport comes back (session repopulated + breaker reset by\n+    the run loop), the next call must go straight through — proving the wedge\n+    is escapable, not just deferred.\n+    \"\"\"\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+\n+    from tools import mcp_tool\n+    from tools.mcp_tool import _make_tool_handler\n+\n+    async def _call_tool_success(*a, **kw):\n+        result = MagicMock()\n+        result.isError = False\n+        block = MagicMock()\n+        block.text = \"ok\"\n+        result.content = [block]\n+        result.structuredContent = None\n+        return result\n+\n+    server = _install_stub_server(mcp_tool, \"srv\", _call_tool_success)\n+    server.session = None  # transport down at first\n+    monkeypatch.setattr(mcp_tool, \"_mcp_loop\", None)\n+    mcp_tool._ensure_mcp_loop()\n+\n+    try:\n+        mcp_tool._server_error_counts[\"srv\"] = mcp_tool._CIRCUIT_BREAKER_THRESHOLD\n+        fake_now = [1000.0]\n+        monkeypatch.setattr(mcp_tool.time, \"monotonic\", lambda: fake_now[0])\n+        mcp_tool._server_breaker_opened_at[\"srv\"] = fake_now[0]\n+        cooldown = getattr(mcp_tool, \"_CIRCUIT_BREAKER_COOLDOWN_SEC\", 60.0)\n+        fake_now[0] += cooldown + 1.0\n+\n+        handler = _make_tool_handler(\"srv\", \"tool1\", 10.0)\n+\n+        # Probe 1: transport down → reconnect requested, clean error.\n+        parsed = json.loads(handler({}))\n+        assert \"reconnect\" in parsed.get(\"error\", \"\").lower(), parsed\n+\n+        # Simulate the run loop rebuilding the session + resetting the breaker\n+        # (what _run_stdio does on successful re-init).\n+        live = MagicMock()\n+        live.call_tool = _call_tool_success\n+        server.session = live\n+        mcp_tool._reset_server_error(\"srv\")\n+\n+        # Advance past the re-armed cooldown so the next call is a fresh probe.\n+        fake_now[0] += cooldown + 1.0\n+\n+        # Next call goes straight through.\n+        parsed = json.loads(handler({}))\n+        assert parsed.get(\"result\") == \"ok\", parsed\n+    finally:\n+        _cleanup(mcp_tool, \"srv\")\n+\n+\n def test_circuit_breaker_cleared_on_reconnect(monkeypatch, tmp_path):\n     \"\"\"When the auth-recovery path successfully reconnects the server,\n     the breaker should be cleared so subsequent calls aren't gated on a","@@ -250,3 +351,98 @@ def _retry_call():\n         )\n     finally:\n         _cleanup(mcp_tool, \"srv\")\n+\n+\n+def test_run_loop_parks_instead_of_exiting_then_revives(monkeypatch, tmp_path):\n+    \"\"\"The run loop must NOT exit when the reconnect budget is exhausted.\n+\n+    It deregisters tools and parks as a dormant listener; a later\n+    ``_reconnect_event`` revives it and re-enters the transport. This is the\n+    structural fix for #16788 — without a live task, no half-open probe could\n+    ever bring a dead stdio server back.\n+    \"\"\"\n+    import asyncio\n+\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+\n+    from tools import mcp_tool\n+    from tools.mcp_tool import MCPServerTask\n+\n+    # Shrink the budget and collapse backoff sleeps (but still yield control\n+    # to the loop) so the test runs fast without starving the scheduler.\n+    monkeypatch.setattr(mcp_tool, \"_MAX_RECONNECT_RETRIES\", 2)\n+\n+    _real_sleep = asyncio.sleep\n+\n+    async def _fast_sleep(_delay, *a, **kw):\n+        await _real_sleep(0)\n+\n+    monkeypatch.setattr(mcp_tool.asyncio, \"sleep\", _fast_sleep)\n+\n+    state = {\"transport_calls\": 0, \"deregistered\": 0, \"revived\": False}\n+\n+    async def _scenario():\n+        class _Task(MCPServerTask):\n+            def _is_http(self):\n+                return False\n+\n+            def _deregister_tools(self):\n+                state[\"deregistered\"] += 1\n+                self._registered_tool_names = []\n+\n+            async def _run_stdio(self, config):\n+                state[\"transport_calls\"] += 1\n+                # First connect succeeds (sets _ready) then immediately\n+                # fails, as if the subprocess died — the post-ready failure\n+                # path that counts toward the reconnect budget.\n+                if state[\"transport_calls\"] == 1:\n+                    self.session = object()\n+                    self._ready.set()\n+                    self.session = None\n+                    raise RuntimeError(\"subprocess died\")\n+                # Keep failing until the budget is exhausted and the loop\n+                # parks, UNLESS we've been revived after parking.\n+                if state[\"revived\"]:\n+                    self.session = object()\n+                    self._ready.set()\n+                    await self._wait_for_lifecycle_event()\n+                    return\n+                raise RuntimeError(\"still down\")\n+\n+        task = _Task(\"srv\")\n+        task._registered_tool_names = [\"srv__tool\"]\n+\n+        run_task = asyncio.ensure_future(task.run({\"command\": \"x\"}))\n+\n+        # Wait until the loop has parked (it deregisters tools right before\n+        # blocking on _wait_for_reconnect_or_shutdown).\n+        for _ in range(500):\n+            await _real_sleep(0)\n+            if state[\"deregistered\"] >= 1:\n+                break\n+        # Give the loop one more tick to settle into the park wait.\n+        await _real_sleep(0)\n+        assert not run_task.done(), \"run loop exited instead of parking\"\n+        assert state[\"deregistered\"] >= 1, \"tools not deregistered on park\"\n+\n+        # Revive it: a reconnect signal must wake the parked task.\n+        state[\"revived\"] = True\n+        before = state[\"transport_calls\"]\n+        task._reconnect_event.set()\n+        for _ in range(500):\n+            await _real_sleep(0)\n+            if state[\"transport_calls\"] > before:\n+                break\n+        assert state[\"transport_calls\"] > before, (\n+            \"parked task did not re-enter transport on reconnect signal\"\n+        )\n+\n+        # Clean shutdown.\n+        task._shutdown_event.set()\n+        task._reconnect_event.set()\n+        try:\n+            await asyncio.wait_for(run_task, timeout=2)\n+        except (asyncio.TimeoutError, asyncio.CancelledError, Exception):\n+            run_task.cancel()\n+\n+    asyncio.run(_scenario())"]},{"file":"tools/mcp_tool.py","hunks":["@@ -1759,6 +1759,41 @@ async def _wait_for_lifecycle_event(self) -> str:\n         self._reconnect_event.clear()\n         return \"reconnect\"\n \n+    async def _wait_for_reconnect_or_shutdown(self) -> str:\n+        \"\"\"Block until a reconnect or shutdown is requested while parked.\n+\n+        Used by :meth:`run` after the reconnect budget is exhausted. The\n+        task stays alive (so ``_reconnect_event`` always has a listener) but\n+        does no work until something explicitly asks it to come back —\n+        the circuit-breaker half-open probe, OAuth recovery, or a manual\n+        ``/mcp`` refresh.\n+\n+        Returns:\n+            ``\"shutdown\"`` if the server should exit the run loop entirely,\n+            ``\"reconnect\"`` if it should rebuild the transport. The reconnect\n+            event is cleared before returning so the next park cycle starts\n+            from a fresh signal. Shutdown takes precedence.\n+        \"\"\"\n+        shutdown_task = asyncio.ensure_future(self._shutdown_event.wait())\n+        reconnect_task = asyncio.ensure_future(self._reconnect_event.wait())\n+        try:\n+            await asyncio.wait(\n+                {shutdown_task, reconnect_task},\n+                return_when=asyncio.FIRST_COMPLETED,\n+            )\n+        finally:\n+            for t in (shutdown_task, reconnect_task):\n+                if not t.done():\n+                    t.cancel()\n+                    try:\n+                        await t\n+                    except (asyncio.CancelledError, Exception):\n+                        pass\n+        if self._shutdown_event.is_set():\n+            return \"shutdown\"\n+        self._reconnect_event.clear()\n+        return \"reconnect\"\n+\n     async def _run_stdio(self, config: dict):\n         \"\"\"Run the server using stdio transport.\"\"\"\n         if not _MCP_AVAILABLE:","@@ -1857,6 +1892,10 @@ async def _run_stdio(self, config: dict):\n                     self.session = session\n                     await self._discover_tools()\n                     self._ready.set()\n+                    # Session is live again: clear any breaker state from a\n+                    # prior outage so the first call after recovery isn't\n+                    # gated on a stale consecutive-failure count (#16788).\n+                    _reset_server_error(self.name)\n                     # stdio transport does not use OAuth, but we still honor\n                     # _reconnect_event (e.g. future manual /mcp refresh) for\n                     # consistency with _run_http.","@@ -2088,6 +2127,10 @@ def _mcp_http_client_factory(\n                     self.session = session\n                     await self._discover_tools()\n                     self._ready.set()\n+                    # Session is live again: clear any breaker state from a\n+                    # prior outage so the first call after recovery isn't\n+                    # gated on a stale consecutive-failure count (#16788).\n+                    _reset_server_error(self.name)\n                     reason = await self._wait_for_lifecycle_event()\n                     if reason == \"reconnect\":\n                         logger.info(","@@ -2137,6 +2180,10 @@ async def _strip_auth_on_cross_origin_redirect(response):\n                         self.session = session\n                         await self._discover_tools()\n                         self._ready.set()\n+                        # Session is live again: clear any breaker state from\n+                        # a prior outage so the first call after recovery\n+                        # isn't gated on a stale failure count (#16788).\n+                        _reset_server_error(self.name)\n                         reason = await self._wait_for_lifecycle_event()\n                         if reason == \"reconnect\":\n                             logger.info(","@@ -2160,6 +2207,10 @@ async def _strip_auth_on_cross_origin_redirect(response):\n                     self.session = session\n                     await self._discover_tools()\n                     self._ready.set()\n+                    # Session is live again: clear any breaker state from a\n+                    # prior outage so the first call after recovery isn't\n+                    # gated on a stale consecutive-failure count (#16788).\n+                    _reset_server_error(self.name)\n                     reason = await self._wait_for_lifecycle_event()\n                     if reason == \"reconnect\":\n                         logger.info(","@@ -2298,6 +2349,15 @@ async def run(self, config: dict):\n                     \"manual refresh)\",\n                     self.name,\n                 )\n+                # A clean transport return only happens after a session was\n+                # successfully established and then asked to rebuild (auth\n+                # recovery / manual refresh / breaker-driven reconnect). That\n+                # is proof the server is reachable, so clear the consecutive-\n+                # failure budget — otherwise transient drops accumulated over\n+                # a long-lived session would eventually exhaust it and\n+                # permanently kill an otherwise-healthy server.\n+                retries = 0\n+                backoff = 1.0\n                 # Reset the session reference; _run_http/_run_stdio will\n                 # repopulate it on successful re-entry.\n                 self.session = None","@@ -2374,10 +2434,32 @@ async def run(self, config: dict):\n                 if retries > _MAX_RECONNECT_RETRIES:\n                     logger.warning(\n                         \"MCP server '%s' failed after %d reconnection attempts, \"\n-                        \"giving up: %s\",\n+                        \"parking until a reconnect is requested: %s\",\n                         self.name, _MAX_RECONNECT_RETRIES, exc,\n                     )\n-                    return\n+                    # Do NOT return — exiting the task orphans the server:\n+                    # nothing would ever listen for _reconnect_event again,\n+                    # so a half-open circuit-breaker probe could never revive\n+                    # it and the server would be permanently wedged for the\n+                    # life of the process (#16788). Instead, drop the phantom\n+                    # tools from the registry and park as a dormant listener.\n+                    # A future _reconnect_event.set() — from the breaker's\n+                    # half-open probe, OAuth recovery, or a manual /mcp\n+                    # refresh — wakes us to rebuild the transport (respawning\n+                    # a dead stdio subprocess in the process).\n+                    self._deregister_tools()\n+                    self._reconnect_event.clear()\n+                    parked = await self._wait_for_reconnect_or_shutdown()\n+                    if parked == \"shutdown\":\n+                        return\n+                    logger.info(\n+                        \"MCP server '%s': reconnect requested while parked; \"\n+                        \"rebuilding transport.\",\n+                        self.name,\n+                    )\n+                    retries = 0\n+                    backoff = 1.0\n+                    continue\n \n                 logger.warning(\n                     \"MCP server '%s' connection lost (attempt %d/%d), \"","@@ -2403,8 +2485,6 @@ async def start(self, config: dict):\n \n     async def shutdown(self):\n         \"\"\"Signal the Task to exit and wait for clean resource teardown.\"\"\"\n-        from tools.registry import registry\n-\n         self._shutdown_event.set()\n         # Defensive: if _wait_for_lifecycle_event is blocking, we need ANY\n         # event to unblock it. _shutdown_event alone is sufficient (the","@@ -2430,11 +2510,24 @@ async def shutdown(self):\n                 task.cancel()\n             await asyncio.gather(*self._pending_refresh_tasks, return_exceptions=True)\n             self._pending_refresh_tasks.clear()\n+        self._deregister_tools()\n+        self.session = None\n+\n+    def _deregister_tools(self) -> None:\n+        \"\"\"Drop this server's tools from the global registry (idempotent).\n+\n+        Pulls the server's tool schemas out of the registry so the agent\n+        stops advertising them to the model. Called on shutdown AND when the\n+        reconnect budget is exhausted, so a dead server never leaves phantom\n+        tool definitions bloating the prompt cache and producing \"not\n+        connected\" errors on every turn.\n+        \"\"\"\n+        from tools.registry import registry\n+\n         for tool_name in list(getattr(self, \"_registered_tool_names\", [])):\n             registry.deregister(tool_name)\n             _forget_mcp_tool_server(tool_name)\n         self._registered_tool_names = []\n-        self.session = None\n \n \n # ---------------------------------------------------------------------------","@@ -2491,6 +2584,31 @@ def _reset_server_error(server_name: str) -> None:\n     _server_error_counts[server_name] = 0\n     _server_breaker_opened_at.pop(server_name, None)\n \n+\n+def _signal_reconnect(server: Any) -> bool:\n+    \"\"\"Ask a server task to rebuild its transport, thread-safely.\n+\n+    The tool handlers run on caller threads, while the server task and its\n+    ``_reconnect_event`` live on the background MCP loop. Setting an\n+    asyncio.Event from another thread must go through\n+    ``loop.call_soon_threadsafe``; only fall back to a direct ``.set()``\n+    when the loop isn't running (e.g. unit tests that drive the handler\n+    synchronously).\n+\n+    Returns True if a reconnect signal was delivered, False if the server\n+    has no reconnect machinery (nothing to revive).\n+    \"\"\"\n+    event = getattr(server, \"_reconnect_event\", None)\n+    if event is None:\n+        return False\n+    loop = _mcp_loop\n+    if loop is not None and loop.is_running():\n+        loop.call_soon_threadsafe(event.set)\n+    else:\n+        event.set()\n+    return True\n+\n+\n # ---------------------------------------------------------------------------\n # Auth-failure detection helpers (Task 6 of MCP OAuth consolidation)\n # ---------------------------------------------------------------------------","@@ -3176,12 +3294,35 @@ def _handler(args: dict, **kwargs) -> str:\n \n         with _lock:\n             server = _servers.get(server_name)\n-        if not server or not server.session:\n+        if not server:\n             _bump_server_error(server_name)\n             return json.dumps({\n                 \"error\": f\"MCP server '{server_name}' is not connected\"\n             }, ensure_ascii=False)\n \n+        if not server.session:\n+            # No live session — the server task is reconnecting, or it has\n+            # exhausted its retry budget and parked (e.g. a dead stdio\n+            # subprocess). Probing here would write into a dead/absent\n+            # transport and re-arm the breaker forever (#16788). Instead,\n+            # ask the (always-present) server task to rebuild the transport\n+            # — which respawns a dead stdio subprocess — and return a clean\n+            # \"reconnecting\" error so the model backs off without burning\n+            # iterations. The breaker resets once the fresh session\n+            # initializes (_run_stdio/_run_http call _reset_server_error).\n+            _bump_server_error(server_name)\n+            if _signal_reconnect(server):\n+                return json.dumps({\n+                    \"error\": (\n+                        f\"MCP server '{server_name}' transport is down; \"\n+                        f\"reconnect requested. Do NOT retry this tool \"\n+                        f\"immediately — give it a few seconds to come back.\"\n+                    )\n+                }, ensure_ascii=False)\n+            return json.dumps({\n+                \"error\": f\"MCP server '{server_name}' is not connected\"\n+            }, ensure_ascii=False)\n+\n         async def _call():\n             async with server._rpc_lock:\n                 # Snapshot the agent's context so an elicitation callback\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}