MCP stdio subprocess memory leak and recycling fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #60509 fix(mcp): close all four stdio subprocess leak paths (reconnect zombies, hung handshake, parent death, idle residency)

  merged to main

  files: scripts/release.py, tests/tools/test_mcp_circuit_breaker.py, tests/tools/test_mcp_stability.py, tests/tools/test_mcp_stdio_init_timeout.py, tests/tools/test_mcp_tool.py, tests/tools/test_mcp_tool_issue_948.py, tools/mcp_stdio_watchdog.py, tools/mcp_tool.py, website/docs/user-guide/features/mcp.md

  Show the diff

  • scripts/release.py

    @@ -45,6 +45,8 @@
     
     # Auto-extracted from noreply emails + manual overrides
     AUTHOR_MAP = {
    +    "[redacted:email]": "rainbowgore",  # PR #59405 salvage (mcp: bound stdio initialize handshake to stop subprocess/FD leak; #59349)
    +    "[redacted:email]": "thestudionorth",  # PR #60015 salvage (mcp: parent-death watchdog for stdio children; commit under unlinked local identity)
         "[redacted:email]": "vampyren",  # PR #59830 salvage (kanban: grab-to-pan board scrolling; original commit under unlinked local identity)
         "[redacted:email]": "spiky02plateau",  # PR #32824 salvage (usage: fetch Codex account limits from the credential pool in pool-only setups; superseded by #60028)
         "[redacted:email]": "hwrdprkns",  # PR #36896 salvage (secrets: [redacted:secret-assignment] op:// secret source + shared _cache substrate, adapted onto the SecretSource interface)

  • tests/tools/test_mcp_circuit_breaker.py

    @@ -74,6 +74,13 @@ def assert_called_once(self):
     
         server._reconnect_event = _ReconnectAdapter()
         server._ready = _ReadyAdapter()
    +    # A bare MagicMock returns a truthy Mock for every method, so
    +    # ``_is_recycled_stdio()`` would spuriously report this stub as a recycled
    +    # stdio server and divert dead-session tool calls into the lazy-reconnect
    +    # wait (which polls the test-frozen ``time.monotonic`` forever). Real
    +    # non-recycled servers return False here; make the stub faithful so the
    +    # dead-session path falls through to the graceful reconnect handler.
    +    server._is_recycled_stdio.return_value = False
     
         mcp_tool_module._servers[name] = server
         mcp_tool_module._server_error_counts.pop(name, None)

  • tests/tools/test_mcp_stability.py

    @@ -109,6 +109,7 @@ def test_kill_orphaned_noop_when_empty(self):
             """_kill_orphaned_mcp_children does nothing when no PIDs tracked."""
             from tools.mcp_tool import (
                 _kill_orphaned_mcp_children,
    +            _orphan_stdio_pid_servers,
                 _orphan_stdio_pids,
                 _stdio_pids,
                 _lock,
    @@ -117,6 +118,7 @@ def test_kill_orphaned_noop_when_empty(self):
             with _lock:
                 _stdio_pids.clear()
                 _orphan_stdio_pids.clear()
    +            _orphan_stdio_pid_servers.clear()
     
             # Should not raise
             _kill_orphaned_mcp_children()
    @@ -125,6 +127,7 @@ def test_kill_orphaned_handles_dead_pids(self):
             """_kill_orphaned_mcp_children gracefully handles already-dead PIDs."""
             from tools.mcp_tool import (
                 _kill_orphaned_mcp_children,
    +            _orphan_stdio_pid_servers,
                 _orphan_stdio_pids,
                 _lock,
             )
    @@ -133,6 +136,7 @@ def test_kill_orphaned_handles_dead_pids(self):
             fake_pid = 999999999
             with _lock:
                 _orphan_stdio_pids.add(fake_pid)
    +            _orphan_stdio_pid_servers[fake_pid] = "orphan"
     
             # Should not raise (ProcessLookupError is caught)
             _kill_orphaned_mcp_children()
    @@ -144,6 +148,7 @@ def test_kill_orphaned_uses_sigkill_when_available(self, monkeypatch):
             """SIGTERM-first then SIGKILL after 2s for orphan cleanup."""
             from tools.mcp_tool import (
                 _kill_orphaned_mcp_children,
    +            _orphan_stdio_pid_servers,
                 _orphan_stdio_pids,
                 _lock,
             )
    @@ -151,7 +156,9 @@ def test_kill_orphaned_uses_sigkill_when_available(self, monkeypatch):
             fake_pid = 424242
             with _lock:
                 _orphan_stdio_pids.clear()
    +            _orphan_stdio_pid_servers.clear()
                 _orphan_stdio_pids.add(fake_pid)
    +            _orphan_stdio_pid_servers[fake_pid] = "orphan"
     
             fake_sigkill = 9
             monkeypatch.setattr(signal, "SIGKILL", fake_sigkill, raising=False)
    @@ -177,6 +184,7 @@ def test_kill_orphaned_falls_back_without_sigkill(self, monkeypatch):
             """Without SIGKILL, SIGTERM is used for both phases."""
             from tools.mcp_tool import (
                 _kill_orphaned_mcp_children,
    +            _orphan_stdio_pid_servers,
                 _orphan_stdio_pids,
                 _lock,
             )
    @@ -184,7 +192,9 @@ def test_kill_orphaned_falls_back_without_sigkill(self, monkeypatch):
             fake_pid = 434343
             with _lock:
                 _orphan_stdio_pids.clear()
    +            _orphan_stdio_pid_servers.clear()
                 _orphan_stdio_pids.add(fake_pid)
    +            _orphan_stdio_pid_servers[fake_pid] = "orphan"
     
             monkeypatch.delattr(signal, "SIGKILL", raising=False)
     
    @@ -199,6 +209,99 @@ def test_kill_orphaned_falls_back_without_sigkill(self, monkeypatch):
             with _lock:
                 assert fake_pid not in _orphan_stdio_pids
     
    +    def test_run_stdio_reaps_orphans_before_spawn(self):
    +        """_run_stdio kills orphaned PIDs from prior failed attempts (#57355)."""
    +        from tools.mcp_tool import (
    +            _kill_orphaned_mcp_children,
    +            _orphan_stdio_pids,
    +            _stdio_pids,
    +            _stdio_pgids,
    +            _lock,
    +            MCPServerTask,
    +        )
    +        from unittest.mock import patch, MagicMock, AsyncMock
    +
    +        # Seed an orphan PID that belongs to a prior failed connection.
    +        fake_pid = 999999997
    +        with _lock:
    +            _orphan_stdio_pids.add(fake_pid)
    +
    +        server = MCPServerTask.__new__(MCPServerTask)
    +        server.name = "test-zombie-reap"
    +        server._ready = MagicMock()
    +        server._shutdown_event = MagicMock()
    +        server._shutdown_event.is_set.return_value = True
    +        server._reconnect_event = MagicMock()
    +        server._sampling = None
    +        server._elicitation = None
    +        server._registered_tool_names = []
    +
    +        config = {"command": "echo", "args": ["hello"]}
    +
    +        import asyncio
    +
    +        async def _run():
    +            # _run_stdio should reap orphans before it gets to the
    +            # stdio_client spawn.  Patch the OSV check (local import)
    +            # and stdio_client so no real subprocess is spawned.
    +            with patch("tools.mcp_tool._MCP_AVAILABLE", True), \
    +                 patch("tools.mcp_tool._build_safe_env", return_value={}), \
    +                 patch("tools.mcp_tool._resolve_stdio_command",
    +                       return_value=("echo", {})), \
    +                 patch("tools.mcp_tool._write_stderr_log_header"), \
    +                 patch("tools.mcp_tool._get_mcp_stderr_log",
    +                       return_value=None), \
    +                 patch("tools.mcp_tool.check_package_for_malware",
    +                       return_value=None, create=True), \
    +                 patch("tools.osv_check.check_package_for_malware",
    +                       return_value=None):
    +                # Patch stdio_client to raise so the test exits quickly
    +                cm = MagicMock()
    +                cm.__aenter__ = AsyncMock(side_effect=RuntimeError("test"))
    +                cm.__aexit__ = AsyncMock(return_value=False)
    +                with patch("tools.mcp_tool.stdio_client", return_value=cm):
    +                    try:
    +                        await server._run_stdio(config)
    +                    except Exception:
    +                        pass
    +
    +        asyncio.run(_run())
    +
    +        # The orphan must have been reaped before the spawn attempt.
    +        with _lock:
    +            assert fake_pid not in _orphan_stdio_pids
    +
    +    def test_kill_orphaned_can_filter_by_server_name(self):
    +        """Reconnect cleanup reaps only the orphan owned by that MCP server."""
    +        from tools.mcp_tool import (
    +            _kill_orphaned_mcp_children,
    +            _orphan_stdio_pid_servers,
    +            _orphan_stdio_pids,
    +            _lock,
    +        )
    +
    +        target_pid = 454545
    +        other_pid = 464646
    +        with _lock:
    +            _orphan_stdio_pids.clear()
    +            _orphan_stdio_pid_servers.clear()
    +            _orphan_stdio_pids.update({target_pid, other_pid})
    +            _orphan_stdio_pid_servers[target_pid] = "feishu"
    +            _orphan_stdio_pid_servers[other_pid] = "mimir"
    +
    +        with patch("tools.mcp_tool.os.kill") as mock_kill, \
    +             patch("gateway.status._pid_exists", return_value=False), \
    +             patch("tools.mcp_tool.time.sleep") as mock_sleep:
    +            _kill_orphaned_mcp_children(server_name="feishu")
    +
    +        mock_kill.assert_called_once_with(target_pid, signal.SIGTERM)
    +        mock_sleep.assert_called_once_with(2)
    +        with _lock:
    +            assert target_pid not in _orphan_stdio_pids
    +            assert target_pid not in _orphan_stdio_pid_servers
    +            assert other_pid in _orphan_stdio_pids
    +            assert _orphan_stdio_pid_servers[other_pid] == "mimir"
    +
     
     # ---------------------------------------------------------------------------
     # Fix 2b: stdio descendant reaping via process group (issue #23799)
    @@ -214,10 +317,17 @@ class TestStdioPgroupReaping:
         """_kill_orphaned_mcp_children reaps via killpg when a pgid is tracked."""
     
         def _reset_state(self):
    -        from tools.mcp_tool import _stdio_pids, _orphan_stdio_pids, _stdio_pgids, _lock
    +        from tools.mcp_tool import (
    +            _orphan_stdio_pid_servers,
    +            _orphan_stdio_pids,
    +            _stdio_pgids,
    +            _stdio_pids,
    +            _lock,
    +        )
             with _lock:
                 _stdio_pids.clear()
                 _orphan_stdio_pids.clear()
    +            _orphan_stdio_pid_servers.clear()
                 _stdio_pgids.clear()
     
         def test_killpg_used_when_pgid_tracked(self, monkeypatch):
    @@ -443,6 +553,7 @@ def test_grandchild_reaped_via_pgroup(self, tmp_path):
             # Drive the reaper: register the parent pid + pgid as an orphan.
             from tools.mcp_tool import (
                 _kill_orphaned_mcp_children,
    +            _orphan_stdio_pid_servers,
                 _orphan_stdio_pids,
                 _stdio_pgids,
                 _stdio_pids,
    @@ -451,8 +562,10 @@ def test_grandchild_reaped_via_pgroup(self, tmp_path):
             with _lock:
                 _stdio_pids.clear()
                 _orphan_stdio_pids.clear()
    +            _orphan_stdio_pid_servers.clear()
                 _stdio_pgids.clear()
                 _orphan_stdio_pids.add(parent.pid)
    +            _orphan_stdio_pid_servers[parent.pid] = "orphan"
                 _stdio_pgids[parent.pid] = parent_pgid
             try:
                 _kill_orphaned_mcp_children()

  • tests/tools/test_mcp_stdio_init_timeout.py

    @@ -0,0 +1,94 @@
    +"""Regression test for the stdio-MCP subprocess/FD leak (#59349).
    +
    +A stdio MCP server that never completes ``initialize`` (e.g. emits a
    +non-JSON-RPC frame and then blocks on stdin) used to hang ``_run_stdio``
    +forever on the background event loop: ``connect_timeout`` bounded only the
    +*caller's* ``.result()`` wait, not the coroutine itself. Because the connect
    +never unwound, the cleanup ``finally`` in ``_run_stdio`` never ran, so the
    +spawned child process and its stdio pipes / pidfd leaked on *every* discovery
    +retry — unbounded, until the gateway hit EMFILE.
    +
    +The fix wraps ``session.initialize()`` in
    +``asyncio.wait_for(..., timeout=connect_timeout)`` so a stalled handshake fails
    +instead of hanging, which lets the existing ``finally`` reap the child.
    +
    +This test drives the *real* ``_run_stdio`` with a fake transport whose
    +``initialize()`` hangs, and asserts the connect is bounded by
    +``connect_timeout`` rather than blocking forever. It is fully hermetic — no real
    +subprocess, no network (the drain-to-zero behaviour was additionally verified
    +manually against the reporter's live repro).
    +"""
    +
    +from __future__ import annotations
    +
    +import asyncio
    +import time
    +from unittest.mock import patch
    +
    +import pytest
    +
    +pytest.importorskip("mcp")
    +
    +
    +class _HangingSession:
    +    """Stand-in ClientSession whose handshake never completes."""
    +
    +    async def initialize(self):
    +        await asyncio.sleep(3600)
    +
    +
    +class _FakeAsyncCM:
    +    """Minimal async context manager yielding a fixed value; spawns nothing."""
    +
    +    def __init__(self, value):
    +        self._value = value
    +
    +    async def __aenter__(self):
    +        return self._value
    +
    +    async def __aexit__(self, *_exc):
    +        return False
    +
    +
    +def _fake_stdio_client(*_args, **_kwargs):
    +    # `async with stdio_client(...) as (read, write)` — no subprocess spawned.
    +    return _FakeAsyncCM((object(), object()))
    +
    +
    +def _fake_client_session(*_args, **_kwargs):
    +    # `async with ClientSession(...) as session` -> a session that hangs.
    +    return _FakeAsyncCM(_HangingSession())
    +
    +
    +class TestStdioInitializeTimeout:
    +    def test_hanging_initialize_is_bounded_not_leaked(self):
    +        """A stdio server that hangs at ``initialize`` must fail within
    +        ``connect_timeout`` — not block ``_run_stdio`` forever (#59349)."""
    +        from tools import mcp_tool
    +
    +        server = mcp_tool.MCPServerTask("leak-guard")
    +        config = {"command": "fake-mcp", "args": [], "connect_timeout": 0.2}
    +
    +        async def drive():
    +            with patch.object(mcp_tool, "stdio_client", _fake_stdio_client), \
    +                 patch.object(mcp_tool, "ClientSession", _fake_client_session), \
    +                 patch.object(mcp_tool, "_resolve_stdio_command", lambda c, e: (c, e)), \
    +                 patch.object(mcp_tool, "_write_stderr_log_header", lambda *_a, **_k: None), \
    +                 patch.object(mcp_tool, "_get_mcp_stderr_log", lambda: None), \
    +                 patch("tools.osv_check.check_package_for_malware",
    +                       lambda *_a, **_k: None):
    +                start = time.monotonic()
    +                # The outer 5s guard exists ONLY so a regression can't hang the
    +                # whole suite. With the fix, the inner connect_timeout (0.2s)
    +                # fires first; the elapsed assertion below is what actually
    +                # distinguishes "bounded" (fixed) from "hung" (regressed).
    +                with pytest.raises(asyncio.TimeoutError):
    +                    await asyncio.wait_for(server._run_stdio(config), timeout=5.0)
    +                return time.monotonic() - start
    +
    +        elapsed = asyncio.run(drive())
    +        assert elapsed < 2.0, (
    +            f"_run_stdio blocked {elapsed:.1f}s on a hanging initialize() — the "
    +            f"connect_timeout ({config['connect_timeout']}s) bound was not applied; "
    +            f"the #59349 subprocess/FD leak has regressed."
    +        )

  • tests/tools/test_mcp_tool.py

    @@ -172,6 +172,56 @@ def test_status_distinguishes_configured_connecting_failed_and_disabled(
             assert statuses["disabled"]["disabled"] is True
     
     
    +class TestLifecycleConfig:
    +    def test_get_lifecycle_seconds_accepts_top_level_and_nested_values(self):
    +        from tools.mcp_tool import _get_lifecycle_seconds
    +
    +        assert (
    +            _get_lifecycle_seconds(
    +                {"idle_timeout_seconds": "3.5"},
    +                "idle_timeout_seconds",
    +            )
    +            == 3.5
    +        )
    +        assert _get_lifecycle_seconds(
    +            {"lifecycle": {"max_lifetime_seconds": 42}},
    +            "max_lifetime_seconds",
    +        ) == 42.0
    +
    +    def test_get_lifecycle_seconds_treats_zero_as_disabled(self):
    +        from tools.mcp_tool import _get_lifecycle_seconds
    +
    +        assert (
    +            _get_lifecycle_seconds(
    +                {"idle_timeout_seconds": 0},
    +                "idle_timeout_seconds",
    +            )
    +            is None
    +        )
    +
    +    def test_get_lifecycle_seconds_ignores_invalid_values(self, caplog):
    +        from tools.mcp_tool import _get_lifecycle_seconds
    +
    +        assert (
    +            _get_lifecycle_seconds(
    +                {"idle_timeout_seconds": "soon"},
    +                "idle_timeout_seconds",
    +            )
    +            is None
    +        )
    +        assert (
    +            _get_lifecycle_seconds(
    +                {"idle_timeout_seconds": -1},
    +                "idle_timeout_seconds",
    +            )
    +            is None
    +        )
    +
    +        messages = [record.getMessage() for record in caplog.records]
    +        assert any("must be a number of seconds" in msg for msg in messages)
    +        assert any("must be positive" in msg for msg in messages)
    +
    +
     # ---------------------------------------------------------------------------
     # Schema conversion
     # ---------------------------------------------------------------------------
    @@ -568,6 +618,19 @@ def test_session_none_returns_false(self):
             finally:
                 _servers.pop("test_server", None)
     
    +    def test_recycled_stdio_server_remains_available_for_lazy_reconnect(self):
    +        from tools.mcp_tool import _make_check_fn, _servers
    +
    +        server = _make_mock_server("test_server", session=None)
    +        server._config = {"command": "npx"}
    +        server._recycled_reason = "idle_timeout_seconds"
    +        _servers["test_server"] = server
    +        try:
    +            check = _make_check_fn("test_server")
    +            assert check() is True
    +        finally:
    +            _servers.pop("test_server", None)
    +
     
     # ---------------------------------------------------------------------------
     # MCP loop runner
    @@ -742,6 +805,36 @@ def _interrupting_run(coro_or_factory, timeout=30):
             finally:
                 _servers.pop("test_srv", None)
     
    +    def test_recycled_stdio_server_reconnects_lazily_on_tool_call(self):
    +        from tools.mcp_tool import _make_tool_handler, _servers
    +
    +        mock_session = MagicMock()
    +        mock_session.call_tool = AsyncMock(
    +            return_value=_make_call_result("reconnected", is_error=False)
    +        )
    +        server = _make_mock_server("test_srv", session=None)
    +        server._config = {"command": "npx"}
    +        server._recycled_reason = "idle_timeout_seconds"
    +        _servers["test_srv"] = server
    +
    +        def fake_lazy_reconnect(server_name, srv):
    +            assert server_name == "test_srv"
    +            assert srv is server
    +            srv.session = mock_session
    +            srv._recycled_reason = None
    +            return True
    +
    +        try:
    +            handler = _make_tool_handler("test_srv", "greet", 120)
    +            with patch("tools.mcp_tool._request_lazy_reconnect", side_effect=fake_lazy_reconnect) as reconnect, \
    +                 self._patch_mcp_loop():
    +                result = json.loads(handler({"name": "world"}))
    +            assert result["result"] == "reconnected"
    +            reconnect.assert_called_once()
    +            mock_session.call_tool.assert_called_once_with("greet", arguments={"name": "world"})
    +        finally:
    +            _servers.pop("test_srv", None)
    +
     
     class TestRunOnMCPLoopInterrupts:
         def test_interrupt_cancels_waiting_mcp_call(self):
    @@ -1148,6 +1241,52 @@ async def _test():
     
             asyncio.run(_test())
     
    +    def test_wait_for_lifecycle_event_recycles_idle_stdio_server(self):
    +        from tools.mcp_tool import MCPServerTask
    +
    +        async def _test():
    +            server = MCPServerTask("srv")
    +            server._config = {"command": "npx"}
    +            server.session = MagicMock()
    +            server._idle_timeout_seconds = 0.01
    +            server._last_tool_call_at = time.monotonic() - 1.0
    +
    +            reason = await server._wait_for_lifecycle_event()
    +
    +            assert reason == "recycle"
    +            assert server._recycled_reason == "idle_timeout_seconds"
    +            assert server.session is None
    +
    +        asyncio.run(_test())
    +
    +    def test_stdio_recycle_reason_uses_max_lifetime(self):
    +        from tools.mcp_tool import MCPServerTask
    +
    +        async def _test():
    +            server = MCPServerTask("srv")
    +            server._config = {"command": "npx"}
    +            server._max_lifetime_seconds = 0.01
    +            server._lifecycle_started_at = time.monotonic() - 1.0
    +
    +            assert server._stdio_recycle_reason() == "max_lifetime_seconds"
    +
    +        asyncio.run(_test())
    +
    +    def test_stdio_recycle_deadline_pauses_while_rpc_active(self):
    +        from tools.mcp_tool import MCPServerTask
    +
    +        async def _test():
    +            server = MCPServerTask("srv")
    +            server._config = {"command": "npx"}
    +            server._idle_timeout_seconds = 0.01
    +            server._last_tool_call_at = time.monotonic() - 1.0
    +
    +            async with server._rpc_lock:
    +                assert server._stdio_recycle_reason() is None
    +                assert server._next_stdio_recycle_deadline() is None
    +
    +        asyncio.run(_test())
    +
     
     # ---------------------------------------------------------------------------
     # discover_mcp_tools toolset injection
    @@ -2000,6 +2139,55 @@ async def _test():
     
             asyncio.run(_test())
     
    +    def test_failed_recycle_reconnect_no_longer_checks_available(self):
    +        """A failed lazy reconnect should not leave recycled availability true."""
    +        from tools.mcp_tool import (
    +            MCPServerTask,
    +            _MAX_INITIAL_CONNECT_RETRIES,
    +            _make_check_fn,
    +            _servers,
    +        )
    +
    +        run_count = 0
    +        target_server = None
    +
    +        original_run_stdio = MCPServerTask._run_stdio
    +
    +        async def patched_run_stdio(self_srv, config):
    +            nonlocal run_count, target_server
    +            run_count += 1
    +            if target_server is not self_srv:
    +                return await original_run_stdio(self_srv, config)
    +            # After the final retry, run() parks in
    +            # _wait_for_reconnect_or_shutdown(timeout=_PARKED_RETRY_INTERVAL)
    +            # (a real asyncio.wait — the patched asyncio.sleep doesn't cover
    +            # it). Signal shutdown so the park exits immediately instead of
    +            # blocking the test for the 300s self-probe interval.
    +            if run_count > _MAX_INITIAL_CONNECT_RETRIES:
    +                self_srv._shutdown_event.set()
    +            raise ConnectionError("recycle reconnect failed")
    +
    +        async def _test():
    +            nonlocal target_server
    +            server = MCPServerTask("test_srv")
    +            target_server = server
    +            server._config = {"command": "test"}
    +            server._ready.clear()
    +            server._recycled_reason = "idle_timeout_seconds"
    +            _servers["test_srv"] = server
    +            try:
    +                with patch.object(MCPServerTask, "_run_stdio", patched_run_stdio), \
    +                     patch("asyncio.sleep", new_callable=AsyncMock):
    +                    await server.run({"command": "test"})
    +
    +                assert run_count == _MAX_INITIAL_CONNECT_RETRIES + 1
    +                assert server._recycled_reason is None
    +                assert _make_check_fn("test_srv")() is False
    +            finally:
    +                _servers.pop("test_srv", None)
    +
    +        asyncio.run(_test())
    +
     
     # ---------------------------------------------------------------------------
     # Configurable timeouts

  • tests/tools/test_mcp_tool_issue_948.py

    @@ -1,5 +1,6 @@
     import asyncio
     import os
    +import sys
     from types import SimpleNamespace
     from unittest.mock import AsyncMock, MagicMock, patch
     
    @@ -119,8 +120,18 @@ async def _test():
                 server = MCPServerTask("srv")
                 await server.start({"command": "npx", "args": ["-y", "pkg"], "env": {"PATH": "/usr/bin"}})
     
    +            # The real (resolved) command no longer reaches StdioServerParameters
    +            # directly -- it's now wrapped in the parent-death watchdog
    +            # supervisor (tools/mcp_stdio_watchdog.py) so an ungraceful exit of
    +            # this process can't orphan it. Assert the resolved npx path and
    +            # its args still flow through correctly as the watchdog's target
    +            # command, preserving this test's original path-resolution intent.
                 call_kwargs = mock_params.call_args.kwargs
    -            assert call_kwargs["command"] == str(npx_path)
    +            assert call_kwargs["command"] == sys.executable
    +            assert call_kwargs["args"][0].endswith("mcp_stdio_watchdog.py")
    +            assert "--" in call_kwargs["args"]
    +            sep = call_kwargs["args"].index("--")
    +            assert call_kwargs["args"][sep + 1:] == [str(npx_path), "-y", "pkg"]
                 assert call_kwargs["env"]["PATH"].split(os.pathsep)[0] == str(node_bin)
     
                 await server.shutdown()

  • tools/mcp_stdio_watchdog.py

    @@ -0,0 +1,184 @@
    +#!/usr/bin/env python3
    +"""Parent-death watchdog supervisor for stdio MCP subprocesses.
    +
    +Problem this fixes (#TBD): a stdio MCP server (e.g. ``npx -y mcp-remote
    +<url>``) is spawned as a direct child of the Hermes process. Hermes's own
    +teardown path (``MCPServerTask.shutdown()`` / ``_kill_orphaned_mcp_children``
    +at final exit) reaps it cleanly on a *graceful* exit. But if the spawning
    +Hermes process dies hard — ``kill -9``, an OS-level crash, a force-quit of
    +the TUI/desktop app — that teardown code never runs, and the child (plus any
    +of its own descendants, e.g. mcp-remote's spawned ``node`` process) is
    +orphaned. macOS has no direct equivalent of Linux's
    +``prctl(PR_SET_PDEATHSIG)`` to make the kernel auto-kill a child when its
    +parent dies, so nothing reaps these until the next Hermes startup's opt-in
    +``_kill_orphaned_mcp_children()`` sweep — which only runs if something calls
    +it. Repeated ungraceful session restarts can pile up N orphaned processes,
    +all racing to hold the same upstream SSE session, producing errors like
    +"Invalid request parameters" / "Received request before initialization was
    +complete" on the *legitimate* new connection.
    +
    +Fix: don't spawn the MCP server command directly. Spawn this supervisor
    +instead, which:
    +  1. execs the real command as its own child (own process group via
    +     ``start_new_session``, so it doesn't inherit the supervisor's
    +     controlling terminal weirdly and so we can killpg it cleanly);
    +  2. transparently passes stdin/stdout/stderr through — the MCP stdio
    +     protocol talks directly over those pipes, so the supervisor must be a
    +     no-op relay, not a bytes-in-the-middle proxy;
    +  3. runs a background thread that polls the ORIGINAL parent PID using the
    +     exact same orphan-detection algorithm already proven in
    +     ``tui_gateway/slash_worker.py`` (``_is_orphaned``): compare current
    +     ``getppid()`` against the recorded original, and guard PID reuse via
    +     ``psutil`` process creation time;
    +  4. the instant the original parent is gone, terminates the real child's
    +     process group (SIGTERM, grace period, then SIGKILL) and exits.
    +
    +This is intentionally a thin, dependency-light script (``psutil`` only,
    +already a hard dependency via ``tui_gateway/slash_worker.py``) so it starts
    +fast and can't itself become a resource leak.
    +
    +Usage (see ``tools/mcp_tool.py::_run_stdio``)::
    +
    +    python3 -m tools.mcp_stdio_watchdog \\
    +        --ppid <original_parent_pid> --create-time <original_parent_create_time> \\
    +        -- <real_command> <arg1> <arg2> ...
    +"""
    +
    +from __future__ import annotations
    +
    +import argparse
    +import os
    +import signal
    +import subprocess
    +import sys
    +import threading
    +import time
    +
    +try:
    +    import psutil
    +except ImportError:  # pragma: no cover - psutil is a hard dependency elsewhere
    +    psutil = None
    +
    +_POLL_INTERVAL_S = 2.0
    +_TERM_GRACE_S = 3.0
    +
    +
    +def _is_orphaned(original_ppid: int, parent_create_time: float, getppid=os.getppid) -> bool:
    +    """Mirrors ``tui_gateway.slash_worker._is_orphaned`` exactly.
    +
    +    True once the process that spawned us is gone. Never trusts a bare
    +    ``getppid() == 1`` check (Linux reparents orphans to a subreaper, not
    +    always PID 1), and guards against PID reuse via the recorded creation
    +    time of the original parent.
    +    """
    +    if getppid() != original_ppid:
    +        return True
    +    if psutil is None:
    +        # No reliable staleness check available; fall back to the ppid
    +        # comparison alone (still catches the common case).
    +        return False
    +    try:
    +        if not psutil.pid_exists(original_ppid):
    +            return True
    +        return psutil.Process(original_ppid).create_time() != parent_create_time
    +    except psutil.Error:
    +        return True
    +
    +
    +def _terminate_process_group(proc: subprocess.Popen) -> None:
    +    """Best-effort SIGTERM-then-SIGKILL of the child's process group.
    +
    +    This module only ever runs on POSIX (the wrap site in tools/mcp_tool.py
    +    gates on ``os.name == "posix"``), but guard the POSIX-only primitives
    +    anyway so an accidental Windows import/execute degrades to a plain
    +    child kill instead of AttributeError.
    +    """
    +    killpg = getattr(os, "killpg", None)
    +    if killpg is None:  # windows-footgun: ok — non-POSIX fallback
    +        try:
    +            proc.terminate()
    +            proc.wait(timeout=_TERM_GRACE_S)
    +        except (OSError, subprocess.TimeoutExpired):
    +            proc.kill()
    +        return
    +    try:
    +        pgid = os.getpgid(proc.pid)
    +    except (ProcessLookupError, OSError):
    +        return
    +    sigkill = getattr(signal, "SIGKILL", signal.SIGTERM)
    +    for sig in (signal.SIGTERM, sigkill):
    +        try:
    +            killpg(pgid, sig)
    +        except (ProcessLookupError, PermissionError, OSError):
    +            return
    +        try:
    +            proc.wait(timeout=_TERM_GRACE_S)
    +            return
    +        except subprocess.TimeoutExpired:
    +            continue
    +
    +
    +def _watchdog_loop(proc: subprocess.Popen, original_ppid: int, parent_create_time: float) -> None:
    +    while proc.poll() is None:
    +        if _is_orphaned(original_ppid, parent_create_time):
    +            _terminate_process_group(proc)
    +            return
    +        time.sleep(_POLL_INTERVAL_S)
    +
    +
    +def main(argv: list[str] | None = None) -> int:
    +    parser = argparse.ArgumentParser(
    +        description="Parent-death watchdog for a stdio MCP subprocess.",
    +    )
    +    parser.add_argument("--ppid", type=int, required=True)
    +    parser.add_argument("--create-time", type=float, required=True)
    +    parser.add_argument("command", nargs=argparse.REMAINDER)
    +    args = parser.parse_args(argv)
    +
    +    real_argv = list(args.command)
    +    if real_argv and real_argv[0] == "--":
    +        real_argv = real_argv[1:]
    +    if not real_argv:
    +        print("mcp_stdio_watchdog: no command given after '--'", file=sys.stderr)
    +        return 2
    +
    +    # New process group so we can killpg() the whole tree the real command
    +    # may spawn (e.g. mcp-remote's own child `node` process), without
    +    # touching our own group or the (already-gone) original parent's.
    +    proc = subprocess.Popen(
    +        real_argv,
    +        stdin=sys.stdin,
    +        stdout=sys.stdout,
    +        stderr=sys.stderr,
    +        start_new_session=True,
    +    )
    +
    +    # Because the real server lives in its OWN process group (above), the
    +    # parent's graceful-shutdown killpg of *our* group no longer reaches it.
    +    # Forward SIGTERM/SIGINT to the child's group so graceful teardown
    +    # (`_kill_orphaned_mcp_children`, shutdown sweeps) still kills a wedged
    +    # server that ignores stdin EOF — otherwise the watchdog wrap would
    +    # invert the bug it fixes.
    +    def _forward_shutdown(signum, frame):  # noqa: ARG001
    +        _terminate_process_group(proc)
    +        sys.exit(128 + signum)
    +
    +    signal.signal(signal.SIGTERM, _forward_shutdown)
    +    signal.signal(signal.SIGINT, _forward_shutdown)
    +
    +    watchdog = threading.Thread(
    +        target=_watchdog_loop,
    +        args=(proc, args.ppid, args.create_time),
    +        daemon=True,
    +    )
    +    watchdog.start()
    +
    +    try:
    +        return proc.wait()
    +    except KeyboardInterrupt:
    +        _terminate_process_group(proc)
    +        return 130
    +
    +
    +if __name__ == "__main__":
    +    sys.exit(main())

  • tools/mcp_tool.py

    @@ -23,6 +23,10 @@
                                     # 180). Set below the server's session TTL for
                                     # servers that GC idle sessions quickly (e.g.
                                     # Unreal Engine editor MCP, ~15s). Floored at 5s.
    +        idle_timeout_seconds: 3600      # optional stdio recycle after idle
    +        max_lifetime_seconds: 86400     # optional stdio recycle after age
    +        # The recycle settings may also live under lifecycle: {...}.
    +        # Use 0 to disable either recycle limit.
           github:
             command: "npx"
             args: ["-y", "@modelcontextprotocol/server-github"]
    @@ -330,6 +334,7 @@ def _check_logging_callback_support() -> bool:
     # server is unrevivable: its tools are out of the registry, so no tool call
     # can ever reach the circuit-breaker half-open probe or _signal_reconnect.
     _PARKED_RETRY_INTERVAL = 300     # seconds between parked self-probes
    +_RECYCLED_RECONNECT_TIMEOUT = 15.0
     
     # Keepalive cadence for HTTP/SSE sessions. The MCP spec lets a server expire
     # idle sessions on any TTL it chooses (Streamable HTTP "Session Management"),
    @@ -616,6 +621,41 @@ def _resolve_stdio_command(command: str, env: dict) -> tuple[str, dict]:
         return resolved_command, resolved_env
     
     
    +def _wrap_command_with_watchdog(command: str, args: list) -> tuple[str, list]:
    +    """Wrap a stdio MCP server command in the parent-death watchdog supervisor.
    +
    +    See ``tools/mcp_stdio_watchdog.py`` module docstring for the full
    +    rationale. Returns the (command, args) unchanged on any platform/failure
    +    where the wrap can't safely apply, so this can never be the reason a
    +    previously-working MCP server stops starting.
    +    """
    +    if os.name != "posix":
    +        # Relies on process groups (os.getpgid/os.killpg); no POSIX
    +        # equivalent wired up here yet, matching the existing killpg-based
    +        # orphan cleanup's platform scope (Windows falls back to plain
    +        # os.kill there too).
    +        return command, args
    +    try:
    +        my_pid = os.getpid()
    +        try:
    +            import psutil
    +            create_time = psutil.Process(my_pid).create_time()
    +        except ImportError:
    +            create_time = time.time()
    +    except Exception:
    +        # Never let watchdog bookkeeping failure block a real MCP connection.
    +        return command, args
    +    watchdog_args = [
    +        os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp_stdio_watchdog.py"),
    +        "--ppid", str(my_pid),
    +        "--create-time", repr(create_time),
    +        "--",
    +        command,
    +        *args,
    +    ]
    +    return sys.executable, watchdog_args
    +
    +
     # ---------------------------------------------------------------------------
     # MCP ImageContent block → Hermes MEDIA tag
     # ---------------------------------------------------------------------------
    @@ -1483,6 +1523,8 @@ class MCPServerTask:
             "_registered_tool_names", "_auth_type", "_refresh_lock",
             "_rpc_lock", "_pending_refresh_tasks",
             "_pending_call_context",
    +        "_lifecycle_started_at", "_last_tool_call_at",
    +        "_idle_timeout_seconds", "_max_lifetime_seconds", "_recycled_reason",
             "initialize_result", "_ping_unsupported",
             "_reconnect_retries",
         )
    @@ -1527,6 +1569,12 @@ def __init__(self, name: str):
             # gateway-platform attribution and routes the approval prompt
             # to the right surface (Telegram, Slack, etc.).
             self._pending_call_context: Optional[contextvars.Context] = None
    +        now = time.monotonic()
    +        self._lifecycle_started_at: float = now
    +        self._last_tool_call_at: float = now
    +        self._idle_timeout_seconds: Optional[float] = None
    +        self._max_lifetime_seconds: Optional[float] = None
    +        self._recycled_reason: Optional[str] = None
             # Captures the ``InitializeResult`` returned by
             # ``await session.initialize()`` so downstream code can inspect the
             # server's real advertised capabilities (``.capabilities.resources``,
    @@ -1564,6 +1612,53 @@ def _advertises_tools(self) -> bool:
                 return True
             return getattr(caps, "tools", None) is not None
     
    +    def _is_recycled_stdio(self) -> bool:
    +        """Return True when a stdio server was intentionally recycled."""
    +        return not self._is_http() and self._recycled_reason is not None
    +
    +    def mark_tool_call(self) -> None:
    +        """Record that a user-visible MCP operation is starting."""
    +        self._last_tool_call_at = time.monotonic()
    +
    +    def _mark_lifecycle_started(self) -> None:
    +        now = time.monotonic()
    +        self._lifecycle_started_at = now
    +        self._last_tool_call_at = now
    +        self._recycled_reason = None
    +
    +    def _stdio_recycle_reason(self, now: Optional[float] = None) -> Optional[str]:
    +        """Return the stdio recycle reason if idle/age limits have elapsed."""
    +        if self._is_http() or self._rpc_lock.locked():
    +            return None
    +        now = time.monotonic() if now is None else now
    +        if (
    +            self._max_lifetime_seconds is not None
    +            and now - self._lifecycle_started_at >= self._max_lifetime_seconds
    +        ):
    +            return "max_lifetime_seconds"
    +        if (
    +            self._idle_timeout_seconds is not None
    +            and now - self._last_tool_call_at >= self._idle_timeout_seconds
    +        ):
    +            return "idle_timeout_seconds"
    +        return None
    +
    +    def _next_stdio_recycle_deadline(self) -> Optional[float]:
    +        """Return the next monotonic recycle deadline for stdio, if any."""
    +        if self._is_http() or self._rpc_lock.locked():
    +            return None
    +        deadlines = []
    +        if self._max_lifetime_seconds is not None:
    +            deadlines.append(self._lifecycle_started_at + self._max_lifetime_seconds)
    +        if self._idle_timeout_seconds is not None:
    +            deadlines.append(self._last_tool_call_at + self._idle_timeout_seconds)
    +        return min(deadlines) if deadlines else None
    +
    +    def _mark_stdio_recycled(self, reason: str) -> None:
    +        """Mark a stdio session dormant before its transport finishes closing."""
    +        self._recycled_reason = reason
    +        self.session = None
    +
         # ----- Dynamic tool discovery (notifications/tools/list_changed) -----
     
         async def _refresh_tools_task(self):
    @@ -1776,6 +1871,9 @@ async def _wait_for_lifecycle_event(self) -> str:
                             tokens, new session ID, etc.). The reconnect event
                             is cleared before return so the next cycle starts
                             with a fresh signal.
    +            "recycle"   if a stdio idle/max-lifetime limit elapsed. The
    +                        current transport is torn down and restarted lazily
    +                        on the next tool call.
     
             Shutdown takes precedence if both events are set simultaneously.
     
    @@ -1807,14 +1905,29 @@ async def _wait_for_lifecycle_event(self) -> str:
             reconnect_task = asyncio.create_task(self._reconnect_event.wait())
             try:
                 while True:
    +                recycle_reason = self._stdio_recycle_reason()
    +                if recycle_reason is not None:
    +                    self._mark_stdio_recycled(recycle_reason)
    +                    return "recycle"
    +
    +                timeout = keepalive_interval
    +                recycle_deadline = self._next_stdio_recycle_deadline()
    +                if recycle_deadline is not None:
    +                    timeout = max(0.0, min(timeout, recycle_deadline - time.monotonic()))
    +
                     done, _pending = await asyncio.wait(
                         {shutdown_task, reconnect_task},
    -                    timeout=keepalive_interval,
    +                    timeout=timeout,
                         return_when=asyncio.FIRST_COMPLETED,
                     )
                     if done:
                         break
     
    +                recycle_reason = self._stdio_recycle_reason()
    +                if recycle_reason is not None:
    +                    self._mark_stdio_recycled(recycle_reason)
    +                    return "recycle"
    +
                     # Timeout — no lifecycle event fired.  Probe the connection
                     # to detect stale/expired sessions. Prefer ``ping`` (MCP base
                     # protocol liveness): it works uniformly and stays a few bytes
    @@ -1919,6 +2032,9 @@ async def _run_stdio(self, config: dict):
             # it with a wall-clock timeout so a stalled SSL handshake can't freeze
             # MCP discovery / gateway startup (#29184). The check is fail-open, so
             # on timeout we log and proceed rather than blocking indefinitely.
    +        # NOTE: must run against the REAL command/args — the watchdog wrap
    +        # below rewrites argv to `python -m tools.mcp_stdio_watchdog …`,
    +        # which would silently turn the preflight into a no-op.
             from tools.osv_check import check_package_for_malware
             try:
                 malware_error = await asyncio.wait_for(
    @@ -1937,6 +2053,18 @@ async def _run_stdio(self, config: dict):
                     f"MCP server '{self.name}': {malware_error}"
                 )
     
    +        # Wrap the real command in a parent-death watchdog supervisor so an
    +        # ungraceful exit of this Hermes process (kill -9, crash, force-quit)
    +        # can't leave the stdio MCP child (and its own descendants, e.g.
    +        # mcp-remote's spawned `node`) running forever. On a clean exit,
    +        # MCPServerTask.shutdown() / _kill_orphaned_mcp_children() still do
    +        # the reaping as before -- this only covers the case where that code
    +        # never gets to run. POSIX-only (relies on process groups); no-op
    +        # elsewhere, matching existing killpg-based cleanup's platform scope.
    +        # Applied AFTER the OSV preflight so the check inspects the real
    +        # package, not the watchdog wrapper.
    +        command, args = _wrap_command_with_watchdog(command, args)
    +
             server_params = StdioServerParameters(
                 command=command,
                 args=args,
    @@ -1951,6 +2079,18 @@ async def _run_stdio(self, config: dict):
             if _MCP_LOGGING_CALLBACK_SUPPORTED:
                 sampling_kwargs["logging_callback"] = self._make_logging_callback()
     
    +        # Reap any orphaned subprocesses from prior failed connection
    +        # attempts before spawning a new one.  Without this, each retry in
    +        # the run() reconnect loop spawns a fresh process pair while the
    +        # previous failed pair lingers — leading to rapid zombie
    +        # accumulation (see #57355, #57228).  The unscoped sweep also
    +        # opportunistically reaps orphans left by *other* servers that
    +        # never reconnect; per-server filtering via ``server_name`` remains
    +        # available for scoped call sites.  Run in a worker thread: the
    +        # reaper blocks up to 2s (SIGTERM → wait → SIGKILL) when orphans
    +        # exist, which would otherwise stall the shared MCP event loop.
    +        await asyncio.to_thread(_kill_orphaned_mcp_children)
    +
             # Snapshot child PIDs before spawning so we can track the new one.
             pids_before = _snapshot_child_pids()
             new_pids: set = set()
    @@ -1995,8 +2135,25 @@ async def _run_stdio(self, config: dict):
                     async with ClientSession(
                         read_stream, write_stream, **sampling_kwargs
                     ) as session:
    -                    self.initialize_result = await session.initialize()
    +                    # Bound the MCP handshake. A stdio server that never
    +                    # completes ``initialize`` (e.g. emits a non-JSON-RPC frame
    +                    # and then blocks on stdin) otherwise hangs this coroutine
    +                    # forever on the background loop: ``connect_timeout`` only
    +                    # bounds the caller's ``.result()`` wait, not the coroutine
    +                    # itself. Because the connect never unwinds, the cleanup
    +                    # ``finally`` below never runs, so the spawned child and its
    +                    # stdio pipes/pidfd leak on every discovery retry — unbounded
    +                    # until the gateway hits EMFILE. Timing out here converts the
    +                    # hang into a normal failure, letting the ``finally`` reap the
    +                    # child. See #59349.
    +                    connect_timeout = float(
    +                        config.get("connect_timeout", _DEFAULT_CONNECT_TIMEOUT)
    +                    )
    +                    self.initialize_result = await asyncio.wait_for(
    +                        session.initialize(), timeout=connect_timeout
    +                    )
                         self.session = session
    +                    self._mark_lifecycle_started()
                         await self._discover_tools()
                         self._ready.set()
                         # Session is live again: clear any breaker state from a
    @@ -2010,7 +2167,7 @@ async def _run_stdio(self, config: dict):
                         # stdio transport does not use OAuth, but we still honor
                         # _reconnect_event (e.g. future manual /mcp refresh) for
                         # consistency with _run_http.
    -                    await self._wait_for_lifecycle_event()
    +                    return await self._wait_for_lifecycle_event()
             finally:
                 # Runs on clean exit, exceptions, AND asyncio cancellation.
                 # If any of the spawned PIDs are still alive, the SDK's
    @@ -2041,6 +2198,7 @@ async def _run_stdio(self, config: dict):
                                     pgroup_alive = False
                             if pid_alive or pgroup_alive:
                                 _orphan_stdio_pids.add(pid)
    +                            _orphan_stdio_pid_servers[pid] = self.name
                             else:
                                 # Nothing left to reap — drop the pgid entry so
                                 # PID-reuse can't surface stale pgroup state later.
    @@ -2283,7 +2441,13 @@ def _mcp_http_client_factory(
                     async with ClientSession(
                         read_stream, write_stream, **sampling_kwargs
                     ) as session:
    -                    self.initialize_result = await session.initialize()
    +                    # Bound the handshake — same orphaned-task hang as the
    +                    # stdio path (#59349): an endpoint that accepts the
    +                    # connection but never answers ``initialize`` parks this
    +                    # coroutine forever on the background loop.
    +                    self.initialize_result = await asyncio.wait_for(
    +                        session.initialize(), timeout=float(connect_timeout)
    +                    )
                         self.session = session
                         await self._discover_tools()
                         self._ready.set()
    @@ -2298,7 +2462,7 @@ def _mcp_http_client_factory(
                                 "MCP server '%s': reconnect requested — "
                                 "tearing down SSE session", self.name,
                             )
    -            return
    +            return reason
     
             if _MCP_NEW_HTTP:
                 # New API (mcp >= 1.24.0): build an explicit httpx.AsyncClient
    @@ -2337,7 +2501,10 @@ async def _strip_auth_on_cross_origin_redirect(response):
                         read_stream, write_stream, _get_session_id,
                     ):
                         async with ClientSession(read_stream, write_stream, **sampling_kwargs) as session:
    -                        self.initialize_result = await session.initialize()
    +                        # Bound the handshake (#59349) — see stdio path.
    +                        self.initialize_result = await asyncio.wait_for(
    +                            session.initialize(), timeout=float(connect_timeout)
    +                        )
                             self.session = session
                             await self._discover_tools()
                             self._ready.set()
    @@ -2352,6 +2519,7 @@ async def _strip_auth_on_cross_origin_redirect(response):
                                     "MCP server '%s': reconnect requested — "
                                     "tearing down HTTP session", self.name,
                                 )
    +            return reason
             else:
                 # Deprecated API (mcp < 1.24.0): manages httpx client internally.
                 _http_kwargs: dict = {
    @@ -2365,7 +2533,10 @@ async def _strip_auth_on_cross_origin_redirect(response):
                     read_stream, write_stream, _get_session_id,
                 ):
                     async with ClientSession(read_stream, write_stream, **sampling_kwargs) as session:
    -                    self.initialize_result = await session.initialize()
    +                    # Bound the handshake (#59349) — see stdio path.
    +                    self.initialize_result = await asyncio.wait_for(
    +                        session.initialize(), timeout=float(connect_timeout)
    +                    )
                         self.session = session
                         await self._discover_tools()
                         self._ready.set()
    @@ -2380,6 +2551,7 @@ async def _strip_auth_on_cross_origin_redirect(response):
                                 "MCP server '%s': reconnect requested — "
                                 "tearing down legacy HTTP session", self.name,
                             )
    +            return reason
     
         async def _discover_tools(self):
             """Discover tools from the connected session.
    @@ -2440,6 +2612,8 @@ async def run(self, config: dict):
             self._config = config
             self.tool_timeout = config.get("timeout", _DEFAULT_TOOL_TIMEOUT)
             self._auth_type = (config.get("auth") or "").lower().strip()
    +        self._idle_timeout_seconds = _get_lifecycle_seconds(config, "idle_timeout_seconds")
    +        self._max_lifetime_seconds = _get_lifecycle_seconds(config, "max_lifetime_seconds")
     
             # Set up sampling handler if enabled and SDK types are available
             sampling_config = config.get("sampling", {})
    @@ -2515,9 +2689,9 @@ async def run(self, config: dict):
             while True:
                 try:
                     if self._is_http():
    -                    await self._run_http(config)
    +                    lifecycle_reason = await self._run_http(config)
                     else:
    -                    await self._run_stdio(config)
    +                    lifecycle_reason = await self._run_stdio(config)
                     # Transport returned cleanly. Two cases:
                     #  - _shutdown_event was set: exit the run loop entirely.
                     #  - _reconnect_event was set (auth recovery): loop back and
    @@ -2525,6 +2699,18 @@ async def run(self, config: dict):
                     #    touch the retry counters — this is not a failure.
                     if self._shutdown_event.is_set():
                         break
    +                if lifecycle_reason == "recycle":
    +                    logger.info(
    +                        "MCP server '%s': stdio session recycled after %s; "
    +                        "waiting for lazy reconnect",
    +                        self.name, self._recycled_reason,
    +                    )
    +                    self.session = None
    +                    await self._wait_for_lazy_reconnect()
    +                    if self._shutdown_event.is_set():
    +                        break
    +                    self._reconnect_event.clear()
    +                    continue
                     logger.info(
                         "MCP server '%s': reconnecting (OAuth recovery or "
                         "manual refresh)",
    @@ -2560,6 +2746,13 @@ async def run(self, config: dict):
                     raise
                 except Exception as exc:
                     self.session = None
    +                if self._is_recycled_stdio():
    +                    logger.warning(
    +                        "MCP server '%s': lazy reconnect after stdio recycle "
    +                        "failed, marking unavailable while retrying: %s",
    +                        self.name, exc,
    +                    )
    +                    self._recycled_reason = None
     
                     # If this is the first connection attempt, retry with backoff
                     # before giving up. A transient DNS/network blip at startup
    @@ -2686,7 +2879,19 @@ async def run(self, config: dict):
         async def start(self, config: dict):
             """Create the background Task and wait until ready (or failed)."""
             self._task = asyncio.ensure_future(self.run(config))
    -        await self._ready.wait()
    +        try:
    +            await self._ready.wait()
    +        except asyncio.CancelledError:
    +            # The caller's connect timeout (discover_mcp_tools wraps start()
    +            # in asyncio.wait_for) cancels *this* coroutine, but the
    +            # ensure_future'd run() task is independent and would otherwise
    +            # keep running detached — parked on a hung transport with no
    +            # owner to reap it (#59349). Propagate the cancellation so the
    +            # transport context managers unwind and their finally blocks
    +            # release the child process / FDs.
    +            if self._task and not self._task.done():
    +                self._task.cancel()
    +            raise
             if self._error:
                 raise self._error
     
    @@ -2736,6 +2941,24 @@ def _deregister_tools(self) -> None:
                 _forget_mcp_tool_server(tool_name)
             self._registered_tool_names = []
     
    +    async def _wait_for_lazy_reconnect(self) -> None:
    +        """Wait while an intentionally recycled stdio server is dormant."""
    +        shutdown_task = asyncio.create_task(self._shutdown_event.wait())
    +        reconnect_task = asyncio.create_task(self._reconnect_event.wait())
    +        try:
    +            await asyncio.wait(
    +                {shutdown_task, reconnect_task},
    +                return_when=asyncio.FIRST_COMPLETED,
    +            )
    +        finally:
    +            for task in (shutdown_task, reconnect_task):
    +                if not task.done():
    +                    task.cancel()
    +                    try:
    +                        await task
    +                    except (asyncio.CancelledError, Exception):
    +                        pass
    +
     
     # ---------------------------------------------------------------------------
     # Module-level state
    @@ -3234,6 +3457,7 @@ def _handle_session_expired_and_retry(
     # Separate from _stdio_pids so cleanup sweeps never race with active
     # sessions (e.g. concurrent cron jobs or live user chats).
     _orphan_stdio_pids: set = set()
    +_orphan_stdio_pid_servers: Dict[int, str] = {}
     
     # Process-group IDs of stdio MCP subprocesses, captured at spawn time.
     # The MCP SDK spawns stdio children with ``start_new_session=True`` so each
    @@ -3580,6 +3804,58 @@ async def _connect_server(name: str, config: dict) -> MCPServerTask:
     # Handler / check-fn factories
     # ---------------------------------------------------------------------------
     
    +def _request_lazy_reconnect(server_name: str, server: MCPServerTask) -> bool:
    +    """Wake a recycled stdio server and wait briefly for a fresh session."""
    +    if not server._is_recycled_stdio():
    +        return False
    +
    +    with _lock:
    +        loop = _mcp_loop
    +    if loop is None or not loop.is_running():
    +        return False
    +
    +    def _signal_reconnect() -> None:
    +        server._ready.clear()
    +        server._reconnect_event.set()
    +
    +    loop.call_soon_threadsafe(_signal_reconnect)
    +
    +    async def _await_ready() -> bool:
    +        deadline = time.monotonic() + _RECYCLED_RECONNECT_TIMEOUT
    +        while time.monotonic() < deadline:
    +            if server.session is not None and server._ready.is_set():
    +                return True
    +            await asyncio.sleep(0.05)
    +        return False
    +
    +    try:
    +        return bool(_run_on_mcp_loop(_await_ready, timeout=_RECYCLED_RECONNECT_TIMEOUT))
    +    except Exception as exc:
    +        logger.warning(
    +            "MCP server '%s': lazy reconnect after stdio recycle failed: %s",
    +            server_name, exc,
    +        )
    +        return False
    +
    +
    +def _get_connected_server_for_call(server_name: str) -> Optional[MCPServerTask]:
    +    """Return a connected server, lazily reconnecting recycled stdio state."""
    +    with _lock:
    +        server = _servers.get(server_name)
    +    if server is not None and server.session is None and server._is_recycled_stdio():
    +        _request_lazy_reconnect(server_name, server)
    +        with _lock:
    +            server = _servers.get(server_name)
    +    return server
    +
    +
    +def _mark_server_call_started(server: Any) -> None:
    +    """Record a user-visible MCP operation when the server supports it."""
    +    mark_tool_call = getattr(server, "mark_tool_call", None)
    +    if callable(mark_tool_call):
    +        mark_tool_call()
    +
    +
     def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float):
         """Return a sync handler that calls an MCP tool via the background loop.
     
    @@ -3614,8 +3890,7 @@ def _handler(args: dict, **kwargs) -> str:
                     }, ensure_ascii=False)
                 # Cooldown elapsed → fall through as a half-open probe.
     
    -        with _lock:
    -            server = _servers.get(server_name)
    +        server = _get_connected_server_for_call(server_name)
             if not server:
                 _bump_server_error(server_name)
                 return json.dumps({
    @@ -3657,6 +3932,7 @@ def _handler(args: dict, **kwargs) -> str:
                     }, ensure_ascii=False)
     
             async def _call():
    +            _mark_server_call_started(server)
                 async with server._rpc_lock:
                     # Snapshot the agent's context so an elicitation callback
                     # triggered during this call (fired on the MCP recv loop
    @@ -3770,14 +4046,14 @@ def _make_list_resources_handler(server_name: str, tool_timeout: float):
         """Return a sync handler that lists resources from an MCP server."""
     
         def _handler(args: dict, **kwargs) -> str:
    -        with _lock:
    -            server = _servers.get(server_name)
    +        server = _get_connected_server_for_call(server_name)
             if not server or not server.session:
                 return json.dumps({
                     "error": f"MCP server '{server_name}' is not connected"
                 }, ensure_ascii=False)
     
             async def _call():
    +            _mark_server_call_started(server)
                 async with server._rpc_lock:
                     result = await server.session.list_resources()
                 resources = []
    @@ -3830,8 +4106,7 @@ def _make_read_resource_handler(server_name: str, tool_timeout: float):
         def _handler(args: dict, **kwargs) -> str:
             from tools.registry import tool_error
     
    -        with _lock:
    -            server = _servers.get(server_name)
    +        server = _get_connected_server_for_call(server_name)
             if not server or not server.session:
                 return json.dumps({
                     "error": f"MCP server '{server_name}' is not connected"
    @@ -3842,6 +4117,7 @@ def _handler(args: dict, **kwargs) -> str:
                 return tool_error("Missing required parameter 'uri'")
     
             async def _call():
    +            _mark_server_call_started(server)
                 async with server._rpc_lock:
                     result = await server.session.read_resource(uri)
                 # read_resource returns ReadResourceResult with .contents list
    @@ -3888,14 +4164,14 @@ def _make_list_prompts_handler(server_name: str, tool_timeout: float):
         """Return a sync handler that lists prompts from an MCP server."""
     
         def _handler(args: dict, **kwargs) -> str:
    -        with _lock:
    -            server = _servers.get(server_name)
    +        server = _get_connected_server_for_call(server_name)
             if not server or not server.session:
                 return json.dumps({
                     "error": f"MCP server '{server_name}' is not connected"
                 }, ensure_ascii=False)
     
             async def _call():
    +            _mark_server_call_started(server)
                 async with server._rpc_lock:
                     result = await server.session.list_prompts()
                 prompts = []
    @@ -3953,8 +4229,7 @@ def _make_get_prompt_handler(server_name: str, tool_timeout: float):
         def _handler(args: dict, **kwargs) -> str:
             from tools.registry import tool_error
     
    -        with _lock:
    -            server = _servers.get(server_name)
    +        server = _get_connected_server_for_call(server_name)
             if not server or not server.session:
                 return json.dumps({
                     "error": f"MCP server '{server_name}' is not connected"
    @@ -3966,6 +4241,7 @@ def _handler(args: dict, **kwargs) -> str:
             arguments = args.get("arguments", {})
     
             async def _call():
    +            _mark_server_call_started(server)
                 async with server._rpc_lock:
                     result = await server.session.get_prompt(name, arguments=arguments)
                 # GetPromptResult has .messages list
    @@ -4024,7 +4300,10 @@ def _make_check_fn(server_name: str):
         def _check() -> bool:
             with _lock:
                 server = _servers.get(server_name)
    -        return server is not None and server.session is not None
    +        return (
    +            server is not None
    +            and (server.session is not None or server._is_recycled_stdio())
    +        )
     
         return _check
     
    @@ -4324,6 +4603,27 @@ def _parse_boolish(value: Any, default: bool = True) -> bool:
         return default
     
     
    +def _get_lifecycle_seconds(config: dict, key: str) -> Optional[float]:
    +    """Return an optional positive lifecycle timeout from top-level/nested config."""
    +    raw = config.get(key)
    +    lifecycle = config.get("lifecycle")
    +    if raw is None and isinstance(lifecycle, dict):
    +        raw = lifecycle.get(key)
    +    if raw is None:
    +        return None
    +    try:
    +        seconds = float(raw)
    +    except (TypeError, ValueError):
    +        logger.warning("MCP config %s must be a number of seconds; ignoring %r", key, raw)
    +        return None
    +    if seconds == 0:
    +        return None
    +    if seconds < 0:
    +        logger.warning("MCP config %s must be positive; ignoring %r", key, raw)
    +        return None
    +    return seconds
    +
    +
     _UTILITY_CAPABILITY_METHODS = {
         "list_resources": "list_resources",
         "read_resource": "read_resource",
    @@ -5161,7 +5461,10 @@ async def _shutdown():
         _stop_mcp_loop()
     
     
    -def _kill_orphaned_mcp_children(include_active: bool = False) -> None:
    +def _kill_orphaned_mcp_children(
    +    include_active: bool = False,
    +    server_name: Optional[str] = None,
    +) -> None:
         """Best-effort graceful shutdown of stdio MCP subprocesses to reap orphans.
     
         Orphans are PIDs that survived their session context exit (SDK teardown
    @@ -5180,6 +5483,10 @@ def _kill_orphaned_mcp_children(include_active: bool = False) -> None:
         first) are reaped alongside the direct child.  Falls back to ``os.kill``
         on Windows and when no pgid is recorded.
     
    +    When ``server_name`` is set, only orphaned PIDs known to belong to that
    +    MCP server are reaped. This lets stdio reconnects clean up their previous
    +    transport without touching unrelated servers.
    +
         With ``include_active=True`` also kills every PID in ``_stdio_pids`` —
         used only at final shutdown, after the MCP event loop has stopped and no
         sessions can still be in flight.
    @@ -5189,11 +5496,24 @@ def _kill_orphaned_mcp_children(include_active: bool = False) -> None:
         with _lock:
             pids: Dict[int, str] = {}
             for opid in _orphan_stdio_pids:
    -            pids[opid] = "orphan"
    -        _orphan_stdio_pids.clear()
    +            owner = _orphan_stdio_pid_servers.get(opid, "orphan")
    +            if server_name is not None and owner != server_name:
    +                continue
    +            pids[opid] = owner
    +        for opid in pids:
    +            _orphan_stdio_pids.discard(opid)
    +            _orphan_stdio_pid_servers.pop(opid, None)
             if include_active:
    -            pids.update(dict(_stdio_pids))
    -            _stdio_pids.clear()
    +            active = dict(_stdio_pids)
    +            if server_name is not None:
    +                active = {
    +                    pid: owner
    +                    for pid, owner in active.items()
    +                    if owner == server_name
    +                }
    +            pids.update(active)
    +            for pid in active:
    +                _stdio_pids.pop(pid, None)
             # Snapshot pgids for the pids we're about to kill, then drop the
             # entries so a future spawn can't collide with stale state.
             pgids: Dict[int, int] = {pid: _stdio_pgids[pid] for pid in pids if pid in _stdio_pgids}

  • website/docs/user-guide/features/mcp.md

    @@ -296,7 +296,9 @@ Hermes reads MCP config from `~/.hermes/config.yaml` under `mcp_servers`.
     | `client_cert` | string \| list | Client certificate for mTLS — a combined PEM path, or `[cert, key]` / `[cert, key, password]` |
     | `client_key` | string | Client private-key PEM path (when separate from `client_cert`) |
     | `timeout` | number | Tool call timeout |
    -| `connect_timeout` | number | Initial connection timeout |
    +| `connect_timeout` | number | Initial connection timeout (also bounds the MCP `initialize` handshake) |
    +| `idle_timeout_seconds` | number | Recycle a stdio server after this many seconds without a tool call (`0` = never, default). The server restarts transparently on the next tool call. |
    +| `max_lifetime_seconds` | number | Recycle a stdio server after this total age (`0` = never, default). Restarts transparently on next use. |
     | `enabled` | bool | If `false`, Hermes skips the server entirely |
     | `supports_parallel_tool_calls` | bool | If `true`, tools from this server may run concurrently |
     | `tools` | mapping | Per-server tool filtering and utility policy |
    @@ -310,6 +312,23 @@ mcp_servers:
         args: ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"]
     ```
     
    +### Recycling memory-heavy stdio servers
    +
    +Browser-based MCP servers (e.g. `@playwright/mcp`) keep a full Chromium
    +resident after their first tool call — hundreds of MB that never get
    +released. Opt in to automatic recycling and the server is torn down after
    +the idle/lifetime limit, then restarted transparently the next time one of
    +its tools is called (its tools stay registered the whole time):
    +
    +```yaml
    +mcp_servers:
    +  playwright:
    +    command: "npx"
    +    args: ["-y", "@playwright/mcp@latest", "--headless"]
    +    idle_timeout_seconds: 900     # recycle after 15 min without a tool call
    +    max_lifetime_seconds: 86400   # and at least once a day regardless
    +```
    +
     ### Minimal HTTP example
     
     ```yaml