{"repo":"NousResearch/hermes-agent","public_id":"21370","display_label":"TUI slash_worker subprocess leak and zombie cleanup","summary":"These issues and fixes all concern the same problem: tui_gateway slash_worker subprocesses are not reliably terminated on session disconnect, PTY EOF, or WebSocket close, causing zombie processes, file descriptor exhaustion, and memory pressure. The fixes address cleanup in various code paths including finalize_session, transport disconnect, and orphan reaping.","subsystem":"gateway","counts":{"members":12,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":6,"merged_prs":2,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":42132},"resolution":{"state":"partially_resolved","by":"pr-42132,pr-42149"},"freshness":{"state":"verified","last_verified_at":"2026-06-08T14:26:06+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.13999999999999999},"fix_edges":[{"issue_number":32377,"pr_number":42132,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":38095,"pr_number":42149,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":21370,"title":"Bug: dashboard chat handler leaks tui_gateway.slash_worker processes after each gateway turn","url":null,"state":"closed"},{"kind":"issue","number":21467,"title":"tui_gateway.slash_worker subprocesses leak under dashboard usage (swap-pinning a 7.8GB box at 128 workers)","url":null,"state":"closed"},{"kind":"issue","number":22855,"title":"Bug: slash_worker lifecycle gaps and system fragility observed during intensive Dashboard usage (#21370)","url":null,"state":"closed"},{"kind":"issue","number":24775,"title":"Dashboard: leaked tui_gateway.slash_worker processes accumulate, exhaust file descriptors on macOS launchd","url":null,"state":"closed"},{"kind":"issue","number":32377,"title":"Dashboard hangs: tui_gateway.slash_worker subprocesses leak on PTY chat disconnect (524 via reverse proxy)","url":null,"state":"closed"},{"kind":"issue","number":38095,"title":"[Bug]: TUI slash worker subprocesses leak after session close, accumulating zombie processes","url":null,"state":"closed"},{"kind":"pr","number":21401,"title":"fix(tui-gateway): close sidecar sessions on disconnect","url":null,"state":"closed","close_reason":{"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":32483,"title":"[codex] fix(tui): reap dashboard slash workers","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":38114,"title":"Fix TUI slash worker zombie subprocess leak (#38095)","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":38218,"title":"fix(tui): close slash_worker in _finalize_session to prevent zombie processes #38095","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":42132,"title":"fix(tui/pty): reap leaked slash_worker subprocesses on PTY chat disconnect","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/pty_bridge.py","scripts/release.py","tests/hermes_cli/test_pty_bridge.py","tests/test_slash_worker_watchdog.py","tui_gateway/slash_worker.py"],"fix_hunks":[{"file":"hermes_cli/pty_bridge.py","hunks":["@@ -250,13 +250,23 @@ def close(self) -> None:\n             return\n         self._closed = True\n \n+        try:\n+            pgid = os.getpgid(self._proc.pid)  # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)\n+        except Exception:\n+            pgid = None\n+\n         # SIGHUP is the conventional \"your terminal went away\" signal.\n-        # We escalate if the child ignores it.\n+        # Send it to the whole foreground process group, not just the PTY\n+        # leader: the dashboard TUI starts helper children such as the Python\n+        # slash worker, and killing only the leader can strand those helpers.\n         for sig in (signal.SIGHUP, signal.SIGTERM, signal.SIGKILL):  # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)\n             if not self._proc.isalive():\n                 break\n             try:\n-                self._proc.kill(sig)\n+                if pgid is not None:\n+                    os.killpg(pgid, sig)  # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)\n+                else:\n+                    self._proc.kill(sig)\n             except Exception:\n                 pass\n             deadline = time.monotonic() + 0.5"]},{"file":"scripts/release.py","hunks":["@@ -1260,6 +1260,7 @@\n     \"[redacted:email]\": \"LeonSGP43\",  # PR #18739 salvage of #14570\n     \"[redacted:email]\": \"Foolafroos\",  # PR #20329 French locale\n     \"[redacted:email]\": \"Montbra\",  # PR #20897 salvage of #16189 (TUI voice PTT)\n+    \"[redacted:email]\": \"paulb26\",  # PR #24135 salvage (pty-bridge killpg)\n     \"[redacted:email]\": \"firefly\",  # PR #18123 salvage of #16660 (ContextVars)\n     \"[redacted:email]\": \"WuTianyi123\",  # PR #20275 salvage of #13723 (feishu markdown)\n     \"[redacted:email]\": \"hanzckernel\",  # PR #20311 (api-server approval events)"]},{"file":"tests/hermes_cli/test_pty_bridge.py","hunks":["@@ -8,6 +8,7 @@\n \n import os\n import shutil\n+import signal\n import sys\n import time\n ","@@ -211,6 +212,75 @@ def test_close_terminates_long_running_child(self):\n                 break\n         assert reaped, f\"pid {pid} still running after close()\"\n \n+    def test_close_signals_child_process_group(self, monkeypatch):\n+        sent: list[tuple[int, signal.Signals]] = []\n+\n+        class _FakeProc:\n+            pid = 12345\n+            fd = -1\n+\n+            def __init__(self):\n+                self.alive = True\n+\n+            def isalive(self):\n+                return self.alive\n+\n+            def kill(self, sig):\n+                raise AssertionError(f\"single-process kill used: {sig}\")\n+\n+            def close(self, force=False):\n+                self.closed = force\n+\n+        fake = _FakeProc()\n+\n+        def fake_killpg(pgid, sig):\n+            sent.append((pgid, sig))\n+            fake.alive = False\n+\n+        monkeypatch.setattr(os, \"getpgid\", lambda pid: 67890)\n+        monkeypatch.setattr(os, \"killpg\", fake_killpg)\n+\n+        bridge = PtyBridge.__new__(PtyBridge)\n+        bridge._proc = fake\n+        bridge._fd = -1\n+        bridge._closed = False\n+\n+        bridge.close()\n+\n+        assert sent == [(67890, signal.SIGHUP)]\n+        assert bridge._closed is True\n+\n+    def test_close_falls_back_to_single_process_signal_when_group_unknown(self, monkeypatch):\n+        sent: list[signal.Signals] = []\n+\n+        class _FakeProc:\n+            pid = 12345\n+            fd = -1\n+\n+            def __init__(self):\n+                self.alive = True\n+\n+            def isalive(self):\n+                return self.alive\n+\n+            def kill(self, sig):\n+                sent.append(sig)\n+                self.alive = False\n+\n+            def close(self, force=False):\n+                self.closed = force\n+\n+        monkeypatch.setattr(os, \"getpgid\", lambda pid: (_ for _ in ()).throw(OSError()))\n+\n+        bridge = PtyBridge.__new__(PtyBridge)\n+        bridge._proc = _FakeProc()\n+        bridge._fd = -1\n+        bridge._closed = False\n+\n+        bridge.close()\n+\n+        assert sent == [signal.SIGHUP]\n+\n \n @skip_on_windows\n class TestPtyBridgeEnv:"]},{"file":"tests/test_slash_worker_watchdog.py","hunks":["@@ -0,0 +1,21 @@\n+import psutil\n+\n+from tui_gateway import slash_worker\n+\n+\n+def test_is_orphaned_true_when_ppid_changes():\n+    # Our parent went away and we were reparented to a subreaper/init.\n+    assert slash_worker._is_orphaned(1234, 1.0, getppid=lambda: 999999) is True\n+\n+\n+def test_is_orphaned_true_when_parent_create_time_mismatch():\n+    # Same ppid but a different create_time means the PID was reused.\n+    me = psutil.Process()\n+    assert slash_worker._is_orphaned(me.pid, 0.0, getppid=lambda: me.pid) is True\n+\n+\n+def test_is_orphaned_false_when_parent_alive_and_matches():\n+    me = psutil.Process()\n+    assert (\n+        slash_worker._is_orphaned(me.pid, me.create_time(), getppid=lambda: me.pid) is False\n+    )"]},{"file":"tui_gateway/slash_worker.py","hunks":["@@ -9,11 +9,46 @@\n import json\n import os\n import sys\n+import threading\n+import time\n+\n+import psutil\n \n import cli as cli_mod\n from cli import HermesCLI\n from rich.console import Console\n \n+# Env-overridable so the integration test can drive sub-second timing.\n+_WATCHDOG_POLL_S = float(os.environ.get(\"HERMES_SLASH_WATCHDOG_POLL_S\", 2.0))\n+_ORPHAN_GRACE_S = float(os.environ.get(\"HERMES_SLASH_WATCHDOG_GRACE_S\", 5.0))\n+_in_flight = threading.Event()  # set while a command is executing\n+\n+\n+def _is_orphaned(original_ppid, parent_create_time, getppid=os.getppid) -> bool:\n+    \"\"\"True once our spawning gateway is gone. Compare to the ORIGINAL ppid\n+    (never ==1: Linux reparents to a subreaper) and guard PID reuse via\n+    create_time.\"\"\"\n+    if getppid() != original_ppid:\n+        return True\n+    try:\n+        if not psutil.pid_exists(original_ppid):\n+            return True\n+        return psutil.Process(original_ppid).create_time() != parent_create_time\n+    except psutil.Error:\n+        return True\n+\n+\n+def _start_parent_death_watchdog(original_ppid, parent_create_time) -> None:\n+    def _loop():\n+        while not _is_orphaned(original_ppid, parent_create_time):\n+            time.sleep(_WATCHDOG_POLL_S)\n+        deadline = time.monotonic() + _ORPHAN_GRACE_S\n+        while _in_flight.is_set() and time.monotonic() < deadline:\n+            time.sleep(0.05)  # let an in-flight command finish/flush\n+        os._exit(0)\n+\n+    threading.Thread(target=_loop, daemon=True).start()\n+\n \n def _run(cli: HermesCLI, command: str) -> str:\n     cmd = (command or \"\").strip()","@@ -52,6 +87,15 @@ def main():\n     os.environ[\"HERMES_SESSION_KEY\"] = args.session_key\n     os.environ[\"HERMES_INTERACTIVE\"] = \"1\"\n \n+    # Start before the (hundreds-of-ms) HermesCLI build — that window is itself\n+    # an orphan risk if the gateway dies mid-spawn.\n+    orig_ppid = os.getppid()\n+    try:\n+        parent_create_time = psutil.Process(orig_ppid).create_time()\n+    except psutil.Error:\n+        parent_create_time = 0.0\n+    _start_parent_death_watchdog(orig_ppid, parent_create_time)\n+\n     with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):\n         cli = HermesCLI(model=args.model or None, compact=True, resume=args.session_key, verbose=False)\n ","@@ -60,6 +104,7 @@ def main():\n         if not line:\n             continue\n \n+        _in_flight.set()\n         rid = None\n         try:\n             req = json.loads(line)","@@ -70,6 +115,8 @@ def main():\n         except Exception as e:\n             sys.stdout.write(json.dumps({\"id\": rid, \"ok\": False, \"error\": str(e)}) + \"\\n\")\n             sys.stdout.flush()\n+        finally:\n+            _in_flight.clear()\n \n \n if __name__ == \"__main__\":\n"]}]},{"kind":"pr","number":42149,"title":"fix(tui): close slash_worker inside _finalize_session (defense-in-depth, #38095)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/test_tui_gateway_server.py","tui_gateway/server.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}