TUI slash_worker subprocess leak and zombie cleanup

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #42132 fix(tui/pty): reap leaked slash_worker subprocesses on PTY chat disconnect

  merged to main

  files: hermes_cli/pty_bridge.py, scripts/release.py, tests/hermes_cli/test_pty_bridge.py, tests/test_slash_worker_watchdog.py, tui_gateway/slash_worker.py

  Show the diff

  • hermes_cli/pty_bridge.py

    @@ -250,13 +250,23 @@ def close(self) -> None:
                 return
             self._closed = True
     
    +        try:
    +            pgid = os.getpgid(self._proc.pid)  # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
    +        except Exception:
    +            pgid = None
    +
             # SIGHUP is the conventional "your terminal went away" signal.
    -        # We escalate if the child ignores it.
    +        # Send it to the whole foreground process group, not just the PTY
    +        # leader: the dashboard TUI starts helper children such as the Python
    +        # slash worker, and killing only the leader can strand those helpers.
             for sig in (signal.SIGHUP, signal.SIGTERM, signal.SIGKILL):  # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
                 if not self._proc.isalive():
                     break
                 try:
    -                self._proc.kill(sig)
    +                if pgid is not None:
    +                    os.killpg(pgid, sig)  # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
    +                else:
    +                    self._proc.kill(sig)
                 except Exception:
                     pass
                 deadline = time.monotonic() + 0.5

  • scripts/release.py

    @@ -1260,6 +1260,7 @@
         "[redacted:email]": "LeonSGP43",  # PR #18739 salvage of #14570
         "[redacted:email]": "Foolafroos",  # PR #20329 French locale
         "[redacted:email]": "Montbra",  # PR #20897 salvage of #16189 (TUI voice PTT)
    +    "[redacted:email]": "paulb26",  # PR #24135 salvage (pty-bridge killpg)
         "[redacted:email]": "firefly",  # PR #18123 salvage of #16660 (ContextVars)
         "[redacted:email]": "WuTianyi123",  # PR #20275 salvage of #13723 (feishu markdown)
         "[redacted:email]": "hanzckernel",  # PR #20311 (api-server approval events)

  • tests/hermes_cli/test_pty_bridge.py

    @@ -8,6 +8,7 @@
     
     import os
     import shutil
    +import signal
     import sys
     import time
     
    @@ -211,6 +212,75 @@ def test_close_terminates_long_running_child(self):
                     break
             assert reaped, f"pid {pid} still running after close()"
     
    +    def test_close_signals_child_process_group(self, monkeypatch):
    +        sent: list[tuple[int, signal.Signals]] = []
    +
    +        class _FakeProc:
    +            pid = 12345
    +            fd = -1
    +
    +            def __init__(self):
    +                self.alive = True
    +
    +            def isalive(self):
    +                return self.alive
    +
    +            def kill(self, sig):
    +                raise AssertionError(f"single-process kill used: {sig}")
    +
    +            def close(self, force=False):
    +                self.closed = force
    +
    +        fake = _FakeProc()
    +
    +        def fake_killpg(pgid, sig):
    +            sent.append((pgid, sig))
    +            fake.alive = False
    +
    +        monkeypatch.setattr(os, "getpgid", lambda pid: 67890)
    +        monkeypatch.setattr(os, "killpg", fake_killpg)
    +
    +        bridge = PtyBridge.__new__(PtyBridge)
    +        bridge._proc = fake
    +        bridge._fd = -1
    +        bridge._closed = False
    +
    +        bridge.close()
    +
    +        assert sent == [(67890, signal.SIGHUP)]
    +        assert bridge._closed is True
    +
    +    def test_close_falls_back_to_single_process_signal_when_group_unknown(self, monkeypatch):
    +        sent: list[signal.Signals] = []
    +
    +        class _FakeProc:
    +            pid = 12345
    +            fd = -1
    +
    +            def __init__(self):
    +                self.alive = True
    +
    +            def isalive(self):
    +                return self.alive
    +
    +            def kill(self, sig):
    +                sent.append(sig)
    +                self.alive = False
    +
    +            def close(self, force=False):
    +                self.closed = force
    +
    +        monkeypatch.setattr(os, "getpgid", lambda pid: (_ for _ in ()).throw(OSError()))
    +
    +        bridge = PtyBridge.__new__(PtyBridge)
    +        bridge._proc = _FakeProc()
    +        bridge._fd = -1
    +        bridge._closed = False
    +
    +        bridge.close()
    +
    +        assert sent == [signal.SIGHUP]
    +
     
     @skip_on_windows
     class TestPtyBridgeEnv:

  • tests/test_slash_worker_watchdog.py

    @@ -0,0 +1,21 @@
    +import psutil
    +
    +from tui_gateway import slash_worker
    +
    +
    +def test_is_orphaned_true_when_ppid_changes():
    +    # Our parent went away and we were reparented to a subreaper/init.
    +    assert slash_worker._is_orphaned(1234, 1.0, getppid=lambda: 999999) is True
    +
    +
    +def test_is_orphaned_true_when_parent_create_time_mismatch():
    +    # Same ppid but a different create_time means the PID was reused.
    +    me = psutil.Process()
    +    assert slash_worker._is_orphaned(me.pid, 0.0, getppid=lambda: me.pid) is True
    +
    +
    +def test_is_orphaned_false_when_parent_alive_and_matches():
    +    me = psutil.Process()
    +    assert (
    +        slash_worker._is_orphaned(me.pid, me.create_time(), getppid=lambda: me.pid) is False
    +    )

  • tui_gateway/slash_worker.py

    @@ -9,11 +9,46 @@
     import json
     import os
     import sys
    +import threading
    +import time
    +
    +import psutil
     
     import cli as cli_mod
     from cli import HermesCLI
     from rich.console import Console
     
    +# Env-overridable so the integration test can drive sub-second timing.
    +_WATCHDOG_POLL_S = float(os.environ.get("HERMES_SLASH_WATCHDOG_POLL_S", 2.0))
    +_ORPHAN_GRACE_S = float(os.environ.get("HERMES_SLASH_WATCHDOG_GRACE_S", 5.0))
    +_in_flight = threading.Event()  # set while a command is executing
    +
    +
    +def _is_orphaned(original_ppid, parent_create_time, getppid=os.getppid) -> bool:
    +    """True once our spawning gateway is gone. Compare to the ORIGINAL ppid
    +    (never ==1: Linux reparents to a subreaper) and guard PID reuse via
    +    create_time."""
    +    if getppid() != original_ppid:
    +        return True
    +    try:
    +        if not psutil.pid_exists(original_ppid):
    +            return True
    +        return psutil.Process(original_ppid).create_time() != parent_create_time
    +    except psutil.Error:
    +        return True
    +
    +
    +def _start_parent_death_watchdog(original_ppid, parent_create_time) -> None:
    +    def _loop():
    +        while not _is_orphaned(original_ppid, parent_create_time):
    +            time.sleep(_WATCHDOG_POLL_S)
    +        deadline = time.monotonic() + _ORPHAN_GRACE_S
    +        while _in_flight.is_set() and time.monotonic() < deadline:
    +            time.sleep(0.05)  # let an in-flight command finish/flush
    +        os._exit(0)
    +
    +    threading.Thread(target=_loop, daemon=True).start()
    +
     
     def _run(cli: HermesCLI, command: str) -> str:
         cmd = (command or "").strip()
    @@ -52,6 +87,15 @@ def main():
         os.environ["HERMES_SESSION_KEY"] = args.session_key
         os.environ["HERMES_INTERACTIVE"] = "1"
     
    +    # Start before the (hundreds-of-ms) HermesCLI build — that window is itself
    +    # an orphan risk if the gateway dies mid-spawn.
    +    orig_ppid = os.getppid()
    +    try:
    +        parent_create_time = psutil.Process(orig_ppid).create_time()
    +    except psutil.Error:
    +        parent_create_time = 0.0
    +    _start_parent_death_watchdog(orig_ppid, parent_create_time)
    +
         with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
             cli = HermesCLI(model=args.model or None, compact=True, resume=args.session_key, verbose=False)
     
    @@ -60,6 +104,7 @@ def main():
             if not line:
                 continue
     
    +        _in_flight.set()
             rid = None
             try:
                 req = json.loads(line)
    @@ -70,6 +115,8 @@ def main():
             except Exception as e:
                 sys.stdout.write(json.dumps({"id": rid, "ok": False, "error": str(e)}) + "\n")
                 sys.stdout.flush()
    +        finally:
    +            _in_flight.clear()
     
     
     if __name__ == "__main__":

• #42149 fix(tui): close slash_worker inside _finalize_session (defense-in-depth, #38095)

  merged to main

  files: tests/test_tui_gateway_server.py, tui_gateway/server.py