{"repo":"NousResearch/hermes-agent","public_id":"20015","display_label":"Kanban zombie process detection and session lease fixes","summary":"These issues and pull requests address zombie worker processes (particularly on macOS/Darwin) that remain detectable as alive, causing session leases to be held indefinitely and blocking new worker spawns. The work centers on improving _pid_alive to correctly identify defunct processes and ensuring SIGTERM properly terminates workers.","subsystem":"cron","counts":{"members":7,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":3,"merged_prs":2,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":20188},"resolution":{"state":"partially_resolved","by":"pr-20188,pr-34045"},"freshness":{"state":"verified","last_verified_at":"2026-05-28T18:59:58+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.22476190476190475},"fix_edges":[{"issue_number":20015,"pr_number":20188,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":28181,"pr_number":34045,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":20015,"title":"kanban dispatcher: macOS zombie detection is a no-op — _pid_alive returns True for defunct workers","url":null,"state":"closed"},{"kind":"issue","number":28181,"title":"[Bug]: Kanban workers stuck in zombie state after SIGTERM — claim never released, task blocked forever","url":null,"state":"closed"},{"kind":"issue","number":55444,"title":"fix(kanban): zombie processes hold session leases forever, blocking worker spawn","url":null,"state":"closed"},{"kind":"pr","number":20023,"title":"fix(kanban): detect darwin zombie workers","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#20188","number":20188}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":20188,"title":"fix(kanban): detect darwin zombie workers (salvages #20023)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/kanban_db.py","tests/hermes_cli/test_kanban_core_functionality.py"],"fix_hunks":[{"file":"hermes_cli/kanban_db.py","hunks":["@@ -76,6 +76,7 @@\n import re\n import secrets\n import sqlite3\n+import subprocess\n import sys\n import time\n from dataclasses import dataclass, field","@@ -2141,16 +2142,16 @@ def _pid_alive(pid: Optional[int]) -> bool:\n     Cross-platform: uses ``os.kill(pid, 0)`` on POSIX and ``OpenProcess``\n     on Windows. Returns False for falsy PIDs or on any OS error.\n \n-    **Zombie handling (Linux):** ``os.kill(pid, 0)`` succeeds against\n+    **Zombie handling:** ``os.kill(pid, 0)`` succeeds against\n     zombie processes (post-exit, pre-reap) because the process table\n     entry still exists. A worker that exits without being reaped by its\n     parent would stay \"alive\" to the dispatcher forever. Dispatcher\n     workers are started via ``start_new_session=True`` + intentional\n     Popen handle abandonment, so init reaps them quickly — but during\n     the window between exit and reap, we'd otherwise see stale \"alive\"\n-    signals. On Linux we additionally peek at ``/proc/<pid>/status``\n-    and treat ``State: Z`` as dead. On other POSIX or on Windows the\n-    zombie check is a no-op.\n+    signals. On Linux we peek at ``/proc/<pid>/status`` and treat\n+    ``State: Z`` as dead. On macOS we ask ``ps`` for the BSD ``stat``\n+    field and treat values containing ``Z`` as dead.\n     \"\"\"\n     if not pid or pid <= 0:\n         return False","@@ -2164,7 +2165,8 @@ def _pid_alive(pid: Optional[int]) -> bool:\n         return True\n     except OSError:\n         return False\n-    # Still here → kill(0) succeeded. Check for zombie on Linux.\n+    # Still here → kill(0) succeeded. Check for zombie on platforms\n+    # where we have a cheap, deterministic process-state probe.\n     if sys.platform == \"linux\":\n         try:\n             with open(f\"/proc/{int(pid)}/status\", \"r\") as f:","@@ -2179,6 +2181,23 @@ def _pid_alive(pid: Optional[int]) -> bool:\n             # PermissionError shouldn't happen for our own children but\n             # be defensive.\n             pass\n+    elif sys.platform == \"darwin\":\n+        try:\n+            proc = subprocess.run(\n+                [\"ps\", \"-o\", \"stat=\", \"-p\", str(int(pid))],\n+                stdout=subprocess.PIPE,\n+                stderr=subprocess.DEVNULL,\n+                text=True,\n+                timeout=1,\n+                check=False,\n+            )\n+            if proc.returncode != 0:\n+                return False\n+            if \"Z\" in (proc.stdout or \"\").strip():\n+                return False\n+        except (OSError, subprocess.SubprocessError, TimeoutError):\n+            # If the secondary probe fails, keep the kill(0) answer.\n+            pass\n     return True\n \n "]},{"file":"tests/hermes_cli/test_kanban_core_functionality.py","hunks":["@@ -13,9 +13,11 @@\n import argparse\n import json\n import os\n+import subprocess\n import threading\n import time\n from pathlib import Path\n+from types import SimpleNamespace\n from typing import Optional\n \n import pytest","@@ -183,6 +185,20 @@ def test_pid_alive_helper():\n     assert not kb._pid_alive(2 ** 30)\n \n \n+def test_pid_alive_detects_darwin_zombie(monkeypatch):\n+    monkeypatch.setattr(kb.sys, \"platform\", \"darwin\")\n+    monkeypatch.setattr(kb.os, \"kill\", lambda pid, sig: None)\n+\n+    def fake_run(args, **kwargs):\n+        assert args == [\"ps\", \"-o\", \"stat=\", \"-p\", \"123\"]\n+        assert kwargs[\"stdout\"] is subprocess.PIPE\n+        return SimpleNamespace(returncode=0, stdout=\"Z+\\n\")\n+\n+    monkeypatch.setattr(kb.subprocess, \"run\", fake_run)\n+\n+    assert kb._pid_alive(123) is False\n+\n+\n def test_detect_crashed_workers_reclaims(kanban_home):\n     \"\"\"A running task whose pid vanished gets dropped to ready with a\n     ``crashed`` event, independent of the claim TTL.\"\"\"\n"]}]},{"kind":"pr","number":34045,"title":"fix(kanban): SIGTERM on worker must terminate the process (closes #28181)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cli.py","tests/hermes_cli/test_signal_handler_kanban_worker.py"],"fix_hunks":[{"file":"cli.py","hunks":["@@ -14968,6 +14968,39 @@ def _signal_handler_q(signum, frame):\n                     time.sleep(_grace)\n         except Exception:\n             pass  # never block signal handling\n+        # Kanban worker exit path (#28181): SIGTERM hits a dispatcher-spawned\n+        # worker that's likely in a non-daemon thread waiting on a child\n+        # subprocess in _wait_for_process. Raising KeyboardInterrupt only\n+        # unwinds the main thread; the worker thread keeps running, the\n+        # process gets reparented to init, and the dispatcher's _pid_alive\n+        # check returns True forever — task stuck in 'running' indefinitely.\n+        # Skip the controlled-unwind dance and call os._exit(0) so the kernel\n+        # reclaims the PID immediately and detect_crashed_workers can reclaim\n+        # the stale claim on the next tick. Flush logging + stdout/stderr\n+        # first so the final debug trace isn't lost; SIGALRM deadman guards\n+        # the flush against any rare blocking-I/O case (the reporter measured\n+        # flush in <1ms; the alarm is a failsafe, not the common path).\n+        if os.environ.get(\"HERMES_KANBAN_TASK\"):\n+            try:\n+                import signal as _sig_mod\n+                if hasattr(_sig_mod, \"SIGALRM\"):\n+                    # Cancel any pre-existing alarm to avoid colliding with\n+                    # caller-installed timers.\n+                    _sig_mod.signal(_sig_mod.SIGALRM, lambda *_: os._exit(0))\n+                    _sig_mod.alarm(2)\n+            except Exception:\n+                pass\n+            try:\n+                import logging as _lg\n+                _lg.shutdown()\n+            except Exception:\n+                pass\n+            for _stream in (sys.stdout, sys.stderr):\n+                try:\n+                    _stream.flush()\n+                except Exception:\n+                    pass\n+            os._exit(0)\n         raise KeyboardInterrupt()\n     try:\n         import signal as _signal"]},{"file":"tests/hermes_cli/test_signal_handler_kanban_worker.py","hunks":["@@ -0,0 +1,230 @@\n+\"\"\"Regression test for #28181 — kanban worker SIGTERM must terminate the process.\n+\n+The single-query signal handler in cli.py (``_signal_handler_q``) raises\n+``KeyboardInterrupt`` to unwind the main thread on SIGTERM/SIGHUP. That works\n+for interactive ``hermes chat -q`` invocations, but kanban workers spawned by\n+the dispatcher are likely to have a non-daemon thread alive (terminal_tool's\n+``_wait_for_process``, custom plugin background workers, etc.). With\n+``KeyboardInterrupt`` only the main thread unwinds; the non-daemon thread\n+keeps the process alive after the gateway has already restarted, the kanban\n+dispatcher's ``_pid_alive`` check returns True forever, and the task stays\n+``running`` indefinitely.\n+\n+The fix: when the process is a dispatcher-spawned worker (``HERMES_KANBAN_TASK``\n+env var set), flush logging + stdout/stderr and call ``os._exit(0)`` instead.\n+The kernel reclaims the PID immediately, and ``detect_crashed_workers``\n+reclaims the stale claim on the next dispatcher tick.\n+\n+These tests use a synthetic Python script that mirrors the cli.py signal\n+handler shape so we can exercise the exit-path contract without booting the\n+full CLI (which needs a real provider config).\n+\"\"\"\n+from __future__ import annotations\n+\n+import os\n+import signal\n+import subprocess\n+import sys\n+import textwrap\n+import time\n+\n+import pytest\n+\n+\n+def _synthetic_worker_script() -> str:\n+    \"\"\"A standalone script that mirrors cli.py's single-query SIGTERM handler.\n+\n+    Keeping the synthetic copy here means the test exercises the exact handler\n+    shape without needing the full hermes_cli boot path (config, providers,\n+    skills, etc.). If the production handler in cli.py drifts, the test\n+    that loads the real handler (test_real_handler_uses_os_exit) will catch it.\n+    \"\"\"\n+    return textwrap.dedent(\n+        \"\"\"\n+        import os, signal, sys, threading, time\n+\n+        # Non-daemon thread that blocks forever — simulates the worker\n+        # thread that would prevent orderly Python shutdown after\n+        # KeyboardInterrupt unwinds main.\n+        stuck = threading.Event()\n+        threading.Thread(target=stuck.wait, daemon=False).start()\n+\n+        def handler(signum, frame):\n+            # Mirrors cli.py:_signal_handler_q. Real handler sleeps 1.5s; the\n+            # test uses a short grace so it runs fast.\n+            try:\n+                time.sleep(0.05)\n+            except Exception:\n+                pass\n+            if os.environ.get(\"HERMES_KANBAN_TASK\"):\n+                try:\n+                    if hasattr(signal, \"SIGALRM\"):\n+                        signal.signal(signal.SIGALRM, lambda *_: os._exit(0))\n+                        signal.alarm(2)\n+                except Exception:\n+                    pass\n+                sys.stdout.flush()\n+                sys.stderr.flush()\n+                os._exit(0)\n+            raise KeyboardInterrupt()\n+\n+        signal.signal(signal.SIGTERM, handler)\n+        print(\"READY\", flush=True)\n+        try:\n+            threading.Event().wait()\n+        except KeyboardInterrupt:\n+            sys.exit(0)\n+        \"\"\"\n+    )\n+\n+\n+def _is_alive_like_dispatcher(pid: int) -> bool:\n+    \"\"\"Mirrors hermes_cli/kanban_db.py:_pid_alive on Linux.\n+\n+    A zombie is treated as dead — the dispatcher's _pid_alive checks\n+    /proc/<pid>/status for State: Z. We replicate that here so a clean\n+    os._exit followed by zombie-state is correctly counted as dead.\n+    \"\"\"\n+    if pid <= 0:\n+        return False\n+    try:\n+        os.kill(pid, 0)\n+    except ProcessLookupError:\n+        return False\n+    except PermissionError:\n+        return True\n+    if sys.platform == \"linux\":\n+        try:\n+            with open(f\"/proc/{pid}/status\") as f:\n+                for line in f:\n+                    if line.startswith(\"State:\"):\n+                        if \"Z\" in line.split(\":\", 1)[1]:\n+                            return False\n+                        break\n+        except (FileNotFoundError, PermissionError, OSError):\n+            pass\n+    return True\n+\n+\n+def _spawn_synthetic(env_overrides: dict) -> subprocess.Popen:\n+    env = dict(os.environ)\n+    env.update(env_overrides)\n+    proc = subprocess.Popen(\n+        [sys.executable, \"-u\", \"-c\", _synthetic_worker_script()],\n+        env=env,\n+        stdout=subprocess.PIPE,\n+        stderr=subprocess.PIPE,\n+        start_new_session=True,\n+    )\n+    # Wait for \"READY\" so we know the signal handler is installed.\n+    assert proc.stdout is not None\n+    deadline = time.time() + 5.0\n+    while time.time() < deadline:\n+        line = proc.stdout.readline()\n+        if line and line.startswith(b\"READY\"):\n+            return proc\n+    proc.kill()\n+    raise RuntimeError(\"synthetic worker never signalled READY\")\n+\n+\n+def _cleanup(proc: subprocess.Popen) -> None:\n+    try:\n+        os.killpg(os.getpgid(proc.pid), signal.SIGKILL)\n+    except (ProcessLookupError, PermissionError):\n+        pass\n+    try:\n+        proc.communicate(timeout=2)\n+    except subprocess.TimeoutExpired:\n+        proc.kill()\n+\n+\n[redacted:email](\n+    sys.platform == \"win32\",\n+    reason=\"SIGTERM semantics differ on Windows; kanban dispatcher is POSIX-only\",\n+)\n+def test_sigterm_with_kanban_task_env_terminates_quickly():\n+    \"\"\"With HERMES_KANBAN_TASK set, SIGTERM should kill the process in <2s\n+    even when a non-daemon thread is still alive.\"\"\"\n+    proc = _spawn_synthetic({\"HERMES_KANBAN_TASK\": \"t_test_28181\"})\n+    try:\n+        t0 = time.time()\n+        os.kill(proc.pid, signal.SIGTERM)\n+\n+        # Should die in <2s. The handler sleeps ~50ms, then os._exit(0)\n+        # is immediate. Give generous headroom for slow CI runners.\n+        deadline = t0 + 2.0\n+        while time.time() < deadline:\n+            if not _is_alive_like_dispatcher(proc.pid):\n+                elapsed = time.time() - t0\n+                assert elapsed < 2.0\n+                return\n+            time.sleep(0.02)\n+        pytest.fail(\n+            f\"process still alive 2s after SIGTERM with HERMES_KANBAN_TASK set \"\n+            f\"(dispatcher would keep extending claim) — fix regressed\"\n+        )\n+    finally:\n+        _cleanup(proc)\n+\n+\n[redacted:email](\n+    sys.platform == \"win32\",\n+    reason=\"SIGTERM semantics differ on Windows; kanban dispatcher is POSIX-only\",\n+)\n+def test_sigterm_without_kanban_task_env_uses_keyboard_interrupt_path():\n+    \"\"\"Without HERMES_KANBAN_TASK, the original KeyboardInterrupt path runs.\n+\n+    This is the contrast case proving the fix is gated on the env var: in\n+    interactive ``hermes chat -q`` (no env var), behavior is unchanged. The\n+    process MAY hang under non-daemon threads, but that's not a kanban-worker\n+    concern. We just verify the handler logs the KeyboardInterrupt branch\n+    rather than os._exit'ing.\n+    \"\"\"\n+    proc = _spawn_synthetic({})\n+    try:\n+        os.kill(proc.pid, signal.SIGTERM)\n+        # Wait a moment for the handler to react.\n+        time.sleep(0.5)\n+        # The process may or may not be dead depending on whether the\n+        # KeyboardInterrupt unwinds cleanly. The behavioral guarantee is\n+        # only that the env-gated path didn't fire.\n+        try:\n+            # Drain stdout up to whatever's available.\n+            if proc.stdout is not None:\n+                proc.stdout.close()\n+            if proc.stderr is not None:\n+                proc.stderr.close()\n+        except Exception:\n+            pass\n+    finally:\n+        _cleanup(proc)\n+\n+\n+def test_real_handler_uses_os_exit_for_kanban_workers():\n+    \"\"\"Source-level invariant: cli.py's _signal_handler_q must call\n+    os._exit(0) when HERMES_KANBAN_TASK is set.\n+\n+    Catches the case where someone refactors the handler and accidentally\n+    drops the env-gated exit, restoring the bug. Reading cli.py directly is\n+    cheap and avoids the heavy CLI import.\n+    \"\"\"\n+    import pathlib\n+\n+    cli_path = (\n+        pathlib.Path(__file__).resolve().parent.parent.parent / \"cli.py\"\n+    )\n+    src = cli_path.read_text()\n+    # Locate the handler body.\n+    start = src.find(\"def _signal_handler_q(signum, frame):\")\n+    assert start != -1, \"cli.py is missing _signal_handler_q\"\n+    # Look ahead for the env-gated os._exit call within ~80 lines.\n+    body = src[start : start + 4000]\n+    assert \"HERMES_KANBAN_TASK\" in body, (\n+        \"_signal_handler_q must gate its kanban-worker exit path on \"\n+        \"HERMES_KANBAN_TASK — see #28181\"\n+    )\n+    assert \"os._exit(0)\" in body, (\n+        \"_signal_handler_q must call os._exit(0) for kanban workers — \"\n+        \"raising KeyboardInterrupt orphans the process when non-daemon \"\n+        \"threads are alive (see #28181)\"\n+    )\n"]}]},{"kind":"pr","number":55533,"title":"fix(active-sessions): reject zombie PIDs in _pid_alive to prevent session lease deadlocks","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"acca526286db53b03c3a8d820171faebd6c8b987","number":0},{"kind":"pr","value":"#55511","number":55511}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true}],"trust":{"default_tier":"verified","includes_suggested":false}}