Fix orphaned gateway process on WSL without systemd

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #51678 fix(gateway): track + reap no-systemd gateway restart runtimes (WSL orphan)

  merged to main

  files: gateway/status.py, hermes_cli/gateway.py, scripts/release.py, tests/gateway/test_gateway_command_line_matcher.py, tests/gateway/test_status.py, tests/hermes_cli/test_gateway.py, tests/hermes_cli/test_gateway_proc_fallback.py

  Show the diff

  • gateway/status.py

    @@ -190,8 +190,8 @@ def _read_process_cmdline(pid: int) -> Optional[str]:
         return None
     
     
    -def looks_like_gateway_command_line(command: str | None) -> bool:
    -    """Return True only for a real ``gateway run`` process command line.
    +def _gateway_command_subcommand(command: str | None) -> str | None:
    +    """Return the Hermes gateway lifecycle subcommand from a command line.
     
         Lifecycle decisions (is the gateway up? did restart relaunch it?) must not
         fire on loose substring matches.  The previous ``"... gateway" in cmdline``
    @@ -211,7 +211,7 @@ def looks_like_gateway_command_line(command: str | None) -> bool:
         either side of the ``gateway`` subcommand.
         """
         if not command:
    -        return False
    +        return None
     
         try:
             raw_tokens = [redacted:secret-assignment] posix=False)
    @@ -220,15 +220,15 @@ def looks_like_gateway_command_line(command: str | None) -> bool:
         # Strip surrounding quotes, normalize slashes + case per token.
         tokens = [redacted:secret-assignment]"\"'").replace("\\", "/").lower() for t in raw_tokens]
         if not tokens:
    -        return False
    +        return None
     
         # Gateway-dedicated entrypoints carry no subcommand to inspect.
         for token in tokens:
             if token == "gateway/run.py" or token.endswith("/gateway/run.py"):
    -            return True
    +            return "run"
             basename = token.rsplit("/", 1)[-1]
             if basename in ("hermes-gateway", "hermes-gateway.exe"):
    -            return True
    +            return "run"
     
         joined = " ".join(tokens)
         has_gateway_entry = (
    @@ -237,7 +237,7 @@ def looks_like_gateway_command_line(command: str | None) -> bool:
             or any(t.rsplit("/", 1)[-1] in ("hermes", "hermes.exe") for t in tokens)
         )
         if not has_gateway_entry:
    -        return False
    +        return None
     
         # Drop profile selectors anywhere: --profile X / -p X / --profile=X / -p=X.
         # This consumes a profile VALUE of "gateway" too, so the real subcommand
    @@ -259,9 +259,28 @@ def looks_like_gateway_command_line(command: str | None) -> bool:
             if token != "gateway":
                 continue
             if i + 1 >= len(filtered):
    -            return True  # bare `hermes gateway` defaults to `run`
    -        return filtered[i + 1] == "run"
    -    return False
    +            return "run"  # bare `hermes gateway` defaults to `run`
    +        return filtered[i + 1]
    +    return None
    +
    +
    +def looks_like_gateway_command_line(command: str | None) -> bool:
    +    """Return True only for a real ``gateway run`` process command line."""
    +    return _gateway_command_subcommand(command) == "run"
    +
    +
    +def looks_like_gateway_runtime_command_line(command: str | None) -> bool:
    +    """Return True for command lines that can host the gateway runtime.
    +
    +    ``gateway restart`` is normally a management command, not the gateway
    +    runtime. On hosts without a service manager, though, the manual restart
    +    fallback executes ``run_gateway()`` in that same process, so its argv stays
    +    as ``gateway restart`` while it owns the webhook port and writes runtime
    +    state. Keep the public ``looks_like_gateway_command_line()`` strict, and
    +    use this broader matcher only when validating Hermes-owned runtime records
    +    or no-supervisor cleanup scans.
    +    """
    +    return _gateway_command_subcommand(command) in {"run", "restart"}
     
     
     def _looks_like_gateway_process(pid: int) -> bool:
    @@ -282,7 +301,7 @@ def _record_looks_like_gateway(record: dict[str, Any]) -> bool:
             return False
     
         cmdline = " ".join(str(part) for part in argv)
    -    return looks_like_gateway_command_line(cmdline)
    +    return looks_like_gateway_runtime_command_line(cmdline)
     
     
     def _build_pid_record() -> dict:
    @@ -1213,6 +1232,10 @@ def get_running_pid(
         resolved_lock_path = _get_gateway_lock_path(resolved_pid_path)
         lock_active = is_gateway_runtime_lock_active(resolved_lock_path)
         if not lock_active:
    +        if pid_path is None:
    +            runtime_pid = get_runtime_status_running_pid()
    +            if runtime_pid is not None:
    +                return runtime_pid
             _cleanup_invalid_pid_path(resolved_pid_path, cleanup_stale=cleanup_stale)
             return None
     
    @@ -1236,6 +1259,10 @@ def get_running_pid(
                 return pid
     
         _cleanup_invalid_pid_path(resolved_pid_path, cleanup_stale=cleanup_stale)
    +    if pid_path is None:
    +        runtime_pid = get_runtime_status_running_pid()
    +        if runtime_pid is not None:
    +            return runtime_pid
         return None
     
     

  • hermes_cli/gateway.py

    @@ -13,6 +13,7 @@
     import subprocess
     import sys
     import textwrap
    +import time
     from dataclasses import dataclass
     from pathlib import Path
     
    @@ -308,7 +309,11 @@ def _append_unique_pid(
         pids.append(pid)
     
     
    -def _scan_gateway_pids(exclude_pids: set[int], all_profiles: bool = False) -> list[int]:
    +def _scan_gateway_pids(
    +    exclude_pids: set[int],
    +    all_profiles: bool = False,
    +    include_restart_managers: bool = False,
    +) -> list[int]:
         """Best-effort process-table scan for gateway PIDs.
     
         This supplements the profile-scoped PID file so status views can still spot
    @@ -325,7 +330,10 @@ def _scan_gateway_pids(exclude_pids: set[int], all_profiles: bool = False) -> li
         # scan no longer false-matches ``gateway status``/``dashboard`` siblings or
         # unrelated processes like ``python -m tui_gateway``. Lazy import mirrors the
         # circular-import avoidance used elsewhere in this module.
    -    from gateway.status import looks_like_gateway_command_line
    +    from gateway.status import (
    +        looks_like_gateway_command_line,
    +        looks_like_gateway_runtime_command_line,
    +    )
         current_home = str(get_hermes_home().resolve())
         current_home_lc = current_home.lower()
         current_profile_arg = _profile_arg(current_home)
    @@ -357,6 +365,11 @@ def _matches_current_profile(command: str) -> bool:
                 return False
             return True
     
    +    def _matches_gateway_runtime(command: str) -> bool:
    +        if looks_like_gateway_command_line(command):
    +            return True
    +        return include_restart_managers and looks_like_gateway_runtime_command_line(command)
    +
         try:
             if is_windows():
                 # Prefer wmic when present (fast, stable output format).  On
    @@ -420,7 +433,7 @@ def _matches_current_profile(command: str) -> bool:
                         current_cmd = line[len("CommandLine=") :]
                     elif line.startswith("ProcessId="):
                         pid_str = line[len("ProcessId=") :]
    -                    if looks_like_gateway_command_line(current_cmd) and (
    +                    if _matches_gateway_runtime(current_cmd) and (
                             all_profiles or _matches_current_profile(current_cmd)
                         ):
                             try:
    @@ -445,7 +458,7 @@ def _matches_current_profile(command: str) -> bool:
                                 with open(f"/proc/{pid}/cmdline", "rb") as _f:
                                     cmdline = _f.read().decode("utf-8", errors="replace")
                                 cmdline = cmdline.replace("\x00", " ")
    -                            if looks_like_gateway_command_line(cmdline) and (
    +                            if _matches_gateway_runtime(cmdline) and (
                                     all_profiles or _matches_current_profile(cmdline)
                                 ):
                                     _append_unique_pid(pids, pid, exclude_pids)
    @@ -488,7 +501,7 @@ def _matches_current_profile(command: str) -> bool:
     
                         if pid is None:
                             continue
    -                    if looks_like_gateway_command_line(command) and (
    +                    if _matches_gateway_runtime(command) and (
                             all_profiles or _matches_current_profile(command)
                         ):
                             _append_unique_pid(pids, pid, exclude_pids)
    @@ -567,7 +580,15 @@ def find_gateway_pids(
                 pass
         for pid in _get_service_pids():
             _append_unique_pid(pids, pid, _exclude)
    -    for pid in _scan_gateway_pids(_exclude, all_profiles=all_profiles):
    +    try:
    +        include_restart_managers = not supports_systemd_services()
    +    except Exception:
    +        include_restart_managers = False
    +    for pid in _scan_gateway_pids(
    +        _exclude,
    +        all_profiles=all_profiles,
    +        include_restart_managers=include_restart_managers,
    +    ):
             _append_unique_pid(pids, pid, _exclude)
         return pids
     
    @@ -1324,12 +1345,85 @@ def kill_gateway_processes(
         return killed
     
     
    +def _reap_unsupervised_gateway_orphans() -> bool:
    +    """Kill no-supervisor gateway orphans the pidfile/runtime record can't see.
    +
    +    On WSL/no-systemd hosts the manual restart fallback runs the gateway
    +    in-process under a ``gateway restart`` argv (hermes_cli/gateway.py restart
    +    branch → ``run_gateway()``). If its pidfile or runtime record goes missing
    +    or stale, ``get_running_pid()`` returns ``None`` even though a live orphan
    +    still holds the webhook port, so a follow-up restart stacks a duplicate on
    +    the same port (#51325). This is a no-op on hosts WITH a service supervisor,
    +    where a ``gateway restart`` argv is a transient management command, not the
    +    running gateway — gating on ``supports_systemd_services()`` keeps the
    +    orphan-aware scan from killing live management processes there.
    +
    +    Returns True if at least one orphan was reaped.
    +    """
    +    try:
    +        if supports_systemd_services():
    +            return False
    +    except Exception:
    +        return False
    +
    +    from gateway.status import _pid_exists, write_planned_stop_marker
    +
    +    own = {os.getpid()}
    +    try:
    +        # find_gateway_pids() includes no-supervisor `gateway restart` runtimes
    +        # for the current profile when no systemd supervisor is present.
    +        orphans = [p for p in find_gateway_pids(exclude_pids=own) if p and p > 0]
    +    except Exception:
    +        return False
    +    if not orphans:
    +        return False
    +
    +    reaped = False
    +    for pid in orphans:
    +        try:
    +            write_planned_stop_marker(pid)
    +        except Exception:
    +            pass
    +        try:
    +            os.kill(pid, signal.SIGTERM)
    +        except ProcessLookupError:
    +            continue
    +        except PermissionError:
    +            print(f"⚠ Permission denied to kill orphaned gateway PID {pid}")
    +            continue
    +        reaped = True
    +
    +    # SIGTERM released the port in the field report but the orphan kept
    +    # running until a follow-up SIGKILL — wait briefly, then force-kill
    +    # any survivor so the replacement can bind the port cleanly.
    +    deadline = time.monotonic() + 5.0
    +    survivors = list(orphans)
    +    while survivors and time.monotonic() < deadline:
    +        survivors = [p for p in survivors if _pid_exists(p)]
    +        if survivors:
    +            time.sleep(0.2)
    +    for pid in survivors:
    +        try:
    +            os.kill(pid, getattr(signal, "SIGKILL", signal.SIGTERM))
    +        except (ProcessLookupError, PermissionError, OSError):
    +            pass
    +
    +    return reaped
    +
    +
     def stop_profile_gateway() -> bool:
         """Stop only the gateway for the current profile (HERMES_HOME-scoped).
     
         Uses the PID file written by start_gateway(), so it only kills the
         gateway belonging to this profile — not gateways from other profiles.
         Returns True if a process was stopped, False if none was found.
    +
    +    On hosts without a service supervisor (e.g. WSL/no-systemd, where the
    +    manual restart fallback runs the gateway in-process under a ``gateway
    +    restart`` argv), the pidfile/runtime record can be missing or stale while
    +    a live orphan still holds the webhook port. In that case fall back to the
    +    orphan-aware process scan so the replacement reaps the prior instance
    +    instead of stacking a duplicate on the same port (#51325).
         """
         try:
             from gateway.status import get_running_pid, remove_pid_file
    @@ -1338,7 +1432,7 @@ def stop_profile_gateway() -> bool:
     
         pid = get_running_pid()
         if pid is None:
    -        return False
    +        return _reap_unsupervised_gateway_orphans()
     
         try:
             from gateway.status import write_planned_stop_marker

  • scripts/release.py

    @@ -45,6 +45,7 @@
     
     # Auto-extracted from noreply emails + manual overrides
     AUTHOR_MAP = {
    +    "[email protected]": "wgu9",  # PR #51468 salvage (WSL/no-systemd orphan gateway tracking, #51325)
         "[redacted:email]": "s010mn",  # PR #29221 salvage (ollama-cloud reasoning_effort xhigh→max)
         "[redacted:email]": "jeeves-assistant",  # PR #50771 (computer-use CuaDriver vision capture routing)
         "[email protected]": "ScotterMonk",  # PR #50145 salvage (cron output truncation: adapter-aware chunking, #50126)

  • tests/gateway/test_gateway_command_line_matcher.py

    @@ -11,7 +11,10 @@
     
     import pytest
     
    -from gateway.status import looks_like_gateway_command_line as matches
    +from gateway.status import (
    +    looks_like_gateway_command_line as matches,
    +    looks_like_gateway_runtime_command_line as matches_runtime,
    +)
     
     
     ACCEPT = [
    @@ -58,3 +61,9 @@ def test_accepts_real_gateway_run(cmd):
     @pytest.mark.parametrize("cmd", REJECT)
     def test_rejects_non_gateway_run(cmd):
         assert matches(cmd) is False
    +
    +
    +def test_runtime_matcher_accepts_no_supervisor_restart_process():
    +    assert matches("python -m hermes_cli.main gateway restart") is False
    +    assert matches_runtime("python -m hermes_cli.main gateway restart") is True
    +    assert matches_runtime("python -m hermes_cli.main gateway status") is False

  • tests/gateway/test_status.py

    @@ -174,6 +174,54 @@ def test_get_running_pid_treats_pid_file_as_stale_without_runtime_lock(self, tmp
             assert status.get_running_pid() is None
             assert not pid_path.exists()
     
    +    def test_get_running_pid_accepts_no_supervisor_restart_runtime(self, tmp_path, monkeypatch):
    +        """WSL/no-systemd restart fallback runs the gateway in a restart argv process."""
    +        monkeypatch.setenv("HERMES_HOME", str(tmp_path))
    +        pid_path = tmp_path / "gateway.pid"
    +        record = {
    +            "pid": os.getpid(),
    +            "kind": "hermes-gateway",
    +            "argv": ["python", "-m", "hermes_cli.main", "gateway", "restart"],
    +            "start_time": 123,
    +        }
    +        pid_path.write_text(json.dumps(record))
    +
    +        monkeypatch.setattr(status.os, "kill", lambda pid, sig: None)
    +        monkeypatch.setattr(status, "_get_process_start_time", lambda pid: 123)
    +        monkeypatch.setattr(
    +            status,
    +            "_read_process_cmdline",
    +            lambda pid: "python -m hermes_cli.main gateway restart",
    +        )
    +
    +        assert status.acquire_gateway_runtime_lock() is True
    +        try:
    +            assert status.get_running_pid() == os.getpid()
    +        finally:
    +            status.release_gateway_runtime_lock()
    +
    +    def test_get_running_pid_falls_back_to_no_supervisor_runtime_state(self, tmp_path, monkeypatch):
    +        """A live gateway_state.json PID should keep status accurate without a pidfile."""
    +        monkeypatch.setenv("HERMES_HOME", str(tmp_path))
    +        state_path = tmp_path / "gateway_state.json"
    +        state_path.write_text(json.dumps({
    +            "gateway_state": "running",
    +            "pid": os.getpid(),
    +            "kind": "hermes-gateway",
    +            "argv": ["python", "-m", "hermes_cli.main", "gateway", "restart"],
    +            "start_time": 123,
    +        }))
    +
    +        monkeypatch.setattr(status.os, "kill", lambda pid, sig: None)
    +        monkeypatch.setattr(status, "_get_process_start_time", lambda pid: 123)
    +        monkeypatch.setattr(
    +            status,
    +            "_read_process_cmdline",
    +            lambda pid: "python -m hermes_cli.main gateway restart",
    +        )
    +
    +        assert status.get_running_pid() == os.getpid()
    +
         def test_get_running_pid_cleans_stale_metadata_from_dead_foreign_pid(self, tmp_path, monkeypatch):
             """Stale PID file from a *different* PID (crashed process) must still be cleaned.
     

  • tests/hermes_cli/test_gateway.py

    @@ -1,6 +1,7 @@
     """Tests for hermes_cli.gateway."""
     
     import argparse
    +import signal
     import sys
     from types import ModuleType, SimpleNamespace
     
    @@ -821,6 +822,70 @@ def fake_run(cmd, **kwargs):
         assert gateway.find_gateway_pids() == [321]
     
     
    +def test_find_gateway_pids_includes_restart_managers_without_systemd(monkeypatch):
    +    calls = []
    +
    +    monkeypatch.setattr(gateway, "_get_service_pids", lambda: set())
    +    monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
    +    monkeypatch.setattr(gateway, "supports_systemd_services", lambda: False)
    +
    +    def fake_scan(exclude_pids, all_profiles=False, include_restart_managers=False):
    +        calls.append((set(exclude_pids), all_profiles, include_restart_managers))
    +        return [708] if include_restart_managers else []
    +
    +    monkeypatch.setattr(gateway, "_scan_gateway_pids", fake_scan)
    +
    +    assert gateway.find_gateway_pids(all_profiles=True) == [708]
    +    assert calls == [(set(), True, True)]
    +
    +
    +def test_reap_unsupervised_orphans_noop_on_systemd_hosts(monkeypatch):
    +    """On supervised hosts a `gateway restart` argv is transient — never reap."""
    +    monkeypatch.setattr(gateway, "supports_systemd_services", lambda: True)
    +    killed = []
    +    monkeypatch.setattr(gateway.os, "kill", lambda pid, sig: killed.append((pid, sig)))
    +    # Should not even consult the scan when a supervisor is present.
    +    monkeypatch.setattr(
    +        gateway, "find_gateway_pids",
    +        lambda *a, **k: (_ for _ in ()).throw(AssertionError("scanned on systemd host")),
    +    )
    +
    +    assert gateway._reap_unsupervised_gateway_orphans() is False
    +    assert killed == []
    +
    +
    +def test_reap_unsupervised_orphans_sigterms_then_sigkills_survivor(monkeypatch):
    +    """No-systemd: orphan gets SIGTERM, and a survivor is force-killed."""
    +    monkeypatch.setattr(gateway, "supports_systemd_services", lambda: False)
    +    monkeypatch.setattr(gateway, "find_gateway_pids", lambda exclude_pids=None: [708])
    +    monkeypatch.setattr("gateway.status.write_planned_stop_marker", lambda pid: True)
    +    # Orphan ignores SIGTERM (matches the field report) and stays alive, so the
    +    # follow-up SIGKILL must fire.
    +    monkeypatch.setattr("gateway.status._pid_exists", lambda pid: True)
    +
    +    sent = []
    +    monkeypatch.setattr(gateway.os, "kill", lambda pid, sig: sent.append((pid, sig)))
    +    # Collapse the drain window: no real sleeping, and jump past the deadline
    +    # after the first check so the loop exits immediately.
    +    monkeypatch.setattr(gateway.time, "sleep", lambda _s: None)
    +    ticks = iter([0.0, 100.0, 200.0])
    +    monkeypatch.setattr(gateway.time, "monotonic", lambda: next(ticks, 200.0))
    +
    +    assert gateway._reap_unsupervised_gateway_orphans() is True
    +    assert (708, signal.SIGTERM) in sent
    +    assert (708, signal.SIGKILL) in sent
    +
    +
    +def test_reap_unsupervised_orphans_returns_false_when_none_found(monkeypatch):
    +    monkeypatch.setattr(gateway, "supports_systemd_services", lambda: False)
    +    monkeypatch.setattr(gateway, "find_gateway_pids", lambda exclude_pids=None: [])
    +    killed = []
    +    monkeypatch.setattr(gateway.os, "kill", lambda pid, sig: killed.append((pid, sig)))
    +
    +    assert gateway._reap_unsupervised_gateway_orphans() is False
    +    assert killed == []
    +
    +
     def test_scan_gateway_pids_detects_windows_hermes_exe_case_variants(monkeypatch):
         monkeypatch.setattr(gateway, "is_windows", lambda: True)
         monkeypatch.setattr(gateway, "_get_ancestor_pids", lambda: set())

  • tests/hermes_cli/test_gateway_proc_fallback.py

    @@ -77,6 +77,43 @@ def test_detects_gateway_pid_via_proc(self):
             assert 99999 not in pids
             mock_ps.assert_not_called()  # ps must NOT be called when /proc worked
     
    +    def test_detects_no_supervisor_restart_process_only_when_enabled(self):
    +        entries = {
    +            12345: "python -m hermes_cli.main gateway restart",
    +            99999: _OTHER_CMD,
    +        }
    +        _isdir, _listdir, _open = _fake_proc_dir(entries)
    +
    +        with (
    +            patch("hermes_cli.gateway.is_windows", return_value=False),
    +            patch("os.path.isdir", side_effect=_isdir),
    +            patch("os.listdir", side_effect=_listdir),
    +            patch("builtins.open", side_effect=_open),
    +            patch("hermes_cli.gateway._get_ancestor_pids", return_value=set()),
    +            patch("subprocess.run") as mock_ps,
    +        ):
    +            strict_pids = gateway_mod._scan_gateway_pids(set(), all_profiles=True)
    +
    +        _isdir, _listdir, _open = _fake_proc_dir(entries)
    +        with (
    +            patch("hermes_cli.gateway.is_windows", return_value=False),
    +            patch("os.path.isdir", side_effect=_isdir),
    +            patch("os.listdir", side_effect=_listdir),
    +            patch("builtins.open", side_effect=_open),
    +            patch("hermes_cli.gateway._get_ancestor_pids", return_value=set()),
    +            patch("subprocess.run") as mock_ps_enabled,
    +        ):
    +            fallback_pids = gateway_mod._scan_gateway_pids(
    +                set(),
    +                all_profiles=True,
    +                include_restart_managers=True,
    +            )
    +
    +        assert strict_pids == []
    +        assert fallback_pids == [12345]
    +        mock_ps.assert_not_called()
    +        mock_ps_enabled.assert_not_called()
    +
         def test_excludes_own_pid_from_proc_scan(self):
             my_pid = os.getpid()
             entries = {my_pid: _GATEWAY_CMD}