Gateway process lifecycle and stale PID handling

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #28341 fix(gateway): exit code 75 on service restart so launchd relaunches (#28200)

  merged to main

  files: gateway/run.py

  Show the diff

  • gateway/run.py

    @@ -17346,6 +17346,19 @@ def restart_signal_handler():
             )
             return False  # → sys.exit(1) in the caller
     
    +    # When the gateway is restarting via the service manager (SIGUSR1 →
    +    # launchd_restart or /restart / /update commands), exit with code 75 so
    +    # that launchd's ``KeepAlive → SuccessfulExit → false`` policy treats
    +    # the exit as *unsuccessful* and relaunches the service.  This mirrors
    +    # the systemd ``RestartForceExitStatus=75`` convention already used by
    +    # the systemd unit template.
    +    if runner._restart_via_service:
    +        logger.info(
    +            "Exiting with code 75 (service-restart requested) so "
    +            "launchd KeepAlive relaunches the gateway."
    +        )
    +        raise SystemExit(75)
    +
         return True
     
     

• #39000 fix(cli): launchd KeepAlive unconditional restart (#37388)

  merged to main

  files: hermes_cli/gateway.py, scripts/release.py, tests/hermes_cli/test_gateway_service.py

  Show the diff

  • hermes_cli/gateway.py

    @@ -227,9 +227,8 @@ def _graceful_restart_via_sigusr1(pid: int, drain_timeout: float) -> bool:
     
         SIGUSR1 is wired in gateway/run.py to ``request_restart(via_service=True)``
         which drains in-flight agent runs (up to ``agent.restart_drain_timeout``
    -    seconds), then exits.  systemd relaunches clean exits via
    -    ``Restart=always``; launchd still uses a non-zero planned-restart exit
    -    because its plist has ``KeepAlive.SuccessfulExit = false``.
    +    seconds), then exits.  Both systemd (``Restart=always``) and launchd
    +    (unconditional ``<key>KeepAlive</key><true/>``) restart on any exit.
     
         This is the drain-aware alternative to ``systemctl restart`` / ``SIGTERM``,
         which SIGKILL in-flight agents after a short timeout.
    @@ -3083,10 +3082,7 @@ def generate_launchd_plist() -> str:
         <true/>
         
         <key>KeepAlive</key>
    -    <dict>
    -        <key>SuccessfulExit</key>
    -        <false/>
    -    </dict>
    +    <true/>
         
         <key>StandardOutPath</key>
         <string>{log_dir}/gateway.log</string>
    @@ -3249,7 +3245,7 @@ def launchd_stop():
             pass
         # bootout unloads the service definition so KeepAlive doesn't respawn
         # the process.  A plain `kill SIGTERM` only signals the process — launchd
    -    # immediately restarts it because KeepAlive.SuccessfulExit = false.
    +    # immediately restarts it because KeepAlive is unconditionally true.
         # `hermes gateway start` re-bootstraps when it detects the job is unloaded.
         try:
             subprocess.run(["launchctl", "bootout", target], check=True, timeout=90)

  • scripts/release.py

    @@ -72,6 +72,7 @@
         "[redacted:email]": "kchuang1015",
         "[redacted:email]": "MaheshtheDev",
         "[redacted:email]": "kyssta-exe",
    +    "[redacted:email]": "ashishpatel26",
         "[redacted:email]": "fujinice",
         "[redacted:email]": "carltonawong",
         "[redacted:email]": "EvilHumphrey",

  • tests/hermes_cli/test_gateway_service.py

    @@ -2579,3 +2579,24 @@ def test_launchd_workingdirectory_is_hermes_home(self, tmp_path, monkeypatch):
             assert m, "plist has no WorkingDirectory entry"
             assert Path(m.group(1)).resolve() == home.resolve()
             assert "/.worktrees/" not in m.group(1)
    +
    +    def test_launchd_plist_keepalive_unconditional(self, tmp_path, monkeypatch):
    +        """KeepAlive must be unconditional <true/> so the gateway restarts on clean exits.
    +
    +        Bug #37388: the old ``KeepAlive.SuccessfulExit = false`` dict form meant
    +        launchd would NOT restart after a zero-exit (e.g. ``gateway run --replace``
    +        causes the old instance to exit cleanly).  Switching to the scalar
    +        ``<key>KeepAlive</key><true/>`` makes launchd restart regardless of exit code.
    +        """
    +        home = tmp_path / ".hermes"
    +        home.mkdir()
    +        monkeypatch.setattr(gateway_cli, "get_hermes_home", lambda: home)
    +        plist = gateway_cli.generate_launchd_plist()
    +
    +        # Scalar <true/> must be present immediately after the KeepAlive key
    +        assert "<key>KeepAlive</key>" in plist
    +        # The unconditional form
    +        assert "<key>KeepAlive</key>\n    <true/>" in plist
    +        # The old conditional dict form must NOT appear
    +        assert "SuccessfulExit" not in plist
    +        assert "<key>KeepAlive</key>\n    <dict>" not in plist

• #51734 fix(gateway): detect launchd in /restart service-manager probe (#43475)

  merged to main

  files: gateway/slash_commands.py, scripts/release.py, tests/gateway/test_restart_service_detection.py

  Show the diff

  • gateway/slash_commands.py

    @@ -960,7 +960,15 @@ async def _handle_restart_command(self, event: MessageEvent) -> Union[str, Ephem
             # us.  The detached subprocess approach (setsid + bash) doesn't work
             # under systemd (KillMode=mixed kills the cgroup) or Docker (tini
             # exits when the gateway dies, taking the detached helper with it).
    -        _under_service = bool(os.environ.get("INVOCATION_ID"))  # systemd sets this
    +        # systemd sets INVOCATION_ID; launchd sets XPC_SERVICE_NAME to the
    +        # job label.  Without the launchd check, macOS /restart takes the
    +        # detached path and exits 0, which KeepAlive.SuccessfulExit=false
    +        # treats as a deliberate stop — the gateway stays dead until next
    +        # login.  Interactive macOS shells inherit XPC_SERVICE_NAME=0, so
    +        # "0" must count as not-under-launchd.
    +        _under_service = bool(os.environ.get("INVOCATION_ID")) or os.environ.get(
    +            "XPC_SERVICE_NAME", "0"
    +        ) not in ("", "0")
             _in_container = os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv")
             if _under_service or _in_container:
                 self.request_restart(detached=False, via_service=True)

  • scripts/release.py

    @@ -1547,6 +1547,7 @@
         "[redacted:email]": "bird",  # PR #25219 (gateway docker exit-75 restart)
         "[redacted:email]": "davidcampbelldc",  # PR #26834 (web_server proxy_headers=False)
         "[redacted:email]": "davidcampbelldc",
    +    "[redacted:email]": "chazmaniandinkle",  # PR #43888 (launchd /restart detection)
         "[redacted:email]": "hehehe0803",  # PR #26212 salvage (codex kanban writable root)
         "[redacted:email]": "hehehe0803",
         "[redacted:email]": "kasunvinod",  # PR #24126 salvage (codex timeout propagation)

  • tests/gateway/test_restart_service_detection.py

    @@ -0,0 +1,85 @@
    +"""Tests for /restart service-manager detection (launchd vs interactive).
    +
    +The /restart handler routes through ``request_restart(via_service=True)``
    +when a service manager supervises the gateway, so the process exits with
    +the service-restart code and the manager relaunches it.  Under macOS
    +launchd the plist uses ``KeepAlive.SuccessfulExit=false`` — a clean exit 0
    +is treated as a deliberate stop and the gateway stays dead (#43475) — so
    +launchd must be detected here in the handler, not only at the exit-code
    +site (which never runs unless ``via_service=True`` is already set).
    +
    +launchd sets ``XPC_SERVICE_NAME`` to the job label for processes it
    +spawns.  Interactive macOS shells inherit ``XPC_SERVICE_NAME=0`` (a
    +truthy string), so the probe must treat ``"0"`` as not-under-launchd:
    +routing an unsupervised interactive gateway to the service path would
    +make it exit non-zero with nothing to revive it.
    +"""
    +from unittest.mock import MagicMock
    +
    +import pytest
    +
    +import gateway.run as gateway_run
    +from gateway.platforms.base import MessageEvent, MessageType
    +from tests.gateway.restart_test_helpers import make_restart_runner, make_restart_source
    +
    +
    +def _make_restart_event(update_id: int | None = 100) -> MessageEvent:
    +    return MessageEvent(
    +        text="/restart",
    +        message_type=MessageType.TEXT,
    +        source=make_restart_source(),
    +        message_id="m1",
    +        platform_update_id=update_id,
    +    )
    +
    +
    +def _make_runner_with_mock_restart(tmp_path, monkeypatch):
    +    monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
    +    monkeypatch.delenv("INVOCATION_ID", raising=False)
    +    monkeypatch.delenv("XPC_SERVICE_NAME", raising=False)
    +    runner, _adapter = make_restart_runner()
    +    runner.request_restart = MagicMock(return_value=True)
    +    return runner
    +
    +
    [redacted:email]
    +async def test_restart_under_launchd_uses_service_path(tmp_path, monkeypatch):
    +    """launchd job label in XPC_SERVICE_NAME routes /restart via the service path."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +    monkeypatch.setenv("XPC_SERVICE_NAME", "ai.hermes.gateway")
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=False, via_service=True)
    +
    +
    [redacted:email]
    +async def test_restart_in_interactive_macos_shell_uses_detached_path(tmp_path, monkeypatch):
    +    """XPC_SERVICE_NAME=0 (inherited by interactive macOS shells) is NOT a service."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +    monkeypatch.setenv("XPC_SERVICE_NAME", "0")
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=True, via_service=False)
    +
    +
    [redacted:email]
    +async def test_restart_without_service_env_uses_detached_path(tmp_path, monkeypatch):
    +    """No service-manager env at all falls back to the detached restart."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=True, via_service=False)
    +
    +
    [redacted:email]
    +async def test_restart_under_systemd_uses_service_path(tmp_path, monkeypatch):
    +    """INVOCATION_ID (systemd) still routes via the service path."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +    monkeypatch.setenv("INVOCATION_ID", "abc123")
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=False, via_service=True)