Gateway service restart loops and exit code fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #614 fix: resolve systemd restart loop with --replace flag

  merged to main

  files: gateway/run.py, hermes_cli/gateway.py, hermes_cli/main.py

• #9875 fix: gateway auto-recovers from unexpected SIGTERM via systemd (#5646)

  merged to main

  files: gateway/run.py, gateway/status.py

  Show the diff

  • gateway/run.py

    @@ -9261,8 +9261,18 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =
     
         runner = GatewayRunner(config)
         
    +    # Track whether a signal initiated the shutdown (vs. internal request).
    +    # When an unexpected SIGTERM kills the gateway, we exit non-zero so
    +    # systemd's Restart=on-failure revives the process.  systemctl stop
    +    # is safe: systemd tracks stop-requested state independently of exit
    +    # code, so Restart= never fires for a deliberate stop.
    +    _signal_initiated_shutdown = False
    +
         # Set up signal handlers
         def shutdown_signal_handler():
    +        nonlocal _signal_initiated_shutdown
    +        _signal_initiated_shutdown = True
    +        logger.info("Received SIGTERM/SIGINT — initiating shutdown")
             asyncio.create_task(runner.stop())
     
         def restart_signal_handler():
    @@ -9332,6 +9342,21 @@ def restart_signal_handler():
         if runner.exit_code is not None:
             raise SystemExit(runner.exit_code)
     
    +    # When a signal (SIGTERM/SIGINT) caused the shutdown and it wasn't a
    +    # planned restart (/restart, /update, SIGUSR1), exit non-zero so
    +    # systemd's Restart=on-failure revives the process.  This covers:
    +    #   - hermes update killing the gateway mid-work
    +    #   - External kill commands
    +    #   - WSL2/container runtime sending unexpected signals
    +    # systemctl stop is safe: systemd tracks "stop requested" state
    +    # independently of exit code, so Restart= never fires for it.
    +    if _signal_initiated_shutdown and not runner._restart_requested:
    +        logger.info(
    +            "Exiting with code 1 (signal-initiated shutdown without restart "
    +            "request) so systemd Restart=on-failure can revive the gateway."
    +        )
    +        return False  # → sys.exit(1) in the caller
    +
         return True
     
     

  • gateway/status.py

    @@ -266,9 +266,25 @@ def read_runtime_status() -> Optional[dict[str, Any]]:
     
     
     def remove_pid_file() -> None:
    -    """Remove the gateway PID file if it exists."""
    +    """Remove the gateway PID file, but only if it belongs to this process.
    +
    +    During --replace handoffs, the old process's atexit handler can fire AFTER
    +    the new process has written its own PID file.  Blindly removing the file
    +    would delete the new process's record, leaving the gateway running with no
    +    PID file (invisible to ``get_running_pid()``).
    +    """
         try:
    -        _get_pid_path().unlink(missing_ok=True)
    +        path = _get_pid_path()
    +        record = _read_json_file(path)
    +        if record is not None:
    +            try:
    +                file_pid = int(record["pid"])
    +            except (KeyError, TypeError, ValueError):
    +                file_pid = None
    +            if file_pid is not None and file_pid != os.getpid():
    +                # PID file belongs to a different process — leave it alone.
    +                return
    +        path.unlink(missing_ok=True)
         except Exception:
             pass
     

• #51734 fix(gateway): detect launchd in /restart service-manager probe (#43475)

  merged to main

  files: gateway/slash_commands.py, scripts/release.py, tests/gateway/test_restart_service_detection.py

  Show the diff

  • gateway/slash_commands.py

    @@ -960,7 +960,15 @@ async def _handle_restart_command(self, event: MessageEvent) -> Union[str, Ephem
             # us.  The detached subprocess approach (setsid + bash) doesn't work
             # under systemd (KillMode=mixed kills the cgroup) or Docker (tini
             # exits when the gateway dies, taking the detached helper with it).
    -        _under_service = bool(os.environ.get("INVOCATION_ID"))  # systemd sets this
    +        # systemd sets INVOCATION_ID; launchd sets XPC_SERVICE_NAME to the
    +        # job label.  Without the launchd check, macOS /restart takes the
    +        # detached path and exits 0, which KeepAlive.SuccessfulExit=false
    +        # treats as a deliberate stop — the gateway stays dead until next
    +        # login.  Interactive macOS shells inherit XPC_SERVICE_NAME=0, so
    +        # "0" must count as not-under-launchd.
    +        _under_service = bool(os.environ.get("INVOCATION_ID")) or os.environ.get(
    +            "XPC_SERVICE_NAME", "0"
    +        ) not in ("", "0")
             _in_container = os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv")
             if _under_service or _in_container:
                 self.request_restart(detached=False, via_service=True)

  • scripts/release.py

    @@ -1547,6 +1547,7 @@
         "[redacted:email]": "bird",  # PR #25219 (gateway docker exit-75 restart)
         "[redacted:email]": "davidcampbelldc",  # PR #26834 (web_server proxy_headers=False)
         "[redacted:email]": "davidcampbelldc",
    +    "[redacted:email]": "chazmaniandinkle",  # PR #43888 (launchd /restart detection)
         "[redacted:email]": "hehehe0803",  # PR #26212 salvage (codex kanban writable root)
         "[redacted:email]": "hehehe0803",
         "[redacted:email]": "kasunvinod",  # PR #24126 salvage (codex timeout propagation)

  • tests/gateway/test_restart_service_detection.py

    @@ -0,0 +1,85 @@
    +"""Tests for /restart service-manager detection (launchd vs interactive).
    +
    +The /restart handler routes through ``request_restart(via_service=True)``
    +when a service manager supervises the gateway, so the process exits with
    +the service-restart code and the manager relaunches it.  Under macOS
    +launchd the plist uses ``KeepAlive.SuccessfulExit=false`` — a clean exit 0
    +is treated as a deliberate stop and the gateway stays dead (#43475) — so
    +launchd must be detected here in the handler, not only at the exit-code
    +site (which never runs unless ``via_service=True`` is already set).
    +
    +launchd sets ``XPC_SERVICE_NAME`` to the job label for processes it
    +spawns.  Interactive macOS shells inherit ``XPC_SERVICE_NAME=0`` (a
    +truthy string), so the probe must treat ``"0"`` as not-under-launchd:
    +routing an unsupervised interactive gateway to the service path would
    +make it exit non-zero with nothing to revive it.
    +"""
    +from unittest.mock import MagicMock
    +
    +import pytest
    +
    +import gateway.run as gateway_run
    +from gateway.platforms.base import MessageEvent, MessageType
    +from tests.gateway.restart_test_helpers import make_restart_runner, make_restart_source
    +
    +
    +def _make_restart_event(update_id: int | None = 100) -> MessageEvent:
    +    return MessageEvent(
    +        text="/restart",
    +        message_type=MessageType.TEXT,
    +        source=make_restart_source(),
    +        message_id="m1",
    +        platform_update_id=update_id,
    +    )
    +
    +
    +def _make_runner_with_mock_restart(tmp_path, monkeypatch):
    +    monkeypatch.setattr(gateway_run, "_hermes_home", tmp_path)
    +    monkeypatch.delenv("INVOCATION_ID", raising=False)
    +    monkeypatch.delenv("XPC_SERVICE_NAME", raising=False)
    +    runner, _adapter = make_restart_runner()
    +    runner.request_restart = MagicMock(return_value=True)
    +    return runner
    +
    +
    [redacted:email]
    +async def test_restart_under_launchd_uses_service_path(tmp_path, monkeypatch):
    +    """launchd job label in XPC_SERVICE_NAME routes /restart via the service path."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +    monkeypatch.setenv("XPC_SERVICE_NAME", "ai.hermes.gateway")
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=False, via_service=True)
    +
    +
    [redacted:email]
    +async def test_restart_in_interactive_macos_shell_uses_detached_path(tmp_path, monkeypatch):
    +    """XPC_SERVICE_NAME=0 (inherited by interactive macOS shells) is NOT a service."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +    monkeypatch.setenv("XPC_SERVICE_NAME", "0")
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=True, via_service=False)
    +
    +
    [redacted:email]
    +async def test_restart_without_service_env_uses_detached_path(tmp_path, monkeypatch):
    +    """No service-manager env at all falls back to the detached restart."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=True, via_service=False)
    +
    +
    [redacted:email]
    +async def test_restart_under_systemd_uses_service_path(tmp_path, monkeypatch):
    +    """INVOCATION_ID (systemd) still routes via the service path."""
    +    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)
    +    monkeypatch.setenv("INVOCATION_ID", "abc123")
    +
    +    await runner._handle_restart_command(_make_restart_event())
    +
    +    runner.request_restart.assert_called_once_with(detached=False, via_service=True)

• #53903 docs(gateway): warn against custom ExecStopPost kill drop-in (restart loop)

  merged to main

  files: website/docs/user-guide/messaging/index.md