{"repo":"NousResearch/hermes-agent","public_id":"576","display_label":"Gateway service restart loops and exit code fixes","summary":"These issues and fixes address gateway daemon behavior under systemd, launchd, and other service managers, focusing on unintended restart loops caused by incorrect exit codes on intentional stops, the --replace flag in generated service definitions, and stale or colliding PID file handling.","subsystem":"gateway","counts":{"members":39,"open_issues":3,"open_issues_distinct":3,"open_prs":6,"closed_issues":9,"merged_prs":4,"attempt_prs":23},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":614},"resolution":{"state":"partially_resolved","by":"pr-614,pr-9875,pr-53903"},"freshness":{"state":"verified","last_verified_at":"2026-06-28T02:04:30+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.042267206477732785},"fix_edges":[{"issue_number":576,"pr_number":614,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":5646,"pr_number":9875,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":21549,"pr_number":51734,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":23272,"pr_number":53903,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":43475,"pr_number":51734,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":576,"title":"[Bug]: systemd restart loop (non-TTY CLI exit and \"gateway already running\" conflict)","url":null,"state":"closed"},{"kind":"issue","number":5646,"title":"[Bug]: gateway restart / --replace flag causes gateway to exit immediately (status=0)","url":null,"state":"closed"},{"kind":"issue","number":21549,"title":"fix(gateway): launchd double-spawn triggers infinite restart death spiral (exit code 1 on instance detection)","url":null,"state":"closed"},{"kind":"issue","number":21915,"title":"[systemd] Incomplete process cleanup during restart causes port conflict and infinite restart loop","url":null,"state":"closed"},{"kind":"issue","number":23272,"title":"Gateway restart loop: hardcoded --replace + systemd ExecStopPost causes infinite restart cycle","url":null,"state":"closed"},{"kind":"issue","number":24344,"title":"[Bug]: Systemd gateway service exits with status 1 on intentional stop","url":null,"state":"open"},{"kind":"issue","number":27275,"title":"Generated Linux gateway service can flap forever because ExecStart uses gateway run --replace under Type=simple","url":null,"state":"closed"},{"kind":"issue","number":32951,"title":"Avoid systemd gateway restart/takeover loops from Restart=always + --replace","url":null,"state":"closed"},{"kind":"issue","number":36213,"title":"[Bug] Windows: hermes gateway restart does not verify old process termination, causing silent Gateway failure","url":null,"state":"closed"},{"kind":"issue","number":41631,"title":"[Bug]: gateway exits code 1 (→ unit 'failed') on systemctl stop; planned stops should exit 0","url":null,"state":"open"},{"kind":"issue","number":42517,"title":"fix(gateway): ExecStop should write planned-stop marker instead of inferring signal source","url":null,"state":"closed"},{"kind":"issue","number":57739,"title":"[Bug]: macOS gateway enters restart loop — hardcoded --replace in launchd plist and incorrect exit code on SIGTERM","url":null,"state":"open"},{"kind":"pr","number":580,"title":"fix: make gateway run idempotent when already running","url":null,"state":"closed","close_reason":{"category":"partial_fix","implication":"needs_llm"},"merged":false},{"kind":"pr","number":582,"title":"fix: prevent systemd restart loop from stale/colliding PID files","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#614","number":614}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":583,"title":"fix(gateway): make systemd startup idempotent and service-safe","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":613,"title":"fix(gateway): avoid systemd restart loop when instance already running","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#614","number":614}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":614,"title":"fix: resolve systemd restart loop with --replace flag","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","hermes_cli/gateway.py","hermes_cli/main.py"]},{"kind":"pr","number":9875,"title":"fix: gateway auto-recovers from unexpected SIGTERM via systemd (#5646)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","gateway/status.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -9261,8 +9261,18 @@ async def start_gateway(config: Optional[GatewayConfig] = None, replace: bool =\n \n     runner = GatewayRunner(config)\n     \n+    # Track whether a signal initiated the shutdown (vs. internal request).\n+    # When an unexpected SIGTERM kills the gateway, we exit non-zero so\n+    # systemd's Restart=on-failure revives the process.  systemctl stop\n+    # is safe: systemd tracks stop-requested state independently of exit\n+    # code, so Restart= never fires for a deliberate stop.\n+    _signal_initiated_shutdown = False\n+\n     # Set up signal handlers\n     def shutdown_signal_handler():\n+        nonlocal _signal_initiated_shutdown\n+        _signal_initiated_shutdown = True\n+        logger.info(\"Received SIGTERM/SIGINT — initiating shutdown\")\n         asyncio.create_task(runner.stop())\n \n     def restart_signal_handler():","@@ -9332,6 +9342,21 @@ def restart_signal_handler():\n     if runner.exit_code is not None:\n         raise SystemExit(runner.exit_code)\n \n+    # When a signal (SIGTERM/SIGINT) caused the shutdown and it wasn't a\n+    # planned restart (/restart, /update, SIGUSR1), exit non-zero so\n+    # systemd's Restart=on-failure revives the process.  This covers:\n+    #   - hermes update killing the gateway mid-work\n+    #   - External kill commands\n+    #   - WSL2/container runtime sending unexpected signals\n+    # systemctl stop is safe: systemd tracks \"stop requested\" state\n+    # independently of exit code, so Restart= never fires for it.\n+    if _signal_initiated_shutdown and not runner._restart_requested:\n+        logger.info(\n+            \"Exiting with code 1 (signal-initiated shutdown without restart \"\n+            \"request) so systemd Restart=on-failure can revive the gateway.\"\n+        )\n+        return False  # → sys.exit(1) in the caller\n+\n     return True\n \n "]},{"file":"gateway/status.py","hunks":["@@ -266,9 +266,25 @@ def read_runtime_status() -> Optional[dict[str, Any]]:\n \n \n def remove_pid_file() -> None:\n-    \"\"\"Remove the gateway PID file if it exists.\"\"\"\n+    \"\"\"Remove the gateway PID file, but only if it belongs to this process.\n+\n+    During --replace handoffs, the old process's atexit handler can fire AFTER\n+    the new process has written its own PID file.  Blindly removing the file\n+    would delete the new process's record, leaving the gateway running with no\n+    PID file (invisible to ``get_running_pid()``).\n+    \"\"\"\n     try:\n-        _get_pid_path().unlink(missing_ok=True)\n+        path = _get_pid_path()\n+        record = _read_json_file(path)\n+        if record is not None:\n+            try:\n+                file_pid = int(record[\"pid\"])\n+            except (KeyError, TypeError, ValueError):\n+                file_pid = None\n+            if file_pid is not None and file_pid != os.getpid():\n+                # PID file belongs to a different process — leave it alone.\n+                return\n+        path.unlink(missing_ok=True)\n     except Exception:\n         pass\n \n"]}]},{"kind":"pr","number":21555,"title":"fix(gateway): exit cleanly when another instance detected to avoid launchd restart loop","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#51734","number":51734},{"kind":"pr","value":"#37388","number":37388},{"kind":"pr","value":"#43475","number":43475},{"kind":"pr","value":"#21549","number":21549},{"kind":"pr","value":"#42446","number":42446},{"kind":"pr","value":"#42450","number":42450},{"kind":"pr","value":"#28561","number":28561},{"kind":"pr","value":"#21549","number":21549}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":21907,"title":"fix(gateway): avoid --replace in generated services","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#23281","number":23281},{"kind":"pr","value":"#23272","number":23272}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":22391,"title":"fix(gateway): exit cleanly under systemd when duplicate instance detected","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":23281,"title":"fix(gateway): stop default replace in service runs","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"f556edc10","number":0}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":23330,"title":"fix(gateway): remove --replace from systemd/launchd service definitions (fixes #23272)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#26186","number":26186}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":23350,"title":"fix(gateway): skip --replace when running under systemd","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#23330","number":23330},{"kind":"pr","value":"#23281","number":23281},{"kind":"pr","value":"#23272","number":23272}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":24351,"title":"fix(gateway): make intentional systemd stops exit cleanly","url":null,"state":"open","close_reason":{"refs":[{"kind":"commit","value":"002c45998","number":0}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":26186,"title":"fix(gateway): remove --replace flag from systemd/launchd/windows service definitions","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":32989,"title":"fix(gateway): avoid systemd restart loops on replace","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4d18717b6c798d4f6bab9e736c6ed10c5a8365f4","number":0},{"kind":"pr","value":"#32951","number":32951}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":36223,"title":"fix(gateway): wait for Windows restart slot before start","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"e7d2f0b93ca29b2cd95d6623fe8af7646173a75b","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":41639,"title":"fix(gateway): treat SIGTERM under systemd as planned stop (exit 0)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#41642","number":41642},{"kind":"pr","value":"#41631","number":41631}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":41642,"title":"fix(gateway): exit 0 on systemctl stop instead of exit 1 (failed unit)","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#41639","number":41639},{"kind":"pr","value":"#41631","number":41631}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":41690,"title":"fix(gateway): exit 0 when systemd sends SIGTERM via systemctl stop","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#41642","number":41642},{"kind":"pr","value":"#41639","number":41639},{"kind":"pr","value":"#41631","number":41631}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":42555,"title":"fix(gateway): write planned-stop marker from systemd ExecStop","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#43236","number":43236},{"kind":"pr","value":"#42675","number":42675},{"kind":"pr","value":"#41631","number":41631},{"kind":"pr","value":"#43236","number":43236},{"kind":"pr","value":"#43236","number":43236}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":51734,"title":"fix(gateway): detect launchd in /restart service-manager probe (#43475)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/slash_commands.py","scripts/release.py","tests/gateway/test_restart_service_detection.py"],"fix_hunks":[{"file":"gateway/slash_commands.py","hunks":["@@ -960,7 +960,15 @@ async def _handle_restart_command(self, event: MessageEvent) -> Union[str, Ephem\n         # us.  The detached subprocess approach (setsid + bash) doesn't work\n         # under systemd (KillMode=mixed kills the cgroup) or Docker (tini\n         # exits when the gateway dies, taking the detached helper with it).\n-        _under_service = bool(os.environ.get(\"INVOCATION_ID\"))  # systemd sets this\n+        # systemd sets INVOCATION_ID; launchd sets XPC_SERVICE_NAME to the\n+        # job label.  Without the launchd check, macOS /restart takes the\n+        # detached path and exits 0, which KeepAlive.SuccessfulExit=false\n+        # treats as a deliberate stop — the gateway stays dead until next\n+        # login.  Interactive macOS shells inherit XPC_SERVICE_NAME=0, so\n+        # \"0\" must count as not-under-launchd.\n+        _under_service = bool(os.environ.get(\"INVOCATION_ID\")) or os.environ.get(\n+            \"XPC_SERVICE_NAME\", \"0\"\n+        ) not in (\"\", \"0\")\n         _in_container = os.path.exists(\"/.dockerenv\") or os.path.exists(\"/run/.containerenv\")\n         if _under_service or _in_container:\n             self.request_restart(detached=False, via_service=True)"]},{"file":"scripts/release.py","hunks":["@@ -1547,6 +1547,7 @@\n     \"[redacted:email]\": \"bird\",  # PR #25219 (gateway docker exit-75 restart)\n     \"[redacted:email]\": \"davidcampbelldc\",  # PR #26834 (web_server proxy_headers=False)\n     \"[redacted:email]\": \"davidcampbelldc\",\n+    \"[redacted:email]\": \"chazmaniandinkle\",  # PR #43888 (launchd /restart detection)\n     \"[redacted:email]\": \"hehehe0803\",  # PR #26212 salvage (codex kanban writable root)\n     \"[redacted:email]\": \"hehehe0803\",\n     \"[redacted:email]\": \"kasunvinod\",  # PR #24126 salvage (codex timeout propagation)"]},{"file":"tests/gateway/test_restart_service_detection.py","hunks":["@@ -0,0 +1,85 @@\n+\"\"\"Tests for /restart service-manager detection (launchd vs interactive).\n+\n+The /restart handler routes through ``request_restart(via_service=True)``\n+when a service manager supervises the gateway, so the process exits with\n+the service-restart code and the manager relaunches it.  Under macOS\n+launchd the plist uses ``KeepAlive.SuccessfulExit=false`` — a clean exit 0\n+is treated as a deliberate stop and the gateway stays dead (#43475) — so\n+launchd must be detected here in the handler, not only at the exit-code\n+site (which never runs unless ``via_service=True`` is already set).\n+\n+launchd sets ``XPC_SERVICE_NAME`` to the job label for processes it\n+spawns.  Interactive macOS shells inherit ``XPC_SERVICE_NAME=0`` (a\n+truthy string), so the probe must treat ``\"0\"`` as not-under-launchd:\n+routing an unsupervised interactive gateway to the service path would\n+make it exit non-zero with nothing to revive it.\n+\"\"\"\n+from unittest.mock import MagicMock\n+\n+import pytest\n+\n+import gateway.run as gateway_run\n+from gateway.platforms.base import MessageEvent, MessageType\n+from tests.gateway.restart_test_helpers import make_restart_runner, make_restart_source\n+\n+\n+def _make_restart_event(update_id: int | None = 100) -> MessageEvent:\n+    return MessageEvent(\n+        text=\"/restart\",\n+        message_type=MessageType.TEXT,\n+        source=make_restart_source(),\n+        message_id=\"m1\",\n+        platform_update_id=update_id,\n+    )\n+\n+\n+def _make_runner_with_mock_restart(tmp_path, monkeypatch):\n+    monkeypatch.setattr(gateway_run, \"_hermes_home\", tmp_path)\n+    monkeypatch.delenv(\"INVOCATION_ID\", raising=False)\n+    monkeypatch.delenv(\"XPC_SERVICE_NAME\", raising=False)\n+    runner, _adapter = make_restart_runner()\n+    runner.request_restart = MagicMock(return_value=True)\n+    return runner\n+\n+\n[redacted:email]\n+async def test_restart_under_launchd_uses_service_path(tmp_path, monkeypatch):\n+    \"\"\"launchd job label in XPC_SERVICE_NAME routes /restart via the service path.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+    monkeypatch.setenv(\"XPC_SERVICE_NAME\", \"ai.hermes.gateway\")\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=False, via_service=True)\n+\n+\n[redacted:email]\n+async def test_restart_in_interactive_macos_shell_uses_detached_path(tmp_path, monkeypatch):\n+    \"\"\"XPC_SERVICE_NAME=0 (inherited by interactive macOS shells) is NOT a service.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+    monkeypatch.setenv(\"XPC_SERVICE_NAME\", \"0\")\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=True, via_service=False)\n+\n+\n[redacted:email]\n+async def test_restart_without_service_env_uses_detached_path(tmp_path, monkeypatch):\n+    \"\"\"No service-manager env at all falls back to the detached restart.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=True, via_service=False)\n+\n+\n[redacted:email]\n+async def test_restart_under_systemd_uses_service_path(tmp_path, monkeypatch):\n+    \"\"\"INVOCATION_ID (systemd) still routes via the service path.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+    monkeypatch.setenv(\"INVOCATION_ID\", \"abc123\")\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=False, via_service=True)\n"]}]},{"kind":"pr","number":53903,"title":"docs(gateway): warn against custom ExecStopPost kill drop-in (restart loop)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["website/docs/user-guide/messaging/index.md"]},{"kind":"pr","number":57743,"title":"fix(gateway): prevent macOS gateway restart loop on unexpected SIGTERM","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":57746,"title":"fix(gateway): prevent macOS gateway restart loop on unexpected SIGTERM","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":63535,"title":"fix(gateway): prevent macOS gateway restart loop on unexpected SIGTERM","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":85901,"title":"fix(gateway): mark direct systemd stops as planned","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#85901","number":85901}],"category":"wontfix_bydesign","implication":"wontfix"},"merged":false},{"kind":"pr","number":113358,"title":"fix(gateway): mark planned systemd stops","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#24351","number":24351},{"kind":"pr","value":"#42517","number":42517},{"kind":"pr","value":"#42555","number":42555},{"kind":"pr","value":"#85901","number":85901},{"kind":"pr","value":"#24351","number":24351}],"category":"stale_abandoned","implication":"active"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}