{"repo":"NousResearch/hermes-agent","public_id":"5126","display_label":"Gateway process lifecycle and stale PID handling","summary":"These members address bugs and fixes related to gateway startup, restart, and shutdown race conditions, particularly around stale or recycled PID detection and recovery from unexpected termination signals.","subsystem":"apps/desktop","counts":{"members":39,"open_issues":2,"open_issues_distinct":2,"open_prs":3,"closed_issues":15,"merged_prs":3,"attempt_prs":19},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":28341},"resolution":{"state":"partially_resolved","by":"pr-28341,pr-39000,pr-51734"},"freshness":{"state":"verified","last_verified_at":"2026-06-24T07:14:25+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.040377867746288786},"fix_edges":[{"issue_number":21549,"pr_number":51734,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":28135,"pr_number":28341,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":37388,"pr_number":39000,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":43475,"pr_number":51734,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":5126,"title":"`hermes gateway restart` hangs on macOS due to KeepAlive race condition","url":null,"state":"closed"},{"kind":"issue","number":9659,"title":"launchd plist KeepAlive should be unconditional, not SuccessfulExit-based","url":null,"state":"closed"},{"kind":"issue","number":11258,"title":"Gateway self-restart/update can exit cleanly into draining state and stay dead under systemd Restart=on-failure","url":null,"state":"closed"},{"kind":"issue","number":14176,"title":"[Bug]: Gateway hang on clean exit / restart race with stale PID","url":null,"state":"closed"},{"kind":"issue","number":20694,"title":"Gateway self-restart from active WhatsApp chat can self-block graceful drain and always hit drain timeout","url":null,"state":"closed"},{"kind":"issue","number":26198,"title":"hermes gateway restart on macOS (launchd) can leave gateway in stopped state","url":null,"state":"closed"},{"kind":"issue","number":27592,"title":"Telegram gateway can stay down after self-restart if launchd job is unloaded","url":null,"state":"closed"},{"kind":"issue","number":28135,"title":"hermes update via gateway leaves launchd service unrestarted on macOS","url":null,"state":"closed"},{"kind":"issue","number":29180,"title":"[Bug]: /restart does not relaunch the gateway under macOS launchd","url":null,"state":"closed"},{"kind":"issue","number":31048,"title":"hermes update fails to restart gateway on Linux systemd (exit code 75 handoff broken)","url":null,"state":"closed"},{"kind":"issue","number":31162,"title":"[BUG] Gateway not auto-restarted after `hermes update`","url":null,"state":"closed"},{"kind":"issue","number":37388,"title":"[Bug]: gateway launchd KeepAlive (SuccessfulExit:false) doesn't restart after a clean --replace exit on macOS","url":null,"state":"closed"},{"kind":"issue","number":43475,"title":"/restart bricks a launchd-managed gateway on macOS — exits 0, KeepAlive.SuccessfulExit=false won't revive it","url":null,"state":"closed"},{"kind":"issue","number":48746,"title":"issue","url":null,"state":"closed"},{"kind":"issue","number":53861,"title":"macOS: gateway stays down after update — launchd defers respawn in on-demand-only mode, update flow doesn't verify restart","url":null,"state":"open"},{"kind":"issue","number":56524,"title":"Telegram in-band hermes update can leave launchd gateway offline after drain timeout","url":null,"state":"open"},{"kind":"issue","number":62128,"title":"[Bug]: macOS launchd: post-update gateway restart left service dead (respawn pended) and desktop app never relaunched itself","url":null,"state":"closed"},{"kind":"pr","number":5127,"title":"fix: use launchctl kickstart for atomic gateway restart on macOS","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"74ff62f5ac803029b498dfb22b7322a9d3267a4b","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":9689,"title":"fix(gateway): use unconditional KeepAlive in launchd plist","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"c9b62061d43feb61a3a69caec99148623d988a5e","number":0},{"kind":"pr","value":"#37388","number":37388}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":10637,"title":"fix(gateway): use unconditional KeepAlive in launchd plist","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#9689","number":9689}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":14332,"title":"fix(gateway): treat recycled PID with unreadable start_time as stale (#14176)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ccf526964a793371d4df97b4d1ae4f7d6fd63439","number":0},{"kind":"commit","value":"d682f320b35a13084371a541a835e1d988c982b8","number":0},{"kind":"pr","value":"#14176","number":14176},{"kind":"pr","value":"#14128","number":14128}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":21036,"title":"fix(gateway): exclude restart requester from drain","url":null,"state":"open","close_reason":{"refs":[{"kind":"commit","value":"619bd782738adfa87544d0fda7a1114defe86e4c","number":0}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":28200,"title":"fix(gateway): exit code 75 on service restart so launchd relaunches","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"5987b24314b3f40043ba150819ff698bce220db0","number":0},{"kind":"pr","value":"#28341","number":28341}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":28341,"title":"fix(gateway): exit code 75 on service restart so launchd relaunches (#28200)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -17346,6 +17346,19 @@ def restart_signal_handler():\n         )\n         return False  # → sys.exit(1) in the caller\n \n+    # When the gateway is restarting via the service manager (SIGUSR1 →\n+    # launchd_restart or /restart / /update commands), exit with code 75 so\n+    # that launchd's ``KeepAlive → SuccessfulExit → false`` policy treats\n+    # the exit as *unsuccessful* and relaunches the service.  This mirrors\n+    # the systemd ``RestartForceExitStatus=75`` convention already used by\n+    # the systemd unit template.\n+    if runner._restart_via_service:\n+        logger.info(\n+            \"Exiting with code 75 (service-restart requested) so \"\n+            \"launchd KeepAlive relaunches the gateway.\"\n+        )\n+        raise SystemExit(75)\n+\n     return True\n \n \n"]}]},{"kind":"pr","number":29181,"title":"fix(gateway): detect macOS launchd in service-restart path","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"abc3662bf6076045e4d4dc1e14a74cb35d69b86e","number":0},{"kind":"pr","value":"#43475","number":43475}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":34464,"title":"fix(gateway): refresh systemd units before update restart","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":37411,"title":"fix(gateway): use unconditional KeepAlive in launchd plist so gateway restarts after clean --replace exit (#37388)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#39000","number":39000},{"kind":"pr","value":"#38796","number":38796}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":37534,"title":"fix(launchd): use unconditional KeepAlive so gateway restarts after clean exit #37388","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#39000","number":39000},{"kind":"pr","value":"#38796","number":38796}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":38796,"title":"fix(cli): launchd KeepAlive unconditional restart (#37388)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"c9b62061d","number":0}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":39000,"title":"fix(cli): launchd KeepAlive unconditional restart (#37388)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/gateway.py","scripts/release.py","tests/hermes_cli/test_gateway_service.py"],"fix_hunks":[{"file":"hermes_cli/gateway.py","hunks":["@@ -227,9 +227,8 @@ def _graceful_restart_via_sigusr1(pid: int, drain_timeout: float) -> bool:\n \n     SIGUSR1 is wired in gateway/run.py to ``request_restart(via_service=True)``\n     which drains in-flight agent runs (up to ``agent.restart_drain_timeout``\n-    seconds), then exits.  systemd relaunches clean exits via\n-    ``Restart=always``; launchd still uses a non-zero planned-restart exit\n-    because its plist has ``KeepAlive.SuccessfulExit = false``.\n+    seconds), then exits.  Both systemd (``Restart=always``) and launchd\n+    (unconditional ``<key>KeepAlive</key><true/>``) restart on any exit.\n \n     This is the drain-aware alternative to ``systemctl restart`` / ``SIGTERM``,\n     which SIGKILL in-flight agents after a short timeout.","@@ -3083,10 +3082,7 @@ def generate_launchd_plist() -> str:\n     <true/>\n     \n     <key>KeepAlive</key>\n-    <dict>\n-        <key>SuccessfulExit</key>\n-        <false/>\n-    </dict>\n+    <true/>\n     \n     <key>StandardOutPath</key>\n     <string>{log_dir}/gateway.log</string>","@@ -3249,7 +3245,7 @@ def launchd_stop():\n         pass\n     # bootout unloads the service definition so KeepAlive doesn't respawn\n     # the process.  A plain `kill SIGTERM` only signals the process — launchd\n-    # immediately restarts it because KeepAlive.SuccessfulExit = false.\n+    # immediately restarts it because KeepAlive is unconditionally true.\n     # `hermes gateway start` re-bootstraps when it detects the job is unloaded.\n     try:\n         subprocess.run([\"launchctl\", \"bootout\", target], check=True, timeout=90)"]},{"file":"scripts/release.py","hunks":["@@ -72,6 +72,7 @@\n     \"[redacted:email]\": \"kchuang1015\",\n     \"[redacted:email]\": \"MaheshtheDev\",\n     \"[redacted:email]\": \"kyssta-exe\",\n+    \"[redacted:email]\": \"ashishpatel26\",\n     \"[redacted:email]\": \"fujinice\",\n     \"[redacted:email]\": \"carltonawong\",\n     \"[redacted:email]\": \"EvilHumphrey\","]},{"file":"tests/hermes_cli/test_gateway_service.py","hunks":["@@ -2579,3 +2579,24 @@ def test_launchd_workingdirectory_is_hermes_home(self, tmp_path, monkeypatch):\n         assert m, \"plist has no WorkingDirectory entry\"\n         assert Path(m.group(1)).resolve() == home.resolve()\n         assert \"/.worktrees/\" not in m.group(1)\n+\n+    def test_launchd_plist_keepalive_unconditional(self, tmp_path, monkeypatch):\n+        \"\"\"KeepAlive must be unconditional <true/> so the gateway restarts on clean exits.\n+\n+        Bug #37388: the old ``KeepAlive.SuccessfulExit = false`` dict form meant\n+        launchd would NOT restart after a zero-exit (e.g. ``gateway run --replace``\n+        causes the old instance to exit cleanly).  Switching to the scalar\n+        ``<key>KeepAlive</key><true/>`` makes launchd restart regardless of exit code.\n+        \"\"\"\n+        home = tmp_path / \".hermes\"\n+        home.mkdir()\n+        monkeypatch.setattr(gateway_cli, \"get_hermes_home\", lambda: home)\n+        plist = gateway_cli.generate_launchd_plist()\n+\n+        # Scalar <true/> must be present immediately after the KeepAlive key\n+        assert \"<key>KeepAlive</key>\" in plist\n+        # The unconditional form\n+        assert \"<key>KeepAlive</key>\\n    <true/>\" in plist\n+        # The old conditional dict form must NOT appear\n+        assert \"SuccessfulExit\" not in plist\n+        assert \"<key>KeepAlive</key>\\n    <dict>\" not in plist\n"]}]},{"kind":"pr","number":42338,"title":"fix(gateway): use canonical _pid_exists in restart watcher; /restart …","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":43498,"title":"fix(gateway): exit non-zero on /restart so launchd revives the gateway","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#51734","number":51734},{"kind":"pr","value":"#43475","number":43475}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":43596,"title":"fix(gateway): exit non-zero on macOS launchd restart path (#43475)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#51734","number":51734},{"kind":"pr","value":"#43475","number":43475}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":43888,"title":"fix(gateway): detect launchd in the /restart service-manager probe","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"abc3662bf6076045e4d4dc1e14a74cb35d69b86e","number":0},{"kind":"pr","value":"#43475","number":43475},{"kind":"pr","value":"#43475","number":43475},{"kind":"pr","value":"#51734","number":51734}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":51112,"title":"fix(gateway): detect launchd via XPC_SERVICE_NAME for Telegram /restart path","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"abc3662bf6076045e4d4dc1e14a74cb35d69b86e","number":0},{"kind":"pr","value":"#43475","number":43475}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":51734,"title":"fix(gateway): detect launchd in /restart service-manager probe (#43475)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/slash_commands.py","scripts/release.py","tests/gateway/test_restart_service_detection.py"],"fix_hunks":[{"file":"gateway/slash_commands.py","hunks":["@@ -960,7 +960,15 @@ async def _handle_restart_command(self, event: MessageEvent) -> Union[str, Ephem\n         # us.  The detached subprocess approach (setsid + bash) doesn't work\n         # under systemd (KillMode=mixed kills the cgroup) or Docker (tini\n         # exits when the gateway dies, taking the detached helper with it).\n-        _under_service = bool(os.environ.get(\"INVOCATION_ID\"))  # systemd sets this\n+        # systemd sets INVOCATION_ID; launchd sets XPC_SERVICE_NAME to the\n+        # job label.  Without the launchd check, macOS /restart takes the\n+        # detached path and exits 0, which KeepAlive.SuccessfulExit=false\n+        # treats as a deliberate stop — the gateway stays dead until next\n+        # login.  Interactive macOS shells inherit XPC_SERVICE_NAME=0, so\n+        # \"0\" must count as not-under-launchd.\n+        _under_service = bool(os.environ.get(\"INVOCATION_ID\")) or os.environ.get(\n+            \"XPC_SERVICE_NAME\", \"0\"\n+        ) not in (\"\", \"0\")\n         _in_container = os.path.exists(\"/.dockerenv\") or os.path.exists(\"/run/.containerenv\")\n         if _under_service or _in_container:\n             self.request_restart(detached=False, via_service=True)"]},{"file":"scripts/release.py","hunks":["@@ -1547,6 +1547,7 @@\n     \"[redacted:email]\": \"bird\",  # PR #25219 (gateway docker exit-75 restart)\n     \"[redacted:email]\": \"davidcampbelldc\",  # PR #26834 (web_server proxy_headers=False)\n     \"[redacted:email]\": \"davidcampbelldc\",\n+    \"[redacted:email]\": \"chazmaniandinkle\",  # PR #43888 (launchd /restart detection)\n     \"[redacted:email]\": \"hehehe0803\",  # PR #26212 salvage (codex kanban writable root)\n     \"[redacted:email]\": \"hehehe0803\",\n     \"[redacted:email]\": \"kasunvinod\",  # PR #24126 salvage (codex timeout propagation)"]},{"file":"tests/gateway/test_restart_service_detection.py","hunks":["@@ -0,0 +1,85 @@\n+\"\"\"Tests for /restart service-manager detection (launchd vs interactive).\n+\n+The /restart handler routes through ``request_restart(via_service=True)``\n+when a service manager supervises the gateway, so the process exits with\n+the service-restart code and the manager relaunches it.  Under macOS\n+launchd the plist uses ``KeepAlive.SuccessfulExit=false`` — a clean exit 0\n+is treated as a deliberate stop and the gateway stays dead (#43475) — so\n+launchd must be detected here in the handler, not only at the exit-code\n+site (which never runs unless ``via_service=True`` is already set).\n+\n+launchd sets ``XPC_SERVICE_NAME`` to the job label for processes it\n+spawns.  Interactive macOS shells inherit ``XPC_SERVICE_NAME=0`` (a\n+truthy string), so the probe must treat ``\"0\"`` as not-under-launchd:\n+routing an unsupervised interactive gateway to the service path would\n+make it exit non-zero with nothing to revive it.\n+\"\"\"\n+from unittest.mock import MagicMock\n+\n+import pytest\n+\n+import gateway.run as gateway_run\n+from gateway.platforms.base import MessageEvent, MessageType\n+from tests.gateway.restart_test_helpers import make_restart_runner, make_restart_source\n+\n+\n+def _make_restart_event(update_id: int | None = 100) -> MessageEvent:\n+    return MessageEvent(\n+        text=\"/restart\",\n+        message_type=MessageType.TEXT,\n+        source=make_restart_source(),\n+        message_id=\"m1\",\n+        platform_update_id=update_id,\n+    )\n+\n+\n+def _make_runner_with_mock_restart(tmp_path, monkeypatch):\n+    monkeypatch.setattr(gateway_run, \"_hermes_home\", tmp_path)\n+    monkeypatch.delenv(\"INVOCATION_ID\", raising=False)\n+    monkeypatch.delenv(\"XPC_SERVICE_NAME\", raising=False)\n+    runner, _adapter = make_restart_runner()\n+    runner.request_restart = MagicMock(return_value=True)\n+    return runner\n+\n+\n[redacted:email]\n+async def test_restart_under_launchd_uses_service_path(tmp_path, monkeypatch):\n+    \"\"\"launchd job label in XPC_SERVICE_NAME routes /restart via the service path.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+    monkeypatch.setenv(\"XPC_SERVICE_NAME\", \"ai.hermes.gateway\")\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=False, via_service=True)\n+\n+\n[redacted:email]\n+async def test_restart_in_interactive_macos_shell_uses_detached_path(tmp_path, monkeypatch):\n+    \"\"\"XPC_SERVICE_NAME=0 (inherited by interactive macOS shells) is NOT a service.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+    monkeypatch.setenv(\"XPC_SERVICE_NAME\", \"0\")\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=True, via_service=False)\n+\n+\n[redacted:email]\n+async def test_restart_without_service_env_uses_detached_path(tmp_path, monkeypatch):\n+    \"\"\"No service-manager env at all falls back to the detached restart.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=True, via_service=False)\n+\n+\n[redacted:email]\n+async def test_restart_under_systemd_uses_service_path(tmp_path, monkeypatch):\n+    \"\"\"INVOCATION_ID (systemd) still routes via the service path.\"\"\"\n+    runner = _make_runner_with_mock_restart(tmp_path, monkeypatch)\n+    monkeypatch.setenv(\"INVOCATION_ID\", \"abc123\")\n+\n+    await runner._handle_restart_command(_make_restart_event())\n+\n+    runner.request_restart.assert_called_once_with(detached=False, via_service=True)\n"]}]},{"kind":"pr","number":53869,"title":"fix(cli): verify gateway liveness after macOS launchd restart (#53861)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"1bf93660f1","number":0},{"kind":"commit","value":"bee489d563","number":0},{"kind":"pr","value":"#88848","number":88848},{"kind":"pr","value":"#74973","number":74973},{"kind":"pr","value":"#53861","number":53861},{"kind":"pr","value":"#99685","number":99685}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":56908,"title":"fix(gateway): wait for launchd restart readiness in-band (#56524)","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#56524","number":56524},{"kind":"pr","value":"#42450","number":42450},{"kind":"pr","value":"#43181","number":43181},{"kind":"pr","value":"#10317","number":10317}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":72752,"title":"fix(gateway): verify launchd service survivability after restart","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"1bf93660f1d","number":0},{"kind":"commit","value":"ac2e4207c11","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}