{"repo":"NousResearch/hermes-agent","public_id":"2894","display_label":"Gateway restart safety across systemd and containers","summary":"These issues and PRs address a cluster of related problems with safely restarting the gateway process across different environments (systemd, Docker, Podman), including self-kill bugs, cgroup inheritance, deadlocks, and preventing agents or slash commands from accidentally terminating the gateway permanently.","subsystem":"apps/desktop","counts":{"members":9,"open_issues":0,"open_issues_distinct":0,"open_prs":1,"closed_issues":4,"merged_prs":2,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":37475,"merged_at":"2026-06-19T06:23:45+00:00"},"resolution":{"state":"partially_resolved","by":"pr-37475"},"freshness":{"state":"verified","last_verified_at":"2026-06-19T06:23:45+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.1688888888888889},"fix_edges":[{"issue_number":2894,"pr_number":9850,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-04-14T21:21:58+00:00"},{"issue_number":6631,"pr_number":9850,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-14T21:21:58+00:00"},{"issue_number":8104,"pr_number":9850,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-14T21:21:58+00:00"},{"issue_number":37453,"pr_number":37475,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-19T06:23:45+00:00"}],"members":[{"kind":"issue","number":2894,"title":"[Feature]: safe /restart slash command for in-chat gateway restarts","url":null,"state":"closed"},{"kind":"issue","number":21189,"title":"cron: Cron jobs inherit gateway cgroup, cannot safely restart the service","url":null,"state":"closed"},{"kind":"issue","number":21912,"title":"[Bug]: gateway restart self-kills: KillMode=mixed SIGKILLs the restart bash process","url":null,"state":"closed"},{"kind":"issue","number":37453,"title":"gateway: systemctl restart from inside terminal tool deadlocks — kills itself mid-execution","url":null,"state":"closed"},{"kind":"pr","number":2929,"title":"feat(gateway): add safe in-chat /restart command","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#9850","number":9850}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":9850,"title":"fix: notify active sessions on gateway shutdown + update health check","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","hermes_cli/main.py","tests/gateway/restart_test_helpers.py","tests/gateway/test_restart_drain.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -1391,6 +1391,65 @@ def _interrupt_running_agents(self, reason: str) -> None:\n             except Exception as e:\n                 logger.debug(\"Failed interrupting agent during shutdown: %s\", e)\n \n+    async def _notify_active_sessions_of_shutdown(self) -> None:\n+        \"\"\"Send a notification to every chat with an active agent.\n+\n+        Called at the very start of stop() — adapters are still connected so\n+        messages can be delivered.  Best-effort: individual send failures are\n+        logged and swallowed so they never block the shutdown sequence.\n+        \"\"\"\n+        active = self._snapshot_running_agents()\n+        if not active:\n+            return\n+\n+        action = \"restarting\" if self._restart_requested else \"shutting down\"\n+        hint = (\n+            \"Your current task will be interrupted. \"\n+            \"Use /retry after restart to continue.\"\n+            if self._restart_requested\n+            else \"Your current task will be interrupted.\"\n+        )\n+        msg = f\"⚠️ Gateway {action} — {hint}\"\n+\n+        notified: set = set()\n+        for session_key in active:\n+            # Parse platform + chat_id from the session key.\n+            # Format: agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]\n+            parts = session_key.split(\":\")\n+            if len(parts) < 5:\n+                continue\n+            platform_str = parts[2]\n+            chat_id = parts[4]\n+\n+            # Deduplicate: one notification per chat, even if multiple\n+            # sessions (different users/threads) share the same chat.\n+            dedup_key = (platform_str, chat_id)\n+            if dedup_key in notified:\n+                continue\n+\n+            try:\n+                platform = Platform(platform_str)\n+                adapter = self.adapters.get(platform)\n+                if not adapter:\n+                    continue\n+\n+                # Include thread_id if present so the message lands in the\n+                # correct forum topic / thread.\n+                thread_id = parts[5] if len(parts) > 5 else None\n+                metadata = {\"thread_id\": thread_id} if thread_id else None\n+\n+                await adapter.send(chat_id, msg, metadata=metadata)\n+                notified.add(dedup_key)\n+                logger.info(\n+                    \"Sent shutdown notification to %s:%s\",\n+                    platform_str, chat_id,\n+                )\n+            except Exception as e:\n+                logger.debug(\n+                    \"Failed to send shutdown notification to %s:%s: %s\",\n+                    platform_str, chat_id, e,\n+                )\n+\n     def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:\n         for agent in active_agents.values():\n             try:","@@ -2018,6 +2077,10 @@ async def _stop_impl() -> None:\n             self._running = False\n             self._draining = True\n \n+            # Notify all chats with active agents BEFORE draining.\n+            # Adapters are still connected here, so messages can be sent.\n+            await self._notify_active_sessions_of_shutdown()\n+\n             timeout = self._restart_drain_timeout\n             active_agents, timed_out = await self._drain_active_agents(timeout)\n             if timed_out:","@@ -2088,12 +2151,23 @@ async def _stop_impl() -> None:\n \n             # Write a clean-shutdown marker so the next startup knows this\n             # wasn't a crash.  suspend_recently_active() only needs to run\n-            # after unexpected exits — graceful shutdowns already drain\n-            # active agents, so there's no stuck-session risk.\n-            try:\n-                (_hermes_home / \".clean_shutdown\").touch()\n-            except Exception:\n-                pass\n+            # after unexpected exits.  However, if the drain timed out and\n+            # agents were force-interrupted, their sessions may be in an\n+            # incomplete state (trailing tool response, no final assistant\n+            # message).  Skip the marker in that case so the next startup\n+            # suspends those sessions — giving users a clean slate instead\n+            # of resuming a half-finished tool loop.\n+            if not timed_out:\n+                try:\n+                    (_hermes_home / \".clean_shutdown\").touch()\n+                except Exception:\n+                    pass\n+            else:\n+                logger.info(\n+                    \"Skipping .clean_shutdown marker — drain timed out with \"\n+                    \"interrupted agents; next startup will suspend recently \"\n+                    \"active sessions.\"\n+                )\n \n             if self._restart_requested and self._restart_via_service:\n                 self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE"]},{"file":"hermes_cli/main.py","hunks":["@@ -4036,7 +4036,40 @@ def cmd_update(args):\n                                     capture_output=True, text=True, timeout=15,\n                                 )\n                                 if restart.returncode == 0:\n-                                    restarted_services.append(svc_name)\n+                                    # Verify the service actually survived the\n+                                    # restart.  systemctl restart returns 0 even\n+                                    # if the new process crashes immediately.\n+                                    import time as _time\n+                                    _time.sleep(3)\n+                                    verify = subprocess.run(\n+                                        scope_cmd + [\"is-active\", svc_name],\n+                                        capture_output=True, text=True, timeout=5,\n+                                    )\n+                                    if verify.stdout.strip() == \"active\":\n+                                        restarted_services.append(svc_name)\n+                                    else:\n+                                        # Retry once — transient startup failures\n+                                        # (stale module cache, import race) often\n+                                        # resolve on the second attempt.\n+                                        print(f\"  ⚠ {svc_name} died after restart, retrying...\")\n+                                        retry = subprocess.run(\n+                                            scope_cmd + [\"restart\", svc_name],\n+                                            capture_output=True, text=True, timeout=15,\n+                                        )\n+                                        _time.sleep(3)\n+                                        verify2 = subprocess.run(\n+                                            scope_cmd + [\"is-active\", svc_name],\n+                                            capture_output=True, text=True, timeout=5,\n+                                        )\n+                                        if verify2.stdout.strip() == \"active\":\n+                                            restarted_services.append(svc_name)\n+                                            print(f\"  ✓ {svc_name} recovered on retry\")\n+                                        else:\n+                                            print(\n+                                                f\"  ✗ {svc_name} failed to stay running after restart.\\n\"\n+                                                f\"    Check logs: journalctl --user -u {svc_name} --since '2 min ago'\\n\"\n+                                                f\"    Restart manually: systemctl {'--user ' if scope == 'user' else ''}restart {svc_name}\"\n+                                            )\n                                 else:\n                                     print(f\"  ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}\")\n                     except (FileNotFoundError, subprocess.TimeoutExpired):"]},{"file":"tests/gateway/restart_test_helpers.py","hunks":["@@ -93,6 +93,12 @@ def make_restart_runner(\n     runner._running_agent_count = GatewayRunner._running_agent_count.__get__(\n         runner, GatewayRunner\n     )\n+    runner._snapshot_running_agents = GatewayRunner._snapshot_running_agents.__get__(\n+        runner, GatewayRunner\n+    )\n+    runner._notify_active_sessions_of_shutdown = (\n+        GatewayRunner._notify_active_sessions_of_shutdown.__get__(runner, GatewayRunner)\n+    )\n     runner._launch_detached_restart_command = GatewayRunner._launch_detached_restart_command.__get__(\n         runner, GatewayRunner\n     )"]},{"file":"tests/gateway/test_restart_drain.py","hunks":["@@ -161,3 +161,84 @@ def fake_popen(cmd, **kwargs):\n     assert kwargs[\"start_new_session\"] is True\n     assert kwargs[\"stdout\"] is subprocess.DEVNULL\n     assert kwargs[\"stderr\"] is subprocess.DEVNULL\n+\n+\n+# ── Shutdown notification tests ──────────────────────────────────────\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_sent_to_active_sessions():\n+    \"\"\"Active sessions receive a notification when the gateway starts shutting down.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    source = make_restart_source(chat_id=\"999\", chat_type=\"dm\")\n+    session_key = f\"agent:main:telegram:dm:999\"\n+    runner._running_agents[session_key] = MagicMock()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 1\n+    assert \"shutting down\" in adapter.sent[0]\n+    assert \"interrupted\" in adapter.sent[0]\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_says_restarting_when_restart_requested():\n+    \"\"\"When _restart_requested is True, the message says 'restarting' and mentions /retry.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    runner._restart_requested = True\n+    session_key = \"agent:main:telegram:dm:999\"\n+    runner._running_agents[session_key] = MagicMock()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 1\n+    assert \"restarting\" in adapter.sent[0]\n+    assert \"/retry\" in adapter.sent[0]\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_deduplicates_per_chat():\n+    \"\"\"Multiple sessions in the same chat only get one notification.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    # Two sessions (different users) in the same chat\n+    runner._running_agents[\"agent:main:telegram:group:chat1:u1\"] = MagicMock()\n+    runner._running_agents[\"agent:main:telegram:group:chat1:u2\"] = MagicMock()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 1\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_skipped_when_no_active_agents():\n+    \"\"\"No notification is sent when there are no active agents.\"\"\"\n+    runner, adapter = make_restart_runner()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 0\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_ignores_pending_sentinels():\n+    \"\"\"Pending sentinels (not-yet-started agents) don't trigger notifications.\"\"\"\n+    from gateway.run import _AGENT_PENDING_SENTINEL\n+\n+    runner, adapter = make_restart_runner()\n+    runner._running_agents[\"agent:main:telegram:dm:999\"] = _AGENT_PENDING_SENTINEL\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 0\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_send_failure_does_not_block():\n+    \"\"\"If sending a notification fails, the method still completes.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    adapter.send = AsyncMock(side_effect=Exception(\"network error\"))\n+    session_key = \"agent:main:telegram:dm:999\"\n+    runner._running_agents[session_key] = MagicMock()\n+\n+    # Should not raise\n+    await runner._notify_active_sessions_of_shutdown()\n"]}]},{"kind":"pr","number":21200,"title":"feat(gateway): add --graceful flag to restart for cgroup-safe SIGUSR1 restart","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#21912","number":21912}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":37475,"title":"fix(terminal): block gateway lifecycle commands from inside the gateway process","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/cron.py","tests/hermes_cli/test_gateway_restart_loop.py","tools/terminal_tool.py"],"fix_hunks":[{"file":"hermes_cli/cron.py","hunks":["@@ -25,7 +25,7 @@\n     r\"(?i)\"\n     r\"(hermes\\s+gateway\\s+(restart|stop|start))\"\n     r\"|(launchctl\\s+(kickstart|unload|load|stop|restart)\\s+.*hermes)\"\n-    r\"|(systemctl\\s+(restart|stop|start)\\s+.*hermes)\"\n+    r\"|(systemctl\\s+(-\\S+\\s+)*(restart|stop|start)\\s+.*hermes)\"\n     r\"|(p?kill\\s+.*hermes.*gateway)\"\n )\n "]},{"file":"tests/hermes_cli/test_gateway_restart_loop.py","hunks":["@@ -6,6 +6,7 @@\n - _contains_gateway_lifecycle_command pattern matching\n \"\"\"\n \n+import json\n import os\n from argparse import Namespace\n ","@@ -250,3 +251,109 @@ def _sentinel(*a, **k):\n         args = Namespace(gateway_command=\"restart\", all=False, system=False)\n         with pytest.raises(_Reached):\n             gw.gateway_command(args)\n+\n+\n+# ---------------------------------------------------------------------------\n+# Defense 3: terminal_tool hard-blocks gateway lifecycle commands inside gateway\n+# ---------------------------------------------------------------------------\n+\n+class TestTerminalToolGatewayLifecycleGuard:\n+    \"\"\"terminal_tool must refuse gateway lifecycle commands when _HERMES_GATEWAY=1.\n+\n+    Issue #37453: systemctl --user restart hermes-gateway runs as a child of the\n+    gateway process.  When systemd delivers SIGTERM the gateway kills its own\n+    restart command mid-execution — the service may never restart.  The guard\n+    must fire before execution, unconditionally (force=True cannot bypass it).\n+    \"\"\"\n+\n+    def _make_fake_env(self):\n+        class _FakeEnv:\n+            env = {}\n+            def execute(self, command, **kwargs):  # pragma: no cover\n+                raise AssertionError(\"execute must not be reached\")\n+        return _FakeEnv()\n+\n+    def _minimal_config(self):\n+        return {\"env_type\": \"local\", \"cwd\": \"/tmp\", \"timeout\": 60, \"lifetime_seconds\": 3600}\n+\n+    def _patch_env(self, monkeypatch, fake_env, *, inside_gateway: bool):\n+        import tools.terminal_tool as tt\n+        eid = \"default\"\n+        monkeypatch.setattr(tt, \"_active_environments\", {eid: fake_env})\n+        monkeypatch.setattr(tt, \"_last_activity\", {eid: 0.0})\n+        monkeypatch.setattr(tt, \"_task_env_overrides\", {})\n+        monkeypatch.setattr(tt, \"_get_env_config\", self._minimal_config)\n+        if inside_gateway:\n+            monkeypatch.setenv(\"_HERMES_GATEWAY\", \"1\")\n+        else:\n+            monkeypatch.delenv(\"_HERMES_GATEWAY\", raising=False)\n+\n+    @pytest.mark.parametrize(\"cmd\", [\n+        \"systemctl restart hermes-gateway\",\n+        \"systemctl --user restart hermes-gateway\",\n+        \"systemctl stop hermes-gateway.service\",\n+        \"hermes gateway restart\",\n+        \"launchctl kickstart gui/501/ai.hermes.gateway\",\n+        \"pkill -f hermes.*gateway\",\n+    ])\n+    def test_blocks_lifecycle_commands_inside_gateway(self, monkeypatch, cmd):\n+        import tools.terminal_tool as tt\n+        self._patch_env(monkeypatch, self._make_fake_env(), inside_gateway=True)\n+\n+        result = json.loads(tt.terminal_tool(command=cmd))\n+\n+        assert result[\"exit_code\"] == 1\n+        assert \"Blocked\" in result[\"error\"]\n+\n+    def test_force_true_cannot_bypass_block(self, monkeypatch):\n+        import tools.terminal_tool as tt\n+        self._patch_env(monkeypatch, self._make_fake_env(), inside_gateway=True)\n+\n+        result = json.loads(tt.terminal_tool(\n+            command=\"systemctl restart hermes-gateway\", force=True\n+        ))\n+\n+        assert result[\"exit_code\"] == 1\n+        assert \"Blocked\" in result[\"error\"]\n+\n+    def test_safe_systemctl_commands_pass_through(self, monkeypatch):\n+        \"\"\"Non-hermes systemctl commands must not be blocked by this guard.\"\"\"\n+        import tools.terminal_tool as tt\n+\n+        calls = []\n+\n+        class _FakeEnv:\n+            env = {}\n+            def execute(self, command, **kwargs):\n+                calls.append(command)\n+                return {\"output\": \"Active: running\", \"returncode\": 0}\n+\n+        self._patch_env(monkeypatch, _FakeEnv(), inside_gateway=True)\n+        monkeypatch.setattr(tt, \"_check_all_guards\", lambda cmd, env: {\"approved\": True})\n+\n+        result = json.loads(tt.terminal_tool(command=\"systemctl status nginx\"))\n+\n+        assert result[\"exit_code\"] == 0\n+        assert calls == [\"systemctl status nginx\"]\n+\n+    def test_guard_inactive_outside_gateway(self, monkeypatch):\n+        \"\"\"Without _HERMES_GATEWAY=1 the lifecycle guard must not fire.\"\"\"\n+        import tools.terminal_tool as tt\n+\n+        calls = []\n+\n+        class _FakeEnv:\n+            env = {}\n+            def execute(self, command, **kwargs):\n+                calls.append(command)\n+                return {\"output\": \"restarting...\", \"returncode\": 0}\n+\n+        self._patch_env(monkeypatch, _FakeEnv(), inside_gateway=False)\n+        monkeypatch.setattr(tt, \"_check_all_guards\", lambda cmd, env: {\"approved\": True})\n+\n+        result = json.loads(tt.terminal_tool(command=\"systemctl restart hermes-gateway\"))\n+\n+        # Outside the gateway the lifecycle guard doesn't block — the normal\n+        # approval flow handles it (here mocked as approved).\n+        assert result[\"exit_code\"] == 0\n+        assert calls == [\"systemctl restart hermes-gateway\"]"]},{"file":"tools/terminal_tool.py","hunks":["@@ -1959,6 +1959,29 @@ def terminal_tool(\n                         env = new_env\n                     logger.info(\"%s environment ready for task %s\", env_type, effective_task_id[:8])\n \n+        # Hard-block: gateway lifecycle commands (systemctl/launchctl/hermes\n+        # restart|stop targeting hermes-gateway) must never run inside the\n+        # gateway process itself. The restart would SIGTERM the gateway, which\n+        # kills this very subprocess before it can complete — the service may\n+        # never restart. This mirrors the `hermes gateway restart` guard in\n+        # hermes_cli/gateway.py and the cron-path guard in hermes_cli/cron.py,\n+        # but applies unconditionally (force=True cannot help here).\n+        if os.environ.get(\"_HERMES_GATEWAY\") == \"1\":\n+            from hermes_cli.cron import _contains_gateway_lifecycle_command\n+            if _contains_gateway_lifecycle_command(command):\n+                return json.dumps({\n+                    \"output\": \"\",\n+                    \"exit_code\": 1,\n+                    \"error\": (\n+                        \"Blocked: cannot restart or stop the gateway from inside the \"\n+                        \"gateway process. The gateway would kill this command before \"\n+                        \"it could complete (SIGTERM propagates to child processes). \"\n+                        \"Run `hermes gateway restart` from a separate shell outside \"\n+                        \"the running gateway.\"\n+                    ),\n+                    \"status\": \"error\",\n+                }, ensure_ascii=False)\n+\n         # Pre-exec security checks (tirith + dangerous command detection)\n         # Skip check if force=True (user has confirmed they want to run it)\n         approval_note = None\n"]}]},{"kind":"pr","number":37603,"title":"Fix desktop IME enter handling and gateway restart guards","url":null,"state":"open","close_reason":{"refs":[{"kind":"commit","value":"e551da6ddb39a712c93e6c48a16aa748e2c7bc95","number":0},{"kind":"commit","value":"245b95b09470bb3887943122a7d0de5bf20da055","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}