{"repo":"NousResearch/hermes-agent","public_id":"8104","display_label":"Gateway restart and shutdown handling, including systemd-managed services","summary":"Members concern gateway lifecycle behavior: one issue reports that the /restart slash command hardcodes detached=True and via_service=False, which is described as permanently killing a systemd-managed gateway. A related PR proposes detecting systemd in /restart, and another PR covers notifying active sessions on shutdown and updating a health check, which is adjacent rather than clearly the same problem.","subsystem":null,"counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":9850,"merged_at":"2026-04-14T21:21:58+00:00"},"resolution":{"state":"resolved","by":"pr-9850"},"freshness":{"state":"stale","last_verified_at":"2026-04-14T21:21:58+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":2894,"pr_number":9850,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-04-14T21:21:58+00:00"},{"issue_number":6631,"pr_number":9850,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-14T21:21:58+00:00"},{"issue_number":8104,"pr_number":9850,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-14T21:21:58+00:00"}],"members":[{"kind":"issue","number":8104,"title":"[Bug]: /restart slash command hardcodes detached=True, via_service=False — kills systemd-managed gateway permanently","url":null,"state":"closed"},{"kind":"pr","number":8150,"title":"fix(gateway): detect systemd in /restart to avoid killing service permanently","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"276d20e62c99defaf9ec0d4ee5291ba43039c489","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":9850,"title":"fix: notify active sessions on gateway shutdown + update health check","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","hermes_cli/main.py","tests/gateway/restart_test_helpers.py","tests/gateway/test_restart_drain.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -1391,6 +1391,65 @@ def _interrupt_running_agents(self, reason: str) -> None:\n             except Exception as e:\n                 logger.debug(\"Failed interrupting agent during shutdown: %s\", e)\n \n+    async def _notify_active_sessions_of_shutdown(self) -> None:\n+        \"\"\"Send a notification to every chat with an active agent.\n+\n+        Called at the very start of stop() — adapters are still connected so\n+        messages can be delivered.  Best-effort: individual send failures are\n+        logged and swallowed so they never block the shutdown sequence.\n+        \"\"\"\n+        active = self._snapshot_running_agents()\n+        if not active:\n+            return\n+\n+        action = \"restarting\" if self._restart_requested else \"shutting down\"\n+        hint = (\n+            \"Your current task will be interrupted. \"\n+            \"Use /retry after restart to continue.\"\n+            if self._restart_requested\n+            else \"Your current task will be interrupted.\"\n+        )\n+        msg = f\"⚠️ Gateway {action} — {hint}\"\n+\n+        notified: set = set()\n+        for session_key in active:\n+            # Parse platform + chat_id from the session key.\n+            # Format: agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]\n+            parts = session_key.split(\":\")\n+            if len(parts) < 5:\n+                continue\n+            platform_str = parts[2]\n+            chat_id = parts[4]\n+\n+            # Deduplicate: one notification per chat, even if multiple\n+            # sessions (different users/threads) share the same chat.\n+            dedup_key = (platform_str, chat_id)\n+            if dedup_key in notified:\n+                continue\n+\n+            try:\n+                platform = Platform(platform_str)\n+                adapter = self.adapters.get(platform)\n+                if not adapter:\n+                    continue\n+\n+                # Include thread_id if present so the message lands in the\n+                # correct forum topic / thread.\n+                thread_id = parts[5] if len(parts) > 5 else None\n+                metadata = {\"thread_id\": thread_id} if thread_id else None\n+\n+                await adapter.send(chat_id, msg, metadata=metadata)\n+                notified.add(dedup_key)\n+                logger.info(\n+                    \"Sent shutdown notification to %s:%s\",\n+                    platform_str, chat_id,\n+                )\n+            except Exception as e:\n+                logger.debug(\n+                    \"Failed to send shutdown notification to %s:%s: %s\",\n+                    platform_str, chat_id, e,\n+                )\n+\n     def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:\n         for agent in active_agents.values():\n             try:","@@ -2018,6 +2077,10 @@ async def _stop_impl() -> None:\n             self._running = False\n             self._draining = True\n \n+            # Notify all chats with active agents BEFORE draining.\n+            # Adapters are still connected here, so messages can be sent.\n+            await self._notify_active_sessions_of_shutdown()\n+\n             timeout = self._restart_drain_timeout\n             active_agents, timed_out = await self._drain_active_agents(timeout)\n             if timed_out:","@@ -2088,12 +2151,23 @@ async def _stop_impl() -> None:\n \n             # Write a clean-shutdown marker so the next startup knows this\n             # wasn't a crash.  suspend_recently_active() only needs to run\n-            # after unexpected exits — graceful shutdowns already drain\n-            # active agents, so there's no stuck-session risk.\n-            try:\n-                (_hermes_home / \".clean_shutdown\").touch()\n-            except Exception:\n-                pass\n+            # after unexpected exits.  However, if the drain timed out and\n+            # agents were force-interrupted, their sessions may be in an\n+            # incomplete state (trailing tool response, no final assistant\n+            # message).  Skip the marker in that case so the next startup\n+            # suspends those sessions — giving users a clean slate instead\n+            # of resuming a half-finished tool loop.\n+            if not timed_out:\n+                try:\n+                    (_hermes_home / \".clean_shutdown\").touch()\n+                except Exception:\n+                    pass\n+            else:\n+                logger.info(\n+                    \"Skipping .clean_shutdown marker — drain timed out with \"\n+                    \"interrupted agents; next startup will suspend recently \"\n+                    \"active sessions.\"\n+                )\n \n             if self._restart_requested and self._restart_via_service:\n                 self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE"]},{"file":"hermes_cli/main.py","hunks":["@@ -4036,7 +4036,40 @@ def cmd_update(args):\n                                     capture_output=True, text=True, timeout=15,\n                                 )\n                                 if restart.returncode == 0:\n-                                    restarted_services.append(svc_name)\n+                                    # Verify the service actually survived the\n+                                    # restart.  systemctl restart returns 0 even\n+                                    # if the new process crashes immediately.\n+                                    import time as _time\n+                                    _time.sleep(3)\n+                                    verify = subprocess.run(\n+                                        scope_cmd + [\"is-active\", svc_name],\n+                                        capture_output=True, text=True, timeout=5,\n+                                    )\n+                                    if verify.stdout.strip() == \"active\":\n+                                        restarted_services.append(svc_name)\n+                                    else:\n+                                        # Retry once — transient startup failures\n+                                        # (stale module cache, import race) often\n+                                        # resolve on the second attempt.\n+                                        print(f\"  ⚠ {svc_name} died after restart, retrying...\")\n+                                        retry = subprocess.run(\n+                                            scope_cmd + [\"restart\", svc_name],\n+                                            capture_output=True, text=True, timeout=15,\n+                                        )\n+                                        _time.sleep(3)\n+                                        verify2 = subprocess.run(\n+                                            scope_cmd + [\"is-active\", svc_name],\n+                                            capture_output=True, text=True, timeout=5,\n+                                        )\n+                                        if verify2.stdout.strip() == \"active\":\n+                                            restarted_services.append(svc_name)\n+                                            print(f\"  ✓ {svc_name} recovered on retry\")\n+                                        else:\n+                                            print(\n+                                                f\"  ✗ {svc_name} failed to stay running after restart.\\n\"\n+                                                f\"    Check logs: journalctl --user -u {svc_name} --since '2 min ago'\\n\"\n+                                                f\"    Restart manually: systemctl {'--user ' if scope == 'user' else ''}restart {svc_name}\"\n+                                            )\n                                 else:\n                                     print(f\"  ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}\")\n                     except (FileNotFoundError, subprocess.TimeoutExpired):"]},{"file":"tests/gateway/restart_test_helpers.py","hunks":["@@ -93,6 +93,12 @@ def make_restart_runner(\n     runner._running_agent_count = GatewayRunner._running_agent_count.__get__(\n         runner, GatewayRunner\n     )\n+    runner._snapshot_running_agents = GatewayRunner._snapshot_running_agents.__get__(\n+        runner, GatewayRunner\n+    )\n+    runner._notify_active_sessions_of_shutdown = (\n+        GatewayRunner._notify_active_sessions_of_shutdown.__get__(runner, GatewayRunner)\n+    )\n     runner._launch_detached_restart_command = GatewayRunner._launch_detached_restart_command.__get__(\n         runner, GatewayRunner\n     )"]},{"file":"tests/gateway/test_restart_drain.py","hunks":["@@ -161,3 +161,84 @@ def fake_popen(cmd, **kwargs):\n     assert kwargs[\"start_new_session\"] is True\n     assert kwargs[\"stdout\"] is subprocess.DEVNULL\n     assert kwargs[\"stderr\"] is subprocess.DEVNULL\n+\n+\n+# ── Shutdown notification tests ──────────────────────────────────────\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_sent_to_active_sessions():\n+    \"\"\"Active sessions receive a notification when the gateway starts shutting down.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    source = make_restart_source(chat_id=\"999\", chat_type=\"dm\")\n+    session_key = f\"agent:main:telegram:dm:999\"\n+    runner._running_agents[session_key] = MagicMock()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 1\n+    assert \"shutting down\" in adapter.sent[0]\n+    assert \"interrupted\" in adapter.sent[0]\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_says_restarting_when_restart_requested():\n+    \"\"\"When _restart_requested is True, the message says 'restarting' and mentions /retry.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    runner._restart_requested = True\n+    session_key = \"agent:main:telegram:dm:999\"\n+    runner._running_agents[session_key] = MagicMock()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 1\n+    assert \"restarting\" in adapter.sent[0]\n+    assert \"/retry\" in adapter.sent[0]\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_deduplicates_per_chat():\n+    \"\"\"Multiple sessions in the same chat only get one notification.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    # Two sessions (different users) in the same chat\n+    runner._running_agents[\"agent:main:telegram:group:chat1:u1\"] = MagicMock()\n+    runner._running_agents[\"agent:main:telegram:group:chat1:u2\"] = MagicMock()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 1\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_skipped_when_no_active_agents():\n+    \"\"\"No notification is sent when there are no active agents.\"\"\"\n+    runner, adapter = make_restart_runner()\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 0\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_ignores_pending_sentinels():\n+    \"\"\"Pending sentinels (not-yet-started agents) don't trigger notifications.\"\"\"\n+    from gateway.run import _AGENT_PENDING_SENTINEL\n+\n+    runner, adapter = make_restart_runner()\n+    runner._running_agents[\"agent:main:telegram:dm:999\"] = _AGENT_PENDING_SENTINEL\n+\n+    await runner._notify_active_sessions_of_shutdown()\n+\n+    assert len(adapter.sent) == 0\n+\n+\n[redacted:email]\n+async def test_shutdown_notification_send_failure_does_not_block():\n+    \"\"\"If sending a notification fails, the method still completes.\"\"\"\n+    runner, adapter = make_restart_runner()\n+    adapter.send = AsyncMock(side_effect=Exception(\"network error\"))\n+    session_key = \"agent:main:telegram:dm:999\"\n+    runner._running_agents[session_key] = MagicMock()\n+\n+    # Should not raise\n+    await runner._notify_active_sessions_of_shutdown()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}