Gateway restart safety across systemd and containers

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #9850 fix: notify active sessions on gateway shutdown + update health check

  merged to main

  files: gateway/run.py, hermes_cli/main.py, tests/gateway/restart_test_helpers.py, tests/gateway/test_restart_drain.py

  Show the diff

  • gateway/run.py

    @@ -1391,6 +1391,65 @@ def _interrupt_running_agents(self, reason: str) -> None:
                 except Exception as e:
                     logger.debug("Failed interrupting agent during shutdown: %s", e)
     
    +    async def _notify_active_sessions_of_shutdown(self) -> None:
    +        """Send a notification to every chat with an active agent.
    +
    +        Called at the very start of stop() — adapters are still connected so
    +        messages can be delivered.  Best-effort: individual send failures are
    +        logged and swallowed so they never block the shutdown sequence.
    +        """
    +        active = self._snapshot_running_agents()
    +        if not active:
    +            return
    +
    +        action = "restarting" if self._restart_requested else "shutting down"
    +        hint = (
    +            "Your current task will be interrupted. "
    +            "Use /retry after restart to continue."
    +            if self._restart_requested
    +            else "Your current task will be interrupted."
    +        )
    +        msg = f"⚠️ Gateway {action} — {hint}"
    +
    +        notified: set = set()
    +        for session_key in active:
    +            # Parse platform + chat_id from the session key.
    +            # Format: agent:main:{platform}:{chat_type}:{chat_id}[:{extra}...]
    +            parts = session_key.split(":")
    +            if len(parts) < 5:
    +                continue
    +            platform_str = parts[2]
    +            chat_id = parts[4]
    +
    +            # Deduplicate: one notification per chat, even if multiple
    +            # sessions (different users/threads) share the same chat.
    +            dedup_key = (platform_str, chat_id)
    +            if dedup_key in notified:
    +                continue
    +
    +            try:
    +                platform = Platform(platform_str)
    +                adapter = self.adapters.get(platform)
    +                if not adapter:
    +                    continue
    +
    +                # Include thread_id if present so the message lands in the
    +                # correct forum topic / thread.
    +                thread_id = parts[5] if len(parts) > 5 else None
    +                metadata = {"thread_id": thread_id} if thread_id else None
    +
    +                await adapter.send(chat_id, msg, metadata=metadata)
    +                notified.add(dedup_key)
    +                logger.info(
    +                    "Sent shutdown notification to %s:%s",
    +                    platform_str, chat_id,
    +                )
    +            except Exception as e:
    +                logger.debug(
    +                    "Failed to send shutdown notification to %s:%s: %s",
    +                    platform_str, chat_id, e,
    +                )
    +
         def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:
             for agent in active_agents.values():
                 try:
    @@ -2018,6 +2077,10 @@ async def _stop_impl() -> None:
                 self._running = False
                 self._draining = True
     
    +            # Notify all chats with active agents BEFORE draining.
    +            # Adapters are still connected here, so messages can be sent.
    +            await self._notify_active_sessions_of_shutdown()
    +
                 timeout = self._restart_drain_timeout
                 active_agents, timed_out = await self._drain_active_agents(timeout)
                 if timed_out:
    @@ -2088,12 +2151,23 @@ async def _stop_impl() -> None:
     
                 # Write a clean-shutdown marker so the next startup knows this
                 # wasn't a crash.  suspend_recently_active() only needs to run
    -            # after unexpected exits — graceful shutdowns already drain
    -            # active agents, so there's no stuck-session risk.
    -            try:
    -                (_hermes_home / ".clean_shutdown").touch()
    -            except Exception:
    -                pass
    +            # after unexpected exits.  However, if the drain timed out and
    +            # agents were force-interrupted, their sessions may be in an
    +            # incomplete state (trailing tool response, no final assistant
    +            # message).  Skip the marker in that case so the next startup
    +            # suspends those sessions — giving users a clean slate instead
    +            # of resuming a half-finished tool loop.
    +            if not timed_out:
    +                try:
    +                    (_hermes_home / ".clean_shutdown").touch()
    +                except Exception:
    +                    pass
    +            else:
    +                logger.info(
    +                    "Skipping .clean_shutdown marker — drain timed out with "
    +                    "interrupted agents; next startup will suspend recently "
    +                    "active sessions."
    +                )
     
                 if self._restart_requested and self._restart_via_service:
                     self._exit_code = GATEWAY_SERVICE_RESTART_EXIT_CODE

  • hermes_cli/main.py

    @@ -4036,7 +4036,40 @@ def cmd_update(args):
                                         capture_output=True, text=True, timeout=15,
                                     )
                                     if restart.returncode == 0:
    -                                    restarted_services.append(svc_name)
    +                                    # Verify the service actually survived the
    +                                    # restart.  systemctl restart returns 0 even
    +                                    # if the new process crashes immediately.
    +                                    import time as _time
    +                                    _time.sleep(3)
    +                                    verify = subprocess.run(
    +                                        scope_cmd + ["is-active", svc_name],
    +                                        capture_output=True, text=True, timeout=5,
    +                                    )
    +                                    if verify.stdout.strip() == "active":
    +                                        restarted_services.append(svc_name)
    +                                    else:
    +                                        # Retry once — transient startup failures
    +                                        # (stale module cache, import race) often
    +                                        # resolve on the second attempt.
    +                                        print(f"  ⚠ {svc_name} died after restart, retrying...")
    +                                        retry = subprocess.run(
    +                                            scope_cmd + ["restart", svc_name],
    +                                            capture_output=True, text=True, timeout=15,
    +                                        )
    +                                        _time.sleep(3)
    +                                        verify2 = subprocess.run(
    +                                            scope_cmd + ["is-active", svc_name],
    +                                            capture_output=True, text=True, timeout=5,
    +                                        )
    +                                        if verify2.stdout.strip() == "active":
    +                                            restarted_services.append(svc_name)
    +                                            print(f"  ✓ {svc_name} recovered on retry")
    +                                        else:
    +                                            print(
    +                                                f"  ✗ {svc_name} failed to stay running after restart.\n"
    +                                                f"    Check logs: journalctl --user -u {svc_name} --since '2 min ago'\n"
    +                                                f"    Restart manually: systemctl {'--user ' if scope == 'user' else ''}restart {svc_name}"
    +                                            )
                                     else:
                                         print(f"  ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}")
                         except (FileNotFoundError, subprocess.TimeoutExpired):

  • tests/gateway/restart_test_helpers.py

    @@ -93,6 +93,12 @@ def make_restart_runner(
         runner._running_agent_count = GatewayRunner._running_agent_count.__get__(
             runner, GatewayRunner
         )
    +    runner._snapshot_running_agents = GatewayRunner._snapshot_running_agents.__get__(
    +        runner, GatewayRunner
    +    )
    +    runner._notify_active_sessions_of_shutdown = (
    +        GatewayRunner._notify_active_sessions_of_shutdown.__get__(runner, GatewayRunner)
    +    )
         runner._launch_detached_restart_command = GatewayRunner._launch_detached_restart_command.__get__(
             runner, GatewayRunner
         )

  • tests/gateway/test_restart_drain.py

    @@ -161,3 +161,84 @@ def fake_popen(cmd, **kwargs):
         assert kwargs["start_new_session"] is True
         assert kwargs["stdout"] is subprocess.DEVNULL
         assert kwargs["stderr"] is subprocess.DEVNULL
    +
    +
    +# ── Shutdown notification tests ──────────────────────────────────────
    +
    +
    [redacted:email]
    +async def test_shutdown_notification_sent_to_active_sessions():
    +    """Active sessions receive a notification when the gateway starts shutting down."""
    +    runner, adapter = make_restart_runner()
    +    source = make_restart_source(chat_id="999", chat_type="dm")
    +    session_key = f"agent:main:telegram:dm:999"
    +    runner._running_agents[session_key] = MagicMock()
    +
    +    await runner._notify_active_sessions_of_shutdown()
    +
    +    assert len(adapter.sent) == 1
    +    assert "shutting down" in adapter.sent[0]
    +    assert "interrupted" in adapter.sent[0]
    +
    +
    [redacted:email]
    +async def test_shutdown_notification_says_restarting_when_restart_requested():
    +    """When _restart_requested is True, the message says 'restarting' and mentions /retry."""
    +    runner, adapter = make_restart_runner()
    +    runner._restart_requested = True
    +    session_key = "agent:main:telegram:dm:999"
    +    runner._running_agents[session_key] = MagicMock()
    +
    +    await runner._notify_active_sessions_of_shutdown()
    +
    +    assert len(adapter.sent) == 1
    +    assert "restarting" in adapter.sent[0]
    +    assert "/retry" in adapter.sent[0]
    +
    +
    [redacted:email]
    +async def test_shutdown_notification_deduplicates_per_chat():
    +    """Multiple sessions in the same chat only get one notification."""
    +    runner, adapter = make_restart_runner()
    +    # Two sessions (different users) in the same chat
    +    runner._running_agents["agent:main:telegram:group:chat1:u1"] = MagicMock()
    +    runner._running_agents["agent:main:telegram:group:chat1:u2"] = MagicMock()
    +
    +    await runner._notify_active_sessions_of_shutdown()
    +
    +    assert len(adapter.sent) == 1
    +
    +
    [redacted:email]
    +async def test_shutdown_notification_skipped_when_no_active_agents():
    +    """No notification is sent when there are no active agents."""
    +    runner, adapter = make_restart_runner()
    +
    +    await runner._notify_active_sessions_of_shutdown()
    +
    +    assert len(adapter.sent) == 0
    +
    +
    [redacted:email]
    +async def test_shutdown_notification_ignores_pending_sentinels():
    +    """Pending sentinels (not-yet-started agents) don't trigger notifications."""
    +    from gateway.run import _AGENT_PENDING_SENTINEL
    +
    +    runner, adapter = make_restart_runner()
    +    runner._running_agents["agent:main:telegram:dm:999"] = _AGENT_PENDING_SENTINEL
    +
    +    await runner._notify_active_sessions_of_shutdown()
    +
    +    assert len(adapter.sent) == 0
    +
    +
    [redacted:email]
    +async def test_shutdown_notification_send_failure_does_not_block():
    +    """If sending a notification fails, the method still completes."""
    +    runner, adapter = make_restart_runner()
    +    adapter.send = AsyncMock(side_effect=Exception("network error"))
    +    session_key = "agent:main:telegram:dm:999"
    +    runner._running_agents[session_key] = MagicMock()
    +
    +    # Should not raise
    +    await runner._notify_active_sessions_of_shutdown()

• #37475 fix(terminal): block gateway lifecycle commands from inside the gateway process

  merged to main

  files: hermes_cli/cron.py, tests/hermes_cli/test_gateway_restart_loop.py, tools/terminal_tool.py

  Show the diff

  • hermes_cli/cron.py

    @@ -25,7 +25,7 @@
         r"(?i)"
         r"(hermes\s+gateway\s+(restart|stop|start))"
         r"|(launchctl\s+(kickstart|unload|load|stop|restart)\s+.*hermes)"
    -    r"|(systemctl\s+(restart|stop|start)\s+.*hermes)"
    +    r"|(systemctl\s+(-\S+\s+)*(restart|stop|start)\s+.*hermes)"
         r"|(p?kill\s+.*hermes.*gateway)"
     )
     

  • tests/hermes_cli/test_gateway_restart_loop.py

    @@ -6,6 +6,7 @@
     - _contains_gateway_lifecycle_command pattern matching
     """
     
    +import json
     import os
     from argparse import Namespace
     
    @@ -250,3 +251,109 @@ def _sentinel(*a, **k):
             args = Namespace(gateway_command="restart", all=False, system=False)
             with pytest.raises(_Reached):
                 gw.gateway_command(args)
    +
    +
    +# ---------------------------------------------------------------------------
    +# Defense 3: terminal_tool hard-blocks gateway lifecycle commands inside gateway
    +# ---------------------------------------------------------------------------
    +
    +class TestTerminalToolGatewayLifecycleGuard:
    +    """terminal_tool must refuse gateway lifecycle commands when _HERMES_GATEWAY=1.
    +
    +    Issue #37453: systemctl --user restart hermes-gateway runs as a child of the
    +    gateway process.  When systemd delivers SIGTERM the gateway kills its own
    +    restart command mid-execution — the service may never restart.  The guard
    +    must fire before execution, unconditionally (force=True cannot bypass it).
    +    """
    +
    +    def _make_fake_env(self):
    +        class _FakeEnv:
    +            env = {}
    +            def execute(self, command, **kwargs):  # pragma: no cover
    +                raise AssertionError("execute must not be reached")
    +        return _FakeEnv()
    +
    +    def _minimal_config(self):
    +        return {"env_type": "local", "cwd": "/tmp", "timeout": 60, "lifetime_seconds": 3600}
    +
    +    def _patch_env(self, monkeypatch, fake_env, *, inside_gateway: bool):
    +        import tools.terminal_tool as tt
    +        eid = "default"
    +        monkeypatch.setattr(tt, "_active_environments", {eid: fake_env})
    +        monkeypatch.setattr(tt, "_last_activity", {eid: 0.0})
    +        monkeypatch.setattr(tt, "_task_env_overrides", {})
    +        monkeypatch.setattr(tt, "_get_env_config", self._minimal_config)
    +        if inside_gateway:
    +            monkeypatch.setenv("_HERMES_GATEWAY", "1")
    +        else:
    +            monkeypatch.delenv("_HERMES_GATEWAY", raising=False)
    +
    +    @pytest.mark.parametrize("cmd", [
    +        "systemctl restart hermes-gateway",
    +        "systemctl --user restart hermes-gateway",
    +        "systemctl stop hermes-gateway.service",
    +        "hermes gateway restart",
    +        "launchctl kickstart gui/501/ai.hermes.gateway",
    +        "pkill -f hermes.*gateway",
    +    ])
    +    def test_blocks_lifecycle_commands_inside_gateway(self, monkeypatch, cmd):
    +        import tools.terminal_tool as tt
    +        self._patch_env(monkeypatch, self._make_fake_env(), inside_gateway=True)
    +
    +        result = json.loads(tt.terminal_tool(command=cmd))
    +
    +        assert result["exit_code"] == 1
    +        assert "Blocked" in result["error"]
    +
    +    def test_force_true_cannot_bypass_block(self, monkeypatch):
    +        import tools.terminal_tool as tt
    +        self._patch_env(monkeypatch, self._make_fake_env(), inside_gateway=True)
    +
    +        result = json.loads(tt.terminal_tool(
    +            command="systemctl restart hermes-gateway", force=True
    +        ))
    +
    +        assert result["exit_code"] == 1
    +        assert "Blocked" in result["error"]
    +
    +    def test_safe_systemctl_commands_pass_through(self, monkeypatch):
    +        """Non-hermes systemctl commands must not be blocked by this guard."""
    +        import tools.terminal_tool as tt
    +
    +        calls = []
    +
    +        class _FakeEnv:
    +            env = {}
    +            def execute(self, command, **kwargs):
    +                calls.append(command)
    +                return {"output": "Active: running", "returncode": 0}
    +
    +        self._patch_env(monkeypatch, _FakeEnv(), inside_gateway=True)
    +        monkeypatch.setattr(tt, "_check_all_guards", lambda cmd, env: {"approved": True})
    +
    +        result = json.loads(tt.terminal_tool(command="systemctl status nginx"))
    +
    +        assert result["exit_code"] == 0
    +        assert calls == ["systemctl status nginx"]
    +
    +    def test_guard_inactive_outside_gateway(self, monkeypatch):
    +        """Without _HERMES_GATEWAY=1 the lifecycle guard must not fire."""
    +        import tools.terminal_tool as tt
    +
    +        calls = []
    +
    +        class _FakeEnv:
    +            env = {}
    +            def execute(self, command, **kwargs):
    +                calls.append(command)
    +                return {"output": "restarting...", "returncode": 0}
    +
    +        self._patch_env(monkeypatch, _FakeEnv(), inside_gateway=False)
    +        monkeypatch.setattr(tt, "_check_all_guards", lambda cmd, env: {"approved": True})
    +
    +        result = json.loads(tt.terminal_tool(command="systemctl restart hermes-gateway"))
    +
    +        # Outside the gateway the lifecycle guard doesn't block — the normal
    +        # approval flow handles it (here mocked as approved).
    +        assert result["exit_code"] == 0
    +        assert calls == ["systemctl restart hermes-gateway"]

  • tools/terminal_tool.py

    @@ -1959,6 +1959,29 @@ def terminal_tool(
                             env = new_env
                         logger.info("%s environment ready for task %s", env_type, effective_task_id[:8])
     
    +        # Hard-block: gateway lifecycle commands (systemctl/launchctl/hermes
    +        # restart|stop targeting hermes-gateway) must never run inside the
    +        # gateway process itself. The restart would SIGTERM the gateway, which
    +        # kills this very subprocess before it can complete — the service may
    +        # never restart. This mirrors the `hermes gateway restart` guard in
    +        # hermes_cli/gateway.py and the cron-path guard in hermes_cli/cron.py,
    +        # but applies unconditionally (force=True cannot help here).
    +        if os.environ.get("_HERMES_GATEWAY") == "1":
    +            from hermes_cli.cron import _contains_gateway_lifecycle_command
    +            if _contains_gateway_lifecycle_command(command):
    +                return json.dumps({
    +                    "output": "",
    +                    "exit_code": 1,
    +                    "error": (
    +                        "Blocked: cannot restart or stop the gateway from inside the "
    +                        "gateway process. The gateway would kill this command before "
    +                        "it could complete (SIGTERM propagates to child processes). "
    +                        "Run `hermes gateway restart` from a separate shell outside "
    +                        "the running gateway."
    +                    ),
    +                    "status": "error",
    +                }, ensure_ascii=False)
    +
             # Pre-exec security checks (tirith + dangerous command detection)
             # Skip check if force=True (user has confirmed they want to run it)
             approval_note = None