{"repo":"NousResearch/hermes-agent","public_id":"5686","display_label":"Gateway restart reliability and test fixes","summary":"These members address how the gateway restart process waits for the service to come back up, replacing a one-shot sleep with polling, and update related tests to mock Linux-specific behavior and reflect changed systemd output.","subsystem":"hermes_cli","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":9945,"merged_at":"2026-04-15T00:12:59+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"stale","last_verified_at":"2026-04-15T00:12:59+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":5686,"pr_number":9945,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-04-15T00:12:59+00:00"},{"issue_number":8260,"pr_number":9945,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-15T00:12:59+00:00"}],"members":[{"kind":"issue","number":5686,"title":"Update gateway restart test: systemd restart message not in output","url":null,"state":"closed"},{"kind":"pr","number":5693,"title":"fix(test): mock is_linux in systemd restart test","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"480dd750f2a9755935778e879c6d211d3838a5bb","number":0},{"kind":"commit","value":"2a98098035ca70459570e99b6b26e1a3ca6fbd27","number":0},{"kind":"pr","value":"#5686","number":5686},{"kind":"pr","value":"#9945","number":9945}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":9945,"title":"fix: hermes gateway restart waits for service to come back up (#8260)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/gateway.py","tests/hermes_cli/test_gateway_service.py"],"fix_hunks":[{"file":"hermes_cli/gateway.py","hunks":["@@ -1128,7 +1128,62 @@ def systemd_restart(system: bool = False):\n \n     pid = get_running_pid()\n     if pid is not None and _request_gateway_self_restart(pid):\n-        print(f\"✓ {_service_scope_label(system).capitalize()} service restart requested\")\n+        # SIGUSR1 sent — the gateway will drain active agents, exit with\n+        # code 75, and systemd will restart it after RestartSec (30s).\n+        # Wait for the old process to die and the new one to become active\n+        # so the CLI doesn't return while the service is still restarting.\n+        import time\n+        scope_label = _service_scope_label(system).capitalize()\n+        svc = get_service_name()\n+        scope_cmd = _systemctl_cmd(system)\n+\n+        # Phase 1: wait for old process to exit (drain + shutdown)\n+        print(f\"⏳ {scope_label} service draining active work...\")\n+        deadline = time.time() + 90\n+        while time.time() < deadline:\n+            try:\n+                os.kill(pid, 0)\n+                time.sleep(1)\n+            except (ProcessLookupError, PermissionError):\n+                break  # old process is gone\n+        else:\n+            print(f\"⚠ Old process (PID {pid}) still alive after 90s\")\n+\n+        # Phase 2: wait for systemd to start the new process\n+        print(f\"⏳ Waiting for {svc} to restart...\")\n+        deadline = time.time() + 60\n+        while time.time() < deadline:\n+            try:\n+                result = subprocess.run(\n+                    scope_cmd + [\"is-active\", svc],\n+                    capture_output=True, text=True, timeout=5,\n+                )\n+                if result.stdout.strip() == \"active\":\n+                    # Verify it's a NEW process, not the old one somehow\n+                    new_pid = get_running_pid()\n+                    if new_pid and new_pid != pid:\n+                        print(f\"✓ {scope_label} service restarted (PID {new_pid})\")\n+                        return\n+            except (subprocess.TimeoutExpired, FileNotFoundError):\n+                pass\n+            time.sleep(2)\n+\n+        # Timed out — check final state\n+        try:\n+            result = subprocess.run(\n+                scope_cmd + [\"is-active\", svc],\n+                capture_output=True, text=True, timeout=5,\n+            )\n+            if result.stdout.strip() == \"active\":\n+                print(f\"✓ {scope_label} service restarted\")\n+                return\n+        except Exception:\n+            pass\n+        print(\n+            f\"⚠ {scope_label} service did not become active within 60s.\\n\"\n+            f\"  Check status: {'sudo ' if system else ''}hermes gateway status\\n\"\n+            f\"  Check logs:   journalctl {'--user ' if not system else ''}-u {svc} --since '2 min ago'\"\n+        )\n         return\n     _run_systemctl([\"reload-or-restart\", get_service_name()], system=system, check=True, timeout=90)\n     print(f\"✓ {_service_scope_label(system).capitalize()} service restarted\")"]},{"file":"tests/hermes_cli/test_gateway_service.py","hunks":["@@ -452,7 +452,7 @@ def fake_run(*args, **kwargs):\n \n \n class TestGatewaySystemServiceRouting:\n-    def test_systemd_restart_self_requests_graceful_restart_without_reload_or_restart(self, monkeypatch, capsys):\n+    def test_systemd_restart_self_requests_graceful_restart_and_waits(self, monkeypatch, capsys):\n         calls = []\n \n         monkeypatch.setattr(gateway_cli, \"_select_systemd_scope\", lambda system=False: False)","@@ -466,16 +466,37 @@ def test_systemd_restart_self_requests_graceful_restart_without_reload_or_restar\n             \"_request_gateway_self_restart\",\n             lambda pid: calls.append((\"self\", pid)) or True,\n         )\n-        monkeypatch.setattr(\n-            gateway_cli.subprocess,\n-            \"run\",\n-            lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError(\"systemctl should not run\")),\n-        )\n+\n+        # Simulate: old process dies immediately, new process becomes active\n+        kill_call_count = [0]\n+        def fake_kill(pid, sig):\n+            kill_call_count[0] += 1\n+            if kill_call_count[0] >= 2:  # first call checks, second = dead\n+                raise ProcessLookupError()\n+        monkeypatch.setattr(os, \"kill\", fake_kill)\n+\n+        # Simulate systemctl is-active returning \"active\" with a new PID\n+        new_pid = [None]\n+        def fake_subprocess_run(cmd, **kwargs):\n+            if \"is-active\" in cmd:\n+                result = SimpleNamespace(stdout=\"active\\n\", returncode=0)\n+                new_pid[0] = 999  # new PID\n+                return result\n+            raise AssertionError(f\"Unexpected systemctl call: {cmd}\")\n+\n+        monkeypatch.setattr(gateway_cli.subprocess, \"run\", fake_subprocess_run)\n+        # get_running_pid returns new PID after restart\n+        pid_calls = [0]\n+        def fake_get_pid():\n+            pid_calls[0] += 1\n+            return 999 if pid_calls[0] > 1 else 654\n+        monkeypatch.setattr(\"gateway.status.get_running_pid\", fake_get_pid)\n \n         gateway_cli.systemd_restart()\n \n-        assert calls == [(\"refresh\", False), (\"self\", 654)]\n-        assert \"restart requested\" in capsys.readouterr().out.lower()\n+        assert (\"self\", 654) in calls\n+        out = capsys.readouterr().out.lower()\n+        assert \"restarted\" in out\n \n     def test_gateway_install_passes_system_flags(self, monkeypatch):\n         monkeypatch.setattr(gateway_cli, \"supports_systemd_services\", lambda: True)\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}