{"repo":"NousResearch/hermes-agent","public_id":"42126","display_label":"Gateway stability: zombie PIDs and platform crash-loop fixes","summary":"These items address two related gateway reliability issues: zombie PID handling in _pid_exists (blocking --replace), and the gateway exiting when platforms fail, with a circuit breaker and /platform endpoint added to keep it running.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":54162,"merged_at":"2026-06-28T11:11:14+00:00"},"resolution":{"state":"resolved","by":"pr-54162"},"freshness":{"state":"verified","last_verified_at":"2026-06-28T11:11:14+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":42126,"pr_number":54162,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-28T11:11:14+00:00"}],"members":[{"kind":"issue","number":42126,"title":"[Bug]: systemd `gateway run` exits 1 right after platforms connect (crash-loop) — regression after auto-update; runs fine standalone","url":null,"state":"closed"},{"kind":"pr","number":44898,"title":"fix(gateway): handle zombie PIDs in _pid_exists (closes #42126)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"acca52628","number":0},{"kind":"pr","value":"#54162","number":54162}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":54162,"title":"fix(gateway): treat zombie PIDs as dead in _pid_exists to unblock --replace (closes #42126)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/status.py","tests/gateway/test_gateway_shutdown.py"],"fix_hunks":[{"file":"gateway/status.py","hunks":["@@ -546,10 +546,29 @@ def _pid_exists(pid: int) -> bool:\n     \"\"\"\n     try:\n         import psutil  # type: ignore\n+\n+        # A zombie (defunct) process is still in the process table, so\n+        # ``psutil.pid_exists()`` returns True for it — but it is already\n+        # dead: SIGKILL has no effect and it cannot be a running gateway.\n+        # Treating a zombie as alive makes ``--replace`` wait for the old\n+        # PID to die (it never does, until its parent reaps it), then abort\n+        # with exit 1 — a silent crash loop under systemd ``Restart=always``,\n+        # which respawns the gateway before reaping the previous process\n+        # (issue #42126). Report zombies as dead so the takeover proceeds.\n+        # Best-effort: any failure to read status (partial/stub psutil,\n+        # access denied, transient race) falls through to the authoritative\n+        # ``pid_exists()`` below rather than raising.\n+        try:\n+            if psutil.Process(int(pid)).status() == psutil.STATUS_ZOMBIE:\n+                return False\n+        except getattr(psutil, \"NoSuchProcess\", ()):\n+            return False\n+        except Exception:\n+            pass\n         return bool(psutil.pid_exists(int(pid)))\n+\n     except ImportError:\n         pass  # Fall through to stdlib fallback.\n-\n     if _IS_WINDOWS:\n         try:\n             import ctypes","@@ -584,6 +603,31 @@ def _pid_exists(pid: int) -> bool:\n         except (OSError, AttributeError):\n             return False\n     else:\n+        # psutil missing (stripped install / scaffold phase). Catch the same\n+        # zombie case as the psutil path above (issue #42126): a zombie\n+        # answers os.kill(pid, 0) successfully, so without this check\n+        # ``--replace`` would wait on a dead PID and abort with exit 1.\n+        try:\n+            stat_fields = (\n+                Path(f\"/proc/{int(pid)}/stat\").read_text(encoding=\"utf-8\").split()\n+            )\n+            if len(stat_fields) > 2 and stat_fields[2] == \"Z\":\n+                return False\n+        except FileNotFoundError:\n+            # No /proc (macOS/BSD) — fall back to ps state.\n+            try:\n+                r = subprocess.run(\n+                    [\"ps\", \"-o\", \"state=\", \"-p\", str(int(pid))],\n+                    capture_output=True,\n+                    text=True,\n+                    timeout=5,\n+                )\n+                if r.returncode == 0 and r.stdout.strip().startswith(\"Z\"):\n+                    return False\n+            except Exception:\n+                pass\n+        except (IndexError, PermissionError, OSError):\n+            pass\n         try:\n             os.kill(int(pid), 0)  # windows-footgun: ok — POSIX-only branch (the whole point of _pid_exists)\n             return True"]},{"file":"tests/gateway/test_gateway_shutdown.py","hunks":["@@ -449,3 +449,105 @@ async def test_signal_initiated_restart_still_persists_stopped(tmp_path, monkeyp\n     assert _stopped_state_persisted(runner), (\n         \"a restart must persist gateway_state=stopped via the normal path\"\n     )\n+\n+\n+# ── #42126: zombie PID must be treated as dead in _pid_exists ────────────────\n+# Under systemd Restart=always, the old gateway becomes a zombie (still in the\n+# process table, not yet reaped) when the replacement starts. _pid_exists must\n+# report it dead so --replace proceeds instead of waiting on it and aborting\n+# with exit 1 (a silent crash loop).\n+\n+\n+def test_pid_exists_zombie_via_psutil_returns_false(monkeypatch):\n+    \"\"\"The live path is psutil. psutil.pid_exists() returns True for a zombie,\n+    so _pid_exists must additionally check Process.status() == STATUS_ZOMBIE.\"\"\"\n+    import sys\n+    import types\n+\n+    from gateway import status\n+\n+    fake_psutil = types.SimpleNamespace()\n+    fake_psutil.STATUS_ZOMBIE = \"zombie\"\n+\n+    class NoSuchProcess(Exception):\n+        pass\n+\n+    class PsutilError(Exception):\n+        pass\n+\n+    fake_psutil.NoSuchProcess = NoSuchProcess\n+    fake_psutil.Error = PsutilError\n+\n+    class _Proc:\n+        def __init__(self, pid):\n+            self.pid = pid\n+\n+        def status(self):\n+            return \"zombie\"\n+\n+    fake_psutil.Process = _Proc\n+    # Without the zombie guard, this True would make the caller treat the\n+    # zombie as a live gateway.\n+    fake_psutil.pid_exists = lambda pid: True\n+\n+    monkeypatch.setitem(sys.modules, \"psutil\", fake_psutil)\n+\n+    assert status._pid_exists(4242) is False\n+\n+\n+def test_pid_exists_live_via_psutil_returns_true(monkeypatch):\n+    \"\"\"A genuinely running (non-zombie) process is still reported alive.\"\"\"\n+    import sys\n+    import types\n+\n+    from gateway import status\n+\n+    fake_psutil = types.SimpleNamespace()\n+    fake_psutil.STATUS_ZOMBIE = \"zombie\"\n+    fake_psutil.NoSuchProcess = type(\"NoSuchProcess\", (Exception,), {})\n+    fake_psutil.Error = type(\"Error\", (Exception,), {})\n+\n+    class _Proc:\n+        def __init__(self, pid):\n+            self.pid = pid\n+\n+        def status(self):\n+            return \"running\"\n+\n+    fake_psutil.Process = _Proc\n+    fake_psutil.pid_exists = lambda pid: True\n+\n+    monkeypatch.setitem(sys.modules, \"psutil\", fake_psutil)\n+\n+    assert status._pid_exists(4242) is True\n+\n+\n+def test_pid_exists_zombie_via_proc_fallback_returns_false(monkeypatch):\n+    \"\"\"When psutil is unavailable, the POSIX fallback reads /proc/<pid>/stat\n+    and must treat state 'Z' as dead before reaching os.kill.\"\"\"\n+    import builtins\n+    import sys\n+\n+    from gateway import status\n+\n+    monkeypatch.setitem(sys.modules, \"psutil\", None)  # force ImportError\n+    real_import = builtins.__import__\n+\n+    def _no_psutil(name, *a, **k):\n+        if name == \"psutil\":\n+            raise ImportError(\"psutil disabled for test\")\n+        return real_import(name, *a, **k)\n+\n+    monkeypatch.setattr(builtins, \"__import__\", _no_psutil)\n+    monkeypatch.setattr(status, \"_IS_WINDOWS\", False)\n+\n+    fake_stat = \"4242 (defunct) Z 1 0 0 0 -1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\"\n+    fake_path = MagicMock()\n+    fake_path.read_text.return_value = fake_stat\n+    monkeypatch.setattr(status, \"Path\", lambda *_a, **_k: fake_path)\n+\n+    kill = MagicMock()\n+    monkeypatch.setattr(status.os, \"kill\", kill)\n+\n+    assert status._pid_exists(4242) is False\n+    kill.assert_not_called()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}