{"repo":"NousResearch/hermes-agent","public_id":"34597","display_label":"Gateway crash from stale planned-stop marker bug","summary":"These issues and fixes address a gateway crash occurring shortly after boot, caused by stale planned-stop markers incorrectly triggering a false UNKNOWN exit. The fix scopes the planned-stop watcher to self-targeting markers only and includes a related Windows-specific correction.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":34749,"merged_at":"2026-05-29T17:36:59+00:00"},"resolution":{"state":"resolved","by":"pr-34749"},"freshness":{"state":"verified","last_verified_at":"2026-05-29T17:36:59+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":34597,"pr_number":34749,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-29T17:36:59+00:00"}],"members":[{"kind":"issue","number":34597,"title":"Windows: Gateway crashes ~400ms after boot — stale planned-stop marker triggers false UNKNOWN exit","url":null,"state":"closed"},{"kind":"pr","number":34599,"title":"fix(gateway): only fire planned-stop watcher for markers targeting self (cross-platform stale-marker crash loop)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"7379f175567bd0f1d833eec3a3599d0665b2a491","number":0},{"kind":"pr","value":"#34749","number":34749},{"kind":"pr","value":"#34599","number":34599}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":34749,"title":"fix(gateway): only fire planned-stop watcher for self-targeting markers + fix Windows consume","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","gateway/status.py","tests/gateway/test_planned_stop_watcher.py","tests/gateway/test_status.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -18442,7 +18442,10 @@ def _run_planned_stop_watcher(\n         poll_interval: seconds between marker checks. 0.5s gives a\n             responsive shutdown without burning CPU.\n     \"\"\"\n-    from gateway.status import _get_planned_stop_marker_path\n+    from gateway.status import (\n+        _get_planned_stop_marker_path,\n+        planned_stop_marker_targets_self,\n+    )\n     marker_path = _get_planned_stop_marker_path()\n     while not stop_event.is_set():\n         try:","@@ -18451,6 +18454,26 @@ def _run_planned_stop_watcher(\n                 and not getattr(runner, \"_draining\", False)\n                 and getattr(runner, \"_running\", False)\n             ):\n+                # A marker existing is NOT sufficient — it may have been\n+                # written for a PREVIOUS gateway instance (different PID)\n+                # and left behind because that process exited before the\n+                # CLI's stop() could clean it up. Firing the handler on a\n+                # stale/foreign marker drives the gateway into shutdown,\n+                # then consume_planned_stop_marker_for_self() correctly\n+                # reports a PID mismatch — but by then we're already\n+                # stopping, so it's logged as an unexpected \"UNKNOWN\" exit\n+                # and the watchdog crash-loops the gateway (issue #34597,\n+                # a regression from PR #33798 which added this watcher\n+                # without the PID check).\n+                #\n+                # Only fire when the marker actually targets us. The probe\n+                # is non-destructive on a match (the handler does the\n+                # authoritative consume on the loop thread) and self-heals\n+                # by unlinking stale/malformed markers so they cannot wedge\n+                # a freshly booted gateway.\n+                if not planned_stop_marker_targets_self():\n+                    stop_event.wait(poll_interval)\n+                    continue\n                 # Drive the same path as a real signal handler.\n                 # Pass signal=None — the handler tolerates that and consumes\n                 # the marker via consume_planned_stop_marker_for_self,"]},{"file":"gateway/status.py","hunks":["@@ -816,12 +816,24 @@ def _consume_pid_marker_for_self(\n \n     our_pid = os.getpid()\n     our_start_time = _get_process_start_time(our_pid)\n-    matches = (\n-        target_pid == our_pid\n-        and target_start_time is not None\n-        and our_start_time is not None\n-        and target_start_time == our_start_time\n-    )\n+    # Start-time is a PID-reuse guard. It is only meaningful when both\n+    # sides actually have it: ``_get_process_start_time`` returns None on\n+    # platforms without ``/proc`` (macOS, native Windows — the very\n+    # platform the planned-stop watcher exists for). Requiring a non-None\n+    # match there would make every consume return False, so a legitimate\n+    # ``hermes gateway stop`` on Windows would be misclassified as an\n+    # unexpected ``UNKNOWN`` exit (exit 1) and revived by the service\n+    # manager. So: when both start_times are known they must match; when\n+    # either is unknown, fall back to PID equality alone (bounded by the\n+    # marker's short TTL). This mirrors ``planned_stop_marker_targets_self``\n+    # so the watcher's non-destructive probe and this authoritative\n+    # consume agree on every platform (issue #34597).\n+    if target_pid != our_pid:\n+        matches = False\n+    elif target_start_time is not None and our_start_time is not None:\n+        matches = target_start_time == our_start_time\n+    else:\n+        matches = True\n \n     try:\n         path.unlink(missing_ok=True)","@@ -914,6 +926,68 @@ def consume_planned_stop_marker_for_self() -> bool:\n     )\n \n \n+def planned_stop_marker_targets_self() -> bool:\n+    \"\"\"Return True only when a live planned-stop marker names the current process.\n+\n+    This is a **non-destructive** probe used by the watcher thread\n+    (``gateway/run.py:_run_planned_stop_watcher``) to decide whether to\n+    trigger shutdown. Unlike :func:`consume_planned_stop_marker_for_self`,\n+    it never unlinks a marker that matches us — the shutdown handler does\n+    the authoritative consume on its own thread.\n+\n+    It *does* clean up markers that can never apply to this process:\n+    malformed markers and markers older than the TTL are unlinked so a\n+    stale file left behind by a previous gateway instance cannot wedge\n+    the new one. Markers naming a different PID/start_time are left in\n+    place (they may still be consumed legitimately by the process they\n+    name) but report False here.\n+\n+    Returns False (without raising) on any read/parse error.\n+    \"\"\"\n+    path = _get_planned_stop_marker_path()\n+    record = _read_json_file(path)\n+    if not record:\n+        return False\n+\n+    try:\n+        target_pid = int(record[\"target_pid\"])\n+        target_start_time = record.get(\"target_start_time\")\n+        written_at = record.get(\"written_at\") or \"\"\n+    except (KeyError, TypeError, ValueError):\n+        # Malformed marker can never match anyone — drop it.\n+        try:\n+            path.unlink(missing_ok=True)\n+        except OSError:\n+            pass\n+        return False\n+\n+    if _marker_is_stale(written_at, _PLANNED_STOP_MARKER_TTL_S):\n+        # A marker this old is past its useful life regardless of target —\n+        # clean it up so it cannot crash-loop a freshly booted gateway.\n+        try:\n+            path.unlink(missing_ok=True)\n+        except OSError:\n+            pass\n+        return False\n+\n+    our_pid = os.getpid()\n+    if target_pid != our_pid:\n+        return False\n+\n+    # Start-time is a PID-reuse guard. It is only meaningful when both\n+    # sides actually have it: ``_get_process_start_time`` returns None on\n+    # platforms without ``/proc`` (macOS, native Windows — the very\n+    # platform this watcher exists for). Requiring a non-None match there\n+    # would make the watcher never fire and re-break the #33778 Windows\n+    # session-resume path. So: when both start_times are known they must\n+    # match; when either is unknown, fall back to PID equality alone\n+    # (the marker is short-lived under a 60s TTL, bounding reuse risk).\n+    our_start_time = _get_process_start_time(our_pid)\n+    if target_start_time is not None and our_start_time is not None:\n+        return target_start_time == our_start_time\n+    return True\n+\n+\n def clear_planned_stop_marker() -> None:\n     \"\"\"Remove the planned-stop marker unconditionally.\"\"\"\n     try:"]},{"file":"tests/gateway/test_planned_stop_watcher.py","hunks":["@@ -12,12 +12,33 @@\n \"\"\"\n \n import asyncio\n+import json\n+import os\n import threading\n import time\n from unittest.mock import MagicMock\n \n \n from gateway.run import _run_planned_stop_watcher\n+from gateway import status as status_mod\n+\n+\n+def _write_self_marker(marker, *, stale: bool = False):\n+    \"\"\"Write a planned-stop marker that targets the CURRENT process.\n+\n+    The watcher only fires for markers naming our PID + start_time (the\n+    fix for issue #34597), so tests that expect a fire must write a\n+    self-targeting marker. Pass ``stale=True`` to backdate ``written_at``\n+    past the TTL.\n+    \"\"\"\n+    written_at = \"2000-01-01T00:00:00+00:00\" if stale else status_mod._utc_now_iso()\n+    record = {\n+        \"target_pid\": os.getpid(),\n+        \"target_start_time\": status_mod._get_process_start_time(os.getpid()),\n+        \"stopper_pid\": os.getpid(),\n+        \"written_at\": written_at,\n+    }\n+    marker.write_text(json.dumps(record), encoding=\"utf-8\")\n \n \n class _FakeRunner:","@@ -41,11 +62,10 @@ def fake_call_soon_threadsafe(fn, *args):\n \n \n def test_watcher_fires_shutdown_when_marker_appears(tmp_path, monkeypatch):\n-    \"\"\"When the marker file exists, the watcher must call the shutdown handler.\"\"\"\n+    \"\"\"When a marker targeting THIS process exists, fire the shutdown handler.\"\"\"\n     marker = tmp_path / \".gateway-planned-stop.json\"\n \n     # Patch the marker-path resolver so the watcher polls our temp location.\n-    from gateway import status as status_mod\n     monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n \n     runner = _FakeRunner(running=True, draining=False)","@@ -53,8 +73,8 @@ def test_watcher_fires_shutdown_when_marker_appears(tmp_path, monkeypatch):\n     shutdown_handler = MagicMock(name=\"shutdown_signal_handler\")\n     stop_event = threading.Event()\n \n-    # Drop the marker before the thread starts.\n-    marker.write_text('{\"target_pid\": 1234}', encoding=\"utf-8\")\n+    # Drop a self-targeting marker before the thread starts.\n+    _write_self_marker(marker)\n \n     watcher = threading.Thread(\n         target=_run_planned_stop_watcher,","@@ -114,9 +134,8 @@ def test_watcher_skips_when_runner_already_draining(tmp_path, monkeypatch):\n     so the watcher backs off once any shutdown is in flight.\n     \"\"\"\n     marker = tmp_path / \".gateway-planned-stop.json\"\n-    marker.write_text('{\"target_pid\": 1234}', encoding=\"utf-8\")\n+    _write_self_marker(marker)\n \n-    from gateway import status as status_mod\n     monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n \n     # Already draining — watcher should be a no-op.","@@ -204,9 +223,8 @@ def test_watcher_fires_only_once_when_marker_persists(tmp_path, monkeypatch):\n     times before the gateway actually shuts down.\n     \"\"\"\n     marker = tmp_path / \".gateway-planned-stop.json\"\n-    marker.write_text('{\"target_pid\": 1234}', encoding=\"utf-8\")\n+    _write_self_marker(marker)\n \n-    from gateway import status as status_mod\n     monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n \n     runner = _FakeRunner(running=True, draining=False)","@@ -263,3 +281,113 @@ def explode():\n     assert not watcher.is_alive(), \"Watcher should still honour stop_event after errors\"\n     # No shutdown fired because the marker never reported existence.\n     assert loop._captured == []\n+\n+\n+# ---------------------------------------------------------------------------\n+# Regression coverage for issue #34597:\n+# A marker left behind by a PREVIOUS gateway instance (different PID, or\n+# past its TTL) must NOT crash the freshly booted gateway. The watcher\n+# only fires when the marker targets the current process, and self-heals\n+# by cleaning up stale/malformed markers.\n+# ---------------------------------------------------------------------------\n+\n+\n+def test_watcher_does_not_fire_for_foreign_pid_marker(tmp_path, monkeypatch):\n+    \"\"\"A marker naming a DIFFERENT process must not trigger our shutdown.\n+\n+    This is the core #34597 regression: a stale marker from a prior\n+    gateway instance was firing the handler, driving the new gateway into\n+    a false \"Received UNKNOWN\" shutdown and a watchdog crash loop.\n+    \"\"\"\n+    marker = tmp_path / \".gateway-planned-stop.json\"\n+    # Foreign PID + a start_time that cannot match ours, freshly written\n+    # so the TTL does NOT remove it — the watcher must still decline.\n+    record = {\n+        \"target_pid\": os.getpid() + 1,\n+        \"target_start_time\": -1,\n+        \"stopper_pid\": os.getpid() + 1,\n+        \"written_at\": status_mod._utc_now_iso(),\n+    }\n+    marker.write_text(json.dumps(record), encoding=\"utf-8\")\n+\n+    monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n+\n+    runner = _FakeRunner(running=True, draining=False)\n+    loop = _make_loop_capturing_calls()\n+    shutdown_handler = MagicMock(name=\"shutdown_signal_handler\")\n+    stop_event = threading.Event()\n+\n+    watcher = threading.Thread(\n+        target=_run_planned_stop_watcher,\n+        args=(stop_event, runner, loop, shutdown_handler),\n+        kwargs={\"poll_interval\": 0.05},\n+        daemon=True,\n+    )\n+    watcher.start()\n+    time.sleep(0.3)  # several poll cycles\n+    stop_event.set()\n+    watcher.join(timeout=2.0)\n+\n+    assert not watcher.is_alive()\n+    assert loop._captured == [], (\n+        f\"Watcher fired on a foreign-PID marker (#34597 regression): {loop._captured}\"\n+    )\n+    shutdown_handler.assert_not_called()\n+    # Foreign (but live) marker is left in place — it may still belong to\n+    # the process it names.\n+    assert marker.exists()\n+\n+\n+def test_watcher_cleans_up_stale_marker_and_keeps_running(tmp_path, monkeypatch):\n+    \"\"\"A marker older than the TTL is unlinked and never fires shutdown.\"\"\"\n+    marker = tmp_path / \".gateway-planned-stop.json\"\n+    # Self-targeting but backdated past the TTL: must be treated as dead.\n+    _write_self_marker(marker, stale=True)\n+\n+    monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n+\n+    runner = _FakeRunner(running=True, draining=False)\n+    loop = _make_loop_capturing_calls()\n+    shutdown_handler = MagicMock(name=\"shutdown_signal_handler\")\n+    stop_event = threading.Event()\n+\n+    watcher = threading.Thread(\n+        target=_run_planned_stop_watcher,\n+        args=(stop_event, runner, loop, shutdown_handler),\n+        kwargs={\"poll_interval\": 0.05},\n+        daemon=True,\n+    )\n+    watcher.start()\n+    time.sleep(0.3)\n+    stop_event.set()\n+    watcher.join(timeout=2.0)\n+\n+    assert not watcher.is_alive()\n+    assert loop._captured == [], \"Stale marker must not fire shutdown\"\n+    shutdown_handler.assert_not_called()\n+    assert not marker.exists(), \"Stale marker should have been cleaned up\"\n+\n+\n+def test_planned_stop_marker_targets_self_probe_is_non_destructive(tmp_path, monkeypatch):\n+    \"\"\"The probe returns True for a self-marker WITHOUT unlinking it.\n+\n+    The shutdown handler performs the authoritative consume on its own\n+    thread, so the watcher's probe must leave a matching marker intact.\n+    \"\"\"\n+    marker = tmp_path / \".gateway-planned-stop.json\"\n+    _write_self_marker(marker)\n+    monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n+\n+    assert status_mod.planned_stop_marker_targets_self() is True\n+    assert marker.exists(), \"Probe must not consume a matching marker\"\n+    # Idempotent: still True on a second call.\n+    assert status_mod.planned_stop_marker_targets_self() is True\n+\n+\n+def test_planned_stop_marker_targets_self_drops_malformed(tmp_path, monkeypatch):\n+    \"\"\"A malformed marker reports False and is cleaned up.\"\"\"\n+    marker = tmp_path / \".gateway-planned-stop.json\"\n+    marker.write_text(\"{not valid json\", encoding=\"utf-8\")\n+    monkeypatch.setattr(status_mod, \"_get_planned_stop_marker_path\", lambda: marker)\n+\n+    assert status_mod.planned_stop_marker_targets_self() is False"]},{"file":"tests/gateway/test_status.py","hunks":["@@ -707,6 +707,33 @@ def test_consume_returns_false_on_start_time_mismatch(self, tmp_path, monkeypatc\n \n         assert result is False\n \n+    def test_consume_returns_true_on_windows_when_start_time_unavailable(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"Takeover consume must also recognise a self-marker on platforms\n+        without ``/proc`` (macOS / native Windows).\n+\n+        ``consume_takeover_marker_for_self`` shares ``_consume_pid_marker_for_self``\n+        with the planned-stop path, so the same start_time fallback applies:\n+        a ``--replace`` SIGTERM on Windows (where start_time is None on both\n+        sides) must be recognised as a planned takeover and exit 0, not be\n+        misclassified as an unexpected UNKNOWN exit. With start_time\n+        unavailable we fall back to PID equality alone, bounded by the TTL.\n+        \"\"\"\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+        # Simulate Windows: no start_time available for any PID.\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: None)\n+\n+        ok = status.write_takeover_marker(target_pid=os.getpid())\n+        assert ok is True\n+        payload = json.loads((tmp_path / \".gateway-takeover.json\").read_text())\n+        assert payload[\"target_start_time\"] is None\n+\n+        result = status.consume_takeover_marker_for_self()\n+\n+        assert result is True\n+        assert not (tmp_path / \".gateway-takeover.json\").exists()\n+\n     def test_consume_returns_false_when_marker_missing(self, tmp_path, monkeypatch):\n         monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n ","@@ -899,6 +926,74 @@ def raise_oserror(*args, **kwargs):\n \n         assert ok is False\n \n+    def test_consume_returns_true_on_windows_when_start_time_unavailable(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"Regression for #34597: a legitimate stop must be recognised on\n+        platforms without ``/proc``.\n+\n+        ``_get_process_start_time`` returns None on macOS / native Windows\n+        (no ``/proc/<pid>/stat``). The planned-stop watcher only runs there,\n+        so if the authoritative consume required a non-None start_time match\n+        it would always return False — and ``hermes gateway stop`` would be\n+        misclassified as an unexpected ``UNKNOWN`` exit, exit 1, and revived\n+        by the service manager (the very crash loop #34597 set out to fix).\n+        With start_time unavailable on BOTH sides we fall back to PID\n+        equality alone, bounded by the marker TTL.\n+        \"\"\"\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+        # Simulate Windows: no start_time available for any PID.\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: None)\n+\n+        ok = status.write_planned_stop_marker(target_pid=os.getpid())\n+        assert ok is True\n+        # Marker carries a null start_time, exactly as written on Windows.\n+        payload = json.loads((tmp_path / \".gateway-planned-stop.json\").read_text())\n+        assert payload[\"target_start_time\"] is None\n+\n+        result = status.consume_planned_stop_marker_for_self()\n+\n+        assert result is True\n+        assert not (tmp_path / \".gateway-planned-stop.json\").exists()\n+\n+    def test_consume_still_rejects_foreign_pid_when_start_time_unavailable(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"The PID-only fallback must NOT match a marker naming another PID.\n+\n+        Falling back to PID equality when start_time is unknown must remain\n+        a PID check — a marker for a different process is never ours.\n+        \"\"\"\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: None)\n+\n+        ok = status.write_planned_stop_marker(target_pid=os.getpid() + 9999)\n+        assert ok is True\n+\n+        result = status.consume_planned_stop_marker_for_self()\n+\n+        assert result is False\n+\n+    def test_consume_still_rejects_start_time_mismatch_when_both_known(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"PID-reuse defence is preserved when BOTH start_times are present.\n+\n+        The Windows fallback only relaxes matching when a start_time is\n+        unavailable. When both sides report one (Linux), a mismatch must\n+        still reject — otherwise PID reuse could resurrect a stale marker.\n+        \"\"\"\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: 100)\n+        status.write_planned_stop_marker(target_pid=os.getpid())\n+\n+        # Simulate PID reuse: same PID, different start_time.\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: 9999)\n+\n+        result = status.consume_planned_stop_marker_for_self()\n+\n+        assert result is False\n+\n \n class TestReadProcessCmdlinePsFallback:\n     \"\"\"Tests for _read_process_cmdline falling back to ps on non-Linux.\"\"\"\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}