{"repo":"NousResearch/hermes-agent","public_id":"16376","display_label":"Gateway stale lock detection broken on macOS/Windows","summary":"These issues and fixes center on the gateway's stale lock detection failing on non-Linux platforms because process start-time retrieval relied on Linux-only mechanisms, causing PID reuse to go undetected and leaving stale locks in place. The work involves adding cross-platform fallbacks (psutil, ps, cmdline inspection) to correctly identify and evict stale scoped and platform locks.","subsystem":"gateway","counts":{"members":25,"open_issues":1,"open_issues_distinct":1,"open_prs":0,"closed_issues":7,"merged_prs":2,"attempt_prs":15},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":68650,"merged_at":"2026-07-21T19:39:13+00:00"},"resolution":{"state":"partially_resolved","by":"pr-24500"},"freshness":{"state":"verified","last_verified_at":"2026-07-21T19:39:13+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.09519999999999997},"fix_edges":[{"issue_number":16376,"pr_number":24500,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-12T22:00:30+00:00"},{"issue_number":31890,"pr_number":68650,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-07-21T19:39:13+00:00"},{"issue_number":42685,"pr_number":68650,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-07-21T19:39:13+00:00"},{"issue_number":53763,"pr_number":68650,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-07-21T19:39:13+00:00"}],"members":[{"kind":"issue","number":16376,"title":"[Bug]: macOS stale Telegram token lock can block gateway after PID reuse","url":null,"state":"closed"},{"kind":"issue","number":16586,"title":"Stale gateway lock files never cleared on macOS — process-start-time check is Linux-only","url":null,"state":"closed"},{"kind":"issue","number":18778,"title":"gateway: scoped lock PID-reuse guard is a no-op on macOS/Windows — stale lockfiles permanently block startup","url":null,"state":"closed"},{"kind":"issue","number":21596,"title":"Stale platform-lock detection broken on macOS: PID reuse causes permanent startup failure","url":null,"state":"closed"},{"kind":"issue","number":21613,"title":"macOS: Telegram gateway lock always has start_time=null, causing PID reuse false positives","url":null,"state":"closed"},{"kind":"issue","number":24067,"title":"[Bug]: macOS PID lock check fails — system process occupies same PID, causing zombie lock on gateway restart","url":null,"state":"closed"},{"kind":"issue","number":31890,"title":"macOS gateway scoped locks can block adapters after PID reuse when start_time is null","url":null,"state":"open"},{"kind":"issue","number":53763,"title":"Bug: Stale lock cleanup fails on macOS due to PID recycling","url":null,"state":"closed"},{"kind":"pr","number":16423,"title":"fix(gateway): detect stale scoped locks via cmdline when start_time is absent","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"653d30429039d1f5f048889e5397c1297c1fda38","number":0},{"kind":"pr","value":"#23953","number":23953}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":16432,"title":"fix(gateway): detect stale scoped lock when PID is reused on macOS (#16376)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16423","number":16423},{"kind":"pr","value":"#15144","number":15144},{"kind":"pr","value":"#15584","number":15584}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":16655,"title":"fix(gateway): detect stale lock files on macOS via `ps -o lstart=`","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#15144","number":15144}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":16942,"title":"fix(gateway): add macOS/BSD fallback for process start-time and cmdline detection in lock files","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16432","number":16432},{"kind":"pr","value":"#12448","number":12448},{"kind":"pr","value":"#15144","number":15144},{"kind":"pr","value":"#16423","number":16423}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":16966,"title":"fix(scoped_lock): treat null start_time lock as stale when live cmdline lacks hermes","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16432","number":16432},{"kind":"pr","value":"#16376","number":16376},{"kind":"pr","value":"#16423","number":16423},{"kind":"pr","value":"#15144","number":15144}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":21018,"title":"fix(gateway): detect recycled macOS lock PIDs","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"653d30429039d1f5f048889e5397c1297c1fda38","number":0},{"kind":"pr","value":"#16586","number":16586}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":22490,"title":"fix(gateway): use psutil for cross-platform process start time","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":22564,"title":"fix(gateway): macOS fallback for _get_process_start_time","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#22490","number":22490},{"kind":"pr","value":"#22564","number":22564}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":23953,"title":"fix(gateway): detect stale scoped locks via cmdline when start_time is absent on macOS","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"653d30429","number":0},{"kind":"pr","value":"#24500","number":24500},{"kind":"pr","value":"#21561","number":21561}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":24500,"title":"fix(gateway): detect stale scoped locks via cmdline when start_time is absent on macOS","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/status.py","scripts/release.py","tests/gateway/test_status.py"],"fix_hunks":[{"file":"gateway/status.py","hunks":["@@ -124,16 +124,33 @@ def get_process_start_time(pid: int) -> Optional[int]:\n \n \n def _read_process_cmdline(pid: int) -> Optional[str]:\n-    \"\"\"Return the process command line as a space-separated string.\"\"\"\n+    \"\"\"Return the process command line as a space-separated string.\n+\n+    On Linux, reads /proc/<pid>/cmdline directly.  On macOS and other\n+    platforms without /proc, falls back to ``ps -p <pid> -o command=``.\n+    \"\"\"\n     cmdline_path = Path(f\"/proc/{pid}/cmdline\")\n     try:\n         raw = cmdline_path.read_bytes()\n     except (FileNotFoundError, PermissionError, OSError):\n-        return None\n+        pass\n+    else:\n+        if raw:\n+            return raw.replace(b\"\\x00\", b\" \").decode(\"utf-8\", errors=\"ignore\").strip()\n \n-    if not raw:\n-        return None\n-    return raw.replace(b\"\\x00\", b\" \").decode(\"utf-8\", errors=\"ignore\").strip()\n+    try:\n+        result = subprocess.run(\n+            [\"ps\", \"-p\", str(pid), \"-o\", \"command=\"],\n+            capture_output=True,\n+            text=True,\n+            timeout=5,\n+        )\n+        if result.returncode == 0 and result.stdout.strip():\n+            return result.stdout.strip()\n+    except (OSError, subprocess.TimeoutExpired):\n+        pass\n+\n+    return None\n \n \n def _looks_like_gateway_process(pid: int) -> bool:","@@ -594,6 +611,17 @@ def acquire_scoped_lock(scope: str, identity: str, metadata: Optional[dict[str,\n                     and current_start != existing.get(\"start_time\")\n                 ):\n                     stale = True\n+                # When start_time comparison is unavailable (macOS / Windows\n+                # have no /proc, so both sides are None), fall back to\n+                # checking the live process command line.  If the PID was\n+                # reused by an unrelated process the lock is stale.\n+                if (\n+                    not stale\n+                    and existing.get(\"start_time\") is None\n+                    and current_start is None\n+                    and not _looks_like_gateway_process(existing_pid)\n+                ):\n+                    stale = True\n                 # Check if process is stopped (Ctrl+Z / SIGTSTP) — stopped\n                 # processes still appear alive to _pid_exists but are not\n                 # actually running. Treat them as stale so --replace works."]},{"file":"scripts/release.py","hunks":["@@ -58,6 +58,7 @@\n     \"[redacted:email]\": \"hrkzogw\",\n     \"[redacted:email]\": \"dandacompany\",\n     \"[redacted:email]\": \"TreyDong\",\n+    \"[redacted:email]\": \"pkyanam\",\n     \"127238744+teknium1@users.noreply.github.com\": \"teknium1\",\n     \"[redacted:email]\": \"Hugo-SEQUIER\",\n     \"128259593+Gutslabs@users.noreply.github.com\": \"Gutslabs\","]},{"file":"tests/gateway/test_status.py","hunks":["@@ -444,6 +444,59 @@ def test_acquire_scoped_lock_rejects_live_other_process(self, tmp_path, monkeypa\n         assert acquired is False\n         assert existing[\"pid\"] == 99999\n \n+    def test_acquire_scoped_lock_replaces_pid_reused_by_unrelated_process(self, tmp_path, monkeypatch):\n+        \"\"\"macOS regression: PID reused by an unrelated process with start_time=None.\n+\n+        On macOS /proc is unavailable, so both the lock record and the live\n+        process report start_time=None.  The live PID is alive (os.kill\n+        succeeds) but belongs to a completely different program.  The lock\n+        must be treated as stale.\n+        \"\"\"\n+        monkeypatch.setenv(\"HERMES_GATEWAY_LOCK_DIR\", str(tmp_path / \"locks\"))\n+        lock_path = tmp_path / \"locks\" / \"telegram-bot-token-2bb80d537b1da3e3.lock\"\n+        lock_path.parent.mkdir(parents=True, exist_ok=True)\n+        lock_path.write_text(json.dumps({\n+            \"pid\": 873,\n+            \"start_time\": None,\n+            \"kind\": \"hermes-gateway\",\n+            \"argv\": [\"/Users/user/.hermes/hermes-agent/hermes_cli/main.py\", \"gateway\", \"run\", \"--replace\"],\n+        }))\n+\n+        # Post-#21561 the liveness probe routes through\n+        # ``gateway.status._pid_exists`` (psutil-first, safe on Windows),\n+        # not ``os.kill``.\n+        monkeypatch.setattr(status, \"_pid_exists\", lambda pid: True)\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: None)\n+        monkeypatch.setattr(status, \"_looks_like_gateway_process\", lambda pid: False)\n+\n+        acquired, existing = status.acquire_scoped_lock(\"telegram-bot-token\", \"secret\", metadata={\"platform\": \"telegram\"})\n+\n+        assert acquired is True\n+        payload = json.loads(lock_path.read_text())\n+        assert payload[\"pid\"] == os.getpid()\n+        assert payload[\"metadata\"][\"platform\"] == \"telegram\"\n+\n+    def test_acquire_scoped_lock_keeps_lock_when_pid_reused_by_gateway(self, tmp_path, monkeypatch):\n+        \"\"\"When start_time is None but the live PID still looks like a gateway, keep the lock.\"\"\"\n+        monkeypatch.setenv(\"HERMES_GATEWAY_LOCK_DIR\", str(tmp_path / \"locks\"))\n+        lock_path = tmp_path / \"locks\" / \"telegram-bot-token-2bb80d537b1da3e3.lock\"\n+        lock_path.parent.mkdir(parents=True, exist_ok=True)\n+        lock_path.write_text(json.dumps({\n+            \"pid\": 99999,\n+            \"start_time\": None,\n+            \"kind\": \"hermes-gateway\",\n+            \"argv\": [\"/Users/user/.hermes/hermes-agent/hermes_cli/main.py\", \"gateway\", \"run\", \"--replace\"],\n+        }))\n+\n+        monkeypatch.setattr(status, \"_pid_exists\", lambda pid: True)\n+        monkeypatch.setattr(status, \"_get_process_start_time\", lambda pid: None)\n+        monkeypatch.setattr(status, \"_looks_like_gateway_process\", lambda pid: True)\n+\n+        acquired, existing = status.acquire_scoped_lock(\"telegram-bot-token\", \"secret\", metadata={\"platform\": \"telegram\"})\n+\n+        assert acquired is False\n+        assert existing[\"pid\"] == 99999\n+\n     def test_acquire_scoped_lock_replaces_stale_record(self, tmp_path, monkeypatch):\n         monkeypatch.setenv(\"HERMES_GATEWAY_LOCK_DIR\", str(tmp_path / \"locks\"))\n         lock_path = tmp_path / \"locks\" / \"telegram-bot-token-2bb80d537b1da3e3.lock\"","@@ -811,3 +864,46 @@ def raise_oserror(*args, **kwargs):\n         ok = status.write_planned_stop_marker(target_pid=12345)\n \n         assert ok is False\n+\n+\n+class TestReadProcessCmdlinePsFallback:\n+    \"\"\"Tests for _read_process_cmdline falling back to ps on non-Linux.\"\"\"\n+\n+    def test_ps_fallback_when_proc_unavailable(self, monkeypatch):\n+        monkeypatch.setattr(status.Path, \"read_bytes\", lambda self: (_ for _ in ()).throw(FileNotFoundError))\n+        monkeypatch.setattr(\n+            status.subprocess, \"run\",\n+            lambda args, **kwargs: SimpleNamespace(returncode=0, stdout=\"/usr/libexec/bluetoothuserd\\n\"),\n+        )\n+        result = status._read_process_cmdline(873)\n+        assert result == \"/usr/libexec/bluetoothuserd\"\n+\n+    def test_ps_fallback_returns_none_on_failure(self, monkeypatch):\n+        monkeypatch.setattr(status.Path, \"read_bytes\", lambda self: (_ for _ in ()).throw(FileNotFoundError))\n+        monkeypatch.setattr(\n+            status.subprocess, \"run\",\n+            lambda args, **kwargs: SimpleNamespace(returncode=1, stdout=\"\"),\n+        )\n+        result = status._read_process_cmdline(99999)\n+        assert result is None\n+\n+    def test_proc_cmdline_takes_priority_over_ps(self, monkeypatch):\n+        calls = []\n+\n+        def fake_read_bytes(self):\n+            calls.append(\"proc\")\n+            return b\"python\\x00hermes_cli/main.py\\x00gateway\\x00\"\n+\n+        monkeypatch.setattr(status.Path, \"read_bytes\", fake_read_bytes)\n+        result = status._read_process_cmdline(12345)\n+        assert \"hermes_cli/main.py\" in result\n+        assert calls == [\"proc\"]\n+\n+    def test_ps_fallback_used_when_proc_returns_empty(self, monkeypatch):\n+        monkeypatch.setattr(status.Path, \"read_bytes\", lambda self: b\"\")\n+        monkeypatch.setattr(\n+            status.subprocess, \"run\",\n+            lambda args, **kwargs: SimpleNamespace(returncode=0, stdout=\"python hermes_cli/main.py gateway run\\n\"),\n+        )\n+        result = status._read_process_cmdline(12345)\n+        assert \"hermes_cli/main.py\" in result\n"]}]},{"kind":"pr","number":24595,"title":"fix(gateway): use psutil for cross-platform process start_time + cmdline detection","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":31923,"title":"fix(gateway): detect scoped-lock PID reuse on macOS when start_time is null (#31890)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"012f40c98","number":0},{"kind":"pr","value":"#012","number":12},{"kind":"pr","value":"#31890","number":31890}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":35625,"title":"fix(gateway): cross-platform process start-time for scoped-lock PID-reuse guard","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"012f40c98c18b6723e355abbba7544b752836276","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":53767,"title":"fix(gateway): detect stale lock when macOS psutil returns valid start_time for recycled PID","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#68650","number":68650},{"kind":"pr","value":"#42689","number":42689}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":54279,"title":"fix(gateway): treat null-start scoped locks as stale","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#68650","number":68650},{"kind":"pr","value":"#53767","number":53767}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":56919,"title":"fix(gateway): honor cmdline fallback for null-start_time locks when live start_time resolves","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#68650","number":68650},{"kind":"pr","value":"#53767","number":53767}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":68650,"title":"fix(gateway): stale scoped-lock detection for mixed-None start times + guarded lock reads + atomic removal","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/status.py","tests/gateway/test_status.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}