{"repo":"NousResearch/hermes-agent","public_id":"17327","display_label":"Fix infinite process poll loop and exit reconciliation","summary":"These members address a bug where process polling could loop infinitely, freezing the Gateway, along with related fixes for reconciling process exit state on Windows and in reader-thread race conditions.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":17430,"merged_at":"2026-04-29T11:59:22+00:00"},"resolution":{"state":"resolved","by":"pr-17430"},"freshness":{"state":"verified","last_verified_at":"2026-04-29T11:59:22+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":17327,"pr_number":17430,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-29T11:59:22+00:00"}],"members":[{"kind":"issue","number":17327,"title":"[Bug]: process poll 无限轮询导致 Gateway 卡死","url":null,"state":"closed"},{"kind":"pr","number":17346,"title":"fix: prevent infinite process poll loop from reader-thread race (#17327)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"20b759cd0","number":0},{"kind":"pr","value":"#17430","number":17430},{"kind":"pr","value":"#17327","number":17327}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":17430,"title":"fix(process): reconcile session.exited against real child exit in poll/wait","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/tools/test_process_registry.py","tools/process_registry.py"],"fix_hunks":[{"file":"tests/tools/test_process_registry.py","hunks":["@@ -103,6 +103,134 @@ def test_poll_exited(self, registry):\n         assert result[\"exit_code\"] == 0\n \n \n+# =========================================================================\n+# Orphaned-pipe reconciliation (issue #17327)\n+# =========================================================================\n+\n[redacted:email](sys.platform == \"win32\", reason=\"POSIX-only: uses setsid/fcntl\")\n+class TestOrphanedPipeReconciliation:\n+    \"\"\"Regression tests for issue #17327.\n+\n+    `hermes update` in Feishu spawned a background subprocess that restarted\n+    the gateway; the direct child exited quickly but a descendant daemon\n+    held the stdout pipe open. `_reader_loop.finally` never ran, so\n+    `session.exited` stayed False and the agent polled 74 times over 7\n+    minutes, all returning `status: running`.\n+\n+    The fix is `_reconcile_local_exit()`: poll() and wait() now check the\n+    direct `Popen.poll()` before trusting `session.exited`.\n+    \"\"\"\n+\n+    def test_reconcile_flips_exited_when_direct_child_done(self, registry):\n+        \"\"\"Direct child exited but reader thread is blocked on orphaned pipe.\"\"\"\n+        # Simulate the orphaned-pipe scenario: direct child exited, but a\n+        # descendant holds stdout open so the reader never sees EOF.\n+        # Approach: spawn `sh -c 'sleep 10 &'` with setsid — sh forks the\n+        # sleep into a new session group, exits immediately, but sleep\n+        # inherits the stdout pipe and keeps it open.\n+        proc = subprocess.Popen(\n+            [\"sh\", \"-c\", \"exec 1>&2; ( sleep 30 ) & disown; exit 0\"],\n+            stdout=subprocess.PIPE,\n+            stderr=subprocess.STDOUT,\n+            preexec_fn=os.setsid,\n+        )\n+\n+        s = _make_session(sid=\"proc_orphan_test\")\n+        s.process = proc\n+        s.pid = proc.pid\n+        registry._running[s.id] = s\n+\n+        # Wait for the direct child to exit. We don't start a reader thread,\n+        # so session.exited stays False (mimicking the stuck-reader state).\n+        assert _wait_until(lambda: proc.poll() is not None, timeout=5.0), (\n+            \"Direct child should exit quickly (sh exits, sleep descendant \"\n+            \"holds the pipe open)\"\n+        )\n+\n+        # Before the fix: poll would return \"running\" forever.\n+        # After the fix: poll reconciles against proc.poll() and flips.\n+        assert s.exited is False  # Precondition: reader hasn't updated it.\n+        result = registry.poll(s.id)\n+        assert result[\"status\"] == \"exited\", (\n+            f\"Expected reconciled 'exited' status; got {result!r}. \"\n+            \"This is issue #17327 — reader is blocked on orphaned pipe.\"\n+        )\n+        assert result[\"exit_code\"] == 0\n+        assert s.exited is True\n+        assert s.id in registry._finished\n+        assert s.id not in registry._running\n+\n+        # Clean up the orphaned descendant.\n+        try:\n+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)\n+        except (ProcessLookupError, PermissionError):\n+            pass\n+\n+    def test_reconcile_noop_when_child_still_running(self, registry):\n+        \"\"\"Reconcile must NOT flip exited when the direct child is alive.\"\"\"\n+        proc = _spawn_python_sleep(5.0)\n+        s = _make_session(sid=\"proc_running_test\")\n+        s.process = proc\n+        s.pid = proc.pid\n+        registry._running[s.id] = s\n+\n+        result = registry.poll(s.id)\n+        assert result[\"status\"] == \"running\"\n+        assert s.exited is False\n+\n+        proc.kill()\n+        proc.wait()\n+\n+    def test_reconcile_noop_on_already_exited(self, registry):\n+        \"\"\"Reconcile is a no-op when session.exited is already True.\"\"\"\n+        s = _make_session(sid=\"proc_already_exited\", exited=True, exit_code=7)\n+        s.process = MagicMock()\n+        s.process.poll = MagicMock(return_value=0)  # Would say exit 0\n+        registry._finished[s.id] = s\n+\n+        registry._reconcile_local_exit(s)\n+        # Must not overwrite the existing exit_code with proc.poll()'s 0.\n+        assert s.exit_code == 7\n+\n+    def test_reconcile_noop_on_no_process(self, registry):\n+        \"\"\"Reconcile is a no-op for sessions without a local Popen (env/PTY).\"\"\"\n+        s = _make_session(sid=\"proc_no_popen\")\n+        assert getattr(s, \"process\", None) is None\n+        # Must not raise.\n+        registry._reconcile_local_exit(s)\n+        assert s.exited is False\n+\n+    def test_wait_returns_when_reader_blocked(self, registry):\n+        \"\"\"wait() must also reconcile — not just poll().\"\"\"\n+        proc = subprocess.Popen(\n+            [\"sh\", \"-c\", \"( sleep 30 ) & disown; exit 0\"],\n+            stdout=subprocess.PIPE,\n+            stderr=subprocess.STDOUT,\n+            preexec_fn=os.setsid,\n+        )\n+\n+        s = _make_session(sid=\"proc_wait_orphan\")\n+        s.process = proc\n+        s.pid = proc.pid\n+        registry._running[s.id] = s\n+\n+        assert _wait_until(lambda: proc.poll() is not None, timeout=5.0)\n+\n+        start = time.monotonic()\n+        result = registry.wait(s.id, timeout=10)\n+        elapsed = time.monotonic() - start\n+\n+        assert result[\"status\"] == \"exited\", result\n+        assert elapsed < 5.0, (\n+            f\"wait() should return ~immediately via reconcile; took {elapsed:.1f}s\"\n+        )\n+\n+        try:\n+            os.killpg(os.getpgid(proc.pid), signal.SIGKILL)\n+        except (ProcessLookupError, PermissionError):\n+            pass\n+\n+\n # =========================================================================\n # Read log\n # ========================================================================="]},{"file":"tools/process_registry.py","hunks":["@@ -800,6 +800,78 @@ def get(self, session_id: str) -> Optional[ProcessSession]:\n             session = self._running.get(session_id) or self._finished.get(session_id)\n         return self._refresh_detached_session(session)\n \n+    def _reconcile_local_exit(self, session: \"ProcessSession\") -> None:\n+        \"\"\"Reconcile session.exited against the real child process state.\n+\n+        The reader thread (`_reader_loop`) sets `session.exited = True` only\n+        in its `finally` block, which runs when `stdout.read()` returns EOF.\n+        If the direct `Popen` child has exited but a descendant process (e.g.\n+        a daemon spawned by `hermes update` restarting the gateway) is still\n+        holding the stdout pipe open, the reader blocks forever and poll()\n+        keeps returning \"running\" indefinitely (issue #17327 — 74 polls over\n+        7 minutes on Feishu).\n+\n+        This helper closes that window: when `session.exited` is still False\n+        but the direct child's `Popen.poll()` reports an exit code, drain any\n+        readable bytes non-blocking and flip `session.exited`. The orphaned\n+        reader thread remains stuck on its blocking `read()` but is a daemon\n+        thread and will be reaped with the process.\n+\n+        Safe no-op on sessions without a local `Popen` (env/PTY), already-\n+        exited sessions, and detached-recovered sessions.\n+        \"\"\"\n+        if session is None or session.exited:\n+            return\n+        proc = getattr(session, \"process\", None)\n+        if proc is None:\n+            return\n+        try:\n+            rc = proc.poll()\n+        except Exception:\n+            return\n+        if rc is None:\n+            return  # Direct child still running — reader block is legitimate.\n+\n+        # Direct child exited. Try to drain any bytes the reader hasn't\n+        # consumed yet. This is best-effort: if the pipe is held open by a\n+        # descendant, the non-blocking read returns what's immediately\n+        # available and we stop.\n+        drained = \"\"\n+        stdout = getattr(proc, \"stdout\", None)\n+        if stdout is not None and not _IS_WINDOWS:\n+            try:\n+                import fcntl\n+                fd = stdout.fileno()\n+                flags = fcntl.fcntl(fd, fcntl.F_GETFL)\n+                fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)\n+                try:\n+                    chunk = stdout.read()\n+                    if chunk:\n+                        drained = chunk if isinstance(chunk, str) else chunk.decode(\"utf-8\", errors=\"replace\")\n+                except (BlockingIOError, OSError, ValueError):\n+                    pass\n+                finally:\n+                    try:\n+                        fcntl.fcntl(fd, fcntl.F_SETFL, flags)\n+                    except Exception:\n+                        pass\n+            except Exception as e:\n+                logger.debug(\"Non-blocking drain failed for %s: %s\", session.id, e)\n+\n+        with session._lock:\n+            if drained:\n+                session.output_buffer += drained\n+                if len(session.output_buffer) > session.max_output_chars:\n+                    session.output_buffer = session.output_buffer[-session.max_output_chars:]\n+            session.exited = True\n+            session.exit_code = rc\n+        logger.info(\n+            \"Reconciled session %s: direct child exited with code %s but reader \"\n+            \"was still blocked (orphaned pipe). Flipped to exited.\",\n+            session.id, rc,\n+        )\n+        self._move_to_finished(session)\n+\n     def poll(self, session_id: str) -> dict:\n         \"\"\"Check status and get new output for a background process.\"\"\"\n         from tools.ansi_strip import strip_ansi","@@ -808,6 +880,10 @@ def poll(self, session_id: str) -> dict:\n         if session is None:\n             return {\"status\": \"not_found\", \"error\": f\"No process with ID {session_id}\"}\n \n+        # Reconcile against real child state before reading session.exited.\n+        # Guards against orphaned-pipe reader hangs (issue #17327).\n+        self._reconcile_local_exit(session)\n+\n         with session._lock:\n             output_preview = strip_ansi(session.output_buffer[-1000:]) if session.output_buffer else \"\"\n ","@@ -898,6 +974,10 @@ def wait(self, session_id: str, timeout: int = None) -> dict:\n \n         while time.monotonic() < deadline:\n             session = self._refresh_detached_session(session)\n+            # Reconcile against real child state — guards against orphaned-\n+            # pipe reader hangs where the reader is blocked but the direct\n+            # child has already exited (issue #17327).\n+            self._reconcile_local_exit(session)\n             if session.exited:\n                 self._completion_consumed.add(session_id)\n                 result = {\n"]}]},{"kind":"pr","number":47733,"title":"fix(process_registry): add Windows drain path in _reconcile_local_exit","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}