{"repo":"NousResearch/hermes-agent","public_id":"17758","display_label":"Gateway recursion causing stack overflow on message drain","summary":"A stack overflow or SIGSEGV was occurring in the gateway's pending message queue drain logic due to direct recursion. The fix replaces the recursive approach with an independent/fresh task.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":17896,"merged_at":"2026-04-30T10:27:08+00:00"},"resolution":{"state":"resolved","by":"pr-17896"},"freshness":{"state":"verified","last_verified_at":"2026-04-30T10:27:08+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":17758,"pr_number":17896,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-30T10:27:08+00:00"}],"members":[{"kind":"issue","number":17758,"title":"Stack overflow / SIGSEGV in _process_message_background due to direct recursion on pending-queue drain","url":null,"state":"closed"},{"kind":"pr","number":17772,"title":"fix(gateway): drain pending messages via fresh task, not recursion (#17758)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#17896","number":17896}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":17863,"title":"fix(gateway): drain pending messages via independent task, not recursive await","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#17772","number":17772},{"kind":"pr","value":"#17896","number":17896},{"kind":"pr","value":"#17758","number":17758}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":17896,"title":"fix(gateway): drain pending messages via fresh task, not recursion (#17758)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/platforms/base.py","tests/gateway/test_duplicate_reply_suppression.py","tests/gateway/test_pending_drain_no_recursion.py"],"fix_hunks":[{"file":"gateway/platforms/base.py","hunks":["@@ -2708,9 +2708,27 @@ async def _stop_typing_task() -> None:\n                 if _active is not None:\n                     _active.clear()\n                 await _stop_typing_task()\n-                # Process pending message in new background task\n-                await self._process_message_background(pending_event, session_key)\n-                return  # Already cleaned up\n+                # Spawn a fresh task for the pending message instead of\n+                # recursing.  Issue #17758: `await\n+                # self._process_message_background(...)` here grew the\n+                # call stack one frame per chained follow-up, and under\n+                # sustained pending-queue activity the C stack would\n+                # exhaust at ~2000 frames and SIGSEGV the process.\n+                # Mirror the late-arrival drain pattern below: hand off\n+                # to a new task and return so this frame can unwind.\n+                drain_task = asyncio.create_task(\n+                    self._process_message_background(pending_event, session_key)\n+                )\n+                # Hand ownership of the session to the drain task so\n+                # stale-lock detection keeps working while it runs.\n+                self._session_tasks[session_key] = drain_task\n+                try:\n+                    self._background_tasks.add(drain_task)\n+                    drain_task.add_done_callback(self._background_tasks.discard)\n+                except TypeError:\n+                    # Tests stub create_task() with non-hashable sentinels; tolerate.\n+                    pass\n+                return  # Drain task owns the session now.\n                 \n         except asyncio.CancelledError:\n             current_task = asyncio.current_task()","@@ -2772,25 +2790,41 @@ async def _stop_typing_task() -> None:\n             # dropped (user never gets a reply).\n             late_pending = self._pending_messages.pop(session_key, None)\n             if late_pending is not None:\n-                logger.debug(\n-                    \"[%s] Late-arrival pending message during cleanup — spawning drain task\",\n-                    self.name,\n-                )\n-                _active = self._active_sessions.get(session_key)\n-                if _active is not None:\n-                    _active.clear()\n-                drain_task = asyncio.create_task(\n-                    self._process_message_background(late_pending, session_key)\n-                )\n-                # Hand ownership of the session to the drain task so stale-lock\n-                # detection keeps working while it runs.\n-                self._session_tasks[session_key] = drain_task\n-                try:\n-                    self._background_tasks.add(drain_task)\n-                    drain_task.add_done_callback(self._background_tasks.discard)\n-                except TypeError:\n-                    # Tests stub create_task() with non-hashable sentinels; tolerate.\n-                    pass\n+                current_task = asyncio.current_task()\n+                existing_task = self._session_tasks.get(session_key)\n+                if (\n+                    existing_task is not None\n+                    and existing_task is not current_task\n+                ):\n+                    # The in-band drain (or an earlier late-arrival drain)\n+                    # already spawned a follow-up task that owns this\n+                    # session.  Re-queue the late-arrival event so that\n+                    # task picks it up — avoids spawning two concurrent\n+                    # _process_message_background tasks for the same key\n+                    # (#17758 follow-up: prevents the create_task path\n+                    # from racing with itself across the in-band/finally\n+                    # boundary).\n+                    self._pending_messages[session_key] = late_pending\n+                else:\n+                    logger.debug(\n+                        \"[%s] Late-arrival pending message during cleanup — spawning drain task\",\n+                        self.name,\n+                    )\n+                    _active = self._active_sessions.get(session_key)\n+                    if _active is not None:\n+                        _active.clear()\n+                    drain_task = asyncio.create_task(\n+                        self._process_message_background(late_pending, session_key)\n+                    )\n+                    # Hand ownership of the session to the drain task so stale-lock\n+                    # detection keeps working while it runs.\n+                    self._session_tasks[session_key] = drain_task\n+                    try:\n+                        self._background_tasks.add(drain_task)\n+                        drain_task.add_done_callback(self._background_tasks.discard)\n+                    except TypeError:\n+                        # Tests stub create_task() with non-hashable sentinels; tolerate.\n+                        pass\n                 # Leave _active_sessions[session_key] populated — the drain\n                 # task's own lifecycle will clean it up.\n             else:","@@ -2798,10 +2832,23 @@ async def _stop_typing_task() -> None:\n                 # reset-like command that already swapped in its own\n                 # command_guard (and cancelled us) can't be accidentally\n                 # cleared by our unwind.  The command owns the session now.\n+                #\n+                # The owner-check also covers the in-band drain handoff\n+                # above: when we spawned a drain_task and transferred\n+                # ownership via ``_session_tasks[session_key] = drain_task``,\n+                # ``_session_tasks.get(session_key) is current_task`` is\n+                # False, so we leave _active_sessions populated.  Without\n+                # this guard, the drain task picks up the same\n+                # interrupt_event in its own _process_message_background\n+                # entry, _release_session_guard's guard-match succeeds,\n+                # and we'd delete the entry while the drain task is still\n+                # running — letting a concurrent inbound message pass\n+                # the Level-1 guard and spawn a second handler for the\n+                # same session.\n                 current_task = asyncio.current_task()\n                 if current_task is not None and self._session_tasks.get(session_key) is current_task:\n                     del self._session_tasks[session_key]\n-                self._release_session_guard(session_key, guard=interrupt_event)\n+                    self._release_session_guard(session_key, guard=interrupt_event)\n     \n     async def cancel_background_tasks(self) -> None:\n         \"\"\"Cancel any in-flight background message-processing tasks."]},{"file":"tests/gateway/test_duplicate_reply_suppression.py","hunks":["@@ -108,6 +108,15 @@ async def fake_handler(event):\n \n         await adapter._process_message_background(event_a, session_key)\n \n+        # The in-band pending-drain now hands off to a fresh task instead\n+        # of recursing (#17758).  Wait for that task to finish before\n+        # checking the sent list.\n+        for _ in range(200):\n+            if any(s[\"content\"] == pending_response for s in adapter.sent):\n+                break\n+            await asyncio.sleep(0.01)\n+        await adapter.cancel_background_tasks()\n+\n         # The stale response should NOT have been sent.\n         stale_sends = [s for s in adapter.sent if s[\"content\"] == stale_response]\n         assert len(stale_sends) == 0, ("]},{"file":"tests/gateway/test_pending_drain_no_recursion.py","hunks":["@@ -0,0 +1,188 @@\n+\"\"\"Regression test for #17758 — chained pending-message drains must not\n+grow the call stack.\n+\n+Before the fix, ``_process_message_background`` finished a turn, found a\n+pending follow-up, and drained it via ``await\n+self._process_message_background(pending_event, session_key)``.  Each\n+queued follow-up added a frame to the call stack instead of starting\n+fresh, so under sustained pending-queue activity the C stack would\n+exhaust at ~2000 nested frames and the process would crash with\n+SIGSEGV.\n+\n+After the fix, the in-band drain spawns a fresh task (mirroring the\n+late-arrival drain pattern), so the stack stays bounded regardless of\n+chain length.\n+\n+We assert the invariant directly: count nested\n+``_process_message_background`` frames at handler entry across a chain\n+of N follow-ups.  Recursion makes depth grow linearly (1, 2, 3, …, N);\n+task spawning keeps it constant (1 every time).\n+\"\"\"\n+\n+import asyncio\n+import sys\n+from unittest.mock import AsyncMock\n+\n+import pytest\n+\n+from gateway.config import Platform, PlatformConfig\n+from gateway.platforms.base import (\n+    BasePlatformAdapter,\n+    MessageEvent,\n+    MessageType,\n+)\n+from gateway.session import SessionSource, build_session_key\n+\n+\n+class _StubAdapter(BasePlatformAdapter):\n+    async def connect(self):\n+        pass\n+\n+    async def disconnect(self):\n+        pass\n+\n+    async def send(self, chat_id, text, **kwargs):\n+        return None\n+\n+    async def get_chat_info(self, chat_id):\n+        return {}\n+\n+\n+def _make_adapter():\n+    adapter = _StubAdapter(PlatformConfig(enabled=True, token=\"t\"), Platform.TELEGRAM)\n+    adapter._send_with_retry = AsyncMock(return_value=None)\n+    return adapter\n+\n+\n+def _make_event(text=\"hi\", chat_id=\"42\"):\n+    return MessageEvent(\n+        text=text,\n+        message_type=MessageType.TEXT,\n+        source=SessionSource(platform=Platform.TELEGRAM, chat_id=chat_id, chat_type=\"dm\"),\n+    )\n+\n+\n+def _sk(chat_id=\"42\"):\n+    return build_session_key(\n+        SessionSource(platform=Platform.TELEGRAM, chat_id=chat_id, chat_type=\"dm\")\n+    )\n+\n+\n+def _count_pmb_frames() -> int:\n+    \"\"\"Walk the current call stack and count nested\n+    ``_process_message_background`` frames.  Used to detect recursive\n+    in-band drains.\"\"\"\n+    f = sys._getframe()\n+    n = 0\n+    while f is not None:\n+        if f.f_code.co_name == \"_process_message_background\":\n+            n += 1\n+        f = f.f_back\n+    return n\n+\n+\n[redacted:email]\n+async def test_in_band_drain_does_not_grow_stack():\n+    \"\"\"Issue #17758: chained pending-message drains must not recurse.\n+\n+    Queue a fresh pending message inside each handler invocation so the\n+    in-band drain block fires for every turn in the chain.  After N\n+    turns, the recorded stack depth at handler entry must stay bounded.\n+    Pre-fix, depths would be 1, 2, 3, …, N; post-fix, depths are 1\n+    every time because each drain runs in its own task.\n+    \"\"\"\n+    N = 12\n+    adapter = _make_adapter()\n+    sk = _sk()\n+\n+    depths: list[int] = []\n+    next_index = [1]\n+\n+    async def handler(event):\n+        depths.append(_count_pmb_frames())\n+        if next_index[0] < N:\n+            adapter._pending_messages[sk] = _make_event(text=f\"M{next_index[0]}\")\n+            next_index[0] += 1\n+        return \"ok\"\n+\n+    adapter._message_handler = handler\n+\n+    await adapter.handle_message(_make_event(text=\"M0\"))\n+\n+    # Drain the chain.  Each turn schedules the next via the in-band\n+    # drain block, so we wait until N handler runs have completed and\n+    # the session has been released.\n+    for _ in range(400):\n+        if len(depths) >= N and sk not in adapter._active_sessions:\n+            break\n+        await asyncio.sleep(0.01)\n+\n+    await adapter.cancel_background_tasks()\n+\n+    assert len(depths) == N, (\n+        f\"expected {N} handler runs in the chain, got {len(depths)}: depths={depths!r}\"\n+    )\n+    max_depth = max(depths)\n+    assert max_depth <= 2, (\n+        f\"in-band drain is recursing instead of spawning a fresh task — \"\n+        f\"stack depth grew with chain length: {depths!r}\"\n+    )\n+\n+\n[redacted:email]\n+async def test_in_band_drain_preserves_active_session_guard():\n+    \"\"\"The original task must NOT release ``_active_sessions[session_key]``\n+    after handing off to the drain task.\n+\n+    When the in-band drain spawns ``drain_task`` and transfers ownership\n+    via ``_session_tasks[session_key] = drain_task``, the original task\n+    still unwinds through the ``finally`` block.  The drain task picks\n+    up the same ``interrupt_event`` in its own\n+    ``_process_message_background`` entry, so a naive\n+    ``_release_session_guard(session_key, guard=interrupt_event)`` in\n+    the unwind matches and deletes ``_active_sessions[session_key]``.\n+    That briefly reopens the Level-1 guard between the original task's\n+    finally and the drain task's first await — a concurrent inbound\n+    arriving in that window passes the guard and spawns a second\n+    handler for the same session.\n+\n+    Invariant: ``_active_sessions[sk]`` must hold the SAME interrupt\n+    Event identity at every handler entry across an in-band drain\n+    chain.  Pre-fix, the original task's finally deletes the entry, so\n+    the drain task falls through to the ``or asyncio.Event()`` branch\n+    in ``_process_message_background`` and installs a *new* Event —\n+    the identity diverges.  Post-fix, the entry is preserved across\n+    handoff and the drain task reuses the original Event.\n+    \"\"\"\n+    adapter = _make_adapter()\n+    sk = _sk()\n+\n+    seen_guards: list = []\n+\n+    async def handler(event):\n+        seen_guards.append(adapter._active_sessions.get(sk))\n+        if len(seen_guards) == 1:\n+            adapter._pending_messages[sk] = _make_event(text=\"M1\")\n+        return \"ok\"\n+\n+    adapter._message_handler = handler\n+\n+    await adapter.handle_message(_make_event(text=\"M0\"))\n+\n+    for _ in range(400):\n+        if len(seen_guards) >= 2 and sk not in adapter._active_sessions:\n+            break\n+        await asyncio.sleep(0.01)\n+\n+    await adapter.cancel_background_tasks()\n+\n+    assert len(seen_guards) == 2, f\"expected 2 handler runs, got {len(seen_guards)}\"\n+    assert seen_guards[0] is not None, \"M0 saw no active-session guard\"\n+    assert seen_guards[1] is not None, \"M1 saw no active-session guard\"\n+    assert seen_guards[0] is seen_guards[1], (\n+        \"in-band drain handoff replaced the active-session guard — the \"\n+        \"original task's finally deleted _active_sessions[sk] and the \"\n+        \"drain task installed a new Event.  Concurrent inbounds during \"\n+        \"the handoff window would bypass the Level-1 guard and spawn a \"\n+        \"second handler for the same session.\"\n+    )\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}