{"repo":"NousResearch/hermes-agent","public_id":"53175","display_label":"Event loop blocking and zombie state from agent cleanup","summary":"These issues and fixes address agent resource cleanup operations blocking or killing the gateway event loop, with changes to offload cleanup work to executor threads with timeouts. A related fix handles Telegram polling recovery when the updater stops unexpectedly.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":54130,"merged_at":"2026-06-28T09:41:37+00:00"},"resolution":{"state":"resolved","by":"pr-54130"},"freshness":{"state":"verified","last_verified_at":"2026-06-28T09:41:37+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":53175,"pr_number":54130,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-28T09:41:37+00:00"}],"members":[{"kind":"issue","number":53175,"title":"Gateway event loop dies silently after long responses + /new (zombie state, 16 crashes in 30h)","url":null,"state":"closed"},{"kind":"pr","number":53183,"title":"fix: offload agent resource cleanup to executor thread with timeout to prevent event loop blocking (#53175)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ea5aaa7a22e0d5d036da0130a659d2e4126f7d41","number":0},{"kind":"commit","value":"4403e950506382147349d4d0a2b18beccee9b9f4","number":0},{"kind":"commit","value":"c38dfba3a737acf1a6e366e0686757cb236d7282","number":0},{"kind":"pr","value":"#53175","number":53175},{"kind":"pr","value":"#54130","number":54130}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":54130,"title":"fix(gateway): offload remaining inline agent cleanup off the event loop (#53175)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","infographic/53175-gateway-cleanup-off-loop/infographic.png","tests/gateway/test_13121_shutdown_inflight_transcript_flush.py","tests/gateway/test_53175_cleanup_off_loop.py","tests/gateway/test_shutdown_cache_cleanup.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -5162,7 +5162,7 @@ async def _notify_active_sessions_of_shutdown(self) -> None:\n                     e,\n                 )\n \n-    def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:\n+    async def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:\n         for agent in active_agents.values():\n             # Persist any in-flight transcript to the SQLite session store\n             # before teardown (#13121).  An agent forcibly interrupted by the","@@ -5208,7 +5208,11 @@ def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:\n                 )\n             except Exception:\n                 pass\n-            self._cleanup_agent_resources(agent)\n+            # Off-loop + bounded: a wedged memory provider here used to hang\n+            # the whole shutdown so SIGTERM never completed (#53175).\n+            await self._cleanup_agent_resources_off_loop(\n+                agent, context=\"shutdown finalize\"\n+            )\n \n     def _should_emit_long_running_notification(\n         self,","@@ -5231,6 +5235,52 @@ def _should_emit_long_running_notification(\n             return False\n         return True\n \n+    # Upper bound on off-loop agent-resource cleanup invoked from coroutines\n+    # running on the gateway's event loop (session-expiry sweep, in-turn\n+    # cache-hygiene re-eviction). _cleanup_agent_resources is synchronous and\n+    # can block for a long time (agent.close() does subprocess teardown;\n+    # shutdown_memory_provider() may do network/SQLite IO via a memory plugin).\n+    # Calling it inline wedges the whole loop — the bot goes silent, the\n+    # runtime-status updated_at heartbeat freezes, and SIGTERM cannot be\n+    # serviced (#53175). Offload to a worker thread under this timeout so the\n+    # loop is never blocked; mirrors the /new reset path's fix (#35994).\n+    _CLEANUP_TIMEOUT_S = 30.0\n+\n+    async def _cleanup_agent_resources_off_loop(\n+        self, agent: Any, *, context: str = \"\"\n+    ) -> None:\n+        \"\"\"Run _cleanup_agent_resources in a worker thread with a bounded wait.\n+\n+        Safe to await from coroutines on the gateway event loop: a slow or\n+        wedged teardown (memory provider IO, subprocess close) can no longer\n+        block message processing. On timeout the await is cancelled and the\n+        worker thread is left to finish (or leak) on its own — the caller\n+        proceeds regardless, exactly as the /new reset path does (#35994).\n+        \"\"\"\n+        if agent is None:\n+            return\n+        try:\n+            await asyncio.wait_for(\n+                self._run_in_executor_with_context(\n+                    self._cleanup_agent_resources, agent\n+                ),\n+                timeout=self._CLEANUP_TIMEOUT_S,\n+            )\n+        except asyncio.TimeoutError:\n+            logger.warning(\n+                \"Agent resource cleanup%s exceeded %ss; proceeding without \"\n+                \"blocking the event loop (the worker thread is left to finish \"\n+                \"on its own). (#53175)\",\n+                f\" ({context})\" if context else \"\",\n+                self._CLEANUP_TIMEOUT_S,\n+            )\n+        except Exception as cleanup_exc:\n+            logger.warning(\n+                \"Agent resource cleanup%s failed: %s (#53175)\",\n+                f\" ({context})\" if context else \"\",\n+                cleanup_exc,\n+            )\n+\n     def _cleanup_agent_resources(self, agent: Any) -> None:\n         \"\"\"Best-effort cleanup for temporary or cached agent instances.\"\"\"\n         if agent is None:","@@ -6743,7 +6793,9 @@ async def _session_expiry_watcher(self, interval: int = 300):\n                         if _cached_agent is None:\n                             _cached_agent = self._running_agents.get(key)\n                         if _cached_agent and _cached_agent is not _AGENT_PENDING_SENTINEL:\n-                            self._cleanup_agent_resources(_cached_agent)\n+                            await self._cleanup_agent_resources_off_loop(\n+                                _cached_agent, context=\"session expiry\"\n+                            )\n                         # Drop the cache entry so the AIAgent (and its LLM\n                         # clients, tool schemas, memory provider refs) can\n                         # be garbage-collected.  Otherwise the cache grows","@@ -7250,7 +7302,7 @@ def _phase_elapsed() -> float:\n                 except Exception as e:\n                     logger.error(\"Failed to launch detached gateway restart: %s\", e)\n \n-            self._finalize_shutdown_agents(active_agents)\n+            await self._finalize_shutdown_agents(active_agents)\n \n             # Also shut down memory providers on idle cached agents.\n             # _finalize_shutdown_agents only handles agents that were","@@ -7267,7 +7319,12 @@ def _phase_elapsed() -> float:\n                     _agent = (\n                         _entry[0] if isinstance(_entry, tuple) else _entry\n                     )\n-                    self._cleanup_agent_resources(_agent)\n+                    # Bounded + off-loop so a wedged memory provider on one\n+                    # idle agent can't hang shutdown indefinitely — that path\n+                    # is why SIGTERM failed to kill the process (#53175).\n+                    await self._cleanup_agent_resources_off_loop(\n+                        _agent, context=\"shutdown idle-cache\"\n+                    )\n \n             for platform, adapter in list(self.adapters.items()):\n                 await self._bounded_adapter_teardown(adapter, platform)","@@ -10089,7 +10146,9 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g\n                                     # rebuilds its system prompt from current\n                                     # SOUL.md, memory, and skills.\n                                     self._evict_cached_agent(session_key)\n-                                    self._cleanup_agent_resources(_hyg_agent)\n+                                    await self._cleanup_agent_resources_off_loop(\n+                                        _hyg_agent, context=\"session hygiene\"\n+                                    )\n \n                     except Exception as e:\n                         logger.warning("]},{"file":"tests/gateway/test_13121_shutdown_inflight_transcript_flush.py","hunks":["@@ -32,6 +32,7 @@\n \n from __future__ import annotations\n \n+import asyncio\n import sys\n import types\n from unittest.mock import MagicMock","@@ -51,9 +52,25 @@ def _make_runner():\n     from gateway.run import GatewayRunner\n \n     runner = object.__new__(GatewayRunner)\n+    # _finalize_shutdown_agents offloads _cleanup_agent_resources to a worker\n+    # thread via _run_in_executor_with_context (#53175). Stub it to run inline\n+    # so the bounded-cleanup path is exercised deterministically in tests.\n+    async def _inline_executor(func, *args):\n+        return func(*args)\n+\n+    runner._run_in_executor_with_context = _inline_executor\n     return runner\n \n \n+def _finalize(runner, agents):\n+    \"\"\"Drive the now-async _finalize_shutdown_agents from sync test bodies.\"\"\"\n+    if not hasattr(runner, \"_run_in_executor_with_context\"):\n+        async def _inline_executor(func, *args):\n+            return func(*args)\n+        runner._run_in_executor_with_context = _inline_executor\n+    asyncio.run(runner._finalize_shutdown_agents(agents))\n+\n+\n # ─────────────────────────────────────────────────────────────────────────\n # Unit: _finalize_shutdown_agents calls the flush hook with the in-flight\n # transcript before teardown.","@@ -84,7 +101,7 @@ def test_inflight_messages_flushed_before_teardown(self):\n         ]\n         agent = _FakeAgent(session_messages=inflight)\n \n-        runner._finalize_shutdown_agents({\"agent:main:discord:dm:42\": agent})\n+        _finalize(runner, {\"agent:main:discord:dm:42\": agent})\n \n         agent._flush_messages_to_session_db.assert_called_once_with(inflight)\n         # Cleanup still happens after the flush.","@@ -96,7 +113,7 @@ def test_empty_session_messages_not_flushed(self):\n         runner = _make_runner()\n         agent = _FakeAgent(session_messages=[])\n \n-        runner._finalize_shutdown_agents({\"k\": agent})\n+        _finalize(runner, {\"k\": agent})\n \n         agent._flush_messages_to_session_db.assert_not_called()\n         agent.close.assert_called_once()","@@ -108,7 +125,7 @@ def test_missing_flush_method_is_tolerated(self):\n         agent = _FakeAgent(session_messages=[{\"role\": \"user\", \"content\": \"x\"}],\n                            has_flush=False)\n \n-        runner._finalize_shutdown_agents({\"k\": agent})\n+        _finalize(runner, {\"k\": agent})\n \n         agent.close.assert_called_once()\n ","@@ -119,7 +136,7 @@ def test_flush_exception_is_swallowed(self):\n         agent = _FakeAgent(session_messages=[{\"role\": \"user\", \"content\": \"x\"}])\n         agent._flush_messages_to_session_db.side_effect = RuntimeError(\"db locked\")\n \n-        runner._finalize_shutdown_agents({\"k\": agent})\n+        _finalize(runner, {\"k\": agent})\n \n         agent.close.assert_called_once()\n ","@@ -186,7 +203,7 @@ def test_interrupted_turn_persisted_and_readable_on_resume(self, tmp_path, monke\n         # Drive the gateway shutdown finalization with this real agent.\n         from gateway.run import GatewayRunner\n         runner = object.__new__(GatewayRunner)\n-        runner._finalize_shutdown_agents({\"agent:main:discord:dm:7\": agent})\n+        _finalize(runner, {\"agent:main:discord:dm:7\": agent})\n \n         # The in-flight turn must now be durable and readable via the SAME\n         # path the resume logic uses (SessionStore.load_transcript → DB).","@@ -237,7 +254,7 @@ def test_graceful_agent_reflush_is_idempotent(self, tmp_path, monkeypatch):\n         # Shutdown re-flush of the SAME list identity must add nothing.\n         from gateway.run import GatewayRunner\n         runner = object.__new__(GatewayRunner)\n-        runner._finalize_shutdown_agents({\"k\": agent})\n+        _finalize(runner, {\"k\": agent})\n \n         after = db.get_messages_as_conversation(session_id)\n         assert len(after) == 2, after"]},{"file":"tests/gateway/test_53175_cleanup_off_loop.py","hunks":["@@ -0,0 +1,174 @@\n+\"\"\"Regression test for #53175: gateway event loop wedged by synchronous\n+agent-resource cleanup run inline from loop coroutines.\n+\n+#35994 fixed the /new reset path, but the same synchronous\n+``_cleanup_agent_resources`` (agent.close() tears down terminal sandboxes /\n+browser daemons / background processes; shutdown_memory_provider() may do\n+SQLite / network IO via a memory plugin) was still called INLINE on the event\n+loop from three other places:\n+\n+  * ``_session_expiry_watcher`` (the 5-minute idle sweep) — live loop\n+  * ``_handle_message_with_agent`` cache-hygiene re-eviction — live loop\n+  * ``_finalize_shutdown_agents`` / ``stop()`` idle-cache loop — shutdown\n+\n+A wedged provider on any of these froze the whole loop: the bot went silent,\n+the runtime-status ``updated_at`` heartbeat stopped advancing (the symptom the\n+reporter's watchdog keyed on), and SIGTERM could not be serviced (requiring\n+``kill -9``).\n+\n+The fix routes all four call sites through ``_cleanup_agent_resources_off_loop``\n+which offloads to a worker thread under a bounded ``asyncio.wait_for``, so the\n+loop is never blocked and a stuck teardown degrades gracefully.\n+\n+These tests drive that shared helper directly — it is the single chokepoint\n+every fixed call site now uses.\n+\"\"\"\n+import asyncio\n+import logging\n+import threading\n+from contextvars import copy_context\n+from types import SimpleNamespace\n+\n+import pytest\n+\n+\n+def _make_runner():\n+    \"\"\"Bare GatewayRunner with a real thread-pool-backed executor helper.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    runner = object.__new__(GatewayRunner)\n+    from concurrent.futures import ThreadPoolExecutor\n+\n+    executor = ThreadPoolExecutor(max_workers=2)\n+    runner._get_executor = lambda: executor\n+\n+    async def _run_in_executor_with_context(func, *args):\n+        loop = asyncio.get_running_loop()\n+        ctx = copy_context()\n+        return await loop.run_in_executor(executor, lambda: ctx.run(func, *args))\n+\n+    runner._run_in_executor_with_context = _run_in_executor_with_context\n+    return runner, executor\n+\n+\n+def _agent_with_close(close_fn):\n+    return SimpleNamespace(\n+        close=close_fn,\n+        shutdown_memory_provider=lambda *a, **k: None,\n+        _session_messages=None,\n+    )\n+\n+\n[redacted:email]\n+async def test_cleanup_off_loop_does_not_block_event_loop():\n+    \"\"\"A slow agent.close() must NOT freeze the loop. A concurrent heartbeat\n+    keeps ticking WHILE close() blocks in its worker thread — proving the\n+    cleanup was offloaded, not run inline (which would freeze the loop and\n+    stall the runtime-status updated_at heartbeat, #53175).\"\"\"\n+    runner, executor = _make_runner()\n+    close_started = threading.Event()\n+    release = threading.Event()\n+\n+    def slow_close():\n+        close_started.set()\n+        release.wait(timeout=5)  # block the WORKER thread, not the loop\n+\n+    agent = _agent_with_close(slow_close)\n+\n+    ticks = {\"n\": 0}\n+    stop = threading.Event()\n+\n+    async def _heartbeat():\n+        while not stop.is_set():\n+            ticks[\"n\"] += 1\n+            await asyncio.sleep(0.005)\n+\n+    hb = asyncio.create_task(_heartbeat())\n+    cleanup_task = asyncio.create_task(\n+        runner._cleanup_agent_resources_off_loop(agent, context=\"test\")\n+    )\n+\n+    for _ in range(200):\n+        if close_started.is_set():\n+            break\n+        await asyncio.sleep(0.005)\n+    assert close_started.is_set(), \"close() never ran\"\n+\n+    ticks_at_block = ticks[\"n\"]\n+    await asyncio.sleep(0.1)\n+    ticks_during_block = ticks[\"n\"] - ticks_at_block\n+\n+    release.set()\n+    await cleanup_task\n+    stop.set()\n+    await hb\n+    executor.shutdown(wait=False)\n+\n+    assert ticks_during_block >= 5, (\n+        f\"event loop was blocked during agent cleanup (#53175): only \"\n+        f\"{ticks_during_block} ticks while close() was running\"\n+    )\n+\n+\n[redacted:email]\n+async def test_cleanup_off_loop_times_out_gracefully(caplog):\n+    \"\"\"A cleanup that exceeds the bounded timeout logs a warning and returns —\n+    the caller (sweep / shutdown / hygiene) proceeds rather than hanging.\"\"\"\n+    runner, executor = _make_runner()\n+\n+    async def _instant_timeout(aw, timeout=None):\n+        if asyncio.iscoroutine(aw):\n+            aw.close()\n+        raise asyncio.TimeoutError\n+\n+    import gateway.run as _run\n+\n+    agent = _agent_with_close(lambda: None)\n+    with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+        # Patch the wait_for the helper uses so we don't actually wait 30s.\n+        orig = _run.asyncio.wait_for\n+        _run.asyncio.wait_for = _instant_timeout\n+        try:\n+            await runner._cleanup_agent_resources_off_loop(agent, context=\"sweep\")\n+        finally:\n+            _run.asyncio.wait_for = orig\n+    executor.shutdown(wait=False)\n+\n+    assert any(\n+        \"exceeded\" in r.message and \"#53175\" in r.message for r in caplog.records\n+    ), \"expected the timeout warning to be logged\"\n+\n+\n[redacted:email]\n+async def test_cleanup_off_loop_swallows_executor_failure(caplog):\n+    \"\"\"If the offloaded cleanup raises, the helper logs and returns — a\n+    teardown failure must never abort the loop coroutine that triggered it.\"\"\"\n+    runner, executor = _make_runner()\n+\n+    def boom():\n+        raise RuntimeError(\"provider shutdown blew up\")\n+\n+    # _cleanup_agent_resources swallows its own internal errors, so to reach\n+    # the helper's except branch make the offloaded call itself raise.\n+    def _boom_cleanup(agent):\n+        raise RuntimeError(\"boom\")\n+\n+    runner._cleanup_agent_resources = _boom_cleanup\n+\n+    with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+        await runner._cleanup_agent_resources_off_loop(\n+            _agent_with_close(boom), context=\"shutdown finalize\"\n+        )\n+    executor.shutdown(wait=False)\n+\n+    assert any(\n+        \"failed\" in r.message and \"#53175\" in r.message for r in caplog.records\n+    ), \"expected the cleanup-failure warning to be logged\"\n+\n+\n[redacted:email]\n+async def test_cleanup_off_loop_none_agent_is_noop():\n+    \"\"\"A None agent (None cache entry) is a no-op and never touches the loop.\"\"\"\n+    runner, executor = _make_runner()\n+    await runner._cleanup_agent_resources_off_loop(None)\n+    executor.shutdown(wait=False)"]},{"file":"tests/gateway/test_shutdown_cache_cleanup.py","hunks":["@@ -57,13 +57,23 @@ def _running_agent_count(self):\n     def _update_runtime_status(self, *_a, **_kw):\n         pass\n \n+    async def _run_in_executor_with_context(self, func, *args):\n+        # stop() offloads agent-resource cleanup off the loop (#53175); run\n+        # inline in tests so the bounded-cleanup path is exercised.\n+        return func(*args)\n+\n+    async def _cleanup_agent_resources_off_loop(self, agent, *, context=\"\"):\n+        # Mirror the real bounded helper, inline (no executor/timeout) so the\n+        # fake exercises the same call shape stop() now uses.\n+        self._cleanup_agent_resources(agent)\n+\n     async def _notify_active_sessions_of_shutdown(self):\n         pass\n \n     async def _drain_active_agents(self, timeout):\n         return {}, False\n \n-    def _finalize_shutdown_agents(self, agents):\n+    async def _finalize_shutdown_agents(self, agents):\n         for agent in agents.values():\n             self._cleanup_agent_resources(agent)\n \n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}