{"repo":"NousResearch/hermes-agent","public_id":"35994","display_label":"Gateway reset/new command hang and cleanup fixes","summary":"These items relate to hangs, deadlocks, and blocking operations in the gateway's /new reset flow, including event loop offloading, confirmation handler timeouts, and a Telegram state machine deadlock.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":52090,"merged_at":"2026-06-24T19:04:21+00:00"},"resolution":{"state":"resolved","by":"pr-52090"},"freshness":{"state":"verified","last_verified_at":"2026-06-24T19:04:21+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":35994,"pr_number":52090,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-24T19:04:21+00:00"}],"members":[{"kind":"issue","number":35994,"title":"Telegram /new confirm button hangs in `_handle_reset_command()` state machine deadlock","url":null,"state":"closed"},{"kind":"pr","number":36006,"title":"fix(gateway): timeout slash confirmation handlers","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"0225480369f576ddac88152b857098309d60fb69","number":0},{"kind":"pr","value":"#35994","number":35994}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":52090,"title":"fix(gateway): offload agent cleanup off the event loop in /new reset (#35994)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/slash_commands.py","tests/gateway/test_35994_reset_button_deadlock.py"],"fix_hunks":[{"file":"gateway/slash_commands.py","hunks":["@@ -44,6 +44,12 @@\n \n logger = logging.getLogger(\"gateway.run\")\n \n+# Upper bound on the off-loop agent-resource cleanup during a /new or /reset\n+# (see _handle_reset_command). A stuck teardown must not block the event loop;\n+# past this the reset proceeds and the cleanup is left to finish (or leak) in\n+# its worker thread. (#35994)\n+_RESET_CLEANUP_TIMEOUT_S = 30.0\n+\n \n def _model_switch_skew_guard() -> Optional[str]:\n     \"\"\"Refuse a model switch when the gateway is running stale code.","@@ -111,13 +117,44 @@ async def _handle_reset_command(self, event: MessageEvent) -> Union[str, Ephemer\n         # Close tool resources on the old agent (terminal sandboxes, browser\n         # daemons, background processes) before evicting from cache.\n         # Guard with getattr because test fixtures may skip __init__.\n+        #\n+        # _cleanup_agent_resources is synchronous and can block for a long time\n+        # (agent.close() does subprocess teardown; shutdown_memory_provider()\n+        # may do network IO). This handler runs ON the event loop when a\n+        # Telegram/Discord/Slack confirm-button click resolves the slash-confirm\n+        # (see _request_slash_confirm), so an inline call wedges the whole loop\n+        # and the bot goes silent until restart (#35994). Offload it to a worker\n+        # thread (via the contextvar-preserving executor helper) with a bounded\n+        # timeout so the loop is never blocked.\n         _cache_lock = getattr(self, \"_agent_cache_lock\", None)\n         if _cache_lock is not None:\n             with _cache_lock:\n                 _cached = self._agent_cache.get(session_key)\n                 _old_agent = _cached[0] if isinstance(_cached, tuple) else _cached if _cached else None\n             if _old_agent is not None:\n-                self._cleanup_agent_resources(_old_agent)\n+                try:\n+                    await asyncio.wait_for(\n+                        self._run_in_executor_with_context(\n+                            self._cleanup_agent_resources, _old_agent\n+                        ),\n+                        timeout=_RESET_CLEANUP_TIMEOUT_S,\n+                    )\n+                except asyncio.TimeoutError:\n+                    # wait_for cancels the await, but the worker thread cannot be\n+                    # cancelled — a wedged teardown keeps running (or leaks) for\n+                    # the gateway's lifetime. The reset proceeds regardless.\n+                    logger.warning(\n+                        \"Agent resource cleanup for session %s exceeded %ss during \"\n+                        \"/new reset; proceeding with reset (the worker thread is left \"\n+                        \"to finish on its own). (#35994)\",\n+                        session_key, _RESET_CLEANUP_TIMEOUT_S,\n+                    )\n+                except Exception as cleanup_exc:\n+                    logger.warning(\n+                        \"Agent resource cleanup for session %s failed during /new \"\n+                        \"reset: %s (#35994)\",\n+                        session_key, cleanup_exc,\n+                    )\n         self._evict_cached_agent(session_key)\n \n         # Discard any /queue overflow for this session — /new is a"]},{"file":"tests/gateway/test_35994_reset_button_deadlock.py","hunks":["@@ -0,0 +1,200 @@\n+\"\"\"Regression test for #35994: Telegram /new confirm-button deadlock.\n+\n+The /new confirmation button callback runs the slash-confirm handler on the\n+asyncio event loop (see GatewayRunner._request_slash_confirm). That handler\n+calls _handle_reset_command, which used to invoke the SYNCHRONOUS, potentially\n+long-blocking _cleanup_agent_resources (agent.close() tears down terminal\n+sandboxes / browser daemons / background processes; shutdown_memory_provider()\n+may make a network call) inline on the loop. A slow teardown wedged the entire\n+event loop, so the bot went silent until a manual restart.\n+\n+The fix offloads _cleanup_agent_resources to a worker thread with a bounded\n+timeout, so the loop is never blocked and a stuck teardown degrades gracefully.\n+\"\"\"\n+import asyncio\n+import logging\n+import threading\n+import time\n+from datetime import datetime\n+from types import SimpleNamespace\n+from unittest.mock import AsyncMock, MagicMock, patch\n+\n+import pytest\n+\n+from gateway.config import GatewayConfig, Platform, PlatformConfig\n+from gateway.platforms.base import MessageEvent\n+from gateway.session import SessionEntry, SessionSource, build_session_key\n+\n+\n+def _make_source() -> SessionSource:\n+    return SessionSource(\n+        platform=Platform.TELEGRAM,\n+        user_id=\"u1\",\n+        chat_id=\"c1\",\n+        user_name=\"tester\",\n+        chat_type=\"dm\",\n+    )\n+\n+\n+def _make_event(text: str) -> MessageEvent:\n+    return MessageEvent(text=text, source=_make_source(), message_id=\"m1\")\n+\n+\n+def _make_runner_with_cached_agent(close_fn):\n+    \"\"\"Build a bare GatewayRunner with a cached agent whose close() runs\n+    ``close_fn`` (used to simulate slow / blocking teardown).\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    runner = object.__new__(GatewayRunner)\n+    runner.config = GatewayConfig(\n+        platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token=\"***\")}\n+    )\n+    adapter = MagicMock()\n+    adapter.send = AsyncMock()\n+    runner.adapters = {Platform.TELEGRAM: adapter}\n+    runner._voice_mode = {}\n+    runner.hooks = SimpleNamespace(emit=AsyncMock(), loaded_hooks=False)\n+    runner._session_model_overrides = {}\n+    runner._pending_model_notes = {}\n+    runner._background_tasks = set()\n+\n+    session_key = build_session_key(_make_source())\n+    session_entry = SessionEntry(\n+        session_key=session_key, session_id=\"sess-old\",\n+        created_at=datetime.now(), updated_at=datetime.now(),\n+        platform=Platform.TELEGRAM, chat_type=\"dm\",\n+    )\n+    new_entry = SessionEntry(\n+        session_key=session_key, session_id=\"sess-new\",\n+        created_at=datetime.now(), updated_at=datetime.now(),\n+        platform=Platform.TELEGRAM, chat_type=\"dm\",\n+    )\n+    runner.session_store = MagicMock()\n+    runner.session_store.reset_session.return_value = new_entry\n+    runner.session_store._entries = {session_key: session_entry}\n+    runner.session_store._generate_session_key.return_value = session_key\n+    runner._running_agents = {}\n+    runner._pending_messages = {}\n+    runner._pending_approvals = {}\n+    runner._session_db = None\n+    runner._is_user_authorized = lambda _source: True\n+    runner._format_session_info = lambda: \"\"\n+\n+    # Enable the cache-lock path (this is what the button callback exercises)\n+    runner._agent_cache_lock = threading.RLock()\n+    agent = MagicMock()\n+    agent.close = close_fn\n+    agent.shutdown_memory_provider = MagicMock()\n+    runner._agent_cache = {session_key: agent}\n+    return runner\n+\n+\n[redacted:email]\n+async def test_reset_does_not_block_event_loop_during_cleanup():\n+    \"\"\"#35994: a slow agent.close() must NOT block the event loop. A\n+    concurrent loop task must keep ticking WHILE close() is still blocking\n+    (proving cleanup was offloaded to a worker thread, not run inline on\n+    the loop). With the pre-fix inline call, the loop is frozen for the\n+    whole duration of close() and no ticks accumulate until it returns.\"\"\"\n+    close_started = threading.Event()\n+    release = threading.Event()\n+\n+    def slow_close():\n+        close_started.set()\n+        # Block the WORKER thread (not the loop) until released.\n+        release.wait(timeout=5)\n+\n+    runner = _make_runner_with_cached_agent(slow_close)\n+\n+    ticks = {\"n\": 0}\n+    stop = threading.Event()\n+\n+    async def _heartbeat():\n+        while not stop.is_set():\n+            ticks[\"n\"] += 1\n+            await asyncio.sleep(0.005)\n+\n+    hb = asyncio.create_task(_heartbeat())\n+    reset_task = asyncio.create_task(\n+        runner._handle_reset_command(_make_event(\"/new\"))\n+    )\n+\n+    # Wait until close() has actually started blocking in its worker thread.\n+    for _ in range(200):\n+        if close_started.is_set():\n+            break\n+        await asyncio.sleep(0.005)\n+    assert close_started.is_set(), \"close() never ran\"\n+\n+    # Now sample ticks while close() is STILL blocking. If the loop were\n+    # frozen (pre-fix inline call), this stays ~0.\n+    ticks_at_block = ticks[\"n\"]\n+    await asyncio.sleep(0.1)\n+    ticks_during_block = ticks[\"n\"] - ticks_at_block\n+\n+    release.set()\n+    await reset_task\n+    stop.set()\n+    await hb\n+\n+    assert ticks_during_block >= 5, (\n+        f\"event loop was blocked during agent cleanup (#35994): only \"\n+        f\"{ticks_during_block} ticks while close() was running\"\n+    )\n+    runner.session_store.reset_session.assert_called_once()\n+\n+\n[redacted:email]\n+async def test_reset_completes_when_cleanup_raises(caplog):\n+    \"\"\"#35994: if the offloaded cleanup itself raises, the handler swallows it\n+    (logs a warning) and still rotates the session — it must not abort /new.\n+\n+    Note: _cleanup_agent_resources swallows its own internal errors, so to\n+    exercise the handler's `except Exception` branch we make the cleanup call\n+    itself raise (patched on the instance), then assert the warning fired —\n+    proving the branch executed rather than the success path.\n+    \"\"\"\n+    runner = _make_runner_with_cached_agent(lambda: None)\n+\n+    def boom_cleanup(_agent):\n+        raise RuntimeError(\"cleanup blew up\")\n+\n+    runner._cleanup_agent_resources = boom_cleanup\n+\n+    with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+        result = await asyncio.wait_for(\n+            runner._handle_reset_command(_make_event(\"/new\")), timeout=3\n+        )\n+\n+    assert any(\n+        \"failed during /new reset\" in r.message and \"#35994\" in r.message\n+        for r in caplog.records\n+    ), \"expected the cleanup-failure warning to be logged (except branch not hit)\"\n+    runner.session_store.reset_session.assert_called_once()\n+    assert result is not None\n+\n+\n[redacted:email]\n+async def test_reset_completes_when_cleanup_times_out(caplog):\n+    \"\"\"#35994: if cleanup exceeds the bounded timeout, the reset still completes\n+    (graceful degradation) and the timeout warning fires.\"\"\"\n+    import gateway.slash_commands as _sc\n+\n+    # Force the wait_for to time out immediately, closing the offloaded awaitable\n+    # so no worker thread dangles past the test.\n+    async def _instant_timeout(aw, timeout=None):\n+        if asyncio.iscoroutine(aw):\n+            aw.close()\n+        raise asyncio.TimeoutError\n+\n+    runner = _make_runner_with_cached_agent(lambda: None)\n+\n+    with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+        with patch.object(_sc.asyncio, \"wait_for\", _instant_timeout):\n+            result = await runner._handle_reset_command(_make_event(\"/new\"))\n+\n+    assert any(\n+        \"exceeded\" in r.message and \"#35994\" in r.message for r in caplog.records\n+    ), \"expected the timeout warning to be logged\"\n+    runner.session_store.reset_session.assert_called_once()\n+    assert result is not None\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}