{"repo":"NousResearch/hermes-agent","public_id":"35309","display_label":"Interrupt state leak fixes and recovery handling","summary":"These members address a bug where interrupted thread state leaks when tool execution raises unhandled exceptions, plus a related fix for model recovery behavior during post-interrupt turns. The fixes involve wrapping cleanup logic in finally blocks and gating fallback status.","subsystem":"agent","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":2,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":35382,"merged_at":"2026-05-30T14:28:12+00:00"},"resolution":{"state":"resolved","by":"pr-35381"},"freshness":{"state":"verified","last_verified_at":"2026-05-30T14:28:12+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":35309,"pr_number":35382,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-30T14:28:12+00:00"}],"members":[{"kind":"issue","number":35309,"title":"[Bug]: ThreadPoolExecutor leaks _interrupted_threads state when _run_tool raises an unhandled exception","url":null,"state":"closed"},{"kind":"pr","number":35320,"title":"fix(tools): wrap _run_tool cleanup in finally to prevent interrupt state leak (#35309)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#35382","number":35382}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":35381,"title":"fix(gateway): recover model on post-interrupt recovery turn; gate fallback status","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true},{"kind":"pr","number":35382,"title":"fix(tools): wrap _run_tool cleanup in finally to prevent interrupt state leak (#35309)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/tool_executor.py","tests/tools/test_interrupt.py"],"fix_hunks":[{"file":"agent/tool_executor.py","hunks":["@@ -306,33 +306,38 @@ def _run_tool(index, tool_call, function_name, function_args):\n         # submit site below (GHSA-qg5c-hvr5-hjgr, #13617).\n         start = time.time()\n         try:\n-            result = agent._invoke_tool(\n-                function_name,\n-                function_args,\n-                effective_task_id,\n-                tool_call.id,\n-                messages=messages,\n-                pre_tool_block_checked=True,\n-            )\n-        except Exception as tool_error:\n-            result = f\"Error executing tool '{function_name}': {tool_error}\"\n-            logger.error(\"_invoke_tool raised for %s: %s\", function_name, tool_error, exc_info=True)\n-        duration = time.time() - start\n-        is_error, _ = _detect_tool_failure(function_name, result)\n-        if is_error:\n-            logger.info(\"tool %s failed (%.2fs): %s\", function_name, duration, result[:200])\n-        else:\n-            logger.info(\"tool %s completed (%.2fs, %d chars)\", function_name, duration, len(result))\n-        results[index] = (function_name, function_args, result, duration, is_error, False)\n-        # Tear down worker-tid tracking.  Clear any interrupt bit we may\n-        # have set so the next task scheduled onto this recycled tid\n-        # starts with a clean slate.\n-        with agent._tool_worker_threads_lock:\n-            agent._tool_worker_threads.discard(_worker_tid)\n-        try:\n-            _ra()._set_interrupt(False, _worker_tid)\n-        except Exception:\n-            pass\n+            try:\n+                result = agent._invoke_tool(\n+                    function_name,\n+                    function_args,\n+                    effective_task_id,\n+                    tool_call.id,\n+                    messages=messages,\n+                    pre_tool_block_checked=True,\n+                )\n+            except Exception as tool_error:\n+                result = f\"Error executing tool '{function_name}': {tool_error}\"\n+                logger.error(\"_invoke_tool raised for %s: %s\", function_name, tool_error, exc_info=True)\n+            duration = time.time() - start\n+            is_error, _ = _detect_tool_failure(function_name, result)\n+            if is_error:\n+                logger.info(\"tool %s failed (%.2fs): %s\", function_name, duration, result[:200])\n+            else:\n+                logger.info(\"tool %s completed (%.2fs, %d chars)\", function_name, duration, len(result))\n+            results[index] = (function_name, function_args, result, duration, is_error, False)\n+        finally:\n+            # Tear down worker-tid tracking.  Clear any interrupt bit we may\n+            # have set so the next task scheduled onto this recycled tid\n+            # starts with a clean slate.  This MUST be in a finally block\n+            # because BaseException subclasses (CancelledError, KeyboardInterrupt)\n+            # bypass ``except Exception`` and would otherwise leak the tid\n+            # into _interrupted_threads, poisoning the recycled thread.\n+            with agent._tool_worker_threads_lock:\n+                agent._tool_worker_threads.discard(_worker_tid)\n+            try:\n+                _ra()._set_interrupt(False, _worker_tid)\n+            except Exception:\n+                pass\n \n     # Start spinner for CLI mode (skip when TUI handles tool progress)\n     spinner = None"]},{"file":"tests/tools/test_interrupt.py","hunks":["@@ -203,6 +203,83 @@ def _run():\n         assert \"interrupted\" in result_holder[\"value\"][\"output\"].lower()\n \n \n+# ---------------------------------------------------------------------------\n+# Regression: _run_tool cleanup on BaseException (issue #35309)\n+# ---------------------------------------------------------------------------\n+\n+class TestRunToolCleanupOnBaseException:\n+    \"\"\"Verify that _run_tool cleans up _interrupted_threads even when\n+    _invoke_tool raises a BaseException (e.g. CancelledError).\n+\n+    Regression test for #35309: without the finally block, a BaseException\n+    bypasses ``except Exception``, leaking the worker tid into\n+    _interrupted_threads.  ThreadPoolExecutor recycles tids, so the next\n+    tool scheduled on the same thread is instantly \"interrupted\".\n+    \"\"\"\n+\n+    def test_cleanup_on_base_exception(self):\n+        from unittest.mock import MagicMock, patch\n+        import types\n+        from tools.interrupt import set_interrupt, is_interrupted, _interrupted_threads, _lock\n+\n+        # Clear global state\n+        with _lock:\n+            _interrupted_threads.clear()\n+\n+        # Build a minimal mock agent with the attributes _run_tool needs\n+        agent = MagicMock()\n+        agent._interrupt_requested = False\n+        agent._tool_worker_threads = set()\n+        agent._tool_worker_threads_lock = threading.Lock()\n+\n+        # _set_interrupt delegates to the real module\n+        def _mock_set_interrupt(active, tid=None):\n+            set_interrupt(active, tid)\n+        agent._set_interrupt = _mock_set_interrupt\n+\n+        # _invoke_tool raises BaseException (simulating CancelledError)\n+        agent._invoke_tool = MagicMock(side_effect=BaseException(\"simulated CancelledError\"))\n+\n+        # Bind the real concurrent method so we get _run_tool\n+        from run_agent import AIAgent\n+        agent._execute_tool_calls_concurrent = types.MethodType(\n+            AIAgent._execute_tool_calls_concurrent, agent\n+        )\n+\n+        # Build a single tool call\n+        tc = MagicMock()\n+        tc.id = \"tc_base_exc\"\n+        tc.function.name = \"dummy_tool\"\n+        tc.function.arguments = \"{}\"\n+\n+        assistant_msg = MagicMock()\n+        assistant_msg.tool_calls = [tc]\n+\n+        # _execute_tool_calls_concurrent will submit _run_tool to a\n+        # ThreadPoolExecutor.  The BaseException propagates out of the\n+        # worker, but the finally block should still clean up.\n+        try:\n+            agent._execute_tool_calls_concurrent(assistant_msg, [], \"default\")\n+        except Exception:\n+            pass  # ThreadPoolExecutor may re-raise\n+\n+        # After the worker finishes (even with BaseException), the worker\n+        # tid should have been removed from _interrupted_threads and\n+        # _tool_worker_threads.\n+        assert len(agent._tool_worker_threads) == 0, (\n+            f\"_tool_worker_threads not cleaned up: {agent._tool_worker_threads}\"\n+        )\n+\n+        # Verify no stale tid is left in the global interrupt set.  The\n+        # worker thread is recycled by ThreadPoolExecutor, so a leaked tid\n+        # would poison the next task on that thread.  We cleared the set at\n+        # the start and never set any interrupt ourselves, so a leak from\n+        # _run_tool is the only way an entry could land here.\n+        with _lock:\n+            leaked = set(_interrupted_threads)\n+        assert leaked == set(), f\"leaked tids in _interrupted_threads: {leaked}\"\n+\n+\n # ---------------------------------------------------------------------------\n # Manual smoke test checklist (not automated)\n # ---------------------------------------------------------------------------\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}