{"repo":"NousResearch/hermes-agent","public_id":"27563","display_label":"Process hangs on exit due to stuck worker threads","summary":"Multiple reports and fixes address scenarios where the process hangs indefinitely at shutdown because non-daemon worker threads block Python finalization or graceful shutdown. Proposed mitigations involve force-exit backstops using os._exit to bound exit time.","subsystem":"gateway","counts":{"members":8,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":3,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":54111},"resolution":{"state":"partially_resolved","by":"pr-54111"},"freshness":{"state":"verified","last_verified_at":"2026-07-02T11:20:44+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.22714285714285712},"fix_edges":[{"issue_number":27563,"pr_number":57000,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":53107,"pr_number":54111,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":53107,"pr_number":54122,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":27563,"title":"Python shutdown deadlock: worker thread blocks Py_FinalizeEx, process hangs forever after exit","url":null,"state":"closed"},{"kind":"issue","number":53107,"title":"[Bug]: Gateway hangs after graceful shutdown — sys.exit(1) blocks on stuck non-daemon tool-worker thread (needs os._exit backstop)","url":null,"state":"closed"},{"kind":"pr","number":27785,"title":"fix(cli): prevent Py_FinalizeEx deadlock via os._exit(0) in _run_cleanup (#27563)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3f2a56d1a4aab9511770b81ee595378440376ac0","number":0},{"kind":"pr","value":"#57000","number":57000},{"kind":"pr","value":"#27563","number":27563}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":53121,"title":"fix(gateway): force exit after graceful shutdown","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#54122","number":54122}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":53122,"title":"fix(gateway): force-exit after graceful shutdown so a wedged worker can't hang the process (#53107)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53121","number":53121},{"kind":"pr","value":"#54122","number":54122}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":54111,"title":"fix(gateway): route SystemExit exit paths through the force-exit backstop too (#53107)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_gateway_process_exit.py"]},{"kind":"pr","number":54122,"title":"fix(gateway): force-exit after graceful shutdown so a wedged worker can't hang the process (#53107)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","infographic/gateway-force-exit-53107/infographic.png","tests/gateway/test_gateway_process_exit.py"]},{"kind":"pr","number":57000,"title":"fix(cli): reliable interrupts, bounded exit, and immediate shutdown feedback","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/memory_manager.py","agent/tool_executor.py","cli.py","tests/cli/test_cli_interrupt_ack_race.py","tests/run_agent/test_run_agent.py","tests/tools/test_daemon_pool.py","tools/async_delegation.py","tools/daemon_pool.py","tools/delegate_tool.py","tools/skills_hub.py"],"fix_hunks":[{"file":"agent/memory_manager.py","hunks":["@@ -651,7 +651,12 @@ def _get_sync_executor(self) -> Optional[ThreadPoolExecutor]:\n         with self._sync_executor_lock:\n             if self._sync_executor is None:\n                 try:\n-                    self._sync_executor = ThreadPoolExecutor(\n+                    # Daemon workers (see tools.daemon_pool): a provider wedged\n+                    # on a network call must never block interpreter exit —\n+                    # stdlib ThreadPoolExecutor's atexit hook would join it\n+                    # unconditionally even after shutdown(wait=False).\n+                    from tools.daemon_pool import DaemonThreadPoolExecutor\n+                    self._sync_executor = DaemonThreadPoolExecutor(\n                         max_workers=1,\n                         thread_name_prefix=\"mem-sync\",\n                     )"]},{"file":"agent/tool_executor.py","hunks":["@@ -638,7 +638,13 @@ def _run_tool(index, tool_call, function_name, function_args, middleware_trace):\n         deadline = time.monotonic() + timeout_s if timeout_s is not None else None\n         if runnable_calls:\n             max_workers = min(len(runnable_calls), _MAX_TOOL_WORKERS)\n-            executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers)\n+            # Daemon workers: an interrupted/timed-out batch is abandoned with\n+            # shutdown(wait=False), but stdlib ThreadPoolExecutor workers are\n+            # non-daemon and registered in concurrent.futures' atexit hook,\n+            # which joins them unconditionally — so one wedged tool thread\n+            # would block interpreter exit forever (multi-minute CLI exits).\n+            from tools.daemon_pool import DaemonThreadPoolExecutor\n+            executor = DaemonThreadPoolExecutor(max_workers=max_workers)\n             abandon_executor = False\n             try:\n                 for submit_index, (i, tc, name, args) in enumerate(runnable_calls):"]},{"file":"cli.py","hunks":["@@ -982,6 +982,71 @@ def _prepare_deferred_agent_startup() -> None:\n             exc_info=True,\n         )\n \n+def _arm_exit_watchdog(timeout_s: float | None = None) -> None:\n+    \"\"\"Guarantee the process actually exits once shutdown has begun.\n+\n+    Two hang classes have kept \"dead\" CLI processes alive for minutes:\n+\n+      1. A cleanup step wedged on network I/O (memory provider\n+         ``on_session_end``, MCP teardown, remote terminal cleanup).\n+      2. Interpreter teardown blocked joining non-daemon threads —\n+         stdlib ``ThreadPoolExecutor`` workers are joined unconditionally\n+         by ``concurrent.futures``' atexit hook even after\n+         ``shutdown(wait=False)``, so one tool thread wedged on a socket\n+         held the process open forever (#27563 class).\n+\n+    The shared daemon pool (``tools.daemon_pool``) removes the main cause\n+    of (2); this watchdog is the backstop for both. It arms a daemon\n+    timer when ``_run_cleanup`` starts; if the process is still alive\n+    after ``timeout_s`` it flushes logging/stdio and calls ``os._exit(0)``.\n+    Daemon threads keep running through ``Py_FinalizeEx``'s thread joins,\n+    so the timer fires even when the main thread is stuck in teardown.\n+\n+    Tune with ``HERMES_EXIT_WATCHDOG_S`` (seconds); ``0`` disables.\n+    \"\"\"\n+    if timeout_s is None:\n+        try:\n+            timeout_s = float(os.getenv(\"HERMES_EXIT_WATCHDOG_S\", \"30\"))\n+        except (TypeError, ValueError):\n+            timeout_s = 30.0\n+    if timeout_s <= 0:\n+        return\n+    # Never arm under pytest: tests invoke _run_cleanup() directly and a\n+    # 30s-delayed os._exit(0) would silently kill the test worker.\n+    if os.environ.get(\"PYTEST_CURRENT_TEST\"):\n+        return\n+\n+    def _watchdog():\n+        time.sleep(timeout_s)\n+        # Still alive — cleanup or interpreter teardown is wedged.\n+        try:\n+            logger.warning(\n+                \"Exit watchdog fired after %.0fs — forcing process exit \"\n+                \"(a cleanup step or non-daemon thread is wedged).\",\n+                timeout_s,\n+            )\n+        except Exception:\n+            pass\n+        try:\n+            import logging as _lg\n+            _lg.shutdown()\n+        except Exception:\n+            pass\n+        for _stream in (sys.stdout, sys.stderr):\n+            try:\n+                _stream.flush()\n+            except Exception:\n+                pass\n+        os._exit(0)\n+\n+    try:\n+        threading.Thread(\n+            target=_watchdog, daemon=True, name=\"exit-watchdog\"\n+        ).start()\n+    except Exception:\n+        pass  # best-effort — never block shutdown on watchdog setup\n+\n+\n def _run_cleanup(*, notify_session_finalize: bool = True):\n     \"\"\"Run resource cleanup exactly once.\"\"\"\n     global _cleanup_done","@@ -989,6 +1054,11 @@ def _run_cleanup(*, notify_session_finalize: bool = True):\n         return\n     _cleanup_done = True\n \n+    # Bound total shutdown time: if cleanup (or the interpreter's\n+    # thread-join teardown after it) wedges, force-exit instead of\n+    # leaving a zombie CLI holding the terminal for minutes.\n+    _arm_exit_watchdog()\n+\n     # Reset terminal input modes first, before the slower resource teardown\n     # below (MCP / browser / memory shutdown can take seconds). On Ctrl+C the\n     # user's terminal becomes usable immediately, and a later step raising","@@ -12161,8 +12231,15 @@ def run_agent():\n                         if interrupt_msg:\n                             # If clarify is active, the Enter handler routes\n                             # input directly; this queue shouldn't have anything.\n-                            # But if it does (race condition), don't interrupt.\n+                            # But if it does (race condition), don't interrupt —\n+                            # and don't drop the message either: park it in\n+                            # _pending_input so it runs as the next turn.\n                             if self._clarify_state or self._clarify_freetext:\n+                                try:\n+                                    self._pending_input.put(interrupt_msg)\n+                                except Exception:\n+                                    pass\n+                                interrupt_msg = None\n                                 continue\n                             print(\"\\n⚡ New message detected, interrupting...\")\n                             # Signal TTS to stop on interrupt","@@ -12334,6 +12411,33 @@ def run_agent():\n                 # Add indicator that we were interrupted\n                 if response and pending_message:\n                     response = response + \"\\n\\n---\\n_[Interrupted - processing new message]_\"\n+            elif interrupt_msg:\n+                # We fired agent.interrupt(interrupt_msg) but the turn result\n+                # doesn't acknowledge it. Two ways this happens, both racy:\n+                #   1. The agent thread had already passed its last interrupt\n+                #      check (or finished) when the interrupt landed — the turn\n+                #      completed normally and finalize_turn() never saw the flag.\n+                #   2. The 10s post-interrupt wait above expired and we\n+                #      abandoned the daemon thread; `result` is still None.\n+                # In both cases the user's message must NOT be dropped —\n+                # re-queue it as the next turn (#interrupt-vacuumed-into-void).\n+                pending_message = interrupt_msg\n+                # If the interrupt landed after finalize_turn()'s\n+                # clear_interrupt(), the stale flag would instantly abort the\n+                # NEXT turn at its first loop check. Clear it now that we've\n+                # claimed the message — but ONLY if the agent thread actually\n+                # exited. If it's still alive (abandoned after the 10s wait),\n+                # the flag is what makes the wedged tool eventually unwind;\n+                # clearing it would un-signal that thread.\n+                try:\n+                    if (\n+                        not agent_thread.is_alive()\n+                        and self.agent\n+                        and getattr(self.agent, \"_interrupt_requested\", False)\n+                    ):\n+                        self.agent.clear_interrupt()\n+                except Exception:\n+                    pass\n \n             response_previewed = result.get(\"response_previewed\", False) if result else False\n ","@@ -15240,6 +15344,15 @@ def new_event_loop(self):\n         finally:\n             self._should_exit = True\n             self._pet_stop_anim()\n+            # Immediate feedback: prompt_toolkit has just torn down the input\n+            # box + status bar, so without a line here the terminal sits\n+            # silent for the whole cleanup window (session flush, memory\n+            # shutdown, MCP/browser/terminal teardown) and the exit looks\n+            # hung. Print before any potentially-slow step.\n+            try:\n+                print(f\"{_DIM}Shutting down… (finalizing session){_RST}\", flush=True)\n+            except Exception:\n+                pass\n             # Interrupt the agent immediately so its daemon thread stops making\n             # API calls and exits promptly (agent_thread is daemon, so the\n             # process will exit once the main thread finishes, but interrupting"]},{"file":"tests/cli/test_cli_interrupt_ack_race.py","hunks":["@@ -0,0 +1,194 @@\n+\"\"\"Regression tests for the CLI interrupt-acknowledgement race.\n+\n+Symptom (user report, July 2026): interrupting an active turn is\n+unreliable — the interrupt message is sometimes \"vacuumed into the void\".\n+\n+Root cause: ``HermesCLI.chat()`` fires ``agent.interrupt(msg)`` from its\n+monitor loop, but only re-queued the message when the turn RESULT carried\n+``interrupted=True``. Two races defeat that:\n+\n+  1. The agent thread passes its last ``_interrupt_requested`` check (or\n+     finishes entirely) just before the interrupt lands — the turn\n+     completes \"normally\", ``finalize_turn()`` never acknowledges the\n+     interrupt, and the user's message was silently dropped.\n+  2. Worse, when the interrupt lands *after* ``finalize_turn()``'s\n+     ``clear_interrupt()``, the stale ``_interrupt_requested`` flag\n+     survives on the agent and instantly aborts the NEXT turn at its\n+     first loop check.\n+\n+The fix: when ``chat()`` consumed an ``interrupt_msg`` but the result\n+doesn't acknowledge the interrupt, re-queue the message as the next turn\n+and clear the stale agent flag (only when the agent thread has exited).\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import importlib\n+import queue\n+import sys\n+import time\n+from unittest.mock import MagicMock, patch\n+\n+\n+def _make_cli():\n+    \"\"\"Build a HermesCLI with prompt_toolkit stubbed (same pattern as\n+    test_cli_interrupt_drain_regression.py).\"\"\"\n+    _clean_config = {\n+        \"model\": {\n+            \"default\": \"anthropic/claude-opus-4.6\",\n+            \"base_url\": \"https://openrouter.ai/api/v1\",\n+            \"provider\": \"auto\",\n+        },\n+        \"display\": {\"compact\": False, \"tool_progress\": \"all\"},\n+        \"agent\": {},\n+        \"terminal\": {\"env_type\": \"local\"},\n+    }\n+    clean_env = {\"LLM_MODEL\": \"\", \"HERMES_MAX_ITERATIONS\": \"\"}\n+    prompt_toolkit_stubs = {\n+        \"prompt_toolkit\": MagicMock(),\n+        \"prompt_toolkit.history\": MagicMock(),\n+        \"prompt_toolkit.styles\": MagicMock(),\n+        \"prompt_toolkit.patch_stdout\": MagicMock(),\n+        \"prompt_toolkit.application\": MagicMock(),\n+        \"prompt_toolkit.layout\": MagicMock(),\n+        \"prompt_toolkit.layout.processors\": MagicMock(),\n+        \"prompt_toolkit.filters\": MagicMock(),\n+        \"prompt_toolkit.layout.dimension\": MagicMock(),\n+        \"prompt_toolkit.layout.menus\": MagicMock(),\n+        \"prompt_toolkit.widgets\": MagicMock(),\n+        \"prompt_toolkit.key_binding\": MagicMock(),\n+        \"prompt_toolkit.completion\": MagicMock(),\n+        \"prompt_toolkit.formatted_text\": MagicMock(),\n+        \"prompt_toolkit.auto_suggest\": MagicMock(),\n+    }\n+    with patch.dict(sys.modules, prompt_toolkit_stubs), patch.dict(\n+        \"os.environ\", clean_env, clear=False\n+    ):\n+        import cli as _cli_mod\n+\n+        _cli_mod = importlib.reload(_cli_mod)\n+        with patch.object(_cli_mod, \"get_tool_definitions\", return_value=[]), patch.dict(\n+            _cli_mod.__dict__, {\"CLI_CONFIG\": _clean_config}\n+        ):\n+            return _cli_mod.HermesCLI()\n+\n+\n+class _StubAgent:\n+    \"\"\"Agent whose turn completes WITHOUT acknowledging the interrupt.\"\"\"\n+\n+    def __init__(self, session_id, turn_seconds=0.5):\n+        self.session_id = session_id\n+        self.turn_seconds = turn_seconds\n+        self._interrupt_requested = False\n+        self._interrupt_message = None\n+        self._active_children = []\n+        self.interrupt_calls = []\n+        self.clear_calls = 0\n+        self.max_iterations = 90\n+        self.model = \"test/model\"\n+        self.platform = \"cli\"\n+\n+    def run_conversation(self, **kwargs):\n+        # Simulate a turn that finishes normally — it never observed the\n+        # interrupt flag (raced past its last check).\n+        time.sleep(self.turn_seconds)\n+        return {\n+            \"final_response\": \"turn finished normally\",\n+            \"messages\": [\n+                {\"role\": \"user\", \"content\": \"original\"},\n+                {\"role\": \"assistant\", \"content\": \"turn finished normally\"},\n+            ],\n+            \"api_calls\": 1,\n+            \"completed\": True,\n+            # NOTE: no \"interrupted\" key — the race means finalize_turn\n+            # never saw the flag (or cleared it before it was re-set).\n+            \"partial\": True,  # skip auto-title thread in the test\n+            # Skip the Rich Panel rendering path (crashes under the\n+            # prompt_toolkit/skin mocks; irrelevant to this regression).\n+            \"response_previewed\": True,\n+        }\n+\n+    def interrupt(self, message=None):\n+        self.interrupt_calls.append(message)\n+        self._interrupt_requested = True\n+        self._interrupt_message = message\n+\n+    def clear_interrupt(self):\n+        self.clear_calls += 1\n+        self._interrupt_requested = False\n+        self._interrupt_message = None\n+\n+\n+def test_unacknowledged_interrupt_message_is_requeued_not_dropped():\n+    cli = _make_cli()\n+    agent = _StubAgent(cli.session_id)\n+    cli.agent = agent\n+\n+    cli._interrupt_queue = queue.Queue()\n+    cli._pending_input = queue.Queue()\n+    cli._interrupt_queue.put(\"urgent new message\")\n+\n+    with patch.object(cli, \"_ensure_runtime_credentials\", return_value=True), \\\n+         patch.object(cli, \"_resolve_turn_agent_config\", return_value={\n+             \"signature\": cli._active_agent_route_signature,\n+             \"model\": None, \"runtime\": None, \"request_overrides\": None,\n+         }), \\\n+         patch.object(cli, \"_init_agent\", return_value=True):\n+        cli.chat(\"original\")\n+\n+    # The interrupt fired against the agent...\n+    assert agent.interrupt_calls == [\"urgent new message\"]\n+    # ...the turn result never acknowledged it, so the message must be\n+    # re-queued as the next turn instead of dropped.\n+    queued = []\n+    while not cli._pending_input.empty():\n+        queued.append(cli._pending_input.get_nowait())\n+    assert any(\"urgent new message\" in str(q) for q in queued), (\n+        f\"interrupt message was dropped; pending_input={queued!r}\"\n+    )\n+    # ...and the stale flag must be cleared so the NEXT turn doesn't\n+    # instantly self-abort at its first _interrupt_requested check.\n+    assert agent._interrupt_requested is False\n+    assert agent.clear_calls >= 1\n+\n+\n+def test_acknowledged_interrupt_still_requeues_message():\n+    \"\"\"The pre-existing path (result carries interrupted=True) still works.\"\"\"\n+    cli = _make_cli()\n+\n+    class _AckAgent(_StubAgent):\n+        def run_conversation(self, **kwargs):\n+            # Wait until the monitor loop delivers the interrupt.\n+            for _ in range(100):\n+                if self._interrupt_requested:\n+                    break\n+                time.sleep(0.05)\n+            return {\n+                \"final_response\": \"partial work\",\n+                \"messages\": [{\"role\": \"assistant\", \"content\": \"partial work\"}],\n+                \"api_calls\": 1,\n+                \"completed\": False,\n+                \"interrupted\": True,\n+                \"interrupt_message\": self._interrupt_message,\n+                \"partial\": True,\n+            }\n+\n+    agent = _AckAgent(cli.session_id)\n+    cli.agent = agent\n+    cli._interrupt_queue = queue.Queue()\n+    cli._pending_input = queue.Queue()\n+    cli._interrupt_queue.put(\"redirect please\")\n+\n+    with patch.object(cli, \"_ensure_runtime_credentials\", return_value=True), \\\n+         patch.object(cli, \"_resolve_turn_agent_config\", return_value={\n+             \"signature\": cli._active_agent_route_signature,\n+             \"model\": None, \"runtime\": None, \"request_overrides\": None,\n+         }), \\\n+         patch.object(cli, \"_init_agent\", return_value=True):\n+        cli.chat(\"original\")\n+\n+    queued = []\n+    while not cli._pending_input.empty():\n+        queued.append(cli._pending_input.get_nowait())\n+    assert any(\"redirect please\" in str(q) for q in queued)\n+    assert cli._last_turn_interrupted is True"]},{"file":"tests/run_agent/test_run_agent.py","hunks":["@@ -2731,7 +2731,7 @@ def shutdown(self, *args, **kwargs):\n         mock_msg = _mock_assistant_msg(content=\"\", tool_calls=[tc1, tc2])\n         messages = []\n \n-        with patch(\"agent.tool_executor.concurrent.futures.ThreadPoolExecutor\", ShutdownExecutor):\n+        with patch(\"tools.daemon_pool.DaemonThreadPoolExecutor\", ShutdownExecutor):\n             agent._execute_tool_calls_concurrent(mock_msg, messages, \"task-1\")\n \n         assert len(messages) == 2"]},{"file":"tests/tools/test_daemon_pool.py","hunks":["@@ -0,0 +1,89 @@\n+\"\"\"Tests for tools.daemon_pool.DaemonThreadPoolExecutor.\n+\n+The daemon pool exists so abandoned workers (interrupted/timed-out tool\n+batches, wedged memory-provider syncs) can never block interpreter exit:\n+stdlib ThreadPoolExecutor workers are non-daemon AND registered in\n+concurrent.futures.thread._threads_queues, whose atexit hook joins every\n+worker unconditionally — even after shutdown(wait=False).\n+\"\"\"\n+\n+import subprocess\n+import sys\n+import threading\n+import time\n+\n+from concurrent.futures.thread import _threads_queues\n+\n+from tools.daemon_pool import DaemonThreadPoolExecutor\n+\n+\n+def test_workers_are_daemon_threads():\n+    pool = DaemonThreadPoolExecutor(max_workers=2)\n+    try:\n+        info = pool.submit(\n+            lambda: (threading.current_thread().daemon, threading.current_thread())\n+        ).result(timeout=10)\n+        is_daemon, worker = info\n+        assert is_daemon is True\n+        # Not registered with concurrent.futures' atexit join hook.\n+        assert worker not in _threads_queues\n+    finally:\n+        pool.shutdown(wait=True)\n+\n+\n+def test_results_and_initializer_work_like_stdlib():\n+    seen = []\n+\n+    def _init(tag):\n+        seen.append(tag)\n+\n+    pool = DaemonThreadPoolExecutor(max_workers=1, initializer=_init, initargs=(\"t\",))\n+    try:\n+        assert pool.submit(lambda: 41 + 1).result(timeout=10) == 42\n+        assert seen == [\"t\"]\n+    finally:\n+        pool.shutdown(wait=True)\n+\n+\n+def test_idle_worker_reuse():\n+    pool = DaemonThreadPoolExecutor(max_workers=4)\n+    try:\n+        tid1 = pool.submit(threading.get_ident).result(timeout=10)\n+        time.sleep(0.05)  # let the worker park on the idle semaphore\n+        tid2 = pool.submit(threading.get_ident).result(timeout=10)\n+        assert tid1 == tid2\n+    finally:\n+        pool.shutdown(wait=True)\n+\n+\n+def test_wedged_worker_does_not_block_interpreter_exit():\n+    \"\"\"A worker stuck in a long sleep must not hold the process open.\n+\n+    With stdlib ThreadPoolExecutor this subprocess hangs until the sleep\n+    finishes (the atexit hook joins the worker); with the daemon pool it\n+    exits as soon as the main thread returns.\n+    \"\"\"\n+    script = (\n+        \"import sys; sys.path.insert(0, %r)\\n\"\n+        \"from tools.daemon_pool import DaemonThreadPoolExecutor\\n\"\n+        \"import time\\n\"\n+        \"pool = DaemonThreadPoolExecutor(max_workers=1)\\n\"\n+        \"pool.submit(time.sleep, 120)\\n\"\n+        \"time.sleep(0.3)\\n\"\n+        \"pool.shutdown(wait=False)\\n\"\n+        \"print('main-done', flush=True)\\n\"\n+    ) % (str(_repo_root()),)\n+    proc = subprocess.run(\n+        [sys.executable, \"-c\", script],\n+        capture_output=True,\n+        text=True,\n+        timeout=30,\n+    )\n+    assert proc.returncode == 0\n+    assert \"main-done\" in proc.stdout\n+\n+\n+def _repo_root():\n+    import pathlib\n+\n+    return pathlib.Path(__file__).resolve().parents[2]"]},{"file":"tools/async_delegation.py","hunks":["@@ -40,48 +40,18 @@\n import threading\n import time\n import uuid\n-import weakref\n from concurrent.futures import ThreadPoolExecutor\n-from concurrent.futures.thread import _worker\n from typing import Any, Callable, Dict, List, Optional\n \n+from tools.daemon_pool import DaemonThreadPoolExecutor\n from tools.thread_context import propagate_context_to_thread\n \n logger = logging.getLogger(__name__)\n \n-\n-class _DaemonThreadPoolExecutor(ThreadPoolExecutor):\n-    \"\"\"ThreadPoolExecutor variant whose workers do not block process exit.\n-\n-    Stdlib ``ThreadPoolExecutor`` workers are non-daemon. Background\n-    delegation is explicitly best-effort detached work, so a long child should\n-    be interruptible by ``/stop``/shutdown but must not keep a CLI process alive\n-    after the user exits.\n-    \"\"\"\n-\n-    def _adjust_thread_count(self) -> None:\n-        if self._idle_semaphore.acquire(timeout=0):\n-            return\n-\n-        def weakref_cb(_, q=self._work_queue):\n-            q.put(None)\n-\n-        num_threads = len(self._threads)\n-        if num_threads < self._max_workers:\n-            thread_name = \"%s_%d\" % (self._thread_name_prefix or self, num_threads)\n-            t = threading.Thread(\n-                name=thread_name,\n-                target=_worker,\n-                args=(\n-                    weakref.ref(self, weakref_cb),\n-                    self._work_queue,\n-                    self._initializer,\n-                    self._initargs,\n-                ),\n-                daemon=True,\n-            )\n-            t.start()\n-            self._threads.add(t)\n+# Back-compat alias — the daemon executor now lives in tools.daemon_pool so\n+# other subsystems (tool_executor, memory_manager, delegate_tool, skills_hub)\n+# can share it. Existing imports of ``_DaemonThreadPoolExecutor`` keep working.\n+_DaemonThreadPoolExecutor = DaemonThreadPoolExecutor\n \n \n # ---------------------------------------------------------------------------"]},{"file":"tools/daemon_pool.py","hunks":["@@ -0,0 +1,64 @@\n+\"\"\"Shared daemon-thread ThreadPoolExecutor.\n+\n+Stdlib ``ThreadPoolExecutor`` workers are non-daemon AND are registered in\n+``concurrent.futures.thread._threads_queues``, whose atexit hook\n+(``_python_exit``) joins every worker unconditionally — even after\n+``shutdown(wait=False)``.  A single wedged worker (tool blocked on network\n+I/O, hung provider daemon, stuck subagent) therefore blocks interpreter\n+exit forever.  This is the root cause of multi-minute CLI exits on long\n+sessions: every abandoned concurrent-tool batch leaves workers that the\n+exit hook insists on joining.\n+\n+``DaemonThreadPoolExecutor`` spawns daemon workers and skips the\n+``_threads_queues`` registration, so:\n+\n+  - ``_python_exit`` never joins them, and\n+  - the interpreter's non-daemon thread join at shutdown skips them.\n+\n+Semantics are otherwise identical (initializer/initargs, work queue,\n+idle-thread reuse).  Use it for any pool whose work is best-effort or\n+independently interruptible and must never hold the process open:\n+concurrent tool execution, background memory sync, catalog fan-out,\n+subagent timeout wrappers.  Do NOT use it for work that must complete\n+before exit (durable writes) — those belong on foreground threads with\n+explicit bounded joins.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import threading\n+import weakref\n+from concurrent.futures import ThreadPoolExecutor\n+from concurrent.futures.thread import _worker\n+\n+__all__ = [\"DaemonThreadPoolExecutor\"]\n+\n+\n+class DaemonThreadPoolExecutor(ThreadPoolExecutor):\n+    \"\"\"ThreadPoolExecutor variant whose workers do not block process exit.\"\"\"\n+\n+    def _adjust_thread_count(self) -> None:\n+        # Mirrors CPython's implementation (3.8–3.13) with two changes:\n+        # daemon=True and no _threads_queues registration.\n+        if self._idle_semaphore.acquire(timeout=0):\n+            return\n+\n+        def weakref_cb(_, q=self._work_queue):\n+            q.put(None)\n+\n+        num_threads = len(self._threads)\n+        if num_threads < self._max_workers:\n+            thread_name = \"%s_%d\" % (self._thread_name_prefix or self, num_threads)\n+            t = threading.Thread(\n+                name=thread_name,\n+                target=_worker,\n+                args=(\n+                    weakref.ref(self, weakref_cb),\n+                    self._work_queue,\n+                    self._initializer,\n+                    self._initargs,\n+                ),\n+                daemon=True,\n+            )\n+            t.start()\n+            self._threads.add(t)"]},{"file":"tools/delegate_tool.py","hunks":["@@ -1887,7 +1887,11 @@ def _heartbeat_loop():\n         # result(timeout=None) blocks until the child finishes). Stuck-child\n         # protection comes from the heartbeat staleness monitor instead.\n         child_timeout = _get_child_timeout()\n-        _timeout_executor = ThreadPoolExecutor(\n+        # Daemon worker (tools.daemon_pool): a timed-out child is abandoned\n+        # below; a stdlib non-daemon worker would then block interpreter\n+        # exit at atexit-join time if the child never unwinds.\n+        from tools.daemon_pool import DaemonThreadPoolExecutor\n+        _timeout_executor = DaemonThreadPoolExecutor(\n             max_workers=1,\n             # Install a non-interactive approval callback in the worker thread\n             # so dangerous-command prompts from the subagent don't fall back to","@@ -2535,7 +2539,11 @@ def _execute_and_aggregate() -> dict:\n             completed_count = 0\n             spinner_ref = getattr(parent_agent, \"_delegate_spinner\", None)\n \n-            with ThreadPoolExecutor(max_workers=max_children) as executor:\n+            # Daemon workers (tools.daemon_pool): the `with` block still joins\n+            # normally, but if the parent is interrupted while a child is\n+            # wedged, the abandoned worker must not block interpreter exit.\n+            from tools.daemon_pool import DaemonThreadPoolExecutor\n+            with DaemonThreadPoolExecutor(max_workers=max_children) as executor:\n                 futures = {}\n                 for i, t, child in children:\n                     future = executor.submit("]},{"file":"tools/skills_hub.py","hunks":["@@ -4005,8 +4005,11 @@ def parallel_search_sources(\n     # worker finishes — so a single slow source (e.g. ClawHub) keeps the\n     # caller blocked for minutes and renders ``overall_timeout`` a no-op.\n     # Manage the executor manually and shut it down with ``wait=False`` so\n-    # the timeout is actually honoured.\n-    pool = ThreadPoolExecutor(max_workers=min(len(active), 8))\n+    # the timeout is actually honoured.  Daemon workers (tools.daemon_pool):\n+    # an abandoned slow source must not block interpreter exit either —\n+    # stdlib workers are joined unconditionally by the atexit hook.\n+    from tools.daemon_pool import DaemonThreadPoolExecutor\n+    pool = DaemonThreadPoolExecutor(max_workers=min(len(active), 8))\n     futures = {}\n     for src in active:\n         lim = per_source_limits.get(src.source_id(), 50)\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}