{"repo":"NousResearch/hermes-agent","public_id":"24453","display_label":"Move external memory sync off main turn thread","summary":"Multiple fixes and performance changes aimed at running external memory synchronization on a background or daemon thread, so it no longer blocks the main turn thread and delays response completion. Includes a reported reproduction of the blocking behavior.","subsystem":"agent","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":41945},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-08T09:19:00+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":24453,"pr_number":41945,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":24453,"title":"run_conversation blocks 30-50s on _sync_external_memory_for_turn, delaying response.completed","url":null,"state":"closed"},{"kind":"pr","number":24484,"title":"perf(agent): run external memory sync on a daemon thread to unblock response.completed (#24453)","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":24486,"title":"fix(agent): dispatch external memory sync to daemon thread","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"aa6f2775fac7c460a73669f35d4d478fed393004","number":0},{"kind":"pr","value":"#41945","number":41945}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":38988,"title":"perf(agent): run _sync_external_memory_for_turn in background to unblock response (#24453)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":41945,"title":"fix(memory): run end-of-turn sync off the turn thread (agent stuck 'running')","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/memory_manager.py","tests/agent/test_memory_async_sync.py","tests/agent/test_memory_provider.py","tests/agent/test_memory_session_switch.py"],"fix_hunks":[{"file":"agent/memory_manager.py","hunks":["@@ -28,6 +28,8 @@\n import logging\n import re\n import inspect\n+import threading\n+from concurrent.futures import ThreadPoolExecutor\n from typing import Any, Dict, List, Optional\n \n from agent.memory_provider import MemoryProvider","@@ -35,6 +37,12 @@\n \n logger = logging.getLogger(__name__)\n \n+# How long shutdown_all() waits for in-flight background sync/prefetch work\n+# to drain before abandoning it. A wedged provider must never block process\n+# teardown indefinitely — the worker threads are daemon, so anything still\n+# running past this window dies with the interpreter.\n+_SYNC_DRAIN_TIMEOUT_S = 5.0\n+\n \n # ---------------------------------------------------------------------------\n # Context fencing helpers","@@ -252,6 +260,13 @@ def __init__(self) -> None:\n         self._providers: List[MemoryProvider] = []\n         self._tool_to_provider: Dict[str, MemoryProvider] = {}\n         self._has_external: bool = False  # True once a non-builtin provider is added\n+        # Background executor for end-of-turn sync/prefetch. Lazily created on\n+        # first use so the common builtin-only path spawns no extra threads.\n+        # A single worker serializes a provider's writes (turn N must land\n+        # before turn N+1) and caps thread growth at one per manager. See\n+        # _submit_background() and the sync_all/queue_prefetch_all rationale.\n+        self._sync_executor: Optional[ThreadPoolExecutor] = None\n+        self._sync_executor_lock = threading.Lock()\n \n     # -- Registration --------------------------------------------------------\n ","@@ -375,15 +390,27 @@ def prefetch_all(self, query: str, *, session_id: str = \"\") -> str:\n         return \"\\n\\n\".join(parts)\n \n     def queue_prefetch_all(self, query: str, *, session_id: str = \"\") -> None:\n-        \"\"\"Queue background prefetch on all providers for the next turn.\"\"\"\n-        for provider in self._providers:\n-            try:\n-                provider.queue_prefetch(query, session_id=session_id)\n-            except Exception as e:\n-                logger.debug(\n-                    \"Memory provider '%s' queue_prefetch failed (non-fatal): %s\",\n-                    provider.name, e,\n-                )\n+        \"\"\"Queue background prefetch on all providers for the next turn.\n+\n+        Provider work is dispatched to a background worker so a slow or\n+        wedged provider can never block the caller. See ``sync_all`` for\n+        the full rationale (agent stuck \"running\" minutes after a turn).\n+        \"\"\"\n+        providers = list(self._providers)\n+        if not providers:\n+            return\n+\n+        def _run() -> None:\n+            for provider in providers:\n+                try:\n+                    provider.queue_prefetch(query, session_id=session_id)\n+                except Exception as e:\n+                    logger.debug(\n+                        \"Memory provider '%s' queue_prefetch failed (non-fatal): %s\",\n+                        provider.name, e,\n+                    )\n+\n+        self._submit_background(_run)\n \n     # -- Sync ----------------------------------------------------------------\n ","@@ -407,27 +434,120 @@ def sync_all(\n         session_id: str = \"\",\n         messages: Optional[List[Dict[str, Any]]] = None,\n     ) -> None:\n-        \"\"\"Sync a completed turn to all providers.\"\"\"\n-        for provider in self._providers:\n-            try:\n-                if messages is not None and self._provider_sync_accepts_messages(provider):\n-                    provider.sync_turn(\n-                        user_content,\n-                        assistant_content,\n-                        session_id=session_id,\n-                        messages=messages,\n+        \"\"\"Sync a completed turn to all providers.\n+\n+        Runs on a background worker thread, NOT inline on the\n+        turn-completion path. A provider's ``sync_turn`` may make a\n+        blocking network/daemon call (a misconfigured Hindsight daemon\n+        was observed blocking ~298s before failing); doing that inline\n+        held ``run_conversation`` open long after the user saw their\n+        response, so every interface (CLI, TUI, gateway) kept the agent\n+        marked \"running\" for minutes and any follow-up message triggered\n+        an aggressive interrupt. Dispatching off-thread means a slow or\n+        broken provider can never stall the turn — the sync simply\n+        completes (or fails, logged) in the background.\n+\n+        Writes are serialized through a single worker so turn N lands\n+        before turn N+1; provider implementations don't need their own\n+        ordering guarantees.\n+        \"\"\"\n+        providers = list(self._providers)\n+        if not providers:\n+            return\n+\n+        def _run() -> None:\n+            for provider in providers:\n+                try:\n+                    if messages is not None and self._provider_sync_accepts_messages(provider):\n+                        provider.sync_turn(\n+                            user_content,\n+                            assistant_content,\n+                            session_id=session_id,\n+                            messages=messages,\n+                        )\n+                    else:\n+                        provider.sync_turn(\n+                            user_content,\n+                            assistant_content,\n+                            session_id=session_id,\n+                        )\n+                except Exception as e:\n+                    logger.warning(\n+                        \"Memory provider '%s' sync_turn failed: %s\",\n+                        provider.name, e,\n                     )\n-                else:\n-                    provider.sync_turn(\n-                        user_content,\n-                        assistant_content,\n-                        session_id=session_id,\n+\n+        self._submit_background(_run)\n+\n+    # -- Background dispatch -------------------------------------------------\n+\n+    def _submit_background(self, fn) -> None:\n+        \"\"\"Run ``fn`` on the manager's background worker.\n+\n+        The executor is created lazily and shared across calls. If the\n+        executor can't be created or has already been shut down, ``fn``\n+        runs inline as a last-resort fallback — losing the async benefit\n+        but never losing the write itself. ``fn`` must do its own\n+        per-provider error handling; this wrapper only guards executor\n+        plumbing.\n+        \"\"\"\n+        executor = self._get_sync_executor()\n+        if executor is None:\n+            # Executor unavailable (shut down / creation failed) — run\n+            # inline rather than drop the work. Slow, but correct.\n+            try:\n+                fn()\n+            except Exception as e:  # pragma: no cover - fn guards internally\n+                logger.debug(\"Inline memory background task failed: %s\", e)\n+            return\n+        try:\n+            executor.submit(fn)\n+        except RuntimeError:\n+            # Executor was shut down between the get and the submit\n+            # (teardown race). Fall back to inline.\n+            try:\n+                fn()\n+            except Exception as e:  # pragma: no cover - fn guards internally\n+                logger.debug(\"Inline memory background task failed: %s\", e)\n+\n+    def _get_sync_executor(self) -> Optional[ThreadPoolExecutor]:\n+        \"\"\"Lazily create the single-worker background executor.\"\"\"\n+        if self._sync_executor is not None:\n+            return self._sync_executor\n+        with self._sync_executor_lock:\n+            if self._sync_executor is None:\n+                try:\n+                    self._sync_executor = ThreadPoolExecutor(\n+                        max_workers=1,\n+                        thread_name_prefix=\"mem-sync\",\n                     )\n-            except Exception as e:\n-                logger.warning(\n-                    \"Memory provider '%s' sync_turn failed: %s\",\n-                    provider.name, e,\n-                )\n+                except Exception as e:  # pragma: no cover - resource exhaustion\n+                    logger.warning(\"Failed to create memory sync executor: %s\", e)\n+                    return None\n+            return self._sync_executor\n+\n+    def flush_pending(self, timeout: Optional[float] = None) -> bool:\n+        \"\"\"Block until queued sync/prefetch work has drained.\n+\n+        Single-worker executor means submitting a sentinel and waiting on\n+        it guarantees every previously-submitted task has run. Returns\n+        True if the barrier completed within ``timeout`` (or no executor\n+        exists), False on timeout. Used at real session boundaries and by\n+        tests that need to assert provider state deterministically.\n+        \"\"\"\n+        executor = self._sync_executor\n+        if executor is None:\n+            return True\n+        try:\n+            fut = executor.submit(lambda: None)\n+        except RuntimeError:\n+            # Executor already shut down — nothing pending.\n+            return True\n+        try:\n+            fut.result(timeout=timeout)\n+            return True\n+        except Exception:\n+            return False\n \n     # -- Tools ---------------------------------------------------------------\n ","@@ -653,7 +773,15 @@ def on_delegation(self, task: str, result: str, *,\n                 )\n \n     def shutdown_all(self) -> None:\n-        \"\"\"Shut down all providers (reverse order for clean teardown).\"\"\"\n+        \"\"\"Shut down all providers (reverse order for clean teardown).\n+\n+        Drains the background sync/prefetch executor first (bounded by\n+        ``_SYNC_DRAIN_TIMEOUT_S``) so a turn's final sync has a chance to\n+        land before providers are torn down. The worker threads are\n+        daemon, so anything still wedged past the drain window dies with\n+        the interpreter rather than blocking exit.\n+        \"\"\"\n+        self._drain_sync_executor()\n         for provider in reversed(self._providers):\n             try:\n                 provider.shutdown()","@@ -663,6 +791,52 @@ def shutdown_all(self) -> None:\n                     provider.name, e,\n                 )\n \n+    def _drain_sync_executor(self) -> None:\n+        \"\"\"Shut down the background executor, waiting briefly for drain.\n+\n+        Bounded by ``_SYNC_DRAIN_TIMEOUT_S``: a wedged provider must never\n+        hang process/session teardown. We stop accepting new work and\n+        cancel anything still queued, then wait at most the drain timeout\n+        for the currently-running task on a watcher thread. The worker is\n+        daemon, so an over-running task dies with the interpreter.\n+        \"\"\"\n+        with self._sync_executor_lock:\n+            executor = self._sync_executor\n+            self._sync_executor = None\n+        if executor is None:\n+            return\n+        try:\n+            # Stop accepting new work and drop anything still queued, but\n+            # do NOT block here — cancel_futures cancels not-yet-started\n+            # tasks; the in-flight one keeps running on its daemon thread.\n+            executor.shutdown(wait=False, cancel_futures=True)\n+        except TypeError:\n+            # Older Python without cancel_futures kwarg.\n+            try:\n+                executor.shutdown(wait=False)\n+            except Exception as e:  # pragma: no cover\n+                logger.debug(\"Memory sync executor shutdown failed: %s\", e)\n+            return\n+        except Exception as e:  # pragma: no cover\n+            logger.debug(\"Memory sync executor shutdown failed: %s\", e)\n+            return\n+        # Give an in-flight sync a bounded chance to finish on a watcher\n+        # thread so we don't block the caller past the drain timeout.\n+        drainer = threading.Thread(\n+            target=lambda: self._bounded_executor_wait(executor),\n+            daemon=True,\n+            name=\"mem-sync-drain\",\n+        )\n+        drainer.start()\n+        drainer.join(timeout=_SYNC_DRAIN_TIMEOUT_S)\n+\n+    @staticmethod\n+    def _bounded_executor_wait(executor: ThreadPoolExecutor) -> None:\n+        try:\n+            executor.shutdown(wait=True)\n+        except Exception as e:  # pragma: no cover\n+            logger.debug(\"Memory sync executor drain wait failed: %s\", e)\n+\n     def initialize_all(self, session_id: str, **kwargs) -> None:\n         \"\"\"Initialize all providers.\n "]},{"file":"tests/agent/test_memory_async_sync.py","hunks":["@@ -0,0 +1,138 @@\n+\"\"\"Regression guard: end-of-turn memory sync must not block the turn.\n+\n+Before this fix, ``MemoryManager.sync_all`` / ``queue_prefetch_all`` looped\n+``provider.sync_turn`` / ``provider.queue_prefetch`` INLINE on the\n+turn-completion path. A provider making a blocking network/daemon call (a\n+misconfigured Hindsight daemon was observed blocking ~298s before failing)\n+held ``run_conversation`` open long after the user saw their response, so\n+every interface (CLI, TUI, gateway) kept the agent marked \"running\" for\n+minutes and any follow-up message triggered an aggressive interrupt that\n+dropped the message.\n+\n+The fix dispatches provider work to a single-worker background executor.\n+``sync_all`` / ``queue_prefetch_all`` return immediately; the work completes\n+(or fails, logged) in the background. ``flush_pending`` provides a barrier\n+for session boundaries and deterministic tests. ``shutdown_all`` drains the\n+executor with a bounded timeout so a wedged provider can't hang teardown.\n+\"\"\"\n+import time\n+\n+import pytest\n+\n+from agent.memory_provider import MemoryProvider\n+from agent.memory_manager import MemoryManager\n+\n+\n+class _SlowProvider(MemoryProvider):\n+    \"\"\"Provider whose sync/prefetch block, simulating a slow backend.\"\"\"\n+\n+    _name = \"slow\"\n+\n+    def __init__(self, delay: float = 1.0):\n+        self._delay = delay\n+        self.sync_done = False\n+        self.prefetch_done = False\n+\n+    @property\n+    def name(self) -> str:\n+        return self._name\n+\n+    def initialize(self, session_id: str = \"\", **kwargs) -> None:\n+        pass\n+\n+    def is_available(self) -> bool:\n+        return True\n+\n+    def system_prompt_block(self) -> str:\n+        return \"\"\n+\n+    def prefetch(self, query, *, session_id: str = \"\") -> str:\n+        return \"\"\n+\n+    def queue_prefetch(self, query, *, session_id: str = \"\") -> None:\n+        time.sleep(self._delay)\n+        self.prefetch_done = True\n+\n+    def sync_turn(self, user_content, assistant_content, *, session_id: str = \"\", messages=None) -> None:\n+        time.sleep(self._delay)\n+        self.sync_done = True\n+\n+    def get_tool_schemas(self):\n+        return []\n+\n+    def handle_tool_call(self, tool_name, args, **kwargs) -> str:\n+        return \"\"\n+\n+\n+def test_sync_all_does_not_block_on_slow_provider():\n+    \"\"\"The crux of the fix: a slow provider must NOT stall the caller.\"\"\"\n+    mgr = MemoryManager()\n+    mgr.add_provider(_SlowProvider(delay=2.0))\n+\n+    t0 = time.time()\n+    mgr.sync_all(\"hi\", \"hey\", session_id=\"s1\")\n+    mgr.queue_prefetch_all(\"hi\", session_id=\"s1\")\n+    elapsed = time.time() - t0\n+\n+    # Provider blocks 2s per call inline; off-thread dispatch returns ~instantly.\n+    assert elapsed < 0.5, f\"turn-completion path blocked {elapsed:.2f}s\"\n+\n+\n+def test_background_work_still_completes():\n+    \"\"\"Dispatching off-thread must not silently drop the write.\"\"\"\n+    mgr = MemoryManager()\n+    p = _SlowProvider(delay=0.1)\n+    mgr.add_provider(p)\n+\n+    mgr.sync_all(\"hi\", \"hey\", session_id=\"s1\")\n+    mgr.queue_prefetch_all(\"hi\", session_id=\"s1\")\n+\n+    assert mgr.flush_pending(timeout=10) is True\n+    assert p.sync_done is True\n+    assert p.prefetch_done is True\n+\n+\n+def test_flush_pending_no_executor_is_true():\n+    \"\"\"flush_pending must be a no-op (return True) before any sync ran.\"\"\"\n+    mgr = MemoryManager()\n+    assert mgr.flush_pending(timeout=1) is True\n+\n+\n+def test_no_providers_does_not_create_executor():\n+    \"\"\"Builtin-only / no-provider sessions must not spawn an executor.\"\"\"\n+    mgr = MemoryManager()\n+    mgr.sync_all(\"hi\", \"hey\")\n+    mgr.queue_prefetch_all(\"hi\")\n+    assert mgr._sync_executor is None\n+\n+\n+def test_shutdown_all_is_bounded_with_wedged_provider():\n+    \"\"\"A provider that never returns must not hang teardown.\"\"\"\n+    mgr = MemoryManager()\n+    mgr.add_provider(_SlowProvider(delay=30.0))\n+    mgr.sync_all(\"hi\", \"hey\")\n+\n+    t0 = time.time()\n+    mgr.shutdown_all()\n+    elapsed = time.time() - t0\n+\n+    # Bounded by _SYNC_DRAIN_TIMEOUT_S (5s) plus a little slack.\n+    assert elapsed < 8.0, f\"shutdown blocked {elapsed:.1f}s on wedged provider\"\n+\n+\n+def test_writes_are_serialized_in_order():\n+    \"\"\"Single-worker executor must preserve turn ordering (N before N+1).\"\"\"\n+    order = []\n+\n+    class _OrderProvider(_SlowProvider):\n+        _name = \"order\"\n+\n+        def sync_turn(self, user_content, assistant_content, *, session_id=\"\", messages=None):\n+            order.append(user_content)\n+\n+    mgr = MemoryManager()\n+    mgr.add_provider(_OrderProvider(delay=0.0))\n+    for i in range(5):\n+        mgr.sync_all(f\"turn-{i}\", \"resp\", session_id=\"s1\")\n+    assert mgr.flush_pending(timeout=10) is True\n+    assert order == [f\"turn-{i}\" for i in range(5)]"]},{"file":"tests/agent/test_memory_provider.py","hunks":["@@ -229,6 +229,7 @@ def test_queue_prefetch_all(self):\n         mgr.add_provider(p2)\n \n         mgr.queue_prefetch_all(\"next turn\")\n+        mgr.flush_pending(timeout=5)\n         assert p1.queued_prefetches == [\"next turn\"]\n         assert p2.queued_prefetches == [\"next turn\"]\n ","@@ -240,6 +241,7 @@ def test_sync_all(self):\n         mgr.add_provider(p2)\n \n         mgr.sync_all(\"user msg\", \"assistant msg\")\n+        mgr.flush_pending(timeout=5)\n         assert p1.synced_turns == [(\"user msg\", \"assistant msg\")]\n         assert p2.synced_turns == [(\"user msg\", \"assistant msg\")]\n ","@@ -253,7 +255,7 @@ def test_sync_all_passes_messages_to_opted_in_provider(self):\n         ]\n \n         mgr.sync_all(\"user msg\", \"assistant msg\", session_id=\"sess-1\", messages=messages)\n-\n+        mgr.flush_pending(timeout=5)\n         assert p.synced_turns == [(\"user msg\", \"assistant msg\", \"sess-1\", messages)]\n \n     def test_sync_all_omits_messages_for_legacy_provider(self):","@@ -262,7 +264,7 @@ def test_sync_all_omits_messages_for_legacy_provider(self):\n         mgr.add_provider(p)\n \n         mgr.sync_all(\"user msg\", \"assistant msg\", messages=[{\"role\": \"tool\"}])\n-\n+        mgr.flush_pending(timeout=5)\n         assert p.synced_turns == [(\"user msg\", \"assistant msg\")]\n \n     def test_sync_failure_doesnt_block_others(self):","@@ -275,6 +277,7 @@ def test_sync_failure_doesnt_block_others(self):\n         mgr.add_provider(p2)\n \n         mgr.sync_all(\"user\", \"assistant\")\n+        mgr.flush_pending(timeout=5)\n         # p1 failed but p2 still synced\n         assert p2.synced_turns == [(\"user\", \"assistant\")]\n "]},{"file":"tests/agent/test_memory_session_switch.py","hunks":["@@ -179,6 +179,7 @@ def test_sync_all_propagates_session_id_to_providers():\n     p = _RecordingProvider()\n     mm.add_provider(p)\n     mm.sync_all(\"hello\", \"world\", session_id=\"sess-42\")\n+    mm.flush_pending(timeout=5)\n     assert p.sync_calls == [\n         {\"user\": \"hello\", \"asst\": \"world\", \"session_id\": \"sess-42\"}\n     ]","@@ -189,6 +190,7 @@ def test_queue_prefetch_all_propagates_session_id_to_providers():\n     p = _RecordingProvider()\n     mm.add_provider(p)\n     mm.queue_prefetch_all(\"next query\", session_id=\"sess-42\")\n+    mm.flush_pending(timeout=5)\n     assert p.queue_calls == [{\"query\": \"next query\", \"session_id\": \"sess-42\"}]\n \n \n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}