{"repo":"NousResearch/hermes-agent","public_id":"52197","display_label":"Gateway agent cache locking and memory issues","summary":"A cluster of issues and fixes addressing the gateway's agent cache: lock contention during cross-process invalidation that stalls the event loop and blocks heartbeats, unbounded cache growth, and resource/memory leaks from unreleased evicted agent clients.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":52761,"merged_at":"2026-06-26T01:58:47+00:00"},"resolution":{"state":"resolved","by":"pr-52761"},"freshness":{"state":"verified","last_verified_at":"2026-06-26T01:58:47+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":52197,"pr_number":52761,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-26T01:58:47+00:00"}],"members":[{"kind":"issue","number":52197,"title":"[Bug]:  gateway cross-process agent-cache invalidation performs cleanup while holding _agent_cache_lock, stalling the event loop and blocking Discord heartbeats","url":null,"state":"closed"},{"kind":"pr","number":52200,"title":"fix(gateway): move cross-process cache invalidation cleanup outside lock (#52197)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"811df74a102a2efb4465d775e218a590b1486034","number":0},{"kind":"pr","value":"#52761","number":52761},{"kind":"pr","value":"#52197","number":52197}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":52761,"title":"fix(gateway): defer cross-process cache cleanup off the cache lock (#52197)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_agent_cache.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -15829,6 +15829,7 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:\n                 except Exception:\n                     pass\n \n+            _xproc_evicted_agent = None\n             if _cache_lock and _cache is not None:\n                 with _cache_lock:\n                     cached = _cache.get(session_key)","@@ -15852,7 +15853,20 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:\n                             evicted = self._agent_cache.pop(session_key, None)\n                             _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None\n                             if _ev_agent and _ev_agent is not _AGENT_PENDING_SENTINEL:\n-                                self._cleanup_agent_resources(_ev_agent)\n+                                # Defer cleanup until AFTER the lock is\n+                                # released — _cleanup_agent_resources /\n+                                # release_clients can block on memory-provider\n+                                # shutdown and socket teardown, and running it\n+                                # here would stall the gateway event loop while\n+                                # _sweep_idle_cached_agents (session-expiry\n+                                # watcher) waits on the same lock, blocking\n+                                # Discord heartbeats (#52197).  The same session\n+                                # rebuilds a fresh agent immediately below, so\n+                                # use the SOFT release that preserves the\n+                                # session's terminal sandbox / browser / bg\n+                                # processes for the rebuilt agent to inherit —\n+                                # mirrors _evict_cached_agent / idle-sweep.\n+                                _xproc_evicted_agent = _ev_agent\n                         else:\n                             agent = cached[0]\n                             # Refresh LRU order so the cap enforcement evicts","@@ -15868,6 +15882,26 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:\n                             agent.max_iterations = max_iterations\n                             logger.debug(\"Reusing cached agent for session %s\", session_key)\n \n+            # Lock released — now schedule cleanup of any cross-process-evicted\n+            # agent on a daemon thread so memory-provider shutdown / socket\n+            # teardown never blocks the gateway event loop or the cache lock\n+            # the session-expiry watcher needs (#52197).\n+            if _xproc_evicted_agent is not None:\n+                try:\n+                    threading.Thread(\n+                        target=self._release_evicted_agent_soft,\n+                        args=(_xproc_evicted_agent,),\n+                        daemon=True,\n+                        name=f\"agent-xproc-evict-{str(session_key)[:24]}\",\n+                    ).start()\n+                except Exception:\n+                    # Interpreter shutdown or thread-spawn failure — release\n+                    # inline as a best-effort fallback.\n+                    try:\n+                        self._release_evicted_agent_soft(_xproc_evicted_agent)\n+                    except Exception:\n+                        pass\n+\n             if agent is None:\n                 # Config changed or first message — create fresh agent\n                 agent = AIAgent("]},{"file":"tests/gateway/test_agent_cache.py","hunks":["@@ -1706,3 +1706,117 @@ def get_session(self, _sid):\n         runner._refresh_agent_cache_message_count(\"telegram:s1\", \"s1\")\n         with runner._agent_cache_lock:\n             assert runner._agent_cache[\"telegram:s1\"][2] == 5\n+\n+\n+class TestCrossProcessInvalidationDefersCleanup:\n+    \"\"\"#52197: cross-process cache invalidation must NOT run agent cleanup\n+    while holding ``_agent_cache_lock``.\n+\n+    The #45966 guard popped the stale cached agent and then called the\n+    blocking ``_cleanup_agent_resources`` (memory-provider shutdown, socket\n+    teardown) *inside* the ``with _agent_cache_lock:`` block, on the gateway\n+    event-loop thread.  While that ran, ``_sweep_idle_cached_agents`` (driven\n+    by the session-expiry watcher) blocked acquiring the same lock and the\n+    asyncio loop stalled, tripping Discord heartbeat-blocked warnings.\n+\n+    The fix mirrors the cap-enforcer / idle-sweep paths: pop under the lock,\n+    release it, then schedule the SOFT release (which preserves the session's\n+    terminal sandbox / browser / bg processes for the immediately-rebuilt\n+    agent) on a daemon thread.\n+\n+    These tests replicate the exact eviction sequence the production guard now\n+    performs and pin the invariant: the lock is free while cleanup runs, and\n+    the hard-teardown path is never used here.\n+    \"\"\"\n+\n+    def _runner(self):\n+        from collections import OrderedDict\n+        from gateway.run import GatewayRunner\n+\n+        runner = GatewayRunner.__new__(GatewayRunner)\n+        runner._agent_cache = OrderedDict()\n+        runner._agent_cache_lock = threading.Lock()\n+        return runner\n+\n+    @staticmethod\n+    def _evict_like_production(runner, session_key):\n+        \"\"\"Run the post-#52197 cross-process eviction sequence verbatim:\n+        pop the stale entry under the lock, then schedule the soft release\n+        on a daemon thread AFTER the lock is released.\"\"\"\n+        _xproc_evicted_agent = None\n+        with runner._agent_cache_lock:\n+            evicted = runner._agent_cache.pop(session_key, None)\n+            _ev_agent = evicted[0] if isinstance(evicted, tuple) and evicted else None\n+            if _ev_agent is not None:\n+                _xproc_evicted_agent = _ev_agent\n+        if _xproc_evicted_agent is not None:\n+            threading.Thread(\n+                target=runner._release_evicted_agent_soft,\n+                args=(_xproc_evicted_agent,),\n+                daemon=True,\n+                name=\"agent-xproc-evict-test\",\n+            ).start()\n+\n+    def test_cleanup_runs_with_lock_released(self):\n+        \"\"\"The cache lock must be acquirable WHILE the evicted agent's\n+        cleanup is running — proving cleanup is off the locked path.\"\"\"\n+        runner = self._runner()\n+\n+        cleanup_started = threading.Event()\n+        release_lock = threading.Event()\n+\n+        def _soft(agent):\n+            cleanup_started.set()\n+            # Block here as if memory-provider shutdown / socket teardown is\n+            # slow.  If cleanup were still holding _agent_cache_lock, the\n+            # assertion below could never acquire it.\n+            release_lock.wait(timeout=2.0)\n+\n+        runner._release_evicted_agent_soft = _soft\n+        runner._cleanup_agent_resources = MagicMock()\n+\n+        old_agent = MagicMock()\n+        with runner._agent_cache_lock:\n+            runner._agent_cache[\"telegram:s1\"] = (old_agent, \"sig\", 3)\n+\n+        self._evict_like_production(runner, \"telegram:s1\")\n+\n+        # Wait until the (blocking) cleanup is mid-flight.\n+        assert cleanup_started.wait(timeout=2.0)\n+\n+        # The lock MUST be free right now — this is the heart of #52197.\n+        # A 0.5s acquire timeout would fire if cleanup held the lock.\n+        acquired = runner._agent_cache_lock.acquire(timeout=0.5)\n+        assert acquired, \"cache lock blocked during cross-process cleanup (#52197)\"\n+        runner._agent_cache_lock.release()\n+\n+        # Let the cleanup thread finish.\n+        release_lock.set()\n+\n+        # Stale entry was popped, hard-teardown path never used.\n+        assert \"telegram:s1\" not in runner._agent_cache\n+        runner._cleanup_agent_resources.assert_not_called()\n+\n+    def test_soft_release_scheduled_for_evicted_agent(self):\n+        \"\"\"The evicted agent is handed to the soft-release path, not the\n+        hard ``_cleanup_agent_resources`` teardown.\"\"\"\n+        runner = self._runner()\n+\n+        release_calls: list = []\n+        runner._release_evicted_agent_soft = lambda agent: release_calls.append(agent)\n+        runner._cleanup_agent_resources = MagicMock()\n+\n+        old_agent = MagicMock()\n+        with runner._agent_cache_lock:\n+            runner._agent_cache[\"telegram:s1\"] = (old_agent, \"sig\", 3)\n+\n+        self._evict_like_production(runner, \"telegram:s1\")\n+\n+        import time as _t\n+        deadline = _t.time() + 2.0\n+        while _t.time() < deadline and not release_calls:\n+            _t.sleep(0.02)\n+\n+        assert release_calls == [old_agent]\n+        runner._cleanup_agent_resources.assert_not_called()\n+\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}