{"repo":"NousResearch/hermes-agent","public_id":"10865","display_label":"Gateway agent cache and resource lifecycle leaks","summary":"A cluster of bug reports and fixes centered on memory and resource leaks in the gateway's agent cache, covering missing cleanup on eviction paths, unclosed aiohttp client sessions, and unreleased memory providers from temporary and cached AIAgent instances.","subsystem":"agent","counts":{"members":21,"open_issues":2,"open_issues_distinct":2,"open_prs":1,"closed_issues":6,"merged_prs":4,"attempt_prs":9},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":11028},"resolution":{"state":"partially_resolved","by":"pr-11028,pr-41975,pr-122925"},"freshness":{"state":"verified","last_verified_at":"2026-09-25T17:49:57+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.08209523809523807},"fix_edges":[{"issue_number":10865,"pr_number":11028,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":14598,"pr_number":41974,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":19251,"pr_number":41975,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":25315,"pr_number":41975,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":50197,"pr_number":122925,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":10865,"title":"Multiple AIAgent instances leak aiohttp client sessions","url":null,"state":"closed"},{"kind":"issue","number":19251,"title":"Investigate residual gateway memory leak under heavy scheduled workload","url":null,"state":"closed"},{"kind":"issue","number":25315,"title":"fix: gateway memory leak — _evict_cached_agent drops agents without cleanup, _session_messages never cleared","url":null,"state":"closed"},{"kind":"issue","number":43657,"title":"aiohttp ClientSession leak after auxiliary tasks (title_generation, etc.)","url":null,"state":"open"},{"kind":"issue","number":48287,"title":"[Bug]: Agent cache (_agent_cache) causes 50GB+ virtual memory leak in long-running gateway sessions","url":null,"state":"closed"},{"kind":"issue","number":50197,"title":"Multiple AIAgent instances lack lifecycle cleanup (shutdown_memory_provider + close)","url":null,"state":"closed"},{"kind":"issue","number":58817,"title":"[Bug]: Gateway memory grows ~5x over 20h on Linux — planned-restart masking leak, not fixing it","url":null,"state":"closed"},{"kind":"issue","number":64278,"title":"[Bug]: Gateway soft agent-cache eviction leaves memory providers running and leaks threads/HTTP clients","url":null,"state":"open"},{"kind":"pr","number":10899,"title":"fix(gateway): close temporary agents after one-off tasks","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#11028","number":11028},{"kind":"pr","value":"#10945","number":10945}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":10945,"title":"fix(gateway): clean up temporary agent resources","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#11028","number":11028},{"kind":"pr","value":"#10899","number":10899}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":11028,"title":"fix(gateway): close temporary agents after one-off tasks","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_background_command.py","tests/gateway/test_compress_command.py","tests/gateway/test_flush_memory_stale_guard.py","tests/gateway/test_session_hygiene.py"]},{"kind":"pr","number":25318,"title":"fix(gateway): prevent memory leak from agent cache eviction and unbounded session data","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3d029a53e","number":0},{"kind":"pr","value":"#41975","number":41975}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":25332,"title":"fix: stop gateway memory leak on cached agent and session expiry","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":25377,"title":"fix(gateway): release cached agents on eviction","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3d029a53ec327fd7a628cde900bf300ca9a0cc0f","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":41974,"title":"fix(gateway): release evicted agent clients to stop RSS leak (#29298)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -12493,25 +12493,67 @@ async def _interrupt_and_clear_session(\n             self._release_running_agent_state(session_key)\n \n     def _evict_cached_agent(self, session_key: str) -> None:\n-        \"\"\"Remove a cached agent for a session (called on /new, /model, etc).\"\"\"\n+        \"\"\"Remove a cached agent for a session (called on /new, /model, etc).\n+\n+        Pops the entry AND soft-releases the evicted agent's LLM client\n+        pool so the httpx connection (sockets + held buffers) is freed\n+        promptly rather than waiting on CPython GC — AIAgent holds\n+        reference cycles (callbacks, tool state) that delay refcount\n+        collection, so a manual release is required to keep gateway RSS\n+        flat across many /new, /model, undo and reset operations (#29298,\n+        same leak class as #25315).\n+\n+        The release is soft (``release_clients()``): it frees the client\n+        pool and per-turn child subagents but PRESERVES the session's\n+        terminal sandbox, browser daemon, and tracked bg processes (keyed\n+        on task_id), because the session may resume with a freshly-built\n+        agent.  Call sites that want a hard teardown (true conversation\n+        boundaries like /new) already call ``_cleanup_agent_resources``\n+        before evicting; ``release_clients`` is idempotent and safe to\n+        run again after that (the client is already None).\n+\n+        Cleanup runs on a daemon thread so we never block holding\n+        ``_agent_cache_lock`` on slow socket teardown — mirrors the\n+        cap-enforcer and idle-sweeper paths.\n+        \"\"\"\n         _lock = getattr(self, \"_agent_cache_lock\", None)\n+        evicted = None\n         if _lock:\n             with _lock:\n-                entry = self._agent_cache.pop(session_key, None)\n-            # Release clients on a daemon thread, same as _enforce_agent_cache_cap.\n-            # Without this, every /new, /model, /reasoning, codex-runtime change,\n-            # and /undo leaked a full agent: OpenAI client, httpx transport, SSL\n-            # context, and conversation history. Only the /new path cleaned up\n-            # first; the rest popped the entry and dropped it on the floor.\n-            if entry is not None:\n-                agent = entry[0] if isinstance(entry, tuple) and entry else None\n-                if agent is not None:\n-                    threading.Thread(\n-                        target=self._release_evicted_agent_soft,\n-                        args=(agent,),\n-                        daemon=True,\n-                        name=f\"agent-cache-cmd-evict-{session_key[:24]}\",\n-                    ).start()\n+                evicted = self._agent_cache.pop(session_key, None)\n+        else:\n+            _cache = getattr(self, \"_agent_cache\", None)\n+            if _cache is not None:\n+                evicted = _cache.pop(session_key, None)\n+\n+        agent = evicted[0] if isinstance(evicted, tuple) and evicted else evicted\n+        if agent is None or agent is _AGENT_PENDING_SENTINEL:\n+            return\n+\n+        # Don't tear down an agent that's actively mid-turn — its client,\n+        # sandbox and child subagents are in use by the running request.\n+        running_ids = {\n+            id(a)\n+            for a in getattr(self, \"_running_agents\", {}).values()\n+            if a is not None and a is not _AGENT_PENDING_SENTINEL\n+        }\n+        if id(agent) in running_ids:\n+            return\n+\n+        try:\n+            threading.Thread(\n+                target=self._release_evicted_agent_soft,\n+                args=(agent,),\n+                daemon=True,\n+                name=f\"agent-evict-{str(session_key)[:24]}\",\n+            ).start()\n+        except Exception:\n+            # If we can't spawn a thread (interpreter shutdown), release\n+            # inline as a best-effort fallback.\n+            try:\n+                self._release_evicted_agent_soft(agent)\n+            except Exception:\n+                pass\n \n     @staticmethod\n     def _init_cached_agent_for_turn(agent: Any, interrupt_depth: int) -> None:\n"]}]},{"kind":"pr","number":41975,"title":"fix(gateway): close residual memory-leak sites under heavy scheduled workload","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","model_tools.py","tests/test_get_tool_definitions_cache_isolation.py"]},{"kind":"pr","number":50383,"title":"fix(tui-gateway): close AIAgent instances to release resources (#50197)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":50403,"title":"fix(tui-gateway): close AIAgent instances to release resources (#50197)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"2fa2f42d41","number":0},{"kind":"commit","value":"94f3dbec9b","number":0},{"kind":"pr","value":"#122925","number":122925},{"kind":"pr","value":"#50197","number":50197}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":64284,"title":"fix: clean up evicted memory providers (salvage of #63703 + review fixes)","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":64644,"title":"fix(gateway): shut down memory provider on soft cache eviction (#64278)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#64284","number":64284},{"kind":"pr","value":"#64278","number":64278},{"kind":"pr","value":"#64644","number":64644},{"kind":"pr","value":"#64284","number":64284},{"kind":"pr","value":"#64284","number":64284}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":122925,"title":"fix(agent): close one-shot AIAgents on every exit path","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["batch_runner.py","hermes_cli/cli_commands_mixin.py","plugins/platforms/feishu/feishu_comment.py","tests/test_agent_close_lifecycle.py","tui_gateway/methods_prompt.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}