{"repo":"NousResearch/hermes-agent","public_id":"14598","display_label":"Stale connection and PID file handling patches","summary":"These items address stale or leaked connections and PID file handling across the gateway and Bedrock components, including cache eviction cleanup, RSS memory leaks, and recovery from SIGKILL/OOM scenarios. No single fix is assumed complete or final.","subsystem":"gateway","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":41974,"merged_at":"2026-06-08T13:44:52+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-08T13:44:52+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":14598,"pr_number":41974,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-06-08T13:44:52+00:00"}],"members":[{"kind":"issue","number":14598,"title":"Gateway run.py: PID file race condition and httpx connection leak on cache eviction","url":null,"state":"closed"},{"kind":"pr","number":14609,"title":"fix(gateway): recover stale pid files and release evicted clients","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"402d048eb","number":0},{"kind":"pr","value":"#14598","number":14598},{"kind":"pr","value":"#29298","number":29298},{"kind":"pr","value":"#41974","number":41974}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":14710,"title":"fix(bedrock): evict cached boto3 client on stale-connection errors","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"a9ccb03cc","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":41974,"title":"fix(gateway): release evicted agent clients to stop RSS leak (#29298)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -12493,25 +12493,67 @@ async def _interrupt_and_clear_session(\n             self._release_running_agent_state(session_key)\n \n     def _evict_cached_agent(self, session_key: str) -> None:\n-        \"\"\"Remove a cached agent for a session (called on /new, /model, etc).\"\"\"\n+        \"\"\"Remove a cached agent for a session (called on /new, /model, etc).\n+\n+        Pops the entry AND soft-releases the evicted agent's LLM client\n+        pool so the httpx connection (sockets + held buffers) is freed\n+        promptly rather than waiting on CPython GC — AIAgent holds\n+        reference cycles (callbacks, tool state) that delay refcount\n+        collection, so a manual release is required to keep gateway RSS\n+        flat across many /new, /model, undo and reset operations (#29298,\n+        same leak class as #25315).\n+\n+        The release is soft (``release_clients()``): it frees the client\n+        pool and per-turn child subagents but PRESERVES the session's\n+        terminal sandbox, browser daemon, and tracked bg processes (keyed\n+        on task_id), because the session may resume with a freshly-built\n+        agent.  Call sites that want a hard teardown (true conversation\n+        boundaries like /new) already call ``_cleanup_agent_resources``\n+        before evicting; ``release_clients`` is idempotent and safe to\n+        run again after that (the client is already None).\n+\n+        Cleanup runs on a daemon thread so we never block holding\n+        ``_agent_cache_lock`` on slow socket teardown — mirrors the\n+        cap-enforcer and idle-sweeper paths.\n+        \"\"\"\n         _lock = getattr(self, \"_agent_cache_lock\", None)\n+        evicted = None\n         if _lock:\n             with _lock:\n-                entry = self._agent_cache.pop(session_key, None)\n-            # Release clients on a daemon thread, same as _enforce_agent_cache_cap.\n-            # Without this, every /new, /model, /reasoning, codex-runtime change,\n-            # and /undo leaked a full agent: OpenAI client, httpx transport, SSL\n-            # context, and conversation history. Only the /new path cleaned up\n-            # first; the rest popped the entry and dropped it on the floor.\n-            if entry is not None:\n-                agent = entry[0] if isinstance(entry, tuple) and entry else None\n-                if agent is not None:\n-                    threading.Thread(\n-                        target=self._release_evicted_agent_soft,\n-                        args=(agent,),\n-                        daemon=True,\n-                        name=f\"agent-cache-cmd-evict-{session_key[:24]}\",\n-                    ).start()\n+                evicted = self._agent_cache.pop(session_key, None)\n+        else:\n+            _cache = getattr(self, \"_agent_cache\", None)\n+            if _cache is not None:\n+                evicted = _cache.pop(session_key, None)\n+\n+        agent = evicted[0] if isinstance(evicted, tuple) and evicted else evicted\n+        if agent is None or agent is _AGENT_PENDING_SENTINEL:\n+            return\n+\n+        # Don't tear down an agent that's actively mid-turn — its client,\n+        # sandbox and child subagents are in use by the running request.\n+        running_ids = {\n+            id(a)\n+            for a in getattr(self, \"_running_agents\", {}).values()\n+            if a is not None and a is not _AGENT_PENDING_SENTINEL\n+        }\n+        if id(agent) in running_ids:\n+            return\n+\n+        try:\n+            threading.Thread(\n+                target=self._release_evicted_agent_soft,\n+                args=(agent,),\n+                daemon=True,\n+                name=f\"agent-evict-{str(session_key)[:24]}\",\n+            ).start()\n+        except Exception:\n+            # If we can't spawn a thread (interpreter shutdown), release\n+            # inline as a best-effort fallback.\n+            try:\n+                self._release_evicted_agent_soft(agent)\n+            except Exception:\n+                pass\n \n     @staticmethod\n     def _init_cached_agent_for_turn(agent: Any, interrupt_depth: int) -> None:\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}