{"repo":"NousResearch/hermes-agent","public_id":"11205","display_label":"Gateway session-end and cleanup hooks not firing","summary":"Multiple bugs and fixes addressing gateway session lifecycle hooks—on_session_end and on_session_finalize—not being called during idle expiry or agent-cache eviction, along with related agent resource cleanup ordering.","subsystem":"gateway","counts":{"members":9,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":2,"attempt_prs":5},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":15132},"resolution":{"state":"resolved","by":"pr-15132,pr-57378"},"freshness":{"state":"verified","last_verified_at":"2026-07-02T22:16:44+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.23222222222222222},"fix_edges":[{"issue_number":7759,"pr_number":57378,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":11205,"pr_number":15132,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":11205,"pr_number":57378,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":14981,"pr_number":15132,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":15165,"pr_number":57378,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":11205,"title":"[Bug]: MemoryProvider.on_session_end() never called on gateway session expiry","url":null,"state":"closed"},{"kind":"issue","number":14981,"title":"[Bug]: The on_session_finalize hook is not being fired when gateway sessions expire due to configured idle time.","url":null,"state":"closed"},{"kind":"pr","number":11410,"title":"fix(gateway): fire MemoryProvider.on_session_end on session expiry","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ea01bdceb","number":0},{"kind":"pr","value":"#15696","number":15696},{"kind":"pr","value":"#15132","number":15132}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":12261,"title":"fix(gateway): finalize session-end memory providers with transcript","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#11205","number":11205}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":12917,"title":"fix(gateway): clean up cached agents on shutdown (#11205)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"bf05b8f4a","number":0},{"kind":"pr","value":"#16207","number":16207}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":15132,"title":"fix(gateway): fire on_session_finalize on idle expiry (salvage #13756)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","scripts/release.py","tests/gateway/test_session_boundary_hooks.py"]},{"kind":"pr","number":31856,"title":"fix(gateway): keep idle cached agents alive until session expires","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#57378","number":57378},{"kind":"pr","value":"#57378","number":57378}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":54293,"title":"fix(gateway): call _cleanup_agent_resources before idle-TTL eviction (#11205)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#11205","number":11205},{"kind":"pr","value":"#57378","number":57378},{"kind":"pr","value":"#57378","number":57378}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":57378,"title":"fix(gateway): fire on_session_end across all agent-cache eviction paths (salvage #31856)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","gateway/session.py","tests/gateway/test_agent_cache.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -15589,6 +15589,72 @@ def _init_cached_agent_for_turn(agent: Any, interrupt_depth: int) -> None:\n                 agent._last_flushed_db_idx = 0\n         agent._api_call_count = 0\n \n+    def _commit_memory_before_soft_evict(self, agent: Any, key: str) -> None:\n+        \"\"\"Fire on_session_end extraction before soft-evicting a live agent.\n+\n+        Soft eviction (``_release_evicted_agent_soft``) deliberately keeps the\n+        session resumable and does NOT fire ``on_session_end`` — that hook is\n+        reserved for the true session boundary, tear-down done by\n+        ``_session_expiry_watcher`` when the session finally expires.\n+\n+        But the watcher tears down whatever agent it finds in ``_agent_cache``\n+        at expiry time.  If cache pressure (the LRU cap) soft-evicts a\n+        finalizable session's agent BEFORE it expires, the watcher later finds\n+        no cached agent and ``on_session_end`` is silently skipped — memory\n+        providers never see the transcript (#11205, LRU-cap variant).\n+\n+        We hold the live, fully-scoped agent right now, so commit its\n+        end-of-session memory extraction here using the agent's own memory\n+        manager (correct per-user/chat scoping, no reconstruction).  This uses\n+        ``commit_memory_session`` — extraction WITHOUT provider teardown — so\n+        the eviction stays soft and a resumed turn keeps working.\n+\n+        Only fires for sessions the expiry watcher will eventually finalize\n+        (finite reset policy).  For ``mode == \"none\"`` sessions the watcher\n+        never runs, so there is no missed-boundary to compensate for and we\n+        skip the commit (the agent is simply released).  Best-effort: any\n+        failure is swallowed so eviction still proceeds.\n+        \"\"\"\n+        if agent is None or not hasattr(agent, \"commit_memory_session\"):\n+            return\n+        if getattr(agent, \"_memory_manager\", None) is None:\n+            return  # no external memory provider — nothing to commit\n+        try:\n+            _store = getattr(self, \"session_store\", None)\n+            if _store is None:\n+                return\n+            _store._ensure_loaded()\n+            entry = _store._entries.get(key)\n+            if entry is None:\n+                return\n+            # Only compensate when the watcher would otherwise expect to find\n+            # this agent at expiry (finite policy, not yet expired). Expired\n+            # sessions are torn down by the watcher directly; mode=\"none\"\n+            # sessions are never finalized.\n+            if not _store.is_session_finalizable(entry):\n+                return\n+            if _store._is_session_expired(entry):\n+                return\n+            messages = getattr(agent, \"_session_messages\", None)\n+            agent.commit_memory_session(messages if isinstance(messages, list) else None)\n+            logger.debug(\n+                \"Committed on_session_end extraction before soft-evicting \"\n+                \"finalizable session=%s (cache pressure, pre-expiry)\", key,\n+            )\n+        except Exception as _e:\n+            logger.debug(\"Pre-evict memory commit failed for %s: %s\", key, _e)\n+\n+    def _commit_then_release_soft(self, agent: Any, key: str) -> None:\n+        \"\"\"Commit end-of-session memory (if warranted), then soft-release.\n+\n+        Runs on the daemon eviction thread so the memory-provider call and the\n+        client teardown never block the caller's held cache lock. Order matters:\n+        commit uses the live agent's memory manager before ``release_clients``\n+        drops the message buffer.\n+        \"\"\"\n+        self._commit_memory_before_soft_evict(agent, key)\n+        self._release_evicted_agent_soft(agent)\n+\n     def _release_evicted_agent_soft(self, agent: Any) -> None:\n         \"\"\"Soft cleanup for cache-evicted agents — preserves session tool state.\n ","@@ -15687,9 +15753,15 @@ def _enforce_agent_cache_cap(self) -> None:\n                 key, len(_cache),\n             )\n             if agent is not None:\n+                # Commit end-of-session memory extraction, then soft-release,\n+                # both on the daemon thread so the (possibly network-bound)\n+                # provider call never blocks the held cache lock. The commit\n+                # only fires for finalizable-not-yet-expired sessions whose\n+                # agent would otherwise vanish before the expiry watcher can\n+                # fire on_session_end (#11205, LRU-cap variant).\n                 threading.Thread(\n-                    target=self._release_evicted_agent_soft,\n-                    args=(agent,),\n+                    target=self._commit_then_release_soft,\n+                    args=(agent, key),\n                     daemon=True,\n                     name=f\"agent-cache-evict-{key[:24]}\",\n                 ).start()","@@ -15727,6 +15799,45 @@ def _sweep_idle_cached_agents(self) -> int:\n                 if last_activity is None:\n                     continue\n                 if (now - last_activity) > _AGENT_CACHE_IDLE_TTL_SECS:\n+                    # Check whether the session has actually expired in the\n+                    # session store.  If it hasn't (e.g. daily-reset mode\n+                    # where the reset fires hours after the user's last\n+                    # message), keep the agent in cache so the session-store\n+                    # expiry watcher can still find it and call\n+                    # on_session_end() with the live transcript.  Skipping\n+                    # eviction here means the agent stays alive until the\n+                    # session genuinely expires, at which point the watcher\n+                    # (gateway/run.py _session_expiry_watcher) tears it down\n+                    # properly.  (#11205 follow-up)\n+                    #\n+                    # BUT only defer when the watcher will EVER finalize this\n+                    # session.  For a mode == \"none\" session the watcher never\n+                    # fires (is_session_finalizable() is False), so deferring\n+                    # would pin the agent in cache for the gateway's entire\n+                    # lifetime — the exact leak this idle sweep exists to\n+                    # relieve.  Those sessions fall through to soft eviction\n+                    # WITHOUT on_session_end, and that is correct: a mode==\"none\"\n+                    # session never reaches a session-end boundary, so there is\n+                    # no missed on_session_end to compensate for.  (The finite\n+                    # case — a session evicted under LRU-cap pressure before it\n+                    # expires — is instead covered by _commit_memory_before_soft_\n+                    # evict on the cap path, which fires on_session_end via the\n+                    # live agent's memory manager before releasing it.)\n+                    session_entry = None\n+                    _store = getattr(self, \"session_store\", None)\n+                    try:\n+                        if _store is not None:\n+                            _store._ensure_loaded()\n+                            session_entry = _store._entries.get(key)\n+                    except Exception:\n+                        session_entry = None\n+                    if (\n+                        session_entry is not None\n+                        and _store is not None\n+                        and _store.is_session_finalizable(session_entry)\n+                        and not _store._is_session_expired(session_entry)\n+                    ):\n+                        continue  # keep agent — finite session hasn't expired\n                     to_evict.append((key, agent))\n             for key, _ in to_evict:\n                 _cache.pop(key, None)"]},{"file":"gateway/session.py","hunks":["@@ -1264,6 +1264,37 @@ def _is_session_expired(self, entry: SessionEntry) -> bool:\n \n         return False\n \n+    def is_session_finalizable(self, entry: SessionEntry) -> bool:\n+        \"\"\"Return True if the expiry watcher will *ever* finalize this session.\n+\n+        The expiry watcher (``GatewayRunner._session_expiry_watcher``) only\n+        tears an agent down — and only then fires ``on_session_end`` — for\n+        sessions whose reset policy eventually expires. A ``mode == \"none\"``\n+        session never expires (``_is_session_expired`` returns ``False``\n+        forever), so the watcher will never finalize it.\n+\n+        This distinction matters for the agent-cache idle sweep: deferring\n+        idle eviction to \"let the watcher finalize it later\" is only correct\n+        when the watcher WILL run for this session. For a ``mode == \"none\"``\n+        session, deferring pins the cached agent in memory for the gateway's\n+        entire lifetime with no finalization ever coming — the exact leak the\n+        idle sweep exists to relieve. Callers use this predicate to decide\n+        whether the session store owns the eviction boundary (finalizable) or\n+        the idle sweep must still reap the agent itself (not finalizable).\n+\n+        Public wrapper so callers don't reach into policy internals. Errors\n+        resolving the policy are treated as \"not finalizable\" (safe: the idle\n+        sweep falls back to reaping the agent rather than pinning it).\n+        \"\"\"\n+        try:\n+            policy = self.config.get_reset_policy(\n+                platform=entry.platform,\n+                session_type=entry.chat_type,\n+            )\n+            return policy.mode != \"none\"\n+        except Exception:\n+            return False\n+\n     def _is_session_ended_in_db(self, session_id: str) -> bool:\n         \"\"\"Return True iff state.db has this session with a non-null end_reason.\n "]},{"file":"tests/gateway/test_agent_cache.py","hunks":["@@ -675,6 +675,89 @@ def _soft(agent):\n         # Hard-cleanup path must NOT have fired — that's for session expiry only.\n         assert cleanup_calls == []\n \n+    def test_cap_commits_memory_before_evicting_finalizable(self, monkeypatch):\n+        \"\"\"LRU-cap eviction of a finalizable, not-yet-expired agent commits\n+        on_session_end extraction before releasing.\n+\n+        The agent would otherwise vanish from _agent_cache before the expiry\n+        watcher runs, so the watcher would never fire on_session_end() and\n+        memory providers would miss the transcript (#11205, LRU-cap variant).\n+        We hold the live agent at eviction time, so commit its memory then.\n+        \"\"\"\n+        from gateway import run as gw_run\n+\n+        monkeypatch.setattr(gw_run, \"_AGENT_CACHE_MAX_SIZE\", 1)\n+        runner = self._bounded_runner()\n+\n+        commit_calls: list = []\n+        release_calls: list = []\n+        runner._release_evicted_agent_soft = lambda agent: release_calls.append(agent)\n+\n+        # Finalizable (finite policy), not yet expired.\n+        runner.session_store = MagicMock()\n+        runner.session_store._entries = {\"old\": MagicMock(), \"new\": MagicMock()}\n+        runner.session_store.is_session_finalizable.return_value = True\n+        runner.session_store._is_session_expired.return_value = False\n+\n+        old_agent = self._fake_agent()\n+        old_agent._memory_manager = MagicMock()  # has an external provider\n+        old_agent._session_messages = [{\"role\": \"user\", \"content\": \"hi\"}]\n+        old_agent.commit_memory_session = lambda msgs=None: commit_calls.append(msgs)\n+        new_agent = self._fake_agent()\n+\n+        with runner._agent_cache_lock:\n+            runner._agent_cache[\"old\"] = (old_agent, \"sig_old\")\n+            runner._agent_cache[\"new\"] = (new_agent, \"sig_new\")\n+            runner._enforce_agent_cache_cap()\n+\n+        import time as _t\n+        deadline = _t.time() + 2.0\n+        while _t.time() < deadline and not release_calls:\n+            _t.sleep(0.02)\n+        # Memory committed with the live transcript, THEN client released.\n+        assert commit_calls == [[{\"role\": \"user\", \"content\": \"hi\"}]]\n+        assert old_agent in release_calls\n+\n+    def test_cap_skips_memory_commit_for_non_finalizable(self, monkeypatch):\n+        \"\"\"LRU-cap eviction of a mode='none' agent does NOT commit memory.\n+\n+        The expiry watcher never finalizes a mode='none' session, so there is\n+        no missed on_session_end boundary to compensate for. Committing here\n+        would fire premature/repeat extraction for a session that simply keeps\n+        living. The agent is released without a commit.\n+        \"\"\"\n+        from gateway import run as gw_run\n+\n+        monkeypatch.setattr(gw_run, \"_AGENT_CACHE_MAX_SIZE\", 1)\n+        runner = self._bounded_runner()\n+\n+        commit_calls: list = []\n+        release_calls: list = []\n+        runner._release_evicted_agent_soft = lambda agent: release_calls.append(agent)\n+\n+        runner.session_store = MagicMock()\n+        runner.session_store._entries = {\"old\": MagicMock(), \"new\": MagicMock()}\n+        runner.session_store.is_session_finalizable.return_value = False  # mode='none'\n+        runner.session_store._is_session_expired.return_value = False\n+\n+        old_agent = self._fake_agent()\n+        old_agent._memory_manager = MagicMock()\n+        old_agent._session_messages = [{\"role\": \"user\", \"content\": \"hi\"}]\n+        old_agent.commit_memory_session = lambda msgs=None: commit_calls.append(msgs)\n+        new_agent = self._fake_agent()\n+\n+        with runner._agent_cache_lock:\n+            runner._agent_cache[\"old\"] = (old_agent, \"sig_old\")\n+            runner._agent_cache[\"new\"] = (new_agent, \"sig_new\")\n+            runner._enforce_agent_cache_cap()\n+\n+        import time as _t\n+        deadline = _t.time() + 2.0\n+        while _t.time() < deadline and not release_calls:\n+            _t.sleep(0.02)\n+        assert commit_calls == []       # no premature extraction\n+        assert old_agent in release_calls  # still released\n+\n     def test_idle_ttl_sweep_evicts_stale_agents(self, monkeypatch):\n         \"\"\"_sweep_idle_cached_agents removes agents idle past the TTL.\"\"\"\n         from gateway import run as gw_run","@@ -708,6 +791,138 @@ def test_idle_sweep_skips_agents_without_activity_ts(self, monkeypatch):\n         assert runner._sweep_idle_cached_agents() == 0\n         assert \"s\" in runner._agent_cache\n \n+    def test_idle_sweep_keeps_agent_when_session_not_expired(self, monkeypatch):\n+        \"\"\"Agents past idle TTL are kept if the session hasn't expired yet.\n+\n+        In daily-reset mode the reset can fire hours after the last\n+        user message — evicting the agent early means the\n+        session-expiry watcher has nothing to call on_session_end()\n+        with, and memory providers miss the live transcript.\n+        \"\"\"\n+        from gateway import run as gw_run\n+\n+        monkeypatch.setattr(gw_run, \"_AGENT_CACHE_IDLE_TTL_SECS\", 0.01)\n+        runner = self._bounded_runner()\n+        runner._cleanup_agent_resources = MagicMock()\n+\n+        import time as _t\n+        stale = self._fake_agent(last_activity=_t.time() - 10.0)\n+\n+        # Session store says the session is still alive AND is finalizable\n+        # (finite reset policy) — so deferring eviction is correct: the expiry\n+        # watcher will find this agent later and fire on_session_end().\n+        session_entry = MagicMock()\n+        runner.session_store = MagicMock()\n+        runner.session_store._entries = {\"stale-session\": session_entry}\n+        runner.session_store.is_session_finalizable.return_value = True\n+        runner.session_store._is_session_expired.return_value = False\n+\n+        runner._agent_cache[\"stale-session\"] = (stale, \"sig\")\n+\n+        evicted = runner._sweep_idle_cached_agents()\n+        assert evicted == 0\n+        assert \"stale-session\" in runner._agent_cache\n+\n+    def test_idle_sweep_evicts_when_session_is_expired(self, monkeypatch):\n+        \"\"\"Agent IS evicted when past idle TTL AND session store says expired.\"\"\"\n+        from gateway import run as gw_run\n+\n+        monkeypatch.setattr(gw_run, \"_AGENT_CACHE_IDLE_TTL_SECS\", 0.01)\n+        runner = self._bounded_runner()\n+        runner._cleanup_agent_resources = MagicMock()\n+\n+        import time as _t\n+        stale = self._fake_agent(last_activity=_t.time() - 10.0)\n+\n+        # Session store says the session has expired.\n+        session_entry = MagicMock()\n+        runner.session_store = MagicMock()\n+        runner.session_store._entries = {\"stale-session\": session_entry}\n+        runner.session_store.is_session_finalizable.return_value = True\n+        runner.session_store._is_session_expired.return_value = True\n+\n+        runner._agent_cache[\"stale-session\"] = (stale, \"sig\")\n+\n+        evicted = runner._sweep_idle_cached_agents()\n+        assert evicted == 1\n+        assert \"stale-session\" not in runner._agent_cache\n+\n+    def test_idle_sweep_evicts_non_finalizable_session(self, monkeypatch):\n+        \"\"\"A mode='none' session's idle agent IS still evicted.\n+\n+        is_session_finalizable() is False for reset-policy 'none': the expiry\n+        watcher never finalizes such a session, so deferring eviction would\n+        pin the cached agent for the gateway's whole lifetime — the exact\n+        leak the idle sweep exists to relieve. The sweep must reap it even\n+        though _is_session_expired() is (and stays) False.\n+        \"\"\"\n+        from gateway import run as gw_run\n+\n+        monkeypatch.setattr(gw_run, \"_AGENT_CACHE_IDLE_TTL_SECS\", 0.01)\n+        runner = self._bounded_runner()\n+        runner._cleanup_agent_resources = MagicMock()\n+\n+        import time as _t\n+        stale = self._fake_agent(last_activity=_t.time() - 10.0)\n+\n+        session_entry = MagicMock()\n+        runner.session_store = MagicMock()\n+        runner.session_store._entries = {\"never-session\": session_entry}\n+        # mode='none' → never finalizable, never expired.\n+        runner.session_store.is_session_finalizable.return_value = False\n+        runner.session_store._is_session_expired.return_value = False\n+\n+        runner._agent_cache[\"never-session\"] = (stale, \"sig\")\n+\n+        evicted = runner._sweep_idle_cached_agents()\n+        assert evicted == 1\n+        assert \"never-session\" not in runner._agent_cache\n+\n+    def test_is_session_finalizable_real_predicate(self, tmp_path):\n+        \"\"\"is_session_finalizable() reflects the real reset policy.\n+\n+        Uses a real SessionStore + GatewayConfig (no mocks) so the predicate\n+        is exercised against actual get_reset_policy() output: True for finite\n+        policies (idle/daily/both), False only for mode='none'.\n+        \"\"\"\n+        from datetime import datetime\n+        from unittest.mock import patch as _patch\n+\n+        from gateway.config import GatewayConfig, Platform, SessionResetPolicy\n+        from gateway.session import (\n+            SessionEntry, SessionSource, SessionStore, build_session_key,\n+        )\n+\n+        def _entry_for(platform: Platform) -> SessionEntry:\n+            src = SessionSource(\n+                platform=platform, user_id=\"u1\", chat_id=\"c1\",\n+                user_name=\"t\", chat_type=\"dm\",\n+            )\n+            return SessionEntry(\n+                session_key=build_session_key(src),\n+                session_id=\"s1\",\n+                created_at=datetime.now(),\n+                updated_at=datetime.now(),\n+                origin=src,\n+                platform=src.platform,\n+                chat_type=src.chat_type,\n+            )\n+\n+        config = GatewayConfig()\n+        # Give Telegram a 'none' policy via the per-platform override; leave the\n+        # default policy finite ('both') for the Discord case.\n+        config.default_reset_policy = SessionResetPolicy(mode=\"both\")\n+        config.reset_by_platform[Platform.TELEGRAM] = SessionResetPolicy(mode=\"none\")\n+\n+        with _patch(\"gateway.session.SessionStore._ensure_loaded\"):\n+            store = SessionStore(sessions_dir=tmp_path, config=config)\n+        store._db = None\n+\n+        # mode='none' → never finalized by the watcher.\n+        assert store.is_session_finalizable(_entry_for(Platform.TELEGRAM)) is False\n+        # default 'both' → finite, will eventually expire.\n+        assert store.is_session_finalizable(_entry_for(Platform.DISCORD)) is True\n+\n     def test_plain_dict_cache_is_tolerated(self):\n         \"\"\"Test fixtures using plain {} don't crash _enforce_agent_cache_cap.\"\"\"\n         from gateway.run import GatewayRunner\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}