{"repo":"NousResearch/hermes-agent","public_id":"54878","display_label":"Stale sessions.json entries causing silently dropped messages","summary":"The members appear to center on issue #54878, where idle-TTL cache eviction ends a session in the database but may leave a stale sessions.json entry, with messages reportedly dropped until restart. Several members propose changes: removing the stale entry on eviction and self-healing stale routing at message time.","subsystem":"gateway","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":55485,"merged_at":"2026-06-30T07:47:52+00:00"},"resolution":{"state":"resolved","by":"pr-55485"},"freshness":{"state":"verified","last_verified_at":"2026-06-30T07:47:52+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":54878,"pr_number":55485,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-30T07:47:52+00:00"}],"members":[{"kind":"issue","number":54878,"title":"[Bug]: idle agent cache eviction ends session in DB but leaves sessions.json stale → messages silently dropped until restart","url":null,"state":"closed"},{"kind":"pr","number":54916,"title":"Fix #54878: Prevent silent message loss due to stale sessions.json entries","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3a83b6bc5","number":0},{"kind":"pr","value":"#55485","number":55485}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":55232,"title":"fix: remove stale session entry from sessions.json on idle-TTL eviction","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":55485,"title":"fix(gateway): self-heal stale sessions.json routing at message time (#54878)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/session.py","tests/gateway/test_session_store_runtime_stale_guard.py"],"fix_hunks":[{"file":"gateway/session.py","hunks":["@@ -905,11 +905,11 @@ def _ensure_loaded_locked(self) -> None:\n         # in state.db. A hard gateway crash (exit code 1) skips the graceful\n         # shutdown path, so sessions.json is never cleared and is left pointing\n         # at ended sessions. On the next startup those stale entries act as live\n-        # routing keys, but get_or_create_session() reuses them as long as the\n-        # time/policy reset checks pass — it never consults end_reason — so every\n-        # incoming message is silently routed into a closed session. Pruning here\n-        # (lock already held) is cheap: one lookup per routing key, once at\n-        # startup, and self-heals into a fresh session on the next message.\n+        # routing keys. get_or_create_session() only consulted end_reason at\n+        # startup (here) until #54878 added a routing-time guard for the\n+        # live-gateway case; this startup prune still self-heals crash-left\n+        # entries before the first message arrives. Pruning here (lock already\n+        # held) is cheap: one lookup per routing key, once at startup.\n         self._prune_stale_sessions_locked()\n \n     def _prune_stale_sessions_locked(self) -> None:","@@ -1148,6 +1148,32 @@ def _is_session_expired(self, entry: SessionEntry) -> bool:\n \n         return False\n \n+    def _is_session_ended_in_db(self, session_id: str) -> bool:\n+        \"\"\"Return True iff state.db has this session with a non-null end_reason.\n+\n+        Mirrors the staleness test in ``_prune_stale_sessions_locked``:\n+          - no DB handle / no session_id -> False (can't tell — keep)\n+          - row absent (legacy / not yet persisted) -> False (keep)\n+          - end_reason is None -> False (alive — keep)\n+          - end_reason not None -> True (ended — stale)\n+\n+        Used by ``get_or_create_session`` to self-heal at routing time:\n+        ``_prune_stale_sessions_locked`` only runs at startup, so a session\n+        ended in the DB while the gateway stays alive (any path that finalizes\n+        the row without clearing sessions.json) would otherwise be reused as a\n+        live routing key and silently swallow every subsequent message until\n+        the next restart (#54878 — the live-gateway variant of #52804/FM9).\n+        DB errors are non-fatal — never block routing on a failed lookup.\n+        \"\"\"\n+        db = getattr(self, \"_db\", None)\n+        if not db or not session_id:\n+            return False\n+        try:\n+            row = db.get_session(session_id)\n+        except Exception:\n+            return False\n+        return bool(row is not None and row.get(\"end_reason\") is not None)\n+\n     def _should_reset(self, entry: SessionEntry, source: SessionSource) -> Optional[str]:\n         \"\"\"\n         Check if a session should be reset based on policy.","@@ -1243,39 +1269,73 @@ def get_or_create_session(\n             if session_key in self._entries and not force_new:\n                 entry = self._entries[session_key]\n \n-                # Auto-reset sessions marked as suspended (e.g. after /stop\n-                # broke a stuck loop — #7536).  ``suspended`` is the hard\n-                # forced-wipe signal and always wins over ``resume_pending``,\n-                # so repeated interrupted restarts that escalate via the\n-                # existing ``.restart_failure_counts`` stuck-loop counter\n-                # still converge to a clean slate.\n-                if entry.suspended:\n-                    reset_reason = \"suspended\"\n-                elif entry.resume_pending:\n-                    # Restart-interrupted session: preserve the session_id\n-                    # and return the existing entry so the transcript reloads\n-                    # intact, but still honour normal daily/idle reset policy.\n-                    reset_reason = self._should_reset(entry, source)\n+                # Self-heal stale routing: if this session_key still points at\n+                # a session that has ALREADY been ended in state.db (end_reason\n+                # set), the in-memory sessions.json entry is stale.  Reusing it\n+                # would route every incoming message into a closed session and\n+                # silently drop it — with no log, no error, no response — until\n+                # the gateway restarts and _prune_stale_sessions_locked() clears\n+                # it (#54878 — the live-gateway variant of #52804/FM9, which\n+                # only the startup prune previously caught).\n+                #\n+                # Drop the stale entry and fall through to the recovery path\n+                # below.  Leaving db_end_session_id None routes us into\n+                # _recover_session_from_db, whose finder\n+                # (hermes_state.find_latest_gateway_session_for_peer) selects\n+                # rows WHERE `ended_at IS NULL OR end_reason = 'agent_close'`\n+                # — so it REOPENS gateway-cleanup-ended ('agent_close') rows and\n+                # resumes the SAME session_id (transcript preserved), but returns\n+                # None for any other end_reason (e.g. /new), which then correctly\n+                # starts a fresh session.\n+                if self._is_session_ended_in_db(entry.session_id):\n+                    logger.warning(\n+                        \"gateway.session: routing key %r -> %s is ended in \"\n+                        \"state.db but still live in sessions.json; dropping \"\n+                        \"stale entry and recovering/recreating the session \"\n+                        \"(#54878)\",\n+                        session_key, entry.session_id,\n+                    )\n+                    self._entries.pop(session_key, None)\n+                    was_auto_reset = False\n+                    auto_reset_reason = None\n+                    reset_had_activity = False\n+                    # Fall through to the recovery/create path below; the\n+                    # stale entry is gone so we must NOT consult its\n+                    # suspended/resume/reset state.\n+                else:\n+                    # Auto-reset sessions marked as suspended (e.g. after /stop\n+                    # broke a stuck loop — #7536).  ``suspended`` is the hard\n+                    # forced-wipe signal and always wins over ``resume_pending``,\n+                    # so repeated interrupted restarts that escalate via the\n+                    # existing ``.restart_failure_counts`` stuck-loop counter\n+                    # still converge to a clean slate.\n+                    if entry.suspended:\n+                        reset_reason = \"suspended\"\n+                    elif entry.resume_pending:\n+                        # Restart-interrupted session: preserve the session_id\n+                        # and return the existing entry so the transcript reloads\n+                        # intact, but still honour normal daily/idle reset policy.\n+                        reset_reason = self._should_reset(entry, source)\n+                        if not reset_reason:\n+                            entry.updated_at = now\n+                            self._save()\n+                            return entry\n+                    else:\n+                        reset_reason = self._should_reset(entry, source)\n                     if not reset_reason:\n                         entry.updated_at = now\n                         self._save()\n                         return entry\n-                else:\n-                    reset_reason = self._should_reset(entry, source)\n-                if not reset_reason:\n-                    entry.updated_at = now\n-                    self._save()\n-                    return entry\n-                else:\n-                    # Session is being auto-reset.\n-                    was_auto_reset = True\n-                    auto_reset_reason = reset_reason\n-                    # Track whether the expired session had any real conversation.\n-                    # total_tokens is never written (token counts migrated to\n-                    # agent-direct persistence) so it is always 0 — use\n-                    # last_prompt_tokens, which is updated on every turn.\n-                    reset_had_activity = entry.last_prompt_tokens > 0\n-                    db_end_session_id = entry.session_id\n+                    else:\n+                        # Session is being auto-reset.\n+                        was_auto_reset = True\n+                        auto_reset_reason = reset_reason\n+                        # Track whether the expired session had any real\n+                        # conversation.  total_tokens is never written (token\n+                        # counts migrated to agent-direct persistence) so it is\n+                        # always 0 — use last_prompt_tokens, updated every turn.\n+                        reset_had_activity = entry.last_prompt_tokens > 0\n+                        db_end_session_id = entry.session_id\n             else:\n                 was_auto_reset = False\n                 auto_reset_reason = None"]},{"file":"tests/gateway/test_session_store_runtime_stale_guard.py","hunks":["@@ -0,0 +1,200 @@\n+\"\"\"Runtime self-heal for stale sessions.json routing entries (#54878).\n+\n+`_prune_stale_sessions_locked` only runs at gateway startup. A session ended\n+in state.db while the gateway stays alive (e.g. any path that finalizes the\n+row without clearing sessions.json) leaves a stale `session_key -> session_id`\n+mapping whose session has `end_reason` set. Before this fix,\n+`get_or_create_session` returned that stale entry as a live routing key (it\n+never consulted end_reason), so every subsequent message was silently routed\n+into a closed session and dropped — no log, no error, no response — until the\n+next restart pruned it.\n+\n+This is the live-gateway variant of #52804/FM9 (#52808/#54138 startup prune),\n+which required an actual gateway *crash*. Here the guard inside\n+`get_or_create_session` detects the ended row at routing time and drops the\n+stale entry, falling through to `_recover_session_from_db` (which reopens\n+`agent_close`-ended rows and resumes the SAME session_id, preserving the\n+transcript) or, failing recovery, to a fresh session.\n+\"\"\"\n+\n+from datetime import datetime, timedelta\n+from unittest.mock import MagicMock, patch\n+\n+from gateway.config import GatewayConfig, Platform, SessionResetPolicy\n+from gateway.session import SessionEntry, SessionSource, SessionStore\n+\n+\n+# ---------------------------------------------------------------------------\n+# Helpers\n+# ---------------------------------------------------------------------------\n+\n+def _make_entry(key: str, session_id: str, **kw) -> SessionEntry:\n+    now = datetime.now()\n+    return SessionEntry(\n+        session_key=key,\n+        session_id=session_id,\n+        created_at=now - timedelta(hours=2),\n+        updated_at=now - timedelta(hours=1),\n+        platform=Platform.TELEGRAM,\n+        chat_type=\"dm\",\n+        **kw,\n+    )\n+\n+\n+def _db_returning(rows: dict) -> MagicMock:\n+    \"\"\"SessionDB mock where get_session maps session_id -> row dict.\"\"\"\n+    db = MagicMock()\n+    db.get_session.side_effect = lambda sid: rows.get(sid)\n+    # By default recovery finds nothing (forces a fresh session).\n+    db.find_latest_gateway_session_for_peer.return_value = None\n+    db.reopen_session.return_value = None\n+    db.create_session.return_value = None\n+    return db\n+\n+\n+def _make_store_with_db(tmp_path, db_mock) -> SessionStore:\n+    \"\"\"Build a SessionStore with a mock SessionDB, bypassing disk load.\"\"\"\n+    config = GatewayConfig(default_reset_policy=SessionResetPolicy(mode=\"none\"))\n+    with patch(\"gateway.session.SessionStore._ensure_loaded\"):\n+        store = SessionStore(sessions_dir=tmp_path, config=config)\n+    store._db = db_mock\n+    store._loaded = True\n+    return store\n+\n+\n+def _source() -> SessionSource:\n+    # session_key for this peer is deterministic; matches the entry key we seed.\n+    return SessionSource(\n+        platform=Platform.TELEGRAM,\n+        chat_id=\"8494508720\",\n+        chat_type=\"dm\",\n+        user_id=\"8494508720\",\n+    )\n+\n+\n+# ---------------------------------------------------------------------------\n+# _is_session_ended_in_db helper\n+# ---------------------------------------------------------------------------\n+\n+class TestIsSessionEndedInDb:\n+    def test_ended_row_is_stale(self, tmp_path):\n+        db = _db_returning({\"sid\": {\"end_reason\": \"agent_close\", \"id\": \"sid\"}})\n+        store = _make_store_with_db(tmp_path, db)\n+        assert store._is_session_ended_in_db(\"sid\") is True\n+\n+    def test_alive_row_not_stale(self, tmp_path):\n+        db = _db_returning({\"sid\": {\"end_reason\": None, \"id\": \"sid\"}})\n+        store = _make_store_with_db(tmp_path, db)\n+        assert store._is_session_ended_in_db(\"sid\") is False\n+\n+    def test_absent_row_not_stale(self, tmp_path):\n+        # Not yet persisted / legacy — must NOT be treated as ended, else a\n+        # freshly-created in-memory session would be wrongly discarded.\n+        db = _db_returning({})\n+        store = _make_store_with_db(tmp_path, db)\n+        assert store._is_session_ended_in_db(\"sid_absent\") is False\n+\n+    def test_no_db_not_stale(self, tmp_path):\n+        store = _make_store_with_db(tmp_path, _db_returning({}))\n+        store._db = None\n+        assert store._is_session_ended_in_db(\"sid\") is False\n+\n+    def test_empty_session_id_not_stale(self, tmp_path):\n+        store = _make_store_with_db(tmp_path, _db_returning({}))\n+        assert store._is_session_ended_in_db(\"\") is False\n+\n+    def test_db_error_not_stale(self, tmp_path):\n+        db = MagicMock()\n+        db.get_session.side_effect = Exception(\"DB locked\")\n+        store = _make_store_with_db(tmp_path, db)\n+        # On error, never block routing — treat as not-stale (keep).\n+        assert store._is_session_ended_in_db(\"sid\") is False\n+\n+\n+# ---------------------------------------------------------------------------\n+# get_or_create_session — runtime self-heal\n+# ---------------------------------------------------------------------------\n+\n+class TestRuntimeStaleGuard:\n+    def test_stale_agent_close_entry_recovered_preserving_session_id(self, tmp_path):\n+        \"\"\"Stale `agent_close` entry → recovery reopens the SAME session_id.\"\"\"\n+        source = _source()\n+        db = _db_returning({\"sid_stale\": {\"end_reason\": \"agent_close\", \"id\": \"sid_stale\"}})\n+        # Recovery finds the agent_close row and reopens it (transcript-preserving).\n+        db.find_latest_gateway_session_for_peer.return_value = {\n+            \"id\": \"sid_stale\",\n+            \"started_at\": (datetime.now() - timedelta(hours=2)).timestamp(),\n+        }\n+        store = _make_store_with_db(tmp_path, db)\n+        key = store._generate_session_key(source)\n+        store._entries[key] = _make_entry(key, \"sid_stale\")\n+\n+        result = store.get_or_create_session(source)\n+\n+        # SAME session_id (resumed), not a brand-new one, and not silently\n+        # routed into the closed entry.\n+        assert result.session_id == \"sid_stale\"\n+        db.reopen_session.assert_called_once_with(\"sid_stale\")\n+        # A brand-new session row must NOT have been created.\n+        db.create_session.assert_not_called()\n+\n+    def test_stale_entry_creates_fresh_when_recovery_returns_none(self, tmp_path):\n+        \"\"\"Stale entry, no recoverable row → brand-new session (no silent drop).\"\"\"\n+        source = _source()\n+        # Ended with a non-recoverable reason (e.g. /new) → finder returns None.\n+        db = _db_returning({\"sid_stale\": {\"end_reason\": \"new_command\", \"id\": \"sid_stale\"}})\n+        db.find_latest_gateway_session_for_peer.return_value = None\n+        store = _make_store_with_db(tmp_path, db)\n+        key = store._generate_session_key(source)\n+        store._entries[key] = _make_entry(key, \"sid_stale\")\n+\n+        result = store.get_or_create_session(source)\n+\n+        assert result.session_id != \"sid_stale\"\n+        # A fresh session row was created for the new session_id.\n+        db.create_session.assert_called_once()\n+        assert store._entries[key].session_id == result.session_id\n+\n+    def test_live_entry_returned_unchanged(self, tmp_path):\n+        \"\"\"A session still alive in the DB is returned as-is (no churn).\"\"\"\n+        source = _source()\n+        db = _db_returning({\"sid_live\": {\"end_reason\": None, \"id\": \"sid_live\"}})\n+        store = _make_store_with_db(tmp_path, db)\n+        key = store._generate_session_key(source)\n+        store._entries[key] = _make_entry(key, \"sid_live\")\n+\n+        result = store.get_or_create_session(source)\n+\n+        assert result.session_id == \"sid_live\"\n+        db.find_latest_gateway_session_for_peer.assert_not_called()\n+        db.create_session.assert_not_called()\n+\n+    def test_stale_check_wins_over_suspended(self, tmp_path):\n+        \"\"\"A stale entry that is ALSO suspended is still dropped via the stale\n+        path — we must not consult the dead entry's reset/suspend state.\"\"\"\n+        source = _source()\n+        db = _db_returning({\"sid_stale\": {\"end_reason\": \"agent_close\", \"id\": \"sid_stale\"}})\n+        db.find_latest_gateway_session_for_peer.return_value = None  # → fresh\n+        store = _make_store_with_db(tmp_path, db)\n+        key = store._generate_session_key(source)\n+        store._entries[key] = _make_entry(key, \"sid_stale\", suspended=True)\n+\n+        result = store.get_or_create_session(source)\n+\n+        # Did not return the stale (suspended) entry; created a fresh session.\n+        assert result.session_id != \"sid_stale\"\n+        db.create_session.assert_called_once()\n+\n+    def test_force_new_skips_stale_check(self, tmp_path):\n+        \"\"\"force_new short-circuits the whole existing-entry branch; the stale\n+        DB lookup must not even run.\"\"\"\n+        source = _source()\n+        db = _db_returning({\"sid_old\": {\"end_reason\": \"agent_close\", \"id\": \"sid_old\"}})\n+        store = _make_store_with_db(tmp_path, db)\n+        key = store._generate_session_key(source)\n+        store._entries[key] = _make_entry(key, \"sid_old\")\n+\n+        result = store.get_or_create_session(source, force_new=True)\n+\n+        assert result.session_id != \"sid_old\"\n+        db.get_session.assert_not_called()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}