{"repo":"NousResearch/hermes-agent","public_id":"13121","display_label":"Gateway restart transcript persistence and resume context loss","summary":"The members concern the gateway losing or not preserving recent transcript context across restarts or shutdown drain timeouts. One issue reports possible context loss on resume with a suspected JSONL vs SQLite mismatch, and two pull requests propose persisting the in-flight transcript and merging the transcript tail on resume.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":50312,"merged_at":"2026-06-21T18:57:15+00:00"},"resolution":{"state":"resolved","by":"pr-50312"},"freshness":{"state":"verified","last_verified_at":"2026-06-21T18:57:15+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":13121,"pr_number":50312,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-21T18:57:15+00:00"}],"members":[{"kind":"issue","number":13121,"title":"Gateway restart resume can lose immediate pre-restart context (possible JSONL vs SQLite transcript mismatch)","url":null,"state":"closed"},{"kind":"pr","number":14400,"title":"fix(gateway): merge restart resume transcript tail","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#13121","number":13121}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":50312,"title":"fix(gateway): persist in-flight transcript on restart/shutdown drain timeout","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_13121_shutdown_inflight_transcript_flush.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -4674,6 +4674,40 @@ async def _notify_active_sessions_of_shutdown(self) -> None:\n \n     def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:\n         for agent in active_agents.values():\n+            # Persist any in-flight transcript to the SQLite session store\n+            # before teardown (#13121).  An agent forcibly interrupted by the\n+            # drain-timeout escalation may never reach\n+            # ``turn_finalizer.finalize_turn`` (the only place that flushes the\n+            # turn to state.db) — e.g. it was blocked in a tool call that did\n+            # not abort within the post-interrupt grace window.  Its in-flight\n+            # tool rounds live only in the in-memory ``_session_messages``\n+            # (refreshed per tool round in ``conversation_loop`` but never\n+            # written to SQLite mid-turn), so the immediate pre-restart turn is\n+            # silently dropped from ``load_transcript()`` on resume.  Flushing\n+            # here closes that gap; the resume_pending / fresh-tool-tail\n+            # branches in ``_handle_message_with_agent`` already expect a\n+            # transcript whose tail may be a pending tool result.  The flush is\n+            # idempotent (identity-tracked in ``_flush_messages_to_session_db``),\n+            # so agents that DID finish gracefully re-flush nothing.\n+            try:\n+                _flush = getattr(agent, \"_flush_messages_to_session_db\", None)\n+                _session_messages = getattr(agent, \"_session_messages\", None)\n+                if callable(_flush) and isinstance(_session_messages, list) and _session_messages:\n+                    # Strip private empty-response retry scaffolding from the\n+                    # tail first, mirroring the graceful ``_persist_session``\n+                    # path, so a resumed turn doesn't replay synthetic recovery\n+                    # nudges.\n+                    _strip = getattr(\n+                        agent, \"_drop_trailing_empty_response_scaffolding\", None\n+                    )\n+                    if callable(_strip):\n+                        try:\n+                            _strip(_session_messages)\n+                        except Exception:\n+                            pass\n+                    _flush(_session_messages)\n+            except Exception as _e:\n+                logger.debug(\"Shutdown transcript flush failed: %s\", _e)\n             try:\n                 from hermes_cli.plugins import invoke_hook as _invoke_hook\n                 _invoke_hook("]},{"file":"tests/gateway/test_13121_shutdown_inflight_transcript_flush.py","hunks":["@@ -0,0 +1,243 @@\n+\"\"\"Regression tests for #13121 — gateway restart/shutdown must persist an\n+in-flight (interrupted) turn's transcript to the SQLite session store so the\n+immediate pre-restart context survives ``load_transcript()`` on resume.\n+\n+The bug: every normal/graceful turn exit funnels through\n+``turn_finalizer.finalize_turn`` which calls ``_persist_session`` →\n+``_flush_messages_to_session_db`` (the only place a turn is written to\n+state.db).  During the tool loop only the *in-memory* ``_session_messages``\n+reference is refreshed per round — there is no incremental SQLite flush\n+mid-turn.\n+\n+When the gateway drain times out it marks the session ``resume_pending``,\n+interrupts the running agents, waits a short grace window, then tears them\n+down via ``_finalize_shutdown_agents`` → ``_cleanup_agent_resources``.  An\n+agent blocked in a tool call that does not abort within the grace window\n+never reaches ``finalize_turn``, so its in-flight tool rounds live only in\n+``_session_messages`` and are never written to state.db.  On resume,\n+``load_transcript()`` (state.db is now the canonical store — the legacy\n+JSONL fallback was dropped) returns the pre-turn state, dropping the\n+immediate pre-restart turn.\n+\n+The fix flushes ``_session_messages`` to the session DB in\n+``_finalize_shutdown_agents`` before teardown.  The flush is idempotent\n+(identity-tracked in ``_flush_messages_to_session_db``), so agents that DID\n+finish gracefully re-flush nothing.\n+\n+These tests exercise BOTH a lightweight unit path (the flush hook is invoked\n+with the in-flight messages) AND a true E2E path (a real ``AIAgent`` flush\n+against a real ``SessionDB`` in a temp ``HERMES_HOME``, read back through the\n+real ``SessionStore.load_transcript``).\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import sys\n+import types\n+from unittest.mock import MagicMock\n+\n+import pytest\n+\n+\n[redacted:email](autouse=True)\n+def _mock_dotenv(monkeypatch):\n+    \"\"\"gateway.run imports dotenv at module load; stub so tests run bare.\"\"\"\n+    fake = types.ModuleType(\"dotenv\")\n+    fake.load_dotenv = lambda *a, **kw: None\n+    monkeypatch.setitem(sys.modules, \"dotenv\", fake)\n+\n+\n+def _make_runner():\n+    from gateway.run import GatewayRunner\n+\n+    runner = object.__new__(GatewayRunner)\n+    return runner\n+\n+\n+# ─────────────────────────────────────────────────────────────────────────\n+# Unit: _finalize_shutdown_agents calls the flush hook with the in-flight\n+# transcript before teardown.\n+# ─────────────────────────────────────────────────────────────────────────\n+class _FakeAgent:\n+    def __init__(self, session_messages=None, has_flush=True):\n+        if session_messages is not None:\n+            self._session_messages = session_messages\n+        if has_flush:\n+            self._flush_messages_to_session_db = MagicMock()\n+            self._drop_trailing_empty_response_scaffolding = MagicMock()\n+        self.shutdown_memory_provider = MagicMock()\n+        self.close = MagicMock()\n+        self.session_id = \"sess-1\"\n+\n+\n+class TestFinalizeShutdownFlushesInflightTranscript:\n+    def test_inflight_messages_flushed_before_teardown(self):\n+        \"\"\"The mid-turn transcript (tail = pending tool result) is flushed\n+        to the session DB during shutdown finalization.\"\"\"\n+        runner = _make_runner()\n+        inflight = [\n+            {\"role\": \"user\", \"content\": \"scan the repo and summarise\"},\n+            {\"role\": \"assistant\", \"content\": \"\", \"tool_calls\": [\n+                {\"id\": \"c1\", \"function\": {\"name\": \"terminal\", \"arguments\": \"{}\"}}\n+            ]},\n+            {\"role\": \"tool\", \"tool_call_id\": \"c1\", \"content\": \"huge output...\"},\n+        ]\n+        agent = _FakeAgent(session_messages=inflight)\n+\n+        runner._finalize_shutdown_agents({\"agent:main:discord:dm:42\": agent})\n+\n+        agent._flush_messages_to_session_db.assert_called_once_with(inflight)\n+        # Cleanup still happens after the flush.\n+        agent.close.assert_called_once()\n+\n+    def test_empty_session_messages_not_flushed(self):\n+        \"\"\"An agent that ran no turns (empty list) triggers no flush — there\n+        is nothing in flight to persist.\"\"\"\n+        runner = _make_runner()\n+        agent = _FakeAgent(session_messages=[])\n+\n+        runner._finalize_shutdown_agents({\"k\": agent})\n+\n+        agent._flush_messages_to_session_db.assert_not_called()\n+        agent.close.assert_called_once()\n+\n+    def test_missing_flush_method_is_tolerated(self):\n+        \"\"\"A stub agent without the flush method (object.__new__ test stubs)\n+        must not break shutdown — teardown still runs.\"\"\"\n+        runner = _make_runner()\n+        agent = _FakeAgent(session_messages=[{\"role\": \"user\", \"content\": \"x\"}],\n+                           has_flush=False)\n+\n+        runner._finalize_shutdown_agents({\"k\": agent})\n+\n+        agent.close.assert_called_once()\n+\n+    def test_flush_exception_is_swallowed(self):\n+        \"\"\"A raising flush must not prevent teardown — a transcript-flush\n+        failure is best-effort, losing tool resources is worse.\"\"\"\n+        runner = _make_runner()\n+        agent = _FakeAgent(session_messages=[{\"role\": \"user\", \"content\": \"x\"}])\n+        agent._flush_messages_to_session_db.side_effect = RuntimeError(\"db locked\")\n+\n+        runner._finalize_shutdown_agents({\"k\": agent})\n+\n+        agent.close.assert_called_once()\n+\n+\n+# ─────────────────────────────────────────────────────────────────────────\n+# E2E: real AIAgent flush → real SessionDB → real load_transcript.\n+# ─────────────────────────────────────────────────────────────────────────\n+class TestShutdownTranscriptSurvivesResumeE2E:\n+    def test_interrupted_turn_persisted_and_readable_on_resume(self, tmp_path, monkeypatch):\n+        \"\"\"Drive the real flush path against a real SessionDB and confirm the\n+        in-flight turn is readable back through SessionStore.load_transcript —\n+        the exact path the resume logic reads on the next message.\"\"\"\n+        # Isolated state.db.\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n+\n+        from hermes_state import SessionDB\n+        from run_agent import AIAgent\n+\n+        db = SessionDB(db_path=tmp_path / \"state.db\")\n+        session_id = \"sess-e2e-13121\"\n+        db.create_session(session_id=session_id, source=\"discord\")\n+\n+        # Simulate a session whose FIRST turn completed and was persisted...\n+        db.append_message(session_id=session_id, role=\"user\",\n+                          content=\"hello, remember my cat is Mochi\")\n+        db.append_message(session_id=session_id, role=\"assistant\",\n+                          content=\"Noted — Mochi the cat.\")\n+\n+        # ...and a SECOND turn that was interrupted mid tool-loop. These rows\n+        # were NEVER flushed to the DB (only live in _session_messages).\n+        prior_history = [\n+            {\"role\": \"user\", \"content\": \"hello, remember my cat is Mochi\"},\n+            {\"role\": \"assistant\", \"content\": \"Noted — Mochi the cat.\"},\n+        ]\n+        inflight_tail = [\n+            {\"role\": \"user\", \"content\": \"now scan the whole repo for TODOs\"},\n+            {\"role\": \"assistant\", \"content\": \"\", \"tool_calls\": [\n+                {\"id\": \"tc1\", \"function\": {\"name\": \"terminal\",\n+                                           \"arguments\": \"{\\\"command\\\": \\\"grep -r TODO\\\"}\"}}\n+            ]},\n+            {\"role\": \"tool\", \"tool_call_id\": \"tc1\", \"name\": \"terminal\",\n+             \"content\": \"src/a.py: TODO fix this\\nsrc/b.py: TODO and that\"},\n+        ]\n+        # _session_messages is the live list: history copy + in-flight tail.\n+        session_messages = list(prior_history) + list(inflight_tail)\n+\n+        # Build a real AIAgent shaped only with what the flush path reads.\n+        agent = object.__new__(AIAgent)\n+        agent._session_db = db\n+        agent._session_db_created = True\n+        agent.session_id = session_id\n+        agent.platform = \"discord\"\n+        agent._session_messages = session_messages\n+        # Model a real agent: turn 1 already flushed, so its message identities\n+        # are recorded in the dedup set. Only the in-flight turn-2 tail is new.\n+        agent._last_flushed_db_idx = len(prior_history)\n+        agent._flushed_db_message_ids = {id(m) for m in prior_history}\n+        agent._flushed_db_message_session_id = session_id\n+\n+        # Sanity: only the 2 first-turn rows are in the DB before shutdown.\n+        before = db.get_messages_as_conversation(session_id)\n+        assert len(before) == 2, before\n+\n+        # Drive the gateway shutdown finalization with this real agent.\n+        from gateway.run import GatewayRunner\n+        runner = object.__new__(GatewayRunner)\n+        runner._finalize_shutdown_agents({\"agent:main:discord:dm:7\": agent})\n+\n+        # The in-flight turn must now be durable and readable via the SAME\n+        # path the resume logic uses (SessionStore.load_transcript → DB).\n+        after = db.get_messages_as_conversation(session_id)\n+        roles = [m.get(\"role\") for m in after]\n+        contents = [m.get(\"content\") for m in after]\n+\n+        assert len(after) == 5, after\n+        # The interrupted user message survived.\n+        assert any(\"scan the whole repo for TODOs\" in (c or \"\") for c in contents), contents\n+        # The pending tool result (the immediate pre-restart context) survived.\n+        assert any(\"TODO fix this\" in (c or \"\") for c in contents), contents\n+        # Tail is a tool result — exactly what the _has_fresh_tool_tail resume\n+        # branch in _handle_message_with_agent expects to handle.\n+        assert roles[-1] == \"tool\", roles\n+\n+    def test_graceful_agent_reflush_is_idempotent(self, tmp_path, monkeypatch):\n+        \"\"\"An agent that already flushed via finalize_turn must not produce\n+        duplicate rows when _finalize_shutdown_agents re-flushes.\"\"\"\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path / \".hermes\"))\n+\n+        from hermes_state import SessionDB\n+        from run_agent import AIAgent\n+\n+        db = SessionDB(db_path=tmp_path / \"state.db\")\n+        session_id = \"sess-e2e-idem\"\n+        db.create_session(session_id=session_id, source=\"discord\")\n+\n+        msgs = [\n+            {\"role\": \"user\", \"content\": \"what is 2+2\"},\n+            {\"role\": \"assistant\", \"content\": \"4\"},\n+        ]\n+\n+        agent = object.__new__(AIAgent)\n+        agent._session_db = db\n+        agent._session_db_created = True\n+        agent.session_id = session_id\n+        agent.platform = \"discord\"\n+        agent._session_messages = msgs\n+        agent._last_flushed_db_idx = 0\n+        agent._flushed_db_message_ids = set()\n+        agent._flushed_db_message_session_id = None\n+\n+        # First flush (simulating finalize_turn).\n+        agent._flush_messages_to_session_db(msgs)\n+        assert len(db.get_messages_as_conversation(session_id)) == 2\n+\n+        # Shutdown re-flush of the SAME list identity must add nothing.\n+        from gateway.run import GatewayRunner\n+        runner = object.__new__(GatewayRunner)\n+        runner._finalize_shutdown_agents({\"k\": agent})\n+\n+        after = db.get_messages_as_conversation(session_id)\n+        assert len(after) == 2, after\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}