{"repo":"NousResearch/hermes-agent","public_id":"44795","display_label":"state.db WAL TRUNCATE checkpoint corruption and PASSIVE checkpoint changes","summary":"Members center on state.db B-tree corruption reported in connection with WAL TRUNCATE checkpoints, especially on large FTS5 databases. Most are pull requests proposing PASSIVE checkpoints in periodic, close, and pre-VACUUM paths, while some also address logging or surfacing checkpoint failures that were silently swallowed.","subsystem":"agent","counts":{"members":11,"open_issues":0,"open_issues_distinct":0,"open_prs":2,"closed_issues":2,"merged_prs":2,"attempt_prs":7},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":84881,"merged_at":"2026-08-13T02:43:23+00:00"},"resolution":{"state":"resolved","by":"pr-64607,pr-84881"},"freshness":{"state":"verified","last_verified_at":"2026-08-13T02:43:23+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.17018181818181818},"fix_edges":[{"issue_number":44795,"pr_number":84881,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-08-13T02:43:23+00:00"},{"issue_number":45383,"pr_number":64607,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-07-15T11:55:46+00:00"},{"issue_number":80255,"pr_number":84881,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-08-13T02:43:23+00:00"}],"members":[{"kind":"issue","number":44795,"title":"[Bug]: _try_wal_checkpoint TRUNCATE silently swallows exceptions, corrupts state.db WAL to zero bytes","url":null,"state":"closed"},{"kind":"issue","number":45383,"title":"state.db recurring B-tree corruption from WAL TRUNCATE checkpoint on large FTS5 databases","url":null,"state":"closed"},{"kind":"pr","number":44834,"title":"fix(state): log WAL checkpoint failures instead of silently swallowing","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":44835,"title":"fix(state): surface failed WAL checkpoints instead of silently swallowing them","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#44834","number":44834},{"kind":"pr","value":"#44795","number":44795}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":44917,"title":"fix(state): use PASSIVE WAL checkpoint to prevent corruption (#44795)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":45396,"title":"fix(state): use PASSIVE checkpoint for periodic WAL flush to prevent B-tree corruption","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"46b2afc56","number":0},{"kind":"commit","value":"9aba95b05","number":0},{"kind":"pr","value":"#45383","number":45383},{"kind":"pr","value":"#45546","number":45546}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":45546,"title":"fix: prevent state.db B-tree corruption from WAL TRUNCATE checkpoint","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#45383","number":45383}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":64607,"title":"fix(state): PASSIVE periodic WAL checkpoint to stop B-tree corruption (#45383)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_state.py","tests/test_wal_checkpoint_strategy.py"],"fix_hunks":[{"file":"hermes_state.py","hunks":["@@ -986,7 +986,7 @@ class SessionDB:\n     _WRITE_MAX_RETRIES = 15\n     _WRITE_RETRY_MIN_S = 0.020   # 20ms\n     _WRITE_RETRY_MAX_S = 0.150   # 150ms\n-    # Attempt a PASSIVE WAL checkpoint every N successful writes.\n+    # Attempt a WAL checkpoint every N successful writes (PASSIVE mode).\n     _CHECKPOINT_EVERY_N_WRITES = 50\n     # Merge fragmented FTS5 segments every N successful writes. The message\n     # triggers append one segment per insert; left unmaintained these grow","@@ -1301,35 +1301,34 @@ def _execute_write(self, fn: Callable[[sqlite3.Connection], T]) -> T:\n         )\n \n     def _try_wal_checkpoint(self) -> None:\n-        \"\"\"Best-effort TRUNCATE WAL checkpoint.  Never raises.\n-\n-        Flushes committed WAL frames back into the main DB file and\n-        truncates the WAL file to zero bytes.  Keeps the WAL from\n-        growing unbounded when many processes hold persistent\n-        connections.\n-\n-        PASSIVE checkpoint was previously used here, but it never\n-        truncates the WAL file — the file stays at its high-water\n-        mark until an explicit TRUNCATE is called (which only\n-        happened inside the infrequent vacuum()).\n-\n-        TRUNCATE may block writers briefly while checkpointing, but\n-        _try_wal_checkpoint is called off the hot path (every 50\n-        writes) and already runs under ``self._lock``, so the\n-        additional hold time is negligible.\n+        \"\"\"Best-effort PASSIVE WAL checkpoint.  Never raises.\n+\n+        Flushes committed WAL frames back into the main DB file without\n+        requiring an exclusive lock.  PASSIVE is safe for frequent\n+        periodic use because it does not block concurrent writers and\n+        cannot corrupt B-tree pages under I/O pressure.\n+\n+        PASSIVE does not truncate the WAL file — it stays at its\n+        high-water mark.  WAL truncation happens in :meth:`close`\n+        (TRUNCATE) and pre-VACUUM checkpoints, which run infrequently\n+        under controlled conditions.\n+\n+        Previous TRUNCATE strategy caused B-tree corruption on large\n+        databases (65K+ pages) due to the exclusive-lock I/O pressure\n+        from checkpointing thousands of frames at once (issue #45383).\n         \"\"\"\n         try:\n             with self._lock:\n                 result = self._conn.execute(\n-                    \"PRAGMA wal_checkpoint(TRUNCATE)\"\n+                    \"PRAGMA wal_checkpoint(PASSIVE)\"\n                 ).fetchone()\n                 if result and result[1] > 0:\n                     logger.debug(\n                         \"WAL checkpoint: %d/%d pages checkpointed\",\n                         result[2], result[1],\n                     )\n-        except Exception:\n-            pass  # Best effort — never fatal.\n+        except Exception as exc:\n+            logger.warning(\"WAL checkpoint (PASSIVE) failed: %s\", exc)\n \n     def _try_optimize_fts(self) -> None:\n         \"\"\"Best-effort FTS5 segment merge. Never raises.","@@ -1357,8 +1356,8 @@ def close(self):\n             if self._conn:\n                 try:\n                     self._conn.execute(\"PRAGMA wal_checkpoint(TRUNCATE)\")\n-                except Exception:\n-                    pass\n+                except Exception as exc:\n+                    logger.debug(\"WAL checkpoint (TRUNCATE) at close failed: %s\", exc)\n                 self._conn.close()\n                 self._conn = None\n ","@@ -7015,8 +7014,8 @@ def vacuum(self) -> int:\n             # Best-effort WAL checkpoint first, then VACUUM.\n             try:\n                 self._conn.execute(\"PRAGMA wal_checkpoint(TRUNCATE)\")\n-            except Exception:\n-                pass\n+            except Exception as exc:\n+                logger.debug(\"WAL checkpoint (TRUNCATE) before VACUUM failed: %s\", exc)\n             self._conn.execute(\"VACUUM\")\n         return optimized\n "]},{"file":"tests/test_wal_checkpoint_strategy.py","hunks":["@@ -0,0 +1,138 @@\n+\"\"\"Tests for SessionDB WAL checkpoint strategy (issue #45383).\n+\n+Verifies that periodic checkpoints use PASSIVE mode (safe for large DBs)\n+while close() and pre-VACUUM paths still use TRUNCATE.\n+\"\"\"\n+\n+import sqlite3\n+import logging\n+from unittest.mock import MagicMock, patch\n+\n+import pytest\n+\n+from hermes_state import SessionDB\n+\n+\n[redacted:email]()\n+def db(tmp_path):\n+    \"\"\"Create a SessionDB with a temp database file.\"\"\"\n+    db_path = tmp_path / \"test_state.db\"\n+    session_db = SessionDB(db_path=db_path)\n+    yield session_db\n+    try:\n+        session_db.close()\n+    except Exception:\n+        pass\n+\n+\n+class TestTryWalCheckpointPassive:\n+    \"\"\"_try_wal_checkpoint() should use PASSIVE mode for periodic use.\"\"\"\n+\n+    def test_checkpoint_uses_passive_mode(self, db):\n+        \"\"\"PASSIVE checkpoint does not require exclusive lock — safe for large DBs.\"\"\"\n+        # Capture the real connection's execute before mocking\n+        real_conn = db._conn\n+        execute_calls = []\n+\n+        def tracking_execute(sql, *args, **kwargs):\n+            execute_calls.append(sql)\n+            return real_conn.execute(sql, *args, **kwargs)\n+\n+        # sqlite3.Connection.execute is read-only (C extension) — replace _conn\n+        mock_conn = MagicMock()\n+        mock_conn.execute.side_effect = tracking_execute\n+        mock_conn.fetchone.return_value = None\n+        db._conn = mock_conn\n+\n+        db._try_wal_checkpoint()\n+\n+        passive_calls = [c for c in execute_calls if \"wal_checkpoint(PASSIVE)\" in c]\n+        truncate_calls = [c for c in execute_calls if \"wal_checkpoint(TRUNCATE)\" in c]\n+        assert len(passive_calls) == 1, (\n+            f\"Expected 1 PASSIVE checkpoint call, got {len(passive_calls)}\"\n+        )\n+        assert len(truncate_calls) == 0, (\n+            \"Periodic checkpoint should NOT use TRUNCATE\"\n+        )\n+\n+    def test_checkpoint_logs_warning_on_failure(self, db, caplog):\n+        \"\"\"Failed PASSIVE checkpoint logs a warning instead of silent pass.\"\"\"\n+        mock_conn = MagicMock()\n+        mock_conn.execute.side_effect = sqlite3.OperationalError(\"disk I/O error\")\n+        db._conn = mock_conn\n+\n+        with caplog.at_level(logging.WARNING):\n+            db._try_wal_checkpoint()\n+\n+        assert any(\"WAL checkpoint (PASSIVE) failed\" in r.message for r in caplog.records), (\n+            f\"Expected warning log about PASSIVE checkpoint failure, got: {caplog.text}\"\n+        )\n+\n+    def test_checkpoint_returns_result_on_success(self, db):\n+        \"\"\"Successful PASSIVE checkpoint does not raise.\"\"\"\n+        db._try_wal_checkpoint()\n+\n+\n+class TestCloseUsesTruncate:\n+    \"\"\"close() should still use TRUNCATE to shrink WAL on shutdown.\"\"\"\n+\n+    def test_close_uses_truncate_mode(self, db):\n+        \"\"\"TRUNCATE at close is safe — no concurrent writers during shutdown.\"\"\"\n+        real_conn = db._conn\n+        execute_calls = []\n+\n+        def tracking_execute(sql, *args, **kwargs):\n+            execute_calls.append(sql)\n+            return real_conn.execute(sql, *args, **kwargs)\n+\n+        mock_conn = MagicMock()\n+        mock_conn.execute.side_effect = tracking_execute\n+        db._conn = mock_conn\n+\n+        db.close()\n+\n+        truncate_calls = [c for c in execute_calls if \"wal_checkpoint(TRUNCATE)\" in c]\n+        assert len(truncate_calls) == 1, (\n+            f\"Expected 1 TRUNCATE checkpoint at close, got {len(truncate_calls)}\"\n+        )\n+\n+    def test_close_logs_debug_on_failure(self, db, caplog):\n+        \"\"\"Failed TRUNCATE at close logs debug (not warning — close is best-effort).\"\"\"\n+        mock_conn = MagicMock()\n+        mock_conn.execute.side_effect = sqlite3.OperationalError(\"database is locked\")\n+        db._conn = mock_conn\n+\n+        with caplog.at_level(logging.DEBUG):\n+            db.close()\n+\n+        assert any(\"WAL checkpoint (TRUNCATE) at close failed\" in r.message for r in caplog.records), (\n+            f\"Expected debug log about TRUNCATE failure at close, got: {caplog.text}\"\n+        )\n+\n+\n+class TestCheckpointFrequency:\n+    \"\"\"Checkpoint triggers every N writes.\"\"\"\n+\n+    def test_checkpoint_triggers_at_interval(self, db):\n+        \"\"\"_try_wal_checkpoint is called every _CHECKPOINT_EVERY_N_WRITES writes.\"\"\"\n+        call_count = [0]\n+        original = db._try_wal_checkpoint\n+\n+        def counting_checkpoint():\n+            call_count[0] += 1\n+            original()\n+\n+        db._try_wal_checkpoint = counting_checkpoint\n+\n+        # Write exactly _CHECKPOINT_EVERY_N_WRITES sessions to trigger one checkpoint\n+        n = db._CHECKPOINT_EVERY_N_WRITES\n+        import time as _time\n+        for i in range(n):\n+            db._execute_write(lambda conn, _i=i: conn.execute(\n+                \"INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)\",\n+                (f\"sess_{_i}\", \"test\", _time.time()),\n+            ))\n+\n+        assert call_count[0] == 1, (\n+            f\"Expected 1 checkpoint after {n} writes, got {call_count[0]}\"\n+        )\n"]}]},{"kind":"pr","number":72546,"title":"fix(state): use PASSIVE WAL checkpoint in close() and pre-VACUUM paths (#45383)","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":72549,"title":"fix(state): use PASSIVE WAL checkpoint in close() and pre-VACUUM paths (#45383)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"7965462d6","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":84881,"title":"fix(state): PASSIVE not TRUNCATE for all shared-DB WAL checkpoints","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["contributors/emails/[redacted:email]","hermes_cli/kanban_db.py","hermes_state.py","hermes_state_search.py","tests/hermes_cli/test_kanban_db_repair.py","tests/test_hermes_state.py","tests/test_wal_checkpoint_strategy.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}