{"repo":"NousResearch/hermes-agent","public_id":"47002","display_label":"SQLite trigram tokenizer absence crashing SessionDB","summary":"Multiple fixes and a feature related to SQLite's optional trigram tokenizer: the absence of this tokenizer causes SessionDB initialization to crash, and several patches address detection, graceful handling, and warnings around this condition. One member also introduces a trigram FTS5 index for CJK search.","subsystem":null,"counts":{"members":6,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":48688},"resolution":{"state":"resolved","by":"pr-48688"},"freshness":{"state":"verified","last_verified_at":"2026-06-19T02:14:52+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.3333333333333333},"fix_edges":[{"issue_number":41386,"pr_number":48688,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":47002,"pr_number":48688,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":47002,"title":"SessionDB.__init__ crashes on SQLite without trigram tokenizer (v0.16.0 regression)","url":null,"state":"closed"},{"kind":"pr","number":47025,"title":"fix(agent): handle missing trigram tokenizer without disabling FTS5","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47049","number":47049},{"kind":"pr","value":"#47049","number":47049}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":47031,"title":"fix: SessionDB crash on SQLite builds without trigram tokenizer","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47002","number":47002},{"kind":"pr","value":"#47049","number":47049}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":47086,"title":"fix(state): handle missing trigram tokenizer in FTS5 error detection (#47002)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47002","number":47002},{"kind":"pr","value":"#47049","number":47049},{"kind":"pr","value":"#47049","number":47049}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":47482,"title":"fix(state): handle missing trigram tokenizer without crashing SessionDB","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47002","number":47002},{"kind":"pr","value":"#47049","number":47049}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":48688,"title":"fix(state): survive SQLite builds without trigram tokenizer + warn on unavailable session store","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cli.py","hermes_state.py","scripts/release.py","tests/test_hermes_state.py"],"fix_hunks":[{"file":"cli.py","hunks":["@@ -3503,11 +3503,36 @@ def __init__(\n         self._last_turn_finished_at: Optional[float] = None  # time.time() when the last agent loop finished\n         # Initialize SQLite session store early so /title works before first message\n         self._session_db = None\n+        self._session_db_unavailable = False\n         try:\n             from hermes_state import SessionDB\n             self._session_db = SessionDB()\n         except Exception as e:\n+            # #41386: a failed session store means the transcript is NOT\n+            # persisted to state.db — the live chat looks healthy but resume\n+            # later shows a truncated/empty session. A buried log line is not\n+            # enough; surface it prominently so the user knows persistence is\n+            # off for this run and can fix the store before relying on resume.\n+            self._session_db_unavailable = True\n             logger.warning(\"Failed to initialize SessionDB — session will NOT be indexed for search: %s\", e)\n+            try:\n+                # Console is imported at module scope; do NOT re-import it here.\n+                # A function-local `import` would make `Console` a local name for\n+                # the whole __init__ body and break the earlier `self.console =\n+                # Console()` with UnboundLocalError.\n+                Console(stderr=True).print(\n+                    \"[bold yellow]⚠ Session store unavailable[/bold yellow] — \"\n+                    \"this conversation will [bold]NOT be saved[/bold] to disk and \"\n+                    \"cannot be resumed later. Searching past sessions is also disabled.\\n\"\n+                    f\"  Reason: {e}\\n\"\n+                    \"  Fix the state.db store (e.g. `hermes update` to rebuild the venv) to restore persistence.\"\n+                )\n+            except Exception:\n+                # Never let the warning path itself break startup.\n+                print(\n+                    \"WARNING: Session store unavailable — this conversation will NOT be \"\n+                    f\"saved to disk and cannot be resumed later. Reason: {e}\"\n+                )\n \n         # Opportunistic state.db maintenance — runs at most once per\n         # min_interval_hours, tracked via state_meta in state.db itself so"]},{"file":"hermes_state.py","hunks":["@@ -684,6 +684,7 @@ def __init__(self, db_path: Path = None, read_only: bool = False):\n         self._lock = threading.Lock()\n         self._write_count = 0\n         self._fts_enabled = False\n+        self._trigram_available = False\n         self._fts_unavailable_warned = False\n         self._conn = None\n         try:","@@ -772,7 +773,33 @@ def _connect_and_init():\n     @staticmethod\n     def _is_fts5_unavailable_error(exc: sqlite3.OperationalError) -> bool:\n         err = str(exc).lower()\n-        return \"no such module\" in err and \"fts5\" in err\n+        if \"no such module\" in err and \"fts5\" in err:\n+            return True\n+        # SQLite builds that have FTS5 but lack the optional trigram tokenizer\n+        # raise \"no such tokenizer: trigram\" instead of \"no such module\".\n+        # Scope to trigram specifically to avoid masking unrelated tokenizer errors.\n+        if \"no such tokenizer: trigram\" in err:\n+            return True\n+        return False\n+\n+    @staticmethod\n+    def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:\n+        \"\"\"True when only the trigram tokenizer is missing (FTS5 itself works).\"\"\"\n+        return \"no such tokenizer: trigram\" in str(exc).lower()\n+\n+    def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None:\n+        \"\"\"Log once that the trigram tokenizer is missing; base FTS5 stays enabled.\"\"\"\n+        if getattr(self, \"_trigram_unavailable_warned\", False):\n+            return\n+        self._trigram_unavailable_warned = True\n+        logger.info(\n+            \"SQLite trigram tokenizer unavailable for %s \"\n+            \"(requires SQLite >= 3.34, this build is %s); \"\n+            \"CJK/substring search will fall back to LIKE: %s\",\n+            self.db_path,\n+            sqlite3.sqlite_version,\n+            exc,\n+        )\n \n     def _warn_fts5_unavailable(self, exc: sqlite3.OperationalError) -> None:\n         self._fts_enabled = False","@@ -818,9 +845,12 @@ def _fts_trigger_count(cursor: sqlite3.Cursor) -> int:\n         return int(row[0] if not isinstance(row, sqlite3.Row) else row[0])\n \n     @staticmethod\n-    def _rebuild_fts_indexes(cursor: sqlite3.Cursor) -> None:\n-        for table_name in (\"messages_fts\", \"messages_fts_trigram\"):\n-            cursor.execute(f\"DELETE FROM {table_name}\")\n+    def _rebuild_fts_indexes(\n+        cursor: sqlite3.Cursor,\n+        *,\n+        include_trigram: bool = True,\n+    ) -> None:\n+        cursor.execute(\"DELETE FROM messages_fts\")\n         cursor.execute(\n             \"INSERT INTO messages_fts(rowid, content) \"\n             \"SELECT id, \"","@@ -829,6 +859,9 @@ def _rebuild_fts_indexes(cursor: sqlite3.Cursor) -> None:\n             \"COALESCE(tool_calls, '') \"\n             \"FROM messages\"\n         )\n+        if not include_trigram:\n+            return\n+        cursor.execute(\"DELETE FROM messages_fts_trigram\")\n         cursor.execute(\n             \"INSERT INTO messages_fts_trigram(rowid, content) \"\n             \"SELECT id, \"","@@ -844,7 +877,12 @@ def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[\n             return True\n         except sqlite3.OperationalError as exc:\n             if self._is_fts5_unavailable_error(exc):\n-                self._warn_fts5_unavailable(exc)\n+                # Only disable FTS entirely when the whole module is missing.\n+                # A missing trigram tokenizer only affects trigram searches.\n+                if self._is_trigram_unavailable_error(exc):\n+                    self._warn_trigram_unavailable(exc)\n+                else:\n+                    self._warn_fts5_unavailable(exc)\n                 return None\n             if \"no such table\" in str(exc).lower():\n                 return False","@@ -868,7 +906,13 @@ def _ensure_fts_schema(\n         except sqlite3.OperationalError as exc:\n             if not self._is_fts5_unavailable_error(exc):\n                 raise\n-            self._warn_fts5_unavailable(exc)\n+            # Only disable FTS entirely when the whole FTS5 module is missing.\n+            # A missing specific tokenizer (e.g. trigram) means only that\n+            # particular table cannot be created — the base FTS5 table is fine.\n+            if self._is_trigram_unavailable_error(exc):\n+                self._warn_trigram_unavailable(exc)\n+            else:\n+                self._warn_fts5_unavailable(exc)\n             return False\n \n     def _execute_write(self, fn: Callable[[sqlite3.Connection], T]) -> T:","@@ -1166,21 +1210,23 @@ def _init_schema(self):\n                         except sqlite3.OperationalError as exc:\n                             if not self._is_fts5_unavailable_error(exc):\n                                 raise\n-                            self._warn_fts5_unavailable(exc)\n-                            fts5_available = False\n-                            fts_migrations_complete = False\n+                            if self._is_trigram_unavailable_error(exc):\n+                                self._warn_trigram_unavailable(exc)\n+                            else:\n+                                self._warn_fts5_unavailable(exc)\n+                                fts5_available = False\n+                                fts_migrations_complete = False\n                             break\n \n                     if fts5_available:\n                         # Recreate virtual tables + triggers with the new inline-mode\n                         # schema that indexes content || tool_name || tool_calls.\n-                        if (\n-                            self._ensure_fts_schema(cursor, \"messages_fts\", FTS_SQL)\n-                            and self._ensure_fts_schema(\n-                                cursor, \"messages_fts_trigram\", FTS_TRIGRAM_SQL\n-                            )\n-                        ):\n-                            # Backfill both indexes from every existing messages row.\n+                        # Handle base and trigram independently — a missing\n+                        # trigram tokenizer should not prevent base FTS backfill.\n+                        base_fts_ok = self._ensure_fts_schema(\n+                            cursor, \"messages_fts\", FTS_SQL\n+                        )\n+                        if base_fts_ok:\n                             cursor.execute(\n                                 \"INSERT INTO messages_fts(rowid, content) \"\n                                 \"SELECT id, \"","@@ -1189,6 +1235,10 @@ def _init_schema(self):\n                                 \"COALESCE(tool_calls, '') \"\n                                 \"FROM messages\"\n                             )\n+                        trigram_ok = self._ensure_fts_schema(\n+                            cursor, \"messages_fts_trigram\", FTS_TRIGRAM_SQL\n+                        )\n+                        if trigram_ok:\n                             cursor.execute(\n                                 \"INSERT INTO messages_fts_trigram(rowid, content) \"\n                                 \"SELECT id, \"","@@ -1197,8 +1247,12 @@ def _init_schema(self):\n                                 \"COALESCE(tool_calls, '') \"\n                                 \"FROM messages\"\n                             )\n-                        else:\n+                        if not base_fts_ok:\n                             fts_migrations_complete = False\n+                        # Track trigram availability for CJK LIKE fallback.\n+                        self._trigram_available = trigram_ok\n+                    else:\n+                        fts_migrations_complete = False\n                 else:\n                     fts_migrations_complete = False\n             if current_version < 12:","@@ -1268,8 +1322,12 @@ def _init_schema(self):\n                 trigram_enabled = self._ensure_fts_schema(\n                     cursor, \"messages_fts_trigram\", FTS_TRIGRAM_SQL\n                 )\n-                if trigram_enabled and triggers_need_repair:\n-                    self._rebuild_fts_indexes(cursor)\n+                self._trigram_available = trigram_enabled\n+                if triggers_need_repair:\n+                    self._rebuild_fts_indexes(\n+                        cursor,\n+                        include_trigram=trigram_enabled,\n+                    )\n \n         self._conn.commit()\n ","@@ -3404,7 +3462,8 @@ def search_messages(\n                 self._count_cjk(t) < 3 for t in _tokens_for_check\n             )\n \n-            if cjk_count >= 3 and not _any_short_cjk:\n+            _trigram_succeeded = False\n+            if cjk_count >= 3 and not _any_short_cjk and self._trigram_available:\n                 # Trigram FTS5 path — quote each non-operator token to handle\n                 # FTS5 special chars (%, *, etc.) while preserving boolean\n                 # operators (AND, OR, NOT) for multi-term queries.","@@ -3453,11 +3512,13 @@ def search_messages(\n                     try:\n                         tri_cursor = self._conn.execute(tri_sql, tri_params)\n                     except sqlite3.OperationalError:\n-                        matches = []\n+                        # Trigram query failed at runtime — fall through to LIKE.\n+                        pass\n                     else:\n                         matches = [dict(row) for row in tri_cursor.fetchall()]\n-            else:\n-                # Short / mixed CJK query: trigram cannot match tokens with\n+                        _trigram_succeeded = True\n+            if not _trigram_succeeded:\n+                # Short / mixed CJK query, trigram unavailable, or trigram\n                 # <3 CJK chars. Fall back to LIKE substring search.\n                 # For multi-token OR queries (e.g. \"广西 OR 桂林 OR 漓江\"),\n                 # build one LIKE condition per non-operator token so each term"]},{"file":"scripts/release.py","hunks":["@@ -102,6 +102,7 @@\n     \"[redacted:email]\": \"synapsesx\",\n     \"[redacted:email]\": \"itsflownium\",\n     \"[redacted:email]\": \"dirtyren\",\n+    \"[redacted:email]\": \"channkim\",\n     \"[redacted:email]\": \"x1erra\",\n     \"[redacted:email]\": \"zimigit2020\",\n     \"[redacted:email]\": \"ft-ioxcs\","]},{"file":"tests/test_hermes_state.py","hunks":["@@ -50,6 +50,20 @@ def cursor(self, factory=None):\n         return super().cursor(factory or _NoFtsExistingTableCursor)\n \n \n+class _NoTrigramCursor(sqlite3.Cursor):\n+    \"\"\"Simulate a SQLite build with FTS5 but without the trigram tokenizer.\"\"\"\n+\n+    def executescript(self, sql_script):\n+        if \"tokenize='[redacted:secret-assignment]'\" in sql_script:\n+            raise sqlite3.OperationalError(\"no such tokenizer: trigram\")\n+        return super().executescript(sql_script)\n+\n+\n+class _NoTrigramConnection(sqlite3.Connection):\n+    def cursor(self, factory=None):\n+        return super().cursor(factory or _NoTrigramCursor)\n+\n+\n @pytest.fixture()\n def db(tmp_path):\n     \"\"\"Create a SessionDB with a temp database file.\"\"\"","@@ -330,6 +344,167 @@ def connect_without_fts(*args, **kwargs):\n         finally:\n             restored.close()\n \n+    def test_base_fts_rebuilds_after_trigger_repair_without_trigram(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"Trigger repair must rebuild base FTS even when trigram is unavailable.\"\"\"\n+        db_path = tmp_path / \"state.db\"\n+        seeded = SessionDB(db_path=db_path)\n+        try:\n+            seeded.create_session(session_id=\"s1\", source=\"cli\")\n+            seeded.append_message(\"s1\", role=\"user\", content=\"already indexed\")\n+            for trigger in (\n+                \"messages_fts_insert\",\n+                \"messages_fts_delete\",\n+                \"messages_fts_update\",\n+                \"messages_fts_trigram_insert\",\n+                \"messages_fts_trigram_delete\",\n+                \"messages_fts_trigram_update\",\n+            ):\n+                seeded._conn.execute(f\"DROP TRIGGER IF EXISTS {trigger}\")\n+            seeded._conn.commit()\n+            seeded.append_message(\"s1\", role=\"assistant\", content=\"repair only base needle\")\n+        finally:\n+            seeded.close()\n+\n+        real_connect = sqlite3.connect\n+\n+        def connect_without_trigram(*args, **kwargs):\n+            kwargs[\"factory\"] = _NoTrigramConnection\n+            return real_connect(*args, **kwargs)\n+\n+        monkeypatch.setattr(\"hermes_state.sqlite3.connect\", connect_without_trigram)\n+        restored = SessionDB(db_path=db_path)\n+        try:\n+            assert restored._fts_enabled is True\n+            assert restored._trigram_available is False\n+            assert restored._fts_table_exists(\"messages_fts\") is True\n+            assert len(restored.search_messages(\"needle\")) == 1\n+        finally:\n+            restored.close()\n+\n+    def test_is_fts5_unavailable_error_catches_trigram_tokenizer(self):\n+        \"\"\"Unit test: _is_fts5_unavailable_error matches 'no such tokenizer: trigram'.\"\"\"\n+        fts5_err = sqlite3.OperationalError(\"no such module: fts5\")\n+        trigram_err = sqlite3.OperationalError(\"no such tokenizer: trigram\")\n+        generic_tokenizer_err = [redacted:secret-assignment]\"no such tokenizer: foo\")\n+        unrelated_err = sqlite3.OperationalError(\"no such table: foo\")\n+\n+        assert SessionDB._is_fts5_unavailable_error(fts5_err) is True\n+        assert SessionDB._is_fts5_unavailable_error(trigram_err) is True\n+        # Generic tokenizer errors should NOT match — only trigram.\n+        assert SessionDB._is_fts5_unavailable_error(generic_tokenizer_err) is False\n+        assert SessionDB._is_fts5_unavailable_error(unrelated_err) is False\n+\n+    def test_is_trigram_unavailable_error(self):\n+        \"\"\"Unit test: _is_trigram_unavailable_error is scoped to trigram.\"\"\"\n+        trigram_err = sqlite3.OperationalError(\"no such tokenizer: trigram\")\n+        generic_err = sqlite3.OperationalError(\"no such tokenizer: foo\")\n+        fts5_err = sqlite3.OperationalError(\"no such module: fts5\")\n+\n+        assert SessionDB._is_trigram_unavailable_error(trigram_err) is True\n+        assert SessionDB._is_trigram_unavailable_error(generic_err) is False\n+        assert SessionDB._is_trigram_unavailable_error(fts5_err) is False\n+\n+    def test_db_initializes_without_trigram_tokenizer(self, tmp_path, monkeypatch):\n+        \"\"\"SessionDB must not crash when FTS5 exists but trigram tokenizer is missing.\"\"\"\n+        real_connect = sqlite3.connect\n+\n+        def connect_without_trigram(*args, **kwargs):\n+            kwargs[\"factory\"] = _NoTrigramConnection\n+            return real_connect(*args, **kwargs)\n+\n+        monkeypatch.setattr(\"hermes_state.sqlite3.connect\", connect_without_trigram)\n+\n+        db = SessionDB(db_path=tmp_path / \"state.db\")\n+        try:\n+            # Base FTS5 should still work (trigram is optional).\n+            assert db._fts_enabled is True\n+            assert db._fts_table_exists(\"messages_fts\") is True\n+            # Trigram table should NOT have been created.\n+            assert db._fts_table_exists(\"messages_fts_trigram\") is False\n+\n+            db.create_session(session_id=\"s1\", source=\"cli\")\n+            db.append_message(\"s1\", role=\"user\", content=\"hello without trigram\")\n+\n+            messages = db.get_messages(\"s1\")\n+            assert len(messages) == 1\n+            assert messages[0][\"content\"] == \"hello without trigram\"\n+\n+            # FTS5 keyword search should still work.\n+            assert len(db.search_messages(\"hello\")) == 1\n+        finally:\n+            db.close()\n+\n+    def test_v11_migration_backfills_base_fts_when_trigram_unavailable(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"Regression: v11 migration must backfill base FTS even when trigram is unavailable.\"\"\"\n+        real_connect = sqlite3.connect\n+        db_path = tmp_path / \"state.db\"\n+\n+        # Phase 1: create a DB at schema v10 with messages.\n+        db = SessionDB(db_path=db_path)\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"legacy message alpha\")\n+        db.append_message(\"s1\", role=\"assistant\", content=\"legacy reply beta\")\n+        # Force schema version to v10 so migration runs on next open.\n+        db._conn.execute(\n+            \"UPDATE schema_version SET version = 10\"\n+        )\n+        db._conn.commit()\n+        db.close()\n+\n+        # Phase 2: reopen with trigram disabled — migration should still\n+        # backfill base FTS and make existing messages searchable.\n+        def connect_without_trigram(*args, **kwargs):\n+            kwargs[\"factory\"] = _NoTrigramConnection\n+            return real_connect(*args, **kwargs)\n+\n+        monkeypatch.setattr(\"hermes_state.sqlite3.connect\", connect_without_trigram)\n+        migrated_db = SessionDB(db_path=db_path)\n+        try:\n+            assert migrated_db._fts_enabled is True\n+            assert migrated_db._trigram_available is False\n+            assert migrated_db._fts_table_exists(\"messages_fts\") is True\n+            assert migrated_db._fts_table_exists(\"messages_fts_trigram\") is False\n+\n+            # Existing messages must be searchable via base FTS.\n+            results = migrated_db.search_messages(\"legacy message\")\n+            assert len(results) == 1\n+            # snippet has FTS5 highlight markers (>>>...<<<); check raw content via get_messages\n+            msgs = migrated_db.get_messages(\"s1\")\n+            assert any(\"legacy message\" in m[\"content\"] for m in msgs)\n+        finally:\n+            migrated_db.close()\n+\n+    def test_cjk_search_falls_back_to_like_when_trigram_unavailable(\n+        self, tmp_path, monkeypatch\n+    ):\n+        \"\"\"Regression: long CJK queries must fall back to LIKE when trigram is missing.\"\"\"\n+        real_connect = sqlite3.connect\n+        db_path = tmp_path / \"state.db\"\n+\n+        def connect_without_trigram(*args, **kwargs):\n+            kwargs[\"factory\"] = _NoTrigramConnection\n+            return real_connect(*args, **kwargs)\n+\n+        monkeypatch.setattr(\"hermes_state.sqlite3.connect\", connect_without_trigram)\n+        db = SessionDB(db_path=db_path)\n+        try:\n+            db.create_session(session_id=\"s1\", source=\"cli\")\n+            db.append_message(\"s1\", role=\"user\", content=\"大别山项目计划书\")\n+            db.append_message(\"s1\", role=\"user\", content=\"长江大桥设计方案\")\n+\n+            # 3+ CJK chars would normally use trigram, but it's unavailable.\n+            # Must fall back to LIKE and still return results.\n+            results = db.search_messages(\"大别山\")\n+            assert len(results) == 1\n+            # Note: search_messages strips 'content' from results; use 'snippet'.\n+            assert \"大别山\" in results[0][\"snippet\"]\n+        finally:\n+            db.close()\n+\n \n # =========================================================================\n # Message storage\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}