Session store unavailability handling and warnings

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #48688 fix(state): survive SQLite builds without trigram tokenizer + warn on unavailable session store

  merged to main

  files: cli.py, hermes_state.py, scripts/release.py, tests/test_hermes_state.py

  Show the diff

  • cli.py

    @@ -3503,11 +3503,36 @@ def __init__(
             self._last_turn_finished_at: Optional[float] = None  # time.time() when the last agent loop finished
             # Initialize SQLite session store early so /title works before first message
             self._session_db = None
    +        self._session_db_unavailable = False
             try:
                 from hermes_state import SessionDB
                 self._session_db = SessionDB()
             except Exception as e:
    +            # #41386: a failed session store means the transcript is NOT
    +            # persisted to state.db — the live chat looks healthy but resume
    +            # later shows a truncated/empty session. A buried log line is not
    +            # enough; surface it prominently so the user knows persistence is
    +            # off for this run and can fix the store before relying on resume.
    +            self._session_db_unavailable = True
                 logger.warning("Failed to initialize SessionDB — session will NOT be indexed for search: %s", e)
    +            try:
    +                # Console is imported at module scope; do NOT re-import it here.
    +                # A function-local `import` would make `Console` a local name for
    +                # the whole __init__ body and break the earlier `self.console =
    +                # Console()` with UnboundLocalError.
    +                Console(stderr=True).print(
    +                    "[bold yellow]⚠ Session store unavailable[/bold yellow] — "
    +                    "this conversation will [bold]NOT be saved[/bold] to disk and "
    +                    "cannot be resumed later. Searching past sessions is also disabled.\n"
    +                    f"  Reason: {e}\n"
    +                    "  Fix the state.db store (e.g. `hermes update` to rebuild the venv) to restore persistence."
    +                )
    +            except Exception:
    +                # Never let the warning path itself break startup.
    +                print(
    +                    "WARNING: Session store unavailable — this conversation will NOT be "
    +                    f"saved to disk and cannot be resumed later. Reason: {e}"
    +                )
     
             # Opportunistic state.db maintenance — runs at most once per
             # min_interval_hours, tracked via state_meta in state.db itself so

  • hermes_state.py

    @@ -684,6 +684,7 @@ def __init__(self, db_path: Path = None, read_only: bool = False):
             self._lock = threading.Lock()
             self._write_count = 0
             self._fts_enabled = False
    +        self._trigram_available = False
             self._fts_unavailable_warned = False
             self._conn = None
             try:
    @@ -772,7 +773,33 @@ def _connect_and_init():
         @staticmethod
         def _is_fts5_unavailable_error(exc: sqlite3.OperationalError) -> bool:
             err = str(exc).lower()
    -        return "no such module" in err and "fts5" in err
    +        if "no such module" in err and "fts5" in err:
    +            return True
    +        # SQLite builds that have FTS5 but lack the optional trigram tokenizer
    +        # raise "no such tokenizer: trigram" instead of "no such module".
    +        # Scope to trigram specifically to avoid masking unrelated tokenizer errors.
    +        if "no such tokenizer: trigram" in err:
    +            return True
    +        return False
    +
    +    @staticmethod
    +    def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:
    +        """True when only the trigram tokenizer is missing (FTS5 itself works)."""
    +        return "no such tokenizer: trigram" in str(exc).lower()
    +
    +    def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None:
    +        """Log once that the trigram tokenizer is missing; base FTS5 stays enabled."""
    +        if getattr(self, "_trigram_unavailable_warned", False):
    +            return
    +        self._trigram_unavailable_warned = True
    +        logger.info(
    +            "SQLite trigram tokenizer unavailable for %s "
    +            "(requires SQLite >= 3.34, this build is %s); "
    +            "CJK/substring search will fall back to LIKE: %s",
    +            self.db_path,
    +            sqlite3.sqlite_version,
    +            exc,
    +        )
     
         def _warn_fts5_unavailable(self, exc: sqlite3.OperationalError) -> None:
             self._fts_enabled = False
    @@ -818,9 +845,12 @@ def _fts_trigger_count(cursor: sqlite3.Cursor) -> int:
             return int(row[0] if not isinstance(row, sqlite3.Row) else row[0])
     
         @staticmethod
    -    def _rebuild_fts_indexes(cursor: sqlite3.Cursor) -> None:
    -        for table_name in ("messages_fts", "messages_fts_trigram"):
    -            cursor.execute(f"DELETE FROM {table_name}")
    +    def _rebuild_fts_indexes(
    +        cursor: sqlite3.Cursor,
    +        *,
    +        include_trigram: bool = True,
    +    ) -> None:
    +        cursor.execute("DELETE FROM messages_fts")
             cursor.execute(
                 "INSERT INTO messages_fts(rowid, content) "
                 "SELECT id, "
    @@ -829,6 +859,9 @@ def _rebuild_fts_indexes(cursor: sqlite3.Cursor) -> None:
                 "COALESCE(tool_calls, '') "
                 "FROM messages"
             )
    +        if not include_trigram:
    +            return
    +        cursor.execute("DELETE FROM messages_fts_trigram")
             cursor.execute(
                 "INSERT INTO messages_fts_trigram(rowid, content) "
                 "SELECT id, "
    @@ -844,7 +877,12 @@ def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[
                 return True
             except sqlite3.OperationalError as exc:
                 if self._is_fts5_unavailable_error(exc):
    -                self._warn_fts5_unavailable(exc)
    +                # Only disable FTS entirely when the whole module is missing.
    +                # A missing trigram tokenizer only affects trigram searches.
    +                if self._is_trigram_unavailable_error(exc):
    +                    self._warn_trigram_unavailable(exc)
    +                else:
    +                    self._warn_fts5_unavailable(exc)
                     return None
                 if "no such table" in str(exc).lower():
                     return False
    @@ -868,7 +906,13 @@ def _ensure_fts_schema(
             except sqlite3.OperationalError as exc:
                 if not self._is_fts5_unavailable_error(exc):
                     raise
    -            self._warn_fts5_unavailable(exc)
    +            # Only disable FTS entirely when the whole FTS5 module is missing.
    +            # A missing specific tokenizer (e.g. trigram) means only that
    +            # particular table cannot be created — the base FTS5 table is fine.
    +            if self._is_trigram_unavailable_error(exc):
    +                self._warn_trigram_unavailable(exc)
    +            else:
    +                self._warn_fts5_unavailable(exc)
                 return False
     
         def _execute_write(self, fn: Callable[[sqlite3.Connection], T]) -> T:
    @@ -1166,21 +1210,23 @@ def _init_schema(self):
                             except sqlite3.OperationalError as exc:
                                 if not self._is_fts5_unavailable_error(exc):
                                     raise
    -                            self._warn_fts5_unavailable(exc)
    -                            fts5_available = False
    -                            fts_migrations_complete = False
    +                            if self._is_trigram_unavailable_error(exc):
    +                                self._warn_trigram_unavailable(exc)
    +                            else:
    +                                self._warn_fts5_unavailable(exc)
    +                                fts5_available = False
    +                                fts_migrations_complete = False
                                 break
     
                         if fts5_available:
                             # Recreate virtual tables + triggers with the new inline-mode
                             # schema that indexes content || tool_name || tool_calls.
    -                        if (
    -                            self._ensure_fts_schema(cursor, "messages_fts", FTS_SQL)
    -                            and self._ensure_fts_schema(
    -                                cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
    -                            )
    -                        ):
    -                            # Backfill both indexes from every existing messages row.
    +                        # Handle base and trigram independently — a missing
    +                        # trigram tokenizer should not prevent base FTS backfill.
    +                        base_fts_ok = self._ensure_fts_schema(
    +                            cursor, "messages_fts", FTS_SQL
    +                        )
    +                        if base_fts_ok:
                                 cursor.execute(
                                     "INSERT INTO messages_fts(rowid, content) "
                                     "SELECT id, "
    @@ -1189,6 +1235,10 @@ def _init_schema(self):
                                     "COALESCE(tool_calls, '') "
                                     "FROM messages"
                                 )
    +                        trigram_ok = self._ensure_fts_schema(
    +                            cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
    +                        )
    +                        if trigram_ok:
                                 cursor.execute(
                                     "INSERT INTO messages_fts_trigram(rowid, content) "
                                     "SELECT id, "
    @@ -1197,8 +1247,12 @@ def _init_schema(self):
                                     "COALESCE(tool_calls, '') "
                                     "FROM messages"
                                 )
    -                        else:
    +                        if not base_fts_ok:
                                 fts_migrations_complete = False
    +                        # Track trigram availability for CJK LIKE fallback.
    +                        self._trigram_available = trigram_ok
    +                    else:
    +                        fts_migrations_complete = False
                     else:
                         fts_migrations_complete = False
                 if current_version < 12:
    @@ -1268,8 +1322,12 @@ def _init_schema(self):
                     trigram_enabled = self._ensure_fts_schema(
                         cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL
                     )
    -                if trigram_enabled and triggers_need_repair:
    -                    self._rebuild_fts_indexes(cursor)
    +                self._trigram_available = trigram_enabled
    +                if triggers_need_repair:
    +                    self._rebuild_fts_indexes(
    +                        cursor,
    +                        include_trigram=trigram_enabled,
    +                    )
     
             self._conn.commit()
     
    @@ -3404,7 +3462,8 @@ def search_messages(
                     self._count_cjk(t) < 3 for t in _tokens_for_check
                 )
     
    -            if cjk_count >= 3 and not _any_short_cjk:
    +            _trigram_succeeded = False
    +            if cjk_count >= 3 and not _any_short_cjk and self._trigram_available:
                     # Trigram FTS5 path — quote each non-operator token to handle
                     # FTS5 special chars (%, *, etc.) while preserving boolean
                     # operators (AND, OR, NOT) for multi-term queries.
    @@ -3453,11 +3512,13 @@ def search_messages(
                         try:
                             tri_cursor = self._conn.execute(tri_sql, tri_params)
                         except sqlite3.OperationalError:
    -                        matches = []
    +                        # Trigram query failed at runtime — fall through to LIKE.
    +                        pass
                         else:
                             matches = [dict(row) for row in tri_cursor.fetchall()]
    -            else:
    -                # Short / mixed CJK query: trigram cannot match tokens with
    +                        _trigram_succeeded = True
    +            if not _trigram_succeeded:
    +                # Short / mixed CJK query, trigram unavailable, or trigram
                     # <3 CJK chars. Fall back to LIKE substring search.
                     # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"),
                     # build one LIKE condition per non-operator token so each term

  • scripts/release.py

    @@ -102,6 +102,7 @@
         "[email protected]": "synapsesx",
         "[email protected]": "itsflownium",
         "[email protected]": "dirtyren",
    +    "[redacted:email]": "channkim",
         "[redacted:email]": "x1erra",
         "[redacted:email]": "zimigit2020",
         "[email protected]": "ft-ioxcs",

  • tests/test_hermes_state.py

    @@ -50,6 +50,20 @@ def cursor(self, factory=None):
             return super().cursor(factory or _NoFtsExistingTableCursor)
     
     
    +class _NoTrigramCursor(sqlite3.Cursor):
    +    """Simulate a SQLite build with FTS5 but without the trigram tokenizer."""
    +
    +    def executescript(self, sql_script):
    +        if "tokenize='[redacted:secret-assignment]'" in sql_script:
    +            raise sqlite3.OperationalError("no such tokenizer: trigram")
    +        return super().executescript(sql_script)
    +
    +
    +class _NoTrigramConnection(sqlite3.Connection):
    +    def cursor(self, factory=None):
    +        return super().cursor(factory or _NoTrigramCursor)
    +
    +
     @pytest.fixture()
     def db(tmp_path):
         """Create a SessionDB with a temp database file."""
    @@ -330,6 +344,167 @@ def connect_without_fts(*args, **kwargs):
             finally:
                 restored.close()
     
    +    def test_base_fts_rebuilds_after_trigger_repair_without_trigram(
    +        self, tmp_path, monkeypatch
    +    ):
    +        """Trigger repair must rebuild base FTS even when trigram is unavailable."""
    +        db_path = tmp_path / "state.db"
    +        seeded = SessionDB(db_path=db_path)
    +        try:
    +            seeded.create_session(session_id="s1", source="cli")
    +            seeded.append_message("s1", role="user", content="already indexed")
    +            for trigger in (
    +                "messages_fts_insert",
    +                "messages_fts_delete",
    +                "messages_fts_update",
    +                "messages_fts_trigram_insert",
    +                "messages_fts_trigram_delete",
    +                "messages_fts_trigram_update",
    +            ):
    +                seeded._conn.execute(f"DROP TRIGGER IF EXISTS {trigger}")
    +            seeded._conn.commit()
    +            seeded.append_message("s1", role="assistant", content="repair only base needle")
    +        finally:
    +            seeded.close()
    +
    +        real_connect = sqlite3.connect
    +
    +        def connect_without_trigram(*args, **kwargs):
    +            kwargs["factory"] = _NoTrigramConnection
    +            return real_connect(*args, **kwargs)
    +
    +        monkeypatch.setattr("hermes_state.sqlite3.connect", connect_without_trigram)
    +        restored = SessionDB(db_path=db_path)
    +        try:
    +            assert restored._fts_enabled is True
    +            assert restored._trigram_available is False
    +            assert restored._fts_table_exists("messages_fts") is True
    +            assert len(restored.search_messages("needle")) == 1
    +        finally:
    +            restored.close()
    +
    +    def test_is_fts5_unavailable_error_catches_trigram_tokenizer(self):
    +        """Unit test: _is_fts5_unavailable_error matches 'no such tokenizer: trigram'."""
    +        fts5_err = sqlite3.OperationalError("no such module: fts5")
    +        trigram_err = sqlite3.OperationalError("no such tokenizer: trigram")
    +        generic_tokenizer_err = [redacted:secret-assignment]"no such tokenizer: foo")
    +        unrelated_err = sqlite3.OperationalError("no such table: foo")
    +
    +        assert SessionDB._is_fts5_unavailable_error(fts5_err) is True
    +        assert SessionDB._is_fts5_unavailable_error(trigram_err) is True
    +        # Generic tokenizer errors should NOT match — only trigram.
    +        assert SessionDB._is_fts5_unavailable_error(generic_tokenizer_err) is False
    +        assert SessionDB._is_fts5_unavailable_error(unrelated_err) is False
    +
    +    def test_is_trigram_unavailable_error(self):
    +        """Unit test: _is_trigram_unavailable_error is scoped to trigram."""
    +        trigram_err = sqlite3.OperationalError("no such tokenizer: trigram")
    +        generic_err = sqlite3.OperationalError("no such tokenizer: foo")
    +        fts5_err = sqlite3.OperationalError("no such module: fts5")
    +
    +        assert SessionDB._is_trigram_unavailable_error(trigram_err) is True
    +        assert SessionDB._is_trigram_unavailable_error(generic_err) is False
    +        assert SessionDB._is_trigram_unavailable_error(fts5_err) is False
    +
    +    def test_db_initializes_without_trigram_tokenizer(self, tmp_path, monkeypatch):
    +        """SessionDB must not crash when FTS5 exists but trigram tokenizer is missing."""
    +        real_connect = sqlite3.connect
    +
    +        def connect_without_trigram(*args, **kwargs):
    +            kwargs["factory"] = _NoTrigramConnection
    +            return real_connect(*args, **kwargs)
    +
    +        monkeypatch.setattr("hermes_state.sqlite3.connect", connect_without_trigram)
    +
    +        db = SessionDB(db_path=tmp_path / "state.db")
    +        try:
    +            # Base FTS5 should still work (trigram is optional).
    +            assert db._fts_enabled is True
    +            assert db._fts_table_exists("messages_fts") is True
    +            # Trigram table should NOT have been created.
    +            assert db._fts_table_exists("messages_fts_trigram") is False
    +
    +            db.create_session(session_id="s1", source="cli")
    +            db.append_message("s1", role="user", content="hello without trigram")
    +
    +            messages = db.get_messages("s1")
    +            assert len(messages) == 1
    +            assert messages[0]["content"] == "hello without trigram"
    +
    +            # FTS5 keyword search should still work.
    +            assert len(db.search_messages("hello")) == 1
    +        finally:
    +            db.close()
    +
    +    def test_v11_migration_backfills_base_fts_when_trigram_unavailable(
    +        self, tmp_path, monkeypatch
    +    ):
    +        """Regression: v11 migration must backfill base FTS even when trigram is unavailable."""
    +        real_connect = sqlite3.connect
    +        db_path = tmp_path / "state.db"
    +
    +        # Phase 1: create a DB at schema v10 with messages.
    +        db = SessionDB(db_path=db_path)
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="legacy message alpha")
    +        db.append_message("s1", role="assistant", content="legacy reply beta")
    +        # Force schema version to v10 so migration runs on next open.
    +        db._conn.execute(
    +            "UPDATE schema_version SET version = 10"
    +        )
    +        db._conn.commit()
    +        db.close()
    +
    +        # Phase 2: reopen with trigram disabled — migration should still
    +        # backfill base FTS and make existing messages searchable.
    +        def connect_without_trigram(*args, **kwargs):
    +            kwargs["factory"] = _NoTrigramConnection
    +            return real_connect(*args, **kwargs)
    +
    +        monkeypatch.setattr("hermes_state.sqlite3.connect", connect_without_trigram)
    +        migrated_db = SessionDB(db_path=db_path)
    +        try:
    +            assert migrated_db._fts_enabled is True
    +            assert migrated_db._trigram_available is False
    +            assert migrated_db._fts_table_exists("messages_fts") is True
    +            assert migrated_db._fts_table_exists("messages_fts_trigram") is False
    +
    +            # Existing messages must be searchable via base FTS.
    +            results = migrated_db.search_messages("legacy message")
    +            assert len(results) == 1
    +            # snippet has FTS5 highlight markers (>>>...<<<); check raw content via get_messages
    +            msgs = migrated_db.get_messages("s1")
    +            assert any("legacy message" in m["content"] for m in msgs)
    +        finally:
    +            migrated_db.close()
    +
    +    def test_cjk_search_falls_back_to_like_when_trigram_unavailable(
    +        self, tmp_path, monkeypatch
    +    ):
    +        """Regression: long CJK queries must fall back to LIKE when trigram is missing."""
    +        real_connect = sqlite3.connect
    +        db_path = tmp_path / "state.db"
    +
    +        def connect_without_trigram(*args, **kwargs):
    +            kwargs["factory"] = _NoTrigramConnection
    +            return real_connect(*args, **kwargs)
    +
    +        monkeypatch.setattr("hermes_state.sqlite3.connect", connect_without_trigram)
    +        db = SessionDB(db_path=db_path)
    +        try:
    +            db.create_session(session_id="s1", source="cli")
    +            db.append_message("s1", role="user", content="大别山项目计划书")
    +            db.append_message("s1", role="user", content="长江大桥设计方案")
    +
    +            # 3+ CJK chars would normally use trigram, but it's unavailable.
    +            # Must fall back to LIKE and still return results.
    +            results = db.search_messages("大别山")
    +            assert len(results) == 1
    +            # Note: search_messages strips 'content' from results; use 'snippet'.
    +            assert "大别山" in results[0]["snippet"]
    +        finally:
    +            db.close()
    +
     
     # =========================================================================
     # Message storage