{"repo":"NousResearch/hermes-agent","public_id":"14829","display_label":"Session search indexing, FTS5 tokenizer, and ranking issues","summary":"These items address session search reliability problems including FTS5/unicode61 tokenizer failures with CJK and Cyrillic text, fallback strategies (LIKE, trigram index) for non-Latin queries, mixed-script boundary issues, and ranking of non-cron sessions in results.","subsystem":"agent","counts":{"members":17,"open_issues":3,"open_issues_distinct":3,"open_prs":1,"closed_issues":3,"merged_prs":3,"attempt_prs":8},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":16651},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-07-22T17:35:05+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.10411764705882352},"fix_edges":[{"issue_number":14829,"pr_number":16651,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":54242,"pr_number":69423,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":54242,"pr_number":69456,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":14829,"title":"FTS5 unicode61 tokenizer silently drops CJK characters, LIKE fallback only triggers on zero results","url":null,"state":"closed"},{"kind":"issue","number":15500,"title":"[BUG] session_search returns zero/partial results despite session files containing matching content","url":null,"state":"closed"},{"kind":"issue","number":16253,"title":"Bug: session_search fails to locate recent non-cron sessions","url":null,"state":"open"},{"kind":"issue","number":24680,"title":"[Bug]: session_search only indexes LLM summaries, not raw session text — cross-session recall unreliable","url":null,"state":"open"},{"kind":"issue","number":54242,"title":"Bug: FTS5 session_search fails for mixed Latin-CJK queries (tokenizer boundary issue)","url":null,"state":"closed"},{"kind":"issue","number":61640,"title":"session_search: Cyrillic queries bypass the trigram index, making recall in Russian near-useless without manual wildcard stemming","url":null,"state":"open"},{"kind":"pr","number":14842,"title":"fix(session_search): supplement FTS5 with LIKE for CJK partial results","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16651","number":16651}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":15509,"title":"fix: bypass FTS5 for CJK queries in session_search","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":16267,"title":"fix(session-search): prioritize non-cron sessions in recent list (#16253)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"142b4bf3c","number":0},{"kind":"pr","value":"#16253","number":16253}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":16276,"title":"fix(session_search): bypass FTS5 for CJK queries, escape LIKE wildcards","url":null,"state":"closed","close_reason":{"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":16651,"title":"feat: trigram FTS5 index for CJK search, replace LIKE fallback","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_state.py","tests/test_hermes_state.py"],"fix_hunks":[{"file":"hermes_state.py","hunks":["@@ -31,7 +31,7 @@\n \n DEFAULT_DB_PATH = get_hermes_home() / \"state.db\"\n \n-SCHEMA_VERSION = 9\n+SCHEMA_VERSION = 10\n \n SCHEMA_SQL = \"\"\"\n CREATE TABLE IF NOT EXISTS schema_version (","@@ -119,6 +119,32 @@\n END;\n \"\"\"\n \n+# Trigram FTS5 table for CJK substring search.  The default unicode61\n+# tokenizer splits CJK characters into individual tokens, breaking phrase\n+# matching.  The trigram tokenizer creates overlapping 3-byte sequences so\n+# substring queries work natively for any script (CJK, Thai, etc.).\n+FTS_TRIGRAM_SQL = \"\"\"\n+CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5(\n+    content,\n+    content=messages,\n+    content_rowid=id,\n+    tokenize='[redacted:secret-assignment]'\n+);\n+\n+CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN\n+    INSERT INTO messages_fts_trigram(rowid, content) VALUES (new.id, new.content);\n+END;\n+\n+CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages BEGIN\n+    INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content) VALUES('delete', old.id, old.content);\n+END;\n+\n+CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages BEGIN\n+    INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content) VALUES('delete', old.id, old.content);\n+    INSERT INTO messages_fts_trigram(rowid, content) VALUES (new.id, new.content);\n+END;\n+\"\"\"\n+\n \n class SessionDB:\n     \"\"\"","@@ -366,6 +392,18 @@ def _init_schema(self):\n                 except sqlite3.OperationalError:\n                     pass  # Column already exists\n                 cursor.execute(\"UPDATE schema_version SET version = 9\")\n+            if current_version < 10:\n+                # v10: trigram FTS5 table for CJK/substring search.\n+                # Created via FTS_TRIGRAM_SQL below; backfill existing messages.\n+                try:\n+                    cursor.execute(\"SELECT * FROM messages_fts_trigram LIMIT 0\")\n+                except sqlite3.OperationalError:\n+                    cursor.executescript(FTS_TRIGRAM_SQL)\n+                    cursor.execute(\n+                        \"INSERT INTO messages_fts_trigram(rowid, content) \"\n+                        \"SELECT id, content FROM messages WHERE content IS NOT NULL\"\n+                    )\n+                cursor.execute(\"UPDATE schema_version SET version = 10\")\n \n         # Unique title index — always ensure it exists (safe to run after migrations\n         # since the title column is guaranteed to exist at this point)","@@ -383,6 +421,12 @@ def _init_schema(self):\n         except sqlite3.OperationalError:\n             cursor.executescript(FTS_SQL)\n \n+        # Trigram FTS5 for CJK/substring search\n+        try:\n+            cursor.execute(\"SELECT * FROM messages_fts_trigram LIMIT 0\")\n+        except sqlite3.OperationalError:\n+            cursor.executescript(FTS_TRIGRAM_SQL)\n+\n         self._conn.commit()\n \n     # =========================================================================","@@ -1246,6 +1290,16 @@ def _preserve_quoted(m: re.Match) -> str:\n         return sanitized.strip()\n \n \n+    @staticmethod\n+    def _is_cjk_codepoint(cp: int) -> bool:\n+        return (0x4E00 <= cp <= 0x9FFF or    # CJK Unified Ideographs\n+                0x3400 <= cp <= 0x4DBF or    # CJK Extension A\n+                0x20000 <= cp <= 0x2A6DF or  # CJK Extension B\n+                0x3000 <= cp <= 0x303F or    # CJK Symbols\n+                0x3040 <= cp <= 0x309F or    # Hiragana\n+                0x30A0 <= cp <= 0x30FF or    # Katakana\n+                0xAC00 <= cp <= 0xD7AF)      # Hangul Syllables\n+\n     @staticmethod\n     def _contains_cjk(text: str) -> bool:\n         \"\"\"Check if text contains CJK (Chinese, Japanese, Korean) characters.\"\"\"","@@ -1261,6 +1315,11 @@ def _contains_cjk(text: str) -> bool:\n                 return True\n         return False\n \n+    @classmethod\n+    def _count_cjk(cls, text: str) -> int:\n+        \"\"\"Count CJK characters in text.\"\"\"\n+        return sum(1 for ch in text if cls._is_cjk_codepoint(ord(ch)))\n+\n     def search_messages(\n         self,\n         query: str,","@@ -1331,52 +1390,113 @@ def search_messages(\n             LIMIT ? OFFSET ?\n         \"\"\"\n \n-        with self._lock:\n-            try:\n-                cursor = self._conn.execute(sql, params)\n-            except sqlite3.OperationalError:\n-                # FTS5 query syntax error despite sanitization — return empty\n-                # unless query contains CJK (fall back to LIKE below)\n-                if not self._contains_cjk(query):\n-                    return []\n-                matches = []\n-            else:\n-                matches = [dict(row) for row in cursor.fetchall()]\n-\n-        # LIKE fallback for CJK queries: FTS5 default tokenizer splits CJK\n-        # characters individually, causing multi-character queries to fail.\n-        if not matches and self._contains_cjk(query):\n+        # CJK queries bypass the unicode61 FTS5 table.  The default tokenizer\n+        # splits CJK characters into individual tokens, so \"大别山项目\" becomes\n+        # \"大 AND 别 AND 山 AND 项 AND 目\" — producing false positives and\n+        # missing exact phrase matches.\n+        #\n+        # For queries with 3+ CJK characters, we use the trigram FTS5 table\n+        # (indexed substring matching with ranking and snippets).  For shorter\n+        # CJK queries (1-2 chars), trigram can't match (it needs ≥9 UTF-8\n+        # bytes = 3 CJK chars), so we fall back to LIKE.\n+        is_cjk = self._contains_cjk(query)\n+        if is_cjk:\n             raw_query = query.strip('\"').strip()\n-            like_where = [\"m.content LIKE ?\"]\n-            like_params: list = [f\"%{raw_query}%\"]\n-            if source_filter is not None:\n-                like_where.append(f\"s.source IN ({','.join('?' for _ in source_filter)})\")\n-                like_params.extend(source_filter)\n-            if exclude_sources is not None:\n-                like_where.append(f\"s.source NOT IN ({','.join('?' for _ in exclude_sources)})\")\n-                like_params.extend(exclude_sources)\n-            if role_filter:\n-                like_where.append(f\"m.role IN ({','.join('?' for _ in role_filter)})\")\n-                like_params.extend(role_filter)\n-            like_sql = f\"\"\"\n-                SELECT m.id, m.session_id, m.role,\n-                       substr(m.content,\n-                              max(1, instr(m.content, ?) - 40),\n-                              120) AS snippet,\n-                       m.content, m.timestamp, m.tool_name,\n-                       s.source, s.model, s.started_at AS session_started\n-                FROM messages m\n-                JOIN sessions s ON s.id = m.session_id\n-                WHERE {' AND '.join(like_where)}\n-                ORDER BY m.timestamp DESC\n-                LIMIT ? OFFSET ?\n-            \"\"\"\n-            like_params.extend([limit, offset])\n-            # instr() parameter goes first in the bound list\n-            like_params = [raw_query] + like_params\n+            cjk_count = self._count_cjk(raw_query)\n+\n+            if cjk_count >= 3:\n+                # Trigram FTS5 path — quote each non-operator token to handle\n+                # FTS5 special chars (%, *, etc.) while preserving boolean\n+                # operators (AND, OR, NOT) for multi-term queries.\n+                tokens = [redacted:secret-assignment]\n+                parts = []\n+                for tok in tokens:\n+                    if tok.upper() in (\"AND\", \"OR\", \"NOT\"):\n+                        parts.append(tok)\n+                    else:\n+                        parts.append('\"' + tok.replace('\"', '\"\"') + '\"')\n+                trigram_query = \" \".join(parts)\n+                tri_where = [\"messages_fts_trigram MATCH ?\"]\n+                tri_params: list = [trigram_query]\n+                if source_filter is not None:\n+                    tri_where.append(f\"s.source IN ({','.join('?' for _ in source_filter)})\")\n+                    tri_params.extend(source_filter)\n+                if exclude_sources is not None:\n+                    tri_where.append(f\"s.source NOT IN ({','.join('?' for _ in exclude_sources)})\")\n+                    tri_params.extend(exclude_sources)\n+                if role_filter:\n+                    tri_where.append(f\"m.role IN ({','.join('?' for _ in role_filter)})\")\n+                    tri_params.extend(role_filter)\n+                tri_sql = f\"\"\"\n+                    SELECT\n+                        m.id,\n+                        m.session_id,\n+                        m.role,\n+                        snippet(messages_fts_trigram, 0, '>>>', '<<<', '...', 40) AS snippet,\n+                        m.content,\n+                        m.timestamp,\n+                        m.tool_name,\n+                        s.source,\n+                        s.model,\n+                        s.started_at AS session_started\n+                    FROM messages_fts_trigram\n+                    JOIN messages m ON m.id = messages_fts_trigram.rowid\n+                    JOIN sessions s ON s.id = m.session_id\n+                    WHERE {' AND '.join(tri_where)}\n+                    ORDER BY rank\n+                    LIMIT ? OFFSET ?\n+                \"\"\"\n+                tri_params.extend([limit, offset])\n+                with self._lock:\n+                    try:\n+                        tri_cursor = self._conn.execute(tri_sql, tri_params)\n+                    except sqlite3.OperationalError:\n+                        matches = []\n+                    else:\n+                        matches = [dict(row) for row in tri_cursor.fetchall()]\n+            else:\n+                # Short CJK query (1-2 chars) — trigram needs ≥3 CJK chars.\n+                # Fall back to LIKE substring search.\n+                escaped = raw_query.replace(\"\\\\\", \"\\\\\\\\\").replace(\"%\", \"\\\\%\").replace(\"_\", \"\\\\_\")\n+                like_where = [\"m.content LIKE ? ESCAPE '\\\\'\"]\n+                like_params: list = [f\"%{escaped}%\"]\n+                if source_filter is not None:\n+                    like_where.append(f\"s.source IN ({','.join('?' for _ in source_filter)})\")\n+                    like_params.extend(source_filter)\n+                if exclude_sources is not None:\n+                    like_where.append(f\"s.source NOT IN ({','.join('?' for _ in exclude_sources)})\")\n+                    like_params.extend(exclude_sources)\n+                if role_filter:\n+                    like_where.append(f\"m.role IN ({','.join('?' for _ in role_filter)})\")\n+                    like_params.extend(role_filter)\n+                like_sql = f\"\"\"\n+                    SELECT m.id, m.session_id, m.role,\n+                           substr(m.content,\n+                                  max(1, instr(m.content, ?) - 40),\n+                                  120) AS snippet,\n+                           m.content, m.timestamp, m.tool_name,\n+                           s.source, s.model, s.started_at AS session_started\n+                    FROM messages m\n+                    JOIN sessions s ON s.id = m.session_id\n+                    WHERE {' AND '.join(like_where)}\n+                    ORDER BY m.timestamp DESC\n+                    LIMIT ? OFFSET ?\n+                \"\"\"\n+                like_params.extend([limit, offset])\n+                # instr() parameter goes first in the bound list\n+                like_params = [raw_query] + like_params\n+                with self._lock:\n+                    like_cursor = self._conn.execute(like_sql, like_params)\n+                    matches = [dict(row) for row in like_cursor.fetchall()]\n+        else:\n             with self._lock:\n-                like_cursor = self._conn.execute(like_sql, like_params)\n-                matches = [dict(row) for row in like_cursor.fetchall()]\n+                try:\n+                    cursor = self._conn.execute(sql, params)\n+                except sqlite3.OperationalError:\n+                    # FTS5 query syntax error despite sanitization — return empty\n+                    return []\n+                else:\n+                    matches = [dict(row) for row in cursor.fetchall()]\n \n         # Add surrounding context (1 message before + after each match).\n         # Done outside the lock so we don't hold it across N sequential queries."]},{"file":"tests/test_hermes_state.py","hunks":["@@ -743,6 +743,51 @@ def test_mixed_cjk_english_query(self, db):\n         results = db.search_messages(\"Agent通信\")\n         assert len(results) == 1\n \n+    def test_cjk_partial_fts5_results_supplemented_by_like(self, db):\n+        \"\"\"When FTS5 returns *some* CJK results, LIKE must still find all matches.\n+\n+        Regression test for #15500 / #14829: FTS5 unicode61 tokenizer drops\n+        certain CJK characters, so multi-character queries may return partial\n+        results.  The LIKE path must always run for CJK queries.\n+        \"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"telegram\")\n+        db.append_message(\"s1\", role=\"user\", content=\"昨晚讨论了记忆系统\")\n+        db.append_message(\"s2\", role=\"user\", content=\"昨晚的会议纪要已发送\")\n+        results = db.search_messages(\"昨晚\")\n+        assert len(results) == 2\n+        session_ids = {r[\"session_id\"] for r in results}\n+        assert session_ids == {\"s1\", \"s2\"}\n+\n+    def test_cjk_like_dedup_no_duplicates(self, db):\n+        \"\"\"When FTS5 and LIKE both find the same message, no duplicates.\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"测试去重逻辑\")\n+        results = db.search_messages(\"测试\")\n+        assert len(results) == 1\n+\n+    def test_cjk_like_escapes_wildcards(self, db):\n+        \"\"\"Special characters (%, _) in CJK queries are treated as literals.\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"达成100%完成率\")\n+        db.append_message(\"s2\", role=\"user\", content=\"达成100完成率是目标\")\n+        # The % in the query must be literal — should only match s1\n+        results = db.search_messages(\"100%完成\")\n+        assert len(results) == 1\n+        assert results[0][\"session_id\"] == \"s1\"\n+\n+    def test_cjk_trigram_preserves_boolean_operators(self, db):\n+        \"\"\"Boolean operators (OR, AND, NOT) work in CJK trigram queries.\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"记忆系统很好用\")\n+        db.append_message(\"s2\", role=\"user\", content=\"断裂连接需要修复\")\n+        results = db.search_messages(\"记忆系统 OR 断裂连接\")\n+        assert len(results) == 2\n+        session_ids = {r[\"session_id\"] for r in results}\n+        assert session_ids == {\"s1\", \"s2\"}\n+\n \n # =========================================================================\n # Session search and listing","@@ -1200,7 +1245,7 @@ def test_tables_exist(self, db):\n     def test_schema_version(self, db):\n         cursor = db._conn.execute(\"SELECT version FROM schema_version\")\n         version = cursor.fetchone()[0]\n-        assert version == 9\n+        assert version == 10\n \n     def test_title_column_exists(self, db):\n         \"\"\"Verify the title column was created in the sessions table.\"\"\"","@@ -1261,7 +1306,7 @@ def test_migration_from_v2(self, tmp_path):\n \n         # Verify migration\n         cursor = migrated_db._conn.execute(\"SELECT version FROM schema_version\")\n-        assert cursor.fetchone()[0] == 9\n+        assert cursor.fetchone()[0] == 10\n \n         # Verify title column exists and is NULL for existing sessions\n         session = migrated_db.get_session(\"existing\")\n"]}]},{"kind":"pr","number":24792,"title":"fix(session_search): LIKE fallback + fast mode for reliable cross-session recall","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":54258,"title":"fix(state): use trigram fallback for mixed Latin-CJK session search (#54242)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#54242","number":54242},{"kind":"pr","value":"#69456","number":69456},{"kind":"pr","value":"#54289","number":54289},{"kind":"pr","value":"#54289","number":54289},{"kind":"pr","value":"#69423","number":69423}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":54289,"title":"fix(agent): recover pure-Latin search matches embedded in CJK text (#54242)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"96560ee60f","number":0},{"kind":"pr","value":"#69456","number":69456},{"kind":"pr","value":"#54258","number":54258},{"kind":"pr","value":"#69423","number":69423}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":61646,"title":"fix(session-search): route Cyrillic queries through trigram index","url":null,"state":"open","close_reason":{"refs":[{"kind":"commit","value":"a991dfc25","number":0},{"kind":"commit","value":"53622f5a9","number":0}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":69423,"title":"perf(state): CJK-bigram FTS index — 2-char CJK terms at index speed (salvage of #65544)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":[".gitignore","cli.py","gateway/run.py","hermes_cli/config.py","hermes_cli/main.py","hermes_state.py","native/fts5_cjk/README.md","native/fts5_cjk/build.sh","native/fts5_cjk/fts5_cjk.c","native/fts5_cjk/vendor/sqlite3.h","native/fts5_cjk/vendor/sqlite3ext.h","tests/gateway/test_cjk_fts_config_bridge.py","tests/test_fts_cjk_bigram.py","tests/test_search_slow_query_log.py"]},{"kind":"pr","number":69456,"title":"fix(state): recover pure-Latin search matches embedded in CJK text (salvage of #54289)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_state.py","tests/test_fts_cjk_bigram.py","tests/test_hermes_state.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}