{"repo":"NousResearch/hermes-agent","public_id":"11511","display_label":"CJK short-query FTS5 failures and LIKE fallback fixes","summary":"These issues and PRs address a recurring problem where session_search returns empty results for short or OR-combined CJK (Chinese/Japanese/Korean) queries due to FTS5/trigram limitations. The proposed fixes involve routing such queries to a LIKE-based fallback instead of FTS5.","subsystem":"agent","counts":{"members":7,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":2,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":12075},"resolution":{"state":"resolved","by":"pr-12075,pr-22847"},"freshness":{"state":"verified","last_verified_at":"2026-05-10T00:53:03+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.2552380952380952},"fix_edges":[{"issue_number":11511,"pr_number":12075,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":20494,"pr_number":22847,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":11511,"title":"session_search: FTS5 returns empty results for Chinese/CJK queries","url":null,"state":"closed"},{"kind":"issue","number":20494,"title":"session_search returns 0 hits for OR-combined ≤2-char CJK queries (trigram picked but can't match)","url":null,"state":"closed"},{"kind":"pr","number":11516,"title":"fix: FTS5 LIKE fallback for CJK (Chinese/Japanese/Korean) queries","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#12075","number":12075},{"kind":"pr","value":"#11511","number":11511},{"kind":"pr","value":"#11516","number":11516},{"kind":"pr","value":"#11517","number":11517},{"kind":"pr","value":"#11541","number":11541},{"kind":"pr","value":"#11517","number":11517}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":11517,"title":"fix: add LIKE fallback for CJK queries in session_search","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#12075","number":12075},{"kind":"pr","value":"#11511","number":11511},{"kind":"pr","value":"#11516","number":11516},{"kind":"pr","value":"#11517","number":11517},{"kind":"pr","value":"#11541","number":11541}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":12075,"title":"fix(session-search): LIKE fallback for CJK queries (salvages #11516, #11517, #11541)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_state.py","scripts/release.py","tests/test_hermes_state.py"],"fix_hunks":[{"file":"hermes_state.py","hunks":["@@ -987,6 +987,22 @@ def _preserve_quoted(m: re.Match) -> str:\n \n         return sanitized.strip()\n \n+\n+    @staticmethod\n+    def _contains_cjk(text: str) -> bool:\n+        \"\"\"Check if text contains CJK (Chinese, Japanese, Korean) characters.\"\"\"\n+        for ch in text:\n+            cp = ord(ch)\n+            if (0x4E00 <= cp <= 0x9FFF or    # CJK Unified Ideographs\n+                0x3400 <= cp <= 0x4DBF or    # CJK Extension A\n+                0x20000 <= cp <= 0x2A6DF or  # CJK Extension B\n+                0x3000 <= cp <= 0x303F or    # CJK Symbols\n+                0x3040 <= cp <= 0x309F or    # Hiragana\n+                0x30A0 <= cp <= 0x30FF or    # Katakana\n+                0xAC00 <= cp <= 0xD7AF):     # Hangul Syllables\n+                return True\n+        return False\n+\n     def search_messages(\n         self,\n         query: str,","@@ -1062,8 +1078,47 @@ def search_messages(\n                 cursor = self._conn.execute(sql, params)\n             except sqlite3.OperationalError:\n                 # FTS5 query syntax error despite sanitization — return empty\n-                return []\n-            matches = [dict(row) for row in cursor.fetchall()]\n+                # unless query contains CJK (fall back to LIKE below)\n+                if not self._contains_cjk(query):\n+                    return []\n+                matches = []\n+            else:\n+                matches = [dict(row) for row in cursor.fetchall()]\n+\n+        # LIKE fallback for CJK queries: FTS5 default tokenizer splits CJK\n+        # characters individually, causing multi-character queries to fail.\n+        if not matches and self._contains_cjk(query):\n+            raw_query = query.strip('\"').strip()\n+            like_where = [\"m.content LIKE ?\"]\n+            like_params: list = [f\"%{raw_query}%\"]\n+            if source_filter is not None:\n+                like_where.append(f\"s.source IN ({','.join('?' for _ in source_filter)})\")\n+                like_params.extend(source_filter)\n+            if exclude_sources is not None:\n+                like_where.append(f\"s.source NOT IN ({','.join('?' for _ in exclude_sources)})\")\n+                like_params.extend(exclude_sources)\n+            if role_filter:\n+                like_where.append(f\"m.role IN ({','.join('?' for _ in role_filter)})\")\n+                like_params.extend(role_filter)\n+            like_sql = f\"\"\"\n+                SELECT m.id, m.session_id, m.role,\n+                       substr(m.content,\n+                              max(1, instr(m.content, ?) - 40),\n+                              120) AS snippet,\n+                       m.content, m.timestamp, m.tool_name,\n+                       s.source, s.model, s.started_at AS session_started\n+                FROM messages m\n+                JOIN sessions s ON s.id = m.session_id\n+                WHERE {' AND '.join(like_where)}\n+                ORDER BY m.timestamp DESC\n+                LIMIT ? OFFSET ?\n+            \"\"\"\n+            like_params.extend([limit, offset])\n+            # instr() parameter goes first in the bound list\n+            like_params = [raw_query] + like_params\n+            with self._lock:\n+                like_cursor = self._conn.execute(like_sql, like_params)\n+                matches = [dict(row) for row in like_cursor.fetchall()]\n \n         # Add surrounding context (1 message before + after each match).\n         # Done outside the lock so we don't hold it across N sequential queries."]},{"file":"scripts/release.py","hunks":["@@ -207,6 +207,7 @@\n     \"[redacted:email]\": \"cola-runner\",\n     \"[redacted:email]\": \"ygd58\",\n     \"[redacted:email]\": \"vominh1919\",\n+    \"[redacted:email]\": \"iamagenius00\",\n     \"[redacted:email]\": \"trevthefoolish\",\n     \"[redacted:email]\": \"ziliangpeng\",\n     \"[redacted:email]\": \"centripetal-star\","]},{"file":"tests/test_hermes_state.py","hunks":["@@ -479,6 +479,141 @@ def test_sanitize_fts5_quotes_dotted_terms(self):\n         assert s('my-app.config.ts') == '\"my-app.config.ts\"'\n \n \n+# =========================================================================\n+# CJK (Chinese/Japanese/Korean) LIKE fallback\n+# =========================================================================\n+\n+class TestCJKSearchFallback:\n+    \"\"\"Regression tests for CJK search (see #11511).\n+\n+    SQLite FTS5's default tokenizer treats contiguous CJK runs as a single\n+    token (\"和其他agent的聊天记录\" → one token), so substring queries like\n+    \"记忆断裂\" return 0 rows despite the data being present. SessionDB falls\n+    back to LIKE substring matching whenever FTS5 returns no results and\n+    the query contains CJK characters.\n+    \"\"\"\n+\n+    def test_cjk_detection_covers_all_ranges(self):\n+        from hermes_state import SessionDB\n+        f = SessionDB._contains_cjk\n+        # Chinese (CJK Unified Ideographs)\n+        assert f(\"记忆断裂\") is True\n+        # Japanese Hiragana + Katakana\n+        assert f(\"こんにちは\") is True\n+        assert f(\"カタカナ\") is True\n+        # Korean Hangul syllables (both early and late — guards against\n+        # the \\ud7a0-\\ud7af typo seen in one of the duplicate PRs)\n+        assert f(\"안녕하세요\") is True\n+        assert f(\"기억\") is True\n+        # Non-CJK\n+        assert f(\"hello world\") is False\n+        assert f(\"日本語mixedwithenglish\") is True\n+        assert f(\"\") is False\n+\n+    def test_chinese_multichar_query_returns_results(self, db):\n+        \"\"\"The headline bug: multi-char Chinese query must not return [].\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\n+            \"s1\", role=\"user\",\n+            content=\"昨天和其他Agent的聊天记录，记忆断裂问题复现了\",\n+        )\n+        results = db.search_messages(\"记忆断裂\")\n+        assert len(results) == 1\n+        assert results[0][\"session_id\"] == \"s1\"\n+\n+    def test_chinese_bigram_query(self, db):\n+        db.create_session(session_id=\"s1\", source=\"telegram\")\n+        db.append_message(\"s1\", role=\"user\", content=\"今天讨论A2A通信协议的实现\")\n+        results = db.search_messages(\"通信\")\n+        assert len(results) == 1\n+\n+    def test_korean_query_returns_results(self, db):\n+        \"\"\"Guards against Hangul range typos (\\\\uac00-\\\\ud7af, not \\\\ud7a0-).\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"안녕하세요 반갑습니다\")\n+        results = db.search_messages(\"안녕\")\n+        assert len(results) == 1\n+\n+    def test_japanese_query_returns_results(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"こんにちは世界\")\n+        assert len(db.search_messages(\"こんにちは\")) == 1\n+        assert len(db.search_messages(\"世界\")) == 1\n+\n+    def test_cjk_fallback_preserves_source_filter(self, db):\n+        \"\"\"Guards against the SQL-builder bug where filter clauses land\n+        after LIMIT/OFFSET (seen in one of the duplicate PRs).\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"telegram\")\n+        db.append_message(\"s1\", role=\"user\", content=\"记忆断裂在CLI\")\n+        db.append_message(\"s2\", role=\"user\", content=\"记忆断裂在Telegram\")\n+\n+        results = db.search_messages(\"记忆断裂\", source_filter=[\"telegram\"])\n+        assert len(results) == 1\n+        assert results[0][\"source\"] == \"telegram\"\n+\n+    def test_cjk_fallback_preserves_exclude_sources(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"tool\")\n+        db.append_message(\"s1\", role=\"user\", content=\"记忆断裂在CLI\")\n+        db.append_message(\"s2\", role=\"assistant\", content=\"记忆断裂在tool\")\n+\n+        results = db.search_messages(\"记忆断裂\", exclude_sources=[\"tool\"])\n+        sources = {r[\"source\"] for r in results}\n+        assert \"tool\" not in sources\n+        assert \"cli\" in sources\n+\n+    def test_cjk_fallback_preserves_role_filter(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"用户说的记忆断裂\")\n+        db.append_message(\"s1\", role=\"assistant\", content=\"助手说的记忆断裂\")\n+\n+        results = db.search_messages(\"记忆断裂\", role_filter=[\"assistant\"])\n+        assert len(results) == 1\n+        assert results[0][\"role\"] == \"assistant\"\n+\n+    def test_cjk_snippet_is_centered_on_match(self, db):\n+        \"\"\"Snippet should contain the search term, not just the first N chars.\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        long_prefix = \"这是一段很长的前缀用来把匹配位置推到文档中间\" * 3\n+        long_suffix = \"这是一段很长的后缀内容填充剩余空间\" * 3\n+        db.append_message(\n+            \"s1\", role=\"user\",\n+            content=f\"{long_prefix}记忆断裂{long_suffix}\",\n+        )\n+        results = db.search_messages(\"记忆断裂\")\n+        assert len(results) == 1\n+        # The centered substr() snippet must include the matched term.\n+        assert \"记忆断裂\" in results[0][\"snippet\"]\n+\n+    def test_english_query_still_uses_fts5_fast_path(self, db):\n+        \"\"\"English queries must not trigger the LIKE fallback (fast path regression).\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"Deploy docker containers\")\n+        results = db.search_messages(\"docker\")\n+        assert len(results) == 1\n+        # No CJK in query → LIKE fallback must not run. We don't assert this\n+        # directly (no instrumentation), but the FTS5 path produces an\n+        # FTS5-style snippet with highlight markers when the term is short.\n+        # At minimum: english queries must still match.\n+\n+    def test_cjk_query_with_no_matches_returns_empty(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"unrelated English content\")\n+        results = db.search_messages(\"记忆断裂\")\n+        assert results == []\n+\n+    def test_mixed_cjk_english_query(self, db):\n+        \"\"\"Mixed queries should still fall back to LIKE when FTS5 misses.\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"讨论Agent通信协议\")\n+        # \"Agent通信\" is CJK+English — FTS5 default tokenizer indexes the\n+        # whole CJK run with embedded \"agent\" as separate tokens; the LIKE\n+        # fallback handles the substring correctly.\n+        results = db.search_messages(\"Agent通信\")\n+        assert len(results) == 1\n+\n+\n # =========================================================================\n # Session search and listing\n # =========================================================================\n"]}]},{"kind":"pr","number":20499,"title":"fix(session-search): route short CJK OR queries to LIKE","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"058c50816c70c5f9a1253a87776d50e8df4c5dcf","number":0},{"kind":"pr","value":"#20494","number":20494}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":22847,"title":"fix(session): route OR-combined short CJK tokens to LIKE fallback (salvage #22649)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_state.py","tests/test_hermes_state.py"],"fix_hunks":[{"file":"hermes_state.py","hunks":["@@ -1958,7 +1958,19 @@ def search_messages(\n             raw_query = query.strip('\"').strip()\n             cjk_count = self._count_cjk(raw_query)\n \n-            if cjk_count >= 3:\n+            # Per-token CJK length check (#20494): trigram needs >=3 CJK chars\n+            # per token. A query like \"广西 OR 桂林 OR 漓江\" has cjk_count=6\n+            # (>=3) but each individual token is only 2 chars — trigram returns 0.\n+            # Route to LIKE when any non-operator CJK token is <3 CJK chars.\n+            _tokens_for_check = [\n+                t for t in raw_query.split()\n+                if t.upper() not in (\"AND\", \"OR\", \"NOT\") and self._contains_cjk(t)\n+            ]\n+            _any_short_cjk = any(\n+                self._count_cjk(t) < 3 for t in _tokens_for_check\n+            )\n+\n+            if cjk_count >= 3 and not _any_short_cjk:\n                 # Trigram FTS5 path — quote each non-operator token to handle\n                 # FTS5 special chars (%, *, etc.) while preserving boolean\n                 # operators (AND, OR, NOT) for multi-term queries.","@@ -2009,11 +2021,24 @@ def search_messages(\n                     else:\n                         matches = [dict(row) for row in tri_cursor.fetchall()]\n             else:\n-                # Short CJK query (1-2 chars) — trigram needs ≥3 CJK chars.\n-                # Fall back to LIKE substring search.\n-                escaped = raw_query.replace(\"\\\\\", \"\\\\\\\\\").replace(\"%\", \"\\\\%\").replace(\"_\", \"\\\\_\")\n-                like_where = [\"(m.content LIKE ? ESCAPE '\\\\' OR m.tool_name LIKE ? ESCAPE '\\\\' OR m.tool_calls LIKE ? ESCAPE '\\\\')\"]\n-                like_params: list = [f\"%{escaped}%\", f\"%{escaped}%\", f\"%{escaped}%\"]\n+                # Short / mixed CJK query: trigram cannot match tokens with\n+                # <3 CJK chars. Fall back to LIKE substring search.\n+                # For multi-token OR queries (e.g. \"广西 OR 桂林 OR 漓江\"),\n+                # build one LIKE condition per non-operator token so each term\n+                # is matched independently (#20494).\n+                non_op_tokens = [\n+                    t for t in raw_query.split()\n+                    if t.upper() not in (\"AND\", \"OR\", \"NOT\")\n+                ] or [raw_query]\n+                token_clauses = []\n+                like_params: list = []\n+                for tok in non_op_tokens:\n+                    esc = tok.replace(\"\\\\\", \"\\\\\\\\\").replace(\"%\", \"\\\\%\").replace(\"_\", \"\\\\_\")\n+                    token_clauses.append(\n+                        \"(m.content LIKE ? ESCAPE '\\\\' OR m.tool_name LIKE ? ESCAPE '\\\\' OR m.tool_calls LIKE ? ESCAPE '\\\\')\"\n+                    )\n+                    like_params += [f\"%{esc}%\", f\"%{esc}%\", f\"%{esc}%\"]\n+                like_where = [f\"({' OR '.join(token_clauses)})\"]\n                 if source_filter is not None:\n                     like_where.append(f\"s.source IN ({','.join('?' for _ in source_filter)})\")\n                     like_params.extend(source_filter)","@@ -2037,8 +2062,8 @@ def search_messages(\n                     LIMIT ? OFFSET ?\n                 \"\"\"\n                 like_params.extend([limit, offset])\n-                # instr() parameter goes first in the bound list\n-                like_params = [raw_query] + like_params\n+                # instr() for snippet uses first search token\n+                like_params = [non_op_tokens[0]] + like_params\n                 with self._lock:\n                     like_cursor = self._conn.execute(like_sql, like_params)\n                     matches = [dict(row) for row in like_cursor.fetchall()]"]},{"file":"tests/test_hermes_state.py","hunks":["@@ -957,6 +957,39 @@ def test_cjk_trigram_preserves_boolean_operators(self, db):\n         session_ids = {r[\"session_id\"] for r in results}\n         assert session_ids == {\"s1\", \"s2\"}\n \n+    def test_cjk_or_combined_short_tokens_returns_results(self, db):\n+        \"\"\"Regression test for #20494.\n+\n+        OR-combined 2-char CJK tokens (e.g. \"广西 OR 桂林 OR 漓江 OR 旅游\")\n+        previously returned 0 results because _count_cjk of the whole query\n+        was >=3 (8 chars here), selecting the trigram path, but each individual\n+        token is only 2 CJK chars and trigram requires >=3 chars per token.\n+        The per-token check must route such queries to the LIKE fallback.\n+        \"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"telegram\")\n+        db.create_session(session_id=\"s3\", source=\"cli\")\n+        db.append_message(\"s1\", role=\"user\", content=\"广西是个好地方，去过桂林\")\n+        db.append_message(\"s2\", role=\"user\", content=\"漓江风景很美，值得旅游\")\n+        db.append_message(\"s3\", role=\"user\", content=\"unrelated English content\")\n+\n+        results = db.search_messages(\"广西 OR 桂林 OR 漓江 OR 旅游\")\n+        session_ids = {r[\"session_id\"] for r in results}\n+        assert \"s1\" in session_ids, \"广西/桂林 terms not matched\"\n+        assert \"s2\" in session_ids, \"漓江/旅游 terms not matched\"\n+        assert \"s3\" not in session_ids, \"unrelated message must not match\"\n+\n+    def test_cjk_short_token_or_query_preserves_filters(self, db):\n+        \"\"\"Source filter applies correctly in the short-token LIKE path (#20494).\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.create_session(session_id=\"s2\", source=\"telegram\")\n+        db.append_message(\"s1\", role=\"user\", content=\"广西旅游攻略cli\")\n+        db.append_message(\"s2\", role=\"user\", content=\"广西旅游攻略telegram\")\n+\n+        results = db.search_messages(\"广西 OR 旅游\", source_filter=[\"telegram\"])\n+        assert len(results) == 1\n+        assert results[0][\"source\"] == \"telegram\"\n+\n \n # =========================================================================\n # Session search and listing\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}