CJK short-query FTS5 failures and LIKE fallback fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #12075 fix(session-search): LIKE fallback for CJK queries (salvages #11516, #11517, #11541)

  merged to main

  files: hermes_state.py, scripts/release.py, tests/test_hermes_state.py

  Show the diff

  • hermes_state.py

    @@ -987,6 +987,22 @@ def _preserve_quoted(m: re.Match) -> str:
     
             return sanitized.strip()
     
    +
    +    @staticmethod
    +    def _contains_cjk(text: str) -> bool:
    +        """Check if text contains CJK (Chinese, Japanese, Korean) characters."""
    +        for ch in text:
    +            cp = ord(ch)
    +            if (0x4E00 <= cp <= 0x9FFF or    # CJK Unified Ideographs
    +                0x3400 <= cp <= 0x4DBF or    # CJK Extension A
    +                0x20000 <= cp <= 0x2A6DF or  # CJK Extension B
    +                0x3000 <= cp <= 0x303F or    # CJK Symbols
    +                0x3040 <= cp <= 0x309F or    # Hiragana
    +                0x30A0 <= cp <= 0x30FF or    # Katakana
    +                0xAC00 <= cp <= 0xD7AF):     # Hangul Syllables
    +                return True
    +        return False
    +
         def search_messages(
             self,
             query: str,
    @@ -1062,8 +1078,47 @@ def search_messages(
                     cursor = self._conn.execute(sql, params)
                 except sqlite3.OperationalError:
                     # FTS5 query syntax error despite sanitization — return empty
    -                return []
    -            matches = [dict(row) for row in cursor.fetchall()]
    +                # unless query contains CJK (fall back to LIKE below)
    +                if not self._contains_cjk(query):
    +                    return []
    +                matches = []
    +            else:
    +                matches = [dict(row) for row in cursor.fetchall()]
    +
    +        # LIKE fallback for CJK queries: FTS5 default tokenizer splits CJK
    +        # characters individually, causing multi-character queries to fail.
    +        if not matches and self._contains_cjk(query):
    +            raw_query = query.strip('"').strip()
    +            like_where = ["m.content LIKE ?"]
    +            like_params: list = [f"%{raw_query}%"]
    +            if source_filter is not None:
    +                like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})")
    +                like_params.extend(source_filter)
    +            if exclude_sources is not None:
    +                like_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})")
    +                like_params.extend(exclude_sources)
    +            if role_filter:
    +                like_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})")
    +                like_params.extend(role_filter)
    +            like_sql = f"""
    +                SELECT m.id, m.session_id, m.role,
    +                       substr(m.content,
    +                              max(1, instr(m.content, ?) - 40),
    +                              120) AS snippet,
    +                       m.content, m.timestamp, m.tool_name,
    +                       s.source, s.model, s.started_at AS session_started
    +                FROM messages m
    +                JOIN sessions s ON s.id = m.session_id
    +                WHERE {' AND '.join(like_where)}
    +                ORDER BY m.timestamp DESC
    +                LIMIT ? OFFSET ?
    +            """
    +            like_params.extend([limit, offset])
    +            # instr() parameter goes first in the bound list
    +            like_params = [raw_query] + like_params
    +            with self._lock:
    +                like_cursor = self._conn.execute(like_sql, like_params)
    +                matches = [dict(row) for row in like_cursor.fetchall()]
     
             # Add surrounding context (1 message before + after each match).
             # Done outside the lock so we don't hold it across N sequential queries.

  • scripts/release.py

    @@ -207,6 +207,7 @@
         "[redacted:email]": "cola-runner",
         "[redacted:email]": "ygd58",
         "[redacted:email]": "vominh1919",
    +    "[redacted:email]": "iamagenius00",
         "[redacted:email]": "trevthefoolish",
         "[redacted:email]": "ziliangpeng",
         "[redacted:email]": "centripetal-star",

  • tests/test_hermes_state.py

    @@ -479,6 +479,141 @@ def test_sanitize_fts5_quotes_dotted_terms(self):
             assert s('my-app.config.ts') == '"my-app.config.ts"'
     
     
    +# =========================================================================
    +# CJK (Chinese/Japanese/Korean) LIKE fallback
    +# =========================================================================
    +
    +class TestCJKSearchFallback:
    +    """Regression tests for CJK search (see #11511).
    +
    +    SQLite FTS5's default tokenizer treats contiguous CJK runs as a single
    +    token ("和其他agent的聊天记录" → one token), so substring queries like
    +    "记忆断裂" return 0 rows despite the data being present. SessionDB falls
    +    back to LIKE substring matching whenever FTS5 returns no results and
    +    the query contains CJK characters.
    +    """
    +
    +    def test_cjk_detection_covers_all_ranges(self):
    +        from hermes_state import SessionDB
    +        f = SessionDB._contains_cjk
    +        # Chinese (CJK Unified Ideographs)
    +        assert f("记忆断裂") is True
    +        # Japanese Hiragana + Katakana
    +        assert f("こんにちは") is True
    +        assert f("カタカナ") is True
    +        # Korean Hangul syllables (both early and late — guards against
    +        # the \ud7a0-\ud7af typo seen in one of the duplicate PRs)
    +        assert f("안녕하세요") is True
    +        assert f("기억") is True
    +        # Non-CJK
    +        assert f("hello world") is False
    +        assert f("日本語mixedwithenglish") is True
    +        assert f("") is False
    +
    +    def test_chinese_multichar_query_returns_results(self, db):
    +        """The headline bug: multi-char Chinese query must not return []."""
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message(
    +            "s1", role="user",
    +            content="昨天和其他Agent的聊天记录，记忆断裂问题复现了",
    +        )
    +        results = db.search_messages("记忆断裂")
    +        assert len(results) == 1
    +        assert results[0]["session_id"] == "s1"
    +
    +    def test_chinese_bigram_query(self, db):
    +        db.create_session(session_id="s1", source="telegram")
    +        db.append_message("s1", role="user", content="今天讨论A2A通信协议的实现")
    +        results = db.search_messages("通信")
    +        assert len(results) == 1
    +
    +    def test_korean_query_returns_results(self, db):
    +        """Guards against Hangul range typos (\\uac00-\\ud7af, not \\ud7a0-)."""
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="안녕하세요 반갑습니다")
    +        results = db.search_messages("안녕")
    +        assert len(results) == 1
    +
    +    def test_japanese_query_returns_results(self, db):
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="こんにちは世界")
    +        assert len(db.search_messages("こんにちは")) == 1
    +        assert len(db.search_messages("世界")) == 1
    +
    +    def test_cjk_fallback_preserves_source_filter(self, db):
    +        """Guards against the SQL-builder bug where filter clauses land
    +        after LIMIT/OFFSET (seen in one of the duplicate PRs)."""
    +        db.create_session(session_id="s1", source="cli")
    +        db.create_session(session_id="s2", source="telegram")
    +        db.append_message("s1", role="user", content="记忆断裂在CLI")
    +        db.append_message("s2", role="user", content="记忆断裂在Telegram")
    +
    +        results = db.search_messages("记忆断裂", source_filter=["telegram"])
    +        assert len(results) == 1
    +        assert results[0]["source"] == "telegram"
    +
    +    def test_cjk_fallback_preserves_exclude_sources(self, db):
    +        db.create_session(session_id="s1", source="cli")
    +        db.create_session(session_id="s2", source="tool")
    +        db.append_message("s1", role="user", content="记忆断裂在CLI")
    +        db.append_message("s2", role="assistant", content="记忆断裂在tool")
    +
    +        results = db.search_messages("记忆断裂", exclude_sources=["tool"])
    +        sources = {r["source"] for r in results}
    +        assert "tool" not in sources
    +        assert "cli" in sources
    +
    +    def test_cjk_fallback_preserves_role_filter(self, db):
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="用户说的记忆断裂")
    +        db.append_message("s1", role="assistant", content="助手说的记忆断裂")
    +
    +        results = db.search_messages("记忆断裂", role_filter=["assistant"])
    +        assert len(results) == 1
    +        assert results[0]["role"] == "assistant"
    +
    +    def test_cjk_snippet_is_centered_on_match(self, db):
    +        """Snippet should contain the search term, not just the first N chars."""
    +        db.create_session(session_id="s1", source="cli")
    +        long_prefix = "这是一段很长的前缀用来把匹配位置推到文档中间" * 3
    +        long_suffix = "这是一段很长的后缀内容填充剩余空间" * 3
    +        db.append_message(
    +            "s1", role="user",
    +            content=f"{long_prefix}记忆断裂{long_suffix}",
    +        )
    +        results = db.search_messages("记忆断裂")
    +        assert len(results) == 1
    +        # The centered substr() snippet must include the matched term.
    +        assert "记忆断裂" in results[0]["snippet"]
    +
    +    def test_english_query_still_uses_fts5_fast_path(self, db):
    +        """English queries must not trigger the LIKE fallback (fast path regression)."""
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="Deploy docker containers")
    +        results = db.search_messages("docker")
    +        assert len(results) == 1
    +        # No CJK in query → LIKE fallback must not run. We don't assert this
    +        # directly (no instrumentation), but the FTS5 path produces an
    +        # FTS5-style snippet with highlight markers when the term is short.
    +        # At minimum: english queries must still match.
    +
    +    def test_cjk_query_with_no_matches_returns_empty(self, db):
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="unrelated English content")
    +        results = db.search_messages("记忆断裂")
    +        assert results == []
    +
    +    def test_mixed_cjk_english_query(self, db):
    +        """Mixed queries should still fall back to LIKE when FTS5 misses."""
    +        db.create_session(session_id="s1", source="cli")
    +        db.append_message("s1", role="user", content="讨论Agent通信协议")
    +        # "Agent通信" is CJK+English — FTS5 default tokenizer indexes the
    +        # whole CJK run with embedded "agent" as separate tokens; the LIKE
    +        # fallback handles the substring correctly.
    +        results = db.search_messages("Agent通信")
    +        assert len(results) == 1
    +
    +
     # =========================================================================
     # Session search and listing
     # =========================================================================

• #22847 fix(session): route OR-combined short CJK tokens to LIKE fallback (salvage #22649)

  merged to main

  files: hermes_state.py, tests/test_hermes_state.py

  Show the diff

  • hermes_state.py

    @@ -1958,7 +1958,19 @@ def search_messages(
                 raw_query = query.strip('"').strip()
                 cjk_count = self._count_cjk(raw_query)
     
    -            if cjk_count >= 3:
    +            # Per-token CJK length check (#20494): trigram needs >=3 CJK chars
    +            # per token. A query like "广西 OR 桂林 OR 漓江" has cjk_count=6
    +            # (>=3) but each individual token is only 2 chars — trigram returns 0.
    +            # Route to LIKE when any non-operator CJK token is <3 CJK chars.
    +            _tokens_for_check = [
    +                t for t in raw_query.split()
    +                if t.upper() not in ("AND", "OR", "NOT") and self._contains_cjk(t)
    +            ]
    +            _any_short_cjk = any(
    +                self._count_cjk(t) < 3 for t in _tokens_for_check
    +            )
    +
    +            if cjk_count >= 3 and not _any_short_cjk:
                     # Trigram FTS5 path — quote each non-operator token to handle
                     # FTS5 special chars (%, *, etc.) while preserving boolean
                     # operators (AND, OR, NOT) for multi-term queries.
    @@ -2009,11 +2021,24 @@ def search_messages(
                         else:
                             matches = [dict(row) for row in tri_cursor.fetchall()]
                 else:
    -                # Short CJK query (1-2 chars) — trigram needs ≥3 CJK chars.
    -                # Fall back to LIKE substring search.
    -                escaped = raw_query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
    -                like_where = ["(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')"]
    -                like_params: list = [f"%{escaped}%", f"%{escaped}%", f"%{escaped}%"]
    +                # Short / mixed CJK query: trigram cannot match tokens with
    +                # <3 CJK chars. Fall back to LIKE substring search.
    +                # For multi-token OR queries (e.g. "广西 OR 桂林 OR 漓江"),
    +                # build one LIKE condition per non-operator token so each term
    +                # is matched independently (#20494).
    +                non_op_tokens = [
    +                    t for t in raw_query.split()
    +                    if t.upper() not in ("AND", "OR", "NOT")
    +                ] or [raw_query]
    +                token_clauses = []
    +                like_params: list = []
    +                for tok in non_op_tokens:
    +                    esc = tok.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
    +                    token_clauses.append(
    +                        "(m.content LIKE ? ESCAPE '\\' OR m.tool_name LIKE ? ESCAPE '\\' OR m.tool_calls LIKE ? ESCAPE '\\')"
    +                    )
    +                    like_params += [f"%{esc}%", f"%{esc}%", f"%{esc}%"]
    +                like_where = [f"({' OR '.join(token_clauses)})"]
                     if source_filter is not None:
                         like_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})")
                         like_params.extend(source_filter)
    @@ -2037,8 +2062,8 @@ def search_messages(
                         LIMIT ? OFFSET ?
                     """
                     like_params.extend([limit, offset])
    -                # instr() parameter goes first in the bound list
    -                like_params = [raw_query] + like_params
    +                # instr() for snippet uses first search token
    +                like_params = [non_op_tokens[0]] + like_params
                     with self._lock:
                         like_cursor = self._conn.execute(like_sql, like_params)
                         matches = [dict(row) for row in like_cursor.fetchall()]

  • tests/test_hermes_state.py

    @@ -957,6 +957,39 @@ def test_cjk_trigram_preserves_boolean_operators(self, db):
             session_ids = {r["session_id"] for r in results}
             assert session_ids == {"s1", "s2"}
     
    +    def test_cjk_or_combined_short_tokens_returns_results(self, db):
    +        """Regression test for #20494.
    +
    +        OR-combined 2-char CJK tokens (e.g. "广西 OR 桂林 OR 漓江 OR 旅游")
    +        previously returned 0 results because _count_cjk of the whole query
    +        was >=3 (8 chars here), selecting the trigram path, but each individual
    +        token is only 2 CJK chars and trigram requires >=3 chars per token.
    +        The per-token check must route such queries to the LIKE fallback.
    +        """
    +        db.create_session(session_id="s1", source="cli")
    +        db.create_session(session_id="s2", source="telegram")
    +        db.create_session(session_id="s3", source="cli")
    +        db.append_message("s1", role="user", content="广西是个好地方，去过桂林")
    +        db.append_message("s2", role="user", content="漓江风景很美，值得旅游")
    +        db.append_message("s3", role="user", content="unrelated English content")
    +
    +        results = db.search_messages("广西 OR 桂林 OR 漓江 OR 旅游")
    +        session_ids = {r["session_id"] for r in results}
    +        assert "s1" in session_ids, "广西/桂林 terms not matched"
    +        assert "s2" in session_ids, "漓江/旅游 terms not matched"
    +        assert "s3" not in session_ids, "unrelated message must not match"
    +
    +    def test_cjk_short_token_or_query_preserves_filters(self, db):
    +        """Source filter applies correctly in the short-token LIKE path (#20494)."""
    +        db.create_session(session_id="s1", source="cli")
    +        db.create_session(session_id="s2", source="telegram")
    +        db.append_message("s1", role="user", content="广西旅游攻略cli")
    +        db.append_message("s2", role="user", content="广西旅游攻略telegram")
    +
    +        results = db.search_messages("广西 OR 旅游", source_filter=["telegram"])
    +        assert len(results) == 1
    +        assert results[0]["source"] == "telegram"
    +
     
     # =========================================================================
     # Session search and listing