{"repo":"NousResearch/hermes-agent","public_id":"16751","display_label":"FTS5 session_search indexing for tool fields","summary":"These issues and PRs address missing FTS5 indexing of tool_calls and tool_name columns in session_search, including migration/repair approaches and a related change to reduce redundant message text storage.","subsystem":"agent","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":16914},"resolution":{"state":"resolved","by":"pr-16914"},"freshness":{"state":"verified","last_verified_at":"2026-04-28T08:33:01+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":16751,"pr_number":16914,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":16751,"title":"`session_search` does not index `tool_calls` or `tool_name`","url":null,"state":"closed"},{"kind":"pr","number":16770,"title":"fix(session-search): index tool_calls and tool_name columns in FTS5 (#16751)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16914","number":16914},{"kind":"pr","value":"#16866","number":16866},{"kind":"pr","value":"#16914","number":16914},{"kind":"pr","value":"#16914","number":16914}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":16866,"title":"fix(state): index tool_calls and tool_name in FTS5 for session_search","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"cfcad80ee","number":0},{"kind":"pr","value":"#16914","number":16914},{"kind":"pr","value":"#16751","number":16751}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":16914,"title":"fix(state): index tool_name and tool_calls in FTS5, with repair + migration (salvages #16866)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_state.py","scripts/release.py","tests/test_hermes_state.py"],"fix_hunks":[{"file":"hermes_state.py","hunks":["@@ -33,7 +33,7 @@\n \n DEFAULT_DB_PATH = get_hermes_home() / \"state.db\"\n \n-SCHEMA_VERSION = 10\n+SCHEMA_VERSION = 11\n \n SCHEMA_SQL = \"\"\"\n CREATE TABLE IF NOT EXISTS schema_version (","@@ -102,22 +102,26 @@\n \n FTS_SQL = \"\"\"\n CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(\n-    content,\n-    content=messages,\n-    content_rowid=id\n+    content\n );\n \n CREATE TRIGGER IF NOT EXISTS messages_fts_insert AFTER INSERT ON messages BEGIN\n-    INSERT INTO messages_fts(rowid, content) VALUES (new.id, new.content);\n+    INSERT INTO messages_fts(rowid, content) VALUES (\n+        new.id,\n+        COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')\n+    );\n END;\n \n CREATE TRIGGER IF NOT EXISTS messages_fts_delete AFTER DELETE ON messages BEGIN\n-    INSERT INTO messages_fts(messages_fts, rowid, content) VALUES('delete', old.id, old.content);\n+    DELETE FROM messages_fts WHERE rowid = old.id;\n END;\n \n CREATE TRIGGER IF NOT EXISTS messages_fts_update AFTER UPDATE ON messages BEGIN\n-    INSERT INTO messages_fts(messages_fts, rowid, content) VALUES('delete', old.id, old.content);\n-    INSERT INTO messages_fts(rowid, content) VALUES (new.id, new.content);\n+    DELETE FROM messages_fts WHERE rowid = old.id;\n+    INSERT INTO messages_fts(rowid, content) VALUES (\n+        new.id,\n+        COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')\n+    );\n END;\n \"\"\"\n ","@@ -128,22 +132,26 @@\n FTS_TRIGRAM_SQL = \"\"\"\n CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5(\n     content,\n-    content=messages,\n-    content_rowid=id,\n     tokenize='[redacted:secret-assignment]'\n );\n \n CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages BEGIN\n-    INSERT INTO messages_fts_trigram(rowid, content) VALUES (new.id, new.content);\n+    INSERT INTO messages_fts_trigram(rowid, content) VALUES (\n+        new.id,\n+        COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')\n+    );\n END;\n \n CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages BEGIN\n-    INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content) VALUES('delete', old.id, old.content);\n+    DELETE FROM messages_fts_trigram WHERE rowid = old.id;\n END;\n \n CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_update AFTER UPDATE ON messages BEGIN\n-    INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content) VALUES('delete', old.id, old.content);\n-    INSERT INTO messages_fts_trigram(rowid, content) VALUES (new.id, new.content);\n+    DELETE FROM messages_fts_trigram WHERE rowid = old.id;\n+    INSERT INTO messages_fts_trigram(rowid, content) VALUES (\n+        new.id,\n+        COALESCE(new.content, '') || ' ' || COALESCE(new.tool_name, '') || ' ' || COALESCE(new.tool_calls, '')\n+    );\n END;\n \"\"\"\n ","@@ -428,6 +436,51 @@ def _init_schema(self):\n                         \"INSERT INTO messages_fts_trigram(rowid, content) \"\n                         \"SELECT id, content FROM messages WHERE content IS NOT NULL\"\n                     )\n+            if current_version < 11:\n+                # v11: re-index FTS5 tables to cover tool_name + tool_calls and\n+                # switch from external-content to inline mode. Existing DBs have\n+                # old-schema FTS tables and triggers that IF NOT EXISTS won't\n+                # overwrite, so we drop them explicitly and let the post-migration\n+                # existence checks (below) recreate them from FTS_SQL /\n+                # FTS_TRIGRAM_SQL, then backfill every message row. Fixes #16751.\n+                for _trig in (\n+                    \"messages_fts_insert\",\n+                    \"messages_fts_delete\",\n+                    \"messages_fts_update\",\n+                    \"messages_fts_trigram_insert\",\n+                    \"messages_fts_trigram_delete\",\n+                    \"messages_fts_trigram_update\",\n+                ):\n+                    try:\n+                        cursor.execute(f\"DROP TRIGGER IF EXISTS {_trig}\")\n+                    except sqlite3.OperationalError:\n+                        pass\n+                for _tbl in (\"messages_fts\", \"messages_fts_trigram\"):\n+                    try:\n+                        cursor.execute(f\"DROP TABLE IF EXISTS {_tbl}\")\n+                    except sqlite3.OperationalError:\n+                        pass\n+                # Recreate virtual tables + triggers with the new inline-mode\n+                # schema that indexes content || tool_name || tool_calls.\n+                cursor.executescript(FTS_SQL)\n+                cursor.executescript(FTS_TRIGRAM_SQL)\n+                # Backfill both indexes from every existing messages row.\n+                cursor.execute(\n+                    \"INSERT INTO messages_fts(rowid, content) \"\n+                    \"SELECT id, \"\n+                    \"COALESCE(content, '') || ' ' || \"\n+                    \"COALESCE(tool_name, '') || ' ' || \"\n+                    \"COALESCE(tool_calls, '') \"\n+                    \"FROM messages\"\n+                )\n+                cursor.execute(\n+                    \"INSERT INTO messages_fts_trigram(rowid, content) \"\n+                    \"SELECT id, \"\n+                    \"COALESCE(content, '') || ' ' || \"\n+                    \"COALESCE(tool_name, '') || ' ' || \"\n+                    \"COALESCE(tool_calls, '') \"\n+                    \"FROM messages\"\n+                )\n             if current_version < SCHEMA_VERSION:\n                 cursor.execute(\n                     \"UPDATE schema_version SET version = ?\",","@@ -1534,8 +1587,8 @@ def search_messages(\n                 # Short CJK query (1-2 chars) — trigram needs ≥3 CJK chars.\n                 # Fall back to LIKE substring search.\n                 escaped = raw_query.replace(\"\\\\\", \"\\\\\\\\\").replace(\"%\", \"\\\\%\").replace(\"_\", \"\\\\_\")\n-                like_where = [\"m.content LIKE ? ESCAPE '\\\\'\"]\n-                like_params: list = [f\"%{escaped}%\"]\n+                like_where = [\"(m.content LIKE ? ESCAPE '\\\\' OR m.tool_name LIKE ? ESCAPE '\\\\' OR m.tool_calls LIKE ? ESCAPE '\\\\')\"]\n+                like_params: list = [f\"%{escaped}%\", f\"%{escaped}%\", f\"%{escaped}%\"]\n                 if source_filter is not None:\n                     like_where.append(f\"s.source IN ({','.join('?' for _ in source_filter)})\")\n                     like_params.extend(source_filter)"]},{"file":"scripts/release.py","hunks":["@@ -54,6 +54,7 @@\n     \"[redacted:email]\": \"dhabibi\",\n     \"[redacted:email]\": \"johnncenae\",\n     \"[redacted:email]\": \"yes999zc\",\n+    \"[redacted:email]\": \"yes999zc\",\n     \"[redacted:email]\": \"DrStrangerUJN\",\n     \"[redacted:email]\": \"dlkakbs\",\n     \"[redacted:email]\": \"Mind-Dragon\","]},{"file":"tests/test_hermes_state.py","hunks":["@@ -1292,7 +1292,7 @@ def test_tables_exist(self, db):\n     def test_schema_version(self, db):\n         cursor = db._conn.execute(\"SELECT version FROM schema_version\")\n         version = cursor.fetchone()[0]\n-        assert version == 10\n+        assert version == 11\n \n     def test_title_column_exists(self, db):\n         \"\"\"Verify the title column was created in the sessions table.\"\"\"","@@ -1353,7 +1353,7 @@ def test_migration_from_v2(self, tmp_path):\n \n         # Verify migration\n         cursor = migrated_db._conn.execute(\"SELECT version FROM schema_version\")\n-        assert cursor.fetchone()[0] == 10\n+        assert cursor.fetchone()[0] == 11\n \n         # Verify title column exists and is NULL for existing sessions\n         session = migrated_db.get_session(\"existing\")","@@ -2266,3 +2266,198 @@ def test_prune_sessions_deletes_files_for_pruned_only(self, db, tmp_path):\n         assert not (sessions_dir / \"old.jsonl\").exists()\n         assert (sessions_dir / \"active.jsonl\").exists()\n \n+\n+# =========================================================================\n+# FTS5 indexing of tool_calls / tool_name (#16751)\n+# =========================================================================\n+\n+class TestFTS5ToolCallIndexing:\n+    \"\"\"Regression tests: search_messages must see tool_name and tool_calls.\n+\n+    Before #16751's fix, `messages_fts` only indexed `messages.content`, so\n+    tokens that only appeared in `tool_name` or the serialized `tool_calls`\n+    JSON were invisible to session_search even though the row was in the DB.\n+    \"\"\"\n+\n+    def test_tool_name_is_searchable(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\n+            \"s1\", role=\"assistant\", content=\"\",\n+            tool_name=\"UNIQUETOOLNAME\",\n+        )\n+        results = db.search_messages(\"UNIQUETOOLNAME\")\n+        assert len(results) == 1\n+\n+    def test_tool_calls_args_are_searchable(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\n+            \"s1\", role=\"assistant\", content=\"\",\n+            tool_calls=[{\n+                \"id\": \"c1\",\n+                \"type\": \"function\",\n+                \"function\": {\n+                    \"name\": \"web_search\",\n+                    \"arguments\": '{\"query\": \"UNIQUESEARCHTOKEN\"}',\n+                },\n+            }],\n+        )\n+        results = db.search_messages(\"UNIQUESEARCHTOKEN\")\n+        assert len(results) == 1\n+\n+    def test_tool_function_name_in_tool_calls_is_searchable(self, db):\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\n+            \"s1\", role=\"assistant\", content=\"\",\n+            tool_calls=[{\n+                \"id\": \"c1\",\n+                \"type\": \"function\",\n+                \"function\": {\"name\": \"UNIQUEFUNCNAME\", \"arguments\": \"{}\"},\n+            }],\n+        )\n+        results = db.search_messages(\"UNIQUEFUNCNAME\")\n+        assert len(results) == 1\n+\n+    def test_delete_message_row_does_not_crash(self, db):\n+        \"\"\"DELETE on messages must not raise when FTS rows reference tool fields.\n+\n+        Previously the messages_fts_delete trigger passed old.content to the\n+        FTS5 delete-command but the inserted row was the concatenation of\n+        content || tool_name || tool_calls, so FTS5 rejected the delete with\n+        'SQL logic error' and every session delete path broke.\n+        \"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\n+            \"s1\", role=\"assistant\", content=\"hello\",\n+            tool_name=\"web_search\",\n+            tool_calls=[{\n+                \"id\": \"c1\",\n+                \"type\": \"function\",\n+                \"function\": {\"name\": \"web_search\", \"arguments\": '{\"q\": \"x\"}'},\n+            }],\n+        )\n+        # end_session + end-time prune path would exercise DELETE; hit the\n+        # row directly through the write helper to keep the regression focused.\n+        def _delete(conn):\n+            conn.execute(\"DELETE FROM messages WHERE session_id = ?\", (\"s1\",))\n+        db._execute_write(_delete)  # must not raise\n+\n+        assert db.search_messages(\"hello\") == []\n+        assert db.search_messages(\"web_search\") == []\n+\n+    def test_update_message_reindexes_tool_fields(self, db):\n+        \"\"\"UPDATE must refresh the FTS row so old tokens drop out and new tokens appear.\"\"\"\n+        db.create_session(session_id=\"s1\", source=\"cli\")\n+        db.append_message(\n+            \"s1\", role=\"assistant\", content=\"\",\n+            tool_name=\"ORIGINALTOOL\",\n+        )\n+        assert len(db.search_messages(\"ORIGINALTOOL\")) == 1\n+\n+        def _update(conn):\n+            conn.execute(\n+                \"UPDATE messages SET tool_name = ? WHERE session_id = ?\",\n+                (\"RENAMEDTOOL\", \"s1\"),\n+            )\n+        db._execute_write(_update)\n+\n+        assert db.search_messages(\"ORIGINALTOOL\") == []\n+        assert len(db.search_messages(\"RENAMEDTOOL\")) == 1\n+\n+\n+class TestFTS5ToolCallMigration:\n+    \"\"\"v11 migration: pre-existing state.db with old external-content FTS tables\n+    must be re-indexed so tool_name / tool_calls become searchable after upgrade.\"\"\"\n+\n+    def test_v10_to_v11_upgrade_backfills_tool_fields(self, tmp_path):\n+        \"\"\"Simulate an existing user: build a v10-shaped DB by hand, insert a\n+        row with tool_calls, then open via SessionDB (which runs migrations).\n+        After upgrade, the tool_calls token must be searchable.\"\"\"\n+        import sqlite3\n+\n+        db_path = tmp_path / \"legacy.db\"\n+\n+        # Build the pre-v11 schema by hand: external-content FTS tables +\n+        # old triggers that only reference new.content.\n+        conn = sqlite3.connect(str(db_path))\n+        conn.executescript(\"\"\"\n+            CREATE TABLE schema_version (version INTEGER NOT NULL);\n+            INSERT INTO schema_version (version) VALUES (10);\n+\n+            CREATE TABLE sessions (\n+                id TEXT PRIMARY KEY,\n+                source TEXT,\n+                started_at REAL,\n+                ended_at REAL,\n+                title TEXT,\n+                parent_session_id TEXT,\n+                message_count INTEGER DEFAULT 0,\n+                tool_call_count INTEGER DEFAULT 0,\n+                api_call_count INTEGER DEFAULT 0\n+            );\n+            CREATE TABLE messages (\n+                id INTEGER PRIMARY KEY,\n+                session_id TEXT NOT NULL,\n+                timestamp REAL NOT NULL,\n+                role TEXT NOT NULL,\n+                content TEXT,\n+                tool_name TEXT,\n+                tool_calls TEXT,\n+                tool_call_id TEXT,\n+                token_count INTEGER,\n+                finish_reason TEXT,\n+                reasoning TEXT,\n+                reasoning_content TEXT,\n+                reasoning_details TEXT,\n+                codex_reasoning_items TEXT,\n+                codex_message_items TEXT\n+            );\n+\n+            CREATE VIRTUAL TABLE messages_fts USING fts5(\n+                content, content=messages, content_rowid=id\n+            );\n+            CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN\n+                INSERT INTO messages_fts(rowid, content) VALUES (new.id, new.content);\n+            END;\n+\n+            CREATE VIRTUAL TABLE messages_fts_trigram USING fts5(\n+                content, content=messages, content_rowid=id, tokenize='[redacted:secret-assignment]'\n+            );\n+            CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages BEGIN\n+                INSERT INTO messages_fts_trigram(rowid, content) VALUES (new.id, new.content);\n+            END;\n+        \"\"\")\n+        conn.execute(\n+            \"INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)\",\n+            (\"s1\", \"cli\", time.time()),\n+        )\n+        conn.execute(\n+            \"INSERT INTO messages (session_id, timestamp, role, content, tool_name, tool_calls) \"\n+            \"VALUES (?, ?, ?, ?, ?, ?)\",\n+            (\"s1\", time.time(), \"assistant\", \"\", \"LEGACYTOOL\",\n+             '{\"function\":{\"name\":\"web_search\",\"arguments\":\"{\\\\\"q\\\\\":\\\\\"LEGACYARG\\\\\"}\"}}'),\n+        )\n+        conn.commit()\n+\n+        # Verify the legacy FTS rows don't contain the tool tokens yet.\n+        legacy_hits = conn.execute(\n+            \"SELECT rowid FROM messages_fts WHERE messages_fts MATCH 'LEGACYTOOL'\"\n+        ).fetchall()\n+        assert legacy_hits == [], \"sanity: legacy FTS must NOT contain tool_name\"\n+        conn.close()\n+\n+        # Now open via SessionDB — migration runs.\n+        session_db = SessionDB(db_path=db_path)\n+        try:\n+            assert len(session_db.search_messages(\"LEGACYTOOL\")) == 1, \\\n+                \"v11 migration must backfill tool_name into FTS\"\n+            assert len(session_db.search_messages(\"LEGACYARG\")) == 1, \\\n+                \"v11 migration must backfill tool_calls JSON into FTS\"\n+            # schema_version bumped\n+            row = session_db._conn.execute(\n+                \"SELECT version FROM schema_version LIMIT 1\"\n+            ).fetchone()\n+            version = row[\"version\"] if hasattr(row, \"keys\") else row[0]\n+            assert version == 11\n+        finally:\n+            session_db.close()\n+\n"]}]},{"kind":"pr","number":57965,"title":"perf(state): external-content FTS5 — stop storing message text three times","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#65798","number":65798}],"category":"uncategorized","implication":"needs_llm"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}