{"repo":"NousResearch/hermes-agent","public_id":"16472","display_label":"Tool call ID corruption causing persistent 400 errors","summary":"Multiple fixes and bug reports addressing how stale or malformed tool_call_ids and tool messages cause repeated HTTP 400 error loops across sessions. Work spans error classification, context compression, and agent run logic.","subsystem":"agent","counts":{"members":6,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":21385},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-05-07T15:35:11+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.29066666666666663},"fix_edges":[{"issue_number":16472,"pr_number":21385,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":16472,"title":"Bug: Tool call ID not invalidated after 400 error — causes persistent failures across sessions","url":null,"state":"closed"},{"kind":"issue","number":27033,"title":"[Bug]: Tool Result Contamination Causes Persistent HTTP 400 Error Loop","url":null,"state":"closed"},{"kind":"pr","number":16602,"title":"fix(agent): invalidate failed tool_call_id on MiniMax 400 errors","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16604","number":16604},{"kind":"pr","value":"#16604","number":16604}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":16604,"title":"fix(error_classifier): purge stale tool_call_id on 400 to prevent cross-session reuse (#16472)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#16472","number":16472}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":18650,"title":"fix(compression): sanitize malformed tool messages + auto-recover on API 400","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"812ce0b9878d1dc9ac1f7c419a620deeb57117f3","number":0},{"kind":"pr","value":"#21385","number":21385}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":21385,"title":"fix(run_agent): break permanent empty-response loop from orphan tool-tail","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["run_agent.py","tests/run_agent/test_empty_response_recovery_persistence.py","tests/run_agent/test_message_sequence_repair.py"],"fix_hunks":[{"file":"run_agent.py","hunks":["@@ -3836,7 +3836,17 @@ def _persist_session(self, messages: List[Dict], conversation_history: List[Dict\n         self._flush_messages_to_session_db(messages, conversation_history)\n \n     def _drop_trailing_empty_response_scaffolding(self, messages: List[Dict]) -> None:\n-        \"\"\"Remove private empty-response retry/failure scaffolding from transcript tails.\"\"\"\n+        \"\"\"Remove private empty-response retry/failure scaffolding from transcript tails.\n+\n+        Also rewinds past any trailing tool-result / assistant(tool_calls) pair\n+        that the failed iteration left hanging. Without this, the tail ends at\n+        a raw ``tool`` message and the next user turn lands as\n+        ``...tool, user, user`` — a protocol-invalid sequence that most\n+        providers silently reject (returns empty content), causing the\n+        empty-retry loop to fire forever. See #<TBD>.\n+        \"\"\"\n+        # Pass 1: strip the flagged scaffolding messages themselves.\n+        dropped_scaffolding = False\n         while (\n             messages\n             and isinstance(messages[-1], dict)","@@ -3846,6 +3856,137 @@ def _drop_trailing_empty_response_scaffolding(self, messages: List[Dict]) -> Non\n             )\n         ):\n             messages.pop()\n+            dropped_scaffolding = True\n+\n+        # Pass 2: if we stripped scaffolding, rewind through any trailing\n+        # tool-result messages plus the assistant(tool_calls) message that\n+        # produced them. This preserves role alternation so the next user\n+        # message follows a user or assistant message, not an orphan tool\n+        # result. Only runs when scaffolding was actually present — normal\n+        # conversation tails (real tool loops mid-progress) are untouched.\n+        if not dropped_scaffolding:\n+            return\n+\n+        # Drop any trailing tool-result messages\n+        while (\n+            messages\n+            and isinstance(messages[-1], dict)\n+            and messages[-1].get(\"role\") == \"tool\"\n+        ):\n+            messages.pop()\n+\n+        # Drop the assistant message that issued the tool calls, if the tail\n+        # now ends in an assistant-with-tool_calls (the pair that owned the\n+        # just-popped tool results). Without this, the tail is\n+        # ``assistant(tool_calls=...)`` with no tool answers, which some\n+        # providers also reject.\n+        if (\n+            messages\n+            and isinstance(messages[-1], dict)\n+            and messages[-1].get(\"role\") == \"assistant\"\n+            and messages[-1].get(\"tool_calls\")\n+        ):\n+            messages.pop()\n+\n+    def _repair_message_sequence(self, messages: List[Dict]) -> int:\n+        \"\"\"Collapse malformed role-alternation left in the live history.\n+\n+        Providers (OpenAI, OpenRouter, Anthropic) expect strict alternation:\n+        after the system message, user/tool alternates with assistant, with\n+        no two consecutive user messages and no tool-result that doesn't\n+        follow an assistant-with-tool_calls. Violations cause silent empty\n+        responses on most providers, which triggers the empty-retry loop.\n+\n+        This runs right before the API call as a defensive belt — by the\n+        time it fires, the scaffolding strip should already have prevented\n+        most shapes, but external callers (gateway multi-queue replay,\n+        session resume, cron, explicit conversation_history passed in by\n+        host code) can feed in already-broken histories.\n+\n+        Repairs applied:\n+          1. Stray ``tool`` messages whose ``tool_call_id`` doesn't match\n+             any preceding assistant tool_call — dropped.\n+          2. Consecutive ``user`` messages — merged with newline separator\n+             so no user input is lost.\n+\n+        Deliberately does NOT rewind orphan ``assistant(tool_calls)+tool``\n+        pairs that precede a user message — that pattern IS valid when the\n+        previous turn completed normally and the user jumped in to redirect\n+        before the model got a continuation turn (the ongoing dialog\n+        pattern). The empty-response scaffolding stripper handles the\n+        genuinely-broken variant via its flag-gated rewind.\n+\n+        Returns the number of repairs made (for logging/telemetry).\n+        \"\"\"\n+        if not messages:\n+            return 0\n+\n+        repairs = 0\n+\n+        # Pass 1: drop stray tool messages that don't follow a known\n+        # assistant tool_call_id. Uses a rolling set of known ids refreshed\n+        # on each assistant message.\n+        known_tool_ids: set = set()\n+        filtered: List[Dict] = []\n+        for msg in messages:\n+            if not isinstance(msg, dict):\n+                filtered.append(msg)\n+                continue\n+            role = msg.get(\"role\")\n+            if role == \"assistant\":\n+                known_tool_ids = set()\n+                for tc in (msg.get(\"tool_calls\") or []):\n+                    tc_id = tc.get(\"id\") if isinstance(tc, dict) else None\n+                    if tc_id:\n+                        known_tool_ids.add(tc_id)\n+                filtered.append(msg)\n+            elif role == \"tool\":\n+                tc_id = msg.get(\"tool_call_id\")\n+                if tc_id and tc_id in known_tool_ids:\n+                    filtered.append(msg)\n+                else:\n+                    repairs += 1\n+            else:\n+                if role == \"user\":\n+                    # A user turn closes the tool-result run; subsequent\n+                    # tool messages without a fresh assistant tool_call\n+                    # are orphans.\n+                    known_tool_ids = set()\n+                filtered.append(msg)\n+\n+        # Pass 2: merge consecutive user messages. Preserves all user input\n+        # so nothing the user typed is lost.\n+        merged: List[Dict] = []\n+        for msg in filtered:\n+            if (\n+                merged\n+                and isinstance(msg, dict)\n+                and msg.get(\"role\") == \"user\"\n+                and isinstance(merged[-1], dict)\n+                and merged[-1].get(\"role\") == \"user\"\n+            ):\n+                prev = merged[-1]\n+                prev_content = prev.get(\"content\", \"\")\n+                new_content = msg.get(\"content\", \"\")\n+                # Only merge plain-text content; leave multimodal (list)\n+                # content alone — collapsing image/audio blocks risks\n+                # mangling the attachment structure.\n+                if isinstance(prev_content, str) and isinstance(new_content, str):\n+                    prev[\"content\"] = (\n+                        (prev_content + \"\\n\\n\" + new_content)\n+                        if prev_content and new_content\n+                        else (prev_content or new_content)\n+                    )\n+                    repairs += 1\n+                    continue\n+            merged.append(msg)\n+\n+        if repairs > 0:\n+            # Rewrite in place so downstream paths (persistence, return\n+            # value, session DB flush) see the repaired sequence.\n+            messages[:] = merged\n+\n+        return repairs\n \n     def _flush_messages_to_session_db(self, messages: List[Dict], conversation_history: List[Dict] = None):\n         \"\"\"Persist any un-flushed messages to the SQLite session store.","@@ -11133,6 +11274,21 @@ def run_conversation(\n                     self.session_id or \"-\",\n                 )\n \n+            # Defensive: repair malformed role-alternation before API call.\n+            # Catches cases where the history got wedged into a\n+            # ``tool → user`` or ``user → user`` tail (e.g. after empty-\n+            # response scaffolding was stripped and a new user message\n+            # landed after an orphan tool result). Most providers return\n+            # empty content on malformed sequences, which would otherwise\n+            # retrigger the empty-retry loop indefinitely.\n+            repaired_seq = self._repair_message_sequence(messages)\n+            if repaired_seq > 0:\n+                request_logger.info(\n+                    \"Repaired %s message-alternation violations before request (session=%s)\",\n+                    repaired_seq,\n+                    self.session_id or \"-\",\n+                )\n+\n             api_messages = []\n             for idx, msg in enumerate(messages):\n                 api_msg = msg.copy()"]},{"file":"tests/run_agent/test_empty_response_recovery_persistence.py","hunks":["@@ -21,9 +21,21 @@ def _agent_with_stubbed_persistence():\n \n \n def test_persist_session_strips_trailing_empty_recovery_scaffolding():\n+    \"\"\"After stripping scaffolding, also rewind past orphan trailing tool-result\n+    messages that the failed iteration left behind. Otherwise the next user\n+    message lands after a bare ``tool`` and produces a protocol-invalid\n+    sequence that most providers silently fail on, retriggering the empty-\n+    retry loop indefinitely.\n+    \"\"\"\n     agent = _agent_with_stubbed_persistence()\n     messages = [\n         {\"role\": \"user\", \"content\": \"run the task\"},\n+        {\n+            \"role\": \"assistant\",\n+            \"content\": \"\",\n+            \"tool_calls\": [{\"id\": \"call_1\", \"type\": \"function\",\n+                            \"function\": {\"name\": \"x\", \"arguments\": \"{}\"}}],\n+        },\n         {\"role\": \"tool\", \"content\": \"{}\", \"tool_call_id\": \"call_1\"},\n         {\n             \"role\": \"assistant\",","@@ -42,9 +54,11 @@ def test_persist_session_strips_trailing_empty_recovery_scaffolding():\n \n     AIAgent._persist_session(agent, messages, conversation_history=[])\n \n+    # After strip + rewind, only the original user message remains. The\n+    # assistant(tool_calls) + tool pair is dropped because its iteration\n+    # never produced a real response.\n     assert messages == [\n         {\"role\": \"user\", \"content\": \"run the task\"},\n-        {\"role\": \"tool\", \"content\": \"{}\", \"tool_call_id\": \"call_1\"},\n     ]\n     assert agent.saved_session_logs[-1] == messages\n     assert all(not msg.get(\"_empty_recovery_synthetic\") for msg in messages)"]},{"file":"tests/run_agent/test_message_sequence_repair.py","hunks":["@@ -0,0 +1,201 @@\n+\"\"\"Tests for pre-API-call message-sequence repair.\n+\n+Covers ``_repair_message_sequence`` and the extended\n+``_drop_trailing_empty_response_scaffolding`` behavior that rewinds past\n+orphan tool-result tails. Together these prevent the self-reinforcing empty-\n+response loop observed in session 20260507_044111_fa7e65, where a tool-result\n+followed directly by a user message produced silent empty responses from\n+providers (violating role alternation), which retriggered the empty-retry\n+recovery every turn.\n+\"\"\"\n+\n+from run_agent import AIAgent\n+\n+\n+def _bare_agent():\n+    return AIAgent.__new__(AIAgent)\n+\n+\n+# ── _drop_trailing_empty_response_scaffolding ──────────────────────────────\n+\n+def test_drop_scaffolding_rewinds_orphan_tool_tail():\n+    \"\"\"When scaffolding is stripped, also rewind the orphan assistant+tool pair.\"\"\"\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"task\"},\n+        {\"role\": \"assistant\", \"content\": \"\",\n+         \"tool_calls\": [{\"id\": \"t1\", \"type\": \"function\",\n+                         \"function\": {\"name\": \"f\", \"arguments\": \"{}\"}}]},\n+        {\"role\": \"tool\", \"tool_call_id\": \"t1\", \"content\": \"out\"},\n+        {\"role\": \"assistant\", \"content\": \"(empty)\",\n+         \"_empty_terminal_sentinel\": True},\n+    ]\n+\n+    AIAgent._drop_trailing_empty_response_scaffolding(agent, messages)\n+\n+    assert messages == [{\"role\": \"user\", \"content\": \"task\"}]\n+\n+\n+def test_drop_scaffolding_keeps_tail_when_no_scaffolding():\n+    \"\"\"Mid-iteration tool results must NOT be rewound — only if scaffolding fires.\"\"\"\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"task\"},\n+        {\"role\": \"assistant\", \"content\": \"\",\n+         \"tool_calls\": [{\"id\": \"t1\", \"type\": \"function\",\n+                         \"function\": {\"name\": \"f\", \"arguments\": \"{}\"}}]},\n+        {\"role\": \"tool\", \"tool_call_id\": \"t1\", \"content\": \"out\"},\n+    ]\n+    original = [dict(m) for m in messages]\n+\n+    AIAgent._drop_trailing_empty_response_scaffolding(agent, messages)\n+\n+    assert messages == original\n+\n+\n+def test_drop_scaffolding_handles_multiple_parallel_tool_results():\n+    \"\"\"Parallel tool calls (one assistant → many tool results) all rewound together.\"\"\"\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"task\"},\n+        {\"role\": \"assistant\", \"content\": \"\",\n+         \"tool_calls\": [\n+             {\"id\": \"t1\", \"type\": \"function\",\n+              \"function\": {\"name\": \"f\", \"arguments\": \"{}\"}},\n+             {\"id\": \"t2\", \"type\": \"function\",\n+              \"function\": {\"name\": \"g\", \"arguments\": \"{}\"}},\n+         ]},\n+        {\"role\": \"tool\", \"tool_call_id\": \"t1\", \"content\": \"out1\"},\n+        {\"role\": \"tool\", \"tool_call_id\": \"t2\", \"content\": \"out2\"},\n+        {\"role\": \"assistant\", \"content\": \"(empty)\",\n+         \"_empty_terminal_sentinel\": True},\n+    ]\n+\n+    AIAgent._drop_trailing_empty_response_scaffolding(agent, messages)\n+\n+    assert messages == [{\"role\": \"user\", \"content\": \"task\"}]\n+\n+\n+# ── _repair_message_sequence ───────────────────────────────────────────────\n+\n+def test_repair_merges_consecutive_user_messages():\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"first\"},\n+        {\"role\": \"user\", \"content\": \"second\"},\n+    ]\n+\n+    repairs = AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert repairs == 1\n+    assert len(messages) == 1\n+    assert messages[0][\"role\"] == \"user\"\n+    assert messages[0][\"content\"] == \"first\\n\\nsecond\"\n+\n+\n+def test_repair_preserves_user_content_when_one_side_empty():\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"\"},\n+        {\"role\": \"user\", \"content\": \"real message\"},\n+    ]\n+\n+    AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert messages == [{\"role\": \"user\", \"content\": \"real message\"}]\n+\n+\n+def test_repair_does_not_rewind_ongoing_dialog_tool_pair():\n+    \"\"\"assistant(tool_calls) + tool + user is a VALID pattern (user redirect\n+    before the model gets its continuation turn). Repair must not touch it —\n+    only the flag-gated scaffolding strip rewinds, and only when the\n+    empty-recovery scaffolding was actually present.\n+    \"\"\"\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"Q1\"},\n+        {\"role\": \"assistant\", \"content\": \"\",\n+         \"tool_calls\": [{\"id\": \"t1\", \"type\": \"function\",\n+                         \"function\": {\"name\": \"f\", \"arguments\": \"{}\"}}]},\n+        {\"role\": \"tool\", \"tool_call_id\": \"t1\", \"content\": \"out\"},\n+        {\"role\": \"user\", \"content\": \"Q2\"},\n+    ]\n+    original = [dict(m) for m in messages]\n+\n+    repairs = AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert repairs == 0\n+    assert messages == original\n+\n+\n+def test_repair_drops_stray_tool_with_unknown_tool_call_id():\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"hi\"},\n+        {\"role\": \"assistant\", \"content\": \"hello\"},\n+        {\"role\": \"tool\", \"tool_call_id\": \"orphan\", \"content\": \"stray\"},\n+        {\"role\": \"user\", \"content\": \"real\"},\n+    ]\n+\n+    repairs = AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert repairs >= 1\n+    assert all(m.get(\"role\") != \"tool\" for m in messages)\n+\n+\n+def test_repair_leaves_valid_conversation_unchanged():\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": \"list files\"},\n+        {\"role\": \"assistant\", \"content\": \"\",\n+         \"tool_calls\": [{\"id\": \"t1\", \"type\": \"function\",\n+                         \"function\": {\"name\": \"ls\", \"arguments\": \"{}\"}}]},\n+        {\"role\": \"tool\", \"tool_call_id\": \"t1\", \"content\": \"a.txt b.txt\"},\n+        {\"role\": \"assistant\", \"content\": \"Found 2 files\"},\n+        {\"role\": \"user\", \"content\": \"more\"},\n+    ]\n+    original = [dict(m) for m in messages]\n+\n+    repairs = AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert repairs == 0\n+    assert messages == original\n+\n+\n+def test_repair_preserves_multimodal_user_content():\n+    \"\"\"Multimodal (list) content must NOT be merged — risks mangling attachments.\"\"\"\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"user\", \"content\": [{\"type\": \"text\", \"text\": \"hi\"},\n+                                     {\"type\": \"image_url\", \"image_url\": {\"url\": \"...\"}}]},\n+        {\"role\": \"user\", \"content\": \"follow-up\"},\n+    ]\n+\n+    AIAgent._repair_message_sequence(agent, messages)\n+\n+    # The multimodal user message stays as a distinct message — no merge\n+    assert len(messages) == 2\n+    assert isinstance(messages[0][\"content\"], list)\n+\n+\n+def test_repair_empty_messages_returns_zero():\n+    agent = _bare_agent()\n+    messages = []\n+\n+    repairs = AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert repairs == 0\n+    assert messages == []\n+\n+\n+def test_repair_preserves_system_messages():\n+    agent = _bare_agent()\n+    messages = [\n+        {\"role\": \"system\", \"content\": \"You are...\"},\n+        {\"role\": \"user\", \"content\": \"hi\"},\n+    ]\n+    original = [dict(m) for m in messages]\n+\n+    AIAgent._repair_message_sequence(agent, messages)\n+\n+    assert messages == original\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}