Multimodal message token count underestimation fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #16369 fix(compressor): use text char sum for multimodal token estimation in _find_tail_cut_by_tokens

  merged to main

  files: agent/context_compressor.py, tests/agent/test_context_compressor.py

  Show the diff

  • agent/context_compressor.py

    @@ -484,7 +484,18 @@ def _prune_old_tool_results(
                 for i in range(len(result) - 1, -1, -1):
                     msg = result[i]
                     raw_content = msg.get("content") or ""
    -                content_len = sum(len(p.get("text", "")) for p in raw_content) if isinstance(raw_content, list) else len(raw_content)
    +                content_len = (
    +                    sum(
    +                        len(p.get("text", ""))
    +                        if isinstance(p, dict)
    +                        else len(p)
    +                        if isinstance(p, str)
    +                        else len(str(p))
    +                        for p in raw_content
    +                    )
    +                    if isinstance(raw_content, list)
    +                    else len(raw_content)
    +                )
                     msg_tokens = [redacted:secret-assignment] // _CHARS_PER_TOKEN + 10
                     for tc in msg.get("tool_calls") or []:
                         if isinstance(tc, dict):
    @@ -1082,8 +1093,20 @@ def _find_tail_cut_by_tokens(
     
             for i in range(n - 1, head_end - 1, -1):
                 msg = messages[i]
    -            content = msg.get("content") or ""
    -            msg_tokens = [redacted:secret-assignment] // _CHARS_PER_TOKEN + 10  # +10 for role/metadata
    +            raw_content = msg.get("content") or ""
    +            content_len = (
    +                sum(
    +                    len(p.get("text", ""))
    +                    if isinstance(p, dict)
    +                    else len(p)
    +                    if isinstance(p, str)
    +                    else len(str(p))
    +                    for p in raw_content
    +                )
    +                if isinstance(raw_content, list)
    +                else len(raw_content)
    +            )
    +            msg_tokens = [redacted:secret-assignment] // _CHARS_PER_TOKEN + 10  # +10 for role/metadata
                 # Include tool call arguments in estimate
                 for tc in msg.get("tool_calls") or []:
                     if isinstance(tc, dict):

  • tests/agent/test_context_compressor.py

    @@ -846,6 +846,97 @@ def test_prune_without_token_budget_uses_message_count(self, budget_compressor):
             # so it might or might not be pruned depending on boundary
             assert isinstance(pruned, int)
     
    +    def test_multimodal_message_accumulates_text_chars_not_block_count(self, budget_compressor):
    +        """_find_tail_cut_by_tokens must use text char count, not list length,
    +        for multimodal content. Regression guard for #16087.
    +
    +        Setup: 6 messages, budget=80 (soft_ceiling=120).  The multimodal message
    +        at index 1 has 500 chars of text → 135 tokens (correct) or 10 tokens (bug).
    +
    +        Fixed path: walk stops at the multimodal (44+135=179 > 120), cut stays at 2,
    +        tail = messages[2:] = 4 messages.
    +
    +        Bug path: walk counts only 10 tokens for the multimodal, exhausts to head_end,
    +        the head_end safeguard forces cut = n - min_tail = 3, tail = only 3 messages.
    +        """
    +        c = budget_compressor
    +        # 500 chars → 500//4 + 10 = 135 tokens; len([text, image]) // 4 + 10 = 10 (bug)
    +        big_text = "x" * 500
    +        multimodal_content = [
    +            {"type": "text", "text": big_text},
    +            {"type": "image_url", "image_url": {"url": "https://example.com/img.jpg"}},
    +        ]
    +        messages = [
    +            {"role": "user", "content": "head1"},               # 0
    +            {"role": "user", "content": multimodal_content},    # 1: BIG (index under test)
    +            {"role": "assistant", "content": "tail1"},           # 2
    +            {"role": "user", "content": "tail2"},                # 3
    +            {"role": "assistant", "content": "tail3"},           # 4
    +            {"role": "user", "content": "tail4"},                # 5
    +        ]
    +        c.tail_token_budget = 80  # soft_ceiling = 120
    +        head_end = 0
    +        cut = c._find_tail_cut_by_tokens(messages, head_end)
    +        # With the fix: cut=2, tail has 4 messages (soft_ceiling not exceeded by tail1-4).
    +        # With the bug: head_end safeguard fires → cut = n - min_tail = 3, only 3 in tail.
    +        assert len(messages) - cut >= 4, (
    +            f"Expected ≥4 messages in tail (got {len(messages) - cut}, cut={cut}). "
    +            "The multimodal message was underestimated — len(list) used instead of text chars."
    +        )
    +
    +    def test_plain_string_content_unchanged(self, budget_compressor):
    +        """Plain string content must still be estimated correctly after the fix."""
    +        c = budget_compressor
    +        # Same layout as the multimodal test but with a plain 500-char string.
    +        # Both buggy and fixed code count plain strings the same way (len(str)).
    +        # With 135 tokens the plain string also exceeds soft_ceiling=120, so
    +        # the walk stops at index 1 and tail has 4 messages — same as the fix path.
    +        big_plain = "x" * 500
    +        messages = [
    +            {"role": "user", "content": "head1"},
    +            {"role": "user", "content": big_plain},   # 1: 135 tokens, plain string
    +            {"role": "assistant", "content": "tail1"},
    +            {"role": "user", "content": "tail2"},
    +            {"role": "assistant", "content": "tail3"},
    +            {"role": "user", "content": "tail4"},
    +        ]
    +        c.tail_token_budget = 80
    +        head_end = 0
    +        cut = c._find_tail_cut_by_tokens(messages, head_end)
    +        assert len(messages) - cut >= 4, (
    +            f"Plain string regression: expected ≥4 messages in tail, got {len(messages) - cut}"
    +        )
    +
    +    def test_image_only_block_contributes_zero_text_chars(self, budget_compressor):
    +        """Image-only content blocks (no 'text' key) contribute 0 chars + base overhead."""
    +        c = budget_compressor
    +        c.tail_token_budget = 500
    +        image_only = [{"type": "image_url", "image_url": {"url": "https://example.com/x.jpg"}}]
    +        messages = [
    +            {"role": "user", "content": "a" * 4000},
    +            {"role": "user", "content": image_only},   # 0 text chars → 10 tokens overhead
    +            {"role": "assistant", "content": "ok"},
    +        ]
    +        head_end = 0
    +        cut = c._find_tail_cut_by_tokens(messages, head_end)
    +        assert isinstance(cut, int)
    +        assert 0 <= cut <= len(messages)
    +
    +    def test_mixed_list_with_bare_strings_does_not_crash(self, budget_compressor):
    +        """Content list may contain bare strings (not dicts) — must not raise AttributeError."""
    +        c = budget_compressor
    +        c.tail_token_budget = 500
    +        # Bare string item alongside a dict item — normalisation elsewhere allows this.
    +        mixed_content = ["Hello, world!", {"type": "text", "text": "extra text"}]
    +        messages = [
    +            {"role": "user", "content": mixed_content},
    +            {"role": "assistant", "content": "ok"},
    +        ]
    +        head_end = 0
    +        cut = c._find_tail_cut_by_tokens(messages, head_end)
    +        assert isinstance(cut, int)
    +        assert 0 <= cut <= len(messages)
    +
     
     class TestUpdateModelBudgets:
         """Regression: update_model() must recalculate token budgets."""