OpenRouter prompt caching cache_control placement bugs

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #2391 fix(prompt-caching): skip top-level cache_control on role:tool for OpenRouter

  merged to main

  files: agent/prompt_caching.py, run_agent.py, tests/agent/test_prompt_caching.py, tests/test_anthropic_adapter.py

• #58129 fix(prompt-caching): land envelope cache breakpoints on honored carriers during tool loops (#57845)

  merged to main

  files: agent/prompt_caching.py, scripts/release.py, tests/agent/test_prompt_caching.py

  Show the diff

  • agent/prompt_caching.py

    @@ -17,12 +17,23 @@ def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool =
         role = msg.get("role", "")
         content = msg.get("content")
     
    -    if role == "tool":
    -        if native_anthropic:
    -            msg["cache_control"] = cache_marker
    +    if role == "tool" and native_anthropic:
    +        # Native Anthropic layout: top-level marker; the adapter moves it
    +        # inside the tool_result block.
    +        msg["cache_control"] = cache_marker
             return
     
         if content is None or content == "":
    +        if role == "tool" and not native_anthropic:
    +            # OpenRouter rejects top-level cache_control on role:tool (silent
    +            # hang) and an empty message has no content part to carry the
    +            # marker — skip. Non-empty tool content falls through below and
    +            # gets the marker on a content part, which OpenRouter honors.
    +            return
    +        if role == "assistant" and not native_anthropic:
    +            # Empty assistant turns are pure tool_calls. A top-level marker
    +            # here is ignored on the envelope layout, so skip.
    +            return
             msg["cache_control"] = cache_marker
             return
     
    @@ -38,6 +49,30 @@ def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool =
                 last["cache_control"] = cache_marker
     
     
    +def _can_carry_marker(msg: dict, native_anthropic: bool) -> bool:
    +    """True if a marker on this message is actually honored by the provider.
    +
    +    On the native Anthropic layout every message works (top-level markers are
    +    relocated by the adapter). On the envelope layout (OpenRouter et al.) only
    +    markers inside content parts are honored: empty-content messages (e.g.
    +    assistant turns that are pure tool_calls) and empty tool messages would
    +    receive a top-level marker the provider ignores — wasting one of the four
    +    breakpoints. Skip those so the breakpoints land on messages that count.
    +    """
    +    if native_anthropic:
    +        return True
    +    content = msg.get("content")
    +    if content is None or content == "":
    +        return False
    +    if isinstance(content, list):
    +        # _apply_cache_marker only marks the LAST content part, so the carrier
    +        # predicate must agree: a list whose last element isn't a dict cannot
    +        # actually receive a marker and would waste a breakpoint. Mirror the
    +        # `content` truthiness + last-element-dict check in _apply_cache_marker.
    +        return bool(content) and isinstance(content[-1], dict)
    +    return isinstance(content, str)
    +
    +
     def _build_marker(ttl: str) -> Dict[str, str]:
         """Build a cache_control marker dict for the given TTL ('5m' or '1h')."""
         marker: Dict[str, str] = {"type": "ephemeral"}
    @@ -72,7 +107,12 @@ def apply_anthropic_cache_control(
             breakpoints_used += 1
     
         remaining = 4 - breakpoints_used
    -    non_sys = [i for i in range(len(messages)) if messages[i].get("role") != "system"]
    +    non_sys = [
    +        i
    +        for i in range(len(messages))
    +        if messages[i].get("role") != "system"
    +        and _can_carry_marker(messages[i], native_anthropic=native_anthropic)
    +    ]
         for idx in non_sys[-remaining:]:
             _apply_cache_marker(messages[idx], marker, native_anthropic=native_anthropic)
     

  • scripts/release.py

    @@ -1857,6 +1857,7 @@
         "[redacted:email]": "wnuuee1",  # commit 9fd2b2cb PR author
         "[redacted:email]": "Eji4h",  # direct email match
         "[redacted:email]": "designnotdrum",  # PR #56480 salvage (toolset static-inference fix)
    +    "[redacted:email]": "LavyaTandel",  # PR #57893 salvage local git identity (envelope-layout cache markers on tool/empty-assistant messages; #57845)
     }
     
     

  • tests/agent/test_prompt_caching.py

    @@ -3,6 +3,7 @@
     
     from agent.prompt_caching import (
         _apply_cache_marker,
    +    _can_carry_marker,
         apply_anthropic_cache_control,
     )
     
    @@ -23,18 +24,37 @@ def test_tool_message_skips_marker_on_openrouter(self):
             _apply_cache_marker(msg, MARKER, native_anthropic=False)
             assert "cache_control" not in msg
     
    -    def test_none_content_gets_top_level_marker(self):
    -        msg = {"role": "assistant", "content": None}
    -        _apply_cache_marker(msg, MARKER)
    -        assert msg["cache_control"] == MARKER
    +    def test_tool_message_wraps_non_empty_content_on_openrouter(self):
    +        """Non-empty tool content should be wrapped so the marker lands on a content part."""
    +        msg = {"role": "tool", "content": "tool result bytes"}
    +        _apply_cache_marker(msg, MARKER, native_anthropic=False)
    +        assert "cache_control" not in msg
    +        assert isinstance(msg["content"], list)
    +        assert msg["content"][0]["cache_control"] == MARKER
     
    -    def test_empty_string_content_gets_top_level_marker(self):
    -        """Empty text blocks cannot have cache_control (Anthropic rejects them)."""
    +    def test_empty_assistant_message_skips_marker_on_openrouter(self):
    +        """OpenRouter path: empty assistant turns are pure tool_calls, marker would be ignored."""
             msg = {"role": "assistant", "content": ""}
    -        _apply_cache_marker(msg, MARKER)
    +        _apply_cache_marker(msg, MARKER, native_anthropic=False)
    +        assert "cache_control" not in msg
    +
    +    def test_native_anthropic_empty_assistant_gets_top_level_marker(self):
    +        """Native Anthropic layout can still carry top-level marker on empty content."""
    +        msg = {"role": "assistant", "content": ""}
    +        _apply_cache_marker(msg, MARKER, native_anthropic=True)
    +        assert msg["cache_control"] == MARKER
    +
    +    def test_none_content_skips_marker_on_openrouter(self):
    +        """OpenRouter path: None-content assistant turns are ignored."""
    +        msg = {"role": "assistant", "content": None}
    +        _apply_cache_marker(msg, MARKER, native_anthropic=False)
    +        assert "cache_control" not in msg
    +
    +    def test_none_content_gets_top_level_marker_on_native_anthropic(self):
    +        """Native Anthropic path: None content still gets top-level marker."""
    +        msg = {"role": "assistant", "content": None}
    +        _apply_cache_marker(msg, MARKER, native_anthropic=True)
             assert msg["cache_control"] == MARKER
    -        # Must NOT wrap into [{"type": "text", "text": "", "cache_control": ...}]
    -        assert msg["content"] == ""
     
         def test_string_content_wrapped_in_list(self):
             msg = {"role": "user", "content": "Hello"}
    @@ -63,6 +83,41 @@ def test_empty_list_content_no_crash(self):
             _apply_cache_marker(msg, MARKER)
     
     
    +class TestCanCarryMarker:
    +    def test_native_anthropic_always_true(self):
    +        assert _can_carry_marker({"role": "assistant", "content": ""}, native_anthropic=True) is True
    +        assert _can_carry_marker({"role": "tool", "content": ""}, native_anthropic=True) is True
    +
    +    def test_openrouter_content_parts_carry_marker(self):
    +        assert _can_carry_marker({"role": "user", "content": "text"}, native_anthropic=False) is True
    +        assert _can_carry_marker({"role": "user", "content": [{"type": "text", "text": "a"}]}, native_anthropic=False) is True
    +
    +    def test_openrouter_empty_or_none_does_not_carry_marker(self):
    +        assert _can_carry_marker({"role": "assistant", "content": ""}, native_anthropic=False) is False
    +        assert _can_carry_marker({"role": "assistant", "content": None}, native_anthropic=False) is False
    +        assert _can_carry_marker({"role": "tool", "content": "result"}, native_anthropic=False) is True
    +        assert _can_carry_marker({"role": "tool", "content": ""}, native_anthropic=False) is False
    +
    +    def test_openrouter_list_carrier_requires_last_part_dict(self):
    +        """Carrier predicate must agree with _apply_cache_marker, which only marks
    +        the LAST content part. A list whose last element isn't a dict cannot carry
    +        a marker and must not consume a breakpoint."""
    +        # Last part is a dict -> carrier.
    +        assert _can_carry_marker(
    +            {"role": "user", "content": [{"type": "text", "text": "a"}]},
    +            native_anthropic=False,
    +        ) is True
    +        # Last part is a non-dict (stray raw string) -> NOT a carrier, even though
    +        # an earlier part is a dict. Previously this passed the gate but got no
    +        # marker, wasting a breakpoint.
    +        assert _can_carry_marker(
    +            {"role": "user", "content": [{"type": "text", "text": "a"}, "trailing raw"]},
    +            native_anthropic=False,
    +        ) is False
    +        # Empty list -> not a carrier.
    +        assert _can_carry_marker({"role": "user", "content": []}, native_anthropic=False) is False
    +
    +
     class TestApplyAnthropicCacheControl:
         def test_empty_messages(self):
             result = apply_anthropic_cache_control([])
    @@ -139,3 +194,32 @@ def test_max_4_breakpoints(self):
                 elif "cache_control" in msg:
                     count += 1
             assert count <= 4
    +
    +    def test_tool_loop_empty_assistant_and_tool_messages_do_not_consume_breakpoints(self):
    +        """Tool loops should keep breakpoints on messages that can carry markers."""
    +        msgs = [
    +            {"role": "system", "content": "You are helpful"},
    +            {"role": "user", "content": "run tool 1", "cache_control": MARKER},
    +            {"role": "assistant", "content": "", "tool_calls": [{"type": "function"}]},
    +            {"role": "tool", "content": "tool result 1"},
    +            {"role": "user", "content": "run tool 2", "cache_control": MARKER},
    +            {"role": "assistant", "content": "", "tool_calls": [{"type": "function"}]},
    +            {"role": "tool", "content": "tool result 2"},
    +        ]
    +        result = apply_anthropic_cache_control(msgs, native_anthropic=False)
    +        # Empty assistant/tool turns should not get broken markers
    +        assert "cache_control" not in result[2]
    +        assert "cache_control" not in result[3]
    +        assert "cache_control" not in result[5]
    +        assert "cache_control" not in result[6]
    +
    +    def test_tool_message_marker_lands_on_content_part_on_openrouter(self):
    +        """Non-empty tool content should be wrapped so the marker lands on a content part."""
    +        msgs = [
    +            {"role": "user", "content": "hello"},
    +            {"role": "tool", "content": "tool output"},
    +        ]
    +        result = apply_anthropic_cache_control(msgs, native_anthropic=False)
    +        assert isinstance(result[1]["content"], list)
    +        assert result[1]["content"][0]["cache_control"] == {"type": "ephemeral"}
    +        assert "cache_control" not in result[1]