Gateway auto-appending MEDIA tags from tool results

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #36275 fix(gateway): salvage MEDIA: false-positive guards (#16721 + #35699)

  merged to main

  files: gateway/platforms/base.py, gateway/run.py, scripts/release.py, tests/gateway/test_media_extraction.py, tests/gateway/test_platform_base.py

  Show the diff

  • gateway/platforms/base.py

    @@ -2579,6 +2579,48 @@ def filter_local_delivery_paths(file_paths) -> List[str]:
                     logger.warning("Skipping unsafe local file path: %s", _log_safe_path(raw))
             return safe_paths
     
    +
    +    @staticmethod
    +    def _mask_protected_spans(content: str) -> str:
    +        """Replace content inside fenced code blocks, inline code spans,
    +        and blockquotes with spaces to prevent MEDIA: false positives.
    +
    +        Preserves character count so regex match offsets stay valid.
    +        Skips masking backtick-quoted paths in MEDIA: tags (e.g.
    +        ``MEDIA:`/path/to/file.png` ``) to avoid breaking path extraction.
    +        """
    +        chars = list(content)
    +        n = len(chars)
    +
    +        # Build list of (start, end) spans to mask
    +        spans: list = []
    +
    +        # Fenced code blocks: ```...```
    +        for m in re.finditer(r'```[^\n]*\n.*?```', content, re.DOTALL):
    +            spans.append((m.start(), m.end()))
    +
    +        # Inline code: `...` but NOT backtick-quoted paths in MEDIA: tags
    +        for m in re.finditer(r'`[^`\n]+`', content):
    +            start = m.start()
    +            # Check if this is a backtick-quoted path after MEDIA:
    +            prefix = content[max(0, start - 20):start]
    +            if re.search(r'MEDIA:\s*$', prefix):
    +                continue  # This is a MEDIA path quote, not inline code
    +            spans.append((start, m.end()))
    +
    +        # Blockquote lines: > at line start
    +        for m in re.finditer(r'^>.*$', content, re.MULTILINE):
    +            spans.append((m.start(), m.end()))
    +
    +        # Apply masking
    +        for start, end in spans:
    +            for i in range(start, end):
    +                if chars[i] != '\n':
    +                    chars[i] = ' '
    +
    +        return ''.join(chars)
    +
    +
         @staticmethod
         def _mask_json_string_media(content: str) -> str:
             """Blank out ``MEDIA:<bare-path>`` occurrences that sit inside a JSON
    @@ -2661,9 +2703,14 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]:
             # set is the shared MEDIA_DELIVERY_EXTS source of truth (built once into
             # MEDIA_TAG_CLEANUP_RE) so it can never drift from extract_local_files.
             media_pattern = MEDIA_TAG_CLEANUP_RE
    -        # Mask MEDIA: embedded inside serialized JSON string values so stale
    -        # paths from stored tool results are never re-delivered (#34375).
    -        scan_content = BasePlatformAdapter._mask_json_string_media(content)
    +        # Mask example/stored MEDIA: paths before scanning so they are never
    +        # delivered as real attachments:
    +        #  - code blocks / inline code / blockquotes hold prose examples (#35695)
    +        #  - serialized JSON string values hold stored tool-result text (#34375)
    +        # Both maskers are offset-preserving (chars -> spaces) so match offsets
    +        # stay valid; chaining them masks the union of both protected regions.
    +        scan_content = BasePlatformAdapter._mask_protected_spans(content)
    +        scan_content = BasePlatformAdapter._mask_json_string_media(scan_content)
             for match in media_pattern.finditer(scan_content):
                 path = match.group("path").strip()
                 if len(path) >= 2 and path[0] == path[-1] and path[0] in "`\"'":
    @@ -2677,15 +2724,17 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]:
                         # and dropping every other attachment in the response.
                         continue
     
    -        # Remove the delivered MEDIA tags from the user-visible text. Mask
    -        # ``cleaned`` (same length, so offsets line up with it), find the real
    -        # tag spans there, and delete those spans from the *unmasked* ``cleaned``.
    -        # This strips real tags while leaving JSON-embedded MEDIA: text intact —
    -        # it is stored data, not a directive, and must read back verbatim
    -        # (#34375). Masking ``cleaned`` (not ``content``) keeps offsets valid
    -        # after the [[audio_as_voice]] / [[as_document]] directives are removed.
    +        # Remove the delivered MEDIA tags from the user-visible text. Mask a
    +        # length-equal copy of ``cleaned`` (same union of protected regions) to
    +        # *locate* the real tag spans, then delete exactly those spans from the
    +        # *unmasked* ``cleaned``. Masking is only a locator — protected spans
    +        # (code blocks, quotes, JSON-embedded MEDIA: text) must survive verbatim
    +        # in the delivered text, not be blanked to whitespace. Masking
    +        # ``cleaned`` (not ``content``) keeps offsets valid after the
    +        # [[audio_as_voice]] / [[as_document]] directives are removed.
             if media:
    -            masked_cleaned = BasePlatformAdapter._mask_json_string_media(cleaned)
    +            masked_cleaned = BasePlatformAdapter._mask_protected_spans(cleaned)
    +            masked_cleaned = BasePlatformAdapter._mask_json_string_media(masked_cleaned)
                 spans = [m.span() for m in media_pattern.finditer(masked_cleaned)]
                 if spans:
                     chars = list(cleaned)

  • gateway/run.py

    @@ -669,6 +669,88 @@ def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any:
         return None
     
     
    +# Tool results can contain literal MEDIA: examples in docs, logs, or other
    +# ordinary outputs. Only tools that intentionally create deliverable media
    +# artifacts should be eligible for automatic append when the model omits them
    +# from the final gateway reply.
    +_AUTO_APPEND_MEDIA_TOOL_NAMES = {"text_to_speech", "text_to_speech_tool"}
    +
    +
    +# Extension-anchored MEDIA: matcher for tool results. Mirrors the dispatch-site
    +# pattern so a bare ``MEDIA:`` token in prose (no deliverable extension) is never
    +# auto-appended. Kept local to the auto-append path; the producer-tool allowlist
    +# below is the primary guard, this is the secondary precision guard.
    +_TOOL_MEDIA_RE = re.compile(
    +    r'MEDIA:((?:[A-Za-z]:[/\\]|/|~\/)\S+\.(?:png|jpe?g|gif|webp|'
    +    r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|'
    +    r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|'
    +    r'txt|csv|apk|ipa))',
    +    re.IGNORECASE,
    +)
    +
    +
    +def _collect_auto_append_media_tags(
    +    messages: List[Dict[str, Any]],
    +    history_offset: int = 0,
    +    history_media_paths: Optional[set] = None,
    +) -> tuple[List[str], bool]:
    +    """Collect real media tags from current-turn producer-tool results only.
    +
    +    Two layered guards keep stale/example MEDIA: strings out of the reply:
    +
    +    1. Producer-tool allowlist: only tools that intentionally emit deliverable
    +       artifacts (TTS) are eligible. Documentation, logs, and search results can
    +       contain example strings such as MEDIA:/absolute/path/to/file, which must
    +       never be delivered as attachments. (Fixes the original report behind #16721.)
    +    2. Current-turn isolation: only messages produced this turn are scanned, so a
    +       tool result from an earlier turn (still present in the full message list)
    +       cannot leak onto a later text-only reply (#34608).
    +
    +    Mid-run context compression can rewrite/shrink the message list below the
    +    original history length. When that happens the slice boundary is no longer
    +    trustworthy, so fall back to scanning every message and rely on
    +    ``history_media_paths`` for dedup, preserving the compression-safe behaviour
    +    of #160. The producer-tool allowlist still applies on the fallback path.
    +    """
    +    history_media_paths = history_media_paths or set()
    +    # Only trust the slice boundary when the message list still contains the
    +    # full history prefix. Otherwise scan everything (compression-safe fallback).
    +    if history_offset and len(messages) >= history_offset:
    +        new_messages = messages[history_offset:]
    +    else:
    +        new_messages = messages
    +
    +    tool_name_by_call_id: Dict[str, str] = {}
    +    for msg in new_messages:
    +        if msg.get("role") != "assistant":
    +            continue
    +        for call in msg.get("tool_calls") or []:
    +            call_id = call.get("id") or call.get("call_id")
    +            fn = call.get("function") or {}
    +            name = str(fn.get("name") or call.get("name") or "")
    +            if call_id and name:
    +                tool_name_by_call_id[str(call_id)] = name
    +
    +    media_tags: List[str] = []
    +    has_voice_directive = False
    +    for msg in new_messages:
    +        if msg.get("role") not in ("tool", "function"):
    +            continue
    +        call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "")
    +        if tool_name_by_call_id.get(call_id) not in _AUTO_APPEND_MEDIA_TOOL_NAMES:
    +            continue
    +        content = str(msg.get("content") or "")
    +        if "MEDIA:" not in content:
    +            continue
    +        for match in _TOOL_MEDIA_RE.finditer(content):
    +            path = match.group(1).strip().rstrip('\",}')
    +            if path and path not in history_media_paths:
    +                media_tags.append(f"MEDIA:{path}")
    +        if "[[audio_as_voice]]" in content:
    +            has_voice_directive = True
    +
    +    return media_tags, has_voice_directive
    +
     # ---------------------------------------------------------------------------
     # SSL certificate auto-detection for NixOS and other non-standard systems.
     # Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported.
    @@ -17931,36 +18013,12 @@ def _approval_notify_sync(approval_data: dict) -> None:
                 # context compression shrinks the message list below the original
                 # history length, preserving the compression-safe behaviour of #160.
                 if "MEDIA:" not in final_response:
    -                media_tags = []
    -                has_voice_directive = False
    -                _all_msgs = result.get("messages", [])
    -                _history_len = len(agent_history)
    -                # Only trust the slice boundary when the message list still
    -                # contains the full history prefix. Mid-run compression can
    -                # rewrite/shrink the list; in that case fall back to scanning
    -                # everything and rely on _history_media_paths for dedup.
    -                if _history_len and len(_all_msgs) >= _history_len:
    -                    _scan_msgs = _all_msgs[_history_len:]
    -                else:
    -                    _scan_msgs = _all_msgs
    -                for msg in _scan_msgs:
    -                    if msg.get("role") in {"tool", "function"}:
    -                        content = msg.get("content", "")
    -                        if "MEDIA:" in content:
    -                            _TOOL_MEDIA_RE = re.compile(
    -                                r'MEDIA:((?:[A-Za-z]:[/\\]|/|~\/)\S+\.(?:png|jpe?g|gif|webp|'
    -                                r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|'
    -                                r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|'
    -                                r'txt|csv|apk|ipa))',
    -                                re.IGNORECASE
    -                            )
    -                            for match in _TOOL_MEDIA_RE.finditer(content):
    -                                path = match.group(1).strip().rstrip('",}')
    -                                if path and path not in _history_media_paths:
    -                                    media_tags.append(f"MEDIA:{path}")
    -                            if "[[audio_as_voice]]" in content:
    -                                has_voice_directive = True
    -                
    +                media_tags, has_voice_directive = _collect_auto_append_media_tags(
    +                    result.get("messages", []),
    +                    history_offset=len(agent_history),
    +                    history_media_paths=_history_media_paths,
    +                )
    +
                     if media_tags:
                         seen = set()
                         unique_tags = []

  • scripts/release.py

    @@ -1032,6 +1032,7 @@
         "[redacted:email]": "holynn-q",
         "[redacted:email]": "jacdevos",
         "[redacted:email]": "liuhao1024",
    +    "[redacted:email]": "VinciZhu",
         "[redacted:email]": "rodrigoeqnit",
         "[email protected]": "sgtworkman",
         "[redacted:email]": "keepcalmqqf",

  • tests/gateway/test_media_extraction.py

    @@ -102,6 +102,63 @@ def extract_media_tags_broken(result_messages):
     
     class TestMediaExtraction:
         """Tests for MEDIA tag extraction from tool results."""
    +
    +    def test_gateway_auto_append_ignores_media_examples_in_skill_docs(self):
    +        """Skill/documentation examples must not be appended as real attachments."""
    +        from gateway.run import _collect_auto_append_media_tags
    +
    +        messages = [
    +            {"role": "user", "content": "How should I format gateway media?"},
    +            {
    +                "role": "assistant",
    +                "tool_calls": [
    +                    {"id": "call_skill", "function": {"name": "skill_view"}}
    +                ],
    +            },
    +            {
    +                "role": "tool",
    +                "tool_call_id": "call_skill",
    +                "content": """
    +Recommended pattern:
    +```text
    +MEDIA:/absolute/path/to/image.png
    +```
    +Second message:
    +```text
    +caption
    +```
    +""",
    +            },
    +            {"role": "assistant", "content": "Use a standalone media message."},
    +        ]
    +
    +        tags, voice = _collect_auto_append_media_tags(messages, history_offset=0)
    +        assert tags == []
    +        assert voice is False
    +
    +    def test_gateway_auto_append_keeps_real_tts_media_tag(self):
    +        """TTS tool media tags are still auto-appended when the model omits them."""
    +        from gateway.run import _collect_auto_append_media_tags
    +
    +        messages = [
    +            {"role": "user", "content": "Say this as audio"},
    +            {
    +                "role": "assistant",
    +                "tool_calls": [
    +                    {"id": "call_tts", "function": {"name": "text_to_speech"}}
    +                ],
    +            },
    +            {
    +                "role": "tool",
    +                "tool_call_id": "call_tts",
    +                "content": '{"success": true, "media_tag": "[[audio_as_voice]]\\nMEDIA:/tmp/voice.ogg"}',
    +            },
    +            {"role": "assistant", "content": "Done."},
    +        ]
    +
    +        tags, voice = _collect_auto_append_media_tags(messages, history_offset=0)
    +        assert tags == ["MEDIA:/tmp/voice.ogg"]
    +        assert voice is True
         
         def test_media_tags_not_extracted_from_history(self):
             """MEDIA tags from previous turns should NOT be extracted again."""

  • tests/gateway/test_platform_base.py

    @@ -400,6 +400,64 @@ def test_relative_path_still_ignored(self):
             )
             assert media == []
     
    +    # --- Code block / inline code / blockquote false-positive guards (#35695) ---
    +
    +    def test_media_in_fenced_code_block_ignored(self):
    +        """MEDIA: inside ``` fenced code blocks must not be extracted."""
    +        content = "Here is an example:\n```text\nMEDIA:/path/to/example.png\n```\nDone."
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert media == []
    +        assert "example" in cleaned.lower()
    +
    +    def test_media_in_inline_code_ignored(self):
    +        """MEDIA: inside backtick inline code must not be extracted."""
    +        content = "Use `MEDIA:/path/to/file.png` in your response."
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert media == []
    +        assert "MEDIA:" in cleaned  # preserved as text
    +
    +    def test_media_in_blockquote_ignored(self):
    +        """MEDIA: inside a > blockquote must not be extracted."""
    +        content = "> To send an image, include MEDIA:/path/to/image.jpg\nEnd."
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert media == []
    +        assert "End." in cleaned
    +
    +    def test_media_outside_code_blocks_still_extracted(self):
    +        """Real MEDIA: tags outside protected regions must still work."""
    +        content = "MEDIA:/real/file.png\n```code\nMEDIA:/fake/file.png\n```"
    +        media, _ = BasePlatformAdapter.extract_media(content)
    +        assert len(media) == 1
    +        assert media[0][0] == "/real/file.png"
    +
    +    def test_media_mixed_code_and_prose(self):
    +        """Real MEDIA: in prose + example in code block: only prose extracted,
    +        and the code block survives verbatim in the delivered text."""
    +        content = (
    +            "Here is your file:\n"
    +            "MEDIA:/output/report.pdf\n"
    +            "Example usage:\n"
    +            "```text\nMEDIA:/example/path.pdf\n```\n"
    +            "Done."
    +        )
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert len(media) == 1
    +        assert media[0][0] == "/output/report.pdf"
    +        assert "Done." in cleaned
    +        # The real tag is stripped from the delivered text...
    +        assert "MEDIA:/output/report.pdf" not in cleaned
    +        # ...but the fenced code block (incl. its example MEDIA: line) must
    +        # survive verbatim — masking is a locator, not a text rewrite.
    +        assert "```text\nMEDIA:/example/path.pdf\n```" in cleaned
    +
    +    def test_inline_code_survives_when_real_media_present(self):
    +        """When a real MEDIA: tag is delivered, an inline-code example in the
    +        same reply must not be blanked to whitespace."""
    +        content = "See MEDIA:/r/a.png and `MEDIA:/ex/b.png` inline"
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert [p for p, _ in media] == ["/r/a.png"]
    +        assert "`MEDIA:/ex/b.png`" in cleaned
    +
     
     class TestMediaInsideSerializedJson:
         """Regression coverage for #34375 — MEDIA: embedded in serialized JSON