MEDIA tag regex missing file extensions in gateway

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #34844 fix(gateway): unify MEDIA: extraction extension set + close the unknown-ext black hole (#34517)

  merged to main

  files: gateway/platforms/base.py, gateway/run.py, gateway/stream_consumer.py, scripts/release.py, tests/gateway/test_platform_base.py

  Show the diff

  • gateway/platforms/base.py

    @@ -1131,6 +1131,75 @@ def _log_safe_path(path: str) -> str:
     }
     
     
    +# ---------------------------------------------------------------------------
    +# Media-delivery extension allowlist — SINGLE SOURCE OF TRUTH
    +#
    +# Both extractors that turn response text into native attachments derive their
    +# extension set from this tuple:
    +#   * ``extract_media()``       — explicit ``MEDIA:<path>`` tags
    +#   * ``extract_local_files()`` — bare absolute/home paths the agent mentions
    +#
    +# Historically these two carried independently-maintained extension lists.
    +# ``extract_media`` had a narrow list (no .md/.json/.yaml/.xml/.html/...) while
    +# ``extract_local_files`` had a broad one. Combined with the unconditional
    +# ``MEDIA:\\s*\\S+`` cleanup at the dispatch sites, that mismatch created a
    +# silent black hole: a ``MEDIA:/report.md`` tag failed the narrow extract_media
    +# match, got stripped from the body by the loose cleanup regex, and was then
    +# invisible to extract_local_files — the file was never delivered (issue
    +# #34517). Keeping one list eliminates the drift; building the cleanup regexes
    +# from the same set means a tag is only stripped when its extension is one we
    +# can actually deliver, so an unknown-extension path survives in the body
    +# instead of vanishing.
    +#
    +# Covers images (inline), video (inline where supported), audio (voice/audio),
    +# documents/spreadsheets/presentations (send_document), archives, and rendered
    +# web output. The dispatch partition (image vs video vs document) lives in
    +# ``gateway/run.py``.
    +# ---------------------------------------------------------------------------
    +
    +MEDIA_DELIVERY_EXTS: Tuple[str, ...] = (
    +    # Images (embed inline)
    +    ".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".svg",
    +    # Video (embed inline where supported)
    +    ".mp4", ".mov", ".avi", ".mkv", ".webm",
    +    # Audio (delivered as voice/audio where supported)
    +    ".mp3", ".wav", ".ogg", ".opus", ".m4a", ".flac",
    +    # Documents (uploaded as file attachments)
    +    ".pdf", ".docx", ".doc", ".odt", ".rtf", ".txt", ".md", ".epub",
    +    # Spreadsheets / data
    +    ".xlsx", ".xls", ".ods", ".csv", ".tsv", ".json", ".xml", ".yaml", ".yml",
    +    # Presentations
    +    ".pptx", ".ppt", ".odp", ".key",
    +    # Archives
    +    ".zip", ".tar", ".gz", ".tgz", ".bz2", ".xz", ".7z", ".rar", ".apk", ".ipa",
    +    # Web / rendered output
    +    ".html", ".htm",
    +)
    +
    +# Regex alternation fragment of bare extensions (no leading dot), e.g.
    +# ``png|jpe?g|...``. ``jpe?g`` collapses jpg/jpeg into one branch. Sorted
    +# longest-first so the alternation never matches a shorter ext as a prefix of
    +# a longer one (e.g. ``.tar`` before ``.tar.gz`` components).
    +_MEDIA_EXT_ALTERNATION = "|".join(
    +    sorted((e.lstrip(".") for e in MEDIA_DELIVERY_EXTS), key=len, reverse=True)
    +)
    +
    +# Anchored ``MEDIA:<path>`` cleanup pattern. Unlike the old loose
    +# ``MEDIA:\\s*\\S+``, this only strips a tag whose path ends in a known
    +# deliverable extension (optionally quoted/backticked). A ``MEDIA:`` tag with
    +# an unknown extension is left in the text so it can still be picked up by the
    +# bare-path detector (extract_local_files) downstream rather than silently
    +# deleted. Shared by the non-streaming dispatch path and the streaming
    +# consumer so both behave identically.
    +MEDIA_TAG_CLEANUP_RE = re.compile(
    +    r'''[`"']?MEDIA:\s*'''
    +    r'''(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|'''
    +    r'''(?:~/|/)\S+(?:[^\S\n]+\S+)*?\.(?:''' + _MEDIA_EXT_ALTERNATION + r'''))'''
    +    r'''(?=[\s`"',;:)\]}]|$)[`"']?''',
    +    re.IGNORECASE,
    +)
    +
    +
     def get_document_cache_dir() -> Path:
         """Return the document cache directory, creating it if it doesn't exist."""
         DOCUMENT_CACHE_DIR.mkdir(parents=True, exist_ok=True)
    @@ -2542,10 +2611,10 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]:
             cleaned = cleaned.replace("[[as_document]]", "")
             
             # Extract MEDIA:<path> tags, allowing optional whitespace after the colon
    -        # and quoted/backticked paths for LLM-formatted outputs.
    -        media_pattern = re.compile(
    -            r'''[`"']?MEDIA:\s*(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|(?:~/|/)\S+(?:[^\S\n]+\S+)*?\.(?:png|jpe?g|gif|webp|mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|txt|csv|apk|ipa)(?=[\s`"',;:)\]}]|$))[`"']?'''
    -        )
    +        # and quoted/backticked paths for LLM-formatted outputs. The extension
    +        # set is the shared MEDIA_DELIVERY_EXTS source of truth (built once into
    +        # MEDIA_TAG_CLEANUP_RE) so it can never drift from extract_local_files.
    +        media_pattern = MEDIA_TAG_CLEANUP_RE
             for match in media_pattern.finditer(content):
                 path = match.group("path").strip()
                 if len(path) >= 2 and path[0] == path[-1] and path[0] in "`\"'":
    @@ -2591,24 +2660,7 @@ def extract_local_files(content: str) -> Tuple[List[str], str]:
                 Tuple of (list of expanded file paths, cleaned text with the
                 raw path strings removed).
             """
    -        _LOCAL_MEDIA_EXTS = (
    -            # Images (embed inline)
    -            '.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.tiff', '.svg',
    -            # Video (embed inline where supported)
    -            '.mp4', '.mov', '.avi', '.mkv', '.webm',
    -            # Audio (delivered as voice/audio where supported)
    -            '.mp3', '.wav', '.ogg', '.m4a', '.flac',
    -            # Documents (uploaded as file attachments)
    -            '.pdf', '.docx', '.doc', '.odt', '.rtf', '.txt', '.md',
    -            # Spreadsheets / data
    -            '.xlsx', '.xls', '.ods', '.csv', '.tsv', '.json', '.xml', '.yaml', '.yml',
    -            # Presentations
    -            '.pptx', '.ppt', '.odp', '.key',
    -            # Archives
    -            '.zip', '.tar', '.gz', '.tgz', '.bz2', '.xz', '.7z', '.rar',
    -            # Web / rendered output
    -            '.html', '.htm',
    -        )
    +        _LOCAL_MEDIA_EXTS = MEDIA_DELIVERY_EXTS
             ext_part = '|'.join(e.lstrip('.') for e in _LOCAL_MEDIA_EXTS)
     
             # (?<![/:\w.]) prevents matching inside URLs (e.g. https://…/img.png)
    @@ -3729,7 +3781,12 @@ async def _stop_typing_task() -> None:
                     # Strip any remaining internal directives from message body (fixes #1561)
                     text_content = text_content.replace("[[audio_as_voice]]", "").strip()
                     text_content = text_content.replace("[[as_document]]", "").strip()
    -                text_content = re.sub(r"MEDIA:\s*\S+", "", text_content).strip()
    +                # Strip only MEDIA: tags whose path has a deliverable extension
    +                # (shared MEDIA_TAG_CLEANUP_RE). A MEDIA: tag with an unknown
    +                # extension is intentionally left in the body so extract_local_files
    +                # below can still pick up the bare path — otherwise the file would
    +                # be silently dropped (issue #34517).
    +                text_content = MEDIA_TAG_CLEANUP_RE.sub("", text_content).strip()
                     if images:
                         logger.info("[%s] extract_images found %d image(s) in response (%d chars)", self.name, len(images), len(response))
     

  • gateway/run.py

    @@ -11743,9 +11743,16 @@ async def _deliver_media_from_response(
     
                 from gateway.platforms.base import BasePlatformAdapter, should_send_media_as_audio
     
    -            media_files, _ = adapter.extract_media(response)
    +            media_files, cleaned = adapter.extract_media(response)
                 media_files = BasePlatformAdapter.filter_media_delivery_paths(media_files)
    -            _, cleaned = adapter.extract_images(response)
    +            # Chain the cleaned text through each extractor (extract_media →
    +            # extract_images → extract_local_files) so MEDIA: tags and image URLs
    +            # are removed before the bare-path auto-detect runs. Previously the
    +            # cleaned text from extract_media was dropped (``_``) and
    +            # extract_local_files scanned text that still contained MEDIA: tags,
    +            # producing false-positive bare-path matches with the MEDIA: prefix
    +            # glued on. This matches the chain order in gateway/platforms/base.py.
    +            _, cleaned = adapter.extract_images(cleaned)
                 local_files, _ = adapter.extract_local_files(cleaned)
                 local_files = BasePlatformAdapter.filter_local_delivery_paths(local_files)
     

  • gateway/stream_consumer.py

    @@ -26,6 +26,7 @@
     
     from gateway.platforms.base import BasePlatformAdapter as _BasePlatformAdapter
     from gateway.platforms.base import _custom_unit_to_cp
    +from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
     from gateway.config import (
         DEFAULT_STREAMING_EDIT_INTERVAL as _DEFAULT_STREAMING_EDIT_INTERVAL,
         DEFAULT_STREAMING_BUFFER_THRESHOLD as _DEFAULT_STREAMING_BUFFER_THRESHOLD,
    @@ -645,10 +646,13 @@ async def run(self) -> None:
             except Exception as e:
                 logger.error("Stream consumer error: %s", e)
     
    -    # Pattern to strip MEDIA:<path> tags (including optional surrounding quotes).
    -    # Matches the simple cleanup regex used by the non-streaming path in
    -    # gateway/platforms/base.py for post-processing.
    -    _MEDIA_RE = re.compile(r'''[`"']?MEDIA:\s*\S+[`"']?''')
    +    # Strip MEDIA:<path> tags before display. Uses the shared anchored
    +    # MEDIA_TAG_CLEANUP_RE from gateway/platforms/base.py — only tags whose
    +    # path ends in a deliverable extension are removed, so an unknown-extension
    +    # path stays visible instead of being silently dropped (issue #34517).
    +    # Streaming and non-streaming paths share the same regex, so a tag is
    +    # treated identically whichever path delivered the text.
    +    _MEDIA_RE = MEDIA_TAG_CLEANUP_RE
     
         @staticmethod
         def _clean_for_display(text: str) -> str:

  • scripts/release.py

    @@ -523,6 +523,8 @@
         "[redacted:email]": "JezzaHehn",
         "[redacted:email]": "dodo-reach",
         "[redacted:email]": "Bartok9",
    +    "[redacted:email]": "banditburai",
    +    "[redacted:email]": "Kyzcreig",
         "[redacted:email]": "crayfish-ai",
         "[redacted:email]": "MestreY0d4-Uninter",
         "[redacted:email]": "Roy-oss1",

  • tests/gateway/test_platform_base.py

    @@ -362,6 +362,54 @@ def test_both_directives_can_coexist(self):
             assert "[[as_document]]" not in cleaned
     
     
    +class TestMediaExtensionAllowlistParity:
    +    """Regression coverage for issue #34517 — the MEDIA: extension black hole.
    +
    +    extract_media used to carry a narrow extension allowlist that omitted
    +    .md/.json/.yaml/.xml/.html etc., while extract_local_files had a broad one.
    +    Combined with an unconditional ``MEDIA:\\s*\\S+`` strip at the dispatch
    +    sites, an unmatched MEDIA: tag for one of those extensions was deleted from
    +    the body before extract_local_files could pick up the bare path — the file
    +    was silently dropped. Both extractors now derive from the single
    +    MEDIA_DELIVERY_EXTS source of truth, and the strip is anchored to that set.
    +    """
    +
    +    DROPPED_BEFORE = ["md", "json", "yaml", "yml", "xml", "html", "htm",
    +                      "tsv", "svg"]
    +
    +    def test_previously_dropped_extensions_now_extract(self):
    +        for ext in self.DROPPED_BEFORE:
    +            path = f"/tmp/report.{ext}"
    +            media, _ = BasePlatformAdapter.extract_media(f"Here: MEDIA:{path}")
    +            assert media == [(path, False)], f".{ext} should extract via MEDIA:"
    +
    +    def test_extract_media_and_local_files_share_one_extension_set(self):
    +        from gateway.platforms.base import MEDIA_DELIVERY_EXTS
    +        # Both functions reference MEDIA_DELIVERY_EXTS; assert the documents
    +        # that motivated the bug are present in the shared set.
    +        for ext in (".md", ".json", ".yaml", ".yml", ".xml", ".html", ".htm"):
    +            assert ext in MEDIA_DELIVERY_EXTS
    +
    +    def test_unknown_extension_not_black_holed_by_cleanup(self):
    +        """A MEDIA: tag with an unknown extension is NOT stripped from the
    +        body — it survives so extract_local_files can still see the bare path,
    +        rather than vanishing entirely (the core of issue #34517)."""
    +        from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
    +        text = "Saved to MEDIA:/tmp/data.weirdext done"
    +        media, _ = BasePlatformAdapter.extract_media(text)
    +        assert media == []  # unknown extension is not a deliverable MEDIA tag
    +        stripped = MEDIA_TAG_CLEANUP_RE.sub("", text)
    +        assert "/tmp/data.weirdext" in stripped  # path preserved, not dropped
    +
    +    def test_known_extension_tag_is_stripped_from_body(self):
    +        from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
    +        text = "Here is your report: MEDIA:/tmp/report.md"
    +        stripped = MEDIA_TAG_CLEANUP_RE.sub("", text).strip()
    +        assert "MEDIA:" not in stripped
    +        assert "/tmp/report.md" not in stripped
    +        assert "Here is your report:" in stripped
    +
    +
     class TestMediaDeliveryPathValidation:
         def _patch_roots(self, monkeypatch, *roots):
             monkeypatch.setattr(

• #65510 fix(gateway): deliver MEDIA: tags for every file type via validated egress

  merged to main

  files: gateway/platforms/base.py, tests/gateway/test_platform_base.py

  Show the diff

  • gateway/platforms/base.py

    @@ -1504,12 +1504,16 @@ def _log_safe_path(path: str) -> str:
         re.IGNORECASE,
     )
     
    -# Extension-less absolute paths (e.g. Caddyfile, Dockerfile, Makefile) are
    -# intentionally excluded from MEDIA_TAG_CLEANUP_RE — they are validated and
    -# delivered via MEDIA_EXTENSIONLESS_TAG_RE so prompt-injection paths that do
    -# not exist on disk are left visible instead of silently dropped. Paths with
    -# an unknown but present extension (e.g. .weirdext) stay on the #34517
    -# bare-path fallback and are not handled here.
    +# Paths NOT covered by MEDIA_TAG_CLEANUP_RE's extension alternation — both
    +# extension-less files (Caddyfile, Dockerfile, Makefile) and files with an
    +# unknown extension (.py, .log, .weirdext, ...) — are validated and delivered
    +# via MEDIA_EXTENSIONLESS_TAG_RE. Every ``MEDIA:`` path is therefore
    +# deliverable regardless of file type (#36060): known extensions extract
    +# unconditionally via the anchored pattern above, everything else extracts
    +# only after ``validate_media_delivery_path`` accepts it (exists on disk, not
    +# under the credential/system denylist, strict-mode rules honored), so
    +# prompt-injection paths that do not validate are left visible instead of
    +# silently dropped.
     MEDIA_EXTENSIONLESS_TAG_RE = re.compile(
         r'''[`"']?MEDIA:\s*'''
         r'''(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|'''
    @@ -1527,8 +1531,16 @@ def _normalize_media_tag_path(raw: str) -> str:
     
     
     def _path_lacks_deliverable_extension(path: str) -> bool:
    -    """True only when the basename has no extension (Caddyfile, Makefile, …)."""
    -    return not Path(path).suffix
    +    """True when MEDIA_TAG_CLEANUP_RE's extension alternation does not cover
    +    ``path`` — either the basename has no extension at all (Caddyfile,
    +    Makefile, …) or the extension is not in MEDIA_DELIVERY_EXTS (.py, .log,
    +    .weirdext, …). Such paths route through the validated delivery pass
    +    (``validate_media_delivery_path``) instead of the unconditional one, so
    +    every file type is deliverable (#36060) while nonexistent / denylisted
    +    paths stay visible in the text.
    +    """
    +    suffix = Path(path).suffix.lower()
    +    return not suffix or suffix not in MEDIA_DELIVERY_EXTS
     
     
     def _strip_media_tag_directives(text: str) -> str:

  • tests/gateway/test_platform_base.py

    @@ -631,12 +631,15 @@ def test_extract_media_and_local_files_share_one_extension_set(self):
     
         def test_unknown_extension_not_black_holed_by_cleanup(self):
             """A MEDIA: tag with an unknown extension is NOT stripped from the
    -        body — it survives so extract_local_files can still see the bare path,
    -        rather than vanishing entirely (the core of issue #34517)."""
    +        body by the extension-anchored cleanup — and when the path does not
    +        validate (nonexistent file here), it is not delivered either, so the
    +        tag survives visibly instead of vanishing (the core of issue #34517).
    +        Validated unknown-extension paths DO deliver — see
    +        TestUniversalMediaEgress (#36060)."""
             from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
             text = "Saved to MEDIA:/tmp/data.weirdext done"
             media, _ = BasePlatformAdapter.extract_media(text)
    -        assert media == []  # unknown extension is not a deliverable MEDIA tag
    +        assert media == []  # nonexistent path fails validation, not delivered
             stripped = MEDIA_TAG_CLEANUP_RE.sub("", text)
             assert "/tmp/data.weirdext" in stripped  # path preserved, not dropped
     
    @@ -712,6 +715,104 @@ def test_as_document_directive_stripped_without_media_tag(self):
             )
     
     
    +class TestUniversalMediaEgress:
    +    """#36060: every MEDIA: path is deliverable regardless of file type.
    +
    +    Known extensions extract unconditionally (MEDIA_TAG_CLEANUP_RE); unknown
    +    extensions and extension-less files extract via the validated pass —
    +    delivered when validate_media_delivery_path accepts them, left visible
    +    when it does not (nonexistent, denylisted).
    +    """
    +
    +    def _patch_allow_root(self, monkeypatch, root):
    +        monkeypatch.setattr(
    +            "gateway.platforms.base.MEDIA_DELIVERY_SAFE_ROOTS",
    +            (str(root),),
    +        )
    +        monkeypatch.delenv("HERMES_MEDIA_DELIVERY_STRICT", raising=False)
    +
    +    @pytest.mark.parametrize("name", [
    +        "script.py", "server.log", "notes.weirdext", "app.ts", "run.sh",
    +        "config.toml", "styles.css", "contract.sol",
    +    ])
    +    def test_unknown_extension_delivered_when_file_validates(
    +        self, tmp_path, monkeypatch, name,
    +    ):
    +        root = tmp_path / "output"
    +        root.mkdir()
    +        f = root / name
    +        f.write_text("content", encoding="utf-8")
    +        self._patch_allow_root(monkeypatch, root)
    +
    +        content = f"Here you go:\nMEDIA:{f}\nDone."
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert len(media) == 1
    +        assert media[0][0] == str(f.resolve())
    +        assert "MEDIA:" not in cleaned
    +        assert "Done." in cleaned
    +
    +    def test_unknown_extension_left_visible_when_not_on_disk(
    +        self, tmp_path, monkeypatch,
    +    ):
    +        root = tmp_path / "output"
    +        root.mkdir()
    +        self._patch_allow_root(monkeypatch, root)
    +
    +        content = "MEDIA:/nonexistent/script.py"
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert media == []
    +        assert "MEDIA:/nonexistent/script.py" in cleaned
    +
    +    def test_denylisted_paths_still_rejected_regardless_of_extension(
    +        self, tmp_path, monkeypatch,
    +    ):
    +        # A denylisted path must not deliver even though .py/.log/etc now
    +        # route through the validated pass. _media_delivery_denied_paths()
    +        # reads _MEDIA_DELIVERY_DENIED_PREFIXES at call time, so patching the
    +        # tuple exercises the real denylist logic.
    +        secret_dir = [redacted:secret-assignment] / "secrets"
    +        secret_dir.mkdir()
    +        f = secret_dir / "creds.py"
    +        f.write_text("TOKEN = 'x'", encoding="utf-8")
    +        monkeypatch.setattr(
    +            "gateway.platforms.base._MEDIA_DELIVERY_DENIED_PREFIXES",
    +            (str(secret_dir),),
    +        )
    +        monkeypatch.delenv("HERMES_MEDIA_DELIVERY_STRICT", raising=False)
    +
    +        content = f"MEDIA:{f}"
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert media == []
    +        assert "MEDIA:" in cleaned  # rejected tag stays visible
    +
    +    def test_strip_for_display_strips_validated_unknown_extension(
    +        self, tmp_path, monkeypatch,
    +    ):
    +        root = tmp_path / "output"
    +        root.mkdir()
    +        f = root / "server.log"
    +        f.write_text("x", encoding="utf-8")
    +        self._patch_allow_root(monkeypatch, root)
    +
    +        text = f"MEDIA:{f}"
    +        stripped = BasePlatformAdapter.strip_media_directives_for_display(text)
    +        assert "MEDIA:" not in stripped
    +
    +    def test_strip_for_display_keeps_unvalidated_unknown_extension(self):
    +        text = "MEDIA:/nonexistent/server.log"
    +        stripped = BasePlatformAdapter.strip_media_directives_for_display(text)
    +        assert "MEDIA:/nonexistent/server.log" in stripped
    +
    +    def test_known_extension_still_unconditional(self):
    +        # Known extensions keep the pre-#36060 behavior: extracted (and the
    +        # tag stripped) even when the file does not exist — downstream
    +        # delivery surfaces the failure.
    +        content = "MEDIA:/nonexistent/report.pdf"
    +        media, cleaned = BasePlatformAdapter.extract_media(content)
    +        assert media == [("/nonexistent/report.pdf", False)]
    +        assert "MEDIA:" not in cleaned
    +
    +
     class TestMediaDeliveryPathValidation:
         def _patch_roots(self, monkeypatch, *roots):
             monkeypatch.setattr(