# NousResearch/hermes-agent · Gateway auto-appending MEDIA tags from tool results ## Open now - Nothing open on GitHub. These items concern the gateway automatically appending MEDIA tags based on tool results. One is a bug report about tags being added from arbitrary tool output, and two are pull requests that propose restricting this to producer tools or adding false-positive guards for MEDIA: detection. ## TL;DR - 3 member(s): 0 open issue(s), 1 closed issue(s), 0 open PR(s), 1 merged PR(s), 1 PR(s) closed without merging. - A fix was merged to main in PR #36275. Verify it addresses your case. - **Recommended:** read PR #36275 (and any competing PRs below) before filing. Confirm the merged change covers your symptom. ## Maintainer resolution verdict - Internal-tracker verdict over this cluster's issue members: `unresolved`. This is distinct from the fix status above. Weigh it yourself. ## How this cluster was grouped - Link evidence (`STRUCTURAL_SPANNING`): every member is connected by structural evidence: shared changed files, or a closing/superseded reference. - Pairwise link density `0.67`: the share of member pairs that carry a link, counting a structural link in full and a relevance-only link at 0.36. `1.00` means every pair is linked. ## Closed / duplicate issues - #16720: \[Bug\] Gateway auto-appends MEDIA tags from arbitrary tool results ## Prior & competing PRs - **#36275: fix(gateway): salvage MEDIA: false-positive guards (#16721 + #35699)**. merged to main · fix tier `verified` - maintainer direction: category `merged` - files: `gateway/platforms/base.py`, `gateway/run.py`, `scripts/release.py`, `tests/gateway/test_media_extraction.py`, `tests/gateway/test_platform_base.py` - fix hunks: - `gateway/platforms/base.py` ```diff @@ -2579,6 +2579,48 @@ def filter_local_delivery_paths(file_paths) -> List[str]: logger.warning("Skipping unsafe local file path: %s", _log_safe_path(raw)) return safe_paths + + @staticmethod + def _mask_protected_spans(content: str) -> str: + """Replace content inside fenced code blocks, inline code spans, + and blockquotes with spaces to prevent MEDIA: false positives. + + Preserves character count so regex match offsets stay valid. + Skips masking backtick-quoted paths in MEDIA: tags (e.g. + ``MEDIA:`/path/to/file.png` ``) to avoid breaking path extraction. + """ + chars = list(content) + n = len(chars) + + # Build list of (start, end) spans to mask + spans: list = [] + + # Fenced code blocks: ```...``` + for m in re.finditer(r'```[^\n]*\n.*?```', content, re.DOTALL): + spans.append((m.start(), m.end())) + + # Inline code: `...` but NOT backtick-quoted paths in MEDIA: tags + for m in re.finditer(r'`[^`\n]+`', content): + start = m.start() + # Check if this is a backtick-quoted path after MEDIA: + prefix = content[max(0, start - 20):start] + if re.search(r'MEDIA:\s*$', prefix): + continue # This is a MEDIA path quote, not inline code + spans.append((start, m.end())) + + # Blockquote lines: > at line start + for m in re.finditer(r'^>.*$', content, re.MULTILINE): + spans.append((m.start(), m.end())) + + # Apply masking + for start, end in spans: + for i in range(start, end): + if chars[i] != '\n': + chars[i] = ' ' + + return ''.join(chars) + + @staticmethod def _mask_json_string_media(content: str) -> str: """Blank out ``MEDIA:`` occurrences that sit inside a JSON ``` ```diff @@ -2661,9 +2703,14 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]: # set is the shared MEDIA_DELIVERY_EXTS source of truth (built once into # MEDIA_TAG_CLEANUP_RE) so it can never drift from extract_local_files. media_pattern = MEDIA_TAG_CLEANUP_RE - # Mask MEDIA: embedded inside serialized JSON string values so stale - # paths from stored tool results are never re-delivered (#34375). - scan_content = BasePlatformAdapter._mask_json_string_media(content) + # Mask example/stored MEDIA: paths before scanning so they are never + # delivered as real attachments: + # - code blocks / inline code / blockquotes hold prose examples (#35695) + # - serialized JSON string values hold stored tool-result text (#34375) + # Both maskers are offset-preserving (chars -> spaces) so match offsets + # stay valid; chaining them masks the union of both protected regions. + scan_content = BasePlatformAdapter._mask_protected_spans(content) + scan_content = BasePlatformAdapter._mask_json_string_media(scan_content) for match in media_pattern.finditer(scan_content): path = match.group("path").strip() if len(path) >= 2 and path[0] == path[-1] and path[0] in "`\"'": ``` ```diff @@ -2677,15 +2724,17 @@ def extract_media(content: str) -> Tuple[List[Tuple[str, bool]], str]: # and dropping every other attachment in the response. continue - # Remove the delivered MEDIA tags from the user-visible text. Mask - # ``cleaned`` (same length, so offsets line up with it), find the real - # tag spans there, and delete those spans from the *unmasked* ``cleaned``. - # This strips real tags while leaving JSON-embedded MEDIA: text intact — - # it is stored data, not a directive, and must read back verbatim - # (#34375). Masking ``cleaned`` (not ``content``) keeps offsets valid - # after the [[audio_as_voice]] / [[as_document]] directives are removed. + # Remove the delivered MEDIA tags from the user-visible text. Mask a + # length-equal copy of ``cleaned`` (same union of protected regions) to + # *locate* the real tag spans, then delete exactly those spans from the + # *unmasked* ``cleaned``. Masking is only a locator — protected spans + # (code blocks, quotes, JSON-embedded MEDIA: text) must survive verbatim + # in the delivered text, not be blanked to whitespace. Masking + # ``cleaned`` (not ``content``) keeps offsets valid after the + # [[audio_as_voice]] / [[as_document]] directives are removed. if media: - masked_cleaned = BasePlatformAdapter._mask_json_string_media(cleaned) + masked_cleaned = BasePlatformAdapter._mask_protected_spans(cleaned) + masked_cleaned = BasePlatformAdapter._mask_json_string_media(masked_cleaned) spans = [m.span() for m in media_pattern.finditer(masked_cleaned)] if spans: chars = list(cleaned) ``` - `gateway/run.py` ```diff @@ -669,6 +669,88 @@ def _last_transcript_timestamp(history: Optional[List[Dict[str, Any]]]) -> Any: return None +# Tool results can contain literal MEDIA: examples in docs, logs, or other +# ordinary outputs. Only tools that intentionally create deliverable media +# artifacts should be eligible for automatic append when the model omits them +# from the final gateway reply. +_AUTO_APPEND_MEDIA_TOOL_NAMES = {"text_to_speech", "text_to_speech_tool"} + + +# Extension-anchored MEDIA: matcher for tool results. Mirrors the dispatch-site +# pattern so a bare ``MEDIA:`` token in prose (no deliverable extension) is never +# auto-appended. Kept local to the auto-append path; the producer-tool allowlist +# below is the primary guard, this is the secondary precision guard. +_TOOL_MEDIA_RE = re.compile( + r'MEDIA:((?:[A-Za-z]:[/\\]|/|~\/)\S+\.(?:png|jpe?g|gif|webp|' + r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|' + r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|' + r'txt|csv|apk|ipa))', + re.IGNORECASE, +) + + +def _collect_auto_append_media_tags( + messages: List[Dict[str, Any]], + history_offset: int = 0, + history_media_paths: Optional[set] = None, +) -> tuple[List[str], bool]: + """Collect real media tags from current-turn producer-tool results only. + + Two layered guards keep stale/example MEDIA: strings out of the reply: + + 1. Producer-tool allowlist: only tools that intentionally emit deliverable + artifacts (TTS) are eligible. Documentation, logs, and search results can + contain example strings such as MEDIA:/absolute/path/to/file, which must + never be delivered as attachments. (Fixes the original report behind #16721.) + 2. Current-turn isolation: only messages produced this turn are scanned, so a + tool result from an earlier turn (still present in the full message list) + cannot leak onto a later text-only reply (#34608). + + Mid-run context compression can rewrite/shrink the message list below the + original history length. When that happens the slice boundary is no longer + trustworthy, so fall back to scanning every message and rely on + ``history_media_paths`` for dedup, preserving the compression-safe behaviour + of #160. The producer-tool allowlist still applies on the fallback path. + """ + history_media_paths = history_media_paths or set() + # Only trust the slice boundary when the message list still contains the + # full history prefix. Otherwise scan everything (compression-safe fallback). + if history_offset and len(messages) >= history_offset: + new_messages = messages[history_offset:] + else: + new_messages = messages + + tool_name_by_call_id: Dict[str, str] = {} + for msg in new_messages: + if msg.get("role") != "assistant": + continue + for call in msg.get("tool_calls") or []: + call_id = call.get("id") or call.get("call_id") + fn = call.get("function") or {} + name = str(fn.get("name") or call.get("name") or "") + if call_id and name: + tool_name_by_call_id[str(call_id)] = name + + media_tags: List[str] = [] + has_voice_directive = False + for msg in new_messages: + if msg.get("role") not in ("tool", "function"): + continue + call_id = str(msg.get("tool_call_id") or msg.get("call_id") or "") + if tool_name_by_call_id.get(call_id) not in _AUTO_APPEND_MEDIA_TOOL_NAMES: + continue + content = str(msg.get("content") or "") + if "MEDIA:" not in content: + continue + for match in _TOOL_MEDIA_RE.finditer(content): + path = match.group(1).strip().rstrip('\",}') + if path and path not in history_media_paths: + media_tags.append(f"MEDIA:{path}") + if "[[audio_as_voice]]" in content: + has_voice_directive = True + + return media_tags, has_voice_directive + # --------------------------------------------------------------------------- # SSL certificate auto-detection for NixOS and other non-standard systems. # Must run BEFORE any HTTP library (discord, aiohttp, etc.) is imported. ``` ```diff @@ -17931,36 +18013,12 @@ def _approval_notify_sync(approval_data: dict) -> None: # context compression shrinks the message list below the original # history length, preserving the compression-safe behaviour of #160. if "MEDIA:" not in final_response: - media_tags = [] - has_voice_directive = False - _all_msgs = result.get("messages", []) - _history_len = len(agent_history) - # Only trust the slice boundary when the message list still - # contains the full history prefix. Mid-run compression can - # rewrite/shrink the list; in that case fall back to scanning - # everything and rely on _history_media_paths for dedup. - if _history_len and len(_all_msgs) >= _history_len: - _scan_msgs = _all_msgs[_history_len:] - else: - _scan_msgs = _all_msgs - for msg in _scan_msgs: - if msg.get("role") in {"tool", "function"}: - content = msg.get("content", "") - if "MEDIA:" in content: - _TOOL_MEDIA_RE = re.compile( - r'MEDIA:((?:[A-Za-z]:[/\\]|/|~\/)\S+\.(?:png|jpe?g|gif|webp|' - r'mp4|mov|avi|mkv|webm|ogg|opus|mp3|wav|m4a|' - r'flac|epub|pdf|zip|rar|7z|docx?|xlsx?|pptx?|' - r'txt|csv|apk|ipa))', - re.IGNORECASE - ) - for match in _TOOL_MEDIA_RE.finditer(content): - path = match.group(1).strip().rstrip('",}') - if path and path not in _history_media_paths: - media_tags.append(f"MEDIA:{path}") - if "[[audio_as_voice]]" in content: - has_voice_directive = True - + media_tags, has_voice_directive = _collect_auto_append_media_tags( + result.get("messages", []), + history_offset=len(agent_history), + history_media_paths=_history_media_paths, + ) + if media_tags: seen = set() unique_tags = [] ``` - `scripts/release.py` ```diff @@ -1032,6 +1032,7 @@ "[redacted:email]": "holynn-q", "[redacted:email]": "jacdevos", "[redacted:email]": "liuhao1024", + "[redacted:email]": "VinciZhu", "[redacted:email]": "rodrigoeqnit", "178342791+sgtworkman@users.noreply.github.com": "sgtworkman", "[redacted:email]": "keepcalmqqf", ``` - `tests/gateway/test_media_extraction.py` ```diff @@ -102,6 +102,63 @@ def extract_media_tags_broken(result_messages): class TestMediaExtraction: """Tests for MEDIA tag extraction from tool results.""" + + def test_gateway_auto_append_ignores_media_examples_in_skill_docs(self): + """Skill/documentation examples must not be appended as real attachments.""" + from gateway.run import _collect_auto_append_media_tags + + messages = [ + {"role": "user", "content": "How should I format gateway media?"}, + { + "role": "assistant", + "tool_calls": [ + {"id": "call_skill", "function": {"name": "skill_view"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call_skill", + "content": """ +Recommended pattern: +```text +MEDIA:/absolute/path/to/image.png +``` +Second message: +```text +caption +``` +""", + }, + {"role": "assistant", "content": "Use a standalone media message."}, + ] + + tags, voice = _collect_auto_append_media_tags(messages, history_offset=0) + assert tags == [] + assert voice is False + + def test_gateway_auto_append_keeps_real_tts_media_tag(self): + """TTS tool media tags are still auto-appended when the model omits them.""" + from gateway.run import _collect_auto_append_media_tags + + messages = [ + {"role": "user", "content": "Say this as audio"}, + { + "role": "assistant", + "tool_calls": [ + {"id": "call_tts", "function": {"name": "text_to_speech"}} + ], + }, + { + "role": "tool", + "tool_call_id": "call_tts", + "content": '{"success": true, "media_tag": "[[audio_as_voice]]\\nMEDIA:/tmp/voice.ogg"}', + }, + {"role": "assistant", "content": "Done."}, + ] + + tags, voice = _collect_auto_append_media_tags(messages, history_offset=0) + assert tags == ["MEDIA:/tmp/voice.ogg"] + assert voice is True def test_media_tags_not_extracted_from_history(self): """MEDIA tags from previous turns should NOT be extracted again.""" ``` - `tests/gateway/test_platform_base.py` ```diff @@ -400,6 +400,64 @@ def test_relative_path_still_ignored(self): ) assert media == [] + # --- Code block / inline code / blockquote false-positive guards (#35695) --- + + def test_media_in_fenced_code_block_ignored(self): + """MEDIA: inside ``` fenced code blocks must not be extracted.""" + content = "Here is an example:\n```text\nMEDIA:/path/to/example.png\n```\nDone." + media, cleaned = BasePlatformAdapter.extract_media(content) + assert media == [] + assert "example" in cleaned.lower() + + def test_media_in_inline_code_ignored(self): + """MEDIA: inside backtick inline code must not be extracted.""" + content = "Use `MEDIA:/path/to/file.png` in your response." + media, cleaned = BasePlatformAdapter.extract_media(content) + assert media == [] + assert "MEDIA:" in cleaned # preserved as text + + def test_media_in_blockquote_ignored(self): + """MEDIA: inside a > blockquote must not be extracted.""" + content = "> To send an image, include MEDIA:/path/to/image.jpg\nEnd." + media, cleaned = BasePlatformAdapter.extract_media(content) + assert media == [] + assert "End." in cleaned + + def test_media_outside_code_blocks_still_extracted(self): + """Real MEDIA: tags outside protected regions must still work.""" + content = "MEDIA:/real/file.png\n```code\nMEDIA:/fake/file.png\n```" + media, _ = BasePlatformAdapter.extract_media(content) + assert len(media) == 1 + assert media[0][0] == "/real/file.png" + + def test_media_mixed_code_and_prose(self): + """Real MEDIA: in prose + example in code block: only prose extracted, + and the code block survives verbatim in the delivered text.""" + content = ( + "Here is your file:\n" + "MEDIA:/output/report.pdf\n" + "Example usage:\n" + "```text\nMEDIA:/example/path.pdf\n```\n" + "Done." + ) + media, cleaned = BasePlatformAdapter.extract_media(content) + assert len(media) == 1 + assert media[0][0] == "/output/report.pdf" + assert "Done." in cleaned + # The real tag is stripped from the delivered text... + assert "MEDIA:/output/report.pdf" not in cleaned + # ...but the fenced code block (incl. its example MEDIA: line) must + # survive verbatim — masking is a locator, not a text rewrite. + assert "```text\nMEDIA:/example/path.pdf\n```" in cleaned + + def test_inline_code_survives_when_real_media_present(self): + """When a real MEDIA: tag is delivered, an inline-code example in the + same reply must not be blanked to whitespace.""" + content = "See MEDIA:/r/a.png and `MEDIA:/ex/b.png` inline" + media, cleaned = BasePlatformAdapter.extract_media(content) + assert [p for p, _ in media] == ["/r/a.png"] + assert "`MEDIA:/ex/b.png`" in cleaned + class TestMediaInsideSerializedJson: """Regression coverage for #34375 — MEDIA: embedded in serialized JSON ``` - **#16721: fix(gateway): restrict auto-appended media to producer tools**. closed without merge - maintainer direction: category `merged_via`; implication `superseded`; refs `#36275`, `#34608`, `#160` ## Fix status - Merged to main in PR #36275 (cited above). No release data is tracked. Verify downstream. - Fix last verified (merged) `2026-06-01`. - Subsystem: `gateway`