Telegram gateway tool progress and audio routing fixes
NousResearch/hermes-agent · cluster 26207
Data as of
Nothing open on GitHub. Fix merged in #28478. Check it covers your case.
The short answer
0open issues
0open PRs
5items grouped
- Known?
- Yes. 5 related items grouped here, 1 issue already closed.
- Fix event
- A fix was merged to main in PR #28478. Check it covers your case. No release data is tracked.
- Tracker
- The maintainers' tracker marks this
resolved, deciding PR #28482. That is separate from the fix event. - Still open
- Nothing open on GitHub.
These issues and pull requests address rolling over Telegram tool progress bubbles before message limits are reached and routing Telegram audio file attachments away from the speech-to-text pipeline. The members share a focused theme around Telegram gateway message handling behavior.
Open issues0
Open PRs0
Fixes on record
-
#28478 fix(gateway): route Telegram audio file attachments away from STT pipeline (#24870)
files:
gateway/run.py,scripts/release.py,tests/gateway/test_telegram_audio_vs_voice.pyShow the diff
-
gateway/run.py@@ -7257,11 +7257,19 @@ async def _prepare_inbound_message_text( if event.media_urls: image_paths = [] audio_paths = [] + audio_file_paths: list[str] = [] # audio file attachments — not for STT for i, path in enumerate(event.media_urls): mtype = event.media_types[i] if i < len(event.media_types) else "" if mtype.startswith("image/") or event.message_type == MessageType.PHOTO: image_paths.append(path) - if mtype.startswith("audio/") or event.message_type in {MessageType.VOICE, MessageType.AUDIO}: + # MessageType.AUDIO = audio file attachment (e.g. .mp3, .m4a) — never STT + # MessageType.VOICE = voice message (Opus/OGG) — always STT + if event.message_type == MessageType.AUDIO: + audio_file_paths.append(path) + elif event.message_type == MessageType.VOICE or ( + mtype.startswith("audio/") + and event.message_type not in {MessageType.AUDIO, MessageType.DOCUMENT} + ): audio_paths.append(path) if image_paths: @@ -7323,6 +7331,21 @@ async def _prepare_inbound_message_text( except Exception: pass + if audio_file_paths: + from tools.credential_files import to_agent_visible_cache_path as _to_agent_path + for _apath in audio_file_paths: + _basename = os.path.basename(_apath) + _parts = _basename.split("_", 2) + _display = _parts[2] if len(_parts) >= 3 else _basename + _display = re.sub(r'[^\w.\- ]', '_', _display) + _agent_path = _to_agent_path(_apath) + _note = ( + f"[The user sent an audio file attachment: '{_display}'. " + f"It is saved at: {_agent_path}. " + f"Ask the user what they'd like you to do with it, or pass the path to a transcription or media tool.]" + ) + message_text = f"{_note}\n\n{message_text}" + if event.media_urls and event.message_type == MessageType.DOCUMENT: import mimetypes as _mimetypes from tools.credential_files import to_agent_visible_cache_path -
scripts/release.py@@ -165,6 +165,7 @@ "[redacted:email]": "dengtaoyuan450-a11y", "[redacted:email]": "Junass1", "[redacted:email]": "Bartok9", + "[redacted:email]": "Bartok9", "[redacted:email]": "hllqkb", "[redacted:email]": "Bongulielmi", "[redacted:email]": "JTroyerOvermatch", -
tests/gateway/test_telegram_audio_vs_voice.py@@ -0,0 +1,184 @@ +""" +Tests for #24870 — Telegram: audio file attachments must NOT be routed to STT. + +Telegram distinguishes three kinds of audio payloads: + - message.voice → Opus/OGG voice message → STT pipeline + - message.audio → audio file attachment → file path note, NOT STT + - message.document (audio mime) → generic file route + +These tests confirm that: + 1. MessageType.VOICE events still flow through the STT pipeline. + 2. MessageType.AUDIO events bypass STT and get a file-path context note instead. + 3. Mixed media lists (voice + audio) split correctly. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from gateway.config import GatewayConfig, Platform +from gateway.platforms.base import MessageEvent, MessageType +from gateway.session import SessionSource + + +def _make_runner(stt_enabled: bool = True) -> "GatewayRunner": # type: ignore[name-defined] + from gateway.run import GatewayRunner + + runner = GatewayRunner.__new__(GatewayRunner) + runner.config = GatewayConfig(stt_enabled=stt_enabled) + runner.adapters = {} + runner._model = "test-model" + runner._base_url = "" + runner._has_setup_skill = lambda: False + return runner + + +def _voice_event(path: str = "/tmp/voice.ogg") -> MessageEvent: + return MessageEvent( + text="", + message_type=MessageType.VOICE, + source=SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm"), + media_urls=[path], + media_types=["audio/ogg"], + ) + + +def _audio_event(path: str = "/tmp/song.mp3") -> MessageEvent: + return MessageEvent( + text="", + message_type=MessageType.AUDIO, + source=SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm"), + media_urls=[path], + media_types=["audio/mpeg"], + ) + + +# --------------------------------------------------------------------------- +# 1. VOICE still goes through STT +# --------------------------------------------------------------------------- + [redacted:email] +async def test_voice_message_still_transcribed(): + """MessageType.VOICE must still be sent through _enrich_message_with_transcription.""" + runner = _make_runner(stt_enabled=True) + source = SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm") + event = _voice_event("/tmp/voice.ogg") + + with patch( + "tools.transcription_tools.transcribe_audio", + return_value={"success": True, "transcript": "hello world", "provider": "whisper"}, + ) as mock_transcribe: + result = await runner._prepare_inbound_message_text( + event=event, + source=source, + history=[], + ) + + mock_transcribe.assert_called_once_with("/tmp/voice.ogg") + assert "hello world" in result + assert "voice message" in result.lower() + + +# --------------------------------------------------------------------------- +# 2. AUDIO file attachment bypasses STT +# --------------------------------------------------------------------------- + [redacted:email] +async def test_audio_attachment_skips_stt(): + """MessageType.AUDIO must NOT be routed to STT — transcribe_audio must not be called.""" + runner = _make_runner(stt_enabled=True) + source = SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm") + event = _audio_event("/tmp/song.mp3") + + with patch( + "tools.transcription_tools.transcribe_audio", + side_effect=AssertionError("transcribe_audio must NOT be called for audio file attachments"), + ): + with patch( + "tools.credential_files.to_agent_visible_cache_path", + side_effect=lambda p: p, + ): + result = await runner._prepare_inbound_message_text( + event=event, + source=source, + history=[], + ) + + assert result is not None + assert "/tmp/song.mp3" in result + assert "audio file attachment" in result.lower() + + [redacted:email] +async def test_audio_attachment_context_note_format(): + """Context note for audio file attachments should include the file path and guidance.""" + runner = _make_runner(stt_enabled=True) + source = SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm") + event = _audio_event("/tmp/cache_12345_my_song.mp3") + + with patch( + "tools.transcription_tools.transcribe_audio", + side_effect=AssertionError("must not be called"), + ): + with patch( + "tools.credential_files.to_agent_visible_cache_path", + side_effect=lambda p: p, + ): + result = await runner._prepare_inbound_message_text( + event=event, + source=source, + history=[], + ) + + assert "my_song.mp3" in result + assert "audio file attachment" in result.lower() + # Should NOT contain the voice-message transcription wrapper text + assert "voice message" not in result.lower() + + +# --------------------------------------------------------------------------- +# 3. STT disabled still results in no transcription for audio file attachments +# --------------------------------------------------------------------------- + [redacted:email] +async def test_audio_attachment_skips_stt_when_stt_disabled(): + """Even with STT disabled, AUDIO must NOT produce STT disabled notice — just a file note.""" + runner = _make_runner(stt_enabled=False) + source = SessionSource(platform=Platform.TELEGRAM, chat_id="1", chat_type="dm") + event = _audio_event("/tmp/podcast.m4a") + + with patch( + "tools.transcription_tools.transcribe_audio", + side_effect=AssertionError("must not be called"), + ): + with patch( + "tools.credential_files.to_agent_visible_cache_path", + side_effect=lambda p: p, + ): + result = await runner._prepare_inbound_message_text( + event=event, + source=source, + history=[], + ) + + # Should NOT see the "transcription is disabled" note — that's only for VOICE + assert "transcription is disabled" not in result.lower() + assert "audio file attachment" in result.lower() + assert "/tmp/podcast.m4a" in result + + +# --------------------------------------------------------------------------- +# 4. Telegram gateway: msg.audio → MessageType.AUDIO (not VOICE) +# --------------------------------------------------------------------------- + +def test_telegram_media_type_detection_audio_vs_voice(): + """The Telegram platform must set MessageType.AUDIO for msg.audio, VOICE for msg.voice.""" + from gateway.platforms.base import MessageType + + # The Telegram adapter's _build_media_type already returns correct values + # via MessageType.AUDIO for .audio and MessageType.VOICE for .voice. + # Check the constants match expected semantic roles. + assert MessageType.AUDIO.value == "audio" + assert MessageType.VOICE.value == "voice" + # Sanity: they are distinct + assert MessageType.AUDIO != MessageType.VOICE
-
-
#28482 fix(gateway): roll over Telegram tool progress bubbles + scope audio_file_paths
files:
gateway/run.py,tests/gateway/test_run_progress_topics.pyShow the diff
-
gateway/run.py@@ -7254,10 +7254,13 @@ async def _prepare_inbound_message_text( if getattr(event, "channel_context", None): message_text = f"{event.channel_context}\n\n[New message]\n{message_text}" + # Declare at outer scope so the audio-file-paths handling block below + # remains safe when ``event.media_urls`` is empty (no inner block runs). + audio_file_paths: list[str] = [] + if event.media_urls: image_paths = [] audio_paths = [] - audio_file_paths: list[str] = [] # audio file attachments — not for STT for i, path in enumerate(event.media_urls): mtype = event.media_types[i] if i < len(event.media_types) else "" if mtype.startswith("image/") or event.message_type == MessageType.PHOTO: @@ -15276,12 +15279,104 @@ async def send_progress_messages(): break return - progress_lines = [] # Accumulated tool lines - progress_msg_id = None # ID of the progress message to edit + progress_lines = [] # Accumulated tool lines for the CURRENT editable bubble + progress_msg_id = None # ID of the current progress message to edit can_edit = True # False once an edit fails (platform doesn't support it) _last_edit_ts = 0.0 # Throttle edits to avoid Telegram flood control _PROGRESS_EDIT_INTERVAL = 1.5 # Minimum seconds between edits + _progress_len_fn = ( + adapter.message_len_fn + if isinstance(adapter, BasePlatformAdapter) + else len + ) + try: + _raw_progress_limit = int(getattr(adapter, "MAX_MESSAGE_LENGTH", 4000) or 4000) + except Exception: + _raw_progress_limit = 4000 + # Leave a little room for platform quirks / formatting. For tiny + # test adapters keep the limit usable instead of clamping to 500+. + _PROGRESS_TEXT_LIMIT = max( + 1, + _raw_progress_limit - (64 if _raw_progress_limit > 128 else 0), + ) + + def _progress_text(lines: list) -> str: + return "\n".join(str(line) for line in lines) + + def _split_progress_groups(lines: list) -> list[list]: + """Partition progress lines into platform-sized editable bubbles.""" + groups: list[list] = [] + current: list = [] + for line in lines: + candidate = current + [line] + if current and _progress_len_fn(_progress_text(candidate)) > _PROGRESS_TEXT_LIMIT: + groups.append(current) + current = [line] + else: + current = candidate + if current: + groups.append(current) + return groups + + def _track_progress_result(result) -> None: + if ( + _cleanup_progress + and getattr(result, "success", False) + and getattr(result, "message_id", None) + ): + _cleanup_msg_ids.append(str(result.message_id)) + + async def _send_progress_text(text: str): + result = await adapter.send( + chat_id=source.chat_id, + content=text, + reply_to=_progress_reply_to, + metadata=_progress_metadata, + ) + _track_progress_result(result) + return result + + async def _roll_progress_overflow_if_needed() -> bool: + """Start fresh editable progress bubbles before a bubble exceeds limit. + + Returns True when it delivered/split the current buffer and the + caller should skip the normal send/edit path for this tick. + """ + nonlocal progress_msg_id, progress_lines, can_edit + if not progress_lines or not can_edit: + return False + groups = _split_progress_groups(progress_lines) + if len(groups) <= 1: + return False + + first_text = _progress_text(groups[0]) + if progress_msg_id is not None: + result = await adapter.edit_message( + chat_id=source.chat_id, + message_id=progress_msg_id, + content=first_text, + ) + if not result.success: + can_edit = False + # Fall back to the existing non-edit behavior below. + return False + else: + result = await _send_progress_text(first_text) + if result.success and result.message_id: + progress_msg_id = result.message_id + + for group in groups[1:]: + result = await _send_progress_text(_progress_text(group)) + if result.success and result.message_id: + progress_msg_id = result.message_id + + # The newest continuation is now the only mutable bubble. Keep + # just its lines so subsequent edits update it instead of + # replaying the full historical transcript into new messages. + progress_lines = groups[-1] + return True + while True: try: if not _run_still_current(): @@ -15334,6 +15429,13 @@ async def send_progress_messages(): msg = raw progress_lines.append(msg) + if await _roll_progress_overflow_if_needed(): + _last_edit_ts = time.monotonic() + await asyncio.sleep(0.3) + if _run_still_current(): + await adapter.send_typing(source.chat_id, metadata=_progress_metadata) + continue + # Throttle edits: batch rapid tool updates into fewer # API calls to avoid hitting Telegram flood control. # (grammY auto-retry pattern: proactively rate-limit @@ -15422,12 +15524,14 @@ async def send_progress_messages(): _, base_msg, count = raw if progress_lines: progress_lines[-1] = f"{base_msg} (×{count + 1})" + await _roll_progress_overflow_if_needed() elif isinstance(raw, tuple) and len(raw) >= 1 and raw[0] == "__reset__": # Content-bubble marker during drain: close off # the current progress bubble and start a fresh # one for any tool lines that arrived after. + await _roll_progress_overflow_if_needed() if can_edit and progress_lines and progress_msg_id: - _pending_text = "\n".join(progress_lines) + _pending_text = _progress_text(progress_lines) try: await adapter.edit_message( chat_id=source.chat_id, @@ -15442,11 +15546,14 @@ async def send_progress_messages(): repeat_count[0] = 0 else: progress_lines.append(raw) + await _roll_progress_overflow_if_needed() except Exception: break # Final edit with all remaining tools (only if editing works) if can_edit and progress_lines and progress_msg_id: - full_text = "\n".join(progress_lines) + await _roll_progress_overflow_if_needed() + if can_edit and progress_lines and progress_msg_id: + full_text = _progress_text(progress_lines) try: await adapter.edit_message( chat_id=source.chat_id, -
tests/gateway/test_run_progress_topics.py@@ -58,6 +58,47 @@ async def get_chat_info(self, chat_id: str): return {"id": chat_id} +class SmallLimitProgressAdapter(ProgressCaptureAdapter): + """Adapter with a tiny platform limit to exercise progress rollover.""" + + MAX_MESSAGE_LENGTH = 180 + + def __init__(self, platform=Platform.TELEGRAM): + super().__init__(platform=platform) + self._next_id = 0 + self.oversized_edits = [] + self.oversized_sends = [] + + def _mint_id(self): + self._next_id += 1 + return f"progress-{self._next_id}" + + async def send(self, chat_id, content, reply_to=None, metadata=None) -> SendResult: + if len(content) > self.MAX_MESSAGE_LENGTH: + self.oversized_sends.append(content) + self.sent.append( + { + "chat_id": chat_id, + "content": content, + "reply_to": reply_to, + "metadata": metadata, + } + ) + return SendResult(success=True, message_id=self._mint_id()) + + async def edit_message(self, chat_id, message_id, content) -> SendResult: + if len(content) > self.MAX_MESSAGE_LENGTH: + self.oversized_edits.append(content) + self.edits.append( + { + "chat_id": chat_id, + "message_id": message_id, + "content": content, + } + ) + return SendResult(success=True, message_id=message_id) + + class NonEditingProgressCaptureAdapter(ProgressCaptureAdapter): SUPPORTS_MESSAGE_EDITING = False @@ -123,6 +164,31 @@ def run_conversation(self, message, conversation_history=None, task_id=None): } +class ManyProgressLinesAgent: + """Emits enough tool-progress lines to exceed a single platform bubble.""" + + def __init__(self, **kwargs): + self.tool_progress_callback = kwargs.get("tool_progress_callback") + self.tools = [] + + def run_conversation(self, message, conversation_history=None, task_id=None): + cb = self.tool_progress_callback + assert cb is not None + cb("tool.started", "terminal", "first-short", {}) + # Let the progress task create the first editable bubble, then enqueue + # the rest quickly. The cancellation drain must roll them into fresh + # editable bubbles instead of trying to edit the first one past limit. + time.sleep(0.35) + for idx in range(1, 8): + cb("tool.started", "terminal", f"overflow-line-{idx}-" + "x" * 45, {}) + time.sleep(0.1) + return { + "final_response": "done", + "messages": [], + "api_calls": 1, + } + + class DelayedInterimAgent: def __init__(self, **kwargs): self.interim_assistant_callback = kwargs.get("interim_assistant_callback") @@ -617,6 +683,39 @@ async def _run_with_agent( return adapter, result [redacted:email] +async def test_run_agent_rolls_progress_bubble_before_platform_limit(monkeypatch, tmp_path): + """Tool progress should start a second editable bubble before Telegram's limit. + + Regression: once the first progress bubble grew past the platform limit, + the gateway kept trying to edit that same oversized full transcript. The + Telegram adapter then split-and-sent a fresh continuation on every update, + causing a noisy trail of one-line messages instead of a new editable bubble. + """ + adapter, result = await _run_with_agent( + monkeypatch, + tmp_path, + ManyProgressLinesAgent, + session_id="sess-progress-overflow-rollover", + config_data={ + "display": { + "tool_progress": "all", + "interim_assistant_messages": False, + "tool_preview_length": 60, + } + }, + adapter_cls=SmallLimitProgressAdapter, + ) + + assert result["final_response"] == "done" + assert isinstance(adapter, SmallLimitProgressAdapter) + assert len(adapter.sent) >= 2, "expected a fresh progress bubble after the first filled" + assert adapter.oversized_sends == [] + assert adapter.oversized_edits == [] + all_bubbles = [call["content"] for call in adapter.sent + adapter.edits] + assert all(len(text) <= adapter.MAX_MESSAGE_LENGTH for text in all_bubbles) + + @pytest.mark.asyncio async def test_run_agent_surfaces_real_interim_commentary(monkeypatch, tmp_path): adapter, result = await _run_with_agent(
-
2 PRs closed without merging
From the close comments: merged via another change 1 · superseded by another PR 1
Closed issues · 1
- Closed#26207Telegram tool progress should roll over before message limit
How this cluster was grouped
-
Link evidence (
structural links throughout): every member is connected by structural evidence: shared changed files, or a closing/superseded reference. -
Pairwise link density
0.40: the share of item pairs that carry a link, counting a structural link in full and a relevance-only link at 0.36.1.00means every pair is linked.
Pick up a Lab task
Check whether a fix covers the open issues here, with your own agent. Bring back evidence for maintainers.
Open the Lab