{"repo":"NousResearch/hermes-agent","public_id":"48495","display_label":"Telegram polling silent-failure detection and watchdogs","summary":"These issues and PRs address a scenario where the Telegram gateway stops receiving messages due to TCP CLOSE-WAIT or stalled long-poll sockets without crashing. The work centers on adding heartbeat loops, polling liveness watchdogs, and related reliability hardening to detect and respond to this silent-offline state.","subsystem":"gateway","counts":{"members":9,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":7},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":52744},"resolution":{"state":"resolved","by":"pr-52744"},"freshness":{"state":"verified","last_verified_at":"2026-06-26T01:50:12+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.2222222222222222},"fix_edges":[{"issue_number":48495,"pr_number":52744,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":48495,"title":"Telegram polling gateway silently stops receiving messages after TCP CLOSE-WAIT on long-poll socket (ep_poll on dead connection)","url":null,"state":"closed"},{"kind":"pr","number":21548,"title":"fix(gateway): detect Telegram polling-pool wedges via correct pool","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#48496","number":48496},{"kind":"pr","value":"#48495","number":48495}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":45223,"title":"fix(telegram): periodic polling watchdog to detect silent poller death","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#48496","number":48496},{"kind":"pr","value":"#48495","number":48495}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":47366,"title":"fix(gateway): polling-liveness + event-loop watchdogs for Telegram silent-offline","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#48496","number":48496},{"kind":"pr","value":"#48495","number":48495}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":48496,"title":"fix(telegram): add _polling_heartbeat_loop to detect CLOSE-WAIT sockets","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#48496","number":48496},{"kind":"pr","value":"#48495","number":48495}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":49681,"title":"fix(hermes): detect Telegram transport-dead PID-alive gateway state","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#48496","number":48496},{"kind":"pr","value":"#48495","number":48495}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":51738,"title":"fix(telegram): persistent heartbeat loop to detect CLOSE-WAIT polling sockets (#48495)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#52744","number":52744}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":51876,"title":"fix(gateway): reliability hardening — stall watchdog, Telegram poll heartbeat, supervised tasks, launchd respawn throttle","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52744","number":52744},{"kind":"pr","value":"#48496","number":48496},{"kind":"pr","value":"#48495","number":48495}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":52744,"title":"fix(telegram): persistent heartbeat loop to detect CLOSE-WAIT polling sockets (#48495)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/platforms/telegram/adapter.py","scripts/release.py","tests/gateway/test_telegram_conflict.py","tests/gateway/test_telegram_network_reconnect.py"],"fix_hunks":[{"file":"plugins/platforms/telegram/adapter.py","hunks":["@@ -537,6 +537,7 @@ def __init__(self, config: PlatformConfig):\n         self._polling_conflict_count: int = 0\n         self._polling_network_error_count: int = 0\n         self._polling_error_callback_ref = None\n+        self._polling_heartbeat_task: Optional[asyncio.Task] = None\n         # After sustained reconnect storms the PTB httpx pool can return\n         # SendResult(success=True) for sends that never actually transmit.\n         # _handle_polling_network_error sets this; _verify_polling_after_reconnect","@@ -1714,6 +1715,63 @@ async def _handle_polling_network_error(self, error: Exception) -> None:\n                 self._background_tasks.add(task)\n                 task.add_done_callback(self._background_tasks.discard)\n \n+    async def _polling_heartbeat_loop(self) -> None:\n+        \"\"\"Detect dead Telegram TCP sockets (CLOSE-WAIT) by periodic probing.\n+\n+        PTB's long-poll task blocks on epoll waiting for Telegram to push an\n+        update.  When the underlying TCP connection enters CLOSE-WAIT (the remote\n+        sent a FIN but the httpx pool has not yet noticed), epoll still reports\n+        the socket as readable and no exception is raised — so PTB's\n+        ``error_callback`` never fires and the gateway silently stops receiving\n+        messages.\n+\n+        This loop probes ``get_me()`` every ``HEARTBEAT_INTERVAL`` seconds on the\n+        *general* request path (not the getUpdates pool), so a healthy long-poll\n+        waiting for the 30-second Telegram window is never interrupted.  On any\n+        connect-level failure the loop hands off to\n+        ``_handle_polling_network_error`` — the same path triggered by PTB's own\n+        ``error_callback`` — which drains the dead pool and restarts polling.\n+\n+        Unlike ``_verify_polling_after_reconnect`` (a one-shot probe scheduled\n+        only after an explicit reconnect), this loop runs for the full lifetime\n+        of the polling connection, so it catches a socket that wedges during\n+        steady-state operation without any prior error event.\n+        \"\"\"\n+        HEARTBEAT_INTERVAL = 90   # seconds between probes\n+        PROBE_TIMEOUT = 15        # seconds before declaring the path dead\n+\n+        while True:\n+            try:\n+                await asyncio.sleep(HEARTBEAT_INTERVAL)\n+                if self.has_fatal_error:\n+                    return\n+                bot = self._app.bot if self._app else None\n+                if bot is None:\n+                    continue\n+                # A real PTB Bot always exposes get_me(); if it's absent the\n+                # app isn't a live polling client (e.g. torn down or a test\n+                # double), so there is nothing to probe — exit rather than spin.\n+                if not callable(getattr(bot, \"get_me\", None)):\n+                    return\n+                await asyncio.wait_for(bot.get_me(), PROBE_TIMEOUT)\n+            except asyncio.CancelledError:\n+                return\n+            except (asyncio.TimeoutError, OSError) as probe_err:\n+                logger.warning(\n+                    \"[%s] Polling heartbeat probe failed (%s); triggering reconnect\",\n+                    self.name, probe_err,\n+                )\n+                if self._polling_error_task and not self._polling_error_task.done():\n+                    continue   # reconnect already in progress\n+                loop = asyncio.get_running_loop()\n+                self._polling_error_task = loop.create_task(\n+                    self._handle_polling_network_error(probe_err)\n+                )\n+            except Exception:\n+                # Non-connectivity errors (e.g. TelegramError 401) are not\n+                # CLOSE-WAIT symptoms — let PTB's own handlers surface them.\n+                pass\n+\n     async def _verify_polling_after_reconnect(self) -> None:\n         \"\"\"Heartbeat probe scheduled after a successful reconnect.\n ","@@ -2505,6 +2563,16 @@ def _polling_error_callback(error: Exception) -> None:\n             mode = \"webhook\" if self._webhook_mode else \"polling\"\n             logger.info(\"[%s] Connected to Telegram (%s mode)\", self.name, mode)\n \n+            # Start the persistent heartbeat loop in polling mode. Webhook mode\n+            # receives updates via incoming pushes — there is no long-poll\n+            # socket to wedge in CLOSE-WAIT, so the loop is not needed there.\n+            if not self._webhook_mode:\n+                if self._polling_heartbeat_task and not self._polling_heartbeat_task.done():\n+                    self._polling_heartbeat_task.cancel()\n+                self._polling_heartbeat_task = asyncio.ensure_future(\n+                    self._polling_heartbeat_loop()\n+                )\n+\n             # Surface the gateway as \"Online\" in the bot's short description\n             # (opt-in via extra.status_indicator). Non-fatal.\n             try:","@@ -2563,6 +2631,16 @@ async def _set_status_indicator(self, online: bool) -> None:\n \n     async def disconnect(self) -> None:\n         \"\"\"Stop polling/webhook, cancel pending album flushes, and disconnect.\"\"\"\n+        # Cancel the heartbeat before tearing down the app so the probe task\n+        # cannot fire get_me() into a half-shutdown bot client.\n+        if self._polling_heartbeat_task and not self._polling_heartbeat_task.done():\n+            self._polling_heartbeat_task.cancel()\n+            try:\n+                await self._polling_heartbeat_task\n+            except asyncio.CancelledError:\n+                pass\n+        self._polling_heartbeat_task = None\n+\n         # Mark the bot \"Offline\" in its short description while the bot's HTTP\n         # client is still alive (before app shutdown closes it). Opt-in via\n         # extra.status_indicator. Non-fatal. This is the clean-shutdown path;"]},{"file":"scripts/release.py","hunks":["@@ -45,6 +45,7 @@\n \n # Auto-extracted from noreply emails + manual overrides\n AUTHOR_MAP = {\n+    \"[redacted:email]\": \"agt-user\",  # PR #48496 salvage (telegram CLOSE-WAIT polling heartbeat, #48495)\n     \"[redacted:email]\": \"herbalizer404\",  # PR #49076 + #51835 salvage (auxiliary compression fallback: 403/session-usage payment errors + honor fallback chain when aux provider auth unavailable)\n     \"[redacted:email]\": \"pyxl-dev\",  # PR #52230 salvage (include rate-limit in auxiliary capacity-error fallback gate; #52228)\n     \"[redacted:email]\": \"benbenlijie\",  # PR #47205 salvage (named custom-provider extra_body + Z.AI Coding overload adaptive backoff; #50663)"]},{"file":"tests/gateway/test_telegram_conflict.py","hunks":["@@ -47,6 +47,25 @@ async def _noop():\n     monkeypatch.setattr(\"plugins.platforms.telegram.adapter.HTTPXRequest\", lambda **kwargs: MagicMock())\n \n \n+async def _cancel_heartbeat(adapter):\n+    \"\"\"Cancel the lifetime heartbeat task connect() starts in polling mode.\n+\n+    These tests call the real connect() but never disconnect(), so the\n+    _polling_heartbeat_loop task would otherwise outlive the test. With\n+    asyncio.sleep monkeypatched to instant, leaving it running busy-spins the\n+    event loop and starves the test (CI per-file timeout). disconnect() does\n+    this in production; tests that only connect() must do it themselves.\n+    \"\"\"\n+    task = getattr(adapter, \"_polling_heartbeat_task\", None)\n+    if task and not task.done():\n+        task.cancel()\n+        try:\n+            await task\n+        except (asyncio.CancelledError, Exception):\n+            pass\n+    adapter._polling_heartbeat_task = None\n+\n+\n @pytest.mark.asyncio\n async def test_connect_rejects_same_host_token_lock(monkeypatch):\n     adapter = TelegramAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))","@@ -127,6 +146,11 @@ async def fake_start_polling(**kwargs):\n     assert adapter.has_fatal_error is False, \"First conflict should not be fatal\"\n     assert adapter._polling_conflict_count == 0, \"Count should reset after successful retry\"\n \n+    # connect() now starts a lifetime _polling_heartbeat_loop task. With\n+    # asyncio.sleep mocked to instant above, it must not be left running or it\n+    # busy-spins on the event loop and starves the test. Cancel it explicitly.\n+    await _cancel_heartbeat(adapter)\n+\n \n @pytest.mark.asyncio\n async def test_polling_conflict_becomes_fatal_after_retries(monkeypatch):","@@ -205,6 +229,7 @@ async def failing_start_polling(**kwargs):\n     )\n     assert adapter.has_fatal_error is True\n     fatal_handler.assert_awaited_once()\n+    await _cancel_heartbeat(adapter)\n \n \n @pytest.mark.asyncio","@@ -285,6 +310,7 @@ async def test_connect_clears_webhook_before_polling(monkeypatch):\n \n     assert ok is True\n     bot.delete_webhook.assert_awaited_once_with(drop_pending_updates=False)\n+    await _cancel_heartbeat(adapter)\n \n \n @pytest.mark.asyncio","@@ -398,3 +424,4 @@ def _boom():\n         await adapter._polling_error_task\n     except (asyncio.CancelledError, Exception):\n         pass\n+    await _cancel_heartbeat(adapter)"]},{"file":"tests/gateway/test_telegram_network_reconnect.py","hunks":["@@ -473,3 +473,211 @@ async def test_reconnect_schedules_heartbeat_probe_on_success():\n             await t\n         except (asyncio.CancelledError, Exception):\n             pass\n+\n+\n+# ── Persistent heartbeat loop (_polling_heartbeat_loop) ──────────────────────\n+#\n+# These tests cover the continuous CLOSE-WAIT detection loop that fixes the bug\n+# (#48495) where a dead Telegram TCP socket caused the gateway to stop receiving\n+# messages silently. The _verify_polling_after_reconnect tests above cover the\n+# one-shot post-reconnect probe; these cover the background loop that runs for\n+# the gateway's full lifetime in polling mode.\n+#\n+# Loop structure: while True: sleep(INTERVAL) → fatal/app checks → get_me().\n+# So with cancel raised on the Nth patched sleep, get_me() fires (N-1) times.\n+\n+\n[redacted:email]\n+async def test_heartbeat_loop_exits_cleanly_on_cancel():\n+    \"\"\"The heartbeat loop must exit without raising when cancelled (normal shutdown).\"\"\"\n+    adapter = _make_adapter()\n+\n+    mock_app = MagicMock()\n+    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())\n+    adapter._app = mock_app\n+\n+    sleep_count = 0\n+\n+    async def fast_sleep(seconds):\n+        nonlocal sleep_count\n+        sleep_count += 1\n+        # sleep #1 → get_me, sleep #2 → get_me, sleep #3 → cancel.\n+        if sleep_count >= 3:\n+            raise asyncio.CancelledError()\n+\n+    with patch(\"asyncio.sleep\", side_effect=fast_sleep):\n+        # Should not raise — CancelledError is swallowed internally.\n+        await adapter._polling_heartbeat_loop()\n+\n+    assert mock_app.bot.get_me.await_count == 2\n+\n+\n[redacted:email]\n+async def test_heartbeat_loop_triggers_reconnect_on_timeout():\n+    \"\"\"A TimeoutError from get_me() must schedule a reconnect via _handle_polling_network_error.\"\"\"\n+    adapter = _make_adapter()\n+    adapter._handle_polling_network_error = AsyncMock()\n+\n+    mock_app = MagicMock()\n+    adapter._app = mock_app\n+\n+    sleep_call = 0\n+\n+    async def fast_sleep(seconds):\n+        nonlocal sleep_call\n+        sleep_call += 1\n+        if sleep_call >= 3:\n+            raise asyncio.CancelledError()\n+\n+    async def fast_wait_for(coro, timeout):\n+        if asyncio.iscoroutine(coro):\n+            coro.close()\n+        raise asyncio.TimeoutError()\n+\n+    with patch(\"asyncio.sleep\", side_effect=fast_sleep):\n+        with patch(\"plugins.platforms.telegram.adapter.asyncio.wait_for\", side_effect=fast_wait_for):\n+            await adapter._polling_heartbeat_loop()\n+\n+    # A reconnect task must have been created.\n+    assert adapter._polling_error_task is not None\n+\n+\n[redacted:email]\n+async def test_heartbeat_loop_triggers_reconnect_on_os_error():\n+    \"\"\"An OSError (e.g. connection reset) from get_me() must trigger a reconnect.\"\"\"\n+    adapter = _make_adapter()\n+    adapter._handle_polling_network_error = AsyncMock()\n+\n+    mock_app = MagicMock()\n+    adapter._app = mock_app\n+\n+    sleep_call = 0\n+\n+    async def fast_sleep(seconds):\n+        nonlocal sleep_call\n+        sleep_call += 1\n+        if sleep_call >= 3:\n+            raise asyncio.CancelledError()\n+\n+    async def os_error_wait_for(coro, timeout):\n+        if asyncio.iscoroutine(coro):\n+            coro.close()\n+        raise OSError(\"Connection reset by peer\")\n+\n+    with patch(\"asyncio.sleep\", side_effect=fast_sleep):\n+        with patch(\"plugins.platforms.telegram.adapter.asyncio.wait_for\", side_effect=os_error_wait_for):\n+            await adapter._polling_heartbeat_loop()\n+\n+    assert adapter._polling_error_task is not None\n+\n+\n[redacted:email]\n+async def test_heartbeat_loop_skips_reconnect_if_already_in_progress():\n+    \"\"\"If a reconnect task is already running, the heartbeat must not spawn another.\"\"\"\n+    adapter = _make_adapter()\n+\n+    # Simulate an already-running reconnect task.\n+    existing_task = asyncio.get_event_loop().create_task(asyncio.sleep(3600))\n+    adapter._polling_error_task = existing_task\n+    adapter._handle_polling_network_error = AsyncMock()\n+\n+    mock_app = MagicMock()\n+    adapter._app = mock_app\n+\n+    sleep_call = 0\n+\n+    async def fast_sleep(seconds):\n+        nonlocal sleep_call\n+        sleep_call += 1\n+        if sleep_call >= 3:\n+            raise asyncio.CancelledError()\n+\n+    async def timeout_wait_for(coro, timeout):\n+        if asyncio.iscoroutine(coro):\n+            coro.close()\n+        raise asyncio.TimeoutError()\n+\n+    with patch(\"asyncio.sleep\", side_effect=fast_sleep):\n+        with patch(\"plugins.platforms.telegram.adapter.asyncio.wait_for\", side_effect=timeout_wait_for):\n+            await adapter._polling_heartbeat_loop()\n+\n+    # _handle_polling_network_error must NOT have been called — existing task still running.\n+    adapter._handle_polling_network_error.assert_not_awaited()\n+\n+    existing_task.cancel()\n+    try:\n+        await existing_task\n+    except (asyncio.CancelledError, Exception):\n+        pass\n+\n+\n[redacted:email]\n+async def test_heartbeat_loop_ignores_non_connectivity_errors():\n+    \"\"\"Errors that are not connectivity failures (e.g. TelegramError) must be swallowed.\"\"\"\n+    adapter = _make_adapter()\n+    adapter._handle_polling_network_error = AsyncMock()\n+\n+    mock_app = MagicMock()\n+    adapter._app = mock_app\n+\n+    sleep_call = 0\n+\n+    async def fast_sleep(seconds):\n+        nonlocal sleep_call\n+        sleep_call += 1\n+        if sleep_call >= 3:\n+            raise asyncio.CancelledError()\n+\n+    async def telegram_error_wait_for(coro, timeout):\n+        if asyncio.iscoroutine(coro):\n+            coro.close()\n+        raise RuntimeError(\"TelegramError: Unauthorized\")  # non-OSError, non-TimeoutError\n+\n+    with patch(\"asyncio.sleep\", side_effect=fast_sleep):\n+        with patch(\"plugins.platforms.telegram.adapter.asyncio.wait_for\", side_effect=telegram_error_wait_for):\n+            await adapter._polling_heartbeat_loop()\n+\n+    # No reconnect should have been triggered for a non-connectivity error.\n+    adapter._handle_polling_network_error.assert_not_awaited()\n+\n+\n[redacted:email]\n+async def test_heartbeat_loop_exits_on_fatal_error():\n+    \"\"\"A fatal error short-circuits the loop before probing get_me().\"\"\"\n+    adapter = _make_adapter()\n+    adapter._set_fatal_error(\"telegram_network_error\", \"boom\", retryable=True)\n+\n+    mock_app = MagicMock()\n+    mock_app.bot.get_me = AsyncMock(return_value=MagicMock())\n+    adapter._app = mock_app\n+\n+    async def fast_sleep(seconds):\n+        return None\n+\n+    with patch(\"asyncio.sleep\", side_effect=fast_sleep):\n+        await adapter._polling_heartbeat_loop()\n+\n+    # Fatal error returns before the get_me() probe.\n+    mock_app.bot.get_me.assert_not_awaited()\n+\n+\n[redacted:email]\n+async def test_disconnect_cancels_heartbeat_task():\n+    \"\"\"disconnect() must cancel the heartbeat task before shutting down the app.\"\"\"\n+    adapter = _make_adapter()\n+\n+    # Simulate a running heartbeat.\n+    heartbeat_task = asyncio.get_event_loop().create_task(asyncio.sleep(3600))\n+    adapter._polling_heartbeat_task = heartbeat_task\n+\n+    mock_app = MagicMock()\n+    mock_app.updater = MagicMock()\n+    mock_app.updater.running = False\n+    mock_app.running = False\n+    mock_app.shutdown = AsyncMock()\n+    adapter._app = mock_app\n+\n+    await adapter.disconnect()\n+\n+    assert heartbeat_task.cancelled(), \"Heartbeat task must be cancelled by disconnect()\"\n+    assert adapter._polling_heartbeat_task is None\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}