{"repo":"NousResearch/hermes-agent","public_id":"59614","display_label":"Telegram start_polling() hang and timeout/watchdog proposals","summary":"The members appear to center on one theme: Telegram polling reconnect reportedly blocking indefinitely in start_polling(), with one bug report and three changes that add a timeout or watchdog around the call. The titles do not indicate whether any of these changes have been merged or whether the problem has been fixed or closed.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":60137,"merged_at":"2026-07-07T10:20:41+00:00"},"resolution":{"state":"resolved","by":"pr-60137"},"freshness":{"state":"verified","last_verified_at":"2026-07-07T10:20:41+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":59614,"pr_number":60137,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-07-07T10:20:41+00:00"}],"members":[{"kind":"issue","number":59614,"title":"[Bug]: Telegram polling reconnect hangs indefinitely when start_polling() blocks on exhausted connection pool — heartbeat and reconnect watcher both deadlocked","url":null,"state":"closed"},{"kind":"pr","number":59618,"title":"fix(telegram): add timeout to start_polling() in network error handler","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4aaaa206aa7d08b1cf72acb84802a2f7d2fd95c0","number":0},{"kind":"commit","value":"aaeba213d90c7234772280a335e6324659c004f9","number":0},{"kind":"pr","value":"#60137","number":60137}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":59721,"title":"fix(gateway): add asyncio.wait_for to Telegram start_polling() to prevent hang (#59614)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"aaeba213d90c7234772280a335e6324659c004f9","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":60137,"title":"fix(telegram): bound start_polling() with a watchdog timeout at all three call sites (#59614)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/platforms/telegram/adapter.py","tests/gateway/test_telegram_start_polling_timeout.py"],"fix_hunks":[{"file":"plugins/platforms/telegram/adapter.py","hunks":["@@ -416,6 +416,12 @@ def _rich_normalize_linebreaks(text: str) -> str:\n # reconnect/teardown ladder. This is an internal safety bound (not a user knob),\n # applied identically at every stop() site so no path can hang on a dead socket.\n _UPDATER_STOP_TIMEOUT = 15.0\n+# start_polling() can also hang when the connection pool is in a degraded state\n+# after _drain_polling_connections(), particularly when both primary and fallback\n+# Telegram endpoints are unreachable. Bounding start_polling() prevents the\n+# reconnect ladder from stalling indefinitely and allows the heartbeat loop to\n+# trigger its own recovery path. Refs: NousResearch/hermes-agent#59614\n+_UPDATER_START_TIMEOUT = 30.0\n \n \n class TelegramAdapter(BasePlatformAdapter):","@@ -1973,10 +1979,19 @@ async def _start_polling_resilient(self, *, drop_pending_updates: bool, error_ca\n         if not (self._app and self._app.updater):\n             raise RuntimeError(\"Telegram application/updater not initialized\")\n         try:\n-            await self._app.updater.start_polling(\n-                allowed_updates=Update.ALL_TYPES,\n-                drop_pending_updates=drop_pending_updates,\n-                error_callback=error_callback,\n+            # Same watchdog bound as the reconnect ladders: a wedged httpx\n+            # connection pool can hang start_polling() forever at bootstrap\n+            # too (#59614). A propagating TimeoutError is a builtins\n+            # TimeoutError (OSError subclass), so the except below classifies\n+            # it via _looks_like_network_error and schedules background\n+            # recovery instead of blocking connect() indefinitely.\n+            await asyncio.wait_for(\n+                self._app.updater.start_polling(\n+                    allowed_updates=Update.ALL_TYPES,\n+                    drop_pending_updates=drop_pending_updates,\n+                    error_callback=error_callback,\n+                ),\n+                timeout=_UPDATER_START_TIMEOUT,\n             )\n             return True\n         except Exception as err:","@@ -2072,11 +2087,26 @@ async def _handle_polling_network_error(self, error: Exception) -> None:\n         try:\n             if not app:\n                 raise RuntimeError(\"Telegram application was torn down during reconnect\")\n-            await app.updater.start_polling(\n-                allowed_updates=Update.ALL_TYPES,\n-                drop_pending_updates=False,\n-                error_callback=self._polling_error_callback_ref,\n-            )\n+            # Guard start_polling() with a timeout: when the connection pool is\n+            # in a degraded state (e.g., after _drain_polling_connections()), the\n+            # httpx client may hold a stale socket that neither connects nor times\n+            # out within PTB's internal flow. Bounding start_polling() prevents\n+            # the reconnect ladder from stalling indefinitely and allows the\n+            # heartbeat loop to trigger its own recovery path.\n+            # Refs: NousResearch/hermes-agent#59614\n+            try:\n+                await asyncio.wait_for(\n+                    app.updater.start_polling(\n+                        allowed_updates=Update.ALL_TYPES,\n+                        drop_pending_updates=False,\n+                        error_callback=self._polling_error_callback_ref,\n+                    ),\n+                    timeout=_UPDATER_START_TIMEOUT,\n+                )\n+            except asyncio.TimeoutError:\n+                raise RuntimeError(\n+                    \"start_polling() timed out — connection pool may be wedged\"\n+                )\n             logger.info(\n                 \"[%s] Telegram polling resumed after network error (attempt %d)\",\n                 self.name, attempt,","@@ -2467,11 +2497,24 @@ async def _handle_polling_conflict(self, error: Exception) -> None:\n             try:\n                 if not app:\n                     raise RuntimeError(\"Telegram application was torn down during conflict reconnect\")\n-                await app.updater.start_polling(\n-                    allowed_updates=Update.ALL_TYPES,\n-                    drop_pending_updates=False,\n-                    error_callback=self._polling_error_callback_ref,\n-                )\n+                # Same watchdog bound as the network-error ladder: an\n+                # exhausted pool hangs start_polling() on the conflict path\n+                # identically (#59614). Timeout converts to RuntimeError so\n+                # the except below logs a readable message and schedules the\n+                # next conflict attempt instead of wedging attempt N forever.\n+                try:\n+                    await asyncio.wait_for(\n+                        app.updater.start_polling(\n+                            allowed_updates=Update.ALL_TYPES,\n+                            drop_pending_updates=False,\n+                            error_callback=self._polling_error_callback_ref,\n+                        ),\n+                        timeout=_UPDATER_START_TIMEOUT,\n+                    )\n+                except asyncio.TimeoutError:\n+                    raise RuntimeError(\n+                        \"start_polling() timed out — connection pool may be wedged\"\n+                    )\n                 logger.info(\n                     \"[%s] Telegram polling resumed after conflict retry %d/%d\",\n                     self.name, self._polling_conflict_count, MAX_CONFLICT_RETRIES,"]},{"file":"tests/gateway/test_telegram_start_polling_timeout.py","hunks":["@@ -0,0 +1,157 @@\n+\"\"\"Regression tests for #59614: start_polling() must be time-bounded.\n+\n+When both the primary Telegram API server and all fallback IPs are unreachable,\n+``await app.updater.start_polling(...)`` can block forever inside an exhausted\n+httpx connection pool — it neither returns nor raises. Unbounded, that wedges:\n+\n+1. the network-error reconnect ladder (stuck inside attempt 1, never advances),\n+2. the heartbeat loop (sees the recovery task as alive-but-wedged and skips),\n+3. the fatal-error escalation (never reached).\n+\n+The fix wraps every ``start_polling()`` await in ``asyncio.wait_for`` with\n+``_UPDATER_START_TIMEOUT`` so a hung call raises and feeds the existing retry\n+ladder. These tests patch the timeout down to keep the suite fast.\n+\"\"\"\n+import asyncio\n+import sys\n+from unittest.mock import AsyncMock, MagicMock, patch\n+\n+import pytest\n+\n+\n+def _ensure_telegram_mock():\n+    if \"telegram\" in sys.modules and hasattr(sys.modules[\"telegram\"], \"__file__\"):\n+        return\n+    telegram_mod = MagicMock()\n+    telegram_mod.ext.ContextTypes.DEFAULT_TYPE = type(None)\n+    telegram_mod.constants.ParseMode.MARKDOWN_V2 = \"MarkdownV2\"\n+    telegram_mod.constants.ChatType.GROUP = \"group\"\n+    telegram_mod.constants.ChatType.SUPERGROUP = \"supergroup\"\n+    telegram_mod.constants.ChatType.CHANNEL = \"channel\"\n+    telegram_mod.constants.ChatType.PRIVATE = \"private\"\n+    telegram_mod.error.NetworkError = type(\"NetworkError\", (OSError,), {})\n+    telegram_mod.error.TimedOut = type(\"TimedOut\", (OSError,), {})\n+    for name in (\"telegram\", \"telegram.ext\", \"telegram.constants\", \"telegram.request\"):\n+        sys.modules.setdefault(name, telegram_mod)\n+    sys.modules.setdefault(\"telegram.error\", telegram_mod.error)\n+\n+\n+_ensure_telegram_mock()\n+\n+from plugins.platforms.telegram import adapter as tg_adapter  # noqa: E402\n+from plugins.platforms.telegram.adapter import TelegramAdapter  # noqa: E402\n+\n+\n+async def _hang_forever(**kwargs):\n+    await asyncio.sleep(1000)\n+\n+\n+def _bare_adapter():\n+    a = TelegramAdapter.__new__(TelegramAdapter)\n+    # `name` / `has_fatal_error` are read-only base-class properties; set the\n+    # backing fields they derive from instead.\n+    from gateway.config import Platform\n+\n+    a.platform = Platform.TELEGRAM\n+    a._fatal_error_code = None\n+    a._fatal_error_message = None\n+    a._fatal_error_retryable = True\n+    a._polling_network_error_count = 0\n+    a._polling_conflict_count = 0\n+    a._polling_error_callback_ref = None\n+    a._background_tasks = set()\n+    a._send_path_degraded = False\n+    return a\n+\n+\n[redacted:email]\n+async def test_network_ladder_start_polling_hang_does_not_wedge(monkeypatch):\n+    \"\"\"A hung start_polling() in _handle_polling_network_error must time out\n+    and advance the ladder instead of blocking forever (#59614 core repro).\"\"\"\n+    monkeypatch.setattr(tg_adapter, \"_UPDATER_START_TIMEOUT\", 0.2)\n+    a = _bare_adapter()\n+    a._polling_network_error_count = 0  # attempt 1 → 5s backoff before start_polling\n+\n+    app = MagicMock()\n+    app.updater = AsyncMock()\n+    app.updater.start_polling = _hang_forever\n+    app.updater.running = False\n+    a._app = app\n+\n+    with patch.object(a, \"_drain_polling_connections\", new=AsyncMock()), \\\n+         patch.object(\n+             tg_adapter.asyncio, \"ensure_future\",\n+             side_effect=lambda coro: (coro.close(), asyncio.get_event_loop().create_future())[1],\n+         ):\n+        # Unbounded, this await hangs past the 30s wait_for and fails the\n+        # test; bounded, the handler waits its 5s backoff, times out the hung\n+        # start_polling() in 0.2s, schedules the chained retry (captured by\n+        # the ensure_future patch), and returns.\n+        await asyncio.wait_for(\n+            a._handle_polling_network_error(Exception(\"net down\")), timeout=30\n+        )\n+\n+\n[redacted:email]\n+async def test_bootstrap_start_polling_hang_schedules_recovery(monkeypatch):\n+    \"\"\"_start_polling_resilient: a hung bootstrap start_polling() must raise\n+    TimeoutError (an OSError → classified as network error) and schedule\n+    background recovery instead of blocking connect() forever.\"\"\"\n+    monkeypatch.setattr(tg_adapter, \"_UPDATER_START_TIMEOUT\", 0.2)\n+    a = _bare_adapter()\n+\n+    app = MagicMock()\n+    app.updater = AsyncMock()\n+    app.updater.start_polling = _hang_forever\n+    a._app = app\n+\n+    scheduled = []\n+    monkeypatch.setattr(\n+        a, \"_schedule_polling_recovery\",\n+        lambda err, reason: scheduled.append((err, reason)),\n+        raising=False,\n+    )\n+\n+    ok = await asyncio.wait_for(\n+        a._start_polling_resilient(drop_pending_updates=False, error_callback=None),\n+        timeout=10,\n+    )\n+    assert ok is False\n+    assert len(scheduled) == 1\n+    assert isinstance(scheduled[0][0], (TimeoutError, asyncio.TimeoutError))\n+\n+\n[redacted:email]\n+async def test_start_polling_success_path_unaffected(monkeypatch):\n+    \"\"\"Sanity: a fast start_polling() still returns True through the wrapper.\"\"\"\n+    monkeypatch.setattr(tg_adapter, \"_UPDATER_START_TIMEOUT\", 5.0)\n+    a = _bare_adapter()\n+\n+    app = MagicMock()\n+    app.updater = AsyncMock()\n+    app.updater.start_polling = AsyncMock(return_value=None)\n+    a._app = app\n+\n+    ok = await a._start_polling_resilient(drop_pending_updates=False, error_callback=None)\n+    assert ok is True\n+    app.updater.start_polling.assert_awaited_once()\n+\n+\n+def test_every_start_polling_call_site_is_time_bounded():\n+    \"\"\"Bug-class contract: every `updater.start_polling(` await in the adapter\n+    must be wrapped in asyncio.wait_for. A new unbounded call site reintroduces\n+    the #59614 wedge.\"\"\"\n+    import inspect\n+    import re\n+\n+    src = inspect.getsource(tg_adapter)\n+    # Find each start_polling( call and check an enclosing wait_for within the\n+    # preceding 6 lines (the wrapper always sits directly above).\n+    lines = src.splitlines()\n+    unbounded = []\n+    for i, line in enumerate(lines):\n+        if re.search(r\"updater\\.start_polling\\(\", line) and \"def \" not in line:\n+            window = \"\\n\".join(lines[max(0, i - 6):i + 1])\n+            if \"wait_for\" not in window:\n+                unbounded.append((i + 1, line.strip()))\n+    assert not unbounded, f\"unbounded start_polling() call sites: {unbounded}\"\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}