{"repo":"NousResearch/hermes-agent","public_id":"55769","display_label":"Telegram polling recovery and bg-review thread isolation","summary":"These items address a bug where failures in a background review thread silently kill the Telegram polling loop, and the fixes involve recovering the polling updater and scoping stdout/stderr silencing to the worker thread.","subsystem":null,"counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":55921},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-30T22:58:06+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.3933333333333333},"fix_edges":[{"issue_number":55769,"pr_number":55921,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":55769,"title":"[Bug] Gateway Telegram polling silently dies after bg-review thread hits LLM connection errors + memory tool failures","url":null,"state":"closed"},{"kind":"issue","number":55925,"title":"Investigate: why does a failing bg-review thread kill the Telegram polling coroutine? (#55769 trigger)","url":null,"state":"closed"},{"kind":"pr","number":55789,"title":"fix(telegram): recover when polling updater stops while process stays alive (#55769)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#55921","number":55921}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":55921,"title":"fix(telegram): recover when polling updater stops while process stays alive (#55769)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/platforms/telegram/adapter.py","tests/gateway/test_telegram_pending_update_probe.py"],"fix_hunks":[{"file":"plugins/platforms/telegram/adapter.py","hunks":["@@ -423,6 +423,13 @@ def __init__(self, config: PlatformConfig):\n         # also probes get_webhook_info().pending_update_count and escalates to\n         # recovery after two consecutive stuck probes (#42909).\n         self._polling_pending_stuck_count: int = 0\n+        # Consecutive heartbeat probes that found the updater stopped entirely\n+        # (running=False) while we are in polling mode with no reconnect in\n+        # flight. Distinct from the wedged-but-running case above: the long-poll\n+        # task is simply gone, so neither the connectivity probe nor PTB's\n+        # error_callback ever fires and the gateway silently stops receiving\n+        # messages with the process still alive (#55769).\n+        self._polling_not_running_count: int = 0\n         # After sustained reconnect storms the PTB httpx pool can return\n         # SendResult(success=True) for sends that never actually transmit.\n         # _handle_polling_network_error sets this; _verify_polling_after_reconnect","@@ -1879,21 +1886,62 @@ async def _probe_pending_updates(self, bot, probe_timeout: float) -> None:\n         not trip a needless recovery. Recovery reuses\n         ``_handle_polling_network_error`` — the same ladder PTB's own\n         ``error_callback`` feeds — so no new restart machinery is introduced.\n+\n+        This also covers the harsher case where the updater has stopped\n+        entirely (``running=False``) with no reconnect in flight: the long-poll\n+        task is gone rather than wedged, so even ``get_webhook_info`` can't\n+        report a queue against a live consumer. We detect the stopped updater\n+        directly and feed the same ladder (#55769).\n         \"\"\"\n-        # Only meaningful in polling mode with a running updater; in webhook\n-        # mode Telegram pushes updates and holds no server-side queue.\n+        # Only meaningful in polling mode; in webhook mode Telegram pushes\n+        # updates and holds no server-side queue.\n         if self._webhook_mode:\n             return\n+        # A reconnect already in flight owns recovery — don't double-trigger,\n+        # and don't misread its brief stop()->start_polling() window (where\n+        # updater.running is transiently False) as a dead updater below.\n+        if self._polling_error_task and not self._polling_error_task.done():\n+            self._polling_not_running_count = 0\n+            return\n         updater = getattr(self._app, \"updater\", None) if self._app else None\n-        if updater is None or not getattr(updater, \"running\", False):\n+        if updater is None:\n             self._polling_pending_stuck_count = 0\n             return\n+        if not getattr(updater, \"running\", False):\n+            # We are in polling mode with no reconnect in flight, yet PTB's\n+            # updater has stopped entirely. This is distinct from the\n+            # wedged-but-running consumer handled below: the long-poll task is\n+            # gone, get_me()/get_webhook_info() on the general request path\n+            # still succeed, so no error_callback or connectivity probe ever\n+            # fires and the gateway silently stops receiving messages while the\n+            # process stays alive (#55769). Escalate through the same reconnect\n+            # ladder as a wedged consumer, debounced over two consecutive probes\n+            # so a just-starting updater never trips it.\n+            self._polling_pending_stuck_count = 0\n+            self._polling_not_running_count += 1\n+            logger.warning(\n+                \"[%s] Telegram polling heartbeat: updater stopped while in \"\n+                \"polling mode (stuck probe %d/2)\",\n+                self.name, self._polling_not_running_count,\n+            )\n+            if self._polling_not_running_count >= 2:\n+                self._polling_not_running_count = 0\n+                logger.warning(\n+                    \"[%s] Telegram updater is not running (long-poll task \"\n+                    \"gone); triggering polling restart\",\n+                    self.name,\n+                )\n+                loop = asyncio.get_running_loop()\n+                self._polling_error_task = loop.create_task(\n+                    self._handle_polling_network_error(\n+                        RuntimeError(\"Telegram updater stopped while in polling mode\")\n+                    )\n+                )\n+            return\n+        self._polling_not_running_count = 0\n         get_webhook_info = getattr(bot, \"get_webhook_info\", None)\n         if not callable(get_webhook_info):\n             return\n-        # A reconnect already in flight owns recovery — don't double-trigger.\n-        if self._polling_error_task and not self._polling_error_task.done():\n-            return\n         try:\n             info = await asyncio.wait_for(get_webhook_info(), probe_timeout)  # type: ignore[arg-type]\n         except (asyncio.TimeoutError, OSError):"]},{"file":"tests/gateway/test_telegram_pending_update_probe.py","hunks":["@@ -6,6 +6,11 @@\n CLOSE-WAIT heartbeat is blind to it. ``_probe_pending_updates`` watches\n ``get_webhook_info().pending_update_count`` and escalates to the existing\n network-error recovery ladder after two consecutive stuck probes.\n+\n+The same probe also covers the harsher case where the updater has stopped\n+entirely (``running=False``) with no reconnect in flight — the long-poll task\n+is gone, so the gateway silently stops receiving messages while the process\n+stays alive (#55769) — and feeds it into the same recovery ladder.\n \"\"\"\n import sys\n from unittest.mock import AsyncMock, MagicMock, patch","@@ -96,14 +101,44 @@ async def test_webhook_mode_is_noop():\n \n \n @pytest.mark.asyncio\n-async def test_no_probe_when_updater_not_running():\n-    \"\"\"If the updater isn't running, recovery is already someone else's job.\"\"\"\n+async def test_single_stopped_updater_probe_does_not_escalate():\n+    \"\"\"One probe finding a stopped updater only increments the counter (#55769).\"\"\"\n     adapter = _make_adapter(pending=9)\n     adapter._app.updater.running = False\n     adapter._polling_pending_stuck_count = 1\n-    await adapter._probe_pending_updates(adapter._app.bot, 5)\n+    with patch.object(adapter, \"_handle_polling_network_error\", new=AsyncMock()) as rec:\n+        await adapter._probe_pending_updates(adapter._app.bot, 5)\n+    # Stopped updater means no live consumer to query for a queue.\n     adapter._app.bot.get_webhook_info.assert_not_called()\n     assert adapter._polling_pending_stuck_count == 0\n+    assert adapter._polling_not_running_count == 1\n+    rec.assert_not_called()\n+\n+\n[redacted:email]\n+async def test_two_stopped_updater_probes_trigger_recovery():\n+    \"\"\"A stopped updater that stays stopped routes into recovery (#55769).\"\"\"\n+    adapter = _make_adapter(pending=9)\n+    adapter._app.updater.running = False\n+    recovery = AsyncMock()\n+    with patch.object(adapter, \"_handle_polling_network_error\", new=recovery):\n+        await adapter._probe_pending_updates(adapter._app.bot, 5)\n+        assert adapter._polling_not_running_count == 1\n+        await adapter._probe_pending_updates(adapter._app.bot, 5)\n+        task = adapter._polling_error_task\n+        assert task is not None\n+        await task\n+    recovery.assert_awaited_once()\n+    assert adapter._polling_not_running_count == 0\n+\n+\n[redacted:email]\n+async def test_running_updater_resets_stopped_counter():\n+    \"\"\"A recovered (running) updater clears any prior stopped-probe count.\"\"\"\n+    adapter = _make_adapter(pending=0)\n+    adapter._polling_not_running_count = 1\n+    await adapter._probe_pending_updates(adapter._app.bot, 5)\n+    assert adapter._polling_not_running_count == 0\n \n \n @pytest.mark.asyncio","@@ -115,3 +150,20 @@ async def test_reconnect_in_flight_skips_probe():\n     adapter._polling_error_task = inflight\n     await adapter._probe_pending_updates(adapter._app.bot, 5)\n     adapter._app.bot.get_webhook_info.assert_not_called()\n+\n+\n[redacted:email]\n+async def test_reconnect_in_flight_skips_stopped_updater_escalation():\n+    \"\"\"A stopped updater during an in-flight reconnect must not re-escalate.\"\"\"\n+    adapter = _make_adapter(pending=9)\n+    adapter._app.updater.running = False\n+    adapter._polling_not_running_count = 1\n+    inflight = MagicMock()\n+    inflight.done.return_value = False\n+    adapter._polling_error_task = inflight\n+    with patch.object(adapter, \"_handle_polling_network_error\", new=AsyncMock()) as rec:\n+        await adapter._probe_pending_updates(adapter._app.bot, 5)\n+    # The in-flight reconnect owns recovery; the stopped-updater counter resets\n+    # so the transient stop()->start_polling() window never trips a re-trigger.\n+    assert adapter._polling_not_running_count == 0\n+    rec.assert_not_called()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}