{"repo":"NousResearch/hermes-agent","public_id":"31066","display_label":"Telegram TimedOut exceptions crashing or disrupting gateway","summary":"These issues and fixes relate to unhandled Telegram TimedOut exceptions propagating up and crashing or disrupting the gateway, including efforts to swallow or handle these errors resiliently and prevent duplicate message delivery on timeout.","subsystem":"gateway","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":31445},"resolution":{"state":"resolved","by":"pr-31445"},"freshness":{"state":"verified","last_verified_at":"2026-05-24T22:03:28+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.43600000000000005},"fix_edges":[{"issue_number":31066,"pr_number":31445,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":31110,"pr_number":31445,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":31066,"title":"Gateway crashes on Telegram Timeout - unhandled `telegram.error.TimedOut` exception","url":null,"state":"closed"},{"kind":"issue","number":31110,"title":"Telegram TimedOut exceptions crash entire gateway (affects all profiles)","url":null,"state":"closed"},{"kind":"pr","number":31445,"title":"fix(gateway): swallow transient Telegram TimedOut at loop level","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_loop_exception_handler.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -139,6 +139,85 @@ def _gateway_platform_value(platform: Any) -> str:\n     return str(getattr(platform, \"value\", platform) or \"\").strip().lower()\n \n \n+def _is_transient_network_error(exc: BaseException) -> bool:\n+    \"\"\"Return True for transient network errors safe to log + swallow.\n+\n+    The crash class targeted by #31066 / #31110: an unhandled Telegram\n+    ``TimedOut`` (or peer ``NetworkError`` / ``httpx`` connection error)\n+    propagating to the event loop and killing the entire gateway\n+    process. These are by definition transient — the next poll cycle or\n+    user action recovers — so they must never crash the process.\n+\n+    Walk the exception cause chain so wrapped errors (e.g. PTB's\n+    ``NetworkError`` wrapping ``httpx.ConnectError``) are still\n+    classified. The chain is bounded to avoid pathological cycles.\n+    \"\"\"\n+    seen: set[int] = set()\n+    cur: Optional[BaseException] = exc\n+    depth = 0\n+    transient_class_names = {\n+        \"TimedOut\",\n+        \"NetworkError\",\n+        \"ReadError\",\n+        \"WriteError\",\n+        \"ConnectError\",\n+        \"ConnectTimeout\",\n+        \"ReadTimeout\",\n+        \"WriteTimeout\",\n+        \"PoolTimeout\",\n+        \"RemoteProtocolError\",\n+        \"ServerDisconnectedError\",\n+        \"ClientConnectorError\",\n+        \"ClientOSError\",\n+    }\n+    while cur is not None and depth < 12:\n+        ident = id(cur)\n+        if ident in seen:\n+            break\n+        seen.add(ident)\n+        depth += 1\n+        name = type(cur).__name__\n+        if name in transient_class_names:\n+            return True\n+        cur = cur.__cause__ or cur.__context__\n+    return False\n+\n+\n+def _gateway_loop_exception_handler(\n+    loop: \"asyncio.AbstractEventLoop\", context: Dict[str, Any]\n+) -> None:\n+    \"\"\"Loop-level safety net for transient network errors.\n+\n+    Installed once during :func:`start_gateway`. Catches the\n+    ``telegram.error.TimedOut`` crash class (issues #31066 / #31110)\n+    and any peer transient network error before it can kill the\n+    gateway process. Logs at WARNING with full traceback so the\n+    originating call site stays diagnosable; non-transient errors\n+    are forwarded to the default loop handler so real bugs still\n+    surface.\n+    \"\"\"\n+    exc = context.get(\"exception\")\n+    if exc is not None and _is_transient_network_error(exc):\n+        message = context.get(\"message\") or \"transient network error\"\n+        task = context.get(\"future\") or context.get(\"task\")\n+        task_name = \"\"\n+        if task is not None:\n+            try:\n+                task_name = task.get_name() if hasattr(task, \"get_name\") else repr(task)\n+            except Exception:\n+                task_name = repr(task)\n+        logger.warning(\n+            \"Gateway swallowed transient network error from %s: %s: %s\",\n+            task_name or \"<unknown task>\",\n+            type(exc).__name__,\n+            exc,\n+            exc_info=(type(exc), exc, exc.__traceback__),\n+        )\n+        return\n+    # Fall back to the default handler for anything we don't recognise.\n+    loop.default_exception_handler(context)\n+\n+\n def _redact_gateway_user_facing_secrets(text: str) -> str:\n     \"\"\"Best-effort secret redaction before text can leave the gateway.\"\"\"\n     redacted = str(text or \"\")","@@ -18140,6 +18219,21 @@ def restart_signal_handler():\n         runner.request_restart(detached=False, via_service=True)\n     \n     loop = asyncio.get_running_loop()\n+\n+    # Install a loop-level exception handler that swallows transient\n+    # network errors from background tasks. Issues #31066 / #31110:\n+    # an unhandled ``telegram.error.TimedOut`` (or peer NetworkError /\n+    # httpx connection error) in any awaited coroutine would propagate\n+    # to the loop and kill the gateway process, taking down every\n+    # profile attached to the same runner. systemd then restarts the\n+    # service after ~5s but the active conversation turn is lost.\n+    #\n+    # The fix is intentionally narrow: only well-known transient\n+    # network errors are swallowed (and logged with full traceback so\n+    # the originating call site is still discoverable). Anything else\n+    # is forwarded to the default handler so real bugs still surface.\n+    loop.set_exception_handler(_gateway_loop_exception_handler)\n+\n     if threading.current_thread() is threading.main_thread():\n         for sig in (signal.SIGINT, signal.SIGTERM):\n             try:"]},{"file":"tests/gateway/test_loop_exception_handler.py","hunks":["@@ -0,0 +1,210 @@\n+\"\"\"Tests for the gateway loop-level transient-network-error safety net.\n+\n+Issues #31066 / #31110: unhandled ``telegram.error.TimedOut`` (or peer\n+``NetworkError`` / ``httpx`` connection error) propagating to the\n+asyncio event loop killed the gateway process, taking down every\n+profile attached to the same runner. The safety net installed in\n+:func:`gateway.run.start_gateway` catches the transient crash class\n+and logs+swallows it; non-transient errors still surface.\n+\n+These tests pin the classifier and the loop handler so the safety net\n+can't silently regress to swallowing every exception.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import asyncio\n+import logging\n+\n+import pytest\n+\n+from gateway.run import (\n+    _gateway_loop_exception_handler,\n+    _is_transient_network_error,\n+)\n+\n+\n+# ----- Fake exception classes that mimic the real wire types ----------\n+# We avoid importing telegram / httpx here so the test runs in environments\n+# without those packages installed (the classifier matches on class name).\n+\n+class TimedOut(Exception):\n+    \"\"\"Stand-in for ``telegram.error.TimedOut``.\"\"\"\n+\n+\n+class NetworkError(Exception):\n+    \"\"\"Stand-in for ``telegram.error.NetworkError``.\"\"\"\n+\n+\n+class ConnectError(Exception):\n+    \"\"\"Stand-in for ``httpx.ConnectError``.\"\"\"\n+\n+\n+class ReadTimeout(Exception):\n+    \"\"\"Stand-in for ``httpx.ReadTimeout``.\"\"\"\n+\n+\n+class PoolTimeout(Exception):\n+    \"\"\"Stand-in for ``httpx.PoolTimeout``.\"\"\"\n+\n+\n+class ClientConnectorError(Exception):\n+    \"\"\"Stand-in for ``aiohttp.ClientConnectorError``.\"\"\"\n+\n+\n+class SomeUnrelatedBug(Exception):\n+    \"\"\"A non-transient error that should NOT be swallowed.\"\"\"\n+\n+\n+# ---------------------------------------------------------------------\n+# Classifier\n+# ---------------------------------------------------------------------\n+\n+\n[redacted:email](\n+    \"exc_cls\",\n+    [\n+        TimedOut,\n+        NetworkError,\n+        ConnectError,\n+        ReadTimeout,\n+        PoolTimeout,\n+        ClientConnectorError,\n+    ],\n+)\n+def test_transient_classifier_matches_known_network_errors(exc_cls):\n+    \"\"\"Every well-known transient network exception class is classified.\"\"\"\n+    assert _is_transient_network_error(exc_cls(\"boom\")) is True\n+\n+\n+def test_transient_classifier_rejects_unrelated_errors():\n+    \"\"\"Real bugs (ValueError, KeyError, custom app errors) are NOT swallowed.\"\"\"\n+    for exc in (ValueError(\"bad\"), KeyError(\"missing\"), SomeUnrelatedBug(\"x\")):\n+        assert _is_transient_network_error(exc) is False\n+\n+\n+def test_transient_classifier_unwraps_cause_chain():\n+    \"\"\"A NetworkError wrapping a ConnectError is still classified.\"\"\"\n+    inner = ConnectError(\"connection refused\")\n+    outer = NetworkError(\"upstream failed\")\n+    outer.__cause__ = inner\n+    assert _is_transient_network_error(outer) is True\n+\n+\n+def test_transient_classifier_unwraps_context_chain():\n+    \"\"\"Implicit ``__context__`` wrapping is also unwrapped.\"\"\"\n+    try:\n+        try:\n+            raise TimedOut(\"upstream timeout\")\n+        except TimedOut:\n+            # Re-raise something else with the original as implicit context\n+            raise SomeUnrelatedBug(\"wrapper\")\n+    except SomeUnrelatedBug as e:\n+        wrapped = e\n+    # The wrapper class name is not transient, but the chained context is.\n+    assert _is_transient_network_error(wrapped) is True\n+\n+\n+def test_transient_classifier_does_not_infinite_loop_on_cyclic_cause():\n+    \"\"\"A pathological self-referential cause chain terminates.\"\"\"\n+    exc = SomeUnrelatedBug(\"loop\")\n+    exc.__cause__ = exc  # cycle\n+    # Must return without hanging.\n+    assert _is_transient_network_error(exc) is False\n+\n+\n+# ---------------------------------------------------------------------\n+# Loop handler\n+# ---------------------------------------------------------------------\n+\n+\n+def test_handler_swallows_transient_error_and_logs_warning(caplog):\n+    \"\"\"Transient errors are logged at WARNING but not re-raised.\"\"\"\n+    loop = asyncio.new_event_loop()\n+    try:\n+        with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+            _gateway_loop_exception_handler(\n+                loop,\n+                {\n+                    \"message\": \"Task exception was never retrieved\",\n+                    \"exception\": TimedOut(\"Timed out\"),\n+                },\n+            )\n+        # Warning emitted, exception class name appears in the log.\n+        assert any(\"TimedOut\" in r.message for r in caplog.records)\n+    finally:\n+        loop.close()\n+\n+\n+def test_handler_delegates_unknown_errors_to_default(monkeypatch):\n+    \"\"\"A non-transient error is forwarded to ``loop.default_exception_handler``.\"\"\"\n+    loop = asyncio.new_event_loop()\n+    try:\n+        forwarded: list[dict] = []\n+\n+        def fake_default(ctx):\n+            forwarded.append(ctx)\n+\n+        monkeypatch.setattr(loop, \"default_exception_handler\", fake_default)\n+\n+        context = {\n+            \"message\": \"Something else broke\",\n+            \"exception\": SomeUnrelatedBug(\"real bug\"),\n+        }\n+        _gateway_loop_exception_handler(loop, context)\n+        assert forwarded == [context]\n+    finally:\n+        loop.close()\n+\n+\n+def test_handler_tolerates_missing_exception_key(monkeypatch):\n+    \"\"\"Contexts without an ``exception`` key fall through to the default handler.\"\"\"\n+    loop = asyncio.new_event_loop()\n+    try:\n+        forwarded: list[dict] = []\n+        monkeypatch.setattr(\n+            loop, \"default_exception_handler\", lambda ctx: forwarded.append(ctx)\n+        )\n+        ctx = {\"message\": \"warning without exception\"}\n+        _gateway_loop_exception_handler(loop, ctx)\n+        assert forwarded == [ctx]\n+    finally:\n+        loop.close()\n+\n+\n+# ---------------------------------------------------------------------\n+# End-to-end: task-level\n+# ---------------------------------------------------------------------\n+\n+\n+def test_unhandled_transient_error_in_task_does_not_propagate_to_loop():\n+    \"\"\"Smoke test the wiring as a loop would actually use it.\n+\n+    Schedules a task that raises TimedOut and is never awaited. With the\n+    handler installed, the loop completes normally and logs a warning\n+    instead of dying. Without the handler, asyncio would emit\n+    ``Task exception was never retrieved`` and (depending on Python's\n+    debug mode) potentially escalate.\n+    \"\"\"\n+\n+    async def raiser():\n+        raise TimedOut(\"upstream timeout\")\n+\n+    async def main():\n+        loop = asyncio.get_running_loop()\n+        loop.set_exception_handler(_gateway_loop_exception_handler)\n+        task = loop.create_task(raiser())\n+        # Give the task a tick to run and raise.\n+        await asyncio.sleep(0)\n+        # Don't await ``task`` — let it become an unhandled-exception task.\n+        del task\n+        import gc\n+\n+        gc.collect()\n+        await asyncio.sleep(0)\n+\n+    # If the safety net works, this returns cleanly. If not, the test\n+    # would still pass (asyncio's default is a warning, not a crash) —\n+    # the real assertion is that no unhandled exception escapes the\n+    # ``run`` boundary.\n+    asyncio.run(main())\n"]}]},{"kind":"pr","number":64037,"title":"fix(telegram): wrap callback_query handler with resilient_handler to swallow TimedOut (#31066)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":64077,"title":"fix(telegram): wrap callback_query handler with resilient_handler to swallow TimedOut (#31066)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"5b52e26d18ca80cb47f335d2acab0c4319d0f5c0","number":0},{"kind":"pr","value":"#31066","number":31066},{"kind":"pr","value":"#31110","number":31110},{"kind":"pr","value":"#31066","number":31066},{"kind":"pr","value":"#31110","number":31110}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}