{"repo":"NousResearch/hermes-agent","public_id":"31599","display_label":"Telegram httpx connection pool fd leak fixes","summary":"These issues and PRs center on file descriptor leaks caused by CLOSE_WAIT and unclaimed sockets accumulating in Telegram adapter httpx connection pools, particularly on proxy paths. Proposed fixes involve bounding pool sizes, applying keepalive limits, and draining or resetting pools on timeout or reconnect.","subsystem":"gateway","counts":{"members":10,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":3,"merged_prs":3,"attempt_prs":4},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":51541},"resolution":{"state":"resolved","by":"pr-51541,pr-54121,pr-61988"},"freshness":{"state":"verified","last_verified_at":"2026-06-28T09:34:17+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.17955555555555555},"fix_edges":[{"issue_number":31599,"pr_number":51541,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":53524,"pr_number":54121,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":31599,"title":"Telegram adapter leaks httpx general-pool connections through HTTP proxy (CLOSED sockets accumulate, fd limit hit after ~2 days)","url":null,"state":"closed"},{"kind":"issue","number":53524,"title":"fix(gateway): Telegram HTTP connection pool exhaustion causes silent reply failure","url":null,"state":"closed"},{"kind":"issue","number":61979,"title":"bug(gateway): force_close_tcp_sockets shutdown-only leaves CLOSED sockets unreclaimed on proxy path, 177+/process","url":null,"state":"closed"},{"kind":"pr","number":37400,"title":"fix: Bound Telegram httpx general-pool and drain after reconnect (#31599)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4b7f3826c","number":0},{"kind":"pr","value":"#31599","number":31599},{"kind":"pr","value":"#31599","number":31599}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":49930,"title":"fix(telegram): apply keepalive limits and TCP keepalive to prevent CLOSE_WAIT fd leak","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4b7f3826c2ced4d1243b64b15785c6781f836639","number":0},{"kind":"commit","value":"74265c8e84c9bb93253c145bb18fd32b2b3946c7","number":0},{"kind":"pr","value":"#51541","number":51541},{"kind":"pr","value":"#31599","number":31599}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":51541,"title":"fix(telegram): wire keepalive limits into general request pool to fix CLOSE_WAIT fd leak (#31599)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/platforms/telegram/adapter.py","tests/gateway/test_telegram_closewait_limits_31599.py"],"fix_hunks":[{"file":"plugins/platforms/telegram/adapter.py","hunks":["@@ -2204,6 +2204,43 @@ def _env_float(name: str, default: float) -> float:\n                 \"write_timeout\": _env_float(\"HERMES_TELEGRAM_HTTP_WRITE_TIMEOUT\", 20.0),\n             }\n \n+            # CLOSE_WAIT fd leak (#31599, same class as #18451): PTB's\n+            # HTTPXRequest builds the underlying httpx.AsyncClient with\n+            # `limits = httpx.Limits(max_connections=connection_pool_size)`\n+            # and *no* keepalive tuning, so httpx's default\n+            # keepalive_expiry=5.0 applies. Behind an HTTP proxy (Cloudflare\n+            # Warp etc.) a peer-initiated FIN can sit in CLOSE_WAIT longer\n+            # than that, leaking fds in the general request pool (_request[1])\n+            # which _drain_polling_connections never resets. Wire the shared\n+            # platform_httpx_limits() helper into the httpx client so idle\n+            # keepalive sockets drain aggressively, while preserving PTB's\n+            # max_connections (= connection_pool_size). httpx_kwargs is spread\n+            # last into PTB's client kwargs, so `limits` here wins.\n+            from gateway.platforms._http_client_limits import platform_httpx_limits\n+\n+            _base_limits = platform_httpx_limits()\n+            if _base_limits is not None:\n+                import httpx as _httpx\n+\n+                _pool_limits = _httpx.Limits(\n+                    max_connections=request_kwargs[\"connection_pool_size\"],\n+                    max_keepalive_connections=_base_limits.max_keepalive_connections,\n+                    keepalive_expiry=_base_limits.keepalive_expiry,\n+                )\n+            else:  # pragma: no cover — httpx always present alongside PTB\n+                _pool_limits = None\n+\n+            def _with_limits(httpx_kwargs: Optional[dict] = None) -> dict:\n+                \"\"\"Merge tuned keepalive limits into httpx client kwargs.\n+\n+                A caller-supplied ``limits`` (none today) is left untouched;\n+                otherwise the CLOSE_WAIT-safe limits are injected.\n+                \"\"\"\n+                kwargs = dict(httpx_kwargs or {})\n+                if _pool_limits is not None and \"limits\" not in kwargs:\n+                    kwargs[\"limits\"] = _pool_limits\n+                return kwargs\n+\n             disable_fallback = (os.getenv(\"HERMES_TELEGRAM_DISABLE_FALLBACK_IPS\", \"\").strip().lower() in {\"1\", \"true\", \"yes\", \"on\"})\n             fallback_ips = self._fallback_ips()\n             if not fallback_ips:","@@ -2226,21 +2263,31 @@ def _env_float(name: str, default: float) -> float:\n                 # polling reconnect + bot API bootstrap/delete_webhook calls.\n                 request = HTTPXRequest(\n                     **request_kwargs,\n-                    httpx_kwargs={\"transport\": TelegramFallbackTransport(fallback_ips)},\n+                    httpx_kwargs=_with_limits(\n+                        {\"transport\": TelegramFallbackTransport(fallback_ips)}\n+                    ),\n                 )\n                 get_updates_request = HTTPXRequest(\n                     **request_kwargs,\n-                    httpx_kwargs={\"transport\": TelegramFallbackTransport(fallback_ips)},\n+                    httpx_kwargs=_with_limits(\n+                        {\"transport\": TelegramFallbackTransport(fallback_ips)}\n+                    ),\n                 )\n             elif proxy_url:\n                 logger.info(\"[%s] Proxy detected; passing explicitly to HTTPXRequest: %s\", self.name, proxy_url)\n-                request = HTTPXRequest(**request_kwargs, proxy=proxy_url)\n-                get_updates_request = HTTPXRequest(**request_kwargs, proxy=proxy_url)\n+                request = HTTPXRequest(\n+                    **request_kwargs, proxy=proxy_url, httpx_kwargs=_with_limits()\n+                )\n+                get_updates_request = HTTPXRequest(\n+                    **request_kwargs, proxy=proxy_url, httpx_kwargs=_with_limits()\n+                )\n             else:\n                 if disable_fallback:\n                     logger.info(\"[%s] Telegram fallback-IP transport disabled via env\", self.name)\n-                request = HTTPXRequest(**request_kwargs)\n-                get_updates_request = HTTPXRequest(**request_kwargs)\n+                request = HTTPXRequest(**request_kwargs, httpx_kwargs=_with_limits())\n+                get_updates_request = HTTPXRequest(\n+                    **request_kwargs, httpx_kwargs=_with_limits()\n+                )\n \n             builder = builder.request(request).get_updates_request(get_updates_request)\n             self._app = builder.build()"]},{"file":"tests/gateway/test_telegram_closewait_limits_31599.py","hunks":["@@ -0,0 +1,177 @@\n+\"\"\"Regression test for #31599 — Telegram general-pool CLOSE_WAIT fd leak.\n+\n+Background\n+----------\n+PTB's ``telegram.request.HTTPXRequest`` builds the underlying\n+``httpx.AsyncClient`` with ``limits = httpx.Limits(max_connections=...)``\n+and *no* keepalive tuning, so httpx's default ``keepalive_expiry=5.0``\n+applies.  Behind an HTTP proxy (Cloudflare Warp etc.) a peer-initiated\n+FIN can sit in ``CLOSE_WAIT`` longer than that, leaking fds in the\n+general request pool (``_request[1]`` — the pool that routes\n+``bot.send_message`` / ``set_my_commands``), which\n+``_drain_polling_connections`` never resets.\n+\n+The fix wires the shared ``gateway.platforms._http_client_limits``\n+``platform_httpx_limits()`` helper into *every* HTTPXRequest the adapter\n+builds — the fallback-transport branch, the proxy branch, and the plain\n+branch — so idle keepalive sockets drain aggressively.\n+\n+Contract asserted here (mutation-survivable)\n+---------------------------------------------\n+Every ``HTTPXRequest`` constructed by ``TelegramAdapter.connect()`` must\n+receive ``httpx_kwargs[\"limits\"]`` that is an ``httpx.Limits`` with a\n+``keepalive_expiry`` strictly below httpx's 5.0 default and a positive,\n+bounded ``max_keepalive_connections``.  Reverting the limits wiring (so\n+HTTPXRequest falls back to PTB's default 5.0s keepalive) fails this test.\n+\"\"\"\n+\n+import asyncio\n+import sys\n+from unittest.mock import MagicMock, patch\n+\n+import httpx\n+import pytest\n+\n+from gateway.config import PlatformConfig\n+\n+\n+def _ensure_telegram_mock():\n+    if \"telegram\" in sys.modules and hasattr(sys.modules[\"telegram\"], \"__file__\"):\n+        return\n+    telegram_mod = MagicMock()\n+    telegram_mod.ext.ContextTypes.DEFAULT_TYPE = type(None)\n+    telegram_mod.constants.ParseMode.MARKDOWN_V2 = \"MarkdownV2\"\n+    telegram_mod.constants.ChatType.GROUP = \"group\"\n+    telegram_mod.constants.ChatType.SUPERGROUP = \"supergroup\"\n+    telegram_mod.constants.ChatType.CHANNEL = \"channel\"\n+    telegram_mod.constants.ChatType.PRIVATE = \"private\"\n+    for name in (\"telegram\", \"telegram.ext\", \"telegram.constants\", \"telegram.request\"):\n+        sys.modules.setdefault(name, telegram_mod)\n+\n+\n+_ensure_telegram_mock()\n+\n+from plugins.platforms.telegram import adapter as tg_adapter  # noqa: E402\n+from plugins.platforms.telegram.adapter import TelegramAdapter  # noqa: E402\n+\n+\n+class _StopConnect(Exception):\n+    \"\"\"Sentinel raised to abort connect() once requests are built.\"\"\"\n+\n+\n+class _RecordingHTTPXRequest:\n+    \"\"\"Stand-in for PTB's HTTPXRequest that records constructor kwargs.\"\"\"\n+\n+    instances: list = []\n+\n+    def __init__(self, *args, **kwargs):\n+        self.args = args\n+        self.kwargs = kwargs\n+        _RecordingHTTPXRequest.instances.append(self)\n+\n+\n+def _make_adapter() -> TelegramAdapter:\n+    return TelegramAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))\n+\n+\n+def _drive_connect(monkeypatch, *, proxy_url):\n+    \"\"\"Run connect() far enough to build the HTTPXRequests, then abort.\n+\n+    Returns the list of recorded _RecordingHTTPXRequest instances.\n+    \"\"\"\n+    _RecordingHTTPXRequest.instances = []\n+\n+    # No DoH auto-discovery → exercise the proxy / plain branches, not fallback.\n+    async def _no_fallback():\n+        return []\n+\n+    monkeypatch.setattr(tg_adapter, \"discover_fallback_ips\", _no_fallback)\n+    monkeypatch.setattr(\n+        tg_adapter, \"resolve_proxy_url\", lambda *a, **k: proxy_url\n+    )\n+    # Replace the real HTTPXRequest with our recorder.\n+    monkeypatch.setattr(tg_adapter, \"HTTPXRequest\", _RecordingHTTPXRequest)\n+\n+    adapter = _make_adapter()\n+    # Skip the cross-process token lock.\n+    monkeypatch.setattr(adapter, \"_acquire_platform_lock\", lambda *a, **k: True)\n+    # Ensure the adapter reports no statically-configured fallback IPs.\n+    monkeypatch.setattr(adapter, \"_fallback_ips\", lambda: [])\n+\n+    # builder.request(...).get_updates_request(...).build() must be harmless;\n+    # make build() raise our sentinel so connect() stops right after the\n+    # HTTPXRequests are constructed (before any real network/init).\n+    fake_built_app = MagicMock()\n+    fake_built_app.initialize = MagicMock(side_effect=_StopConnect)\n+\n+    chainable = MagicMock()\n+    chainable.token.return_value = chainable\n+    chainable.base_url.return_value = chainable\n+    chainable.base_file_url.return_value = chainable\n+    chainable.local_mode.return_value = chainable\n+    chainable.request.return_value = chainable\n+    chainable.get_updates_request.return_value = chainable\n+    chainable.build.side_effect = _StopConnect\n+\n+    builder_root = MagicMock()\n+    builder_root.builder.return_value = chainable\n+    monkeypatch.setattr(tg_adapter, \"Application\", builder_root)\n+\n+    try:\n+        asyncio.run(adapter.connect())\n+    except _StopConnect:\n+        pass\n+    except Exception:\n+        # connect() wraps work in a try; if it swallows the sentinel and\n+        # continues to real init, the recorded instances are still valid.\n+        pass\n+\n+    return list(_RecordingHTTPXRequest.instances)\n+\n+\n+def _assert_keepalive_tight(instances):\n+    assert instances, \"connect() built no HTTPXRequest — test setup is wrong\"\n+    for inst in instances:\n+        limits = inst.kwargs.get(\"httpx_kwargs\", {}).get(\"limits\")\n+        assert isinstance(limits, httpx.Limits), (\n+            \"HTTPXRequest must receive httpx_kwargs['limits'] = httpx.Limits \"\n+            \"wired from platform_httpx_limits() (#31599). Missing → PTB falls \"\n+            \"back to default keepalive_expiry=5.0 and leaks CLOSE_WAIT fds.\"\n+        )\n+        # The whole point: keepalive must be tighter than httpx's 5.0 default.\n+        assert limits.keepalive_expiry is not None\n+        assert limits.keepalive_expiry < 5.0, (\n+            \"keepalive_expiry must be < httpx default 5.0 so idle/CLOSE_WAIT \"\n+            \"sockets drain promptly behind a proxy (#31599).\"\n+        )\n+        assert limits.max_keepalive_connections is not None\n+        assert 1 <= limits.max_keepalive_connections <= 50\n+        # PTB's connection_pool_size (max_connections) must be preserved.\n+        assert limits.max_connections is not None and limits.max_connections > 0\n+\n+\n+def test_proxy_branch_general_pool_has_tight_keepalive(monkeypatch):\n+    \"\"\"The proxy path the #31599 reporter hit must wire tuned limits.\"\"\"\n+    instances = _drive_connect(monkeypatch, proxy_url=\"http://127.0.0.1:9/\")\n+    # Both the general request pool and the get_updates pool are built here.\n+    assert len(instances) >= 2\n+    _assert_keepalive_tight(instances)\n+    # Sanity: the proxy was actually threaded through (we're on the proxy branch).\n+    assert any(inst.kwargs.get(\"proxy\") == \"http://127.0.0.1:9/\" for inst in instances)\n+\n+\n+def test_plain_branch_general_pool_has_tight_keepalive(monkeypatch):\n+    \"\"\"No proxy / no fallback IPs → plain branch must also wire tuned limits.\"\"\"\n+    instances = _drive_connect(monkeypatch, proxy_url=None)\n+    assert len(instances) >= 2\n+    _assert_keepalive_tight(instances)\n+\n+\n+def test_limits_keepalive_below_ptb_default_is_the_contract():\n+    \"\"\"Document the invariant independent of adapter wiring: the shared\n+    helper itself must tighten keepalive below httpx's 5.0 default.\"\"\"\n+    from gateway.platforms._http_client_limits import platform_httpx_limits\n+\n+    limits = platform_httpx_limits()\n+    assert isinstance(limits, httpx.Limits)\n+    assert limits.keepalive_expiry is not None and limits.keepalive_expiry < 5.0\n"]}]},{"kind":"pr","number":53622,"title":"fix(telegram): reset send pool after pool timeouts","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#54121","number":54121}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":53624,"title":"fix(telegram): refresh general request pool after pool timeout","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53524","number":53524},{"kind":"pr","value":"#53622","number":53622},{"kind":"pr","value":"#53622","number":53622},{"kind":"pr","value":"#54121","number":54121}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":54121,"title":"fix(telegram): drain general send pool on pool timeout before retry (#53524)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/platforms/telegram/adapter.py","scripts/release.py","tests/gateway/test_telegram_thread_fallback.py"],"fix_hunks":[{"file":"plugins/platforms/telegram/adapter.py","hunks":["@@ -424,6 +424,7 @@ def __init__(self, config: PlatformConfig):\n         # While True, send() short-circuits to a failure so callers\n         # (cron live-adapter branch) fall through to standalone delivery.\n         self._send_path_degraded: bool = False\n+        self._general_request_drain_lock = asyncio.Lock()\n         # DM Topics: map of topic_name -> message_thread_id (populated at startup)\n         self._dm_topics: Dict[str, int] = {}\n         # Track forum chats where we've already registered bot commands","@@ -1513,6 +1514,50 @@ async def _drain_polling_connections(self) -> None:\n                 self.name, exc_info=True,\n             )\n \n+    def _get_general_request_drain_lock(self) -> asyncio.Lock:\n+        lock = getattr(self, \"_general_request_drain_lock\", None)\n+        if lock is None:\n+            lock = asyncio.Lock()\n+            self._general_request_drain_lock = lock\n+        return lock\n+\n+    async def _drain_general_connections_after_pool_timeout(self) -> None:\n+        \"\"\"Reset the Bot API request pool after a confirmed send pool timeout.\n+\n+        ``send_message`` uses PTB's general request pool (``_request[1]``).\n+        When httpx reports that this pool is exhausted, PTB says the request\n+        was not sent, so it is safe to reset the wedged pool before retrying.\n+        \"\"\"\n+        bot = getattr(getattr(self, \"_app\", None), \"bot\", None)\n+        if bot is None:\n+            bot = getattr(self, \"_bot\", None)\n+        if bot is None:\n+            return\n+        try:\n+            # PTB 22.x: _request is (get_updates_request, general_request).\n+            general_req = bot._request[1]  # noqa: SLF001\n+        except Exception:\n+            return\n+        async with self._get_general_request_drain_lock():\n+            try:\n+                await general_req.shutdown()\n+            except Exception:\n+                logger.debug(\n+                    \"[%s] General request shutdown failed after pool timeout (non-fatal)\",\n+                    self.name, exc_info=True,\n+                )\n+            try:\n+                await general_req.initialize()\n+                logger.warning(\n+                    \"[%s] General request pool drained after Telegram pool timeout\",\n+                    self.name,\n+                )\n+            except Exception:\n+                logger.debug(\n+                    \"[%s] General request re-initialize failed after pool timeout (non-fatal)\",\n+                    self.name, exc_info=True,\n+                )\n+\n     async def _handle_polling_network_error(self, error: Exception) -> None:\n         \"\"\"Reconnect polling after a transient network interruption.\n ","@@ -2912,13 +2957,16 @@ async def send(\n                         # (httpx pool exhausted) is explicitly \"not sent to\n                         # Telegram\" -- retrying through the loop is safe and\n                         # prevents silent drops when the pool frees up.\n+                        is_pool_timeout = self._looks_like_pool_timeout(send_err)\n                         if (\n                             _TimedOut\n                             and isinstance(send_err, _TimedOut)\n                             and not self._looks_like_connect_timeout(send_err)\n-                            and not self._looks_like_pool_timeout(send_err)\n+                            and not is_pool_timeout\n                         ):\n                             raise\n+                        if is_pool_timeout:\n+                            await self._drain_general_connections_after_pool_timeout()\n                         if _send_attempt < 2:\n                             wait = 2 ** _send_attempt\n                             logger.warning(\"[%s] Network error on send (attempt %d/3), retrying in %ds: %s\","]},{"file":"scripts/release.py","hunks":["@@ -67,6 +67,7 @@\n     \"[redacted:email]\": \"herbalizer404\",  # PR #49076 + #51835 salvage (auxiliary compression fallback: 403/session-usage payment errors + honor fallback chain when aux provider auth unavailable)\n     \"[redacted:email]\": \"pyxl-dev\",  # PR #52230 salvage (include rate-limit in auxiliary capacity-error fallback gate; #52228)\n     \"[redacted:email]\": \"yashiels\",  # PR #53284 salvage (discord markdown table-to-bullet conversion; #21168)\n+    \"[redacted:email]\": \"yungchentang\",  # PR #53622 salvage (drain Telegram general send pool on pool timeout before retry; #53524)\n     \"[redacted:email]\": \"595650661\",  # PR #37851 salvage (classify MiniMax new_sensitive content filter → content_policy_blocked; #32421)\n     \"[redacted:email]\": \"benbenlijie\",  # PR #47205 salvage (named custom-provider extra_body + Z.AI Coding overload adaptive backoff; #50663)\n     \"[redacted:email]\": \"Danamove\",  # PR #46726 salvage (kill venv-resident pythonw gateway before recreating venv on Windows; #47036/#47557/#47910)"]},{"file":"tests/gateway/test_telegram_thread_fallback.py","hunks":["@@ -11,6 +11,7 @@\n import sys\n import types\n from types import SimpleNamespace\n+from unittest.mock import AsyncMock\n \n import pytest\n ","@@ -1338,6 +1339,46 @@ async def mock_send_message(**kwargs):\n     assert attempt[0] == 3\n \n \n[redacted:email]\n+async def test_send_drains_general_request_pool_before_retrying_pool_timeout():\n+    \"\"\"Pool timeout should reset the send-message request pool before retrying.\"\"\"\n+    adapter = _make_adapter()\n+    general_request = SimpleNamespace(\n+        shutdown=AsyncMock(),\n+        initialize=AsyncMock(),\n+    )\n+    polling_request = SimpleNamespace(\n+        shutdown=AsyncMock(),\n+        initialize=AsyncMock(),\n+    )\n+    adapter._app = SimpleNamespace(\n+        bot=SimpleNamespace(_request=(polling_request, general_request))\n+    )\n+\n+    attempt = [0]\n+\n+    async def mock_send_message(**kwargs):\n+        attempt[0] += 1\n+        if attempt[0] == 1:\n+            raise FakeTimedOut(\n+                \"Pool timeout: All connections in the connection pool are \"\n+                \"occupied. Request was *not* sent to Telegram.\"\n+            )\n+        return SimpleNamespace(message_id=203)\n+\n+    adapter._bot = SimpleNamespace(send_message=mock_send_message)\n+\n+    result = await adapter.send(chat_id=\"123\", content=\"test message\")\n+\n+    assert result.success is True\n+    assert result.message_id == \"203\"\n+    assert attempt[0] == 2\n+    general_request.shutdown.assert_awaited_once()\n+    general_request.initialize.assert_awaited_once()\n+    polling_request.shutdown.assert_not_awaited()\n+    polling_request.initialize.assert_not_awaited()\n+\n+\n @pytest.mark.asyncio\n async def test_send_marks_pool_timeout_retryable_after_exhaustion():\n     \"\"\"Pool timeout that never clears stays retryable for outer retry handling.\"\"\"\n"]}]},{"kind":"pr","number":61988,"title":"fix(agent): release pool FDs on owning-thread client close (#61979)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true}],"trust":{"default_tier":"verified","includes_suggested":false}}