{"repo":"NousResearch/hermes-agent","public_id":"54049","display_label":"httpx transport keepalive fix for streaming issues","summary":"These items address a problem where custom socket_options transport caused issues (including dropped chunked streaming with OpenResty/DeepSeek), proposing replacement with httpx pool-level keepalive expiry configuration.","subsystem":"agent","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":58766,"merged_at":"2026-07-05T10:14:55+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-07-05T10:14:55+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":54049,"pr_number":58766,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-07-05T10:14:55+00:00"}],"members":[{"kind":"issue","number":54049,"title":"DeepSeek: OpenResty drops chunked streaming with custom httpx transport","url":null,"state":"closed"},{"kind":"pr","number":54550,"title":"fix(agent): replace custom socket_options transport with httpx pool-level keepalive expiry","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"51c1ba697","number":0},{"kind":"pr","value":"#58766","number":58766}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":58766,"title":"fix(agent): replace socket_options transport with httpx pool-level keepalive expiry (#54550 salvage)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/process_bootstrap.py","run_agent.py","tests/run_agent/test_create_openai_client_proxy_env.py"],"fix_hunks":[{"file":"agent/process_bootstrap.py","hunks":["@@ -151,42 +151,51 @@ def build_keepalive_http_client(\n     \"\"\"Build an httpx client for OpenAI SDK calls with env-only proxy policy.\n \n     Uses explicit ``HTTPS_PROXY`` / ``NO_PROXY`` env vars via\n-    ``_get_proxy_for_base_url``. A custom transport disables httpx's default\n-    ``trust_env`` path, so macOS system proxy settings from\n+    ``_get_proxy_for_base_url``. Plain no-proxy mounts disable httpx's default\n+    ``trust_env`` proxy path, so macOS system proxy settings from\n     ``urllib.request.getproxies()`` (which omit the ExceptionsList) are not\n     applied. Mirrors ``AIAgent._build_keepalive_http_client``.\n \n+    Connection lifecycle is managed at the HTTP pool layer\n+    (``keepalive_expiry=20.0`` reaps idle connections before reverse proxies'\n+    typical 30-60 s timeouts) instead of the former custom\n+    ``socket_options`` transport, which broke streaming behind reverse\n+    proxies (#54049, #12952) and stalled TLS handshakes by stripping\n+    ``TCP_NODELAY``.\n+\n     ``verify`` is forwarded to httpx so auxiliary-client calls (compression,\n     vision, web_extract, title generation, etc.) honor the same per-provider\n     ``ssl_ca_cert`` / ``ssl_verify`` and ``HERMES_CA_BUNDLE`` settings the main\n-    client uses. It is passed on the ``HTTPTransport`` (which owns the SSL\n-    context when a custom transport is supplied) and, for the copilot branch\n-    that has no custom transport, on the client itself.\n+    client uses. It is passed on the client AND on the plain no-proxy mounts\n+    (a mounted transport owns the SSL context for its scheme).\n     \"\"\"\n     try:\n         import httpx\n-        import socket\n \n-        if \"api.githubcopilot.com\" in str(base_url or \"\").lower():\n-            client_cls = httpx.AsyncClient if async_mode else httpx.Client\n-            return client_cls(verify=verify)\n+        proxy = _get_proxy_for_base_url(base_url)\n \n-        sock_opts = [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]\n-        if hasattr(socket, \"TCP_KEEPIDLE\"):\n-            sock_opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30))\n-            sock_opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 10))\n-            sock_opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 3))\n-        elif hasattr(socket, \"TCP_KEEPALIVE\"):\n-            sock_opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPALIVE, 30))\n+        limits = httpx.Limits(\n+            max_keepalive_connections=20,\n+            max_connections=100,\n+            keepalive_expiry=20.0,\n+        )\n+        # Generous read=None for SSE streaming endpoints.\n+        timeout = httpx.Timeout(connect=15.0, read=None, write=15.0, pool=10.0)\n \n-        proxy = _get_proxy_for_base_url(base_url)\n         transport_cls = httpx.AsyncHTTPTransport if async_mode else httpx.HTTPTransport\n         client_cls = httpx.AsyncClient if async_mode else httpx.Client\n-        # verify lives on the transport: httpx ignores the client-level\n-        # ``verify`` when a custom ``transport=`` is supplied.\n+        mounts = {}\n+        if proxy is None:\n+            mounts = {\n+                \"http://\": transport_cls(verify=verify),\n+                \"https://\": transport_cls(verify=verify),\n+            }\n         return client_cls(\n-            transport=transport_cls(socket_options=sock_opts, verify=verify),\n+            limits=limits,\n+            timeout=timeout,\n             proxy=proxy,\n+            mounts=mounts or None,\n+            verify=verify,\n         )\n     except Exception:\n         return None"]},{"file":"run_agent.py","hunks":["@@ -3896,30 +3896,70 @@ def _is_openai_client_closed(client: Any) -> bool:\n \n     @staticmethod\n     def _build_keepalive_http_client(base_url: str = \"\", *, verify: Any = True) -> Any:\n+        \"\"\"Build an httpx.Client with proactive idle-connection reaping.\n+\n+        Previously this method injected a custom ``httpx.HTTPTransport``\n+        with ``socket_options`` (``SO_KEEPALIVE``, ``TCP_KEEPIDLE``, …) to\n+        prevent CLOSE-WAIT accumulation on long-lived connections (#10324).\n+\n+        That approach broke streaming for providers behind reverse proxies\n+        (OpenResty, Cloudflare, etc.) because the custom socket options\n+        conflict with the proxy's chunked-transfer handling (#54049,\n+        #12952).  It also stripped ``TCP_NODELAY``, stalling TLS handshakes\n+        and SSE encoding.\n+\n+        The fix moves connection lifecycle management from the socket layer\n+        to the HTTP pool layer: ``keepalive_expiry=20.0`` tells httpx to\n+        close idle pooled connections *before* a reverse proxy's typical\n+        30–60 s timeout drops them, preventing CLOSE-WAIT accumulation\n+        without modifying socket options.  The default httpx transport\n+        preserves OS TCP defaults (including ``TCP_NODELAY``).\n+\n+        ``verify`` carries per-provider ``ssl_ca_cert`` / ``ssl_verify`` and\n+        ``HERMES_CA_BUNDLE`` settings.  It is passed on the client AND on\n+        the plain no-proxy mounts (a mounted transport owns the SSL context\n+        for its scheme).\n+        \"\"\"\n         try:\n             import httpx as _httpx\n-            import socket as _socket\n-\n-            if \"api.githubcopilot.com\" in str(base_url or \"\").lower():\n-                return _httpx.Client(verify=verify)\n-\n-            _sock_opts = [(_socket.SOL_SOCKET, _socket.SO_KEEPALIVE, 1)]\n-            if hasattr(_socket, \"TCP_KEEPIDLE\"):\n-                _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPIDLE, 30))\n-                _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPINTVL, 10))\n-                _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPCNT, 3))\n-            elif hasattr(_socket, \"TCP_KEEPALIVE\"):\n-                _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPALIVE, 30))\n-            # When a custom transport is provided, httpx won't auto-read proxy\n-            # from env vars (allow_env_proxies = trust_env and transport is None).\n-            # Explicitly read proxy settings while still honoring NO_PROXY for\n-            # loopback / local endpoints such as a locally hosted sub2api.\n+\n+            # Explicitly read proxy settings so requests route through\n+            # HTTP_PROXY / HTTPS_PROXY / NO_PROXY correctly.\n             _proxy = _get_proxy_for_base_url(base_url)\n-            # verify lives on the transport: httpx ignores the client-level\n-            # ``verify`` when a custom ``transport=`` is supplied.\n+\n+            # Proactive pool reaping: close idle connections at 20 s,\n+            # before reverse proxies (30–60 s typical) send FIN and\n+            # cause CLOSE-WAIT accumulation.\n+            _limits = _httpx.Limits(\n+                max_keepalive_connections=20,\n+                max_connections=100,\n+                keepalive_expiry=20.0,\n+            )\n+\n+            # Timeouts: generous read=None for SSE streaming endpoints.\n+            _timeout = _httpx.Timeout(\n+                connect=15.0,\n+                read=None,\n+                write=15.0,\n+                pool=10.0,\n+            )\n+\n+            # When _proxy is None (NO_PROXY bypass or no proxy configured),\n+            # mount plain transports to prevent httpx from reading env proxy\n+            # vars and creating an HTTPProxy mount that would bypass our\n+            # NO_PROXY resolution.\n+            _mounts = {}\n+            if _proxy is None:\n+                _mounts = {\n+                    \"http://\": _httpx.HTTPTransport(verify=verify),\n+                    \"https://\": _httpx.HTTPTransport(verify=verify),\n+                }\n             return _httpx.Client(\n-                transport=_httpx.HTTPTransport(socket_options=_sock_opts, verify=verify),\n+                limits=_limits,\n+                timeout=_timeout,\n                 proxy=_proxy,\n+                mounts=_mounts or None,\n+                verify=verify,\n             )\n         except Exception:\n             return None"]},{"file":"tests/run_agent/test_create_openai_client_proxy_env.py","hunks":["@@ -1,22 +1,14 @@\n \"\"\"Regression guard: _create_openai_client must honor HTTP(S)_PROXY env vars.\n \n-When #11277 re-landed TCP keepalives, ``_create_openai_client`` began passing\n-a custom ``transport=httpx.HTTPTransport(...)`` to ``httpx.Client``. httpx only\n-auto-reads ``HTTP_PROXY`` / ``HTTPS_PROXY`` / ``ALL_PROXY`` when\n-``transport is None`` (see ``Client.__init__``:\n-``allow_env_proxies = trust_env and transport is None``). As a result, proxy\n-env vars were silently ignored for the primary chat client, causing requests\n-to bypass local proxies (Clash, corporate egress, etc.) and hit upstream\n-directly from the raw interface.\n-\n-For users on WSL2 + Clash TUN this surfaced as Cloudflare ``cf-mitigated:\n-challenge`` 403s against ``chatgpt.com/backend-api/codex`` once they upgraded\n-past #11277. The fix forwards the proxy URL explicitly to ``httpx.Client``\n-while keeping the keepalive-enabled transport in place.\n-\n-This test pins that the constructed ``httpx.Client`` mounts an ``HTTPProxy``\n-pool when a proxy env var is set, AND that the socket-level keepalive\n-transport is still installed on the no-proxy default path.\n+The keepalive client now uses ``httpx.Limits(keepalive_expiry=20.0)``\n+instead of a custom ``httpx.HTTPTransport(socket_options=...)`` to\n+prevent CLOSE-WAIT accumulation.  This avoids breaking streaming for\n+providers behind reverse proxies (#54049, #12952) while still reaping\n+idle connections before a proxy's timeout drops them.\n+\n+This test pins that the constructed ``httpx.Client`` mounts an\n+``HTTPProxy`` pool when a proxy env var is set, and that no\n+custom socket-options transport is used (default httpx transport).\n \"\"\"\n from unittest.mock import patch\n ","@@ -117,8 +109,8 @@ def test_create_openai_client_routes_via_proxy_when_env_set(mock_openai, monkeyp\n \n @patch(\"run_agent.OpenAI\")\n def test_create_openai_client_no_proxy_when_env_unset(mock_openai, monkeypatch):\n-    \"\"\"Without proxy env vars, the keepalive transport must still be installed\n-    and no HTTPProxy mount should exist.\"\"\"\n+    \"\"\"Without proxy env vars, no HTTPProxy mount should exist and\n+    no custom socket-options transport should be installed.\"\"\"\n     for key in (\"HTTPS_PROXY\", \"HTTP_PROXY\", \"ALL_PROXY\",\n                 \"https_proxy\", \"http_proxy\", \"all_proxy\"):\n         monkeypatch.delenv(key, raising=False)","@@ -147,7 +139,8 @@ def test_create_openai_client_no_proxy_when_env_unset(mock_openai, monkeypatch):\n \n @patch(\"run_agent.OpenAI\")\n def test_create_openai_client_uses_plain_httpx_client_for_copilot(mock_openai, monkeypatch):\n-    \"\"\"Copilot Claude chat-completions rejects the custom socket-options transport.\"\"\"\n+    \"\"\"All providers now use a standard httpx.Client (no custom socket-options\n+    transport) so Copilot Claude chat-completions works without a host bypass.\"\"\"\n     for key in (\"HTTPS_PROXY\", \"HTTP_PROXY\", \"ALL_PROXY\",\n                 \"https_proxy\", \"http_proxy\", \"all_proxy\"):\n         monkeypatch.delenv(key, raising=False)\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}