{"repo":"NousResearch/hermes-agent","public_id":"10324","display_label":"TCP keepalives for detecting dropped provider connections","summary":"These issues and PRs address a bug where the agent hangs indefinitely in CLOSE-WAIT state when a custom provider drops a connection mid-response. The fix enables TCP keepalives to detect dead provider connections.","subsystem":null,"counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":2,"attempt_prs":0},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":11277,"merged_at":"2026-04-17T03:04:54+00:00"},"resolution":{"state":"resolved","by":"pr-10933"},"freshness":{"state":"stale","last_verified_at":"2026-04-17T03:04:54+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":10324,"pr_number":10933,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-16T09:32:21+00:00"},{"issue_number":10324,"pr_number":11277,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-17T03:04:54+00:00"}],"members":[{"kind":"issue","number":10324,"title":"Agent hangs indefinitely on CLOSE-WAIT when custom provider drops connection mid-response","url":null,"state":"closed"},{"kind":"pr","number":10933,"title":"fix: enable TCP keepalives to detect dead provider connections","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["run_agent.py"],"fix_hunks":[{"file":"run_agent.py","hunks":["@@ -4366,6 +4366,29 @@ def _create_openai_client(self, client_kwargs: dict, *, reason: str, shared: boo\n                 self._client_log_context(),\n             )\n             return client\n+        # Inject TCP keepalives to detect dead connections faster (#10324).\n+        # Without keepalives, a provider that drops mid-stream leaves the\n+        # socket in CLOSE-WAIT and epoll_wait may never fire, causing the\n+        # agent to hang indefinitely.  Keepalive probes detect the dead\n+        # peer within ~60s (30s idle + 3×10s probes).\n+        if \"http_client\" not in client_kwargs:\n+            try:\n+                import httpx as _httpx\n+                import socket as _socket\n+                _sock_opts = [(_socket.SOL_SOCKET, _socket.SO_KEEPALIVE, 1)]\n+                if hasattr(_socket, \"TCP_KEEPIDLE\"):\n+                    # Linux\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPIDLE, 30))\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPINTVL, 10))\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPCNT, 3))\n+                elif hasattr(_socket, \"TCP_KEEPALIVE\"):\n+                    # macOS (uses TCP_KEEPALIVE instead of TCP_KEEPIDLE)\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPALIVE, 30))\n+                client_kwargs[\"http_client\"] = _httpx.Client(\n+                    transport=_httpx.HTTPTransport(socket_options=_sock_opts),\n+                )\n+            except Exception:\n+                pass  # Fall through to default transport if socket opts fail\n         client = OpenAI(**client_kwargs)\n         logger.info(\n             \"OpenAI client created (%s, shared=%s) %s\",\n"]}]},{"kind":"pr","number":11277,"title":"fix: enable TCP keepalives to detect dead provider connections (#10324)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["run_agent.py"],"fix_hunks":[{"file":"run_agent.py","hunks":["@@ -4395,6 +4395,41 @@ def _create_openai_client(self, client_kwargs: dict, *, reason: str, shared: boo\n                 self._client_log_context(),\n             )\n             return client\n+        # Inject TCP keepalives so the kernel detects dead provider connections\n+        # instead of letting them sit silently in CLOSE-WAIT (#10324).  Without\n+        # this, a peer that drops mid-stream leaves the socket in a state where\n+        # epoll_wait never fires, ``httpx`` read timeout may not trigger, and\n+        # the agent hangs until manually killed.  Probes after 30s idle, retry\n+        # every 10s, give up after 3 → dead peer detected within ~60s.\n+        #\n+        # Safety against #10933: the ``client_kwargs = dict(client_kwargs)``\n+        # above means this injection only lands in the local per-call copy,\n+        # never back into ``self._client_kwargs``.  Each ``_create_openai_client``\n+        # invocation therefore gets its OWN fresh ``httpx.Client`` whose\n+        # lifetime is tied to the OpenAI client it is passed to.  When the\n+        # OpenAI client is closed (rebuild, teardown, credential rotation),\n+        # the paired ``httpx.Client`` closes with it, and the next call\n+        # constructs a fresh one — no stale closed transport can be reused.\n+        # Tests in ``tests/run_agent/test_create_openai_client_reuse.py`` and\n+        # ``tests/run_agent/test_sequential_chats_live.py`` pin this invariant.\n+        if \"http_client\" not in client_kwargs:\n+            try:\n+                import httpx as _httpx\n+                import socket as _socket\n+                _sock_opts = [(_socket.SOL_SOCKET, _socket.SO_KEEPALIVE, 1)]\n+                if hasattr(_socket, \"TCP_KEEPIDLE\"):\n+                    # Linux\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPIDLE, 30))\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPINTVL, 10))\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPCNT, 3))\n+                elif hasattr(_socket, \"TCP_KEEPALIVE\"):\n+                    # macOS (uses TCP_KEEPALIVE instead of TCP_KEEPIDLE)\n+                    _sock_opts.append((_socket.IPPROTO_TCP, _socket.TCP_KEEPALIVE, 30))\n+                client_kwargs[\"http_client\"] = _httpx.Client(\n+                    transport=_httpx.HTTPTransport(socket_options=_sock_opts),\n+                )\n+            except Exception:\n+                pass  # Fall through to default transport if socket opts fail\n         client = OpenAI(**client_kwargs)\n         logger.info(\n             \"OpenAI client created (%s, shared=%s) %s\",\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}