{"repo":"NousResearch/hermes-agent","public_id":"14128","display_label":"Gateway shutdown hangs and timeout fixes","summary":"Multiple fixes addressing gateway shutdown hanging due to unbounded adapter disconnect/teardown awaits, with added timeouts and related systemd restart edge cases. Covers WSL/Feishu/Weixin websocket scenarios and PID file race conditions on restart.","subsystem":"gateway","counts":{"members":9,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":2,"attempt_prs":5},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":22144},"resolution":{"state":"partially_resolved","by":"pr-22144"},"freshness":{"state":"verified","last_verified_at":"2026-06-28T02:05:04+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.20444444444444443},"fix_edges":[{"issue_number":14128,"pr_number":53897,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":19937,"pr_number":22144,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":14128,"title":"fix(gateway): Gateway shutdown hangs causing 'PID file race lost' on restart","url":null,"state":"closed"},{"kind":"issue","number":19937,"title":"Gateway drain hangs on wedged adapter websockets (WSL + Feishu/Weixin) — 90s systemctl stop traceback","url":null,"state":"closed"},{"kind":"pr","number":14130,"title":"fix(gateway): add timeout to adapter.disconnect() during shutdown","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53897","number":53897}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":14332,"title":"fix(gateway): treat recycled PID with unreadable start_time as stale (#14176)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ccf526964a793371d4df97b4d1ae4f7d6fd63439","number":0},{"kind":"commit","value":"d682f320b35a13084371a541a835e1d988c982b8","number":0},{"kind":"pr","value":"#14176","number":14176},{"kind":"pr","value":"#14128","number":14128}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":19994,"title":"fix(gateway): cap adapter disconnect during stop","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"7f36eebe0","number":0},{"kind":"pr","value":"#22144","number":22144},{"kind":"pr","value":"#22144","number":22144}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":22144,"title":"fix(gateway): cap adapter disconnect + catch TimeoutExpired on stop AND restart","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","hermes_cli/gateway.py","tests/gateway/test_safe_adapter_disconnect.py","tests/hermes_cli/test_gateway_service.py"]},{"kind":"pr","number":36913,"title":"fix(gateway): bound adapter network awaits on the shutdown path","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#53897","number":53897},{"kind":"pr","value":"#14128","number":14128}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":48277,"title":"fix(gateway): add --replace to systemd service ExecStart templates","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"ccf526964a793371d4df97b4d1ae4f7d6fd63439","number":0},{"kind":"commit","value":"4d18717b6c798d4f6bab9e736c6ed10c5a8365f4","number":0},{"kind":"pr","value":"#14128","number":14128}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":53897,"title":"fix(gateway): bound adapter teardown awaits on the stop path (#14128)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_bounded_adapter_teardown.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -3093,6 +3093,60 @@ async def _safe_adapter_disconnect(self, adapter, platform) -> None:\n                 e,\n             )\n \n+    async def _bounded_adapter_teardown(\n+        self, adapter, platform, *, profile: Optional[str] = None\n+    ) -> None:\n+        \"\"\"Tear down one adapter on the shutdown path with bounded awaits.\n+\n+        Both ``cancel_background_tasks()`` and ``disconnect()`` can block\n+        indefinitely when a platform's network state is half-dead (e.g. a\n+        wedged Feishu/Lark WebSocket thread waiting on I/O). An unbounded\n+        await here stalls the entire shutdown sequence past systemd's\n+        ``TimeoutStopSec``; the resulting SIGKILL skips ``atexit`` PID-file\n+        cleanup, so the next start dies with \"PID file race lost\" (#14128).\n+\n+        Each await is wrapped in the existing per-adapter timeout budget\n+        (``HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT``). On timeout we log\n+        and force forward progress; the loop never hangs regardless of any\n+        adapter's internal behavior. Never raises.\n+        \"\"\"\n+        timeout = self._adapter_disconnect_timeout_secs()\n+        suffix = f\" (profile: {profile})\" if profile else \"\"\n+        started_at = time.monotonic()\n+        try:\n+            if timeout <= 0:\n+                await adapter.cancel_background_tasks()\n+            else:\n+                await asyncio.wait_for(\n+                    adapter.cancel_background_tasks(), timeout=timeout\n+                )\n+        except asyncio.TimeoutError:\n+            logger.warning(\n+                \"✗ %s background-task cancel timed out after %.1fs - forcing continue%s\",\n+                platform.value, timeout, suffix,\n+            )\n+        except Exception as e:\n+            logger.debug(\"✗ %s background-task cancel error%s: %s\", platform.value, suffix, e)\n+        try:\n+            if timeout <= 0:\n+                await adapter.disconnect()\n+            else:\n+                await asyncio.wait_for(adapter.disconnect(), timeout=timeout)\n+            logger.info(\n+                \"✓ %s disconnected (%.2fs)%s\",\n+                platform.value, time.monotonic() - started_at, suffix,\n+            )\n+        except asyncio.TimeoutError:\n+            logger.warning(\n+                \"✗ %s disconnect timed out after %.1fs - forcing continue%s\",\n+                platform.value, timeout, suffix,\n+            )\n+        except Exception as e:\n+            logger.error(\n+                \"✗ %s disconnect error after %.2fs%s: %s\",\n+                platform.value, time.monotonic() - started_at, suffix, e,\n+            )\n+\n     def _adapter_disconnect_timeout_secs(self) -> float:\n         \"\"\"Return the per-adapter disconnect timeout used during shutdown.\"\"\"\n         raw = os.getenv(\"HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT\", \"\").strip()","@@ -7204,38 +7258,14 @@ def _phase_elapsed() -> float:\n                     self._cleanup_agent_resources(_agent)\n \n             for platform, adapter in list(self.adapters.items()):\n-                _adapter_started_at = time.monotonic()\n-                try:\n-                    await adapter.cancel_background_tasks()\n-                except Exception as e:\n-                    logger.debug(\"✗ %s background-task cancel error: %s\", platform.value, e)\n-                try:\n-                    await adapter.disconnect()\n-                    logger.info(\n-                        \"✓ %s disconnected (%.2fs)\",\n-                        platform.value,\n-                        time.monotonic() - _adapter_started_at,\n-                    )\n-                except Exception as e:\n-                    logger.error(\n-                        \"✗ %s disconnect error after %.2fs: %s\",\n-                        platform.value,\n-                        time.monotonic() - _adapter_started_at,\n-                        e,\n-                    )\n+                await self._bounded_adapter_teardown(adapter, platform)\n \n             # Disconnect secondary-profile adapters (multiplex mode).\n             for _prof, _amap in list(getattr(self, \"_profile_adapters\", {}).items()):\n                 for platform, adapter in list(_amap.items()):\n-                    try:\n-                        await adapter.cancel_background_tasks()\n-                    except Exception as e:\n-                        logger.debug(\"✗ %s bg-cancel error (profile %s): %s\", platform.value, _prof, e)\n-                    try:\n-                        await adapter.disconnect()\n-                        logger.info(\"✓ %s disconnected (profile: %s)\", platform.value, _prof)\n-                    except Exception as e:\n-                        logger.error(\"✗ %s disconnect error (profile %s): %s\", platform.value, _prof, e)\n+                    await self._bounded_adapter_teardown(\n+                        adapter, platform, profile=_prof\n+                    )\n                 _amap.clear()\n             if hasattr(self, \"_profile_adapters\"):\n                 self._profile_adapters.clear()"]},{"file":"tests/gateway/test_bounded_adapter_teardown.py","hunks":["@@ -0,0 +1,134 @@\n+\"\"\"Regression tests: the shutdown teardown loop must not hang on a wedged adapter.\n+\n+`GatewayRunner._stop_impl()` tears down every adapter by awaiting\n+`cancel_background_tasks()` then `disconnect()`. Both calls can block\n+indefinitely when a platform's network state is half-dead (e.g. a wedged\n+Feishu/Lark WebSocket thread waiting on I/O). An unbounded await stalls the\n+whole shutdown past systemd's TimeoutStopSec; the resulting SIGKILL skips\n+atexit PID-file cleanup, so the next start dies with \"PID file race lost\"\n+(#14128).\n+\n+The fix routes both teardown loops through `_bounded_adapter_teardown`,\n+which wraps each await in the existing per-adapter timeout budget\n+(HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT) and always returns.\n+\"\"\"\n+\n+import asyncio\n+import logging\n+from unittest.mock import AsyncMock, MagicMock\n+\n+import pytest\n+\n+from gateway.config import Platform\n+from gateway.run import GatewayRunner\n+\n+\n[redacted:email]\n+def bare_runner():\n+    \"\"\"A GatewayRunner shell that only needs _bounded_adapter_teardown.\"\"\"\n+    return object.__new__(GatewayRunner)\n+\n+\n[redacted:email]\n+async def test_teardown_calls_both_methods(bare_runner):\n+    \"\"\"The helper cancels background tasks AND disconnects, in that order.\"\"\"\n+    calls = []\n+    adapter = MagicMock()\n+    adapter.cancel_background_tasks = AsyncMock(\n+        side_effect=lambda: calls.append(\"cancel\")\n+    )\n+    adapter.disconnect = AsyncMock(side_effect=lambda: calls.append(\"disconnect\"))\n+\n+    await bare_runner._bounded_adapter_teardown(adapter, Platform.TELEGRAM)\n+\n+    adapter.cancel_background_tasks.assert_awaited_once()\n+    adapter.disconnect.assert_awaited_once()\n+    assert calls == [\"cancel\", \"disconnect\"]\n+\n+\n[redacted:email]\n+async def test_teardown_bounds_hanging_disconnect(bare_runner, monkeypatch, caplog):\n+    \"\"\"A wedged disconnect() must time out instead of hanging the loop.\"\"\"\n+    monkeypatch.setenv(\"HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT\", \"0.01\")\n+    adapter = MagicMock()\n+    adapter.cancel_background_tasks = AsyncMock(return_value=None)\n+\n+    async def hang():\n+        await asyncio.sleep(60)\n+\n+    adapter.disconnect = AsyncMock(side_effect=hang)\n+\n+    with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+        await asyncio.wait_for(\n+            bare_runner._bounded_adapter_teardown(adapter, Platform.FEISHU),\n+            timeout=5.0,  # the helper itself must return well under this\n+        )\n+\n+    adapter.disconnect.assert_awaited_once()\n+    assert \"feishu disconnect timed out\" in caplog.text\n+\n+\n[redacted:email]\n+async def test_teardown_bounds_hanging_cancel(bare_runner, monkeypatch, caplog):\n+    \"\"\"A wedged cancel_background_tasks() must time out, then disconnect runs.\"\"\"\n+    monkeypatch.setenv(\"HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT\", \"0.01\")\n+    adapter = MagicMock()\n+\n+    async def hang():\n+        await asyncio.sleep(60)\n+\n+    adapter.cancel_background_tasks = AsyncMock(side_effect=hang)\n+    adapter.disconnect = AsyncMock(return_value=None)\n+\n+    with caplog.at_level(logging.WARNING, logger=\"gateway.run\"):\n+        await asyncio.wait_for(\n+            bare_runner._bounded_adapter_teardown(adapter, Platform.FEISHU),\n+            timeout=5.0,\n+        )\n+\n+    assert \"feishu background-task cancel timed out\" in caplog.text\n+    # disconnect still attempted after the cancel timeout — forward progress.\n+    adapter.disconnect.assert_awaited_once()\n+\n+\n[redacted:email]\n+async def test_teardown_swallows_exceptions(bare_runner):\n+    \"\"\"Errors in either await must not propagate — shutdown continues.\"\"\"\n+    adapter = MagicMock()\n+    adapter.cancel_background_tasks = AsyncMock(side_effect=RuntimeError(\"bg\"))\n+    adapter.disconnect = AsyncMock(side_effect=RuntimeError(\"disc\"))\n+\n+    # Must NOT raise.\n+    await bare_runner._bounded_adapter_teardown(adapter, Platform.TELEGRAM)\n+\n+    adapter.cancel_background_tasks.assert_awaited_once()\n+    adapter.disconnect.assert_awaited_once()\n+\n+\n[redacted:email]\n+async def test_teardown_profile_suffix_in_logs(bare_runner, caplog):\n+    \"\"\"Multiplex (secondary-profile) teardown tags log lines with the profile.\"\"\"\n+    adapter = MagicMock()\n+    adapter.cancel_background_tasks = AsyncMock(return_value=None)\n+    adapter.disconnect = AsyncMock(return_value=None)\n+\n+    with caplog.at_level(logging.INFO, logger=\"gateway.run\"):\n+        await bare_runner._bounded_adapter_teardown(\n+            adapter, Platform.TELEGRAM, profile=\"acct2\"\n+        )\n+\n+    assert \"(profile: acct2)\" in caplog.text\n+\n+\n[redacted:email]\n+async def test_teardown_timeout_zero_disables_bound(bare_runner, monkeypatch):\n+    \"\"\"timeout=0 disables the wait_for wrapper but still calls through.\"\"\"\n+    monkeypatch.setenv(\"HERMES_GATEWAY_ADAPTER_DISCONNECT_TIMEOUT\", \"0\")\n+    adapter = MagicMock()\n+    adapter.cancel_background_tasks = AsyncMock(return_value=None)\n+    adapter.disconnect = AsyncMock(return_value=None)\n+\n+    await bare_runner._bounded_adapter_teardown(adapter, Platform.TELEGRAM)\n+\n+    adapter.cancel_background_tasks.assert_awaited_once()\n+    adapter.disconnect.assert_awaited_once()\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}