{"repo":"NousResearch/hermes-agent","public_id":"12438","display_label":"Gateway shutdown, restart, and hang hardening","summary":"Most members address problems with gateway processes hanging, failing to restart cleanly, or becoming unresponsive during SIGTERM/drain on Linux (systemd) and macOS (launchd), with a smaller cluster covering concurrent ddgs web search deadlocks and timeout gaps.","subsystem":"agent","counts":{"members":27,"open_issues":2,"open_issues_distinct":2,"open_prs":1,"closed_issues":7,"merged_prs":4,"attempt_prs":14},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":20949},"resolution":{"state":"partially_resolved","by":"pr-52118,pr-66955,pr-68444"},"freshness":{"state":"verified","last_verified_at":"2026-07-21T06:21:02+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.059487179487179485},"fix_edges":[{"issue_number":12438,"pr_number":20949,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":36776,"pr_number":52118,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":66892,"pr_number":66955,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":68096,"pr_number":68444,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":12438,"title":"hermes gateway restart fails when gateway is in crashed/unresponsive state","url":null,"state":"closed"},{"kind":"issue","number":17576,"title":"fix(gateway): replace bash restart watcher with fcntl file-lock based Python process","url":null,"state":"open"},{"kind":"issue","number":29603,"title":"gateway restart kills new instance on non-systemd environments (Termux/Android)","url":null,"state":"open"},{"kind":"issue","number":29966,"title":"Bug: Concurrent ddgs web searches cause hard freeze (futex_do_wait deadlock, CTRL-C immune)","url":null,"state":"closed"},{"kind":"issue","number":36776,"title":"ddgs web search provider hangs indefinitely — no overall timeout on search calls","url":null,"state":"closed"},{"kind":"issue","number":40321,"title":"[Bug]: hermes can run ~15 min past deadline when a parent orchestrator's timeout is delayed — needs in-process hard backstop","url":null,"state":"closed"},{"kind":"issue","number":45361,"title":"macOS launchd: /restart can leave the gateway process alive-but-torn-down — kill -0 respawn watcher deadlocks, KeepAlive sees it 'running' (distinct from #43475)","url":null,"state":"closed"},{"kind":"issue","number":66892,"title":"Gateway event loop froze during SIGTERM drain — zombie process for 26h (KeepAlive can't detect it)","url":null,"state":"closed"},{"kind":"issue","number":68096,"title":"[Bug]: DDGS native primp hang holds the GIL, defeating timeout and Ctrl+C","url":null,"state":"closed"},{"kind":"pr","number":12587,"title":"fix(gateway): recover systemd restart when gateway is unresponsive","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#12438","number":12438}],"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":16621,"title":"fix(gateway): replace bash restart watcher with fcntl file-lock based Python process","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":17588,"title":"fix(gateway): force-kill unresponsive gateway during systemd restart (#12438)","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":17675,"title":"fix(gateway): force-kill unresponsive gateway during systemd restart (#12438)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"d797755a1c17566b0aef4d77548a4b460142d26a","number":0},{"kind":"pr","value":"#20949","number":20949},{"kind":"pr","value":"#17675","number":17675},{"kind":"pr","value":"#17588","number":17588}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":20949,"title":"fix(gateway): wait for systemd restart readiness + harden Discord slash-command sync","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/platforms/discord.py","hermes_cli/gateway.py","tests/gateway/test_discord_connect.py","tests/hermes_cli/test_gateway_service.py"],"fix_hunks":[{"file":"gateway/platforms/discord.py","hunks":["@@ -10,6 +10,8 @@\n \"\"\"\n \n import asyncio\n+import hashlib\n+import json\n import logging\n import os\n import struct","@@ -24,6 +26,10 @@\n \n VALID_THREAD_AUTO_ARCHIVE_MINUTES = {60, 1440, 4320, 10080}\n _DISCORD_COMMAND_SYNC_POLICIES = {\"safe\", \"bulk\", \"off\"}\n+_DISCORD_COMMAND_SYNC_STATE_SUBDIR = \"gateway\"\n+_DISCORD_COMMAND_SYNC_STATE_FILENAME = \"discord_command_sync_state.json\"\n+_DISCORD_COMMAND_SYNC_MUTATION_INTERVAL_SECONDS = 4.5\n+_DISCORD_COMMAND_SYNC_MAX_RATE_LIMIT_SLEEP_SECONDS = 30.0\n \n try:\n     import discord","@@ -45,6 +51,7 @@\n import re\n \n from gateway.platforms.helpers import MessageDeduplicator, ThreadParticipationTracker\n+from utils import atomic_json_write\n from gateway.platforms.base import (\n     BasePlatformAdapter,\n     MessageEvent,","@@ -825,6 +832,167 @@ async def disconnect(self) -> None:\n \n         logger.info(\"[%s] Disconnected\", self.name)\n \n+    def _command_sync_state_path(self) -> _Path:\n+        from hermes_constants import get_hermes_home\n+\n+        directory = get_hermes_home() / _DISCORD_COMMAND_SYNC_STATE_SUBDIR\n+        try:\n+            directory.mkdir(parents=True, exist_ok=True)\n+        except Exception:\n+            pass\n+        return directory / _DISCORD_COMMAND_SYNC_STATE_FILENAME\n+\n+    def _read_command_sync_state(self) -> dict:\n+        try:\n+            path = self._command_sync_state_path()\n+            if not path.exists():\n+                return {}\n+            data = json.loads(path.read_text(encoding=\"utf-8\"))\n+        except Exception:\n+            return {}\n+        return data if isinstance(data, dict) else {}\n+\n+    def _write_command_sync_state(self, state: dict) -> None:\n+        atomic_json_write(\n+            self._command_sync_state_path(),\n+            state,\n+            indent=None,\n+            separators=(\",\", \":\"),\n+        )\n+\n+    def _command_sync_state_key(self, app_id: Any) -> str:\n+        return str(app_id or \"unknown\")\n+\n+    def _desired_command_sync_fingerprint(self) -> str:\n+        tree = self._client.tree if self._client else None\n+        desired = []\n+        if tree is not None:\n+            desired = [\n+                self._canonicalize_app_command_payload(command.to_dict(tree))\n+                for command in tree.get_commands()\n+            ]\n+        desired.sort(key=lambda item: (item.get(\"type\", 1), item.get(\"name\", \"\")))\n+        payload = json.dumps(desired, sort_keys=True, separators=(\",\", \":\"))\n+        return hashlib.sha256(payload.encode(\"utf-8\")).hexdigest()\n+\n+    def _command_sync_skip_reason(self, app_id: Any, fingerprint: str) -> Optional[str]:\n+        entry = self._read_command_sync_state().get(self._command_sync_state_key(app_id))\n+        if not isinstance(entry, dict):\n+            return None\n+        now = time.time()\n+        retry_after_until = float(entry.get(\"retry_after_until\") or 0)\n+        if retry_after_until > now:\n+            remaining = max(1, int(retry_after_until - now))\n+            return f\"Discord asked us to wait before syncing slash commands; retry in {remaining}s\"\n+        if entry.get(\"fingerprint\") == fingerprint and entry.get(\"last_success_at\"):\n+            return \"same slash-command fingerprint already synced\"\n+        return None\n+\n+    def _record_command_sync_attempt(self, app_id: Any, fingerprint: str) -> None:\n+        state = self._read_command_sync_state()\n+        state[self._command_sync_state_key(app_id)] = {\n+            **(\n+                state.get(self._command_sync_state_key(app_id))\n+                if isinstance(state.get(self._command_sync_state_key(app_id)), dict)\n+                else {}\n+            ),\n+            \"fingerprint\": fingerprint,\n+            \"last_attempt_at\": time.time(),\n+        }\n+        self._write_command_sync_state(state)\n+\n+    def _record_command_sync_rate_limit(self, app_id: Any, fingerprint: str, retry_after: float) -> None:\n+        retry_after = max(1.0, float(retry_after))\n+        state = self._read_command_sync_state()\n+        state[self._command_sync_state_key(app_id)] = {\n+            **(\n+                state.get(self._command_sync_state_key(app_id))\n+                if isinstance(state.get(self._command_sync_state_key(app_id)), dict)\n+                else {}\n+            ),\n+            \"fingerprint\": fingerprint,\n+            \"last_attempt_at\": time.time(),\n+            \"retry_after_until\": time.time() + retry_after,\n+            \"retry_after\": retry_after,\n+        }\n+        self._write_command_sync_state(state)\n+\n+    def _record_command_sync_success(self, app_id: Any, fingerprint: str, summary: dict) -> None:\n+        state = self._read_command_sync_state()\n+        state[self._command_sync_state_key(app_id)] = {\n+            \"fingerprint\": fingerprint,\n+            \"last_attempt_at\": time.time(),\n+            \"last_success_at\": time.time(),\n+            \"summary\": summary,\n+        }\n+        self._write_command_sync_state(state)\n+\n+    @staticmethod\n+    def _extract_discord_retry_after(exc: BaseException) -> Optional[float]:\n+        value = getattr(exc, \"retry_after\", None)\n+        if value is not None:\n+            try:\n+                return max(1.0, float(value))\n+            except (TypeError, ValueError):\n+                return None\n+        response = getattr(exc, \"response\", None)\n+        headers = getattr(response, \"headers\", None)\n+        if headers:\n+            for key in (\"Retry-After\", \"X-RateLimit-Reset-After\"):\n+                try:\n+                    raw = headers.get(key)\n+                except Exception:\n+                    raw = None\n+                if raw is None:\n+                    continue\n+                try:\n+                    return max(1.0, float(raw))\n+                except (TypeError, ValueError):\n+                    continue\n+        return None\n+\n+    @staticmethod\n+    def _is_discord_rate_limit(exc: BaseException) -> bool:\n+        \"\"\"True only for exceptions that look like Discord 429 rate limits.\n+\n+        Narrower than ``hasattr(exc, 'retry_after')``: discord.py's own\n+        ``RateLimited`` exception and any HTTPException with status 429\n+        qualify. This prevents suppressing unrelated failures that happen\n+        to expose a ``retry_after`` attribute.\"\"\"\n+        # discord.py emits RateLimited / HTTPException subclasses for 429s.\n+        # Guard with isinstance-of-class so a mocked ``discord`` module\n+        # (where attrs are MagicMocks, not types) doesn't trip isinstance.\n+        if DISCORD_AVAILABLE and discord is not None:\n+            for attr_name in (\"RateLimited\", \"HTTPException\"):\n+                cls = getattr(discord, attr_name, None)\n+                if not isinstance(cls, type):\n+                    continue\n+                if isinstance(exc, cls):\n+                    if attr_name == \"RateLimited\":\n+                        return True\n+                    status = getattr(exc, \"status\", None)\n+                    if status == 429:\n+                        return True\n+        # Fallback duck-type: something named like a rate-limit with a\n+        # numeric retry_after. Covers mocked clients in tests and exotic\n+        # transports, without swallowing arbitrary exceptions.\n+        name = type(exc).__name__.lower()\n+        if (\"ratelimit\" in name or \"rate_limit\" in name) and getattr(exc, \"retry_after\", None) is not None:\n+            return True\n+        response = getattr(exc, \"response\", None)\n+        status = getattr(response, \"status\", None) or getattr(response, \"status_code\", None)\n+        if status == 429:\n+            return True\n+        return False\n+\n+    def _command_sync_mutation_interval_seconds(self) -> float:\n+        return _DISCORD_COMMAND_SYNC_MUTATION_INTERVAL_SECONDS\n+\n+    async def _sleep_between_command_sync_mutations(self) -> None:\n+        interval = self._command_sync_mutation_interval_seconds()\n+        if interval > 0:\n+            await asyncio.sleep(interval)\n+\n     async def _run_post_connect_initialization(self) -> None:\n         \"\"\"Finish non-critical startup work after Discord is connected.\"\"\"\n         if not self._client:","@@ -840,14 +1008,46 @@ async def _run_post_connect_initialization(self) -> None:\n                 logger.info(\"[%s] Synced %d slash command(s) via bulk tree sync\", self.name, len(synced))\n                 return\n \n-            # Discord's per-app command-management bucket is ~5 writes / 20 s,\n-            # so a mass-prune-plus-upsert reconcile (e.g. 77 orphans + 30\n-            # desired = 107 writes) takes several minutes of forced waits.\n-            # A flat 30 s budget blew up reliably under bucket pressure and\n-            # left slash commands broken for ~60 min until the bucket fully\n-            # recovered. Use a wide ceiling; the cap still guards against a\n-            # true hang. (#16713)\n-            summary = await asyncio.wait_for(self._safe_sync_slash_commands(), timeout=600)\n+            app_id = getattr(self._client, \"application_id\", None) or getattr(getattr(self._client, \"user\", None), \"id\", None)\n+            fingerprint = self._desired_command_sync_fingerprint()\n+            skip_reason = self._command_sync_skip_reason(app_id, fingerprint)\n+            if skip_reason:\n+                logger.info(\"[%s] Skipping Discord slash command sync: %s\", self.name, skip_reason)\n+                return\n+            self._record_command_sync_attempt(app_id, fingerprint)\n+\n+            http = getattr(self._client, \"http\", None)\n+            has_ratelimit_timeout = http is not None and hasattr(http, \"max_ratelimit_timeout\")\n+            previous_ratelimit_timeout = getattr(http, \"max_ratelimit_timeout\", None) if has_ratelimit_timeout else None\n+            if has_ratelimit_timeout:\n+                http.max_ratelimit_timeout = _DISCORD_COMMAND_SYNC_MAX_RATE_LIMIT_SLEEP_SECONDS\n+\n+            try:\n+                # Discord's per-app command-management bucket is small, and\n+                # discord.py can otherwise sit inside one long retry sleep\n+                # before surfacing the 429. Keep the whole sync bounded and\n+                # persist Discord's retry-after when it refuses the batch.\n+                summary = await asyncio.wait_for(self._safe_sync_slash_commands(), timeout=600)\n+            except Exception as e:\n+                if not self._is_discord_rate_limit(e):\n+                    raise\n+                retry_after = self._extract_discord_retry_after(e)\n+                if retry_after is None:\n+                    # Rate-limited but no retry-after signal — back off for a\n+                    # conservative default so we don't slam the bucket again.\n+                    retry_after = _DISCORD_COMMAND_SYNC_MAX_RATE_LIMIT_SLEEP_SECONDS\n+                self._record_command_sync_rate_limit(app_id, fingerprint, retry_after)\n+                logger.warning(\n+                    \"[%s] Discord rate-limited slash command sync; retrying after %.0fs\",\n+                    self.name,\n+                    retry_after,\n+                )\n+                return\n+            finally:\n+                if has_ratelimit_timeout:\n+                    http.max_ratelimit_timeout = previous_ratelimit_timeout\n+\n+            self._record_command_sync_success(app_id, fingerprint, summary)\n             logger.info(\n                 \"[%s] Safely reconciled %d slash command(s): unchanged=%d updated=%d recreated=%d created=%d deleted=%d\",\n                 self.name,","@@ -1009,11 +1209,20 @@ async def _safe_sync_slash_commands(self) -> Dict[str, int]:\n         created = 0\n         deleted = 0\n         http = self._client.http\n+        mutation_count = 0\n+\n+        async def mutate(call, *args):\n+            nonlocal mutation_count\n+            if mutation_count:\n+                await self._sleep_between_command_sync_mutations()\n+            result = await call(*args)\n+            mutation_count += 1\n+            return result\n \n         for key, desired in desired_by_key.items():\n             current = existing_by_key.pop(key, None)\n             if current is None:\n-                await http.upsert_global_command(app_id, desired)\n+                await mutate(http.upsert_global_command, app_id, desired)\n                 created += 1\n                 continue\n ","@@ -1025,16 +1234,16 @@ async def _safe_sync_slash_commands(self) -> Dict[str, int]:\n                 continue\n \n             if self._patchable_app_command_payload(current_existing_payload) == self._patchable_app_command_payload(desired):\n-                await http.delete_global_command(app_id, current.id)\n-                await http.upsert_global_command(app_id, desired)\n+                await mutate(http.delete_global_command, app_id, current.id)\n+                await mutate(http.upsert_global_command, app_id, desired)\n                 recreated += 1\n                 continue\n \n-            await http.edit_global_command(app_id, current.id, desired)\n+            await mutate(http.edit_global_command, app_id, current.id, desired)\n             updated += 1\n \n         for current in existing_by_key.values():\n-            await http.delete_global_command(app_id, current.id)\n+            await mutate(http.delete_global_command, app_id, current.id)\n             deleted += 1\n \n         return {"]},{"file":"hermes_cli/gateway.py","hunks":["@@ -505,6 +505,7 @@ def _read_systemd_unit_properties(\n         \"SubState\",\n         \"Result\",\n         \"ExecMainStatus\",\n+        \"MainPID\",\n     ),\n ) -> dict[str, str]:\n     \"\"\"Return selected ``systemctl show`` properties for the gateway unit.\"\"\"","@@ -538,6 +539,41 @@ def _read_systemd_unit_properties(\n     return parsed\n \n \n+def _systemd_main_pid_from_props(props: dict[str, str]) -> int | None:\n+    try:\n+        pid = int(props.get(\"MainPID\", \"0\") or \"0\")\n+    except (TypeError, ValueError):\n+        return None\n+    return pid if pid > 0 else None\n+\n+\n+def _systemd_main_pid(system: bool = False) -> int | None:\n+    return _systemd_main_pid_from_props(_read_systemd_unit_properties(system=system))\n+\n+\n+def _read_gateway_runtime_status() -> dict | None:\n+    try:\n+        from gateway.status import read_runtime_status\n+\n+        state = read_runtime_status()\n+    except Exception:\n+        return None\n+    return state if isinstance(state, dict) else None\n+\n+\n+def _gateway_runtime_status_for_pid(pid: int | None) -> dict | None:\n+    if not pid:\n+        return None\n+    state = _read_gateway_runtime_status()\n+    if not state:\n+        return None\n+    try:\n+        state_pid = int(state.get(\"pid\", 0) or 0)\n+    except (TypeError, ValueError):\n+        return None\n+    return state if state_pid == pid else None\n+\n+\n def _wait_for_systemd_service_restart(\n     *,\n     system: bool = False,","@@ -550,6 +586,7 @@ def _wait_for_systemd_service_restart(\n     svc = get_service_name()\n     scope_label = _service_scope_label(system).capitalize()\n     deadline = time.time() + timeout\n+    printed_runtime_wait = False\n \n     while time.time() < deadline:\n         props = _read_systemd_unit_properties(system=system)","@@ -562,19 +599,32 @@ def _wait_for_systemd_service_restart(\n             new_pid = get_running_pid()\n         except Exception:\n             new_pid = None\n+        if not new_pid:\n+            new_pid = _systemd_main_pid_from_props(props)\n \n         if active_state == \"active\":\n             if new_pid and (previous_pid is None or new_pid != previous_pid):\n-                print(f\"✓ {scope_label} service restarted (PID {new_pid})\")\n-                return True\n-            if previous_pid is None:\n-                print(f\"✓ {scope_label} service restarted\")\n-                return True\n+                runtime_state = _gateway_runtime_status_for_pid(new_pid)\n+                gateway_state = (runtime_state or {}).get(\"gateway_state\")\n+                if gateway_state == \"running\":\n+                    print(f\"✓ {scope_label} service restarted (PID {new_pid})\")\n+                    return True\n+                if gateway_state == \"startup_failed\":\n+                    reason = (runtime_state or {}).get(\"exit_reason\") or \"startup failed\"\n+                    print(f\"⚠ {scope_label} service process restarted (PID {new_pid}), but gateway startup failed: {reason}\")\n+                    return False\n+                if not printed_runtime_wait:\n+                    print(f\"⏳ {scope_label} service process started (PID {new_pid}); waiting for gateway runtime...\")\n+                    printed_runtime_wait = True\n \n         if active_state == \"activating\" and sub_state == \"auto-restart\":\n             time.sleep(1)\n             continue\n \n+        if _systemd_unit_is_start_limited(props):\n+            _print_systemd_start_limit_wait(system=system)\n+            return False\n+\n         time.sleep(2)\n \n     print(","@@ -585,6 +635,46 @@ def _wait_for_systemd_service_restart(\n     return False\n \n \n+def _systemd_unit_is_start_limited(props: dict[str, str]) -> bool:\n+    result = props.get(\"Result\", \"\").lower()\n+    sub_state = props.get(\"SubState\", \"\").lower()\n+    return result == \"start-limit-hit\" or sub_state == \"start-limit-hit\"\n+\n+\n+def _systemd_error_indicates_start_limit(exc: subprocess.CalledProcessError) -> bool:\n+    parts: list[str] = []\n+    for attr in (\"stderr\", \"stdout\", \"output\"):\n+        value = getattr(exc, attr, None)\n+        if not value:\n+            continue\n+        if isinstance(value, bytes):\n+            value = value.decode(errors=\"replace\")\n+        parts.append(str(value))\n+    text = \"\\n\".join(parts).lower()\n+    return (\n+        \"start-limit-hit\" in text\n+        or \"start request repeated too quickly\" in text\n+        or \"start-limit\" in text\n+    )\n+\n+\n+def _systemd_service_is_start_limited(system: bool = False) -> bool:\n+    return _systemd_unit_is_start_limited(_read_systemd_unit_properties(system=system))\n+\n+\n+def _print_systemd_start_limit_wait(system: bool = False) -> None:\n+    svc = get_service_name()\n+    scope_label = _service_scope_label(system).capitalize()\n+    scope_flag = \" --system\" if system else \"\"\n+    systemctl_prefix = \"systemctl \" if system else \"systemctl --user \"\n+    journal_prefix = \"journalctl \" if system else \"journalctl --user \"\n+    print(f\"⏳ {scope_label} service is temporarily rate-limited by systemd.\")\n+    print(\"  systemd is refusing another immediate start after repeated exits.\")\n+    print(f\"  Wait for the start-limit window to expire, then run: {'sudo ' if system else ''}hermes gateway restart{scope_flag}\")\n+    print(f\"  Or clear the failed state manually: {systemctl_prefix}reset-failed {svc}\")\n+    print(f\"  Check logs: {journal_prefix}-u {svc} -l --since '5 min ago'\")\n+\n+\n def _recover_pending_systemd_restart(system: bool = False, previous_pid: int | None = None) -> bool:\n     \"\"\"Recover a planned service restart that is stuck in systemd state.\"\"\"\n     props = _read_systemd_unit_properties(system=system)","@@ -2135,41 +2225,52 @@ def systemd_restart(system: bool = False):\n     refresh_systemd_unit_if_needed(system=system)\n     from gateway.status import get_running_pid\n \n-    pid = get_running_pid()\n-    if pid is not None and _request_gateway_self_restart(pid):\n-        import time\n+    pid = get_running_pid() or _systemd_main_pid(system=system)\n+    if pid is not None:\n         scope_label = _service_scope_label(system).capitalize()\n         svc = get_service_name()\n+        drain_timeout = _get_restart_drain_timeout()\n+\n+        print(f\"⏳ {scope_label} service restarting gracefully (PID {pid})...\")\n+        if _graceful_restart_via_sigusr1(pid, drain_timeout + 5):\n+            # The gateway exits with code 75 for a planned service restart.\n+            # RestartSec can otherwise delay the relaunch even though the\n+            # operator asked for an immediate restart, so kick the unit once\n+            # the old PID has exited and then wait for the replacement PID.\n+            _run_systemctl(\n+                [\"reset-failed\", svc],\n+                system=system,\n+                check=False,\n+                timeout=30,\n+            )\n+            _run_systemctl(\n+                [\"restart\", svc],\n+                system=system,\n+                check=False,\n+                timeout=90,\n+            )\n+            if _wait_for_systemd_service_restart(system=system, previous_pid=pid):\n+                return\n+            if _systemd_service_is_start_limited(system=system):\n+                return\n \n-        # Phase 1: wait for old process to exit (drain + shutdown)\n-        print(f\"⏳ {scope_label} service draining active work...\")\n-        deadline = time.time() + 90\n-        while time.time() < deadline:\n-            try:\n-                os.kill(pid, 0)\n-                time.sleep(1)\n-            except (ProcessLookupError, PermissionError):\n-                break  # old process is gone\n-        else:\n-            print(f\"⚠ Old process (PID {pid}) still alive after 90s\")\n-\n-        # The gateway exits with code 75 for a planned service restart.\n-        # systemd can sit in the RestartSec window or even wedge itself into a\n-        # failed/rate-limited state if the operator asks for another restart in\n-        # the middle of that handoff. Clear any stale failed state and kick the\n-        # unit immediately so `hermes gateway restart` behaves idempotently.\n+        print(\n+            f\"⚠ Graceful restart did not complete within {int(drain_timeout + 5)}s; \"\n+            \"forcing a service restart...\"\n+        )\n         _run_systemctl(\n             [\"reset-failed\", svc],\n             system=system,\n             check=False,\n             timeout=30,\n         )\n-        _run_systemctl(\n-            [\"start\", svc],\n-            system=system,\n-            check=False,\n-            timeout=90,\n-        )\n+        try:\n+            _run_systemctl([\"restart\", svc], system=system, check=True, timeout=90)\n+        except subprocess.CalledProcessError as exc:\n+            if _systemd_error_indicates_start_limit(exc) or _systemd_service_is_start_limited(system=system):\n+                _print_systemd_start_limit_wait(system=system)\n+                return\n+            raise\n         _wait_for_systemd_service_restart(system=system, previous_pid=pid)\n         return\n ","@@ -2182,8 +2283,14 @@ def systemd_restart(system: bool = False):\n         check=False,\n         timeout=30,\n     )\n-    _run_systemctl([\"reload-or-restart\", get_service_name()], system=system, check=True, timeout=90)\n-    print(f\"✓ {_service_scope_label(system).capitalize()} service restarted\")\n+    try:\n+        _run_systemctl([\"restart\", get_service_name()], system=system, check=True, timeout=90)\n+    except subprocess.CalledProcessError as exc:\n+        if _systemd_error_indicates_start_limit(exc) or _systemd_service_is_start_limited(system=system):\n+            _print_systemd_start_limit_wait(system=system)\n+            return\n+        raise\n+    _wait_for_systemd_service_restart(system=system, previous_pid=pid)\n \n \n ","@@ -2255,6 +2362,10 @@ def systemd_status(deep: bool = False, system: bool = False, full: bool = False)\n     result_code = unit_props.get(\"Result\", \"\")\n     if active_state == \"activating\" and sub_state == \"auto-restart\":\n         print(\"  ⏳ Restart pending: systemd is waiting to relaunch the gateway\")\n+    elif _systemd_unit_is_start_limited(unit_props):\n+        print(\"  ⏳ Restart pending: systemd is temporarily rate-limiting starts\")\n+        print(f\"  Run after the start-limit window expires: {'sudo ' if system else ''}hermes gateway restart{scope_flag}\")\n+        print(f\"  Or clear it manually: systemctl {'--user ' if not system else ''}reset-failed {get_service_name()}\")\n     elif active_state == \"failed\" and exec_main_status == str(GATEWAY_SERVICE_RESTART_EXIT_CODE):\n         print(\"  ⚠ Planned restart is stuck in systemd failed state (exit 75)\")\n         print(f\"  Run: systemctl {'--user ' if not system else ''}reset-failed {get_service_name()} && {'sudo ' if system else ''}hermes gateway start{scope_flag}\")"]},{"file":"tests/gateway/test_discord_connect.py","hunks":["@@ -1,4 +1,5 @@\n import asyncio\n+import json\n import sys\n from types import SimpleNamespace\n from unittest.mock import AsyncMock, MagicMock","@@ -70,6 +71,15 @@ def _ensure_discord_mock():\n from gateway.platforms.discord import DiscordAdapter  # noqa: E402\n \n \n[redacted:email](autouse=True)\n+def _speed_up_command_sync_mutation_pacing(monkeypatch):\n+    monkeypatch.setattr(\n+        DiscordAdapter,\n+        \"_command_sync_mutation_interval_seconds\",\n+        lambda self: 0.0,\n+    )\n+\n+\n class FakeTree:\n     def __init__(self):\n         self.sync = AsyncMock(return_value=[])","@@ -536,6 +546,183 @@ async def test_post_connect_initialization_skips_sync_when_policy_off(monkeypatc\n     fake_tree.sync.assert_not_called()\n \n \n[redacted:email]\n+async def test_post_connect_initialization_skips_same_fingerprint_after_success(tmp_path, monkeypatch):\n+    adapter = DiscordAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))\n+    monkeypatch.setattr(\"hermes_constants.get_hermes_home\", lambda: tmp_path)\n+\n+    class _DesiredCommand:\n+        def to_dict(self, tree):\n+            return {\n+                \"name\": \"status\",\n+                \"description\": \"Show Hermes status\",\n+                \"type\": 1,\n+                \"options\": [],\n+            }\n+\n+    fake_tree = SimpleNamespace(\n+        get_commands=lambda: [_DesiredCommand()],\n+        fetch_commands=AsyncMock(return_value=[]),\n+    )\n+    fake_http = SimpleNamespace(\n+        upsert_global_command=AsyncMock(),\n+        edit_global_command=AsyncMock(),\n+        delete_global_command=AsyncMock(),\n+    )\n+    adapter._client = SimpleNamespace(\n+        tree=fake_tree,\n+        http=fake_http,\n+        application_id=999,\n+        user=SimpleNamespace(id=999),\n+    )\n+\n+    await adapter._run_post_connect_initialization()\n+    await adapter._run_post_connect_initialization()\n+\n+    fake_tree.fetch_commands.assert_awaited_once()\n+    fake_http.upsert_global_command.assert_awaited_once()\n+\n+\n[redacted:email]\n+async def test_post_connect_initialization_respects_discord_retry_after(tmp_path, monkeypatch):\n+    adapter = DiscordAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))\n+    monkeypatch.setattr(\"hermes_constants.get_hermes_home\", lambda: tmp_path)\n+\n+    class _DesiredCommand:\n+        def to_dict(self, tree):\n+            return {\n+                \"name\": \"status\",\n+                \"description\": \"Show Hermes status\",\n+                \"type\": 1,\n+                \"options\": [],\n+            }\n+\n+    adapter._client = SimpleNamespace(\n+        tree=SimpleNamespace(get_commands=lambda: [_DesiredCommand()]),\n+        application_id=999,\n+        user=SimpleNamespace(id=999),\n+    )\n+    class _DiscordRateLimit(RuntimeError):\n+        retry_after = 123.0\n+\n+    sync = AsyncMock(side_effect=_DiscordRateLimit(\"discord rate limited\"))\n+    monkeypatch.setattr(adapter, \"_safe_sync_slash_commands\", sync)\n+\n+    await adapter._run_post_connect_initialization()\n+    await adapter._run_post_connect_initialization()\n+\n+    sync.assert_awaited_once()\n+    state_path = (\n+        tmp_path\n+        / discord_platform._DISCORD_COMMAND_SYNC_STATE_SUBDIR\n+        / discord_platform._DISCORD_COMMAND_SYNC_STATE_FILENAME\n+    )\n+    state = json.loads(state_path.read_text())\n+    entry = state[\"999\"]\n+    assert entry[\"retry_after\"] == 123.0\n+    assert entry[\"retry_after_until\"] > entry[\"last_attempt_at\"]\n+\n+\n[redacted:email]\n+async def test_post_connect_initialization_reraises_non_rate_limit_exceptions(tmp_path, monkeypatch):\n+    \"\"\"Arbitrary failures during sync must surface, not be swallowed as rate-limits.\"\"\"\n+    adapter = DiscordAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))\n+    monkeypatch.setattr(\"hermes_constants.get_hermes_home\", lambda: tmp_path)\n+\n+    class _DesiredCommand:\n+        def to_dict(self, tree):\n+            return {\"name\": \"status\", \"description\": \"Show Hermes status\", \"type\": 1, \"options\": []}\n+\n+    adapter._client = SimpleNamespace(\n+        tree=SimpleNamespace(get_commands=lambda: [_DesiredCommand()]),\n+        application_id=4242,\n+        user=SimpleNamespace(id=4242),\n+    )\n+\n+    # Unrelated failure that happens to expose retry_after. Must NOT be\n+    # caught by the rate-limit handler — it has nothing to do with 429s.\n+    class _UnrelatedError(RuntimeError):\n+        retry_after = 999.0\n+\n+    sync = AsyncMock(side_effect=_UnrelatedError(\"database is down\"))\n+    monkeypatch.setattr(adapter, \"_safe_sync_slash_commands\", sync)\n+\n+    # The outer _run_post_connect_initialization has a broad except Exception\n+    # that logs defensively — so we assert on state NOT being written.\n+    await adapter._run_post_connect_initialization()\n+\n+    sync.assert_awaited_once()\n+    state_path = (\n+        tmp_path\n+        / discord_platform._DISCORD_COMMAND_SYNC_STATE_SUBDIR\n+        / discord_platform._DISCORD_COMMAND_SYNC_STATE_FILENAME\n+    )\n+    state = json.loads(state_path.read_text()) if state_path.exists() else {}\n+    entry = state.get(\"4242\", {})\n+    # Attempt was recorded before the sync call, but no rate-limit cooldown\n+    # should have been persisted from the unrelated exception.\n+    assert \"retry_after_until\" not in entry\n+    assert \"retry_after\" not in entry\n+\n+\n[redacted:email]\n+async def test_safe_sync_slash_commands_paces_mutation_writes(monkeypatch):\n+    adapter = DiscordAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))\n+    monkeypatch.setattr(\n+        DiscordAdapter,\n+        \"_command_sync_mutation_interval_seconds\",\n+        lambda self: 1.25,\n+    )\n+    sleeps = []\n+\n+    async def fake_sleep(delay):\n+        sleeps.append(delay)\n+\n+    monkeypatch.setattr(discord_platform.asyncio, \"sleep\", fake_sleep)\n+\n+    class _DesiredCommand:\n+        def __init__(self, payload):\n+            self._payload = payload\n+\n+        def to_dict(self, tree):\n+            assert tree is not None\n+            return dict(self._payload)\n+\n+    desired_one = {\n+        \"name\": \"status\",\n+        \"description\": \"Show Hermes status\",\n+        \"type\": 1,\n+        \"options\": [],\n+    }\n+    desired_two = {\n+        \"name\": \"debug\",\n+        \"description\": \"Generate a debug report\",\n+        \"type\": 1,\n+        \"options\": [],\n+    }\n+    fake_tree = SimpleNamespace(\n+        get_commands=lambda: [_DesiredCommand(desired_one), _DesiredCommand(desired_two)],\n+        fetch_commands=AsyncMock(return_value=[]),\n+    )\n+    fake_http = SimpleNamespace(\n+        upsert_global_command=AsyncMock(),\n+        edit_global_command=AsyncMock(),\n+        delete_global_command=AsyncMock(),\n+    )\n+    adapter._client = SimpleNamespace(\n+        tree=fake_tree,\n+        http=fake_http,\n+        application_id=999,\n+        user=SimpleNamespace(id=999),\n+    )\n+\n+    summary = await adapter._safe_sync_slash_commands()\n+\n+    assert summary[\"created\"] == 2\n+    assert fake_http.upsert_global_command.await_count == 2\n+    assert sleeps == [1.25]\n+\n+\n @pytest.mark.asyncio\n async def test_safe_sync_reads_permission_attrs_from_existing_command():\n     \"\"\"Regression: AppCommand.to_dict() in discord.py does NOT include"]},{"file":"tests/hermes_cli/test_gateway_service.py","hunks":["@@ -2,6 +2,7 @@\n \n import os\n import pwd\n+import subprocess\n from pathlib import Path\n from types import SimpleNamespace\n ","@@ -90,6 +91,13 @@ def test_systemd_restart_refreshes_outdated_unit(self, tmp_path, monkeypatch):\n         monkeypatch.setattr(gateway_cli, \"generate_systemd_unit\", lambda system=False, run_as_user=None: \"new unit\\n\")\n \n         calls = []\n+        monkeypatch.setattr(\"gateway.status.get_running_pid\", lambda: None)\n+        monkeypatch.setattr(gateway_cli, \"_recover_pending_systemd_restart\", lambda system=False, previous_pid=None: False)\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_wait_for_systemd_service_restart\",\n+            lambda system=False, previous_pid=None: calls.append((\"wait\", system, previous_pid)) or True,\n+        )\n \n         def fake_run(cmd, check=True, **kwargs):\n             calls.append(cmd)","@@ -100,11 +108,12 @@ def fake_run(cmd, check=True, **kwargs):\n         gateway_cli.systemd_restart()\n \n         assert unit_path.read_text(encoding=\"utf-8\") == \"new unit\\n\"\n-        assert calls[:4] == [\n+        assert calls[:5] == [\n             [\"systemctl\", \"--user\", \"daemon-reload\"],\n-            [\"systemctl\", \"--user\", \"show\", gateway_cli.get_service_name(), \"--no-pager\", \"--property\", \"ActiveState,SubState,Result,ExecMainStatus\"],\n+            [\"systemctl\", \"--user\", \"show\", gateway_cli.get_service_name(), \"--no-pager\", \"--property\", \"ActiveState,SubState,Result,ExecMainStatus,MainPID\"],\n             [\"systemctl\", \"--user\", \"reset-failed\", gateway_cli.get_service_name()],\n-            [\"systemctl\", \"--user\", \"reload-or-restart\", gateway_cli.get_service_name()],\n+            [\"systemctl\", \"--user\", \"restart\", gateway_cli.get_service_name()],\n+            (\"wait\", False, None),\n         ]\n \n     def test_systemd_stop_marks_running_gateway_as_planned_stop(self, monkeypatch):","@@ -611,62 +620,141 @@ def fake_run(*args, **kwargs):\n         assert gateway_cli._is_service_running() is False\n \n class TestGatewaySystemServiceRouting:\n-    def test_systemd_restart_self_requests_graceful_restart_and_waits(self, monkeypatch, capsys):\n+    def test_systemd_restart_gracefully_restarts_running_service_and_waits(self, monkeypatch, capsys):\n         calls = []\n \n         monkeypatch.setattr(gateway_cli, \"_select_systemd_scope\", lambda system=False: False)\n         monkeypatch.setattr(gateway_cli, \"_require_service_installed\", lambda action, system=False: None)\n         monkeypatch.setattr(gateway_cli, \"refresh_systemd_unit_if_needed\", lambda system=False: calls.append((\"refresh\", system)))\n+        monkeypatch.setattr(gateway_cli, \"_get_restart_drain_timeout\", lambda: 12.0)\n         monkeypatch.setattr(\n             \"gateway.status.get_running_pid\",\n             lambda: 654,\n         )\n         monkeypatch.setattr(\n             gateway_cli,\n-            \"_request_gateway_self_restart\",\n-            lambda pid: calls.append((\"self\", pid)) or True,\n+            \"_graceful_restart_via_sigusr1\",\n+            lambda pid, timeout: calls.append((\"graceful\", pid, timeout)) or True,\n         )\n \n-        # Simulate: old process dies immediately, new process becomes active\n-        kill_call_count = [0]\n-        def fake_kill(pid, sig):\n-            kill_call_count[0] += 1\n-            if kill_call_count[0] >= 2:  # first call checks, second = dead\n-                raise ProcessLookupError()\n-        monkeypatch.setattr(os, \"kill\", fake_kill)\n-\n-        # Simulate systemctl reset-failed/start followed by an active unit\n-        new_pid = [None]\n+        # Simulate systemctl reset-failed/restart followed by an active unit.\n+        # A plain start does not break systemd's auto-restart timer once the\n+        # old gateway has exited with the planned restart code.\n         def fake_subprocess_run(cmd, **kwargs):\n             if \"reset-failed\" in cmd:\n                 calls.append((\"reset-failed\", cmd))\n                 return SimpleNamespace(stdout=\"\", returncode=0)\n-            if \"start\" in cmd:\n-                calls.append((\"start\", cmd))\n+            if \"restart\" in cmd:\n+                calls.append((\"restart\", cmd))\n                 return SimpleNamespace(stdout=\"\", returncode=0)\n-            if \"show\" in cmd:\n-                new_pid[0] = 999\n-                return SimpleNamespace(\n-                    stdout=\"ActiveState=active\\nSubState=running\\nResult=success\\nExecMainStatus=0\\n\",\n-                    returncode=0,\n-                )\n             raise AssertionError(f\"Unexpected systemctl call: {cmd}\")\n \n         monkeypatch.setattr(gateway_cli.subprocess, \"run\", fake_subprocess_run)\n-        # get_running_pid returns new PID after restart\n-        pid_calls = [0]\n-        def fake_get_pid():\n-            pid_calls[0] += 1\n-            return 999 if pid_calls[0] > 1 else 654\n-        monkeypatch.setattr(\"gateway.status.get_running_pid\", fake_get_pid)\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_wait_for_systemd_service_restart\",\n+            lambda system=False, previous_pid=None: calls.append((\"wait\", system, previous_pid)) or True,\n+        )\n \n         gateway_cli.systemd_restart()\n \n-        assert (\"self\", 654) in calls\n+        assert (\"graceful\", 654, 17.0) in calls\n         assert any(call[0] == \"reset-failed\" for call in calls)\n-        assert any(call[0] == \"start\" for call in calls)\n+        assert any(call[0] == \"restart\" for call in calls)\n+        assert (\"wait\", False, 654) in calls\n         out = capsys.readouterr().out.lower()\n-        assert \"restarted\" in out\n+        assert \"restarting gracefully\" in out\n+\n+    def test_systemd_restart_uses_systemd_main_pid_when_pid_file_is_missing(self, monkeypatch, capsys):\n+        calls = []\n+\n+        monkeypatch.setattr(gateway_cli, \"_select_systemd_scope\", lambda system=False: False)\n+        monkeypatch.setattr(gateway_cli, \"_require_service_installed\", lambda action, system=False: None)\n+        monkeypatch.setattr(gateway_cli, \"refresh_systemd_unit_if_needed\", lambda system=False: None)\n+        monkeypatch.setattr(gateway_cli, \"_get_restart_drain_timeout\", lambda: 10.0)\n+        monkeypatch.setattr(\"gateway.status.get_running_pid\", lambda: None)\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_read_systemd_unit_properties\",\n+            lambda system=False: {\n+                \"ActiveState\": \"active\",\n+                \"SubState\": \"running\",\n+                \"Result\": \"success\",\n+                \"ExecMainStatus\": \"0\",\n+                \"MainPID\": \"777\",\n+            },\n+        )\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_graceful_restart_via_sigusr1\",\n+            lambda pid, timeout: calls.append((\"graceful\", pid, timeout)) or True,\n+        )\n+        monkeypatch.setattr(gateway_cli, \"_run_systemctl\", lambda args, **kwargs: calls.append(args) or SimpleNamespace(stdout=\"\", returncode=0))\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_wait_for_systemd_service_restart\",\n+            lambda system=False, previous_pid=None: calls.append((\"wait\", system, previous_pid)) or True,\n+        )\n+\n+        gateway_cli.systemd_restart()\n+\n+        assert (\"graceful\", 777, 15.0) in calls\n+        assert (\"wait\", False, 777) in calls\n+        assert \"restarting gracefully (pid 777)\" in capsys.readouterr().out.lower()\n+\n+    def test_wait_for_systemd_restart_waits_for_runtime_running(self, monkeypatch, capsys):\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_read_systemd_unit_properties\",\n+            lambda system=False: {\n+                \"ActiveState\": \"active\",\n+                \"SubState\": \"running\",\n+                \"Result\": \"success\",\n+                \"ExecMainStatus\": \"0\",\n+                \"MainPID\": \"999\",\n+            },\n+        )\n+        monkeypatch.setattr(\"gateway.status.get_running_pid\", lambda: None)\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_gateway_runtime_status_for_pid\",\n+            lambda pid: {\"pid\": pid, \"gateway_state\": \"running\"},\n+        )\n+\n+        assert gateway_cli._wait_for_systemd_service_restart(previous_pid=777, timeout=0.1) is True\n+        assert \"restarted (pid 999)\" in capsys.readouterr().out.lower()\n+\n+    def test_systemd_restart_reports_start_limit_hit(self, monkeypatch, capsys):\n+        calls = []\n+\n+        monkeypatch.setattr(gateway_cli, \"_select_systemd_scope\", lambda system=False: False)\n+        monkeypatch.setattr(gateway_cli, \"_require_service_installed\", lambda action, system=False: None)\n+        monkeypatch.setattr(gateway_cli, \"refresh_systemd_unit_if_needed\", lambda system=False: None)\n+        monkeypatch.setattr(\"gateway.status.get_running_pid\", lambda: None)\n+        monkeypatch.setattr(gateway_cli, \"_recover_pending_systemd_restart\", lambda system=False, previous_pid=None: False)\n+\n+        def fake_run_systemctl(args, **kwargs):\n+            calls.append(args)\n+            if args[0] == \"show\":\n+                return SimpleNamespace(stdout=\"ActiveState=inactive\\nSubState=dead\\nResult=success\\nExecMainStatus=0\\nMainPID=0\\n\", stderr=\"\", returncode=0)\n+            if args[0] == \"reset-failed\":\n+                return SimpleNamespace(stdout=\"\", stderr=\"\", returncode=0)\n+            if args[0] == \"restart\":\n+                raise subprocess.CalledProcessError(\n+                    1,\n+                    [\"systemctl\", \"--user\", *args],\n+                    stderr=\"Job failed. See result 'start-limit-hit'.\",\n+                )\n+            raise AssertionError(f\"Unexpected args: {args}\")\n+\n+        monkeypatch.setattr(gateway_cli, \"_run_systemctl\", fake_run_systemctl)\n+\n+        gateway_cli.systemd_restart()\n+\n+        assert [\"restart\", gateway_cli.get_service_name()] in calls\n+        out = capsys.readouterr().out.lower()\n+        assert \"rate-limited by systemd\" in out\n+        assert \"reset-failed\" in out\n \n     def test_systemd_restart_recovers_failed_planned_restart(self, monkeypatch, capsys):\n         monkeypatch.setattr(gateway_cli, \"_select_systemd_scope\", lambda system=False: False)","@@ -711,6 +799,11 @@ def fake_subprocess_run(cmd, **kwargs):\n             \"gateway.status.get_running_pid\",\n             lambda: 999 if started[\"value\"] else None,\n         )\n+        monkeypatch.setattr(\n+            gateway_cli,\n+            \"_gateway_runtime_status_for_pid\",\n+            lambda pid: {\"pid\": pid, \"gateway_state\": \"running\"},\n+        )\n \n         gateway_cli.systemd_restart()\n \n"]}]},{"kind":"pr","number":29619,"title":"fix(gateway): respawn via Python watcher so /restart doesn't SIGHUP new gateway (#29603)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":29977,"title":"fix(tools): serialize concurrent ddgs web_search calls to prevent libcurl deadlock","url":null,"state":"closed","close_reason":{"category":"stale_abandoned","implication":"active"},"merged":false},{"kind":"pr","number":36799,"title":"fix(ddgs): add 30s wall-clock timeout to prevent indefinite search hangs","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52118","number":52118},{"kind":"pr","value":"#36776","number":36776},{"kind":"pr","value":"#37422","number":37422}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":36897,"title":"Fix cron, CLI, doctor, and Termux regressions","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"b94397fe7","number":0},{"kind":"commit","value":"6b21a935a","number":0},{"kind":"pr","value":"#56168","number":56168},{"kind":"pr","value":"#36815","number":36815},{"kind":"pr","value":"#11336","number":11336},{"kind":"pr","value":"#36867","number":36867},{"kind":"pr","value":"#36771","number":36771},{"kind":"pr","value":"#29603","number":29603},{"kind":"pr","value":"#36287","number":36287}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":37422,"title":"fix(ddgs): add 30s overall timeout to prevent indefinite search hang","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52118","number":52118}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":38714,"title":"fix(plugins): add 30s overall timeout to DDGS web search (#36776)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#52118","number":52118},{"kind":"pr","value":"#36776","number":36776},{"kind":"pr","value":"#37422","number":37422}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":45380,"title":"fix(gateway): hard-exit launchd service restarts after teardown","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"cde3ca4ebf59e42a422c2239d917b5124a59ec5e","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":45390,"title":"[verified] Avoid hung detached /restart watcher on launchd","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"abc3662bf6076045e4d4dc1e14a74cb35d69b86e","number":0},{"kind":"commit","value":"e860a40e14b1e936ab2e5ad646546380ad0a4e57","number":0}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":52118,"title":"fix(ddgs): bound DuckDuckGo search with a wall-clock timeout (#36776)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/web/ddgs/provider.py","tests/tools/test_web_providers_ddgs.py"]},{"kind":"pr","number":66897,"title":"fix(gateway): thread shutdown watchdog + loop heartbeat for frozen SIGTERM drain (#66892)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#66955","number":66955}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":66918,"title":"fix(gateway): thread-based shutdown watchdog for drain freeze recovery (#66892)","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":66955,"title":"fix(gateway): thread shutdown watchdog + loop heartbeat for frozen SIGTERM drain (#66892)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","gateway/shutdown_watchdog.py","tests/gateway/test_shutdown_watchdog.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -1917,6 +1917,12 @@ def _platform_has_bot_credential(platform: \"Platform\", platform_config: \"Platfor\n     merge_pending_message_event,\n     utf16_len,\n )\n+from gateway.shutdown_watchdog import (\n+    DEFAULT_HEARTBEAT_INTERVAL_S,\n+    arm_shutdown_watchdog,\n+    loop_heartbeat_forever,\n+    resolve_shutdown_watchdog_delay,\n+)\n from gateway.restart import (\n     DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,\n     GATEWAY_FATAL_CONFIG_EXIT_CODE,","@@ -2972,6 +2978,12 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew\n     _session_model_overrides: Dict[str, Dict[str, str]] = {}\n     _session_reasoning_overrides: Dict[str, Dict[str, Any]] = {}\n     _startup_restore_in_progress: bool = False\n+    # Loop-liveness heartbeat / shutdown-watchdog handles (#66892). Class-level\n+    # defaults so partial construction in tests doesn't blow up on access; the\n+    # real values are set in __init__ / start() / stop().\n+    _loop_heartbeat_task: Optional[\"asyncio.Task\"] = None\n+    _gateway_started_at: float = 0.0\n+    _shutdown_watchdog_done: Optional[\"threading.Event\"] = None\n \n     def __init__(self, config: Optional[GatewayConfig] = None):\n         global _gateway_runner_ref","@@ -3297,6 +3309,12 @@ def __init__(self, config: Optional[GatewayConfig] = None):\n         # Track background tasks to prevent garbage collection mid-execution\n         self._background_tasks: set = set()\n \n+        # Event-loop liveness heartbeat (#66892): rewritten every 30s while\n+        # the loop is dispatching. External supervisors use the file mtime /\n+        # updated_at to distinguish \"process alive\" from \"loop frozen\".\n+        self._gateway_started_at: float = time.time()\n+        self._loop_heartbeat_task: Optional[asyncio.Task] = None\n+\n         # scale-to-zero (Phase 0, F13): gateway-scoped \"last inbound seen\" clock.\n         # There is no such clock today (only a per-agent _last_activity_ts), so the\n         # idle predicate needs this. Stamped in _handle_message (the single inbound","@@ -7535,7 +7553,27 @@ async def start(self) -> bool:\n \n         self._running = True\n         self._update_runtime_status(\"running\")\n-        \n+\n+        # Loop-liveness heartbeat (#66892): an asyncio task so a frozen loop\n+        # stops refreshing ``state/gateway.heartbeat``. Cancelled with the\n+        # other background tasks during stop(). Best-effort — a liveness probe\n+        # must never be able to abort startup.\n+        try:\n+            _existing_hb = getattr(self, \"_loop_heartbeat_task\", None)\n+            if _existing_hb is None or _existing_hb.done():\n+                self._loop_heartbeat_task = asyncio.create_task(\n+                    loop_heartbeat_forever(\n+                        interval_s=DEFAULT_HEARTBEAT_INTERVAL_S,\n+                        start_time=getattr(self, \"_gateway_started_at\", 0.0),\n+                    )\n+                )\n+                _bg = getattr(self, \"_background_tasks\", None)\n+                if _bg is not None:\n+                    _bg.add(self._loop_heartbeat_task)\n+                    self._loop_heartbeat_task.add_done_callback(_bg.discard)\n+        except Exception:\n+            logger.debug(\"Failed to start gateway loop heartbeat\", exc_info=True)\n+\n         # Emit gateway:startup hook\n         hook_count = len(self.hooks.loaded_hooks)\n         if hook_count:","@@ -8499,11 +8537,57 @@ def _kill_tool_subprocesses(phase: str) -> None:\n                 except Exception as _e:\n                     logger.debug(\"cleanup_all_browsers (%s) error: %s\", phase, _e)\n \n+            # Thread-based shutdown watchdog (#66892): asyncio timeouts cannot\n+            # recover a frozen loop. Arm a plain OS thread at the start of\n+            # stop(); if teardown never finishes within drain+grace it dumps\n+            # faulthandler stacks and os._exit so KeepAlive/systemd can revive.\n+            # Skip under pytest so stop()-driving unit tests don't get a\n+            # delayed hard-exit in the worker.\n+            _watchdog_done = threading.Event()\n+            self._shutdown_watchdog_done = _watchdog_done\n+            _stop_started_at_box: dict[str, float] = {}\n+\n+            def _shutdown_watchdog_snapshot() -> dict:\n+                started = _stop_started_at_box.get(\"t\")\n+                return {\n+                    \"restart_requested\": bool(self._restart_requested),\n+                    \"draining\": bool(self._draining),\n+                    \"running\": bool(self._running),\n+                    \"active_agents\": self._running_agent_count(),\n+                    \"active_cron_jobs\": self._active_cron_job_count(),\n+                    \"active_api_runs\": self._active_api_run_count(),\n+                    \"restart_drain_timeout\": self._restart_drain_timeout,\n+                    \"watchdog_delay_s\": resolve_shutdown_watchdog_delay(\n+                        self._restart_drain_timeout\n+                    ),\n+                    \"phase_elapsed_s\": (\n+                        time.monotonic() - started if started is not None else None\n+                    ),\n+                }\n+\n+            if not os.environ.get(\"PYTEST_CURRENT_TEST\"):\n+                arm_shutdown_watchdog(\n+                    resolve_shutdown_watchdog_delay(self._restart_drain_timeout),\n+                    done_event=_watchdog_done,\n+                    snapshot_fn=_shutdown_watchdog_snapshot,\n+                    exit_code=1,\n+                )\n+\n+            try:\n+                await _stop_impl_body(\n+                    _kill_tool_subprocesses,\n+                    _stop_started_at_box,\n+                )\n+            finally:\n+                _watchdog_done.set()\n+\n+        async def _stop_impl_body(_kill_tool_subprocesses, _stop_started_at_box) -> None:\n             logger.info(\n                 \"Stopping gateway%s...\",\n                 \" for restart\" if self._restart_requested else \"\",\n             )\n             _stop_started_at = time.monotonic()\n+            _stop_started_at_box[\"t\"] = _stop_started_at\n \n             def _phase_elapsed() -> float:\n                 return time.monotonic() - _stop_started_at"]},{"file":"gateway/shutdown_watchdog.py","hunks":["@@ -0,0 +1,272 @@\n+\"\"\"Out-of-loop shutdown backstop + event-loop liveness heartbeat (#66892).\n+\n+When the asyncio loop freezes mid-drain, every asyncio-based recovery path is\n+structurally unable to fire: the drain deadline, status rewrites, and forensics\n+all need the same loop that is stuck. launchd/systemd KeepAlive only restarts a\n+*dead* process, so a wedged-but-alive gateway sits as a zombie until manual\n+SIGKILL.\n+\n+This module provides:\n+\n+1. A plain OS-thread shutdown watchdog armed at ``stop()``. If shutdown has not\n+   completed within ``restart_drain_timeout + grace``, it dumps all-thread\n+   stacks via ``faulthandler`` plus a metadata snapshot, then ``os._exit`` so\n+   the service manager can revive the process.\n+2. An event-loop heartbeat file at ``<HERMES_HOME>/state/gateway.heartbeat`` so\n+   external supervision can distinguish \"process alive\" from \"loop frozen\"\n+   (``gateway_state.json`` alone can't — it only rewrites on transitions/turns).\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import asyncio\n+import faulthandler\n+import json\n+import logging\n+import os\n+import sys\n+import threading\n+import time\n+from datetime import datetime, timezone\n+from pathlib import Path\n+from typing import Any, Callable, Dict, Optional\n+\n+from hermes_constants import get_hermes_home\n+from utils import atomic_json_write\n+\n+logger = logging.getLogger(__name__)\n+\n+# Extra leash beyond ``agent.restart_drain_timeout`` so a slow-but-progressing\n+# drain is not cut short. Matches the issue #66892 suggested hardening.\n+DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S = 60.0\n+DEFAULT_HEARTBEAT_INTERVAL_S = 30.0\n+_HEARTBEAT_RELATIVE = (\"state\", \"gateway.heartbeat\")\n+_WATCHDOG_DUMP_RELATIVE = (\"logs\", \"gateway-shutdown-watchdog.log\")\n+\n+\n+def _process_hermes_home() -> Path:\n+    \"\"\"HERMES_HOME for process-level identity files (ignore profile overrides).\"\"\"\n+    val = os.environ.get(\"HERMES_HOME\", \"\").strip()\n+    if val:\n+        return Path(val)\n+    return get_hermes_home()\n+\n+\n+def get_loop_heartbeat_path(home: Optional[Path] = None) -> Path:\n+    \"\"\"Return ``<HERMES_HOME>/state/gateway.heartbeat``.\"\"\"\n+    base = home if home is not None else _process_hermes_home()\n+    return base.joinpath(*_HEARTBEAT_RELATIVE)\n+\n+\n+def get_shutdown_watchdog_dump_path(home: Optional[Path] = None) -> Path:\n+    \"\"\"Return the faulthandler / metadata dump path for a fired watchdog.\"\"\"\n+    base = home if home is not None else _process_hermes_home()\n+    return base.joinpath(*_WATCHDOG_DUMP_RELATIVE)\n+\n+\n+def write_loop_heartbeat(\n+    *,\n+    pid: Optional[int] = None,\n+    start_time: Optional[float] = None,\n+    home: Optional[Path] = None,\n+    extra: Optional[Dict[str, Any]] = None,\n+) -> Path:\n+    \"\"\"Atomically rewrite the loop-liveness heartbeat file.\n+\n+    ``start_time`` is the gateway process start (``time.time()`` epoch seconds)\n+    so supervisors can detect PID reuse. Best-effort — never raises.\n+    \"\"\"\n+    path = get_loop_heartbeat_path(home)\n+    payload: Dict[str, Any] = {\n+        \"pid\": int(pid if pid is not None else os.getpid()),\n+        \"updated_at\": datetime.now(timezone.utc).isoformat(),\n+        \"monotonic\": time.monotonic(),\n+    }\n+    if start_time is not None:\n+        payload[\"start_time\"] = float(start_time)\n+    if extra:\n+        payload.update(extra)\n+    try:\n+        atomic_json_write(path, payload, indent=None)\n+    except Exception:\n+        logger.debug(\"Failed to write gateway loop heartbeat\", exc_info=True)\n+    return path\n+\n+\n+def resolve_shutdown_watchdog_delay(\n+    drain_timeout: float,\n+    *,\n+    grace_s: float = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S,\n+) -> float:\n+    \"\"\"Return the wall-clock leash for the shutdown watchdog thread.\"\"\"\n+    try:\n+        drain = max(float(drain_timeout), 0.0)\n+    except (TypeError, ValueError):\n+        drain = 0.0\n+    try:\n+        grace = max(float(grace_s), 0.0)\n+    except (TypeError, ValueError):\n+        grace = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S\n+    return drain + grace\n+\n+\n+def _write_watchdog_dump(\n+    dump_path: Path,\n+    *,\n+    delay_s: float,\n+    snapshot: Optional[Dict[str, Any]],\n+) -> None:\n+    \"\"\"Best-effort faulthandler + metadata dump before hard-exit.\"\"\"\n+    try:\n+        dump_path.parent.mkdir(parents=True, exist_ok=True)\n+    except OSError:\n+        return\n+\n+    header = {\n+        \"event\": \"shutdown_watchdog_fired\",\n+        \"pid\": os.getpid(),\n+        \"delay_s\": delay_s,\n+        \"fired_at\": datetime.now(timezone.utc).isoformat(),\n+        \"snapshot\": snapshot or {},\n+    }\n+    try:\n+        with open(dump_path, \"a\", encoding=\"utf-8\") as fh:\n+            fh.write(json.dumps(header, default=str) + \"\\n\")\n+            fh.write(\"--- faulthandler dump (all threads) ---\\n\")\n+            fh.flush()\n+            try:\n+                faulthandler.dump_traceback(file=fh, all_threads=True)\n+            except Exception:\n+                fh.write(\"(faulthandler.dump_traceback failed)\\n\")\n+            fh.write(\"--- end dump ---\\n\")\n+            fh.flush()\n+    except Exception:\n+        pass\n+\n+    # Also dump to stderr so journald/launchd capture it even if the file\n+    # write failed (wedged disk was one of the #66892 hypotheses).\n+    try:\n+        sys.stderr.write(\n+            f\"Gateway shutdown watchdog fired after {delay_s:.0f}s \"\n+            f\"(pid={os.getpid()}); dumping all thread stacks.\\n\"\n+        )\n+        sys.stderr.flush()\n+        faulthandler.dump_traceback(all_threads=True)\n+    except Exception:\n+        pass\n+\n+\n+def arm_shutdown_watchdog(\n+    delay_s: float,\n+    *,\n+    done_event: Optional[threading.Event] = None,\n+    snapshot_fn: Optional[Callable[[], Dict[str, Any]]] = None,\n+    exit_code: int = 1,\n+    dump_path: Optional[Path] = None,\n+    name: str = \"gateway-shutdown-watchdog\",\n+) -> threading.Event:\n+    \"\"\"Arm a daemon-thread hard-exit backstop for a wedged shutdown path.\n+\n+    If ``done_event`` is set before ``delay_s`` elapses, the thread exits\n+    quietly (normal / progressing shutdown completed). Otherwise it dumps\n+    diagnostics and calls ``os._exit(exit_code)``.\n+\n+    Never raises. Returns the ``done_event`` (creating one when omitted) so\n+    the caller can disarm on successful completion.\n+    \"\"\"\n+    done = done_event if done_event is not None else threading.Event()\n+    try:\n+        delay = max(float(delay_s), 0.0)\n+    except (TypeError, ValueError):\n+        delay = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S\n+\n+    if delay <= 0:\n+        return done\n+\n+    def _watchdog() -> None:\n+        # Wait with interruptible chunks so a late disarm doesn't need the\n+        # full remaining sleep to observe done_event.\n+        deadline = time.monotonic() + delay\n+        while time.monotonic() < deadline:\n+            remaining = deadline - time.monotonic()\n+            if done.wait(timeout=min(remaining, 1.0)):\n+                return\n+        if done.is_set():\n+            return\n+\n+        snapshot: Optional[Dict[str, Any]] = None\n+        if snapshot_fn is not None:\n+            try:\n+                snapshot = snapshot_fn()\n+            except Exception as exc:\n+                snapshot = {\"snapshot_error\": repr(exc)}\n+\n+        target = dump_path if dump_path is not None else get_shutdown_watchdog_dump_path()\n+        _write_watchdog_dump(target, delay_s=delay, snapshot=snapshot)\n+\n+        try:\n+            logger.critical(\n+                \"Shutdown watchdog fired after %.0fs — forcing process exit \"\n+                \"(asyncio drain path appears wedged; see %s)\",\n+                delay,\n+                target,\n+            )\n+        except Exception:\n+            pass\n+\n+        for stream in (sys.stdout, sys.stderr):\n+            try:\n+                stream.flush()\n+            except Exception:\n+                pass\n+        # Mirror _exit_after_graceful_shutdown: release PID file + runtime\n+        # lock BEFORE the log drain (locks must never be stranded), then\n+        # drain the async log queue so the logger.critical above actually\n+        # reaches the file before os._exit bypasses atexit. (#66892)\n+        try:\n+            from gateway.status import remove_pid_file, release_gateway_runtime_lock\n+            remove_pid_file()\n+            release_gateway_runtime_lock()\n+        except Exception:\n+            pass\n+        try:\n+            from hermes_logging import drain_log_queue\n+            drain_log_queue(timeout=1.0)\n+        except Exception:\n+            pass\n+        os._exit(exit_code)\n+\n+    try:\n+        threading.Thread(target=_watchdog, daemon=True, name=name).start()\n+    except Exception:\n+        logger.debug(\"Failed to arm shutdown watchdog\", exc_info=True)\n+    return done\n+\n+\n+async def loop_heartbeat_forever(\n+    *,\n+    interval_s: float = DEFAULT_HEARTBEAT_INTERVAL_S,\n+    start_time: Optional[float] = None,\n+    home: Optional[Path] = None,\n+    should_continue: Optional[Callable[[], bool]] = None,\n+) -> None:\n+    \"\"\"Rewrite the loop heartbeat file on a cadence until cancelled / gated off.\n+\n+    Runs as an asyncio task on the gateway loop — if the loop freezes, this\n+    task stops and the file mtime/updated_at goes stale for external monitors.\n+    \"\"\"\n+    try:\n+        interval = max(float(interval_s), 1.0)\n+    except (TypeError, ValueError):\n+        interval = DEFAULT_HEARTBEAT_INTERVAL_S\n+\n+    # Immediate first write so monitors see a fresh file as soon as the\n+    # gateway is running, not after the first interval.\n+    write_loop_heartbeat(start_time=start_time, home=home)\n+    while True:\n+        if should_continue is not None and not should_continue():\n+            return\n+        await asyncio.sleep(interval)\n+        if should_continue is not None and not should_continue():\n+            return\n+        write_loop_heartbeat(start_time=start_time, home=home)"]},{"file":"tests/gateway/test_shutdown_watchdog.py","hunks":["@@ -0,0 +1,150 @@\n+\"\"\"Shutdown watchdog + loop heartbeat coverage for #66892.\n+\n+The drain path is asyncio-based; a frozen loop makes every asyncio timeout\n+structurally unable to fire. These tests pin the out-of-loop backstop\n+(thread watchdog) and the loop-liveness heartbeat file contract.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import asyncio\n+import json\n+import threading\n+import time\n+from unittest.mock import patch\n+\n+import pytest\n+\n+from gateway.shutdown_watchdog import (\n+    DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S,\n+    arm_shutdown_watchdog,\n+    get_loop_heartbeat_path,\n+    get_shutdown_watchdog_dump_path,\n+    loop_heartbeat_forever,\n+    resolve_shutdown_watchdog_delay,\n+    write_loop_heartbeat,\n+)\n+\n+def test_resolve_shutdown_watchdog_delay_adds_grace():\n+    assert resolve_shutdown_watchdog_delay(180) == 180 + DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S\n+    assert resolve_shutdown_watchdog_delay(0) == DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S\n+    assert resolve_shutdown_watchdog_delay(\"bad\") == DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S\n+    assert resolve_shutdown_watchdog_delay(10, grace_s=5) == 15.0\n+\n+\n+def test_write_loop_heartbeat_atomic_json(tmp_path, monkeypatch):\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+    path = write_loop_heartbeat(pid=4242, start_time=100.5, home=tmp_path)\n+    assert path == tmp_path / \"state\" / \"gateway.heartbeat\"\n+    assert path.is_file()\n+    data = json.loads(path.read_text(encoding=\"utf-8\"))\n+    assert data[\"pid\"] == 4242\n+    assert data[\"start_time\"] == 100.5\n+    assert \"updated_at\" in data\n+    assert \"monotonic\" in data\n+    assert get_loop_heartbeat_path(tmp_path) == path\n+\n+\n+def test_arm_shutdown_watchdog_disarm_before_fire(tmp_path):\n+    done = threading.Event()\n+    exited = []\n+\n+    def fake_exit(code):\n+        exited.append(code)\n+        raise _ExitCalled(code)\n+\n+    with patch(\"gateway.shutdown_watchdog.os._exit\", side_effect=fake_exit):\n+        arm_shutdown_watchdog(\n+            0.4,\n+            done_event=done,\n+            dump_path=tmp_path / \"dump.log\",\n+            exit_code=7,\n+        )\n+        time.sleep(0.1)\n+        done.set()\n+        time.sleep(0.5)\n+\n+    assert exited == []\n+\n+\n+def test_arm_shutdown_watchdog_fires_with_dump_and_exit(tmp_path):\n+    done = threading.Event()\n+    fired = threading.Event()\n+    dump = tmp_path / \"logs\" / \"watchdog.log\"\n+    snapshot_calls = []\n+    exit_codes = []\n+\n+    def snapshot():\n+        snapshot_calls.append(1)\n+        return {\"active_agents\": 1, \"draining\": True}\n+\n+    def fake_exit(code):\n+        exit_codes.append(code)\n+        fired.set()\n+\n+    with patch(\"gateway.shutdown_watchdog.os._exit\", side_effect=fake_exit):\n+        arm_shutdown_watchdog(\n+            0.15,\n+            done_event=done,\n+            snapshot_fn=snapshot,\n+            dump_path=dump,\n+            exit_code=9,\n+        )\n+        assert fired.wait(timeout=5.0), \"watchdog did not fire\"\n+\n+    assert exit_codes == [9]\n+    assert snapshot_calls == [1]\n+    assert dump.is_file()\n+    text = dump.read_text(encoding=\"utf-8\")\n+    assert \"shutdown_watchdog_fired\" in text\n+    assert \"faulthandler dump\" in text\n+    assert get_shutdown_watchdog_dump_path(tmp_path).name == \"gateway-shutdown-watchdog.log\"\n+\n+\n[redacted:email]\n+async def test_loop_heartbeat_rewrites_until_cancelled(tmp_path):\n+    path = get_loop_heartbeat_path(tmp_path)\n+    task = asyncio.create_task(\n+        loop_heartbeat_forever(\n+            interval_s=0.05,\n+            start_time=12.0,\n+            home=tmp_path,\n+        )\n+    )\n+    try:\n+        # First write is immediate.\n+        for _ in range(50):\n+            if path.is_file():\n+                break\n+            await asyncio.sleep(0.02)\n+        assert path.is_file()\n+        first = path.read_text(encoding=\"utf-8\")\n+        assert json.loads(first)[\"start_time\"] == 12.0\n+\n+        # Poll until a refresh lands (monotonic / updated_at change).\n+        second = first\n+        for _ in range(100):\n+            await asyncio.sleep(0.03)\n+            second = path.read_text(encoding=\"utf-8\")\n+            if second != first:\n+                break\n+        assert second != first\n+        assert json.loads(second)[\"start_time\"] == 12.0\n+    finally:\n+        task.cancel()\n+        with pytest.raises(asyncio.CancelledError):\n+            await task\n+\n+\n+def test_gateway_runner_exposes_shutdown_watchdog_state():\n+    \"\"\"Attrs used by stop()/start() exist after normal construction hooks.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    runner = object.__new__(GatewayRunner)\n+    runner._shutdown_watchdog_done = threading.Event()\n+    runner._loop_heartbeat_task = None\n+    runner._gateway_started_at = time.time()\n+    assert not runner._shutdown_watchdog_done.is_set()\n+    runner._shutdown_watchdog_done.set()\n+    assert runner._shutdown_watchdog_done.is_set()\n+    assert runner._loop_heartbeat_task is None\n"]}]},{"kind":"pr","number":68444,"title":"fix(web/ddgs): isolate search in a process so GIL hangs cannot freeze Hermes (#68096)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/web/ddgs/_search_worker.py","plugins/web/ddgs/provider.py","tests/tools/test_web_providers_ddgs.py"],"fix_hunks":[{"file":"plugins/web/ddgs/_search_worker.py","hunks":["@@ -0,0 +1,113 @@\n+\"\"\"DDGS search child-process entrypoint (#68096).\n+\n+Invoked as ``python plugins/web/ddgs/_search_worker.py`` (script path from the\n+parent provider). Reads one JSON request from stdin, writes one JSON envelope\n+to stdout, then exits.\n+\n+Request::\n+    {\"query\": str, \"safe_limit\": int}\n+\n+Envelope::\n+    {\"ok\": true, \"results\": [...]}\n+    {\"ok\": false, \"error\": str}\n+\n+Optional test hooks (only when ``HERMES_DDGS_ALLOW_TEST_HOOKS=1``)::\n+    {\"query\": ..., \"safe_limit\": ..., \"test_hook\": \"sleep\"|\"gil\"|\"success\"|\"error\"|\"empty\"}\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import json\n+import os\n+import sys\n+import time\n+\n+\n+def _hold_gil(secs: int) -> None:\n+    \"\"\"Block in a foreign call that keeps the GIL (ctypes.PyDLL).\n+\n+    Mirrors native ``primp`` holding the interpreter lock. ``PyDLL`` (unlike\n+    ``CDLL``/``WinDLL``) does not release the GIL around the call.\n+    \"\"\"\n+    import ctypes\n+\n+    if sys.platform == \"win32\":\n+        lib = ctypes.PyDLL(\"kernel32\")\n+        lib.Sleep.argtypes = [ctypes.c_uint]\n+        lib.Sleep(int(secs * 1000))\n+        return\n+\n+    lib = ctypes.PyDLL(None)\n+    try:\n+        sleep = lib.sleep\n+    except AttributeError:  # pragma: no cover — macOS libSystem fallback\n+        sleep = ctypes.PyDLL(\"/usr/lib/libSystem.B.dylib\").sleep\n+    sleep.argtypes = [ctypes.c_uint]\n+    sleep(int(secs))\n+\n+\n+def _run_test_hook(hook: str) -> dict:\n+    if hook == \"sleep\":\n+        time.sleep(30)\n+        return {\"ok\": False, \"error\": \"sleep hook returned unexpectedly\"}\n+    if hook == \"gil\":\n+        _hold_gil(30)\n+        return {\"ok\": False, \"error\": \"gil hook returned unexpectedly\"}\n+    if hook == \"success\":\n+        return {\n+            \"ok\": True,\n+            \"results\": [\n+                {\n+                    \"title\": \"Hit\",\n+                    \"url\": \"https://example.com\",\n+                    \"description\": \"body\",\n+                    \"position\": 1,\n+                }\n+            ],\n+        }\n+    if hook == \"empty\":\n+        return {\"ok\": True, \"results\": []}\n+    if hook == \"error\":\n+        return {\"ok\": False, \"error\": \"RuntimeError: boom\"}\n+    return {\"ok\": False, \"error\": f\"unknown test_hook: {hook!r}\"}\n+\n+\n+def _write_envelope(envelope: dict) -> None:\n+    json.dump(envelope, sys.stdout)\n+    sys.stdout.flush()\n+\n+\n+def main() -> int:\n+    try:\n+        request = json.load(sys.stdin)\n+    except Exception as exc:  # noqa: BLE001\n+        _write_envelope({\"ok\": False, \"error\": f\"invalid request: {exc}\"})\n+        return 2\n+\n+    hook = request.get(\"test_hook\")\n+    if hook:\n+        if os.environ.get(\"HERMES_DDGS_ALLOW_TEST_HOOKS\") != \"1\":\n+            _write_envelope(\n+                {\"ok\": False, \"error\": \"test_hook refused (hooks not enabled)\"}\n+            )\n+            return 3\n+        envelope = _run_test_hook(str(hook))\n+        _write_envelope(envelope)\n+        return 0 if envelope.get(\"ok\") else 1\n+\n+    query = str(request.get(\"query\") or \"\")\n+    safe_limit = max(1, int(request.get(\"safe_limit\") or 1))\n+    try:\n+        # Import inside main so script startup stays light / patchable.\n+        from plugins.web.ddgs.provider import _run_ddgs_search\n+\n+        results = _run_ddgs_search(query, safe_limit)\n+        _write_envelope({\"ok\": True, \"results\": results})\n+        return 0\n+    except Exception as exc:  # noqa: BLE001\n+        _write_envelope({\"ok\": False, \"error\": f\"{type(exc).__name__}: {exc}\"})\n+        return 1\n+\n+\n+if __name__ == \"__main__\":\n+    raise SystemExit(main())"]},{"file":"plugins/web/ddgs/provider.py","hunks":["@@ -8,13 +8,24 @@\n The ``ddgs`` package is an optional dependency. ``is_available()`` reflects\n whether the package is importable; the plugin still registers either way so\n ``hermes tools`` can prompt the user to install it.\n+\n+Isolation note (#68096): ``ddgs``/``primp`` can block inside native code while\n+holding the Python GIL. A ``ThreadPoolExecutor`` + ``future.result(timeout=…)``\n+cap (see #52118) cannot fire in that state — the waiter never reacquires the\n+GIL — so the whole Hermes process freezes through Ctrl+C/SIGTERM. Each search\n+therefore runs in a disposable child process the parent can terminate/kill.\n \"\"\"\n \n from __future__ import annotations\n \n-import concurrent.futures as _cf\n+import concurrent.futures as cf\n+import json\n import logging\n-from typing import Any, Dict\n+import os\n+import subprocess\n+import sys\n+import time\n+from typing import Any, Dict, Optional\n \n from agent.web_search_provider import WebSearchProvider\n ","@@ -23,18 +34,28 @@\n # Overall wall-clock cap for a single ddgs search. The DDGS constructor's\n # ``timeout`` only bounds individual HTTP requests; ddgs's multi-engine retry\n # loop has no overall cap, so a slow/rate-limited DuckDuckGo response can hang\n-# the (single, shared) agent loop indefinitely and block every platform\n-# (#36776). Enforce a hard cap here via a worker thread.\n+# the (single, shared) agent loop indefinitely (#36776). Enforce a hard cap\n+# here by killing a disposable worker process (#68096).\n _SEARCH_TIMEOUT_SECS = 30\n \n+# How often the parent polls stdout / interrupt flag while waiting.\n+_POLL_INTERVAL_SECS = 0.1\n+\n+# After terminate(), wait this long before escalating to kill().\n+_TERMINATE_GRACE_SECS = 1.0\n+\n+\n+class _SearchInterrupted(Exception):\n+    \"\"\"Raised when tools.interrupt.is_interrupted() trips during a search wait.\"\"\"\n+\n \n def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]:\n     \"\"\"Run the blocking ddgs query and return normalized hits.\n \n-    Module-level (not a closure) so tests can patch it directly without\n-    spawning a real multi-second worker thread. ``DDGS(timeout=...)`` bounds\n+    Module-level (not a closure) so the child worker can import it and so\n+    tests can patch it for in-process unit tests. ``DDGS(timeout=…)`` bounds\n     each individual HTTP request; the overall wall-clock cap is enforced by\n-    the caller via a future timeout.\n+    the parent via process timeout (#68096).\n     \"\"\"\n     from ddgs import DDGS  # type: ignore\n ","@@ -55,6 +76,192 @@ def _run_ddgs_search(query: str, safe_limit: int) -> list[dict[str, Any]]:\n     return results\n \n \n+# Optional test-only hook name forwarded to the child (see _search_worker.py).\n+# Production search() never sets this.\n+_test_hook: Optional[str] = None\n+\n+# Last worker Popen started by ``_run_ddgs_search_bounded`` (test reap checks).\n+_last_worker_proc: Optional[subprocess.Popen] = None\n+\n+\n+def _plugins_path_entry() -> str:\n+    \"\"\"Return the ``sys.path`` entry that makes ``import plugins`` work.\n+\n+    Prefer the live ``plugins`` package location over counting ``dirname``s from\n+    this file — that stays correct for source checkouts and site-packages.\n+    \"\"\"\n+    try:\n+        import plugins as plugins_pkg\n+\n+        pkg_file = getattr(plugins_pkg, \"__file__\", None)\n+        if pkg_file:\n+            return os.path.dirname(os.path.dirname(os.path.abspath(pkg_file)))\n+    except Exception:  # noqa: BLE001 — fall through to path-walk fallback\n+        pass\n+    return os.path.dirname(\n+        os.path.dirname(\n+            os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n+        )\n+    )\n+\n+\n+def _terminate_and_reap(\n+    proc: Optional[subprocess.Popen],\n+    *,\n+    grace: float = _TERMINATE_GRACE_SECS,\n+) -> None:\n+    \"\"\"Terminate a worker, escalate to kill, and wait so no orphan remains.\n+\n+    Does not close the parent's pipe ends — the caller must finish any\n+    ``communicate()``/reader first. Closing stdout while another thread is\n+    blocked in ``read()`` deadlocks on some platforms.\n+    \"\"\"\n+    if proc is None:\n+        return\n+\n+    def _wait_until_dead(seconds: float) -> bool:\n+        deadline = time.monotonic() + seconds\n+        while time.monotonic() < deadline:\n+            if proc.poll() is not None:\n+                return True\n+            time.sleep(0.05)\n+        return proc.poll() is not None\n+\n+    try:\n+        if proc.poll() is None:\n+            proc.terminate()\n+            _wait_until_dead(grace)\n+        if proc.poll() is None:\n+            proc.kill()\n+            if not _wait_until_dead(grace):\n+                logger.warning(\"DDGS worker pid=%s did not exit after kill\", proc.pid)\n+    except Exception as exc:  # noqa: BLE001 — best-effort cleanup\n+        logger.debug(\"DDGS worker reap error: %s\", exc)\n+\n+\n+def _run_ddgs_search_bounded(query: str, safe_limit: int) -> list[dict[str, Any]]:\n+    \"\"\"Run ``_run_ddgs_search`` in a disposable process with a hard deadline.\n+\n+    The parent never joins the child while it may be inside native code holding\n+    *its* GIL — it only polls a communicator thread and, on timeout/interrupt,\n+    terminates the child OS process. Raises ``TimeoutError``,\n+    ``_SearchInterrupted``, or ``RuntimeError``.\n+    \"\"\"\n+    # Imported lazily so plugin import stays light for ``hermes tools`` probes.\n+    from tools.interrupt import is_interrupted\n+\n+    global _last_worker_proc\n+\n+    request: dict[str, Any] = {\"query\": query, \"safe_limit\": safe_limit}\n+    if _test_hook:\n+        request[\"test_hook\"] = _test_hook\n+\n+    from tools.environments.local import _sanitize_subprocess_env\n+\n+    env = _sanitize_subprocess_env(dict(os.environ))\n+    if _test_hook:\n+        env[\"HERMES_DDGS_ALLOW_TEST_HOOKS\"] = \"1\"\n+\n+    # Running the worker as a script puts ``plugins/web/ddgs/`` on ``sys.path[0]``,\n+    # which breaks ``import plugins...``. Prepend the path entry that makes the\n+    # live ``plugins`` package importable (source tree or site-packages).\n+    child_pythonpath = env.get(\"PYTHONPATH\", \"\")\n+    path_entry = _plugins_path_entry()\n+    if path_entry and path_entry not in child_pythonpath.split(os.pathsep):\n+        env[\"PYTHONPATH\"] = (\n+            path_entry + os.pathsep + child_pythonpath if child_pythonpath else path_entry\n+        )\n+\n+    worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), \"_search_worker.py\")\n+    # Platform-only spawn knobs — stdin/stdout/stderr must stay as explicit\n+    # keyword args on the Popen call so scripts/check_subprocess_stdin.py can\n+    # see them (TUI gateway inherits stdin; #14036).\n+    extra_kwargs: dict[str, Any] = {}\n+    if sys.platform == \"win32\":\n+        # New process group so terminate/kill reach the worker cleanly on Windows.\n+        extra_kwargs[\"creationflags\"] = subprocess.CREATE_NEW_PROCESS_GROUP\n+    else:\n+        # Own session so a hung primp/libcurl grandchild can be reaped with the worker.\n+        extra_kwargs[\"start_new_session\"] = True\n+\n+    proc = subprocess.Popen(\n+        [sys.executable, worker_path],\n+        stdin=subprocess.PIPE,\n+        stdout=subprocess.PIPE,\n+        # DEVNULL avoids the classic deadlock where a chatty child fills the\n+        # stderr pipe buffer while the parent only drains stdout.\n+        stderr=subprocess.DEVNULL,\n+        env=env,\n+        text=True,\n+        **extra_kwargs,\n+    )\n+    _last_worker_proc = proc\n+\n+    # ``communicate`` runs in a side thread so the parent can poll interrupt /\n+    # deadline without blocking. Killing the child unblocks communicate.\n+    pool = cf.ThreadPoolExecutor(max_workers=1)\n+    fut = pool.submit(proc.communicate, json.dumps(request))\n+    timed_out = False\n+    interrupted = False\n+    raw = \"\"\n+    try:\n+        deadline = time.monotonic() + _SEARCH_TIMEOUT_SECS\n+        while True:\n+            if is_interrupted():\n+                interrupted = True\n+                break\n+            remaining = deadline - time.monotonic()\n+            if remaining <= 0:\n+                timed_out = True\n+                break\n+            try:\n+                out, _err = fut.result(timeout=min(_POLL_INTERVAL_SECS, remaining))\n+                raw = out or \"\"\n+                break\n+            except cf.TimeoutError:\n+                continue\n+    finally:\n+        _terminate_and_reap(proc)\n+        # After kill, communicate should return promptly; don't block forever.\n+        if not fut.done():\n+            try:\n+                out, _err = fut.result(timeout=_TERMINATE_GRACE_SECS)\n+                if not raw:\n+                    raw = out or \"\"\n+            except Exception:  # noqa: BLE001\n+                pass\n+        pool.shutdown(wait=False, cancel_futures=True)\n+\n+    if interrupted:\n+        raise _SearchInterrupted(\"DuckDuckGo search interrupted\")\n+    if timed_out:\n+        raise TimeoutError(\n+            f\"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s\"\n+        )\n+\n+    raw = raw.strip()\n+    if not raw:\n+        raise RuntimeError(\n+            f\"DDGS worker exited without a result (code={proc.poll()})\"\n+        )\n+\n+    try:\n+        envelope = json.loads(raw)\n+    except json.JSONDecodeError as exc:\n+        raise RuntimeError(\n+            f\"DDGS worker returned invalid JSON: {raw[:200]!r}\"\n+        ) from exc\n+\n+    if not isinstance(envelope, dict):\n+        raise RuntimeError(f\"DDGS worker returned an invalid envelope: {envelope!r}\")\n+    if envelope.get(\"ok\"):\n+        results = envelope.get(\"results\") or []\n+        if not isinstance(results, list):\n+            raise RuntimeError(\"DDGS worker returned non-list results\")\n+        return results\n+    raise RuntimeError(str(envelope.get(\"error\") or \"DDGS worker failed\"))\n+\n+\n class DDGSWebSearchProvider(WebSearchProvider):\n     \"\"\"DuckDuckGo HTML-scrape search provider.\n ","@@ -94,9 +301,9 @@ def supports_extract(self) -> bool:\n     def search(self, query: str, limit: int = 5) -> Dict[str, Any]:\n         \"\"\"Execute a DuckDuckGo search and return normalized results.\n \n-        The synchronous ``ddgs`` call is run in a worker thread with a hard\n-        wall-clock timeout (``_SEARCH_TIMEOUT_SECS``) so a hung search cannot\n-        block the shared agent loop indefinitely (#36776).\n+        The synchronous ``ddgs`` call runs in a disposable child process with\n+        a hard wall-clock timeout (``_SEARCH_TIMEOUT_SECS``) so a hung native\n+        ``primp`` call cannot freeze the Hermes process (#36776, #68096).\n         \"\"\"\n         try:\n             import ddgs  # type: ignore  # noqa: F401 — availability probe","@@ -110,40 +317,35 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]:\n         # in case the package ignores the hint.\n         safe_limit = max(1, int(limit))\n \n-        # A fresh single-worker pool per call (rather than a module-level one)\n-        # is intentional: on timeout the blocking ddgs call cannot be cancelled\n-        # and keeps running, so a shared pool would serialise every later search\n-        # behind that hung worker. A per-call pool isolates each search from a\n-        # previously-hung one.\n-        pool = _cf.ThreadPoolExecutor(max_workers=1)\n         try:\n-            future = pool.submit(_run_ddgs_search, query, safe_limit)\n-            try:\n-                web_results = future.result(timeout=_SEARCH_TIMEOUT_SECS)\n-            except _cf.TimeoutError:\n-                logger.warning(\n-                    \"DDGS search timed out after %ds for query: %r\",\n-                    _SEARCH_TIMEOUT_SECS, query,\n-                )\n-                return {\n-                    \"success\": False,\n-                    \"error\": (\n-                        f\"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — \"\n-                        \"DuckDuckGo may be rate-limiting or slow. Try again later \"\n-                        \"or switch to a different search provider.\"\n-                    ),\n-                }\n+            web_results = _run_ddgs_search_bounded(query, safe_limit)\n+        except TimeoutError:\n+            logger.warning(\n+                \"DDGS search timed out after %ds for query: %r\",\n+                _SEARCH_TIMEOUT_SECS,\n+                query,\n+            )\n+            return {\n+                \"success\": False,\n+                \"error\": (\n+                    f\"DuckDuckGo search timed out after {_SEARCH_TIMEOUT_SECS}s — \"\n+                    \"DuckDuckGo may be rate-limiting or slow. Try again later \"\n+                    \"or switch to a different search provider.\"\n+                ),\n+            }\n+        except _SearchInterrupted:\n+            logger.info(\"DDGS search interrupted for query: %r\", query)\n+            return {\n+                \"success\": False,\n+                \"error\": \"DuckDuckGo search interrupted\",\n+            }\n         except Exception as exc:  # noqa: BLE001 — ddgs raises its own exceptions\n             logger.warning(\"DDGS search error: %s\", exc)\n             return {\"success\": False, \"error\": f\"DuckDuckGo search failed: {exc}\"}\n-        finally:\n-            # Return immediately without joining the worker. On timeout the\n-            # already-running ddgs call can't be cancelled (cancel_futures only\n-            # affects not-yet-started work), so the worker runs to completion\n-            # on its own; it writes nothing shared, so leaking it is safe.\n-            pool.shutdown(wait=False, cancel_futures=True)\n-\n-        logger.info(\"DDGS search '%s': %d results (limit %d)\", query, len(web_results), limit)\n+\n+        logger.info(\n+            \"DDGS search '%s': %d results (limit %d)\", query, len(web_results), limit\n+        )\n         return {\"success\": True, \"data\": {\"web\": web_results}}\n \n     def get_setup_schema(self) -> Dict[str, Any]:"]},{"file":"tests/tools/test_web_providers_ddgs.py","hunks":["@@ -4,6 +4,7 @@\n - DDGSWebSearchProvider.is_available() — reflects package importability\n - DDGSWebSearchProvider.search() — happy path, missing package, runtime error\n - Result normalization (title, url, description, position)\n+- Process-isolated timeout / interrupt / GIL-hold / reap (#68096)\n - _is_backend_available(\"ddgs\") / _get_backend() integration\n - web_extract returns a search-only error when ddgs is active\n \"\"\"","@@ -11,6 +12,7 @@\n \n import json\n import sys\n+import time\n import types\n \n import pytest","@@ -52,6 +54,21 @@ def text(self, query, max_results=5):\n     return fake\n \n \n+def _force_inprocess_search(monkeypatch, prov):\n+    \"\"\"Route bounded search through the in-process helper.\n+\n+    Happy-path unit tests install a fake ``ddgs`` in the parent interpreter;\n+    spawn workers would not see that fake. Isolation behavior is covered by\n+    dedicated process tests below.\n+    \"\"\"\n+    monkeypatch.setattr(\n+        prov,\n+        \"_run_ddgs_search_bounded\",\n+        lambda query, safe_limit: prov._run_ddgs_search(query, safe_limit),\n+        raising=True,\n+    )\n+\n+\n # ---------------------------------------------------------------------------\n # DDGSWebSearchProvider unit tests\n # ---------------------------------------------------------------------------","@@ -98,9 +115,10 @@ def test_happy_path_normalizes_results(self, monkeypatch):\n             {\"title\": \"B\", \"href\": \"https://b.example.com\", \"body\": \"desc B\"},\n             {\"title\": \"C\", \"href\": \"https://c.example.com\", \"body\": \"desc C\"},\n         ])\n-        from plugins.web.ddgs.provider import DDGSWebSearchProvider\n+        import plugins.web.ddgs.provider as prov\n+        _force_inprocess_search(monkeypatch, prov)\n \n-        result = DDGSWebSearchProvider().search(\"q\", limit=5)\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n \n         assert result[\"success\"] is True\n         web = result[\"data\"][\"web\"]","@@ -112,9 +130,10 @@ def test_accepts_url_key_as_fallback_for_href(self, monkeypatch):\n         _install_fake_ddgs(monkeypatch, text_results=[\n             {\"title\": \"A\", \"url\": \"https://a.example.com\", \"body\": \"desc A\"},\n         ])\n-        from plugins.web.ddgs.provider import DDGSWebSearchProvider\n+        import plugins.web.ddgs.provider as prov\n+        _force_inprocess_search(monkeypatch, prov)\n \n-        result = DDGSWebSearchProvider().search(\"q\", limit=5)\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n \n         assert result[\"success\"] is True\n         assert result[\"data\"][\"web\"][0][\"url\"] == \"https://a.example.com\"","@@ -124,9 +143,10 @@ def test_limit_is_respected(self, monkeypatch):\n             {\"title\": f\"R{i}\", \"href\": f\"https://r{i}.example.com\", \"body\": \"\"}\n             for i in range(10)\n         ])\n-        from plugins.web.ddgs.provider import DDGSWebSearchProvider\n+        import plugins.web.ddgs.provider as prov\n+        _force_inprocess_search(monkeypatch, prov)\n \n-        result = DDGSWebSearchProvider().search(\"q\", limit=3)\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=3)\n \n         assert result[\"success\"] is True\n         assert len(result[\"data\"][\"web\"]) == 3","@@ -151,54 +171,42 @@ def blocked_import(name, *args, **kwargs):\n \n     def test_runtime_error_returns_failure(self, monkeypatch):\n         _install_fake_ddgs(monkeypatch, text_raises=RuntimeError(\"rate limited 202\"))\n-        from plugins.web.ddgs.provider import DDGSWebSearchProvider\n+        import plugins.web.ddgs.provider as prov\n+        _force_inprocess_search(monkeypatch, prov)\n \n-        result = DDGSWebSearchProvider().search(\"q\", limit=5)\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n         assert result[\"success\"] is False\n         assert \"rate limited\" in result[\"error\"] or \"failed\" in result[\"error\"].lower()\n \n     def test_empty_results(self, monkeypatch):\n         _install_fake_ddgs(monkeypatch, text_results=[])\n-        from plugins.web.ddgs.provider import DDGSWebSearchProvider\n+        import plugins.web.ddgs.provider as prov\n+        _force_inprocess_search(monkeypatch, prov)\n \n-        result = DDGSWebSearchProvider().search(\"nothing\", limit=5)\n+        result = prov.DDGSWebSearchProvider().search(\"nothing\", limit=5)\n         assert result[\"success\"] is True\n         assert result[\"data\"][\"web\"] == []\n \n+    @pytest.mark.live_system_guard_bypass\n     def test_hung_search_times_out_and_returns_failure(self, monkeypatch):\n-        \"\"\"#36776: a ddgs call that never returns must be bounded by the\n-        wall-clock timeout and surface a failure instead of hanging the\n-        shared agent loop. We patch the blocking helper to wait on an Event\n-        (released in finally so no worker thread leaks past the test) and\n-        shrink the timeout; search() must return success=False promptly.\"\"\"\n-        import threading\n-        import time\n-\n-        # ddgs must import-probe True for search() to proceed.\n+        \"\"\"#36776 / #68096: a hung worker must be bounded by the wall-clock\n+        timeout and reaped — even when the child never returns to Python.\"\"\"\n         _install_fake_ddgs(monkeypatch)\n-        monkeypatch.delitem(sys.modules, \"plugins.web.ddgs.provider\", raising=False)\n-        import plugins.web.ddgs.provider as _prov\n-\n-        release = threading.Event()\n+        import plugins.web.ddgs.provider as prov\n \n-        def _blocking_search(query, safe_limit):\n-            release.wait(timeout=10)  # bounded so the worker can never truly leak\n-            return []\n+        monkeypatch.setattr(prov, \"_test_hook\", \"sleep\", raising=True)\n+        monkeypatch.setattr(prov, \"_SEARCH_TIMEOUT_SECS\", 0.4, raising=True)\n+        monkeypatch.setattr(prov, \"_TERMINATE_GRACE_SECS\", 0.5, raising=True)\n+        monkeypatch.setattr(\"tools.interrupt.is_interrupted\", lambda: False)\n \n-        monkeypatch.setattr(_prov, \"_run_ddgs_search\", _blocking_search, raising=True)\n-        monkeypatch.setattr(_prov, \"_SEARCH_TIMEOUT_SECS\", 0.3, raising=True)\n+        start = time.monotonic()\n+        result = prov.DDGSWebSearchProvider().search(\"hangs forever\", limit=5)\n+        elapsed = time.monotonic() - start\n \n-        try:\n-            start = time.monotonic()\n-            result = _prov.DDGSWebSearchProvider().search(\"hangs forever\", limit=5)\n-            elapsed = time.monotonic() - start\n-\n-            assert result[\"success\"] is False\n-            assert \"timed out\" in result[\"error\"].lower()\n-            # Returned well before the worker's 10s wait — proves the cap fired.\n-            assert elapsed < 3.0, f\"search did not return promptly ({elapsed:.1f}s)\"\n-        finally:\n-            release.set()  # let the orphaned worker finish immediately\n+        assert result[\"success\"] is False\n+        assert \"timed out\" in result[\"error\"].lower()\n+        assert elapsed < 5.0, f\"search did not return promptly ({elapsed:.1f}s)\"\n+        _assert_worker_reaped(prov)\n \n     def test_fast_search_not_affected_by_timeout_wrapper(self, monkeypatch):\n         \"\"\"Happy-path guard: the timeout wrapper must not break a normal,","@@ -207,14 +215,115 @@ def test_fast_search_not_affected_by_timeout_wrapper(self, monkeypatch):\n             monkeypatch,\n             text_results=[{\"title\": \"T\", \"href\": \"https://e.com\", \"body\": \"B\"}],\n         )\n-        from plugins.web.ddgs.provider import DDGSWebSearchProvider\n+        import plugins.web.ddgs.provider as prov\n+        _force_inprocess_search(monkeypatch, prov)\n \n-        result = DDGSWebSearchProvider().search(\"q\", limit=5)\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n         assert result[\"success\"] is True\n         assert result[\"data\"][\"web\"][0][\"url\"] == \"https://e.com\"\n         assert result[\"data\"][\"web\"][0][\"title\"] == \"T\"\n \n \n+# ---------------------------------------------------------------------------\n+# Process isolation (#68096)\n+# ---------------------------------------------------------------------------\n+\n+\n+def _assert_worker_reaped(prov) -> None:\n+    \"\"\"Assert the last DDGS worker process has exited.\"\"\"\n+    proc = prov._last_worker_proc\n+    assert proc is not None, \"expected a DDGS worker process to have been started\"\n+    assert proc.poll() is not None, (\n+        f\"DDGS worker still alive (pid={proc.pid}, returncode={proc.returncode})\"\n+    )\n+\n+\n[redacted:email]_system_guard_bypass\n+class TestDDGSProcessIsolation:\n+    def test_gil_holding_worker_times_out_and_is_reaped(self, monkeypatch):\n+        \"\"\"#68096: parent deadline still fires when the child holds its GIL.\"\"\"\n+        _install_fake_ddgs(monkeypatch)\n+        import plugins.web.ddgs.provider as prov\n+\n+        monkeypatch.setattr(prov, \"_test_hook\", \"gil\", raising=True)\n+        monkeypatch.setattr(prov, \"_SEARCH_TIMEOUT_SECS\", 0.5, raising=True)\n+        monkeypatch.setattr(prov, \"_TERMINATE_GRACE_SECS\", 0.5, raising=True)\n+        monkeypatch.setattr(\"tools.interrupt.is_interrupted\", lambda: False)\n+\n+        start = time.monotonic()\n+        result = prov.DDGSWebSearchProvider().search(\"gil hold\", limit=5)\n+        elapsed = time.monotonic() - start\n+\n+        assert result[\"success\"] is False\n+        assert \"timed out\" in result[\"error\"].lower()\n+        assert elapsed < 5.0, f\"GIL-hold search did not time out promptly ({elapsed:.1f}s)\"\n+        _assert_worker_reaped(prov)\n+\n+    def test_interrupt_terminates_worker_promptly(self, monkeypatch):\n+        \"\"\"TUI/gateway interrupt must kill the DDGS child before the deadline.\"\"\"\n+        _install_fake_ddgs(monkeypatch)\n+        import plugins.web.ddgs.provider as prov\n+\n+        # Flip interrupt after the first poll so the wait loop observes it.\n+        calls = {\"n\": 0}\n+\n+        def _interrupt_after_poll():\n+            calls[\"n\"] += 1\n+            return calls[\"n\"] >= 2\n+\n+        monkeypatch.setattr(prov, \"_test_hook\", \"sleep\", raising=True)\n+        monkeypatch.setattr(prov, \"_SEARCH_TIMEOUT_SECS\", 30, raising=True)\n+        monkeypatch.setattr(prov, \"_TERMINATE_GRACE_SECS\", 0.5, raising=True)\n+        monkeypatch.setattr(\"tools.interrupt.is_interrupted\", _interrupt_after_poll)\n+\n+        start = time.monotonic()\n+        result = prov.DDGSWebSearchProvider().search(\"interrupt me\", limit=5)\n+        elapsed = time.monotonic() - start\n+\n+        assert result[\"success\"] is False\n+        assert \"interrupted\" in result[\"error\"].lower()\n+        assert elapsed < 5.0, f\"interrupt did not return promptly ({elapsed:.1f}s)\"\n+        _assert_worker_reaped(prov)\n+\n+    def test_spawned_worker_success_envelope(self, monkeypatch):\n+        \"\"\"Real spawn path: success envelope round-trips through the pipe.\"\"\"\n+        _install_fake_ddgs(monkeypatch)\n+        import plugins.web.ddgs.provider as prov\n+\n+        monkeypatch.setattr(prov, \"_test_hook\", \"success\", raising=True)\n+        monkeypatch.setattr(prov, \"_SEARCH_TIMEOUT_SECS\", 5, raising=True)\n+        monkeypatch.setattr(\"tools.interrupt.is_interrupted\", lambda: False)\n+\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n+        assert result[\"success\"] is True\n+        assert result[\"data\"][\"web\"][0][\"url\"] == \"https://example.com\"\n+        _assert_worker_reaped(prov)\n+\n+    def test_spawned_worker_error_envelope(self, monkeypatch):\n+        \"\"\"Real spawn path: error envelope becomes success=False.\"\"\"\n+        _install_fake_ddgs(monkeypatch)\n+        import plugins.web.ddgs.provider as prov\n+\n+        monkeypatch.setattr(prov, \"_test_hook\", \"error\", raising=True)\n+        monkeypatch.setattr(prov, \"_SEARCH_TIMEOUT_SECS\", 5, raising=True)\n+        monkeypatch.setattr(\"tools.interrupt.is_interrupted\", lambda: False)\n+\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n+        assert result[\"success\"] is False\n+        assert \"boom\" in result[\"error\"]\n+        _assert_worker_reaped(prov)\n+\n+    def test_no_orphan_after_successful_search(self, monkeypatch):\n+        _install_fake_ddgs(monkeypatch)\n+        import plugins.web.ddgs.provider as prov\n+\n+        monkeypatch.setattr(prov, \"_test_hook\", \"empty\", raising=True)\n+        monkeypatch.setattr(\"tools.interrupt.is_interrupted\", lambda: False)\n+\n+        result = prov.DDGSWebSearchProvider().search(\"q\", limit=5)\n+        assert result[\"success\"] is True\n+        _assert_worker_reaped(prov)\n+\n # ---------------------------------------------------------------------------\n # Integration: _is_backend_available / _get_backend / check_web_api_key\n # ---------------------------------------------------------------------------\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}