{"repo":"NousResearch/hermes-agent","public_id":"4815","display_label":"Agent timeout: wall-clock versus inactivity-based behavior","summary":"The members share one theme: the gateway agent timeout (HERMES_AGENT_TIMEOUT) is described as interrupting long-running tasks, and two pull requests propose replacing the wall-clock timeout with an inactivity-based one. Titles alone do not show the current status of these changes.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":5389,"merged_at":"2026-04-06T02:38:21+00:00"},"resolution":{"state":"resolved","by":"pr-5389"},"freshness":{"state":"stale","last_verified_at":"2026-04-06T02:38:21+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":4815,"pr_number":5389,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-04-06T02:38:21+00:00"}],"members":[{"kind":"issue","number":4815,"title":"gateway agent timeout (HERMES_AGENT_TIMEOUT) kills legitimate long-running tasks","url":null,"state":"closed"},{"kind":"pr","number":4864,"title":"fix: replace wall-clock agent timeout with inactivity-based timeout","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#5389","number":5389}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":5389,"title":"fix(gateway): replace wall-clock agent timeout with inactivity-based timeout","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","hermes_cli/config.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -182,6 +182,10 @@ def _ensure_ssl_certs() -> None:\n         if _agent_cfg and isinstance(_agent_cfg, dict):\n             if \"max_turns\" in _agent_cfg:\n                 os.environ[\"HERMES_MAX_ITERATIONS\"] = str(_agent_cfg[\"max_turns\"])\n+            # Bridge agent.gateway_timeout → HERMES_AGENT_TIMEOUT env var.\n+            # Env var from .env takes precedence (already in os.environ).\n+            if \"gateway_timeout\" in _agent_cfg and \"HERMES_AGENT_TIMEOUT\" not in os.environ:\n+                os.environ[\"HERMES_AGENT_TIMEOUT\"] = str(_agent_cfg[\"gateway_timeout\"])\n         # Timezone: bridge config.yaml → HERMES_TIMEZONE env var.\n         # HERMES_TIMEZONE from .env takes precedence (already in os.environ).\n         _tz_cfg = _cfg.get(\"timezone\", \"\")","@@ -1800,32 +1804,46 @@ async def _handle_message(self, event: MessageEvent) -> Optional[str]:\n         # simultaneous updates. Do NOT interrupt for photo-only follow-ups here;\n         # let the adapter-level batching/queueing logic absorb them.\n \n-        # Staleness eviction: if an entry has been in _running_agents for\n-        # longer than the agent timeout, it's a leaked lock from a hung or\n-        # crashed handler.  Evict it so the session isn't permanently stuck.\n+        # Staleness eviction: detect leaked locks from hung/crashed handlers.\n+        # With inactivity-based timeout, active tasks can run for hours, so\n+        # wall-clock age alone isn't sufficient.  Evict only when the agent\n+        # has been *idle* beyond the inactivity threshold (or when the agent\n+        # object has no activity tracker and wall-clock age is extreme).\n         _raw_stale_timeout = float(os.getenv(\"HERMES_AGENT_TIMEOUT\", 1800))\n-        _STALE_TTL = (_raw_stale_timeout + 60) if _raw_stale_timeout > 0 else float(\"inf\")\n         _stale_ts = self._running_agents_ts.get(_quick_key, 0)\n-        if _quick_key in self._running_agents and _stale_ts and (time.time() - _stale_ts) > _STALE_TTL:\n+        if _quick_key in self._running_agents and _stale_ts:\n             _stale_age = time.time() - _stale_ts\n             _stale_agent = self._running_agents.get(_quick_key)\n+            _stale_idle = float(\"inf\")  # assume idle if we can't check\n             _stale_detail = \"\"\n             if _stale_agent and hasattr(_stale_agent, \"get_activity_summary\"):\n                 try:\n                     _sa = _stale_agent.get_activity_summary()\n+                    _stale_idle = _sa.get(\"seconds_since_activity\", float(\"inf\"))\n                     _stale_detail = (\n                         f\" | last_activity={_sa.get('last_activity_desc', 'unknown')} \"\n-                        f\"({_sa.get('seconds_since_activity', 0):.0f}s ago) \"\n+                        f\"({_stale_idle:.0f}s ago) \"\n                         f\"| iteration={_sa.get('api_call_count', 0)}/{_sa.get('max_iterations', 0)}\"\n                     )\n                 except Exception:\n                     pass\n-            logger.warning(\n-                \"Evicting stale _running_agents entry for %s (age: %.0fs, TTL: %.0fs)%s\",\n-                _quick_key[:30], _stale_age, _STALE_TTL, _stale_detail,\n+            # Evict if: agent is idle beyond timeout, OR wall-clock age is\n+            # extreme (10x timeout or 2h, whichever is larger — catches\n+            # cases where the agent object was garbage-collected).\n+            _wall_ttl = max(_raw_stale_timeout * 10, 7200) if _raw_stale_timeout > 0 else float(\"inf\")\n+            _should_evict = (\n+                (_raw_stale_timeout > 0 and _stale_idle >= _raw_stale_timeout)\n+                or _stale_age > _wall_ttl\n             )\n-            del self._running_agents[_quick_key]\n-            self._running_agents_ts.pop(_quick_key, None)\n+            if _should_evict:\n+                logger.warning(\n+                    \"Evicting stale _running_agents entry for %s \"\n+                    \"(age: %.0fs, idle: %.0fs, timeout: %.0fs)%s\",\n+                    _quick_key[:30], _stale_age, _stale_idle,\n+                    _raw_stale_timeout, _stale_detail,\n+                )\n+                del self._running_agents[_quick_key]\n+                self._running_agents_ts.pop(_quick_key, None)\n \n         if _quick_key in self._running_agents:\n             if event.get_command() == \"status\":","@@ -6766,19 +6784,54 @@ async def _notify_long_running():\n         _notify_task = asyncio.create_task(_notify_long_running())\n \n         try:\n-            # Run in thread pool to not block.  Cap total execution time\n-            # so a hung API call or runaway tool doesn't permanently lock\n-            # the session.  Default 30 minutes; override with env var.\n-            # Set to 0 for no limit (infinite).\n+            # Run in thread pool to not block.  Use an *inactivity*-based\n+            # timeout instead of a wall-clock limit: the agent can run for\n+            # hours if it's actively calling tools / receiving stream tokens,\n+            # but a hung API call or stuck tool with no activity for the\n+            # configured duration is caught and killed.  (#4815)\n+            #\n+            # Config: agent.gateway_timeout in config.yaml, or\n+            # HERMES_AGENT_TIMEOUT env var (env var takes precedence).\n+            # Default 1800s (30 min inactivity).  0 = unlimited.\n             _agent_timeout_raw = float(os.getenv(\"HERMES_AGENT_TIMEOUT\", 1800))\n             _agent_timeout = _agent_timeout_raw if _agent_timeout_raw > 0 else None\n             loop = asyncio.get_event_loop()\n-            try:\n-                response = await asyncio.wait_for(\n-                    loop.run_in_executor(None, run_sync),\n-                    timeout=_agent_timeout,\n-                )\n-            except asyncio.TimeoutError:\n+            _executor_task = asyncio.ensure_future(\n+                loop.run_in_executor(None, run_sync)\n+            )\n+\n+            _inactivity_timeout = False\n+            _POLL_INTERVAL = 5.0\n+\n+            if _agent_timeout is None:\n+                # Unlimited — just await the result.\n+                response = await _executor_task\n+            else:\n+                # Poll loop: check the agent's built-in activity tracker\n+                # (updated by _touch_activity() on every tool call, API\n+                # call, and stream delta) every few seconds.\n+                response = None\n+                while True:\n+                    done, _ = await asyncio.wait(\n+                        {_executor_task}, timeout=_POLL_INTERVAL\n+                    )\n+                    if done:\n+                        response = _executor_task.result()\n+                        break\n+                    # Agent still running — check inactivity.\n+                    _agent_ref = agent_holder[0]\n+                    _idle_secs = 0.0\n+                    if _agent_ref and hasattr(_agent_ref, \"get_activity_summary\"):\n+                        try:\n+                            _act = _agent_ref.get_activity_summary()\n+                            _idle_secs = _act.get(\"seconds_since_activity\", 0.0)\n+                        except Exception:\n+                            pass\n+                    if _idle_secs >= _agent_timeout:\n+                        _inactivity_timeout = True\n+                        break\n+\n+            if _inactivity_timeout:\n                 # Build a diagnostic summary from the agent's activity tracker.\n                 _timed_out_agent = agent_holder[0]\n                 _activity = {}","@@ -6795,29 +6848,26 @@ async def _notify_long_running():\n                 _iter_max = _activity.get(\"max_iterations\", 0)\n \n                 logger.error(\n-                    \"Agent execution timed out after %.0fs for session %s \"\n-                    \"| last_activity=%.0fs ago (%s) | iteration=%s/%s | tool=%s\",\n-                    _agent_timeout, session_key,\n-                    _secs_ago, _last_desc, _iter_n, _iter_max,\n+                    \"Agent idle for %.0fs (timeout %.0fs) in session %s \"\n+                    \"| last_activity=%s | iteration=%s/%s | tool=%s\",\n+                    _secs_ago, _agent_timeout, session_key,\n+                    _last_desc, _iter_n, _iter_max,\n                     _cur_tool or \"none\",\n                 )\n \n                 # Interrupt the agent if it's still running so the thread\n                 # pool worker is freed.\n                 if _timed_out_agent and hasattr(_timed_out_agent, \"interrupt\"):\n-                    _timed_out_agent.interrupt(\"Execution timed out\")\n+                    _timed_out_agent.interrupt(\"Execution timed out (inactivity)\")\n \n-                _timeout_mins = int(_agent_timeout // 60)\n+                _timeout_mins = int(_agent_timeout // 60) or 1\n \n                 # Construct a user-facing message with diagnostic context.\n-                _diag_lines = [f\"⏱️ Request timed out after {_timeout_mins} minutes.\"]\n-                if _secs_ago < 30:\n-                    _diag_lines.append(\n-                        f\"The agent was actively working when the timeout fired \"\n-                        f\"(last activity: {_last_desc}, {_secs_ago:.0f}s ago, \"\n-                        f\"iteration {_iter_n}/{_iter_max}).\"\n-                    )\n-                elif _cur_tool:\n+                _diag_lines = [\n+                    f\"⏱️ Agent inactive for {_timeout_mins} min — no tool calls \"\n+                    f\"or API responses.\"\n+                ]\n+                if _cur_tool:\n                     _diag_lines.append(\n                         f\"The agent appears stuck on tool `{_cur_tool}` \"\n                         f\"({_secs_ago:.0f}s since last activity, \"","@@ -6830,7 +6880,7 @@ async def _notify_long_running():\n                         \"The agent may have been waiting on an API response.\"\n                     )\n                 _diag_lines.append(\n-                    \"To increase the limit, set HERMES_AGENT_TIMEOUT in your .env \"\n+                    \"To increase the limit, set agent.gateway_timeout in config.yaml \"\n                     \"(value in seconds, 0 = no limit) and restart the gateway.\\n\"\n                     \"Try again, or use /reset to start fresh.\"\n                 )"]},{"file":"hermes_cli/config.py","hunks":["@@ -205,6 +205,11 @@ def ensure_hermes_home():\n     \"toolsets\": [\"hermes-cli\"],\n     \"agent\": {\n         \"max_turns\": 90,\n+        # Inactivity timeout for gateway agent execution (seconds).\n+        # The agent can run indefinitely as long as it's actively calling\n+        # tools or receiving API responses.  Only fires when the agent has\n+        # been completely idle for this duration.  0 = unlimited.\n+        \"gateway_timeout\": 1800,\n         # Tool-use enforcement: injects system prompt guidance that tells the\n         # model to actually call tools instead of describing intended actions.\n         # Values: \"auto\" (default — applies to gpt/codex models), true/false\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}