{"repo":"NousResearch/hermes-agent","public_id":"28686","display_label":"Gateway session_reset zombie slot cleanup fixes","summary":"A bug and several fixes addressing zombie agent slots left behind when session_reset races with in-flight runs or credential pool exhaustion, causing subsequent messages to be silently dropped. Related cleanup includes consolidating per-session teardown, closing SessionDB on shutdown, and stripping a stray environment variable from the Windows restart helper.","subsystem":"gateway","counts":{"members":5,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":39060},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-04T14:50:45+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.4},"fix_edges":[{"issue_number":28686,"pr_number":39060,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":28686,"title":"Bug: session_reset + credential pool exhaustion leaves thread session in zombie state — subsequent messages silently dropped","url":null,"state":"closed"},{"kind":"pr","number":28689,"title":"fix(gateway): clear stale agent slot after session_reset to prevent zombie thread","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"693f4c7e9ce583ed740daaa5aa8eb738390c643b","number":0},{"kind":"commit","value":"ae29484f9a643db8328f8fdc672d2d4717318696","number":0},{"kind":"pr","value":"#39060","number":39060},{"kind":"pr","value":"#39060","number":39060},{"kind":"pr","value":"#28686","number":28686}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":30834,"title":"fix(gateway): clear zombie agent slot when session_reset races credential pool exhaustion","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"693f4c7e9","number":0},{"kind":"pr","value":"#39060","number":39060}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":39060,"title":"fix(gateway): clear zombie agent slot when session_reset races in-flight run","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_session_state_cleanup.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -8470,18 +8470,14 @@ async def _do_undo():\n                 logger.debug(\"goal continuation hook failed: %s\", _goal_exc)\n             return _agent_result\n         finally:\n-            # If _run_agent replaced the sentinel with a real agent and\n-            # then cleaned it up, this is a no-op.  If we exited early\n-            # (exception, command fallthrough, etc.) the sentinel must\n-            # not linger or the session would be permanently locked out.\n-            if self._running_agents.get(_quick_key) is _AGENT_PENDING_SENTINEL:\n-                self._release_running_agent_state(_quick_key)\n-            else:\n-                # Agent path already cleaned _running_agents; make sure\n-                # the paired metadata dicts are gone too.\n-                self._running_agents_ts.pop(_quick_key, None)\n-                if hasattr(self, \"_busy_ack_ts\"):\n-                    self._busy_ack_ts.pop(_quick_key, None)\n+            # Unconditional release covers every exit path. _release_running_agent_state\n+            # is idempotent (pop-on-absent is harmless) and, called without a\n+            # run_generation guard, always clears the slot regardless of which\n+            # generation it holds. This evicts the zombie left when session_reset\n+            # bumps the generation (N -> N+1) mid-flight: gen-N's guarded release\n+            # inside _run_agent returns False, and the old sentinel-only check here\n+            # missed the leftover real agent — locking the session out forever (#28686).\n+            self._release_running_agent_state(_quick_key)\n \n     async def _prepare_inbound_message_text(\n         self,","@@ -9978,6 +9974,12 @@ async def _handle_reset_command(self, event: MessageEvent) -> Union[str, Ephemer\n         # Get existing session key\n         session_key = self._session_key_for_source(source)\n         self._invalidate_session_run_generation(session_key, reason=\"session_reset\")\n+        # Evict the running-agent slot now that the generation is bumped. The\n+        # in-flight run's own guarded release (run_generation=old) will return\n+        # False and leave its dead agent behind; clearing here keeps the slot\n+        # from becoming a zombie that silently drops all later messages (#28686).\n+        # Idempotent, so the run's finally calling it again is harmless.\n+        self._release_running_agent_state(session_key)\n \n         # Snapshot the old entry so on_session_finalize can report the\n         # expiring session id before reset_session() rotates it."]},{"file":"tests/gateway/test_session_state_cleanup.py","hunks":["@@ -228,3 +228,54 @@ def test_shutdown_tolerates_close_raising(self):\n \n         flaky_db.close.assert_called_once()\n         healthy_db.close.assert_called_once()\n+\n+\n+class TestSessionResetZombieRace:\n+    \"\"\"Regression for #28686 — a session_reset racing the in-flight run's\n+    guarded release must not leave a dead agent locking the slot forever.\n+    \"\"\"\n+\n+    def test_generation_guard_blocks_then_unconditional_release_evicts(self):\n+        runner = _make_runner()\n+        runner._session_run_generation = {}\n+        key = \"agent:main:telegram:private:1\"\n+\n+        gen_n = runner._begin_session_run_generation(key)\n+        dead_agent = MagicMock()\n+        runner._running_agents[key] = dead_agent\n+        runner._running_agents_ts[key] = 1.0\n+        runner._busy_ack_ts[key] = 1.0\n+\n+        # session_reset bumps the generation while gen-N is still in flight.\n+        runner._invalidate_session_run_generation(key, reason=\"session_reset\")\n+\n+        # gen-N's own guarded release is correctly blocked — slot would be a\n+        # zombie if nothing else cleared it (the pre-fix behaviour).\n+        assert runner._release_running_agent_state(key, run_generation=gen_n) is False\n+        assert runner._running_agents.get(key) is dead_agent\n+\n+        # The fix: unconditional release (no run_generation) always clears it.\n+        assert runner._release_running_agent_state(key) is True\n+        assert key not in runner._running_agents\n+        assert key not in runner._running_agents_ts\n+        assert key not in runner._busy_ack_ts\n+\n+    def test_normal_completion_is_not_evicted_by_outer_release(self):\n+        \"\"\"Guarded release with the current generation succeeds; the outer\n+        unconditional release that follows is a harmless no-op.\n+        \"\"\"\n+        runner = _make_runner()\n+        runner._session_run_generation = {}\n+        key = \"agent:main:telegram:private:2\"\n+\n+        gen = runner._begin_session_run_generation(key)\n+        runner._running_agents[key] = MagicMock()\n+        runner._running_agents_ts[key] = 1.0\n+        runner._busy_ack_ts[key] = 1.0\n+\n+        assert runner._release_running_agent_state(key, run_generation=gen) is True\n+        assert key not in runner._running_agents\n+        # Outer finally runs the unconditional release after — nothing stranded.\n+        assert runner._release_running_agent_state(key) is True\n+        assert key not in runner._running_agents_ts\n+        assert key not in runner._busy_ack_ts\n"]}]},{"kind":"pr","number":40059,"title":"fix(gateway): strip _HERMES_GATEWAY from Windows detached restart helper env","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}