Session rows left open or stale across agent lifecycle paths

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #50299 fix(session): finalize owned SQLite session rows on AIAgent.close() (#12029)

  merged to main

  files: agent/agent_init.py, agent/background_review.py, gateway/run.py, run_agent.py, tests/gateway/test_busy_session_ack.py, tests/run_agent/test_background_review.py, tests/tools/test_zombie_process_cleanup.py

  Show the diff

  • agent/agent_init.py

    @@ -1096,6 +1096,12 @@ def init_agent(
         agent._parent_session_id = parent_session_id
         agent._last_flushed_db_idx = 0  # tracks DB-write cursor to prevent duplicate writes
         agent._session_db_created = False  # DB row deferred to run_conversation()
    +    # Most agents own their session row and should finalize it on close().
    +    # Some temporary helper agents (manual compression / session-hygiene /
    +    # background-review forks) rotate or share the session forward to a
    +    # continuation row that must remain open after the helper is torn down;
    +    # those callers explicitly set this flag to False.
    +    agent._end_session_on_close = True
         agent._session_init_model_config = {
             "max_iterations": agent.max_iterations,
             "reasoning_config": reasoning_config,

  • agent/background_review.py

    @@ -575,6 +575,13 @@ def _bg_review_auto_deny(command, description, **kwargs):
                 # if a future code path bypasses the cache.
                 review_agent.session_start = agent.session_start
                 review_agent.session_id = agent.session_id
    +            # The fork shares the parent's live session_id (pinned above for
    +            # prefix-cache parity). It is single-lifecycle and calls close()
    +            # right after this run_conversation(); without opting out, close()
    +            # would finalize the parent's still-active session row mid
    +            # conversation (the review fires every ~10 turns). Leave session
    +            # finalization to the real owner (CLI close / gateway reset / cron).
    +            review_agent._end_session_on_close = False
                 # Never let the review fork compress. It shares the parent's
                 # session_id, so if it won a compression race it would rotate the
                 # parent into a NEW child that the gateway never adopts (the fork

  • gateway/run.py

    @@ -4686,6 +4686,27 @@ def _finalize_shutdown_agents(self, active_agents: Dict[str, Any]) -> None:
                     pass
                 self._cleanup_agent_resources(agent)
     
    +    def _should_emit_long_running_notification(
    +        self,
    +        session_key: Optional[str],
    +        agent: Any,
    +        executor_task: Optional[Any],
    +    ) -> bool:
    +        """Only emit the heartbeat while this task still owns the live run.
    +
    +        Guards against a stale ``running: delegate_task`` heartbeat outliving the
    +        run that started it: stop once the executor finishes, the agent is gone,
    +        or the session key has been rebound to a different live agent (e.g. the
    +        user sent ``/new`` and a fresh agent took the slot mid-run, #12029).
    +        """
    +        if agent is None:
    +            return False
    +        if executor_task is not None and executor_task.done():
    +            return False
    +        if session_key and self._running_agents.get(session_key) is not agent:
    +            return False
    +        return True
    +
         def _cleanup_agent_resources(self, agent: Any) -> None:
             """Best-effort cleanup for temporary or cached agent instances."""
             if agent is None:
    @@ -9194,6 +9215,13 @@ async def _handle_message_with_agent(self, event, source, _quick_key: str, run_g
                                         session_id=session_entry.session_id,
                                     )
                                     try:
    +                                    # The hygiene agent rotates the session
    +                                    # forward to a continuation id that becomes
    +                                    # the gateway session's live row. It must
    +                                    # never finalize on close() (today it has no
    +                                    # session_db so close() no-ops, but this
    +                                    # guards a future where one is wired in).
    +                                    _hyg_agent._end_session_on_close = False
                                         _hyg_agent._print_fn = lambda *a, **kw: None
     
                                         loop = asyncio.get_running_loop()
    @@ -16274,6 +16302,20 @@ async def _notify_long_running():
                 _heartbeat_msg_id: Optional[str] = None
                 while True:
                     await asyncio.sleep(_NOTIFY_INTERVAL)
    +                # Stop heartbeating once this run no longer owns the session
    +                # slot or the executor has finished — otherwise a stale
    +                # "running: delegate_task" bubble can outlive the run that
    +                # spawned it (#12029). _executor_task is a closure var bound
    +                # just after this task is scheduled; tolerate the brief window
    +                # before then (the first wake is _NOTIFY_INTERVAL away anyway).
    +                try:
    +                    _exec_ref = _executor_task
    +                except NameError:
    +                    _exec_ref = None
    +                if not self._should_emit_long_running_notification(
    +                    session_key, agent_holder[0], _exec_ref
    +                ):
    +                    break
                     _elapsed_mins = int((time.time() - _notify_start) // 60)
                     # Include agent activity context if available. Default
                     # heartbeat is terse: elapsed + current tool. Verbose

  • run_agent.py

    @@ -3250,6 +3250,22 @@ def close(self) -> None:
             except Exception:
                 pass
     
    +        # 7. Finalize the owned SQLite session row unless this agent is only a
    +        # temporary helper that deliberately handed session ownership forward
    +        # (manual compression helpers that rotate to a continuation session_id,
    +        # or background-review forks that share the live parent's session_id and
    +        # must leave it open). end_session() is first-reason-wins and no-ops on
    +        # an already-ended row, so this never clobbers a 'compression' /
    +        # 'cron_complete' / 'cli_close' reason set by an earlier terminal path.
    +        try:
    +            if getattr(self, "_end_session_on_close", True):
    +                session_db = getattr(self, "_session_db", None)
    +                session_id = getattr(self, "session_id", None)
    +                if session_db and session_id:
    +                    session_db.end_session(session_id, "agent_close")
    +        except Exception:
    +            pass
    +
         def _hydrate_todo_store(self, history: List[Dict[str, Any]]) -> None:
             """
             Recover todo state from conversation history.

  • tests/gateway/test_busy_session_ack.py

    @@ -715,3 +715,62 @@ async def test_queue_mode_hint_points_to_interrupt(self, tmp_path, monkeypatch):
             assert "/busy interrupt" in content
             # Must NOT tell the user to /busy queue when they're already on queue.
             assert "/busy queue" not in content
    +
    +
    +class TestLongRunningNotificationOwnership:
    +    """The long-running heartbeat must stop once its run no longer owns the
    +    session slot or the executor finished — otherwise a stale
    +    'running: delegate_task' bubble outlives the run that spawned it (#12029).
    +    """
    +
    +    def test_notification_stops_after_session_ownership_moves(self):
    +        from gateway.run import GatewayRunner
    +
    +        runner = object.__new__(GatewayRunner)
    +        runner._running_agents = {}
    +
    +        original_agent = MagicMock()
    +        replacement_agent = MagicMock()
    +        runner._running_agents["sess"] = replacement_agent
    +
    +        assert runner._should_emit_long_running_notification(
    +            "sess", original_agent, executor_task=None
    +        ) is False
    +
    +    def test_notification_stops_after_executor_finishes(self):
    +        from gateway.run import GatewayRunner
    +
    +        runner = object.__new__(GatewayRunner)
    +        agent = MagicMock()
    +        runner._running_agents = {"sess": agent}
    +
    +        done_task = MagicMock()
    +        done_task.done.return_value = True
    +
    +        assert runner._should_emit_long_running_notification(
    +            "sess", agent, executor_task=done_task
    +        ) is False
    +
    +    def test_notification_stops_when_agent_is_gone(self):
    +        from gateway.run import GatewayRunner
    +
    +        runner = object.__new__(GatewayRunner)
    +        runner._running_agents = {}
    +
    +        assert runner._should_emit_long_running_notification(
    +            "sess", None, executor_task=None
    +        ) is False
    +
    +    def test_notification_continues_for_live_active_run(self):
    +        from gateway.run import GatewayRunner
    +
    +        runner = object.__new__(GatewayRunner)
    +        agent = MagicMock()
    +        runner._running_agents = {"sess": agent}
    +
    +        live_task = MagicMock()
    +        live_task.done.return_value = False
    +
    +        assert runner._should_emit_long_running_notification(
    +            "sess", agent, executor_task=live_task
    +        ) is True

  • tests/run_agent/test_background_review.py

    @@ -76,6 +76,50 @@ def close(self):
         ]
     
     
    +def test_background_review_fork_opts_out_of_session_finalization(monkeypatch):
    +    """The review fork shares the parent's live session_id, so it must set
    +    ``_end_session_on_close = False``. Otherwise close() (now finalizing owned
    +    session rows) would end the still-active parent session mid-conversation
    +    every time the review fires (~every 10 turns). Regression for #12029.
    +    """
    +    seen = {}
    +
    +    class FakeReviewAgent:
    +        def __init__(self, **kwargs):
    +            self._session_messages = []
    +            # Default matches AIAgent.__init__ (agent_init.py): owns its row.
    +            self._end_session_on_close = True
    +
    +        def __setattr__(self, name, value):
    +            object.__setattr__(self, name, value)
    +            if name == "_end_session_on_close":
    +                seen["end_session_on_close"] = value
    +
    +        def run_conversation(self, **kwargs):
    +            # By the time the fork runs, the opt-out must already be applied.
    +            seen["at_run_time"] = self._end_session_on_close
    +
    +        def shutdown_memory_provider(self):
    +            pass
    +
    +        def close(self):
    +            pass
    +
    +    monkeypatch.setattr(run_agent_module, "AIAgent", FakeReviewAgent)
    +    monkeypatch.setattr(run_agent_module.threading, "Thread", ImmediateThread)
    +
    +    agent = _bare_agent()
    +
    +    AIAgent._spawn_background_review(
    +        agent,
    +        messages_snapshot=[{"role": "user", "content": "hello"}],
    +        review_memory=True,
    +    )
    +
    +    assert seen.get("end_session_on_close") is False
    +    assert seen.get("at_run_time") is False
    +
    +
     def test_background_review_summarizer_receives_captured_messages_after_close(monkeypatch):
         """The action summarizer must see review messages even after close cleanup.
     

  • tests/tools/test_zombie_process_cleanup.py

    @@ -155,6 +155,59 @@ def test_close_propagates_to_children(self):
                 child_2.close.assert_called_once()
                 assert agent._active_children == []
     
    +    def test_close_ends_owned_session_row(self):
    +        """close() finalizes the agent's owned SQLite session row."""
    +        from unittest.mock import MagicMock, patch
    +
    +        with patch("run_agent.AIAgent.__init__", return_value=None):
    +            from run_agent import AIAgent
    +            agent = AIAgent.__new__(AIAgent)
    +            agent.session_id = "test-close-session-row"
    +            agent._active_children = []
    +            agent._active_children_lock = threading.Lock()
    +            agent.client = None
    +            agent._end_session_on_close = True
    +            agent._session_db = MagicMock()
    +
    +            agent.close()
    +
    +            agent._session_db.end_session.assert_called_once_with(
    +                "test-close-session-row", "agent_close"
    +            )
    +
    +    def test_close_skips_session_end_for_forwarded_continuation_agents(self):
    +        """Helper agents that handed session ownership forward opt out."""
    +        from unittest.mock import MagicMock, patch
    +
    +        with patch("run_agent.AIAgent.__init__", return_value=None):
    +            from run_agent import AIAgent
    +            agent = AIAgent.__new__(AIAgent)
    +            agent.session_id = "test-close-forwarded-session"
    +            agent._active_children = []
    +            agent._active_children_lock = threading.Lock()
    +            agent.client = None
    +            agent._end_session_on_close = False
    +            agent._session_db = MagicMock()
    +
    +            agent.close()
    +
    +            agent._session_db.end_session.assert_not_called()
    +
    +    def test_close_session_end_noops_without_session_db(self):
    +        """close() is a no-op for session finalization when no DB is wired in."""
    +        from unittest.mock import patch
    +
    +        with patch("run_agent.AIAgent.__init__", return_value=None):
    +            from run_agent import AIAgent
    +            agent = AIAgent.__new__(AIAgent)
    +            agent.session_id = "test-close-no-db"
    +            agent._active_children = []
    +            agent._active_children_lock = threading.Lock()
    +            agent.client = None
    +            # No _session_db / _end_session_on_close attributes at all —
    +            # getattr defaults must keep close() from raising.
    +            agent.close()  # must not raise
    +
         def test_close_survives_partial_failures(self):
             """close() continues cleanup even if one step fails."""
             from unittest.mock import patch