Scheduler graceful-delivery exemption and budget-exhausted report delivery

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #61900 fix(cron): preserve reports across verification budget exhaustion

  merged to main

  files: agent/conversation_loop.py, agent/turn_finalizer.py, tests/agent/test_turn_finalizer_iteration_limit_exit.py, tests/run_agent/test_verification_continuation_budget.py

  Show the diff

  • agent/conversation_loop.py

    @@ -613,6 +613,11 @@ def run_conversation(
         truncated_response_parts: List[str] = []
         compression_attempts = 0
         _turn_exit_reason = "unknown"  # Diagnostic: why the loop ended
    +    # Last composed answer intentionally held back by a verification gate. If
    +    # that continuation consumes the remaining budget, this is the best
    +    # user-facing result available; it must not be confused with error or
    +    # recovery text produced by unrelated exit paths.
    +    _pending_verification_response = None
     
         # Per-turn tally of consecutive successful credential-pool token refreshes,
         # keyed by (provider, pool-entry-id). A persistent upstream 401 lets
    @@ -5099,6 +5104,10 @@ def _perform_api_call(next_api_kwargs):
                         }
                         messages.append(continue_msg)
                         agent._session_messages = messages
    +                    # An acknowledgment is explicitly non-final. Do not let its
    +                    # text suppress iteration-limit summarization if this
    +                    # continuation consumes the remaining budget.
    +                    final_response = None
                         continue
     
                     codex_ack_continuations = 0
    @@ -5173,6 +5182,12 @@ def _perform_api_call(next_api_kwargs):
                         # terminal. Keep a debug breadcrumb in agent.log for tracing.
                         logger.debug("verification stop-loop nudge issued (attempt %d)",
                                      agent._verification_stop_nudges)
    +                    # Keep the attempted answer only as an explicit fallback for
    +                    # continuation-budget exhaustion.  ``final_response`` itself
    +                    # must be cleared so the finalizer can distinguish this gate
    +                    # from unrelated error/recovery exits. (#61631)
    +                    _pending_verification_response = final_response
    +                    final_response = None
                         continue
     
                     # User verification-loop gate: when the agent edited code this
    @@ -5224,6 +5239,8 @@ def _perform_api_call(next_api_kwargs):
                         agent._session_messages = messages
                         logger.debug("pre_verify nudge issued (attempt %d)",
                                      agent._pre_verify_nudges)
    +                    _pending_verification_response = final_response
    +                    final_response = None
                         continue
     
                     messages.append(final_msg)
    @@ -5308,6 +5325,7 @@ def _perform_api_call(next_api_kwargs):
             original_user_message=original_user_message,
             _should_review_memory=_should_review_memory,
             _turn_exit_reason=_turn_exit_reason,
    +        _pending_verification_response=_pending_verification_response,
         )
     
     

  • agent/turn_finalizer.py

    @@ -42,6 +42,7 @@ def finalize_turn(
         original_user_message,
         _should_review_memory,
         _turn_exit_reason,
    +    _pending_verification_response=None,
     ):
         """Run the post-loop finalization and return the turn ``result`` dict.
     
    @@ -50,10 +51,35 @@ def finalize_turn(
         """
         from agent.conversation_loop import logger
     
    -    if final_response is None and (
    +    budget_exhausted = (
             api_call_count >= agent.max_iterations
             or agent.iteration_budget.remaining <= 0
    -    ):
    +    )
    +    budget_fallback_eligible = (
    +        budget_exhausted
    +        and not interrupted
    +        and not failed
    +        and str(_turn_exit_reason) in {"unknown", "budget_exhausted"}
    +    )
    +    continuation_budget_exhausted = (
    +        final_response is None
    +        and bool(_pending_verification_response)
    +        and budget_fallback_eligible
    +    )
    +
    +    iteration_limit_fallback = False
    +    preserved_verification_fallback = False
    +    if continuation_budget_exhausted:
    +        # A verification/continuation gate deliberately withheld a composed
    +        # answer, then consumed the remaining budget before producing a newer
    +        # one. Preserve that exact answer instead of replacing it with another
    +        # fallible model call. The explicit pending value is the provenance
    +        # guard: unrelated error/recovery exits can never enter this branch.
    +        final_response = _pending_verification_response
    +        _turn_exit_reason = f"max_iterations_reached({api_call_count}/{agent.max_iterations})"
    +        iteration_limit_fallback = True
    +        preserved_verification_fallback = True
    +    elif final_response is None and budget_fallback_eligible:
             # Budget exhausted — ask the model for a summary via one extra
             # API call with tools stripped.  _handle_max_iterations injects a
             # user message and makes a single toolless request.
    @@ -68,20 +94,18 @@ def finalize_turn(
                     "— requesting summary..."
                 )
             final_response = agent._handle_max_iterations(messages, api_call_count)
    +        iteration_limit_fallback = True
     
    +    if iteration_limit_fallback:
             # If running as a kanban worker, signal the dispatcher that the
             # worker could not complete (rather than treating it as a
    -        # protocol violation).  The agent loop strips tools before calling
    -        # _handle_max_iterations, so the model cannot call kanban_block
    -        # itself — we must do it on its behalf.
    +        # protocol violation). This applies whether the user-facing fallback
    +        # came from the summary call or an explicitly pending continuation;
    +        # both exhausted the task budget and must advance the failure circuit.
             #
             # We route through ``_record_task_failure(outcome="timed_out")``
    -        # rather than ``kanban_block`` so this counts toward the
    -        # ``consecutive_failures`` counter and the dispatcher's
    -        # ``failure_limit`` circuit breaker (#29747 gap 2).  Without this,
    -        # a task whose worker keeps exhausting its budget would block
    -        # silently each run, get auto-promoted by the operator (or never
    -        # surface), and re-block in an endless loop with no signal.
    +        # rather than ``kanban_block`` so this counts toward the dispatcher's
    +        # consecutive-failure circuit breaker (#29747 gap 2).
             _kanban_task = os.environ.get("HERMES_KANBAN_TASK")
             if _kanban_task:
                 try:
    @@ -304,6 +328,7 @@ def finalize_turn(
                     # truncated partial (the "The" case from #34452).
                     _is_partial_fragment = (
                         not _is_empty_terminal
    +                    and not preserved_verification_fallback
                         and not str(_turn_exit_reason).startswith("text_response")
                         and len(_stripped) <= 24
                         and _stripped[-1:] not in {".", "!", "?", "。", "！", "？", "`", ")"}

  • tests/agent/test_turn_finalizer_iteration_limit_exit.py

    @@ -0,0 +1,298 @@
    +"""Regression tests for iteration-limit exit normalization (#61631)."""
    +
    +from types import SimpleNamespace
    +from unittest.mock import MagicMock
    +
    +import pytest
    +
    +from agent.turn_finalizer import finalize_turn
    +
    +
    +class _LimitAgent:
    +    def __init__(
    +        self,
    +        *,
    +        max_iterations=60,
    +        budget_remaining=0,
    +        completion_explainer=False,
    +    ):
    +        self.max_iterations = max_iterations
    +        self.iteration_budget = SimpleNamespace(
    +            remaining=budget_remaining, used=max_iterations, max_total=max_iterations
    +        )
    +        self.quiet_mode = True
    +        self.model = "test-model"
    +        self.provider = "test-provider"
    +        self.base_url = ""
    +        self.session_id = "sess-test"
    +        self.context_compressor = SimpleNamespace(last_prompt_tokens=0)
    +        self.session_input_tokens = 0
    +        self.session_output_tokens = 0
    +        self.session_cache_read_tokens = 0
    +        self.session_cache_write_tokens = 0
    +        self.session_reasoning_tokens = 0
    +        self.session_prompt_tokens = 0
    +        self.session_completion_tokens = 0
    +        self.session_total_tokens = 0
    +        self.session_estimated_cost_usd = 0
    +        self.session_cost_status = "unknown"
    +        self.session_cost_source = "test"
    +        self._tool_guardrail_halt_decision = None
    +        self._interrupt_message = None
    +        self._response_was_previewed = False
    +        self._skill_nudge_interval = 0
    +        self._iters_since_skill = 0
    +        self.valid_tool_names = []
    +        self.persisted_messages = None
    +        self._handle_max_iterations_called = False
    +        self._completion_explainer = completion_explainer
    +
    +    def _handle_max_iterations(self, messages, api_call_count):
    +        self._handle_max_iterations_called = True
    +        return "summary from extra call"
    +
    +    def _emit_status(self, *_args, **_kwargs):
    +        pass
    +
    +    def _safe_print(self, *_args, **_kwargs):
    +        pass
    +
    +    def _save_trajectory(self, *_args, **_kwargs):
    +        pass
    +
    +    def _cleanup_task_resources(self, *_args, **_kwargs):
    +        pass
    +
    +    def _drop_trailing_empty_response_scaffolding(self, messages):
    +        pass
    +
    +    def _persist_session(self, messages, conversation_history):
    +        self.persisted_messages = list(messages)
    +
    +    def _file_mutation_verifier_enabled(self):
    +        return False
    +
    +    def _turn_completion_explainer_enabled(self):
    +        return self._completion_explainer
    +
    +    def _format_turn_completion_explanation(self, _reason):
    +        return "iteration-limit explanation"
    +
    +    def _drain_pending_steer(self):
    +        return None
    +
    +    def clear_interrupt(self):
    +        pass
    +
    +    def _sync_external_memory_for_turn(self, **_kwargs):
    +        pass
    +
    +
    +def _finalize(
    +    agent,
    +    *,
    +    final_response,
    +    exit_reason,
    +    api_call_count=60,
    +    pending_verification_response=None,
    +):
    +    return finalize_turn(
    +        agent,
    +        final_response=final_response,
    +        api_call_count=api_call_count,
    +        interrupted=False,
    +        failed=False,
    +        messages=[{"role": "user", "content": "task"}],
    +        conversation_history=[],
    +        effective_task_id="task",
    +        turn_id="turn",
    +        user_message="task",
    +        original_user_message="task",
    +        _should_review_memory=False,
    +        _turn_exit_reason=exit_reason,
    +        _pending_verification_response=pending_verification_response,
    +    )
    +
    +
    +def test_pending_verify_response_is_preserved_for_cron_delivery(monkeypatch):
    +    """A held-back verification response survives last-turn exhaustion."""
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent()
    +    report = "complete cron report body"
    +
    +    result = _finalize(
    +        agent,
    +        final_response=None,
    +        exit_reason="unknown",
    +        pending_verification_response=report,
    +    )
    +
    +    assert result["final_response"] == report
    +    assert result["turn_exit_reason"] == "max_iterations_reached(60/60)"
    +    assert agent._handle_max_iterations_called is False
    +
    +
    +def test_pending_pre_verify_response_is_preserved_on_budget_exhaustion(monkeypatch):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent()
    +    report = "budget exhausted but complete"
    +
    +    result = _finalize(
    +        agent,
    +        final_response=None,
    +        exit_reason="budget_exhausted",
    +        pending_verification_response=report,
    +    )
    +
    +    assert result["final_response"] == report
    +    assert result["turn_exit_reason"] == "max_iterations_reached(60/60)"
    +    assert agent._handle_max_iterations_called is False
    +
    +
    +def test_empty_pending_verification_response_uses_summary_fallback(monkeypatch):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent()
    +
    +    result = _finalize(
    +        agent,
    +        final_response=None,
    +        exit_reason="unknown",
    +        pending_verification_response="",
    +    )
    +
    +    assert result["final_response"] == "summary from extra call"
    +    assert result["turn_exit_reason"] == "max_iterations_reached(60/60)"
    +    assert agent._handle_max_iterations_called is True
    +
    +
    +def test_short_generated_summary_keeps_abnormal_turn_explainer(monkeypatch):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent(completion_explainer=True)
    +    agent._handle_max_iterations = lambda *_args: "The"
    +
    +    result = _finalize(agent, final_response=None, exit_reason="unknown")
    +
    +    assert result["final_response"] == "The\n\niteration-limit explanation"
    +
    +
    +def test_short_preserved_verification_response_is_not_rewritten(monkeypatch):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent(completion_explainer=True)
    +
    +    result = _finalize(
    +        agent,
    +        final_response=None,
    +        exit_reason="unknown",
    +        pending_verification_response="The",
    +    )
    +
    +    assert result["final_response"] == "The"
    +
    +
    +def test_text_response_exit_not_rewritten_at_iteration_limit(monkeypatch):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent(budget_remaining=5)
    +    exit_reason = "text_response(finish_reason=stop)"
    +
    +    result = _finalize(
    +        agent,
    +        final_response="normal answer",
    +        exit_reason=exit_reason,
    +        api_call_count=59,
    +    )
    +
    +    assert result["turn_exit_reason"] == exit_reason
    +    assert agent._handle_max_iterations_called is False
    +
    +
    [redacted:email](
    +    "exit_reason",
    +    [
    +        "error_near_max_iterations(boom)",
    +        "guardrail_halt",
    +        "partial_stream_recovery",
    +        "fallback_prior_turn_content",
    +        "empty_response_exhausted",
    +    ],
    +)
    +def test_unrelated_non_success_response_is_not_reclassified(monkeypatch, exit_reason):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent()
    +
    +    result = _finalize(
    +        agent,
    +        final_response="diagnostic or partial content",
    +        exit_reason=exit_reason,
    +    )
    +
    +    assert result["turn_exit_reason"] == exit_reason
    +    assert result["completed"] is False
    +    assert agent._handle_max_iterations_called is False
    +
    +
    [redacted:email](
    +    ("exit_reason", "interrupted", "failed"),
    +    [
    +        ("interrupted_by_user", True, False),
    +        ("all_retries_exhausted_no_response", False, False),
    +        ("provider_failure", False, True),
    +    ],
    +)
    +def test_pending_response_does_not_mask_later_terminal_exit(
    +    monkeypatch, exit_reason, interrupted, failed
    +):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    agent = _LimitAgent()
    +
    +    result = finalize_turn(
    +        agent,
    +        final_response=None,
    +        api_call_count=60,
    +        interrupted=interrupted,
    +        failed=failed,
    +        messages=[{"role": "user", "content": "task"}],
    +        conversation_history=[],
    +        effective_task_id="task",
    +        turn_id="turn",
    +        user_message="task",
    +        original_user_message="task",
    +        _should_review_memory=False,
    +        _turn_exit_reason=exit_reason,
    +        _pending_verification_response="stale premature report",
    +    )
    +
    +    assert result["final_response"] is None
    +    assert result["turn_exit_reason"] == exit_reason
    +    assert result["completed"] is False
    +    assert agent._handle_max_iterations_called is False
    +
    +
    +def test_pending_response_records_kanban_timeout(monkeypatch):
    +    monkeypatch.setattr("hermes_cli.plugins.invoke_hook", lambda *_a, **_kw: [])
    +    monkeypatch.setenv("HERMES_KANBAN_TASK", "task-123")
    +    record = MagicMock(name="record_task_failure")
    +    conn = SimpleNamespace(close=lambda: None)
    +    monkeypatch.setattr("hermes_cli.kanban_db.connect", lambda: conn)
    +    monkeypatch.setattr("hermes_cli.kanban_db._record_task_failure", record)
    +    agent = _LimitAgent()
    +
    +    result = _finalize(
    +        agent,
    +        final_response=None,
    +        exit_reason="unknown",
    +        pending_verification_response="composed report",
    +    )
    +
    +    assert result["turn_exit_reason"] == "max_iterations_reached(60/60)"
    +    record.assert_called_once_with(
    +        conn,
    +        "task-123",
    +        error=(
    +            "Iteration budget exhausted (60/60) — task could not complete "
    +            "within the allowed iterations"
    +        ),
    +        outcome="timed_out",
    +        release_claim=True,
    +        end_run=True,
    +        event_payload_extra={"budget_used": 60, "budget_max": 60},
    +    )

  • tests/run_agent/test_verification_continuation_budget.py

    @@ -0,0 +1,146 @@
    +"""End-to-end regression coverage for verification budget exhaustion (#61631)."""
    +
    +from types import SimpleNamespace
    +from unittest.mock import MagicMock, patch
    +
    +import pytest
    +
    +from run_agent import AIAgent
    +
    +
    +def _response(content="composed report"):
    +    message = SimpleNamespace(content=content, tool_calls=None)
    +    return SimpleNamespace(
    +        choices=[SimpleNamespace(message=message, finish_reason="stop")],
    +        model="test/model",
    +        usage=None,
    +    )
    +
    +
    [redacted:email]
    +def agent(tmp_path, monkeypatch):
    +    monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
    +    with (
    +        patch("run_agent.get_tool_definitions", return_value=[]),
    +        patch("run_agent.check_toolset_requirements", return_value={}),
    +        patch("run_agent.OpenAI"),
    +    ):
    +        instance = AIAgent(
    +            session_id="verify-budget-test",
    +            api_key="[redacted:secret-assignment]",
    +            base_url="https://example.invalid/v1",
    +            provider="openai-compat",
    +            model="test/model",
    +            max_iterations=1,
    +            quiet_mode=True,
    +            skip_context_files=True,
    +            skip_memory=True,
    +        )
    +    instance._cached_system_prompt = "stable test prompt"
    +    instance._session_db = None
    +    instance._session_json_enabled = False
    +    instance.save_trajectories = False
    +    instance.compression_enabled = False
    +    instance._cleanup_task_resources = lambda *_a, **_kw: None
    +    instance._save_trajectory = lambda *_a, **_kw: None
    +    return instance
    +
    +
    +def _assert_pending_response_survives(agent, result):
    +    assert result["final_response"] == "composed report"
    +    assert result["turn_exit_reason"] == "max_iterations_reached(1/1)"
    +    assert result["completed"] is False
    +    assert agent._handle_max_iterations.call_count == 0
    +    assert [message["role"] for message in result["messages"]] == [
    +        "user",
    +        "assistant",
    +        "user",
    +        "assistant",
    +    ]
    +
    +
    +def test_verify_on_stop_preserves_composed_report_at_budget_limit(agent, monkeypatch):
    +    def model_call(_api_kwargs):
    +        agent._turn_file_mutation_paths = {"changed.py"}
    +        return _response()
    +
    +    agent._interruptible_api_call = model_call
    +    agent._handle_max_iterations = MagicMock(return_value="replacement summary")
    +    monkeypatch.setenv("HERMES_VERIFY_ON_STOP", "1")
    +
    +    with (
    +        patch("agent.verification_stop.build_verify_on_stop_nudge", return_value="verify it"),
    +        patch("hermes_cli.plugins.invoke_hook", return_value=[]),
    +    ):
    +        result = agent.run_conversation("edit changed.py")
    +
    +    _assert_pending_response_survives(agent, result)
    +    assert result["messages"][1]["_verification_stop_synthetic"] is True
    +    assert result["messages"][2]["_verification_stop_synthetic"] is True
    +
    +
    +def test_pre_verify_preserves_composed_report_at_budget_limit(agent, monkeypatch):
    +    def model_call(_api_kwargs):
    +        agent._turn_file_mutation_paths = {"changed.py"}
    +        return _response()
    +
    +    agent._interruptible_api_call = model_call
    +    agent._handle_max_iterations = MagicMock(return_value="replacement summary")
    +    monkeypatch.setenv("HERMES_VERIFY_ON_STOP", "0")
    +
    +    with (
    +        patch("hermes_cli.plugins.has_hook", side_effect=lambda name: name == "pre_verify"),
    +        patch(
    +            "hermes_cli.plugins.get_pre_verify_continue_message",
    +            return_value="run project tests",
    +        ),
    +        patch("agent.verify_hooks.max_verify_nudges", return_value=2),
    +        patch("hermes_cli.plugins.invoke_hook", return_value=[]),
    +    ):
    +        result = agent.run_conversation("edit changed.py")
    +
    +    _assert_pending_response_survives(agent, result)
    +    assert result["messages"][1]["_pre_verify_synthetic"] is True
    +    assert result["messages"][2]["_pre_verify_synthetic"] is True
    +
    +
    +def test_intermediate_ack_uses_summary_instead_of_premature_text(agent, monkeypatch):
    +    agent.valid_tool_names = ["web_search"]
    +    agent._intent_ack_continuation = True
    +    agent._looks_like_codex_intermediate_ack = MagicMock(return_value=True)
    +    agent._interruptible_api_call = lambda _kwargs: _response("I'll inspect the files now")
    +    agent._handle_max_iterations = MagicMock(return_value="verified summary.")
    +    monkeypatch.setenv("HERMES_VERIFY_ON_STOP", "0")
    +
    +    with (
    +        patch("hermes_cli.plugins.has_hook", return_value=False),
    +        patch("hermes_cli.plugins.invoke_hook", return_value=[]),
    +    ):
    +        result = agent.run_conversation("inspect /tmp/project")
    +
    +    assert result["final_response"] == "verified summary."
    +    assert result["turn_exit_reason"] == "max_iterations_reached(1/1)"
    +    agent._handle_max_iterations.assert_called_once()
    +
    +
    +def test_later_verified_response_supersedes_pending_report(agent, monkeypatch):
    +    agent.max_iterations = 2
    +    agent.iteration_budget.max_total = 2
    +    answers = iter([_response("premature report"), _response("verified final report")])
    +    agent._interruptible_api_call = lambda _kwargs: next(answers)
    +    agent._handle_max_iterations = MagicMock(return_value="replacement summary")
    +    monkeypatch.setenv("HERMES_VERIFY_ON_STOP", "1")
    +
    +    with (
    +        patch(
    +            "agent.verification_stop.build_verify_on_stop_nudge",
    +            side_effect=["verify it", None],
    +        ),
    +        patch("hermes_cli.plugins.invoke_hook", return_value=[]),
    +    ):
    +        result = agent.run_conversation("edit changed.py")
    +
    +    assert result["final_response"] == "verified final report"
    +    assert result["turn_exit_reason"] == "text_response(finish_reason=stop)"
    +    assert result["completed"] is True
    +    agent._handle_max_iterations.assert_not_called()