Truncation handling, chunking, output limit fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #1644 fix(tools): chunk long messages in send_message_tool before dispatch (#1552)

  merged to main

  files: gateway/platforms/base.py, tests/tools/test_send_message_tool.py, tools/send_message_tool.py

• #1646 fix(tools): chunk long messages in send_message_tool before platform dispatch

  merged to main

  files: gateway/platforms/base.py, optional-skills/blockchain/base/SKILL.md, optional-skills/blockchain/base/scripts/base_client.py, tests/tools/test_send_message_tool.py, tools/send_message_tool.py

• #3426 fix(anthropic): use model-native output limits instead of hardcoded 16K

  merged to main

  files: agent/anthropic_adapter.py, run_agent.py, tests/test_anthropic_adapter.py

• #22775 fix(api-server): emit length/error finish_reason for truncation/failure

  merged to main

  files: gateway/platforms/api_server.py, tests/gateway/test_api_server.py

  Show the diff

  • gateway/platforms/api_server.py

    @@ -1206,10 +1206,49 @@ async def _compute_completion():
                         status=500,
                     )
     
    -        final_response = result.get("final_response", "")
    -        if not final_response:
    -            final_response = result.get("error", "(No response generated)")
    +        final_response = result.get("final_response") or ""
    +        is_partial = bool(result.get("partial"))
    +        is_failed = bool(result.get("failed"))
    +        completed = bool(result.get("completed", True))
    +        err_msg = result.get("error")
    +
    +        # Decide finish_reason. OpenAI uses "length" for truncation, "stop"
    +        # for normal completion, and downstream SDKs accept "error" / custom
    +        # codes. See issue #22496.
    +        if is_partial and err_msg and "truncat" in err_msg.lower():
    +            finish_reason = "length"
    +        elif is_failed or (not completed and err_msg):
    +            finish_reason = "error"
    +        else:
    +            finish_reason = "stop"
    +
    +        response_headers = {
    +            "X-Hermes-Session-Id": result.get("session_id", session_id),
    +        }
    +        if gateway_session_key:
    +            response_headers["X-Hermes-Session-Key"] = gateway_session_key
     
    +        # Hard-fail path: no usable assistant text AND a real failure → 5xx
    +        # with OpenAI-style error envelope so SDK clients raise instead of
    +        # silently rendering the internal failure string as message.content.
    +        if not final_response and (is_failed or is_partial):
    +            err_body = _openai_error(
    +                err_msg or "Agent run did not produce a response.",
    +                err_type="server_error",
    +                code="agent_incomplete",
    +            )
    +            err_body["error"]["hermes"] = {
    +                "completed": completed,
    +                "partial": is_partial,
    +                "failed": is_failed,
    +            }
    +            response_headers["X-Hermes-Completed"] = "false"
    +            response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
    +            return web.json_response(err_body, status=502, headers=response_headers)
    +
    +        # Soft-partial path: we have *some* text but the run did not complete
    +        # (e.g. truncation with partial buffered output). Still 200 but signal
    +        # truncation via finish_reason="length" + Hermes-specific extras.
             response_data = {
                 "id": completion_id,
                 "object": "chat.completion",
    @@ -1222,7 +1261,7 @@ async def _compute_completion():
                             "role": "assistant",
                             "content": final_response,
                         },
    -                    "finish_reason": "stop",
    +                    "finish_reason": finish_reason,
                     }
                 ],
                 "usage": {
    @@ -1231,12 +1270,19 @@ async def _compute_completion():
                     "total_tokens": usage.get("total_tokens", 0),
                 },
             }
    +        if is_partial or is_failed or not completed:
    +            response_data["hermes"] = {
    +                "completed": completed,
    +                "partial": is_partial,
    +                "failed": is_failed,
    +                "error": err_msg,
    +                "error_code": "output_truncated" if finish_reason == "length" else "agent_error",
    +            }
    +            response_headers["X-Hermes-Completed"] = "false"
    +            response_headers["X-Hermes-Partial"] = "true" if is_partial else "false"
    +            if err_msg:
    +                response_headers["X-Hermes-Error"] = err_msg[:200]
     
    -        response_headers = {
    -            "X-Hermes-Session-Id": result.get("session_id", session_id),
    -        }
    -        if gateway_session_key:
    -            response_headers["X-Hermes-Session-Key"] = gateway_session_key
             return web.json_response(response_data, headers=response_headers)
     
         async def _write_sse_chat_completion(

  • tests/gateway/test_api_server.py

    @@ -2418,6 +2418,109 @@ async def test_no_truncation_keeps_full_history(self, adapter):
             assert len(call_kwargs["conversation_history"]) == 150
     
     
    +# ---------------------------------------------------------------------------
    +# Response-side truncation / failure handling (issue #22496)
    +# ---------------------------------------------------------------------------
    +
    +
    +class TestChatCompletionsAgentIncomplete:
    +    """When the agent run yields a partial / failed result, the API server
    +    must NOT pretend it succeeded. Either signal truncation via
    +    finish_reason='length' (with the partial text), or 502 with an OpenAI
    +    error envelope (no usable text). Issue #22496."""
    +
    +    @pytest.mark.asyncio
    +    async def test_truncation_with_partial_text_uses_length_finish_reason(self, adapter):
    +        """Partial text + truncation marker → finish_reason='length', 200 OK,
    +        plus hermes extras + headers."""
    +        mock_result = {
    +            "final_response": "Here is part one of the answer",
    +            "completed": False,
    +            "partial": True,
    +            "error": "Response truncated due to output length limit",
    +            "messages": [],
    +            "api_calls": 1,
    +        }
    +        app = _create_app(adapter)
    +        async with TestClient(TestServer(app)) as cli:
    +            with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
    +                mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0})
    +                resp = await cli.post(
    +                    "/v1/chat/completions",
    +                    json={"model": "hermes-agent", "messages": [{"role": "user", "content": "tell me everything"}]},
    +                )
    +            assert resp.status == 200
    +            data = await resp.json()
    +            assert data["choices"][0]["finish_reason"] == "length"
    +            assert data["choices"][0]["message"]["content"] == "Here is part one of the answer"
    +            assert data["hermes"]["partial"] is True
    +            assert data["hermes"]["completed"] is False
    +            assert data["hermes"]["error_code"] == "output_truncated"
    +            assert resp.headers.get("X-Hermes-Completed") == "false"
    +            assert resp.headers.get("X-Hermes-Partial") == "true"
    +
    +    @pytest.mark.asyncio
    +    async def test_failure_with_no_text_returns_502_error_envelope(self, adapter):
    +        """No usable assistant text + failure → 502 with OpenAI error envelope.
    +
    +        Pre-fix behavior: the failure string ('Response remained truncated...')
    +        was substituted into message.content with finish_reason='stop',
    +        making API clients think the agent had answered.
    +        """
    +        mock_result = {
    +            "final_response": None,
    +            "completed": False,
    +            "partial": True,
    +            "failed": True,
    +            "error": "Response remained truncated after 3 continuation attempts",
    +            "messages": [],
    +            "api_calls": 1,
    +        }
    +        app = _create_app(adapter)
    +        async with TestClient(TestServer(app)) as cli:
    +            with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
    +                mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0})
    +                resp = await cli.post(
    +                    "/v1/chat/completions",
    +                    json={"model": "hermes-agent", "messages": [{"role": "user", "content": "x"}]},
    +                )
    +            # Hard fail: SDK clients will raise on this status
    +            assert resp.status == 502
    +            data = await resp.json()
    +            assert data["error"]["code"] == "agent_incomplete"
    +            assert "truncated" in data["error"]["message"].lower()
    +            assert data["error"]["hermes"]["partial"] is True
    +            assert data["error"]["hermes"]["failed"] is True
    +            assert resp.headers.get("X-Hermes-Completed") == "false"
    +
    +    @pytest.mark.asyncio
    +    async def test_normal_completion_unchanged(self, adapter):
    +        """Sanity: a completed-True result still returns finish_reason='stop'
    +        and no hermes extras (preserves the existing happy-path contract)."""
    +        mock_result = {
    +            "final_response": "All good.",
    +            "completed": True,
    +            "partial": False,
    +            "failed": False,
    +            "messages": [],
    +            "api_calls": 1,
    +        }
    +        app = _create_app(adapter)
    +        async with TestClient(TestServer(app)) as cli:
    +            with patch.object(adapter, "_run_agent", new_callable=AsyncMock) as mock_run:
    +                mock_run.return_value = (mock_result, {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0})
    +                resp = await cli.post(
    +                    "/v1/chat/completions",
    +                    json={"model": "hermes-agent", "messages": [{"role": "user", "content": "hi"}]},
    +                )
    +            assert resp.status == 200
    +            data = await resp.json()
    +            assert data["choices"][0]["finish_reason"] == "stop"
    +            assert data["choices"][0]["message"]["content"] == "All good."
    +            assert "hermes" not in data
    +            assert "X-Hermes-Completed" not in resp.headers
    +
    +
     # ---------------------------------------------------------------------------
     # CORS
     # ---------------------------------------------------------------------------

• #36705 fix(agent): stop reporting broken streams as output-length truncation

  merged to main

  files: agent/conversation_loop.py, run_agent.py, tests/run_agent/test_run_agent.py

  Show the diff

  • agent/conversation_loop.py

    @@ -1739,20 +1739,52 @@ def _stop_spinner():
                         if agent.api_mode in {"chat_completions", "bedrock_converse", "anthropic_messages"}:
                             assistant_message = _trunc_msg
                             if assistant_message is not None and _trunc_has_tool_calls:
    -                            if truncated_tool_call_retries < 1:
    +                            _is_stub_stall = (
    +                                getattr(response, "id", "") == PARTIAL_STREAM_STUB_ID
    +                            )
    +                            if truncated_tool_call_retries < 3:
                                     truncated_tool_call_retries += 1
    -                                agent._buffer_vprint(
    -                                    f"⚠️  Truncated tool call detected — retrying API call..."
    -                                )
    +                                if _is_stub_stall:
    +                                    # The stream broke mid tool-call (network /
    +                                    # peer-closed connection), not a real output
    +                                    # cap — say so instead of "max output tokens".
    +                                    agent._buffer_vprint(
    +                                        f"⚠️  Stream interrupted mid tool-call — "
    +                                        f"retrying ({truncated_tool_call_retries}/3)..."
    +                                    )
    +                                else:
    +                                    agent._buffer_vprint(
    +                                        f"⚠️  Truncated tool call detected — "
    +                                        f"retrying API call "
    +                                        f"({truncated_tool_call_retries}/3)..."
    +                                    )
    +                                # Boost max_tokens on each retry so the model has
    +                                # more room to complete the tool-call JSON. A
    +                                # network stall doesn't need a bigger budget, but
    +                                # a genuine output-cap truncation does, and the
    +                                # boost is harmless for the stall case.
    +                                _tc_boost_base = agent.max_tokens if agent.max_tokens else 4096
    +                                _tc_boost = _tc_boost_base * (truncated_tool_call_retries + 1)
    +                                _tc_requested_cap = agent._requested_output_cap_from_api_kwargs(api_kwargs)
    +                                if _tc_requested_cap is not None:
    +                                    _tc_boost = max(_tc_boost, _tc_requested_cap)
    +                                _tc_boost_cap = max(32768, _tc_requested_cap or 0)
    +                                agent._ephemeral_max_output_tokens = [redacted:secret-assignment] _tc_boost_cap)
                                     # Don't append the broken response to messages;
                                     # just re-run the same API call from the current
                                     # message state, giving the model another chance.
                                     continue
                                 agent._flush_status_buffer()
    -                            agent._vprint(
    -                                f"{agent.log_prefix}⚠️  Truncated tool call response detected again — refusing to execute incomplete tool arguments.",
    -                                force=True,
    -                            )
    +                            if _is_stub_stall:
    +                                agent._vprint(
    +                                    f"{agent.log_prefix}⚠️  Stream kept dropping mid tool-call after 3 retries — the action was not executed.",
    +                                    force=True,
    +                                )
    +                            else:
    +                                agent._vprint(
    +                                    f"{agent.log_prefix}⚠️  Truncated tool call response detected again — refusing to execute incomplete tool arguments.",
    +                                    force=True,
    +                                )
                                 agent._cleanup_task_resources(effective_task_id)
                                 agent._persist_session(messages, conversation_history)
                                 return {
    @@ -1761,7 +1793,12 @@ def _stop_spinner():
                                     "api_calls": api_call_count,
                                     "completed": False,
                                     "partial": True,
    -                                "error": "Response truncated due to output length limit",
    +                                "error": (
    +                                    "Stream repeatedly dropped mid tool-call (network); "
    +                                    "the tool was not executed"
    +                                    if _is_stub_stall
    +                                    else "Response truncated due to output length limit"
    +                                ),
                                 }
     
                         # If we have prior messages, roll back to last complete state
    @@ -3412,9 +3449,16 @@ def _stop_spinner():
                 # Progressively boost the output token budget on each retry.
                 # Retry 1 → 2× base, retry 2 → 3× base, capped at 32 768.
                 # Applies to all providers via _ephemeral_max_output_tokens.
    +            # If the original request already used a larger provider/model
    +            # default budget, keep that floor so continuation retries do
    +            # not accidentally downshift to a much smaller cap.
                 _boost_base = agent.max_tokens if agent.max_tokens else 4096
                 _boost = _boost_base * (length_continue_retries + 1)
    -            agent._ephemeral_max_output_tokens = [redacted:secret-assignment] 32768)
    +            _requested_cap = agent._requested_output_cap_from_api_kwargs(api_kwargs)
    +            if _requested_cap is not None:
    +                _boost = max(_boost, _requested_cap)
    +            _boost_cap = max(32768, _requested_cap or 0)
    +            agent._ephemeral_max_output_tokens = [redacted:secret-assignment] _boost_cap)
                 continue
     
             # Guard: if all retries exhausted without a successful response

  • run_agent.py

    @@ -1206,6 +1206,21 @@ def _max_tokens_param(self, value: int) -> dict:
                 return {"max_completion_tokens": value}
             return {"max_tokens": value}
     
    +    @staticmethod
    +    def _requested_output_cap_from_api_kwargs(api_kwargs: Any) -> Optional[int]:
    +        """Extract the outgoing response token cap from a prepared request."""
    +        if not isinstance(api_kwargs, dict):
    +            return None
    +        for key in ("max_output_tokens", "max_completion_tokens", "max_tokens"):
    +            raw = api_kwargs.get(key)
    +            try:
    +                value = int(raw)
    +            except (TypeError, ValueError):
    +                continue
    +            if value > 0:
    +                return value
    +        return None
    +
         def _has_content_after_think_block(self, content: str) -> bool:
             """
             Check if content has actual text after any reasoning/thinking blocks.

  • tests/run_agent/test_run_agent.py

    @@ -3699,6 +3699,36 @@ def test_length_finish_reason_requests_continuation(self, agent):
             assert second_call_messages[-1]["role"] == "user"
             assert "truncated by the output length limit" in second_call_messages[-1]["content"]
     
    +    def test_length_continuation_preserves_large_provider_default_output_cap(self, agent):
    +        """Continuation retries must not shrink a higher provider default cap."""
    +        self._setup_agent(agent)
    +        agent.max_tokens = None
    +        requested_caps = []
    +
    +        def _fake_build_api_kwargs(api_messages):
    +            ephemeral = getattr(agent, "_ephemeral_max_output_tokens", None)
    +            if ephemeral is not None:
    +                agent._ephemeral_max_output_tokens = None
    +            cap = ephemeral if ephemeral is not None else 65536
    +            requested_caps.append(cap)
    +            return {"model": agent.model, "messages": api_messages, "max_tokens": cap}
    +
    +        first = _mock_response(content="Part 1 ", finish_reason="length")
    +        second = _mock_response(content="Part 2", finish_reason="stop")
    +        agent.client.chat.completions.create.side_effect = [first, second]
    +
    +        with (
    +            patch.object(agent, "_build_api_kwargs", side_effect=_fake_build_api_kwargs),
    +            patch.object(agent, "_persist_session"),
    +            patch.object(agent, "_save_trajectory"),
    +            patch.object(agent, "_cleanup_task_resources"),
    +        ):
    +            result = agent.run_conversation("hello")
    +
    +        assert result["completed"] is True
    +        assert result["final_response"] == "Part 1 Part 2"
    +        assert requested_caps == [65536, 65536]
    +
         def test_ollama_glm_stop_after_tools_without_terminal_boundary_requests_continuation(self, agent):
             """Ollama-hosted GLM responses can misreport truncated output as stop."""
             self._setup_agent(agent)
    @@ -3877,7 +3907,8 @@ def test_length_with_tool_calls_returns_partial_without_executing_tools(self, ag
     
         def test_truncated_tool_call_retries_once_before_refusing(self, agent):
             """When tool call args are truncated, the agent retries the API call
    -        once. If the retry succeeds (valid JSON args), tool execution proceeds."""
    +        (up to 3 times). If a retry succeeds (valid JSON args), tool execution
    +        proceeds."""
             self._setup_agent(agent)
             agent.valid_tool_names.add("write_file")
             bad_tc = _mock_tool_call(
    @@ -3914,6 +3945,48 @@ def test_truncated_tool_call_retries_once_before_refusing(self, agent):
             mock_hfc.assert_called_once()
             assert result["final_response"] == "Done!"
     
    +    def test_stub_stall_mid_tool_call_recovers_within_3_retries(self, agent):
    +        """A network stream stall mid tool-call (PARTIAL_STREAM_STUB_ID) must
    +        retry up to 3 times rather than hard-failing after one — and recover
    +        if a retry produces a complete tool call. Regression for the false
    +        'model hit max output tokens' on Opus when the stream simply dropped."""
    +        from hermes_constants import PARTIAL_STREAM_STUB_ID
    +
    +        self._setup_agent(agent)
    +        agent.valid_tool_names.add("write_file")
    +        bad_tc = _mock_tool_call(
    +            name="write_file",
    +            arguments='{"path":"report.md","content":"partial',
    +            call_id="c1",
    +        )
    +        # Two consecutive stub-stall responses, then a clean tool call.
    +        stall1 = _mock_response(content="", finish_reason="length", tool_calls=[bad_tc])
    +        stall1.id = PARTIAL_STREAM_STUB_ID
    +        stall2 = _mock_response(content="", finish_reason="length", tool_calls=[bad_tc])
    +        stall2.id = PARTIAL_STREAM_STUB_ID
    +        good_tc = _mock_tool_call(
    +            name="write_file",
    +            arguments='{"path":"report.md","content":"full content"}',
    +            call_id="c2",
    +        )
    +        good_resp = _mock_response(content="", finish_reason="stop", tool_calls=[good_tc])
    +        final_resp = _mock_response(content="Done!", finish_reason="stop")
    +
    +        with (
    +            patch("run_agent.handle_function_call", return_value='{"success":true}') as mock_hfc,
    +            patch.object(agent, "_persist_session"),
    +            patch.object(agent, "_save_trajectory"),
    +            patch.object(agent, "_cleanup_task_resources"),
    +        ):
    +            agent.client.chat.completions.create.side_effect = [
    +                stall1, stall2, good_resp, final_resp,
    +            ]
    +            result = agent.run_conversation("write the report")
    +
    +        # Recovered on the 3rd attempt instead of refusing after the 1st.
    +        mock_hfc.assert_called_once()
    +        assert result["final_response"] == "Done!"
    +
         def test_truncated_tool_args_detected_when_finish_reason_not_length(self, agent):
             """When a router rewrites finish_reason from 'length' to 'tool_calls',
             truncated JSON arguments should still be detected and refused rather

• #115825 fix(agent): router 'Connect timeout' HTTP-200 shim is retried instead of returned as the answer, on every path (#68396, salvage #68433)

  merged to main

  files: agent/auxiliary_client.py, agent/chat_completion_helpers.py, agent/transports/chat_completions.py, tests/agent/test_router_timeout_shim.py, tests/agent/transports/test_chat_completions.py, website/docs/user-guide/features/fallback-providers.md