Internal message fields leaking to strict API providers

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #31456 fix(transport): strip Hermes-internal scaffolding keys + classify 5xx validation errors (salvage #26054)

  merged to main

  files: agent/error_classifier.py, agent/transports/chat_completions.py, scripts/release.py, tests/agent/test_error_classifier.py, tests/agent/transports/test_chat_completions.py

  Show the diff

  • agent/error_classifier.py

    @@ -240,6 +240,24 @@ def is_auth(self) -> bool:
         "unsupported model",
     ]
     
    +# Request-validation patterns — the request is malformed and will fail
    +# identically on every retry. Some OpenAI-compatible gateways (notably
    +# codex.nekos.me) return these as 5xx instead of the standard 4xx, which
    +# makes the generic "5xx → retryable server_error" rule misfire: the retry
    +# loop hammers the same deterministic rejection 3+ times, then the
    +# transport-recovery path resets the counter and does it again, producing
    +# a request flood. When a 5xx body carries one of these unambiguous
    +# request-validation signals, classify as a non-retryable format_error so
    +# the loop fails fast and falls back instead of looping.
    +_REQUEST_VALIDATION_PATTERNS = [
    +    "unknown parameter",
    +    "unsupported parameter",
    +    "unrecognized request argument",
    +    "invalid_request_error",
    +    "unknown_parameter",
    +    "unsupported_parameter",
    +]
    +
     # OpenRouter aggregator policy-block patterns.
     #
     # When a user's OpenRouter account privacy setting (or a per-request
    @@ -745,6 +763,23 @@ def _classify_by_status(
             )
     
         if status_code in {500, 502}:
    +        # Some OpenAI-compatible gateways return request-validation errors
    +        # with a 5xx status (codex.nekos.me returns 502 for unknown/
    +        # unsupported parameters). These are deterministic — every retry
    +        # gets the identical rejection — so the generic "5xx → retryable
    +        # server_error" rule turns one bad request into a retry flood.
    +        # Detect the unambiguous request-validation signals (in either the
    +        # message text or the structured error code) and fail fast.
    +        if (
    +            any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS)
    +            or error_code.lower() in {"invalid_request_error", "unknown_parameter",
    +                                      "unsupported_parameter"}
    +        ):
    +            return result_fn(
    +                FailoverReason.format_error,
    +                retryable=False,
    +                should_fallback=True,
    +            )
             return result_fn(FailoverReason.server_error, retryable=True)
     
         if status_code in {503, 529}:

  • agent/transports/chat_completions.py

    @@ -113,9 +113,8 @@ def convert_messages(
             self, messages: list[dict[str, Any]], **kwargs
         ) -> list[dict[str, Any]]:
             """Messages are already in OpenAI format — strip internal fields
    -        that strict chat-completions providers reject with HTTP 400/422.
    -
    -        Strips:
    +        that strict chat-completions providers reject with HTTP 400/422
    +        (or, in the case of some OpenAI-compatible gateways, 5xx):
     
             - Codex Responses API fields: ``codex_reasoning_items`` /
               ``codex_message_items`` on the message, ``call_id`` /
    @@ -127,6 +126,16 @@ def convert_messages(
               ``Extra inputs are not permitted, field: 'messages[N].tool_name'``.
               Permissive providers (OpenRouter, MiniMax) silently ignore the
               field, which masked the bug for months.
    +        - Hermes-internal scaffolding markers — any top-level message key
    +          starting with ``_`` (e.g. ``_empty_recovery_synthetic``,
    +          ``_empty_terminal_sentinel``, ``_thinking_prefill``). These are
    +          bookkeeping flags the agent loop attaches to messages so the
    +          persistence layer can later strip its own scaffolding; they must
    +          never reach the wire. Permissive providers (real OpenAI,
    +          Anthropic) silently drop unknown message keys, but strict
    +          gateways (e.g. opencode-go, codex.nekos.me) reject with
    +          ``Extra inputs are not permitted, field: 'messages[N]._empty_recovery_synthetic'``,
    +          which then poisons every subsequent request in the session.
             """
             needs_sanitize = False
             for msg in messages:
    @@ -139,6 +148,9 @@ def convert_messages(
                 ):
                     needs_sanitize = True
                     break
    +            if any(isinstance(k, str) and k.startswith("_") for k in msg):
    +                needs_sanitize = True
    +                break
                 tool_calls = msg.get("tool_calls")
                 if isinstance(tool_calls, list):
                     for tc in tool_calls:
    @@ -160,6 +172,11 @@ def convert_messages(
                 msg.pop("codex_reasoning_items", None)
                 msg.pop("codex_message_items", None)
                 msg.pop("tool_name", None)
    +            # Drop all Hermes-internal scaffolding markers (``_``-prefixed).
    +            # OpenAI's message schema has no ``_``-prefixed fields, so this
    +            # is safe and future-proofs against new markers being added.
    +            for key in [k for k in msg if isinstance(k, str) and k.startswith("_")]:
    +                msg.pop(key, None)
                 tool_calls = msg.get("tool_calls")
                 if isinstance(tool_calls, list):
                     for tc in tool_calls:

  • scripts/release.py

    @@ -70,6 +70,8 @@
         "[redacted:email]": "qiyin-code",
         "[redacted:email]": "aashizpoudel",
         "[redacted:email]": "shaun0927",
    +    "[redacted:email]": "Soju06",
    +    "[redacted:email]": "Soju06",
         "[redacted:email]": "Bihruze",
         "[redacted:email]": "Lempkey",
         "[redacted:email]": "m0n3r0",

  • tests/agent/test_error_classifier.py

    @@ -293,6 +293,64 @@ def test_529_anthropic_overloaded(self):
             result = classify_api_error(e)
             assert result.reason == FailoverReason.overloaded
     
    +    # ── 5xx that are actually request-validation errors ──
    +    # Some OpenAI-compatible gateways (e.g. codex.nekos.me) return
    +    # request-validation failures with a 5xx status. These are
    +    # deterministic, so they must NOT be retried — otherwise the retry
    +    # loop hammers the identical bad request into a flood.
    +
    +    def test_502_with_unknown_parameter_is_non_retryable(self):
    +        e = MockAPIError(
    +            "Unknown parameter: 'input[617]._empty_recovery_synthetic'",
    +            status_code=502,
    +            body={
    +                "error": {
    +                    "type": "invalid_request_error",
    +                    "message": (
    +                        "[ObjectParam] [input[617]._empty_recovery_synthetic] "
    +                        "[unknown_parameter] Unknown parameter: "
    +                        "'input[617]._empty_recovery_synthetic'."
    +                    ),
    +                }
    +            },
    +        )
    +        result = classify_api_error(e)
    +        assert result.reason == FailoverReason.format_error
    +        assert result.retryable is False
    +        assert result.should_fallback is True
    +
    +    def test_502_with_unsupported_parameter_is_non_retryable(self):
    +        e = MockAPIError(
    +            "Unsupported parameter: logprobs",
    +            status_code=502,
    +            body={
    +                "error": {
    +                    "type": "invalid_request_error",
    +                    "message": "Unsupported parameter: logprobs",
    +                }
    +            },
    +        )
    +        result = classify_api_error(e)
    +        assert result.reason == FailoverReason.format_error
    +        assert result.retryable is False
    +
    +    def test_500_with_invalid_request_error_type_is_non_retryable(self):
    +        e = MockAPIError(
    +            "bad request",
    +            status_code=500,
    +            body={"error": {"type": "invalid_request_error", "message": "bad request"}},
    +        )
    +        result = classify_api_error(e)
    +        assert result.reason == FailoverReason.format_error
    +        assert result.retryable is False
    +
    +    def test_502_plain_bad_gateway_still_retryable(self):
    +        """A genuine 502 with no request-validation signal stays retryable."""
    +        e = MockAPIError("Bad Gateway", status_code=502)
    +        result = classify_api_error(e)
    +        assert result.reason == FailoverReason.server_error
    +        assert result.retryable is True
    +
         # ── Model not found ──
     
         def test_404_model_not_found(self):

  • tests/agent/transports/test_chat_completions.py

    @@ -66,6 +66,38 @@ def test_convert_messages_strips_tool_name(self, transport):
             # Original list untouched (deepcopy-on-demand)
             assert msgs[2]["tool_name"] == "execute_code"
     
    +    def test_convert_messages_strips_internal_scaffolding_markers(self, transport):
    +        """Hermes-internal ``_``-prefixed markers must never reach the wire.
    +
    +        The empty-response recovery path appends synthetic messages tagged
    +        with ``_empty_recovery_synthetic``; permissive providers ignore the
    +        unknown key, but strict gateways (opencode-go, codex.nekos.me)
    +        reject the request, poisoning every later turn in the session.
    +        """
    +        msgs = [
    +            {"role": "user", "content": "run the task"},
    +            {"role": "assistant", "content": "(empty)", "_empty_recovery_synthetic": True},
    +            {"role": "user", "content": "continue", "_empty_recovery_synthetic": True},
    +            {"role": "assistant", "content": "done", "_thinking_prefill": True,
    +             "_empty_terminal_sentinel": True},
    +        ]
    +        result = transport.convert_messages(msgs)
    +        for m in result:
    +            assert not any(k.startswith("_") for k in m), m
    +        # Visible content preserved
    +        assert result[1]["content"] == "(empty)"
    +        assert result[2]["content"] == "continue"
    +        # Original list untouched (deepcopy-on-demand)
    +        assert msgs[1]["_empty_recovery_synthetic"] is True
    +
    +    def test_convert_messages_clean_list_is_identity(self, transport):
    +        """A list with no internal/codex keys is returned as-is (no copy)."""
    +        msgs = [
    +            {"role": "user", "content": "hi"},
    +            {"role": "assistant", "content": "hello"},
    +        ]
    +        assert transport.convert_messages(msgs) is msgs
    +
     
     class TestChatCompletionsBuildKwargs:
     

• #49840 fix(chat-completions): strip internal timestamp field before strict providers

  merged to main

  files: agent/transports/chat_completions.py, scripts/release.py, tests/agent/transports/test_chat_completions.py

  Show the diff

  • agent/transports/chat_completions.py

    @@ -172,6 +172,7 @@ def convert_messages(
                     "codex_reasoning_items" in msg
                     or "codex_message_items" in msg
                     or "tool_name" in msg
    +                or "timestamp" in msg  # #47868 — strict providers reject this
                 ):
                     needs_sanitize = True
                     break
    @@ -201,6 +202,7 @@ def convert_messages(
                 msg.pop("codex_reasoning_items", None)
                 msg.pop("codex_message_items", None)
                 msg.pop("tool_name", None)
    +            msg.pop("timestamp", None)  # #47868 — leak into strict providers
                 # Drop all Hermes-internal scaffolding markers (``_``-prefixed).
                 # OpenAI's message schema has no ``_``-prefixed fields, so this
                 # is safe and future-proofs against new markers being added.

  • scripts/release.py

    @@ -46,6 +46,7 @@
     # Auto-extracted from noreply emails + manual overrides
     AUTHOR_MAP = {
         "[redacted:email]": "w31rdm4ch1nZ",
    +    "[redacted:email]": "x7peeps",
         "[redacted:email]": "rratmansky",
         "[redacted:email]": "lkz-de",
         "[redacted:email]": "salesondemandio",

  • tests/agent/transports/test_chat_completions.py

    @@ -104,6 +104,31 @@ def test_convert_messages_strips_tool_name(self, transport):
             # Original list untouched (deepcopy-on-demand)
             assert msgs[2]["tool_name"] == "execute_code"
     
    +    def test_convert_messages_strips_timestamp(self, transport):
    +        """Internal per-message ``timestamp`` metadata (stamped by
    +        ``_apply_persist_user_message_override`` to preserve platform event
    +        time without embedding it in content, and persisted to the SQLite
    +        store) is not part of the OpenAI Chat Completions schema. Strict
    +        providers like Mistral / Fireworks-backed endpoints reject it with
    +        HTTP 422 'Extra inputs are not permitted, field: messages[N].timestamp'.
    +        Regression test for #47868.
    +        """
    +        msgs = [
    +            {"role": "user", "content": "hi", "timestamp": 1781976577.0},
    +        ]
    +        result = transport.convert_messages(msgs)
    +        assert "timestamp" not in result[0]
    +        assert result[0]["content"] == "hi"
    +        assert result[0]["role"] == "user"
    +        # Original list untouched (deepcopy-on-demand)
    +        assert msgs[0]["timestamp"] == 1781976577.0
    +
    +    def test_convert_messages_no_copy_without_timestamp(self, transport):
    +        """A timestamp-free message list needs no sanitize pass and is
    +        returned by identity (preserves the deepcopy-on-demand contract)."""
    +        msgs = [{"role": "user", "content": "hi"}]
    +        assert transport.convert_messages(msgs) is msgs
    +
         def test_convert_messages_strips_internal_scaffolding_markers(self, transport):
             """Hermes-internal ``_``-prefixed markers must never reach the wire.