{"repo":"NousResearch/hermes-agent","public_id":"30959","display_label":"Internal message fields leaking to strict API providers","summary":"Multiple bug reports and fixes address internal bookkeeping fields (timestamp, message_id, observed, etc.) being included in outgoing API message payloads, causing HTTP 400/422 errors on strict-schema providers like Mistral and GLM. The fixes involve stripping these fields before sending requests to external LLM endpoints.","subsystem":"agent","counts":{"members":29,"open_issues":1,"open_issues_distinct":1,"open_prs":3,"closed_issues":10,"merged_prs":2,"attempt_prs":16},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":31456},"resolution":{"state":"partially_resolved","by":"pr-31456,pr-49840"},"freshness":{"state":"verified","last_verified_at":"2026-06-21T00:05:17+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.06985221674876846},"fix_edges":[{"issue_number":30959,"pr_number":31456,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":47868,"pr_number":49840,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":30959,"title":"Internal bookkeeping fields leak to strict providers, causing HTTP 400 retry loops","url":null,"state":"closed"},{"kind":"issue","number":47868,"title":"Strict chat-completions providers reject leaked messages[].timestamp metadata","url":null,"state":"closed"},{"kind":"issue","number":48263,"title":"[Bug]: GLM 5.2 via OpenCode Go fails with HTTP 400 — internal `timestamp` field leaked into API messages","url":null,"state":"closed"},{"kind":"issue","number":48386,"title":"[Bug]: Gateway sends timestamp metadata field in message objects → HTTP 422 on strict OpenAI-compatible endpoints (api.mistral.ai)","url":null,"state":"closed"},{"kind":"issue","number":48523,"title":"convert_messages doesn't strip timestamp/message_id/observed/finish_reason — causes 400 with strict providers in gateway mode","url":null,"state":"open"},{"kind":"issue","number":48772,"title":"cannot disable the timestamp in config","url":null,"state":"closed"},{"kind":"issue","number":49490,"title":"[Bug]: OpenCode Zen provider does not work on v0.17.0","url":null,"state":"closed"},{"kind":"issue","number":49572,"title":"chat_completions transport: timestamp field on message dicts causes HTTP 400 on strict providers","url":null,"state":"closed"},{"kind":"issue","number":53139,"title":"[BUG] GLM 5.2 multi-turn sessions fail via OpenCode Go due to timestamp field injection","url":null,"state":"closed"},{"kind":"issue","number":55902,"title":"HTTP 400 with OpenCode Go + Kimi K2.5: 'messages[N].timestamp' is not permitted","url":null,"state":"closed"},{"kind":"issue","number":68077,"title":"sanitize_api_messages() doesn't strip gateway-only timestamp/message_id keys, causing HTTP 400 on strict-schema providers (GLM-5.2)","url":null,"state":"closed"},{"kind":"pr","number":30982,"title":"fix(transport): strip internal bookkeeping fields before API calls","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"775a17284f00a0ab7014d0cea35a74f0c66cf3e5","number":0},{"kind":"pr","value":"#30959","number":30959},{"kind":"pr","value":"#31456","number":31456}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":31456,"title":"fix(transport): strip Hermes-internal scaffolding keys + classify 5xx validation errors (salvage #26054)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/error_classifier.py","agent/transports/chat_completions.py","scripts/release.py","tests/agent/test_error_classifier.py","tests/agent/transports/test_chat_completions.py"],"fix_hunks":[{"file":"agent/error_classifier.py","hunks":["@@ -240,6 +240,24 @@ def is_auth(self) -> bool:\n     \"unsupported model\",\n ]\n \n+# Request-validation patterns — the request is malformed and will fail\n+# identically on every retry. Some OpenAI-compatible gateways (notably\n+# codex.nekos.me) return these as 5xx instead of the standard 4xx, which\n+# makes the generic \"5xx → retryable server_error\" rule misfire: the retry\n+# loop hammers the same deterministic rejection 3+ times, then the\n+# transport-recovery path resets the counter and does it again, producing\n+# a request flood. When a 5xx body carries one of these unambiguous\n+# request-validation signals, classify as a non-retryable format_error so\n+# the loop fails fast and falls back instead of looping.\n+_REQUEST_VALIDATION_PATTERNS = [\n+    \"unknown parameter\",\n+    \"unsupported parameter\",\n+    \"unrecognized request argument\",\n+    \"invalid_request_error\",\n+    \"unknown_parameter\",\n+    \"unsupported_parameter\",\n+]\n+\n # OpenRouter aggregator policy-block patterns.\n #\n # When a user's OpenRouter account privacy setting (or a per-request","@@ -745,6 +763,23 @@ def _classify_by_status(\n         )\n \n     if status_code in {500, 502}:\n+        # Some OpenAI-compatible gateways return request-validation errors\n+        # with a 5xx status (codex.nekos.me returns 502 for unknown/\n+        # unsupported parameters). These are deterministic — every retry\n+        # gets the identical rejection — so the generic \"5xx → retryable\n+        # server_error\" rule turns one bad request into a retry flood.\n+        # Detect the unambiguous request-validation signals (in either the\n+        # message text or the structured error code) and fail fast.\n+        if (\n+            any(p in error_msg for p in _REQUEST_VALIDATION_PATTERNS)\n+            or error_code.lower() in {\"invalid_request_error\", \"unknown_parameter\",\n+                                      \"unsupported_parameter\"}\n+        ):\n+            return result_fn(\n+                FailoverReason.format_error,\n+                retryable=False,\n+                should_fallback=True,\n+            )\n         return result_fn(FailoverReason.server_error, retryable=True)\n \n     if status_code in {503, 529}:"]},{"file":"agent/transports/chat_completions.py","hunks":["@@ -113,9 +113,8 @@ def convert_messages(\n         self, messages: list[dict[str, Any]], **kwargs\n     ) -> list[dict[str, Any]]:\n         \"\"\"Messages are already in OpenAI format — strip internal fields\n-        that strict chat-completions providers reject with HTTP 400/422.\n-\n-        Strips:\n+        that strict chat-completions providers reject with HTTP 400/422\n+        (or, in the case of some OpenAI-compatible gateways, 5xx):\n \n         - Codex Responses API fields: ``codex_reasoning_items`` /\n           ``codex_message_items`` on the message, ``call_id`` /","@@ -127,6 +126,16 @@ def convert_messages(\n           ``Extra inputs are not permitted, field: 'messages[N].tool_name'``.\n           Permissive providers (OpenRouter, MiniMax) silently ignore the\n           field, which masked the bug for months.\n+        - Hermes-internal scaffolding markers — any top-level message key\n+          starting with ``_`` (e.g. ``_empty_recovery_synthetic``,\n+          ``_empty_terminal_sentinel``, ``_thinking_prefill``). These are\n+          bookkeeping flags the agent loop attaches to messages so the\n+          persistence layer can later strip its own scaffolding; they must\n+          never reach the wire. Permissive providers (real OpenAI,\n+          Anthropic) silently drop unknown message keys, but strict\n+          gateways (e.g. opencode-go, codex.nekos.me) reject with\n+          ``Extra inputs are not permitted, field: 'messages[N]._empty_recovery_synthetic'``,\n+          which then poisons every subsequent request in the session.\n         \"\"\"\n         needs_sanitize = False\n         for msg in messages:","@@ -139,6 +148,9 @@ def convert_messages(\n             ):\n                 needs_sanitize = True\n                 break\n+            if any(isinstance(k, str) and k.startswith(\"_\") for k in msg):\n+                needs_sanitize = True\n+                break\n             tool_calls = msg.get(\"tool_calls\")\n             if isinstance(tool_calls, list):\n                 for tc in tool_calls:","@@ -160,6 +172,11 @@ def convert_messages(\n             msg.pop(\"codex_reasoning_items\", None)\n             msg.pop(\"codex_message_items\", None)\n             msg.pop(\"tool_name\", None)\n+            # Drop all Hermes-internal scaffolding markers (``_``-prefixed).\n+            # OpenAI's message schema has no ``_``-prefixed fields, so this\n+            # is safe and future-proofs against new markers being added.\n+            for key in [k for k in msg if isinstance(k, str) and k.startswith(\"_\")]:\n+                msg.pop(key, None)\n             tool_calls = msg.get(\"tool_calls\")\n             if isinstance(tool_calls, list):\n                 for tc in tool_calls:"]},{"file":"scripts/release.py","hunks":["@@ -70,6 +70,8 @@\n     \"[redacted:email]\": \"qiyin-code\",\n     \"[redacted:email]\": \"aashizpoudel\",\n     \"[redacted:email]\": \"shaun0927\",\n+    \"[redacted:email]\": \"Soju06\",\n+    \"[redacted:email]\": \"Soju06\",\n     \"[redacted:email]\": \"Bihruze\",\n     \"[redacted:email]\": \"Lempkey\",\n     \"[redacted:email]\": \"m0n3r0\","]},{"file":"tests/agent/test_error_classifier.py","hunks":["@@ -293,6 +293,64 @@ def test_529_anthropic_overloaded(self):\n         result = classify_api_error(e)\n         assert result.reason == FailoverReason.overloaded\n \n+    # ── 5xx that are actually request-validation errors ──\n+    # Some OpenAI-compatible gateways (e.g. codex.nekos.me) return\n+    # request-validation failures with a 5xx status. These are\n+    # deterministic, so they must NOT be retried — otherwise the retry\n+    # loop hammers the identical bad request into a flood.\n+\n+    def test_502_with_unknown_parameter_is_non_retryable(self):\n+        e = MockAPIError(\n+            \"Unknown parameter: 'input[617]._empty_recovery_synthetic'\",\n+            status_code=502,\n+            body={\n+                \"error\": {\n+                    \"type\": \"invalid_request_error\",\n+                    \"message\": (\n+                        \"[ObjectParam] [input[617]._empty_recovery_synthetic] \"\n+                        \"[unknown_parameter] Unknown parameter: \"\n+                        \"'input[617]._empty_recovery_synthetic'.\"\n+                    ),\n+                }\n+            },\n+        )\n+        result = classify_api_error(e)\n+        assert result.reason == FailoverReason.format_error\n+        assert result.retryable is False\n+        assert result.should_fallback is True\n+\n+    def test_502_with_unsupported_parameter_is_non_retryable(self):\n+        e = MockAPIError(\n+            \"Unsupported parameter: logprobs\",\n+            status_code=502,\n+            body={\n+                \"error\": {\n+                    \"type\": \"invalid_request_error\",\n+                    \"message\": \"Unsupported parameter: logprobs\",\n+                }\n+            },\n+        )\n+        result = classify_api_error(e)\n+        assert result.reason == FailoverReason.format_error\n+        assert result.retryable is False\n+\n+    def test_500_with_invalid_request_error_type_is_non_retryable(self):\n+        e = MockAPIError(\n+            \"bad request\",\n+            status_code=500,\n+            body={\"error\": {\"type\": \"invalid_request_error\", \"message\": \"bad request\"}},\n+        )\n+        result = classify_api_error(e)\n+        assert result.reason == FailoverReason.format_error\n+        assert result.retryable is False\n+\n+    def test_502_plain_bad_gateway_still_retryable(self):\n+        \"\"\"A genuine 502 with no request-validation signal stays retryable.\"\"\"\n+        e = MockAPIError(\"Bad Gateway\", status_code=502)\n+        result = classify_api_error(e)\n+        assert result.reason == FailoverReason.server_error\n+        assert result.retryable is True\n+\n     # ── Model not found ──\n \n     def test_404_model_not_found(self):"]},{"file":"tests/agent/transports/test_chat_completions.py","hunks":["@@ -66,6 +66,38 @@ def test_convert_messages_strips_tool_name(self, transport):\n         # Original list untouched (deepcopy-on-demand)\n         assert msgs[2][\"tool_name\"] == \"execute_code\"\n \n+    def test_convert_messages_strips_internal_scaffolding_markers(self, transport):\n+        \"\"\"Hermes-internal ``_``-prefixed markers must never reach the wire.\n+\n+        The empty-response recovery path appends synthetic messages tagged\n+        with ``_empty_recovery_synthetic``; permissive providers ignore the\n+        unknown key, but strict gateways (opencode-go, codex.nekos.me)\n+        reject the request, poisoning every later turn in the session.\n+        \"\"\"\n+        msgs = [\n+            {\"role\": \"user\", \"content\": \"run the task\"},\n+            {\"role\": \"assistant\", \"content\": \"(empty)\", \"_empty_recovery_synthetic\": True},\n+            {\"role\": \"user\", \"content\": \"continue\", \"_empty_recovery_synthetic\": True},\n+            {\"role\": \"assistant\", \"content\": \"done\", \"_thinking_prefill\": True,\n+             \"_empty_terminal_sentinel\": True},\n+        ]\n+        result = transport.convert_messages(msgs)\n+        for m in result:\n+            assert not any(k.startswith(\"_\") for k in m), m\n+        # Visible content preserved\n+        assert result[1][\"content\"] == \"(empty)\"\n+        assert result[2][\"content\"] == \"continue\"\n+        # Original list untouched (deepcopy-on-demand)\n+        assert msgs[1][\"_empty_recovery_synthetic\"] is True\n+\n+    def test_convert_messages_clean_list_is_identity(self, transport):\n+        \"\"\"A list with no internal/codex keys is returned as-is (no copy).\"\"\"\n+        msgs = [\n+            {\"role\": \"user\", \"content\": \"hi\"},\n+            {\"role\": \"assistant\", \"content\": \"hello\"},\n+        ]\n+        assert transport.convert_messages(msgs) is msgs\n+\n \n class TestChatCompletionsBuildKwargs:\n \n"]}]},{"kind":"pr","number":47875,"title":"fix(chat-completions): strip timestamp from messages before sending to strict providers","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4467c22c8f","number":0},{"kind":"pr","value":"#49840","number":49840},{"kind":"pr","value":"#47868","number":47868}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":47909,"title":"fix: strip internal message metadata before LLM calls","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47868","number":47868}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":48170,"title":"fix(agent): strip timestamp metadata from chat payloads","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47868","number":47868},{"kind":"pr","value":"#47868","number":47868},{"kind":"pr","value":"#47875","number":47875}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":48266,"title":"fix:sanitize internal fields (timestamp) from API messages #48263","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47868","number":47868},{"kind":"pr","value":"#48263","number":48263}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":48295,"title":"fix(transports/chat_completions): strip session timestamp from API-facing messages","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":48358,"title":"fix(transport): strip internal timestamp from API messages (#48263)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#48295","number":48295},{"kind":"pr","value":"#48331","number":48331},{"kind":"pr","value":"#48263","number":48263},{"kind":"pr","value":"#47868","number":47868}],"category":"merged_via","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":48592,"title":"fix(chat_completions): strip timestamp/observed/platform_message_id from API messages","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#48358","number":48358},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#48170","number":48170},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47868","number":47868}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":48604,"title":"fix(agent): strip gateway replay metadata from messages before API calls (#48523)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#48592","number":48592},{"kind":"pr","value":"#48358","number":48358},{"kind":"pr","value":"#48575","number":48575},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#48523","number":48523},{"kind":"pr","value":"#47868","number":47868}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":48916,"title":"fix(prompt): honor explicit system prompt timestamp config","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#48772","number":48772},{"kind":"pr","value":"#48772","number":48772}],"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":49573,"title":"fix(transport): strip timestamp and observed from chat completions messages","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#49595","number":49595},{"kind":"pr","value":"#49572","number":49572}],"category":"superseded_by_pr","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":49595,"title":"fix(transport): strip timestamp and observed from chat completions messages","url":null,"state":"open","close_reason":{"refs":[{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47875","number":47875},{"kind":"pr","value":"#47868","number":47868},{"kind":"pr","value":"#49573","number":49573}],"category":"duplicate_of","implication":"duplicate"},"merged":false},{"kind":"pr","number":49840,"title":"fix(chat-completions): strip internal timestamp field before strict providers","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["agent/transports/chat_completions.py","scripts/release.py","tests/agent/transports/test_chat_completions.py"],"fix_hunks":[{"file":"agent/transports/chat_completions.py","hunks":["@@ -172,6 +172,7 @@ def convert_messages(\n                 \"codex_reasoning_items\" in msg\n                 or \"codex_message_items\" in msg\n                 or \"tool_name\" in msg\n+                or \"timestamp\" in msg  # #47868 — strict providers reject this\n             ):\n                 needs_sanitize = True\n                 break","@@ -201,6 +202,7 @@ def convert_messages(\n             msg.pop(\"codex_reasoning_items\", None)\n             msg.pop(\"codex_message_items\", None)\n             msg.pop(\"tool_name\", None)\n+            msg.pop(\"timestamp\", None)  # #47868 — leak into strict providers\n             # Drop all Hermes-internal scaffolding markers (``_``-prefixed).\n             # OpenAI's message schema has no ``_``-prefixed fields, so this\n             # is safe and future-proofs against new markers being added."]},{"file":"scripts/release.py","hunks":["@@ -46,6 +46,7 @@\n # Auto-extracted from noreply emails + manual overrides\n AUTHOR_MAP = {\n     \"[redacted:email]\": \"w31rdm4ch1nZ\",\n+    \"[redacted:email]\": \"x7peeps\",\n     \"[redacted:email]\": \"rratmansky\",\n     \"[redacted:email]\": \"lkz-de\",\n     \"[redacted:email]\": \"salesondemandio\","]},{"file":"tests/agent/transports/test_chat_completions.py","hunks":["@@ -104,6 +104,31 @@ def test_convert_messages_strips_tool_name(self, transport):\n         # Original list untouched (deepcopy-on-demand)\n         assert msgs[2][\"tool_name\"] == \"execute_code\"\n \n+    def test_convert_messages_strips_timestamp(self, transport):\n+        \"\"\"Internal per-message ``timestamp`` metadata (stamped by\n+        ``_apply_persist_user_message_override`` to preserve platform event\n+        time without embedding it in content, and persisted to the SQLite\n+        store) is not part of the OpenAI Chat Completions schema. Strict\n+        providers like Mistral / Fireworks-backed endpoints reject it with\n+        HTTP 422 'Extra inputs are not permitted, field: messages[N].timestamp'.\n+        Regression test for #47868.\n+        \"\"\"\n+        msgs = [\n+            {\"role\": \"user\", \"content\": \"hi\", \"timestamp\": 1781976577.0},\n+        ]\n+        result = transport.convert_messages(msgs)\n+        assert \"timestamp\" not in result[0]\n+        assert result[0][\"content\"] == \"hi\"\n+        assert result[0][\"role\"] == \"user\"\n+        # Original list untouched (deepcopy-on-demand)\n+        assert msgs[0][\"timestamp\"] == 1781976577.0\n+\n+    def test_convert_messages_no_copy_without_timestamp(self, transport):\n+        \"\"\"A timestamp-free message list needs no sanitize pass and is\n+        returned by identity (preserves the deepcopy-on-demand contract).\"\"\"\n+        msgs = [{\"role\": \"user\", \"content\": \"hi\"}]\n+        assert transport.convert_messages(msgs) is msgs\n+\n     def test_convert_messages_strips_internal_scaffolding_markers(self, transport):\n         \"\"\"Hermes-internal ``_``-prefixed markers must never reach the wire.\n \n"]}]},{"kind":"pr","number":55907,"title":"fix(agent): strip message timestamp before API calls","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":55987,"title":"fix(agent): strip timestamp field from API messages (#55902)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4467c22c8","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":56112,"title":"fix(agent): strip timestamp field in summary path sanitizer","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"6d3d9d0baf18291e5306691b35f81e5b2e0eecb3","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":68166,"title":"fix(agent): strip gateway-only timestamp/message_id keys before LLM call","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"4467c22c8f097cce5b670e81852d69bfbb6aadea","number":0},{"kind":"pr","value":"#68077","number":68077}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}