{"repo":"NousResearch/hermes-agent","public_id":"17537","display_label":"SSE token batching for Open WebUI streaming performance","summary":"These members concern implementing server-side SSE token batching in the API server to address streaming lag in Open WebUI. They also touch on related concerns such as response trimming and error handling.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":20445,"merged_at":"2026-05-05T22:13:37+00:00"},"resolution":{"state":"resolved","by":"pr-20445"},"freshness":{"state":"verified","last_verified_at":"2026-05-05T22:13:37+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":17537,"pr_number":20445,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-05-05T22:13:37+00:00"}],"members":[{"kind":"issue","number":17537,"title":"[Feature]: Server-side SSE token batching to fix Open WebUI streaming lag","url":null,"state":"closed"},{"kind":"pr","number":17541,"title":"fix(api_server): SSE token batching + response trimming for Open WebUI performance","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3188e63b05a1902baecfcd7c30da3301d74b8737","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":20445,"title":"fix(api_server): SSE token batching + error handling for Open WebUI (salvage #17552)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/platforms/api_server.py","scripts/release.py"],"fix_hunks":[{"file":"gateway/platforms/api_server.py","hunks":["@@ -56,7 +56,7 @@\n DEFAULT_HOST = \"127.0.0.1\"\n DEFAULT_PORT = 8642\n MAX_STORED_RESPONSES = 100\n-MAX_REQUEST_BYTES = 1_000_000  # 1 MB default limit for POST bodies\n+MAX_REQUEST_BYTES = 10_000_000  # 10 MB — accommodates long agent conversations with tool calls\n CHAT_COMPLETIONS_SSE_KEEPALIVE_SECONDS = 30.0\n MAX_NORMALIZED_TEXT_LENGTH = 65_536  # 64 KB cap for normalized content parts\n MAX_CONTENT_LIST_SIZE = 1_000  # Max items when content is an array","@@ -1349,6 +1349,22 @@ async def _emit(item):\n                 except (asyncio.CancelledError, Exception):\n                     pass\n             logger.info(\"SSE client disconnected; interrupted agent task %s\", completion_id)\n+        except Exception as _exc:\n+            # Agent crashed mid-stream.  Try to emit an error chunk\n+            # so the client gets a proper response instead of a\n+            # TransferEncodingError from incomplete chunked encoding.\n+            import traceback as _tb\n+            logger.error(\"Agent crashed mid-stream for %s: %s\", completion_id, _tb.format_exc()[:300])\n+            try:\n+                error_chunk = {\n+                    \"id\": completion_id, \"object\": \"chat.completion.chunk\",\n+                    \"created\": created, \"model\": model,\n+                    \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"error\"}],\n+                }\n+                await response.write(f\"data: {json.dumps(error_chunk)}\\n\\n\".encode())\n+                await response.write(b\"data: [DONE]\\n\\n\")\n+            except Exception:\n+                pass\n \n         return response\n ","@@ -1669,20 +1685,54 @@ async def _emit_tool_completed(payload: Dict[str, Any]) -> None:\n             async def _dispatch(it) -> None:\n                 \"\"\"Route a queue item to the correct SSE emitter.\n \n-                Plain strings are text deltas.  Tagged tuples with\n-                ``__tool_started__`` / ``__tool_completed__`` prefixes\n-                are tool lifecycle events.\n+                Plain strings are text deltas — they are batched (50ms)\n+                to reduce Open WebUI re-render storms.  Tagged tuples\n+                with ``__tool_started__`` / ``__tool_completed__``\n+                prefixes are tool lifecycle events and flush the buffer\n+                before emitting.\n                 \"\"\"\n+                nonlocal _batch_timer\n                 if isinstance(it, tuple) and len(it) == 2 and isinstance(it[0], str):\n                     tag, payload = it\n+                    # Flush batched text before tool events\n+                    if _batch_buf:\n+                        await _flush_batch()\n                     if tag == \"__tool_started__\":\n                         await _emit_tool_started(payload)\n                     elif tag == \"__tool_completed__\":\n                         await _emit_tool_completed(payload)\n-                    # Unknown tags are silently ignored (forward-compat).\n                 elif isinstance(it, str):\n-                    await _emit_text_delta(it)\n-                # Other types (non-string, non-tuple) are silently dropped.\n+                    # Batch text deltas — append to buffer, flush on timer\n+                    _batch_buf.append(it)\n+                    if _batch_timer is None:\n+                        _batch_timer = asyncio.create_task(_batch_flush_after(0.05))\n+                # Other types are silently dropped.\n+\n+            # ── Batching state ──\n+            _batch_buf: List[str] = []\n+            _batch_timer: Optional[asyncio.Task] = None\n+            _batch_lock = asyncio.Lock()\n+\n+            async def _batch_flush_after(delay: float) -> None:\n+                \"\"\"Wait delay seconds, then flush accumulated text deltas.\"\"\"\n+                try:\n+                    await asyncio.sleep(delay)\n+                except asyncio.CancelledError:\n+                    return\n+                # Clear timer reference BEFORE flush so new deltas\n+                # can start a fresh timer while we emit\n+                nonlocal _batch_buf, _batch_timer\n+                _batch_timer = None\n+                await _flush_batch()\n+\n+            async def _flush_batch() -> None:\n+                \"\"\"Emit a single SSE delta for all accumulated text.\"\"\"\n+                nonlocal _batch_buf\n+                async with _batch_lock:\n+                    if _batch_buf:\n+                        combined = \"\".join(_batch_buf)\n+                        _batch_buf = []\n+                        await _emit_text_delta(combined)\n \n             loop = asyncio.get_running_loop()\n             while True:","@@ -1707,11 +1757,21 @@ async def _dispatch(it) -> None:\n                     continue\n \n                 if item is None:  # EOS sentinel\n+                    # Cancel pending timer and flush remaining batched text\n+                    if _batch_timer and not _batch_timer.done():\n+                        _batch_timer.cancel()\n+                        _batch_timer = None\n+                    if _batch_buf:\n+                        await _flush_batch()\n                     break\n \n                 await _dispatch(item)\n                 last_activity = time.monotonic()\n \n+            # Flush any final batched text before processing result\n+            if _batch_buf:\n+                await _flush_batch()\n+\n             # Pick up agent result + usage from the completed task\n             try:\n                 result, agent_usage = await agent_task","@@ -1762,6 +1822,31 @@ async def _dispatch(it) -> None:\n             # payload still see the assistant text.  This mirrors the\n             # shape produced by _extract_output_items in the batch path.\n             final_items: List[Dict[str, Any]] = list(emitted_items)\n+\n+            # Trim large content from tool call arguments to keep the\n+            # response.completed event under ~100KB.  Clients already\n+            # received full details via incremental events.\n+            for _item in final_items:\n+                if _item.get(\"type\") == \"function_call\":\n+                    try:\n+                        _args = json.loads(_item.get(\"arguments\", \"{}\")) if isinstance(_item.get(\"arguments\"), str) else _item.get(\"arguments\", {})\n+                        if isinstance(_args, dict):\n+                            for _k in (\"content\", \"query\", \"pattern\", \"old_string\", \"new_string\"):\n+                                if isinstance(_args.get(_k), str) and len(_args[_k]) > 500:\n+                                    _args[_k] = \"[\" + str(len(_args[_k])) + \" chars — truncated for response.completed]\"\n+                            _item[\"arguments\"] = json.dumps(_args)\n+                    except Exception:\n+                        pass\n+                elif _item.get(\"type\") == \"function_call_output\":\n+                    _output = _item.get(\"output\", [])\n+                    if isinstance(_output, list) and _output:\n+                        _first = _output[0]\n+                        if isinstance(_first, dict) and _first.get(\"type\") == \"input_text\":\n+                            _text = _first.get(\"text\", \"\")\n+                            if len(_text) > 1000:\n+                                _first[\"text\"] = _text[:500] + \"...[\" + str(len(_text) - 500) + \" more chars]\"\n+                                _item[\"output\"] = [_first]\n+\n             final_items.append({\n                 \"type\": \"message\",\n                 \"role\": \"assistant\",","@@ -1852,6 +1937,30 @@ async def _dispatch(it) -> None:\n                 agent_task.cancel()\n             logger.info(\"SSE task cancelled; persisted incomplete snapshot for %s\", response_id)\n             raise\n+        except Exception as _exc:\n+            # Agent crashed with an unhandled error (e.g. model API error like\n+            # BadRequestError, AuthenticationError).  Emit a response.failed\n+            # event and properly terminate the SSE stream so the client doesn't\n+            # get a TransferEncodingError from incomplete chunked encoding.\n+            import traceback as _tb\n+            _persist_incomplete_if_needed()\n+            agent_error = _tb.format_exc()\n+            try:\n+                failed_env = _envelope(\"failed\")\n+                failed_env[\"output\"] = list(emitted_items)\n+                failed_env[\"error\"] = {\"message\": str(_exc)[:500], \"type\": \"server_error\"}\n+                failed_env[\"usage\"] = {\n+                    \"input_tokens\": usage.get(\"input_tokens\", 0),\n+                    \"output_tokens\": usage.get(\"output_tokens\", 0),\n+                    \"total_tokens\": usage.get(\"total_tokens\", 0),\n+                }\n+                await _write_event(\"response.failed\", {\n+                    \"type\": \"response.failed\",\n+                    \"response\": failed_env,\n+                })\n+            except Exception:\n+                pass\n+            logger.error(\"Agent crashed mid-stream for %s: %s\", response_id, str(agent_error)[:300])\n \n         return response\n ","@@ -2935,7 +3044,7 @@ async def connect(self) -> bool:\n \n         try:\n             mws = [mw for mw in (cors_middleware, body_limit_middleware, security_headers_middleware) if mw is not None]\n-            self._app = web.Application(middlewares=mws)\n+            self._app = web.Application(middlewares=mws, client_max_size=MAX_REQUEST_BYTES)\n             self._app[\"api_server_adapter\"] = self\n             self._app.router.add_get(\"/health\", self._handle_health)\n             self._app.router.add_get(\"/health/detailed\", self._handle_health_detailed)"]},{"file":"scripts/release.py","hunks":["@@ -100,6 +100,7 @@\n     \"[redacted:email]\": \"haidao1919\",\n     \"50561768+zhanggttry@users.noreply.github.com\": \"zhanggttry\",\n     \"[redacted:email]\": \"formulahendry\",\n+    \"93757150+bogerman1@users.noreply.github.com\": \"bogerman1\",\n     # Matrix parity salvage batch (April 2026)\n     \"sr@samirusani\": \"samrusani\",\n     \"[redacted:email]\": \"angel12\",\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}