max_tokens config propagation bugs and fixes

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #39864 fix(gateway): propagate max_tokens from config.yaml to AIAgent (#20741)

  merged to main

  files: cli.py, gateway/run.py, hermes_cli/runtime_provider.py, scripts/release.py, tests/gateway/test_max_tokens_propagation.py

  Show the diff

  • cli.py

    @@ -3194,6 +3194,18 @@ def __init__(
             _config_model = (_model_config.get("default") or _model_config.get("model") or "") if isinstance(_model_config, dict) else (_model_config or "")
             _DEFAULT_CONFIG_MODEL = ""
             self.model = model or _config_model or _DEFAULT_CONFIG_MODEL
    +        # Read max_tokens from config (env var override: HERMES_MAX_TOKENS)
    +        _env_mt = os.environ.get("HERMES_MAX_TOKENS")
    +        if _env_mt:
    +            try:
    +                self.max_tokens = [redacted:secret-assignment]
    +            except (ValueError, TypeError):
    +                self.max_tokens = None
    +        elif isinstance(_model_config, dict):
    +            _mt = _model_config.get("max_tokens")
    +            self.max_tokens = _mt if isinstance(_mt, int) else None
    +        else:
    +            self.max_tokens = None
             # Auto-detect model from local server if still on default
             if self.model == _DEFAULT_CONFIG_MODEL:
                 _base_url = (_model_config.get("base_url") or "") if isinstance(_model_config, dict) else ""
    @@ -5168,6 +5180,7 @@ def _init_agent(self, *, model_override: str = None, runtime_override: dict = No
                     acp_command=runtime.get("command"),
                     acp_args=runtime.get("args"),
                     credential_pool=[redacted:secret-assignment]"credential_pool"),
    +                max_tokens=[redacted:secret-assignment]
                     max_iterations=self.max_turns,
                     enabled_toolsets=self.enabled_toolsets,
                     disabled_toolsets=self.disabled_toolsets,
    @@ -9284,6 +9297,7 @@ def run_background():
                         api_mode=turn_route["runtime"].get("api_mode"),
                         acp_command=turn_route["runtime"].get("command"),
                         acp_args=turn_route["runtime"].get("args"),
    +                    max_tokens=[redacted:secret-assignment]"runtime"].get("max_tokens"),
                         max_iterations=self.max_turns,
                         enabled_toolsets=self.enabled_toolsets,
                         quiet_mode=True,

  • gateway/run.py

    @@ -1179,6 +1179,7 @@ def _resolve_runtime_agent_kwargs() -> dict:
         from hermes_cli.runtime_provider import (
             resolve_runtime_provider,
             format_runtime_provider_error,
    +        _get_model_config,
         )
         from hermes_cli.auth import AuthError, is_rate_limited_auth_error
     
    @@ -1200,6 +1201,26 @@ def _resolve_runtime_agent_kwargs() -> dict:
         except Exception as exc:
             raise RuntimeError(format_runtime_provider_error(exc)) from exc
     
    +    model_cfg = _get_model_config()
    +    max_tokens = None
    +    _env_mt = os.environ.get("HERMES_MAX_TOKENS")
    +    if _env_mt:
    +        try:
    +            max_tokens = [redacted:secret-assignment]
    +        except (ValueError, TypeError):
    +            max_tokens = None
    +    elif isinstance(model_cfg, dict):
    +        mt = model_cfg.get("max_tokens")
    +        if isinstance(mt, int):
    +            max_tokens = mt
    +    # Fall back to a per-provider output cap (custom_providers max_output_tokens)
    +    # only when the documented global model.max_tokens isn't set, so the global
    +    # key always wins.
    +    if max_tokens is None:
    +        _runtime_mot = runtime.get("max_output_tokens")
    +        if isinstance(_runtime_mot, int) and _runtime_mot > 0:
    +            max_tokens = [redacted:secret-assignment]
    +
         return {
             "api_key": runtime.get("api_key"),
             "base_url": runtime.get("base_url"),
    @@ -1208,6 +1229,7 @@ def _resolve_runtime_agent_kwargs() -> dict:
             "command": runtime.get("command"),
             "args": list(runtime.get("args") or []),
             "credential_pool": runtime.get("credential_pool"),
    +        "max_tokens": max_tokens,
         }
     
     
    @@ -2596,6 +2618,7 @@ def _resolve_session_agent_runtime(
                     "api_key": override.get("api_key"),
                     "base_url": override.get("base_url"),
                     "api_mode": override.get("api_mode"),
    +                "max_tokens": override.get("max_tokens"),
                 }
                 if override_runtime.get("api_key"):
                     logger.debug(
    @@ -2693,6 +2716,7 @@ def _resolve_turn_agent_config(self, user_message: str, model: str, runtime_kwar
                 "command": runtime_kwargs.get("command"),
                 "args": list(runtime_kwargs.get("args") or []),
                 "credential_pool": runtime_kwargs.get("credential_pool"),
    +            "max_tokens": runtime_kwargs.get("max_tokens"),
             }
             route = {
                 "model": model,

  • hermes_cli/runtime_provider.py

    @@ -474,6 +474,21 @@ def _try_resolve_from_custom_pool(
             return None
     
     
    +def _lift_max_output_tokens(entry: Dict[str, Any], result: Dict[str, Any]) -> None:
    +    """Propagate a per-provider output cap onto the resolved runtime dict.
    +
    +    Accepts ``max_output_tokens`` or ``max_tokens`` on a ``custom_providers``
    +    entry so a provider block can pin its own output limit. Gateway and CLI
    +    map this onto ``AIAgent.max_tokens`` only when the top-level
    +    ``model.max_tokens`` isn't set, so the documented global key still wins.
    +    """
    +    for _k in ("max_output_tokens", "max_tokens"):
    +        _v = entry.get(_k)
    +        if isinstance(_v, int) and _v > 0:
    +            result["max_output_tokens"] = _v
    +            return
    +
    +
     def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, Any]]:
         requested_norm = _normalize_custom_provider_name(requested_provider or "")
         if not requested_norm or requested_norm == "custom":
    @@ -541,6 +556,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
                         api_mode = _parse_api_mode(entry.get("api_mode") or entry.get("transport"))
                         if api_mode:
                             result["api_mode"] = api_mode
    +                    _lift_max_output_tokens(entry, result)
                         return result
                 # Also check the 'name' field if present
                 display_name = entry.get("name", "")
    @@ -562,6 +578,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
                             api_mode = _parse_api_mode(entry.get("api_mode") or entry.get("transport"))
                             if api_mode:
                                 result["api_mode"] = api_mode
    +                        _lift_max_output_tokens(entry, result)
                             return result
     
         # Fall back to custom_providers: list (legacy format)
    @@ -611,6 +628,7 @@ def _get_named_custom_provider(requested_provider: str) -> Optional[Dict[str, An
             model_name = str(entry.get("model", "") or "").strip()
             if model_name:
                 result["model"] = model_name
    +        _lift_max_output_tokens(entry, result)
             return result
     
         return None
    @@ -699,6 +717,8 @@ def _resolve_named_custom_runtime(
             model_name = custom_provider.get("model")
             if model_name:
                 pool_result["model"] = model_name
    +        if isinstance(custom_provider.get("max_output_tokens"), int):
    +            pool_result["max_output_tokens"] = custom_provider["max_output_tokens"]
             request_overrides = _custom_provider_request_overrides(custom_provider)
             if request_overrides:
                 pool_result["request_overrides"] = {
    @@ -736,6 +756,8 @@ def _resolve_named_custom_runtime(
         # provider name differs from the actual model string the API expects.
         if custom_provider.get("model"):
             result["model"] = custom_provider["model"]
    +    if isinstance(custom_provider.get("max_output_tokens"), int):
    +        result["max_output_tokens"] = custom_provider["max_output_tokens"]
         request_overrides = _custom_provider_request_overrides(custom_provider)
         if request_overrides:
             result["request_overrides"] = request_overrides

  • scripts/release.py

    @@ -45,6 +45,7 @@
     
     # Auto-extracted from noreply emails + manual overrides
     AUTHOR_MAP = {
    +    "[redacted:email]": "ViewWay",
         "[redacted:email]": "harjothkhara",
         "[redacted:email]": "HeLLGURD",
         "[redacted:email]": "dirtyren",

  • tests/gateway/test_max_tokens_propagation.py

    @@ -0,0 +1,178 @@
    +"""Regression tests for max_tokens propagation from config.yaml to AIAgent.
    +
    +Covers #20741: `model.max_tokens` was silently dropped before reaching the
    +gateway-spawned agent, so providers without a hardcoded default (OpenRouter
    +free models, Ollama Cloud, custom OpenAI-compatible endpoints) truncated long
    +generations with `finish_reason="length"`.
    +
    +Precedence verified here:
    +    HERMES_MAX_TOKENS env  >  model.max_tokens  >  per-provider
    +    max_output_tokens  >  None
    +"""
    +
    +import importlib
    +import os
    +import sys
    +import textwrap
    +
    +import pytest
    +
    +
    [redacted:email]
    +def isolated_home(tmp_path, monkeypatch):
    +    """Isolated HERMES_HOME with a writable config.yaml and a clean module cache.
    +
    +    These tests deliberately re-import ``hermes_cli`` / ``gateway`` so each
    +    config write is read fresh. To avoid leaking that purge into sibling test
    +    files in the same worker (which breaks their import-time mocks), we snapshot
    +    the affected modules and restore them on teardown.
    +    """
    +    hermes_home = tmp_path / ".hermes"
    +    hermes_home.mkdir()
    +    monkeypatch.setenv("HERMES_HOME", str(hermes_home))
    +    monkeypatch.delenv("HERMES_MAX_TOKENS", raising=False)
    +
    +    _saved = {
    +        k: v
    +        for k, v in sys.modules.items()
    +        if k.startswith(("hermes_cli", "gateway"))
    +    }
    +
    +    def write_cfg(body: str) -> None:
    +        (hermes_home / "config.yaml").write_text(textwrap.dedent(body))
    +
    +    def fresh_gateway():
    +        for mod in list(sys.modules.keys()):
    +            if mod.startswith(("hermes_cli", "gateway")):
    +                del sys.modules[mod]
    +        return importlib.import_module("gateway.run")
    +
    +    try:
    +        yield write_cfg, fresh_gateway
    +    finally:
    +        # Drop anything we (re)imported, then restore the pre-test snapshot so
    +        # the next test file sees the module objects it was loaded with.
    +        for k in list(sys.modules.keys()):
    +            if k.startswith(("hermes_cli", "gateway")):
    +                del sys.modules[k]
    +        sys.modules.update(_saved)
    +
    +
    +def test_top_level_max_tokens_propagates(isolated_home):
    +    """model.max_tokens is read into the gateway runtime kwargs (#20741)."""
    +    write_cfg, fresh_gateway = isolated_home
    +    write_cfg(
    +        """
    +        model:
    +          default: glm-5.1
    +          provider: openrouter
    +          max_tokens: 16384
    +        """
    +    )
    +    grun = fresh_gateway()
    +    kw = grun._resolve_runtime_agent_kwargs()
    +    assert kw["max_tokens"] == 16384
    +
    +
    +def test_per_provider_max_output_tokens_fallback(isolated_home):
    +    """A custom provider's max_output_tokens fills in when no global is set."""
    +    write_cfg, fresh_gateway = isolated_home
    +    write_cfg(
    +        """
    +        model:
    +          default: glm-5.1
    +          provider: mylocal
    +        providers:
    +          mylocal:
    +            api: http://localhost:11434/v1
    +            api_key: [redacted:secret-assignment]
    +            default_model: glm-5.1
    +            max_output_tokens: 12000
    +        """
    +    )
    +    grun = fresh_gateway()
    +    kw = grun._resolve_runtime_agent_kwargs()
    +    assert kw["max_tokens"] == 12000
    +
    +
    +def test_global_max_tokens_beats_per_provider(isolated_home):
    +    """The documented global model.max_tokens wins over a provider cap."""
    +    write_cfg, fresh_gateway = isolated_home
    +    write_cfg(
    +        """
    +        model:
    +          default: glm-5.1
    +          provider: mylocal
    +          max_tokens: 16384
    +        providers:
    +          mylocal:
    +            api: http://localhost:11434/v1
    +            api_key: [redacted:secret-assignment]
    +            default_model: glm-5.1
    +            max_output_tokens: 12000
    +        """
    +    )
    +    grun = fresh_gateway()
    +    kw = grun._resolve_runtime_agent_kwargs()
    +    assert kw["max_tokens"] == 16384
    +
    +
    +def test_env_override_beats_everything(isolated_home, monkeypatch):
    +    """HERMES_MAX_TOKENS is the internal override mechanism (highest priority)."""
    +    write_cfg, fresh_gateway = isolated_home
    +    monkeypatch.setenv("HERMES_MAX_TOKENS", "2048")
    +    write_cfg(
    +        """
    +        model:
    +          default: glm-5.1
    +          provider: mylocal
    +          max_tokens: 16384
    +        providers:
    +          mylocal:
    +            api: http://localhost:11434/v1
    +            api_key: [redacted:secret-assignment]
    +            default_model: glm-5.1
    +            max_output_tokens: 12000
    +        """
    +    )
    +    grun = fresh_gateway()
    +    kw = grun._resolve_runtime_agent_kwargs()
    +    assert kw["max_tokens"] == 2048
    +
    +
    +def test_no_config_leaves_max_tokens_none(isolated_home):
    +    """No cap configured anywhere -> max_tokens is None (no spurious limit)."""
    +    write_cfg, fresh_gateway = isolated_home
    +    write_cfg(
    +        """
    +        model:
    +          default: glm-5.1
    +          provider: openrouter
    +        """
    +    )
    +    grun = fresh_gateway()
    +    kw = grun._resolve_runtime_agent_kwargs()
    +    assert kw["max_tokens"] is None
    +
    +
    +def test_lift_helper_accepts_alias_and_rejects_garbage(isolated_home):
    +    """_lift_max_output_tokens accepts both keys, ignores non-positive/non-int."""
    +    write_cfg, _ = isolated_home
    +    write_cfg("model:\n  provider: openrouter\n")
    +    for mod in list(sys.modules.keys()):
    +        if mod.startswith("hermes_cli"):
    +            del sys.modules[mod]
    +    rp = importlib.import_module("hermes_cli.runtime_provider")
    +
    +    out: dict = {}
    +    rp._lift_max_output_tokens({"max_output_tokens": 8192}, out)
    +    assert out["max_output_tokens"] == 8192
    +
    +    out = {}
    +    rp._lift_max_output_tokens({"max_tokens": 4096}, out)
    +    assert out["max_output_tokens"] == 4096
    +
    +    for bad in ({"max_output_tokens": 0}, {"max_output_tokens": "x"}, {}):
    +        out = {}
    +        rp._lift_max_output_tokens(bad, out)
    +        assert "max_output_tokens" not in out

• #41694 fix(ollama): set default_max_tokens=65536 for custom/Ollama provider

  merged to main

  files: plugins/model-providers/custom/__init__.py, scripts/release.py