Vision override lookup for stripped custom provider keys

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #57649 fix(vision): custom-provider routing — stripped custom:<name> override, session model, canonical slug

  merged to main

  files: agent/image_routing.py, gateway/run.py, hermes_cli/model_setup_flows.py, tests/agent/test_image_routing.py, tests/gateway/test_image_input_routing_runtime.py, tests/gateway/test_native_image_buffer_isolation.py, tests/gateway/test_queued_native_image_session_key.py, tests/hermes_cli/test_model_provider_persistence.py, tests/hermes_cli/test_runtime_provider_resolution.py

  Show the diff

  • agent/image_routing.py

    @@ -185,7 +185,8 @@ def _supports_vision_override(
           2. ``providers.<provider>.models.<model>.supports_vision``
              (named custom providers — ``provider`` may be the runtime-resolved
              value ``"custom"`` and/or the user-declared name under
    -         ``model.provider``; both are tried)
    +         ``model.provider``; both are tried. For ``custom:<name>`` syntax,
    +         the stripped ``<name>`` is also tried as a provider key.)
     
         Returns None when no override is set, so the caller falls through to
         models.dev. Returns False explicitly only when the user wrote a
    @@ -205,11 +206,16 @@ def _supports_vision_override(
         # get rewritten to provider="custom" at runtime
         # (hermes_cli/runtime_provider.py:_resolve_named_custom_runtime), so the
         # config still holds the user-declared name under model.provider. Try
    -    # both as candidate provider keys.
    +    # both as candidate provider keys, plus the stripped suffix from
    +    # "custom:<name>" (where <name> is the key under providers:).
         config_provider = str(model_cfg.get("provider") or "").strip()
    +    # Extract the stripped name from "custom:<name>" if present
    +    stripped_suffix = ""
    +    if config_provider.startswith("custom:"):
    +        stripped_suffix = config_provider[len("custom:"):]
         providers_raw = cfg.get("providers")
         providers_cfg: Dict[str, Any] = providers_raw if isinstance(providers_raw, dict) else {}
    -    for p in dict.fromkeys(filter(None, (provider, config_provider))):
    +    for p in dict.fromkeys(filter(None, (provider, config_provider, stripped_suffix))):
             entry_raw = providers_cfg.get(p)
             entry: Dict[str, Any] = entry_raw if isinstance(entry_raw, dict) else {}
             models_raw = entry.get("models")

  • gateway/run.py

    @@ -10035,7 +10035,10 @@ async def _prepare_inbound_message_text(
                 if image_paths:
                     # Decide routing: native (attach pixels) vs text (vision_analyze
                     # pre-run + prepend description).  See agent/image_routing.py.
    -                _img_mode = self._decide_image_input_mode()
    +                _img_mode = self._decide_image_input_mode(
    +                    source=source,
    +                    session_key=session_key,
    +                )
                     if _img_mode == "native":
                         # Defer attachment to the run_conversation call site.
                         pending_native = getattr(self, "_pending_native_image_paths_by_session", None)
    @@ -14381,25 +14384,62 @@ def _shutdown_executor(self) -> None:
             except TypeError:
                 executor.shutdown(wait=False)
     
    -    def _decide_image_input_mode(self) -> str:
    -        """Resolve the image-input routing for the currently active model.
    +    def _decide_image_input_mode(
    +        self,
    +        *,
    +        source: Optional[SessionSource] = None,
    +        session_key: Optional[str] = None,
    +        user_config: Optional[dict] = None,
    +        provider: Optional[str] = None,
    +        model: Optional[str] = None,
    +    ) -> str:
    +        """Resolve image-input routing for the effective model this turn.
     
             Returns ``"native"`` (attach pixels on the user turn) or ``"text"``
             (pre-analyze with vision_analyze and prepend the description). See
             agent/image_routing.py for the full decision table.
     
    -        The active provider/model are read from config.yaml so the decision
    -        tracks ``/model`` switches automatically on the next message.
    +        Gateway sessions can have /model overrides that live outside
    +        config.yaml. Image preprocessing runs before AIAgent sets the
    +        auxiliary_client runtime globals, so resolve the same per-session
    +        runtime bundle the upcoming agent turn will use instead of consulting
    +        only the persisted default model.
             """
             try:
                 from agent.image_routing import decide_image_input_mode
                 from agent.auxiliary_client import _read_main_model, _read_main_provider
                 from hermes_cli.config import load_config
     
    -            cfg = load_config()
    -            provider = _read_main_provider()
    -            model = _read_main_model()
    -            return decide_image_input_mode(provider, model, cfg)
    +            cfg = user_config if isinstance(user_config, dict) else load_config()
    +            resolved_provider = (provider or "").strip()
    +            resolved_model = (model or "").strip()
    +
    +            needs_session_runtime = not resolved_provider or not resolved_model
    +            has_session_identity = source is not None or session_key
    +            if needs_session_runtime and has_session_identity:
    +                try:
    +                    turn_model, runtime_kwargs = self._resolve_session_agent_runtime(
    +                        source=source,
    +                        session_key=session_key,
    +                        user_config=cfg,
    +                    )
    +                    if not resolved_model and isinstance(turn_model, str):
    +                        resolved_model = turn_model.strip()
    +                    runtime_provider = runtime_kwargs.get("provider") if isinstance(runtime_kwargs, dict) else None
    +                    if not resolved_provider and isinstance(runtime_provider, str):
    +                        resolved_provider = runtime_provider.strip()
    +                except Exception as exc:
    +                    logger.debug(
    +                        "image_routing: session runtime resolution failed, falling back to config — %s",
    +                        exc,
    +                    )
    +
    +            if not resolved_provider:
    +                resolved_provider = _read_main_provider()
    +            if not resolved_model:
    +                resolved_model = _read_main_model()
    +
    +            return decide_image_input_mode(resolved_provider, resolved_model, cfg)
             except Exception as exc:
                 logger.debug("image_routing: decision failed, falling back to text — %s", exc)
                 return "text"

  • hermes_cli/model_setup_flows.py

    @@ -1446,7 +1446,7 @@ def _model_flow_named_custom(config, provider_info):
             model = {"default": model} if model else {}
             cfg["model"] = model
         if provider_key:
    -        model["provider"] = provider_key
    +        model["provider"] = "custom:" + provider_key.strip().lower().replace(" ", "-")
             model.pop("base_url", None)
             model.pop("api_key", None)
         else:

  • tests/agent/test_image_routing.py

    @@ -224,6 +224,37 @@ def test_malformed_sections_are_ignored(self):
             cfg = {"model": "some-string", "providers": ["not-a-dict"]}
             assert _supports_vision_override(cfg, "custom", "my-llava") is None
     
    +    def test_custom_colon_name_stripped_suffix_lookup(self):
    +        # model.provider: custom:my-proxy → should resolve stripped key "my-proxy"
    +        cfg = {
    +            "model": {"provider": "custom:my-proxy"},
    +            "providers": {
    +                "my-proxy": {"models": {"gpt-5.5": {"supports_vision": True}}},
    +            },
    +        }
    +        assert _supports_vision_override(cfg, "custom", "gpt-5.5") is True
    +
    +    def test_custom_colon_name_stripped_suffix_false(self):
    +        # Explicitly disabled vision on the stripped key.
    +        cfg = {
    +            "model": {"provider": "custom:my-proxy"},
    +            "providers": {
    +                "my-proxy": {"models": {"gpt-5.5": {"supports_vision": False}}},
    +            },
    +        }
    +        assert _supports_vision_override(cfg, "custom", "gpt-5.5") is False
    +
    +    def test_custom_colon_name_no_stripped_key_falls_through(self):
    +        # custom:my-proxy but providers only has "custom" — stripped key
    +        # doesn't match, but "custom" does via runtime provider.
    +        cfg = {
    +            "model": {"provider": "custom:my-proxy"},
    +            "providers": {
    +                "custom": {"models": {"gpt-5.5": {"supports_vision": True}}},
    +            },
    +        }
    +        assert _supports_vision_override(cfg, "custom", "gpt-5.5") is True
    +
     
     # ─── _lookup_supports_vision (override-aware) ────────────────────────────────
     

  • tests/gateway/test_image_input_routing_runtime.py

    @@ -0,0 +1,140 @@
    +import pytest
    +
    +from gateway.config import GatewayConfig, Platform, PlatformConfig
    +from gateway.platforms.base import MessageEvent, MessageType
    +from gateway.run import GatewayRunner
    +from gateway.session import SessionSource
    +
    +
    +def _make_runner() -> GatewayRunner:
    +    runner = object.__new__(GatewayRunner)
    +    runner.config = GatewayConfig(
    +        platforms={Platform.TELEGRAM: PlatformConfig(enabled=True, token="fake")}
    +    )
    +    runner.adapters = {}
    +    runner._pending_native_image_paths_by_session = {}
    +    runner._session_model_overrides = {}
    +    runner._session_reasoning_overrides = {}
    +    return runner
    +
    +
    +def _source() -> SessionSource:
    +    return SessionSource(
    +        platform=Platform.TELEGRAM,
    +        chat_id="273403055",
    +        chat_type="dm",
    +        user_id="42",
    +        user_name="Maxim",
    +    )
    +
    +
    +def _image_event(text: str = "look") -> MessageEvent:
    +    return MessageEvent(
    +        text=text,
    +        message_type=MessageType.PHOTO,
    +        source=_source(),
    +        media_urls=["/tmp/cashback.png"],
    +        media_types=["image/png"],
    +    )
    +
    +
    +def _auto_config() -> dict:
    +    return {
    +        "agent": {"image_input_mode": "auto"},
    +        "auxiliary": {"vision": {"provider": "auto", "model": "", "base_url": ""}},
    +        "model": {"provider": "xiaomi", "default": "mimo-v2.5-pro"},
    +    }
    +
    +
    [redacted:email]
    +async def test_prepare_image_routing_uses_session_vision_model_override(monkeypatch):
    +    """Telegram /model overrides must affect native-vs-text image routing.
    +
    +    Regression: _prepare_inbound_message_text used config.yaml's default model
    +    before the per-session model override was installed on auxiliary_client's
    +    runtime globals. A Telegram session switched to a vision model still had
    +    screenshots pre-analyzed as text when config.default was text-only.
    +    """
    +    runner = _make_runner()
    +    source = _source()
    +    event = _image_event()
    +    cfg = _auto_config()
    +
    +    monkeypatch.setattr("gateway.run._load_gateway_config", lambda: cfg)
    +    monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg)
    +    monkeypatch.setattr("agent.auxiliary_client._read_main_provider", lambda: "xiaomi")
    +    monkeypatch.setattr("agent.auxiliary_client._read_main_model", lambda: "mimo-v2.5-pro")
    +    monkeypatch.setattr(
    +        runner,
    +        "_resolve_session_agent_runtime",
    +        lambda **_: ("gpt-5.5", {"provider": "openai-codex"}),
    +    )
    +
    +    def fake_supports(provider, model, config):
    +        return provider == "openai-codex" and model == "gpt-5.5"
    +
    +    monkeypatch.setattr("agent.image_routing._lookup_supports_vision", fake_supports)
    +
    +    async def fail_enrich(*_args, **_kwargs):
    +        pytest.fail("vision-capable session override should use native image routing")
    +
    +    monkeypatch.setattr(runner, "_enrich_message_with_vision", fail_enrich)
    +
    +    result = await runner._prepare_inbound_message_text(
    +        event=event,
    +        source=source,
    +        history=[],
    +    )
    +
    +    session_key = runner._session_key_for_source(source)
    +    assert result == "look"
    +    assert runner._pending_native_image_paths_by_session[session_key] == [
    +        "/tmp/cashback.png"
    +    ]
    +
    +
    [redacted:email]
    +async def test_prepare_image_routing_falls_back_to_text_for_text_only_session_override(monkeypatch):
    +    """A text-only session override should get vision_analyze text fallback.
    +
    +    Regression mirror case: if config.default is a vision model but the current
    +    Telegram session is switched to a text-only provider (for example Mimo),
    +    auto routing must not attach pixels natively to the text-only model.
    +    """
    +    runner = _make_runner()
    +    source = _source()
    +    event = _image_event()
    +    cfg = _auto_config()
    +    cfg["model"] = {"provider": "openai-codex", "default": "gpt-5.5"}
    +
    +    monkeypatch.setattr("gateway.run._load_gateway_config", lambda: cfg)
    +    monkeypatch.setattr("hermes_cli.config.load_config", lambda: cfg)
    +    monkeypatch.setattr("agent.auxiliary_client._read_main_provider", lambda: "openai-codex")
    +    monkeypatch.setattr("agent.auxiliary_client._read_main_model", lambda: "gpt-5.5")
    +    monkeypatch.setattr(
    +        runner,
    +        "_resolve_session_agent_runtime",
    +        lambda **_: ("mimo-v2.5-pro", {"provider": "xiaomi"}),
    +    )
    +
    +    def fake_supports(provider, model, config):
    +        return provider == "openai-codex" and model == "gpt-5.5"
    +
    +    monkeypatch.setattr("agent.image_routing._lookup_supports_vision", fake_supports)
    +
    +    async def fake_enrich(user_text, image_paths):
    +        assert user_text == "look"
    +        assert image_paths == ["/tmp/cashback.png"]
    +        return "[vision summary]\n\nlook"
    +
    +    monkeypatch.setattr(runner, "_enrich_message_with_vision", fake_enrich)
    +
    +    result = await runner._prepare_inbound_message_text(
    +        event=event,
    +        source=source,
    +        history=[],
    +    )
    +
    +    session_key = runner._session_key_for_source(source)
    +    assert result == "[vision summary]\n\nlook"
    +    assert runner._pending_native_image_paths_by_session.get(session_key) is None

  • tests/gateway/test_native_image_buffer_isolation.py

    @@ -14,7 +14,7 @@ def _make_runner() -> GatewayRunner:
         runner.adapters = {}
         runner._model = "openai/gpt-4.1-mini"
         runner._base_url = None
    -    runner._decide_image_input_mode = lambda: "native"
    +    runner._decide_image_input_mode = lambda **_: "native"
         return runner
     
     

  • tests/gateway/test_queued_native_image_session_key.py

    @@ -86,7 +86,7 @@ def _make_runner(adapter):
         )
         runner._model = "openai/gpt-4.1-mini"
         runner._base_url = None
    -    runner._decide_image_input_mode = lambda: "native"
    +    runner._decide_image_input_mode = lambda **_kw: "native"
         return runner
     
     

  • tests/hermes_cli/test_model_provider_persistence.py

    @@ -207,6 +207,51 @@ def test_named_custom_provider_preserves_explicit_api_mode(self, config_home):
             assert model.get("base_url") == "https://packy.example.com/v1"
             assert model.get("api_mode") == "codex_responses"
     
    +    def test_named_custom_provider_with_builtin_slug_persists_custom_prefix(
    +        self, config_home, monkeypatch
    +    ):
    +        """providers.<builtin-slug> must persist as a named custom provider."""
    +        import yaml
    +
    +        from hermes_cli.main import _model_flow_named_custom
    +
    +        config_path = config_home / "config.yaml"
    +        config_path.write_text(
    +            "providers:\n"
    +            "  minimax-cn:\n"
    +            "    name: MiniMax CN Proxy\n"
    +            "    api: https://mimimax.cn/v1\n"
    +            "    key_env: MINIMAX_CN_PROXY_KEY\n"
    +            "    transport: chat_completions\n"
    +            "    model: MiniMax-M3\n"
    +            "    default_model: MiniMax-M3\n"
    +        )
    +        monkeypatch.setenv("MINIMAX_CN_PROXY_KEY", "proxy-secret")
    +
    +        provider_info = {
    +            "name": "MiniMax CN Proxy",
    +            "base_url": "https://mimimax.cn/v1",
    +            "api_key": "",
    +            "key_env": "MINIMAX_CN_PROXY_KEY",
    +            "model": "MiniMax-M3",
    +            "api_mode": "chat_completions",
    +            "provider_key": "minimax-cn",
    +        }
    +
    +        with patch("hermes_cli.auth._save_model_choice"), \
    +             patch("hermes_cli.auth.deactivate_provider"), \
    +             patch("hermes_cli.models.fetch_api_models", return_value=["MiniMax-M3"]), \
    +             patch("hermes_cli.curses_ui.curses_radiolist", side_effect=OSError("no tty in test")), \
    +             patch("builtins.input", return_value="1"):
    +            _model_flow_named_custom({}, provider_info)
    +
    +        config = yaml.safe_load(config_path.read_text()) or {}
    +        model = config.get("model")
    +        assert isinstance(model, dict)
    +        assert model.get("provider") == "custom:minimax-cn"
    +        assert "base_url" not in model
    +        assert "api_key" not in model
    +
         def test_copilot_acp_provider_saved_when_selected(self, config_home):
             """_model_flow_copilot_acp should persist provider/base_url/model together."""
             from hermes_cli.main import _model_flow_copilot_acp
    @@ -555,4 +600,3 @@ def fake_choice(choices, *, default=0, title=""):
                 _select_zai_endpoint(custom_url)
     
             assert captured["default"] == expected_default
    -

  • tests/hermes_cli/test_runtime_provider_resolution.py

    @@ -1272,6 +1272,33 @@ def test_resolve_requested_provider_precedence(monkeypatch):
         assert rp.resolve_requested_provider() == "auto"
     
     
    +def test_resolve_runtime_provider_named_custom_with_builtin_slug(monkeypatch):
    +    monkeypatch.setenv("MINIMAX_CN_PROXY_KEY", "proxy-secret")
    +    monkeypatch.setattr(
    +        rp,
    +        "load_config",
    +        lambda: {
    +            "model": {"provider": "custom:minimax-cn"},
    +            "providers": {
    +                "minimax-cn": {
    +                    "name": "MiniMax CN Proxy",
    +                    "api": "https://mimimax.cn/v1",
    +                    "key_env": "MINIMAX_CN_PROXY_KEY",
    +                    "transport": "chat_completions",
    +                    "default_model": "MiniMax-M3",
    +                }
    +            },
    +        },
    +    )
    +
    +    resolved = rp.resolve_runtime_provider()
    +
    +    assert resolved["provider"] == "custom"
    +    assert resolved["base_url"] == "https://mimimax.cn/v1"
    +    assert resolved["api_key"] == "proxy-secret"
    +    assert resolved["api_mode"] == "chat_completions"
    +
    +
     # ── api_mode config override tests ──────────────────────────────────────