Model picker behavior for multi-model custom providers

Fixes on record

Merged to main. A stamp counts the issues a fix is linked to.

• #7187 fix(model): salvage model/provider resolution PRs #6860 #6844 #6863 #6943

  merged to main

  files: agent/auxiliary_client.py, agent/error_classifier.py, cli.py, hermes_cli/main.py, hermes_cli/model_normalize.py, hermes_cli/model_switch.py, run_agent.py, tests/agent/test_auxiliary_named_custom_providers.py, tests/agent/test_error_classifier.py, tests/hermes_cli/test_codex_models.py, tests/hermes_cli/test_custom_provider_model_switch.py, tests/hermes_cli/test_model_normalize.py, tests/hermes_cli/test_opencode_go_in_model_list.py, tests/run_agent/test_fallback_model.py, tests/run_agent/test_run_agent.py, tests/run_agent/test_switch_model_context.py

• #21211 fix(model_switch): live model discovery for custom_providers in /model picker (salvage #19072)

  merged to main

  files: hermes_cli/model_switch.py, tests/hermes_cli/test_model_switch_custom_providers.py

  Show the diff

  • hermes_cli/model_switch.py

    @@ -1637,7 +1637,8 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool:
                             groups[group_key]["models"].append(m)
     
             _section4_emitted_slugs: set = set()
    -        for grp in groups.values():
    +        for grp_key, grp in groups.items():
    +            api_url, api_key = [redacted:secret-assignment]
                 slug = grp["slug"]
                 # If the slug is already claimed by a built-in / overlay /
                 # user-provider row (sections 1-3), skip this custom group
    @@ -1675,6 +1676,18 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool:
                 _grp_url_norm = _pair_key[1]
                 if _grp_url_norm and _grp_url_norm in _builtin_endpoints:
                     continue
    +            # Live model discovery from custom provider endpoints (matches
    +            # Section 3 behavior for user ``providers:`` entries).
    +            if api_url and api_key:
    +                try:
    +                    from hermes_cli.models import fetch_api_models
    +
    +                    live_models = fetch_api_models(api_key, api_url)
    +                    if live_models:
    +                        grp["models"] = live_models
    +                        grp["total_models"] = len(live_models)
    +                except Exception:
    +                    pass
                 results.append({
                     "slug": slug,
                     "name": grp["name"],

  • tests/hermes_cli/test_model_switch_custom_providers.py

    @@ -506,3 +506,64 @@ def _fake_fetch(api_key=None, base_url=None, timeout=5.0):
         )
     
         assert "base_url" not in captured
    +
    +
    +def test_custom_providers_uses_live_models_for_multi_model_endpoint(monkeypatch):
    +    """Custom providers with api_key + base_url should prefer live /models.
    +
    +    Custom providers (section 4 of list_authenticated_providers) point at
    +    gateways like Bifrost that expose hundreds of models.  Reading only the
    +    static ``models:`` dict from config.yaml leaves the /model picker with
    +    a stale subset.  Live discovery fills the picker with all available
    +    models from the endpoint.
    +    """
    +    monkeypatch.setattr("agent.models_dev.fetch_models_dev", lambda: {})
    +    monkeypatch.setattr("hermes_cli.providers.HERMES_OVERLAYS", {})
    +
    +    calls = []
    +
    +    def fake_fetch_api_models(api_key, base_url):
    +        calls.append((api_key, base_url))
    +        return ["gateway-model-a", "gateway-model-b", "gateway-model-c"]
    +
    +    monkeypatch.setattr("hermes_cli.models.fetch_api_models", fake_fetch_api_models)
    +
    +    custom_providers = [
    +        {
    +            "name": "my-gateway",
    +            "api_key": "[redacted:secret-assignment]",
    +            "base_url": "https://gateway.example.com/v1",
    +            "model": "gateway-model-a",
    +            "models": {
    +                "gateway-model-a": {"context_length": 128000},
    +                "gateway-model-b": {"context_length": 128000},
    +            },
    +        }
    +    ]
    +
    +    providers = list_authenticated_providers(
    +        current_provider="openrouter",
    +        current_base_url="https://openrouter.ai/api/v1",
    +        custom_providers=custom_providers,
    +        max_models=50,
    +    )
    +
    +    gateway_prov = next(
    +        (
    +            p
    +            for p in providers
    +            if p.get("api_url") == "https://gateway.example.com/v1"
    +        ),
    +        None,
    +    )
    +
    +    assert gateway_prov is not None, "Custom provider group not found in results"
    +    assert calls == [("sk-gateway-key", "https://gateway.example.com/v1")], (
    +        "fetch_api_models must be called with the custom provider's credentials"
    +    )
    +    assert gateway_prov["models"] == [
    +        "gateway-model-a",
    +        "gateway-model-b",
    +        "gateway-model-c",
    +    ], "Live models must replace the static subset"
    +    assert gateway_prov["total_models"] == 3