{"repo":"NousResearch/hermes-agent","public_id":"7266","display_label":"Model validation fixes across multiple providers","summary":"A cluster of bugs and fixes centered on the /model command and picker incorrectly rejecting valid models for Gemini, Anthropic, Alibaba/DashScope, and MiniMax providers. Fixes address validation logic, unreachable /models endpoints, static catalog fallbacks, and provider-specific edge cases.","subsystem":"hermes_cli","counts":{"members":20,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":8,"merged_prs":3,"attempt_prs":9},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":12842},"resolution":{"state":"partially_resolved","by":"pr-12842,pr-15136,pr-109195"},"freshness":{"state":"verified","last_verified_at":"2026-09-12T15:26:24+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.09115789473684209},"fix_edges":[{"issue_number":12220,"pr_number":109195,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":12399,"pr_number":12842,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":12460,"pr_number":12842,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":12532,"pr_number":15136,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":12547,"pr_number":12842,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":12611,"pr_number":12842,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":13186,"pr_number":15136,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":14651,"pr_number":15136,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":7266,"title":"[Bug] coding.dashscope.aliyuncs.com 端点模型验证警告问题","url":null,"state":"closed"},{"kind":"issue","number":11954,"title":"Bug: `hermes model` command fails to validate DashScope provider models","url":null,"state":"closed"},{"kind":"issue","number":12153,"title":"Custom provider model validation fails when /v1/models endpoint is unavailable","url":null,"state":"closed"},{"kind":"issue","number":12220,"title":"Support Bailian/DashScope endpoints that don't expose /v1/models","url":null,"state":"closed"},{"kind":"issue","number":12272,"title":"fix(models): /model fails for Alibaba/DashScope coding endpoint — /models returns 404","url":null,"state":"closed"},{"kind":"issue","number":12460,"title":"MiniMax provider model validation fails because /anthropic/models endpoint returns 404","url":null,"state":"closed"},{"kind":"issue","number":12532,"title":"Gateway /model picker fails for Gemini and Anthropic providers (validate_requested_model rejects curated models)","url":null,"state":"closed"},{"kind":"issue","number":13186,"title":"[Bug]: Regression: /model command validation blocks switching to Gemini models after native adapter update","url":null,"state":"closed"},{"kind":"pr","number":11967,"title":"fix(models): accept known DashScope models offline","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3f72b2fe1574fea279198f5e8f234ec386e945f3","number":0},{"kind":"pr","value":"#12287","number":12287},{"kind":"pr","value":"#12272","number":12272}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":12157,"title":"Fix custom provider model validation when /v1/models endpoint is unavailable","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#12153","number":12153}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":12287,"title":"fix(models): validate Alibaba coding-endpoint switches via catalog when /models 404s (#12272)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3f72b2fe1574fea279198f5e8f234ec386e945f3","number":0},{"kind":"pr","value":"#12272","number":12272}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":12585,"title":"fix(models): accept Gemini + Anthropic in gateway /model picker (#12532)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"7f26cea39","number":0},{"kind":"commit","value":"2303dd868","number":0},{"kind":"pr","value":"#15136","number":15136},{"kind":"pr","value":"#12618","number":12618}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":12829,"title":"fix(models): validate MiniMax models against static catalog","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#12842","number":12842},{"kind":"pr","value":"#12611","number":12611},{"kind":"pr","value":"#12460","number":12460},{"kind":"pr","value":"#12399","number":12399},{"kind":"pr","value":"#12547","number":12547}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":12842,"title":"fix(models): validate MiniMax models against static catalog","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/models.py","tests/test_minimax_model_validation.py"]},{"kind":"pr","number":13557,"title":"fix(gemini): validate native model switches against the AI Studio catalog","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3f72b2fe1574fea279198f5e8f234ec386e945f3","number":0},{"kind":"commit","value":"7f26cea390664c0cd39da27447f40556ee8664ac","number":0},{"kind":"pr","value":"#15136","number":15136},{"kind":"pr","value":"#13186","number":13186},{"kind":"pr","value":"#13952","number":13952}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":13952,"title":"fix(cli): validate Gemini models against curated catalog instead of probing /models","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"3f72b2fe1574fea279198f5e8f234ec386e945f3","number":0}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":15136,"title":"fix(models): consolidated validation + picker — anthropic_messages, native Anthropic, Gemini prefix, OpenAI catalog","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/main.py","hermes_cli/model_switch.py","hermes_cli/models.py","scripts/release.py","tests/hermes_cli/test_model_validation.py","tests/hermes_cli/test_user_providers_model_switch.py"],"fix_hunks":[{"file":"hermes_cli/main.py","hunks":["@@ -2900,11 +2900,16 @@ def _model_flow_named_custom(config, provider_info):\n \n     name = provider_info[\"name\"]\n     base_url = provider_info[\"base_url\"]\n+    api_mode = provider_info.get(\"api_mode\", \"\")\n     api_key = [redacted:secret-assignment]\"api_key\", \"\")\n     key_env = provider_info.get(\"key_env\", \"\")\n     saved_model = provider_info.get(\"model\", \"\")\n     provider_key = (provider_info.get(\"provider_key\") or \"\").strip()\n \n+    # Resolve key from env var if api_key not set directly\n+    if not api_key and key_env:\n+        api_key = [redacted:secret-assignment] \"\")\n+\n     print(f\"  Provider: {name}\")\n     print(f\"  URL:      {base_url}\")\n     if saved_model:","@@ -2912,7 +2917,10 @@ def _model_flow_named_custom(config, provider_info):\n     print()\n \n     print(\"Fetching available models...\")\n-    models = fetch_api_models(api_key, base_url, timeout=8.0)\n+    models = fetch_api_models(\n+        api_key, base_url, timeout=8.0,\n+        api_mode=api_mode or None,\n+    )\n \n     if models:\n         default_idx = 0"]},{"file":"hermes_cli/model_switch.py","hunks":["@@ -821,6 +821,7 @@ def switch_model(\n             target_provider,\n             api_key=[redacted:secret-assignment]\n             base_url=base_url,\n+            api_mode=api_mode or None,\n         )\n     except Exception as e:\n         validation = {","@@ -1239,6 +1240,15 @@ def list_authenticated_providers(\n                     if m and m not in models_list:\n                         models_list.append(m)\n \n+            # Official OpenAI API rows in providers: often have base_url but no\n+            # explicit models: dict — avoid a misleading zero count in /model.\n+            if not models_list:\n+                url_lower = str(api_url).strip().lower()\n+                if \"api.openai.com\" in url_lower:\n+                    fb = curated.get(\"openai\") or []\n+                    if fb:\n+                        models_list = list(fb)\n+\n             # Try to probe /v1/models if URL is set (but don't block on it)\n             # For now just show what we know from config\n             results.append({"]},{"file":"hermes_cli/models.py","hunks":["@@ -142,6 +142,18 @@ def _codex_curated_models() -> list[str]:\n         \"openai/gpt-5.4-pro\",\n         \"openai/gpt-5.4-nano\",\n     ],\n+    # Native OpenAI Chat Completions (api.openai.com). Used by /model counts and\n+    # provider_model_ids fallback when /v1/models is unavailable.\n+    \"openai\": [\n+        \"gpt-5.4\",\n+        \"gpt-5.4-mini\",\n+        \"gpt-5-mini\",\n+        \"gpt-5.3-codex\",\n+        \"gpt-5.2-codex\",\n+        \"gpt-4.1\",\n+        \"gpt-4o\",\n+        \"gpt-4o-mini\",\n+    ],\n     \"openai-codex\": _codex_curated_models(),\n     \"copilot-acp\": [\n         \"copilot-acp\",","@@ -1748,6 +1760,17 @@ def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False)\n         live = fetch_ollama_cloud_models(force_refresh=force_refresh)\n         if live:\n             return live\n+    if normalized == \"openai\":\n+        api_key = [redacted:secret-assignment]\"OPENAI_API_KEY\", \"\").strip()\n+        if api_key:\n+            base_raw = os.getenv(\"OPENAI_BASE_URL\", \"\").strip().rstrip(\"/\")\n+            base = base_raw or \"https://api.openai.com/v1\"\n+            try:\n+                live = fetch_api_models(api_key, base)\n+                if live:\n+                    return live\n+            except Exception:\n+                pass\n     if normalized == \"custom\":\n         base_url = _get_custom_base_url()\n         if base_url:","@@ -2220,8 +2243,15 @@ def probe_api_models(\n     api_key: [redacted:secret-assignment]\n     base_url: Optional[str],\n     timeout: float = 5.0,\n+    api_mode: Optional[str] = None,\n ) -> dict[str, Any]:\n-    \"\"\"Probe an OpenAI-compatible ``/models`` endpoint with light URL heuristics.\"\"\"\n+    \"\"\"Probe a ``/models`` endpoint with light URL heuristics.\n+\n+    For ``anthropic_messages`` mode, uses ``x-api-key`` and\n+    ``anthropic-version`` headers (Anthropic's native auth) instead of\n+    ``Authorization: Bearer``.  The response shape (``data[].id``) is\n+    identical, so the same parser works for both.\n+    \"\"\"\n     normalized = (base_url or \"\").strip().rstrip(\"/\")\n     if not normalized:\n         return {","@@ -2253,7 +2283,10 @@ def probe_api_models(\n \n     tried: list[str] = []\n     headers: dict[str, str] = {\"User-Agent\": _HERMES_USER_AGENT}\n-    if api_key:\n+    if api_key and api_mode == \"anthropic_messages\":\n+        headers[\"x-api-key\"] = api_key\n+        headers[\"anthropic-version\"] = \"2023-06-01\"\n+    elif api_key:\n         [redacted:secret-assignment]\"Authorization\"] = f\"Bearer {api_key}\"\n     if normalized.startswith(COPILOT_BASE_URL):\n         headers.update(copilot_default_headers())","@@ -2318,13 +2351,14 @@ def fetch_api_models(\n     api_key: [redacted:secret-assignment]\n     base_url: Optional[str],\n     timeout: float = 5.0,\n+    api_mode: Optional[str] = None,\n ) -> Optional[list[str]]:\n     \"\"\"Fetch the list of available model IDs from the provider's ``/models`` endpoint.\n \n     Returns a list of model ID strings, or ``None`` if the endpoint could not\n     be reached (network error, timeout, auth failure, etc.).\n     \"\"\"\n-    return probe_api_models(api_key, base_url, timeout=timeout).get(\"models\")\n+    return probe_api_models(api_key, base_url, timeout=timeout, api_mode=api_mode).get(\"models\")\n \n \n # ---------------------------------------------------------------------------","@@ -2452,6 +2486,7 @@ def validate_requested_model(\n     *,\n     api_key: [redacted:secret-assignment] = None,\n     base_url: Optional[str] = None,\n+    api_mode: Optional[str] = None,\n ) -> dict[str, Any]:\n     \"\"\"\n     Validate a ``/model`` value for the active provider.","@@ -2493,7 +2528,11 @@ def validate_requested_model(\n         }\n \n     if normalized == \"custom\":\n-        probe = probe_api_models(api_key, base_url)\n+        # Try probing with correct auth for the api_mode.\n+        if api_mode == \"anthropic_messages\":\n+            probe = probe_api_models(api_key, base_url, api_mode=api_mode)\n+        else:\n+            probe = probe_api_models(api_key, base_url)\n         api_models = probe.get(\"models\")\n         if api_models is not None:\n             if requested_for_lookup in set(api_models):","@@ -2542,12 +2581,17 @@ def validate_requested_model(\n             f\"Note: could not reach this custom endpoint's model listing at `{probe.get('probed_url')}`. \"\n             f\"Hermes will still save `{requested}`, but the endpoint should expose `/models` for verification.\"\n         )\n+        if api_mode == \"anthropic_messages\":\n+            message += (\n+                \"\\n  Many Anthropic-compatible proxies do not implement the Models API \"\n+                \"(GET /v1/models).  The model name has been accepted without verification.\"\n+            )\n         if probe.get(\"suggested_base_url\"):\n             message += f\"\\n  If this server expects `/v1`, try base URL: `{probe.get('suggested_base_url')}`\"\n \n         return {\n-            \"accepted\": False,\n-            \"persist\": False,\n+            \"accepted\": api_mode == \"anthropic_messages\",\n+            \"persist\": True,\n             \"recognized\": False,\n             \"message\": message,\n         }","@@ -2635,10 +2679,100 @@ def validate_requested_model(\n                 ),\n             }\n \n+    # Native Anthropic provider: /v1/models requires x-api-key (or Bearer for\n+    # OAuth) plus anthropic-version headers.  The generic OpenAI-style probe\n+    # below uses plain Bearer auth and 401s against Anthropic, so dispatch to\n+    # the native fetcher which handles both API keys and Claude-Code OAuth\n+    # tokens.  (The api_mode==\"anthropic_messages\" branch below handles the\n+    # Messages-API transport case separately.)\n+    if normalized == \"anthropic\":\n+        anthropic_models = _fetch_anthropic_models()\n+        if anthropic_models is not None:\n+            if requested_for_lookup in set(anthropic_models):\n+                return {\n+                    \"accepted\": True,\n+                    \"persist\": True,\n+                    \"recognized\": True,\n+                    \"message\": None,\n+                }\n+            auto = get_close_matches(requested_for_lookup, anthropic_models, n=1, cutoff=0.9)\n+            if auto:\n+                return {\n+                    \"accepted\": True,\n+                    \"persist\": True,\n+                    \"recognized\": True,\n+                    \"corrected_model\": auto[0],\n+                    \"message\": f\"Auto-corrected `{requested}` → `{auto[0]}`\",\n+                }\n+            suggestions = get_close_matches(requested, anthropic_models, n=3, cutoff=0.5)\n+            suggestion_text = \"\"\n+            if suggestions:\n+                suggestion_text = \"\\n  Similar models: \" + \", \".join(f\"`{s}`\" for s in suggestions)\n+            # Accept anyway — Anthropic sometimes gates newer/preview models\n+            # (e.g. snapshot IDs, early-access releases) behind accounts\n+            # even though they aren't listed on /v1/models.\n+            return {\n+                \"accepted\": True,\n+                \"persist\": True,\n+                \"recognized\": False,\n+                \"message\": (\n+                    f\"Note: `{requested}` was not found in Anthropic's /v1/models listing. \"\n+                    f\"It may still work if you have early-access or snapshot IDs.\"\n+                    f\"{suggestion_text}\"\n+                ),\n+            }\n+        # _fetch_anthropic_models returned None — no token resolvable or\n+        # network failure.  Fall through to the generic warning below.\n+\n+    # Anthropic Messages API: many proxies don't implement /v1/models.\n+    # Try probing with correct auth; if it fails, accept with a warning.\n+    if api_mode == \"anthropic_messages\":\n+        api_models = fetch_api_models(api_key, base_url, api_mode=api_mode)\n+        if api_models is not None:\n+            if requested_for_lookup in set(api_models):\n+                return {\n+                    \"accepted\": True,\n+                    \"persist\": True,\n+                    \"recognized\": True,\n+                    \"message\": None,\n+                }\n+            auto = get_close_matches(requested_for_lookup, api_models, n=1, cutoff=0.9)\n+            if auto:\n+                return {\n+                    \"accepted\": True,\n+                    \"persist\": True,\n+                    \"recognized\": True,\n+                    \"corrected_model\": auto[0],\n+                    \"message\": f\"Auto-corrected `{requested}` → `{auto[0]}`\",\n+                }\n+        # Probe failed or model not found — accept anyway (proxy likely\n+        # doesn't implement the Anthropic Models API).\n+        return {\n+            \"accepted\": True,\n+            \"persist\": True,\n+            \"recognized\": False,\n+            \"message\": (\n+                f\"Note: could not verify `{requested}` against this endpoint's \"\n+                f\"model listing.  Many Anthropic-compatible proxies do not \"\n+                f\"implement GET /v1/models.  The model name has been accepted \"\n+                f\"without verification.\"\n+            ),\n+        }\n+\n     # Probe the live API to check if the model actually exists\n     api_models = fetch_api_models(api_key, base_url)\n \n     if api_models is not None:\n+        # Gemini's OpenAI-compat /v1beta/openai/models endpoint returns IDs\n+        # prefixed with \"models/\" (e.g. \"models/gemini-2.5-flash\") — native\n+        # Gemini-API convention.  Our curated list and user input both use\n+        # the bare ID, so a direct set-membership check drops every known\n+        # Gemini model.  Strip the prefix before comparison.  See #12532.\n+        if normalized == \"gemini\":\n+            api_models = [\n+                m[len(\"models/\"):] if isinstance(m, str) and m.startswith(\"models/\") else m\n+                for m in api_models\n+            ]\n         if requested_for_lookup in set(api_models):\n             # API confirmed the model exists\n             return {"]},{"file":"scripts/release.py","hunks":["@@ -192,6 +192,9 @@\n     \"[redacted:email]\": \"LeonSGP43\",\n     \"[redacted:email]\": \"vlwkaos\",\n     \"[redacted:email]\": \"Tranquil-Flow\",\n+    \"[redacted:email]\": \"Wangshengyang2004\",\n+    \"[redacted:email]\": \"H-Ali13381\",\n+    \"[redacted:email]\": \"XieNBi\",\n     \"[redacted:email]\": \"nummanali\",\n     \"[redacted:email]\": \"whitehatjr1001\",\n     \"[redacted:email]\": \"0xNyk\","]},{"file":"tests/hermes_cli/test_model_validation.py","hunks":["@@ -566,8 +566,11 @@ def test_custom_endpoint_warns_with_probed_url_and_v1_hint(self):\n                 base_url=\"http://localhost:8000\",\n             )\n \n+        # Unreachable /models on a custom endpoint no longer hard-rejects —\n+        # the model is persisted with a warning so Cloudflare-protected /\n+        # proxy endpoints that don't expose /models still work. See #12950.\n         assert result[\"accepted\"] is False\n-        assert result[\"persist\"] is False\n+        assert result[\"persist\"] is True\n         assert \"http://localhost:8000/v1/models\" in result[\"message\"]\n         assert \"http://localhost:8000/v1\" in result[\"message\"]\n "]},{"file":"tests/hermes_cli/test_user_providers_model_switch.py","hunks":["@@ -197,6 +197,58 @@ def test_list_authenticated_providers_dict_models_dedupe_with_default(monkeypatc\n     assert user_prov[\"models\"].count(\"model-a\") == 1\n \n \n+def test_openai_native_curated_catalog_is_non_empty():\n+    \"\"\"Regression: built-in openai must have a static catalog for picker totals.\"\"\"\n+    from hermes_cli.models import _PROVIDER_MODELS\n+\n+    assert _PROVIDER_MODELS.get(\"openai\")\n+    assert len(_PROVIDER_MODELS[\"openai\"]) >= 4\n+\n+\n+def test_list_authenticated_providers_openai_built_in_nonzero_total(monkeypatch):\n+    \"\"\"Built-in openai row must not report total_models=0 when creds exist.\"\"\"\n+    monkeypatch.setenv(\"OPENAI_API_KEY\", \"sk-test\")\n+    monkeypatch.setattr(\n+        \"agent.models_dev.fetch_models_dev\",\n+        lambda: {\"openai\": {\"env\": [\"OPENAI_API_KEY\"]}},\n+    )\n+    monkeypatch.setattr(\"hermes_cli.providers.HERMES_OVERLAYS\", {})\n+\n+    providers = list_authenticated_providers(\n+        current_provider=\"\",\n+        current_base_url=\"\",\n+        user_providers={},\n+        custom_providers=[],\n+        max_models=50,\n+    )\n+    row = next((p for p in providers if p.get(\"slug\") == \"openai\"), None)\n+    assert row is not None\n+    assert row[\"total_models\"] > 0\n+\n+\n+def test_list_authenticated_providers_user_openai_official_url_fallback(monkeypatch):\n+    \"\"\"User providers: api.openai.com with no models list uses native curated fallback.\"\"\"\n+    monkeypatch.setattr(\"agent.models_dev.fetch_models_dev\", lambda: {})\n+    monkeypatch.setattr(\"hermes_cli.providers.HERMES_OVERLAYS\", {})\n+\n+    user_providers = {\n+        \"openai-direct\": {\n+            \"name\": \"OpenAI Direct\",\n+            \"api\": \"https://api.openai.com/v1\",\n+        }\n+    }\n+    providers = list_authenticated_providers(\n+        current_provider=\"\",\n+        current_base_url=\"\",\n+        user_providers=user_providers,\n+        custom_providers=[],\n+        max_models=50,\n+    )\n+    row = next((p for p in providers if p.get(\"slug\") == \"openai-direct\"), None)\n+    assert row is not None\n+    assert row[\"total_models\"] > 0\n+\n+\n def test_list_authenticated_providers_fallback_to_default_only(monkeypatch):\n     \"\"\"When no models array is provided, should fall back to default_model.\"\"\"\n     monkeypatch.setattr(\"agent.models_dev.fetch_models_dev\", lambda: {})\n"]}]},{"kind":"pr","number":94587,"title":"fix(cli): accept custom models without catalog listings","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#96379","number":96379}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":96379,"title":"fix(cli): accept custom models without catalog listings","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"71c72e078a","number":0},{"kind":"pr","value":"#109195","number":109195}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":109195,"title":"fix(cli): custom endpoints without GET /models no longer reject `/model` switches (#12220, salvage #96379)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["hermes_cli/models_validate.py","tests/hermes_cli/test_model_validation.py"]}],"trust":{"default_tier":"verified","includes_suggested":false}}