{"repo":"NousResearch/hermes-agent","public_id":"65650","display_label":"CLI model picker slow with custom provider discovery","summary":"Reports and fixes addressing slow performance in the CLI /model picker when custom providers have discover_models enabled. The work involves parallelizing or skipping provider probing to reduce latency.","subsystem":"hermes_cli","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":65652,"merged_at":"2026-07-19T23:33:50+00:00"},"resolution":{"state":"resolved","by":"pr-65652"},"freshness":{"state":"verified","last_verified_at":"2026-07-19T23:33:50+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":65650,"pr_number":65652,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-07-19T23:33:50+00:00"}],"members":[{"kind":"issue","number":65650,"title":"/model picker slow (~5s) when custom providers have discover_models enabled (CLI)","url":null,"state":"closed"},{"kind":"pr","number":65652,"title":"fix: speed up CLI /model picker by skipping non-current custom provider probing","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cli.py","hermes_cli/model_switch.py","tests/hermes_cli/test_inventory.py","tests/hermes_cli/test_model_switch_custom_providers.py"],"fix_hunks":[{"file":"cli.py","hunks":["@@ -8087,7 +8087,11 @@ def _handle_model_switch(self, cmd_original: str):\n             try:\n                 if ctx is None:\n                     raise RuntimeError(\"inventory context unavailable\")\n-                providers = build_models_payload(ctx)[\"providers\"]\n+                providers = build_models_payload(\n+                    ctx,\n+                    probe_custom_providers=force_refresh,\n+                    probe_current_custom_provider=not force_refresh,\n+                )[\"providers\"]\n             except Exception:\n                 providers = []\n "]},{"file":"hermes_cli/model_switch.py","hunks":["@@ -100,6 +100,52 @@ def _add(candidate: Any) -> None:\n     return ids\n \n \n+def _save_discovered_models_to_config(\n+    api_url: str, model_ids: list[str]\n+) -> None:\n+    \"\"\"Persist discovered models into ``custom_providers`` in config.yaml.\n+\n+    Called after a successful ``/v1/models`` probe so that the next read\n+    with ``discover_models: false`` uses the cached list instead of a stale\n+    or minimal manually-configured subset.\n+\n+    Matches entries by ``base_url`` (trailing-slash-normalised).  A failed\n+    config write is swallowed — the picker still shows the live models for\n+    this session.\n+    \"\"\"\n+    if not api_url or not model_ids:\n+        return\n+    try:\n+        from hermes_cli.config import load_config, save_config\n+\n+        cfg = load_config()\n+        providers = cfg.get(\"custom_providers\") or []\n+        if not isinstance(providers, list):\n+            return\n+\n+        norm_url = api_url.strip().rstrip(\"/\").lower()\n+        changed = False\n+        for entry in providers:\n+            if not isinstance(entry, dict):\n+                continue\n+            entry_url = (entry.get(\"base_url\", \"\") or entry.get(\"url\", \"\") or \"\").strip()\n+            if entry_url.rstrip(\"/\").lower() != norm_url:\n+                continue\n+            existing = entry.get(\"models\")\n+            # Only update when models are stale — avoids unnecessary\n+            # config writes on every picker open.\n+            if isinstance(existing, list) and existing == model_ids:\n+                continue\n+            entry[\"models\"] = model_ids\n+            changed = True\n+\n+        if changed:\n+            cfg[\"custom_providers\"] = providers\n+            save_config(cfg)\n+    except Exception:\n+        pass\n+\n+\n def _bare_custom_provider_def(current_base_url: str) -> Optional[ProviderDef]:\n     \"\"\"ProviderDef for a direct ``model.provider: custom`` endpoint.\"\"\"\n     base_url = str(current_base_url or \"\").strip()","@@ -2376,6 +2422,12 @@ def _has_aws_sdk_creds_for_listing(slug: str) -> bool:\n                     if live_models:\n                         grp[\"models\"] = live_models\n                         grp[\"total_models\"] = len(live_models)\n+                        # Auto-save discovered models back to config so\n+                        # ``discover_models: false`` has a populated cache\n+                        # on the next read.  A failed save is non-fatal.\n+                        _save_discovered_models_to_config(\n+                            api_url, live_models\n+                        )\n                 except Exception:\n                     pass\n             results.append({"]},{"file":"tests/hermes_cli/test_inventory.py","hunks":["@@ -276,6 +276,43 @@ def test_build_models_payload_can_probe_only_current_custom_provider():\n     assert mock_list.call_args.kwargs[\"probe_current_custom_provider\"] is True\n \n \n+def test_cli_model_picker_forwards_force_refresh_to_probe_flags():\n+    \"\"\"CLI /model picker must pass force_refresh to probe flags (#65652, #65650).\n+\n+    Normal open (/model bare) skips non-current probes; /model --refresh probes\n+    all custom providers to freshen their model lists.\n+    \"\"\"\n+    ctx = _empty_ctx()\n+\n+    # Normal open — skip non-current probes\n+    force_refresh = False\n+    with patch(\n+        \"hermes_cli.model_switch.list_authenticated_providers\",\n+        return_value=[],\n+    ) as mock_list:\n+        build_models_payload(\n+            ctx,\n+            probe_custom_providers=force_refresh,\n+            probe_current_custom_provider=not force_refresh,\n+        )\n+    assert mock_list.call_args.kwargs[\"probe_custom_providers\"] is False\n+    assert mock_list.call_args.kwargs[\"probe_current_custom_provider\"] is True\n+\n+    # Refresh open — probe everything\n+    force_refresh = True\n+    with patch(\n+        \"hermes_cli.model_switch.list_authenticated_providers\",\n+        return_value=[],\n+    ) as mock_list:\n+        build_models_payload(\n+            ctx,\n+            probe_custom_providers=force_refresh,\n+            probe_current_custom_provider=not force_refresh,\n+        )\n+    assert mock_list.call_args.kwargs[\"probe_custom_providers\"] is True\n+    assert mock_list.call_args.kwargs[\"probe_current_custom_provider\"] is False\n+\n+\n def test_list_authenticated_providers_force_fresh_is_keyword_only():\n     \"\"\"``force_fresh_nous_tier`` must be keyword-only on the public listing API.\n "]},{"file":"tests/hermes_cli/test_model_switch_custom_providers.py","hunks":["@@ -1326,3 +1326,156 @@ def test_resolve_custom_provider_bare_custom_self_heal_passes_key_env():\n \n     assert resolved is not None\n     assert resolved.api_key_env_vars == (\"XIAOMI_MIMO_API_KEY\",)\n+\n+\n+def test_discovered_models_auto_saved_to_cache(monkeypatch):\n+    \"\"\"Discovered models are persisted to config so ``discover_models: false``\n+    has a populated cache on the next read (#65652).\n+\n+    When a successful probe returns live models, ``_save_discovered_models_to_config``\n+    must be called with the provider's base_url and the discovered model list.\n+    \"\"\"\n+    monkeypatch.setattr(\"agent.models_dev.fetch_models_dev\", lambda: {})\n+    monkeypatch.setattr(\"hermes_cli.providers.HERMES_OVERLAYS\", {})\n+\n+    save_calls = []\n+\n+    def fake_fetch_api_models(api_key, base_url, **kwargs):\n+        return [\"discovered-a\", \"discovered-b\", \"discovered-c\"]\n+\n+    monkeypatch.setattr(\"hermes_cli.models.fetch_api_models\", fake_fetch_api_models)\n+    monkeypatch.setattr(\n+        \"hermes_cli.model_switch._save_discovered_models_to_config\",\n+        lambda api_url, model_ids: save_calls.append((api_url, model_ids)),\n+    )\n+\n+    custom_providers = [\n+        {\n+            \"name\": \"my-gateway\",\n+            \"api_key\": \"***\",\n+            \"base_url\": \"https://gateway.example.com/v1\",\n+            \"discover_models\": True,\n+            \"model\": \"only-model\",\n+            \"models\": {\"only-model\": {\"context_length\": 128000}},\n+        }\n+    ]\n+\n+    providers = list_authenticated_providers(\n+        current_provider=\"my-gateway\",\n+        current_base_url=\"https://gateway.example.com/v1\",\n+        custom_providers=custom_providers,\n+        max_models=50,\n+        probe_custom_providers=True,\n+    )\n+\n+    assert len(save_calls) == 1, (\n+        \"_save_discovered_models_to_config must be called after a successful probe\"\n+    )\n+    assert save_calls[0][0] == \"https://gateway.example.com/v1\"\n+    assert save_calls[0][1] == [\"discovered-a\", \"discovered-b\", \"discovered-c\"]\n+\n+    gateway_prov = next(\n+        (p for p in providers if p.get(\"api_url\") == \"https://gateway.example.com/v1\"),\n+        None,\n+    )\n+    assert gateway_prov is not None\n+    assert gateway_prov[\"models\"] == [\"discovered-a\", \"discovered-b\", \"discovered-c\"]\n+\n+\n+def test_discovered_models_not_saved_on_empty_probe(monkeypatch):\n+    \"\"\"When a probe returns an empty list, no auto-save must happen (#65652).\"\"\"\n+    monkeypatch.setattr(\"agent.models_dev.fetch_models_dev\", lambda: {})\n+    monkeypatch.setattr(\"hermes_cli.providers.HERMES_OVERLAYS\", {})\n+\n+    save_calls = []\n+\n+    def fake_fetch_api_models(api_key, base_url, **kwargs):\n+        return []\n+\n+    monkeypatch.setattr(\"hermes_cli.models.fetch_api_models\", fake_fetch_api_models)\n+    monkeypatch.setattr(\n+        \"hermes_cli.model_switch._save_discovered_models_to_config\",\n+        lambda api_url, model_ids: save_calls.append((api_url, model_ids)),\n+    )\n+\n+    custom_providers = [\n+        {\n+            \"name\": \"my-gateway\",\n+            \"api_key\": \"***\",\n+            \"base_url\": \"https://gateway.example.com/v1\",\n+            \"discover_models\": True,\n+            \"model\": \"only-model\",\n+        }\n+    ]\n+\n+    list_authenticated_providers(\n+        current_provider=\"my-gateway\",\n+        current_base_url=\"https://gateway.example.com/v1\",\n+        custom_providers=custom_providers,\n+        max_models=50,\n+        probe_custom_providers=True,\n+    )\n+\n+    assert save_calls == [], \"Empty probe must not trigger a save\"\n+\n+\n+def test_save_discovered_models_skips_unchanged(monkeypatch):\n+    \"\"\"``_save_discovered_models_to_config`` must not write config when the\n+    model list hasn't changed (#65652).\"\"\"\n+    from hermes_cli.model_switch import _save_discovered_models_to_config\n+\n+    save_calls = []\n+\n+    def fake_save(config):\n+        save_calls.append(dict(config))\n+\n+    monkeypatch.setattr(\"hermes_cli.config.save_config\", fake_save)\n+    monkeypatch.setattr(\n+        \"hermes_cli.config.load_config\",\n+        lambda: {\n+            \"custom_providers\": [\n+                {\n+                    \"name\": \"my-gateway\",\n+                    \"base_url\": \"https://gateway.example.com/v1\",\n+                    \"models\": [\"model-a\", \"model-b\"],\n+                }\n+            ]\n+        },\n+    )\n+\n+    # Same list — no write\n+    _save_discovered_models_to_config(\n+        \"https://gateway.example.com/v1\",\n+        [\"model-a\", \"model-b\"],\n+    )\n+    assert save_calls == [], \"Unchanged models must not trigger config write\"\n+\n+    # Changed list — write\n+    _save_discovered_models_to_config(\n+        \"https://gateway.example.com/v1\",\n+        [\"model-a\", \"model-b\", \"model-c\"],\n+    )\n+    assert len(save_calls) == 1, \"Changed models must trigger config write\"\n+    updated = save_calls[0][\"custom_providers\"][0]\n+    assert updated[\"models\"] == [\"model-a\", \"model-b\", \"model-c\"]\n+\n+\n+def test_save_discovered_models_noop_on_empty_args(monkeypatch):\n+    \"\"\"``_save_discovered_models_to_config`` is a no-op when api_url or\n+    model_ids are blank (#65652).\"\"\"\n+    from hermes_cli.model_switch import _save_discovered_models_to_config\n+\n+    load_calls = 0\n+\n+    def fake_load():\n+        nonlocal load_calls\n+        load_calls += 1\n+        return {\"custom_providers\": []}\n+\n+    monkeypatch.setattr(\"hermes_cli.config.load_config\", fake_load)\n+\n+    _save_discovered_models_to_config(\"\", [\"a\"])\n+    _save_discovered_models_to_config(\"https://x.com\", [])\n+    _save_discovered_models_to_config(\"\", [])\n+\n+    assert load_calls == 0, \"load_config must not be called for empty args\"\n"]}]},{"kind":"pr","number":65766,"title":"perf(cli): parallelize custom provider model discovery in /model picker","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}