{"repo":"NousResearch/hermes-agent","public_id":"37159","display_label":"MCP servers falsely shown as failing at startup","summary":"A race condition causes the TUI/CLI startup banner to display MCP servers as `\"failed\"` while async discovery is still in progress. Multiple fixes address stale status display, lazy-initialized tools shown in red instead of yellow, and refreshing tool state after background discovery completes.","subsystem":"gateway","counts":{"members":10,"open_issues":1,"open_issues_distinct":1,"open_prs":1,"closed_issues":3,"merged_prs":1,"attempt_prs":5},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":48403,"merged_at":"2026-06-18T12:41:23+00:00"},"resolution":{"state":"unresolved","by":null},"freshness":{"state":"verified","last_verified_at":"2026-06-18T12:41:23+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.16533333333333333},"fix_edges":[{"issue_number":37159,"pr_number":48403,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false,"pr_merged_at":"2026-06-18T12:41:23+00:00"}],"members":[{"kind":"issue","number":37159,"title":"TUI branding shows all MCP servers as \"failed\" due to race condition with background discovery","url":null,"state":"closed"},{"kind":"issue","number":37311,"title":"[Bug] TUI sidebar shows MCP servers as 'failed' despite successful connection","url":null,"state":"closed"},{"kind":"issue","number":38650,"title":"Bug: `hermes dump` / welcome screen reports MCP servers as \"failed\" despite successful async discovery","url":null,"state":"open"},{"kind":"issue","number":39901,"title":"[Bug]: TUI/CLI startup banner shows MCP servers as \"failed\" while async connection is still in progress","url":null,"state":"closed"},{"kind":"pr","number":37249,"title":"fix(tui): re-emit session.info after MCP discovery to fix stale branding status","url":null,"state":"closed","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":37302,"title":"fix(tui): re-emit session.info after MCP discovery to fix stale branding","url":null,"state":"closed","close_reason":{"category":"silent_close","implication":"active"},"merged":false},{"kind":"pr","number":39181,"title":"fix(mcp): show live server count + don't label pending servers as failed","url":null,"state":"open","close_reason":{"category":"uncategorized","implication":"needs_llm"},"merged":false},{"kind":"pr","number":39678,"title":"fix(tui): re-emit session.info after MCP discovery to fix stale branding","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"2f7c4858a","number":0},{"kind":"pr","value":"#48403","number":48403}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":39920,"title":"fix(mcp): show startup servers as connecting","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"e71d746820bf262214e4e1887683d3f65d211cc1","number":0},{"kind":"pr","value":"#39901","number":39901}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":48403,"title":"fix(tui): refresh tool snapshot when MCP discovery lands after agent build","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["tests/test_tui_mcp_late_refresh.py","tui_gateway/entry.py","tui_gateway/server.py"],"fix_hunks":[{"file":"tests/test_tui_mcp_late_refresh.py","hunks":["@@ -0,0 +1,167 @@\n+\"\"\"Tests for the TUI gateway's late MCP tool-snapshot refresh.\n+\n+When an MCP server connects slower than the bounded wait in ``_make_agent``,\n+the agent is built without its tools and the banner/tool count is stale for the\n+session. ``_schedule_mcp_late_refresh`` waits for discovery to land, then\n+rebuilds the snapshot and re-emits ``session.info`` — but only while the\n+session is still pre-first-turn, so it never invalidates a cached prompt.\n+\"\"\"\n+\n+import threading\n+import time\n+import types\n+\n+import model_tools\n+from tui_gateway import server\n+from tui_gateway import entry\n+\n+\n+def _make_fake_agent(initial_tools, *, user_turns=0, api_calls=0):\n+    agent = types.SimpleNamespace()\n+    agent.tools = list(initial_tools)\n+    agent.valid_tool_names = {t[\"function\"][\"name\"] for t in initial_tools}\n+    agent._user_turn_count = user_turns\n+    agent._api_call_count = api_calls\n+    return agent\n+\n+\n+def _tool(name):\n+    return {\"type\": \"function\", \"function\": {\"name\": name, \"description\": \"\", \"parameters\": {}}}\n+\n+\n+def _drain_refresh_threads(timeout=5.0):\n+    deadline = time.time() + timeout\n+    for th in list(threading.enumerate()):\n+        if th.name.startswith(\"tui-mcp-late-refresh-\"):\n+            th.join(timeout=max(0.0, deadline - time.time()))\n+\n+\n+def _install(monkeypatch, *, in_flight, join_result, new_defs):\n+    \"\"\"Wire entry discovery accessors + get_tool_definitions, capture emits.\"\"\"\n+    monkeypatch.setattr(entry, \"mcp_discovery_in_flight\", lambda: in_flight)\n+    monkeypatch.setattr(entry, \"join_mcp_discovery\", lambda timeout=None: join_result)\n+    monkeypatch.setattr(model_tools, \"get_tool_definitions\", lambda **kw: list(new_defs))\n+    monkeypatch.setattr(server, \"_load_enabled_toolsets\", lambda: None)\n+    monkeypatch.setattr(server, \"_session_info\", lambda agent, session: {\"tools_len\": len(agent.tools)})\n+\n+    emitted = []\n+    monkeypatch.setattr(server, \"_emit\", lambda event, sid, payload=None: emitted.append((event, sid, payload)))\n+    return emitted\n+\n+\n+def test_late_refresh_adds_tools_and_reemits_when_pre_first_turn(monkeypatch):\n+    base = [_tool(\"read_file\"), _tool(\"write_file\")]\n+    full = base + [_tool(\"mcp__nous_support__a\")]  # discovery added one tool\n+    agent = _make_fake_agent(base)\n+    sid = \"sess-late-1\"\n+    server._sessions[sid] = {\"agent\": agent}\n+    try:\n+        emitted = _install(monkeypatch, in_flight=True, join_result=True, new_defs=full)\n+        server._schedule_mcp_late_refresh(sid, agent)\n+        _drain_refresh_threads()\n+\n+        assert len(agent.tools) == 3\n+        assert \"mcp__nous_support__a\" in agent.valid_tool_names\n+        assert (\"session.info\", sid, {\"tools_len\": 3}) in emitted\n+    finally:\n+        server._sessions.pop(sid, None)\n+\n+\n+def test_no_refresh_when_discovery_not_in_flight(monkeypatch):\n+    base = [_tool(\"read_file\")]\n+    agent = _make_fake_agent(base)\n+    sid = \"sess-late-2\"\n+    server._sessions[sid] = {\"agent\": agent}\n+    try:\n+        # in_flight=False → helper returns immediately, no thread, no rebuild.\n+        emitted = _install(monkeypatch, in_flight=False, join_result=True, new_defs=base + [_tool(\"x\")])\n+        server._schedule_mcp_late_refresh(sid, agent)\n+        _drain_refresh_threads()\n+\n+        assert len(agent.tools) == 1\n+        assert emitted == []\n+    finally:\n+        server._sessions.pop(sid, None)\n+\n+\n+def test_no_refresh_once_conversation_started(monkeypatch):\n+    \"\"\"Cache safety: never rebuild the tool list after the first turn.\"\"\"\n+    base = [_tool(\"read_file\")]\n+    full = base + [_tool(\"mcp__late__b\")]\n+    agent = _make_fake_agent(base, user_turns=1)  # a turn already happened\n+    sid = \"sess-late-3\"\n+    server._sessions[sid] = {\"agent\": agent}\n+    try:\n+        emitted = _install(monkeypatch, in_flight=True, join_result=True, new_defs=full)\n+        server._schedule_mcp_late_refresh(sid, agent)\n+        _drain_refresh_threads()\n+\n+        # Snapshot frozen; no re-emit that would invalidate the prompt cache.\n+        assert len(agent.tools) == 1\n+        assert emitted == []\n+    finally:\n+        server._sessions.pop(sid, None)\n+\n+\n+def test_no_reemit_when_discovery_added_nothing(monkeypatch):\n+    base = [_tool(\"read_file\"), _tool(\"write_file\")]\n+    agent = _make_fake_agent(base)\n+    sid = \"sess-late-4\"\n+    server._sessions[sid] = {\"agent\": agent}\n+    try:\n+        # Discovery finished but the registry is unchanged (same count) →\n+        # don't churn the client with a redundant session.info.\n+        emitted = _install(monkeypatch, in_flight=True, join_result=True, new_defs=list(base))\n+        server._schedule_mcp_late_refresh(sid, agent)\n+        _drain_refresh_threads()\n+\n+        assert len(agent.tools) == 2\n+        assert emitted == []\n+    finally:\n+        server._sessions.pop(sid, None)\n+\n+\n+def test_no_refresh_when_join_times_out(monkeypatch):\n+    base = [_tool(\"read_file\")]\n+    full = base + [_tool(\"mcp__slow__c\")]\n+    agent = _make_fake_agent(base)\n+    sid = \"sess-late-5\"\n+    server._sessions[sid] = {\"agent\": agent}\n+    try:\n+        # Server never connected within the bound → join returns False, no rebuild.\n+        emitted = _install(monkeypatch, in_flight=True, join_result=False, new_defs=full)\n+        server._schedule_mcp_late_refresh(sid, agent)\n+        _drain_refresh_threads()\n+\n+        assert len(agent.tools) == 1\n+        assert emitted == []\n+    finally:\n+        server._sessions.pop(sid, None)\n+\n+\n+def test_no_refresh_when_session_replaced(monkeypatch):\n+    \"\"\"If the session's agent was swapped (e.g. /new) while we waited, bail.\"\"\"\n+    base = [_tool(\"read_file\")]\n+    full = base + [_tool(\"mcp__late__d\")]\n+    agent = _make_fake_agent(base)\n+    other_agent = _make_fake_agent(base)\n+    sid = \"sess-late-6\"\n+    server._sessions[sid] = {\"agent\": agent}\n+    try:\n+        emitted = _install(monkeypatch, in_flight=True, join_result=True, new_defs=full)\n+\n+        # Swap the stored agent out the moment join is awaited.\n+        def _swap_join(timeout=None):\n+            server._sessions[sid][\"agent\"] = other_agent\n+            return True\n+\n+        monkeypatch.setattr(entry, \"join_mcp_discovery\", _swap_join)\n+        server._schedule_mcp_late_refresh(sid, agent)\n+        _drain_refresh_threads()\n+\n+        # Neither agent's snapshot was rebuilt; no emit.\n+        assert len(agent.tools) == 1\n+        assert len(other_agent.tools) == 1\n+        assert emitted == []\n+    finally:\n+        server._sessions.pop(sid, None)"]},{"file":"tui_gateway/entry.py","hunks":["@@ -210,6 +210,33 @@ def wait_for_mcp_discovery(timeout: float = 0.75) -> None:\n     thread.join(timeout=timeout)\n \n \n+def mcp_discovery_in_flight() -> bool:\n+    \"\"\"Return True if the background MCP discovery thread is still running.\n+\n+    Used by the agent-build path to decide whether to schedule a late tool\n+    snapshot refresh: if discovery didn't land within the bounded\n+    ``wait_for_mcp_discovery`` join, the agent was built without those tools\n+    and the banner/tool count will be stale until they arrive.\n+    \"\"\"\n+    thread = _mcp_discovery_thread\n+    return thread is not None and thread.is_alive()\n+\n+\n+def join_mcp_discovery(timeout: float | None = None) -> bool:\n+    \"\"\"Block until background MCP discovery finishes, up to ``timeout`` seconds.\n+\n+    Returns True if discovery has completed (thread absent or no longer alive),\n+    False if it is still running after the timeout. Unlike\n+    ``wait_for_mcp_discovery`` this accepts an unbounded/long wait and reports\n+    the outcome, for the off-critical-path late-refresh waiter.\n+    \"\"\"\n+    thread = _mcp_discovery_thread\n+    if thread is None:\n+        return True\n+    thread.join(timeout=timeout)\n+    return not thread.is_alive()\n+\n+\n def main():\n     _install_sidecar_publisher()\n "]},{"file":"tui_gateway/server.py","hunks":["@@ -1015,6 +1015,11 @@ def _build() -> None:\n                 info[\"config_warning\"] = cfg_warn\n                 logger.warning(cfg_warn)\n             _emit(\"session.info\", sid, info)\n+            # If MCP discovery is still in flight (a server slower than the\n+            # bounded wait_for_mcp_discovery join in _make_agent), the agent\n+            # was built without those tools. Catch up once they land — see\n+            # _schedule_mcp_late_refresh. Cache-safe (pre-first-turn only).\n+            _schedule_mcp_late_refresh(sid, agent)\n         except Exception as e:\n             current[\"agent_error\"] = str(e)\n             _emit(\"error\", sid, {\"message\": f\"agent init failed: {e}\"})","@@ -3405,6 +3410,87 @@ def _reset_session_agent(sid: str, session: dict) -> dict:\n     return info\n \n \n+def _schedule_mcp_late_refresh(sid: str, agent) -> None:\n+    \"\"\"Refresh a session's tool snapshot when MCP discovery lands late.\n+\n+    The agent snapshots ``agent.tools`` once at build time and never re-reads\n+    the registry (run_agent/agent_init). ``_make_agent`` briefly joins the\n+    background MCP discovery thread (``wait_for_mcp_discovery``, ~0.75s) so\n+    already-spawning servers land in that snapshot — but a server that takes\n+    longer than the bound to connect (common for an HTTP MCP server on first\n+    connect) lands *after* the agent is built. Its tools are then absent from\n+    both the agent and the banner for the whole session, even though the\n+    classic CLI shows them (the CLI re-derives ``get_tool_definitions`` at\n+    banner render time, which re-waits, so it picks them up).\n+\n+    This schedules an off-critical-path daemon that waits for discovery to\n+    finish, then rebuilds the snapshot and re-emits ``session.info`` so both\n+    the agent's callable tools and the banner count catch up — the same\n+    rebuild ``/reload-mcp`` performs, but automatic.\n+\n+    Cache safety: the rebuild only runs while the session is still pre-first-\n+    turn (no API call made yet → nothing cached to invalidate). If the user\n+    has already sent a message, we leave the snapshot frozen rather than\n+    invalidate the prompt cache mid-conversation — those late tools then\n+    require an explicit ``/reload-mcp`` (which gates on user consent), exactly\n+    as today. No-op when discovery already finished before the agent build.\n+    \"\"\"\n+    try:\n+        from tui_gateway.entry import mcp_discovery_in_flight, join_mcp_discovery\n+    except Exception:\n+        return\n+    if not mcp_discovery_in_flight():\n+        return\n+\n+    def _wait_then_refresh() -> None:\n+        # Bounded but generous — a server still not connected after this is\n+        # genuinely slow/dead; the user can /reload-mcp once it recovers.\n+        if not join_mcp_discovery(timeout=30.0):\n+            return\n+        with _sessions_lock:\n+            session = _sessions.get(sid)\n+            # Session may have been closed/reset while we waited.\n+            if session is None or session.get(\"agent\") is not agent:\n+                return\n+            # Cache safety: never rebuild the tool list once the conversation\n+            # has started — that would invalidate the cached prompt prefix.\n+            if (\n+                int(getattr(agent, \"_user_turn_count\", 0) or 0) > 0\n+                or int(getattr(agent, \"_api_call_count\", 0) or 0) > 0\n+            ):\n+                return\n+            try:\n+                from model_tools import get_tool_definitions\n+\n+                new_defs = get_tool_definitions(\n+                    enabled_toolsets=_load_enabled_toolsets(),\n+                    quiet_mode=True,\n+                )\n+            except Exception as exc:\n+                logger.warning(\n+                    \"Late MCP refresh: get_tool_definitions failed for %s: %s\",\n+                    sid,\n+                    exc,\n+                )\n+                return\n+            # No change (discovery added nothing new) → don't churn the client.\n+            if len(new_defs or []) == len(getattr(agent, \"tools\", []) or []):\n+                return\n+            agent.tools = new_defs\n+            agent.valid_tool_names = (\n+                {t[\"function\"][\"name\"] for t in new_defs} if new_defs else set()\n+            )\n+            info = _session_info(agent, session)\n+        # Emit outside the lock — write_json must not block under _sessions_lock.\n+        _emit(\"session.info\", sid, info)\n+\n+    threading.Thread(\n+        target=_wait_then_refresh,\n+        name=f\"tui-mcp-late-refresh-{sid}\",\n+        daemon=True,\n+    ).start()\n+\n+\n def _make_agent(\n     sid: str,\n     key: str,","@@ -3643,6 +3729,7 @@ def _init_session(\n             _sessions[sid][\"_notif_stop\"] = _start_notification_poller(sid, _sessions[sid])\n     _notify_session_boundary(\"on_session_reset\", key)\n     _emit(\"session.info\", sid, _session_info(agent, _sessions.get(sid, {})))\n+    _schedule_mcp_late_refresh(sid, agent)\n \n \n def _new_session_key() -> str:\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}