{"repo":"NousResearch/hermes-agent","public_id":"454","display_label":"Provider/model fallback chain and failover features","summary":"The majority of members propose or implement automatic fallback to secondary providers or models when a primary fails, hits quota, or is invalid, including ordered chains, multi-hop hops, and failure classification. One outlier (Feishu/Lark platform support) does not share this theme.","subsystem":"agent","counts":{"members":12,"open_issues":2,"open_issues_distinct":2,"open_prs":0,"closed_issues":7,"merged_prs":2,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":3799},"resolution":{"state":"partially_resolved","by":"pr-3799"},"freshness":{"state":"stale","last_verified_at":"2026-03-29T23:04:53+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.0890909090909091},"fix_edges":[{"issue_number":1734,"pr_number":3799,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":1734,"pr_number":3813,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":454,"title":"[Feature]: Fallback provider","url":null,"state":"closed"},{"kind":"issue","number":737,"title":"Feature Request: Native Model/Provider Fallback With Failure Classification","url":null,"state":"closed"},{"kind":"issue","number":1734,"title":"[Feature] Support multiple configured providers with ordered fallback sequence (like pi-mono)","url":null,"state":"closed"},{"kind":"issue","number":8947,"title":"feat: model failover — automatic fallback when primary provider fails","url":null,"state":"closed"},{"kind":"issue","number":11737,"title":"[Feature]: Multi-provider credential pools for cross-provider failover and rotation","url":null,"state":"open"},{"kind":"issue","number":41590,"title":"Add smart fallback model routing when primary provider hits usage limit","url":null,"state":"open"},{"kind":"issue","number":51430,"title":"[Feature Request] Auto-failover on invalid model selection","url":null,"state":"closed"},{"kind":"issue","number":59002,"title":"[Feature]: Bounded main-agent fallback when the active model hits quota or session limits","url":null,"state":"closed"},{"kind":"issue","number":68630,"title":"[Feature]: Multi-hop fallback — walk fallback_providers chain within a turn (fallback_max_hops)","url":null,"state":"closed"},{"kind":"pr","number":1761,"title":"feat(providers): add ordered fallback sequence for primary model","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#3813","number":3813}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":3799,"title":"feat(gateway): add Feishu/Lark platform support","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":[".env.example","optional-skills/productivity/canvas/SKILL.md","optional-skills/productivity/canvas/scripts/canvas_api.py"]},{"kind":"pr","number":3813,"title":"feat(providers): add ordered fallback provider chain (salvage #1761)","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["cli.py","gateway/run.py","hermes_cli/config.py","run_agent.py","tests/test_compressor_fallback_update.py","tests/test_provider_fallback.py","tests/test_run_agent.py"],"fix_hunks":[{"file":"cli.py","hunks":["@@ -1182,9 +1182,13 @@ def __init__(\n         self._provider_require_params = pr.get(\"require_parameters\", False)\n         self._provider_data_collection = pr.get(\"data_collection\")\n         \n-        # Fallback model config — tried when primary provider fails after retries\n-        fb = CLI_CONFIG.get(\"fallback_model\") or {}\n-        self._fallback_model = fb if fb.get(\"provider\") and fb.get(\"model\") else None\n+        # Fallback provider chain — tried in order when primary fails after retries.\n+        # Supports new list format (fallback_providers) and legacy single-dict (fallback_model).\n+        fb = CLI_CONFIG.get(\"fallback_providers\") or CLI_CONFIG.get(\"fallback_model\") or []\n+        # Normalize legacy single-dict to a one-element list\n+        if isinstance(fb, dict):\n+            fb = [fb] if fb.get(\"provider\") and fb.get(\"model\") else []\n+        self._fallback_model = fb\n \n         # Optional cheap-vs-strong routing for simple turns\n         self._smart_model_routing = CLI_CONFIG.get(\"smart_model_routing\", {}) or {}"]},{"file":"gateway/run.py","hunks":["@@ -919,11 +919,12 @@ def _load_provider_routing() -> dict:\n         return {}\n \n     @staticmethod\n-    def _load_fallback_model() -> dict | None:\n-        \"\"\"Load fallback model config from config.yaml.\n+    def _load_fallback_model() -> list | dict | None:\n+        \"\"\"Load fallback provider chain from config.yaml.\n \n-        Returns a dict with 'provider' and 'model' keys, or None if\n-        not configured / both fields empty.\n+        Returns a list of provider dicts (``fallback_providers``), a single\n+        dict (legacy ``fallback_model``), or None if not configured.\n+        AIAgent.__init__ normalizes both formats into a chain.\n         \"\"\"\n         try:\n             import yaml as _y","@@ -931,8 +932,8 @@ def _load_fallback_model() -> dict | None:\n             if cfg_path.exists():\n                 with open(cfg_path, encoding=\"utf-8\") as _f:\n                     cfg = _y.safe_load(_f) or {}\n-                fb = cfg.get(\"fallback_model\", {}) or {}\n-                if fb.get(\"provider\") and fb.get(\"model\"):\n+                fb = cfg.get(\"fallback_providers\") or cfg.get(\"fallback_model\") or None\n+                if fb:\n                     return fb\n         except Exception:\n             pass"]},{"file":"hermes_cli/config.py","hunks":["@@ -135,6 +135,7 @@ def ensure_hermes_home():\n \n DEFAULT_CONFIG = {\n     \"model\": \"anthropic/claude-opus-4.6\",\n+    \"fallback_providers\": [],\n     \"toolsets\": [\"hermes-cli\"],\n     \"agent\": {\n         \"max_turns\": 90,"]},{"file":"run_agent.py","hunks":["@@ -896,16 +896,30 @@ def __init__(\n             except Exception as e:\n                 raise RuntimeError(f\"Failed to initialize OpenAI client: {e}\")\n         \n-        # Provider fallback — a single backup model/provider tried when the\n-        # primary is exhausted (rate-limit, overload, connection failure).\n-        # Config shape: {\"provider\": \"openrouter\", \"model\": \"anthropic/claude-sonnet-4\"}\n-        self._fallback_model = fallback_model if isinstance(fallback_model, dict) else None\n+        # Provider fallback chain — ordered list of backup providers tried\n+        # when the primary is exhausted (rate-limit, overload, connection\n+        # failure).  Supports both legacy single-dict ``fallback_model`` and\n+        # new list ``fallback_providers`` format.\n+        if isinstance(fallback_model, list):\n+            self._fallback_chain = [\n+                f for f in fallback_model\n+                if isinstance(f, dict) and f.get(\"provider\") and f.get(\"model\")\n+            ]\n+        elif isinstance(fallback_model, dict) and fallback_model.get(\"provider\") and fallback_model.get(\"model\"):\n+            self._fallback_chain = [fallback_model]\n+        else:\n+            self._fallback_chain = []\n+        self._fallback_index = 0\n         self._fallback_activated = False\n-        if self._fallback_model:\n-            fb_p = self._fallback_model.get(\"provider\", \"\")\n-            fb_m = self._fallback_model.get(\"model\", \"\")\n-            if fb_p and fb_m and not self.quiet_mode:\n-                print(f\"🔄 Fallback model: {fb_m} ({fb_p})\")\n+        # Legacy attribute kept for backward compat (tests, external callers)\n+        self._fallback_model = self._fallback_chain[0] if self._fallback_chain else None\n+        if self._fallback_chain and not self.quiet_mode:\n+            if len(self._fallback_chain) == 1:\n+                fb = self._fallback_chain[0]\n+                print(f\"🔄 Fallback model: {fb['model']} ({fb['provider']})\")\n+            else:\n+                print(f\"🔄 Fallback chain ({len(self._fallback_chain)} providers): \" +\n+                      \" → \".join(f\"{f['model']} ({f['provider']})\" for f in self._fallback_chain))\n \n         # Get available tools with filtering\n         self.tools = get_tool_definitions(","@@ -4318,25 +4332,26 @@ def _call():\n     # ── Provider fallback ──────────────────────────────────────────────────\n \n     def _try_activate_fallback(self) -> bool:\n-        \"\"\"Switch to the configured fallback model/provider.\n+        \"\"\"Switch to the next fallback model/provider in the chain.\n \n-        Called when the primary model is failing after retries.  Swaps the\n+        Called when the current model is failing after retries.  Swaps the\n         OpenAI client, model slug, and provider in-place so the retry loop\n-        can continue with the new backend.  One-shot: returns False if\n-        already activated or not configured.\n+        can continue with the new backend.  Advances through the chain on\n+        each call; returns False when exhausted.\n \n         Uses the centralized provider router (resolve_provider_client) for\n         auth resolution and client construction — no duplicated provider→key\n         mappings.\n         \"\"\"\n-        if self._fallback_activated or not self._fallback_model:\n+        if self._fallback_index >= len(self._fallback_chain):\n             return False\n \n-        fb = self._fallback_model\n+        fb = self._fallback_chain[self._fallback_index]\n+        self._fallback_index += 1\n         fb_provider = (fb.get(\"provider\") or \"\").strip().lower()\n         fb_model = (fb.get(\"model\") or \"\").strip()\n         if not fb_provider or not fb_model:\n-            return False\n+            return self._try_activate_fallback()  # skip invalid, try next\n \n         # Use centralized router for client construction.\n         # raw_codex=True because the main agent needs direct responses.stream()","@@ -4349,7 +4364,7 @@ def _try_activate_fallback(self) -> bool:\n                 logging.warning(\n                     \"Fallback to %s failed: provider not configured\",\n                     fb_provider)\n-                return False\n+                return self._try_activate_fallback()  # try next in chain\n \n             # Determine api_mode from provider / base URL\n             fb_api_mode = \"chat_completions\"","@@ -4424,8 +4439,8 @@ def _try_activate_fallback(self) -> bool:\n             )\n             return True\n         except Exception as e:\n-            logging.error(\"Failed to activate fallback model: %s\", e)\n-            return False\n+            logging.error(\"Failed to activate fallback %s: %s\", fb_model, e)\n+            return self._try_activate_fallback()  # try next in chain\n \n     # ── End provider fallback ──────────────────────────────────────────────\n ","@@ -6528,9 +6543,9 @@ def _stop_spinner():\n                         # Eager fallback: empty/malformed responses are a common\n                         # rate-limit symptom.  Switch to fallback immediately\n                         # rather than retrying with extended backoff.\n-                        if not self._fallback_activated:\n+                        if self._fallback_index < len(self._fallback_chain):\n                             self._emit_status(\"⚠️ Empty/malformed response — switching to fallback...\")\n-                        if not self._fallback_activated and self._try_activate_fallback():\n+                        if self._try_activate_fallback():\n                             retry_count = 0\n                             continue\n ","@@ -6993,7 +7008,7 @@ def _stop_spinner():\n                         or \"usage limit\" in error_msg\n                         or \"quota\" in error_msg\n                     )\n-                    if is_rate_limited and not self._fallback_activated:\n+                    if is_rate_limited and self._fallback_index < len(self._fallback_chain):\n                         self._emit_status(\"⚠️ Rate limited — switching to fallback provider...\")\n                         if self._try_activate_fallback():\n                             retry_count = 0"]},{"file":"tests/test_compressor_fallback_update.py","hunks":["@@ -25,6 +25,8 @@ def _make_agent_with_compressor() -> AIAgent:\n         \"provider\": \"openai\",\n         \"model\": \"gpt-4o\",\n     }\n+    agent._fallback_chain = [agent._fallback_model]\n+    agent._fallback_index = 0\n \n     # Context compressor with primary model values\n     compressor = ContextCompressor("]},{"file":"tests/test_provider_fallback.py","hunks":["@@ -0,0 +1,156 @@\n+\"\"\"Tests for ordered provider fallback chain (salvage of PR #1761).\n+\n+Extends the single-fallback tests in test_fallback_model.py to cover\n+the new list-based ``fallback_providers`` config format and chain\n+advancement through multiple providers.\n+\"\"\"\n+\n+from unittest.mock import MagicMock, patch\n+\n+from run_agent import AIAgent\n+\n+\n+def _make_agent(fallback_model=None):\n+    \"\"\"Create a minimal AIAgent with optional fallback config.\"\"\"\n+    with (\n+        patch(\"run_agent.get_tool_definitions\", return_value=[]),\n+        patch(\"run_agent.check_toolset_requirements\", return_value={}),\n+        patch(\"run_agent.OpenAI\"),\n+    ):\n+        agent = AIAgent(\n+            api_key=\"[redacted:secret-assignment]\",\n+            quiet_mode=True,\n+            skip_context_files=True,\n+            skip_memory=True,\n+            fallback_model=fallback_model,\n+        )\n+        agent.client = MagicMock()\n+        return agent\n+\n+\n+def _mock_client(base_url=\"https://openrouter.ai/api/v1\", api_key=\"[redacted:secret-assignment]\"):\n+    mock = MagicMock()\n+    mock.base_url = base_url\n+    mock.api_key = [redacted:secret-assignment]\n+    return mock\n+\n+\n+# ── Chain initialisation ──────────────────────────────────────────────────\n+\n+\n+class TestFallbackChainInit:\n+    def test_no_fallback(self):\n+        agent = _make_agent(fallback_model=None)\n+        assert agent._fallback_chain == []\n+        assert agent._fallback_index == 0\n+        assert agent._fallback_model is None\n+\n+    def test_single_dict_backwards_compat(self):\n+        fb = {\"provider\": \"openai\", \"model\": \"gpt-4o\"}\n+        agent = _make_agent(fallback_model=fb)\n+        assert agent._fallback_chain == [fb]\n+        assert agent._fallback_model == fb\n+\n+    def test_list_of_providers(self):\n+        fbs = [\n+            {\"provider\": \"openai\", \"model\": \"gpt-4o\"},\n+            {\"provider\": \"zai\", \"model\": \"glm-4.7\"},\n+        ]\n+        agent = _make_agent(fallback_model=fbs)\n+        assert len(agent._fallback_chain) == 2\n+        assert agent._fallback_model == fbs[0]\n+\n+    def test_invalid_entries_filtered(self):\n+        fbs = [\n+            {\"provider\": \"openai\", \"model\": \"gpt-4o\"},\n+            {\"provider\": \"\", \"model\": \"glm-4.7\"},\n+            {\"provider\": \"zai\"},\n+            \"not-a-dict\",\n+        ]\n+        agent = _make_agent(fallback_model=fbs)\n+        assert len(agent._fallback_chain) == 1\n+        assert agent._fallback_chain[0][\"provider\"] == \"openai\"\n+\n+    def test_empty_list(self):\n+        agent = _make_agent(fallback_model=[])\n+        assert agent._fallback_chain == []\n+        assert agent._fallback_model is None\n+\n+    def test_invalid_dict_no_provider(self):\n+        agent = _make_agent(fallback_model={\"model\": \"gpt-4o\"})\n+        assert agent._fallback_chain == []\n+\n+\n+# ── Chain advancement ─────────────────────────────────────────────────────\n+\n+\n+class TestFallbackChainAdvancement:\n+    def test_exhausted_returns_false(self):\n+        agent = _make_agent(fallback_model=None)\n+        assert agent._try_activate_fallback() is False\n+\n+    def test_advances_index(self):\n+        fbs = [\n+            {\"provider\": \"openai\", \"model\": \"gpt-4o\"},\n+            {\"provider\": \"zai\", \"model\": \"glm-4.7\"},\n+        ]\n+        agent = _make_agent(fallback_model=fbs)\n+        with patch(\"agent.auxiliary_client.resolve_provider_client\",\n+                    return_value=(_mock_client(), \"gpt-4o\")):\n+            assert agent._try_activate_fallback() is True\n+            assert agent._fallback_index == 1\n+            assert agent.model == \"gpt-4o\"\n+            assert agent._fallback_activated is True\n+\n+    def test_second_fallback_works(self):\n+        fbs = [\n+            {\"provider\": \"openai\", \"model\": \"gpt-4o\"},\n+            {\"provider\": \"zai\", \"model\": \"glm-4.7\"},\n+        ]\n+        agent = _make_agent(fallback_model=fbs)\n+        with patch(\"agent.auxiliary_client.resolve_provider_client\",\n+                    return_value=(_mock_client(), \"resolved\")):\n+            assert agent._try_activate_fallback() is True\n+            assert agent.model == \"gpt-4o\"\n+            assert agent._try_activate_fallback() is True\n+            assert agent.model == \"glm-4.7\"\n+            assert agent._fallback_index == 2\n+\n+    def test_all_exhausted_returns_false(self):\n+        fbs = [{\"provider\": \"openai\", \"model\": \"gpt-4o\"}]\n+        agent = _make_agent(fallback_model=fbs)\n+        with patch(\"agent.auxiliary_client.resolve_provider_client\",\n+                    return_value=(_mock_client(), \"gpt-4o\")):\n+            assert agent._try_activate_fallback() is True\n+            assert agent._try_activate_fallback() is False\n+\n+    def test_skips_unconfigured_provider_to_next(self):\n+        \"\"\"If resolve_provider_client returns None, skip to next in chain.\"\"\"\n+        fbs = [\n+            {\"provider\": \"broken\", \"model\": \"nope\"},\n+            {\"provider\": \"openai\", \"model\": \"gpt-4o\"},\n+        ]\n+        agent = _make_agent(fallback_model=fbs)\n+        with patch(\"agent.auxiliary_client.resolve_provider_client\") as mock_rpc:\n+            mock_rpc.side_effect = [\n+                (None, None),                    # broken provider\n+                (_mock_client(), \"gpt-4o\"),       # fallback succeeds\n+            ]\n+            assert agent._try_activate_fallback() is True\n+            assert agent.model == \"gpt-4o\"\n+            assert agent._fallback_index == 2\n+\n+    def test_skips_provider_that_raises_to_next(self):\n+        \"\"\"If resolve_provider_client raises, skip to next in chain.\"\"\"\n+        fbs = [\n+            {\"provider\": \"broken\", \"model\": \"nope\"},\n+            {\"provider\": \"openai\", \"model\": \"gpt-4o\"},\n+        ]\n+        agent = _make_agent(fallback_model=fbs)\n+        with patch(\"agent.auxiliary_client.resolve_provider_client\") as mock_rpc:\n+            mock_rpc.side_effect = [\n+                RuntimeError(\"auth failed\"),\n+                (_mock_client(), \"gpt-4o\"),\n+            ]\n+            assert agent._try_activate_fallback() is True\n+            assert agent.model == \"gpt-4o\""]},{"file":"tests/test_run_agent.py","hunks":["@@ -2507,6 +2507,8 @@ class TestFallbackAnthropicProvider:\n     def test_fallback_to_anthropic_sets_api_mode(self, agent):\n         agent._fallback_activated = False\n         agent._fallback_model = {\"provider\": \"anthropic\", \"model\": \"claude-sonnet-4-20250514\"}\n+        agent._fallback_chain = [agent._fallback_model]\n+        agent._fallback_index = 0\n \n         mock_client = MagicMock()\n         mock_client.base_url = \"https://api.anthropic.com/v1\"","@@ -2528,6 +2530,8 @@ def test_fallback_to_anthropic_sets_api_mode(self, agent):\n     def test_fallback_to_anthropic_enables_prompt_caching(self, agent):\n         agent._fallback_activated = False\n         agent._fallback_model = {\"provider\": \"anthropic\", \"model\": \"claude-sonnet-4-20250514\"}\n+        agent._fallback_chain = [agent._fallback_model]\n+        agent._fallback_index = 0\n \n         mock_client = MagicMock()\n         mock_client.base_url = \"https://api.anthropic.com/v1\"","@@ -2545,6 +2549,8 @@ def test_fallback_to_anthropic_enables_prompt_caching(self, agent):\n     def test_fallback_to_openrouter_uses_openai_client(self, agent):\n         agent._fallback_activated = False\n         agent._fallback_model = {\"provider\": \"openrouter\", \"model\": \"anthropic/claude-sonnet-4\"}\n+        agent._fallback_chain = [agent._fallback_model]\n+        agent._fallback_index = 0\n \n         mock_client = MagicMock()\n         mock_client.base_url = \"https://openrouter.ai/api/v1\"","@@ -3238,6 +3244,8 @@ class TestFallbackSetsOAuthFlag:\n     def test_fallback_to_anthropic_oauth_sets_flag(self, agent):\n         agent._fallback_activated = False\n         agent._fallback_model = {\"provider\": \"anthropic\", \"model\": \"claude-sonnet-4-6\"}\n+        agent._fallback_chain = [agent._fallback_model]\n+        agent._fallback_index = 0\n \n         mock_client = MagicMock()\n         mock_client.base_url = \"https://api.anthropic.com/v1\"","@@ -3259,6 +3267,8 @@ def test_fallback_to_anthropic_oauth_sets_flag(self, agent):\n     def test_fallback_to_anthropic_api_key_clears_flag(self, agent):\n         agent._fallback_activated = False\n         agent._fallback_model = {\"provider\": \"anthropic\", \"model\": \"claude-sonnet-4-6\"}\n+        agent._fallback_chain = [agent._fallback_model]\n+        agent._fallback_index = 0\n \n         mock_client = MagicMock()\n         mock_client.base_url = \"https://api.anthropic.com/v1\"\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}