{"repo":"NousResearch/hermes-agent","public_id":"60955","display_label":"Fallback provider handling in agent and gateway sessions","summary":"The three members appear to relate to fallback behavior: one concerns resetting a fallback chain index after an early activation that did not succeed, one concerns reloading fallback_providers for live messaging sessions, and one is a fallback bug report. Only titles are available, so the exact relationship between them and their details are not confirmed.","subsystem":null,"counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":61119,"merged_at":"2026-07-08T20:52:13+00:00"},"resolution":{"state":"resolved","by":"pr-61119"},"freshness":{"state":"verified","last_verified_at":"2026-07-08T20:52:13+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":60955,"pr_number":61119,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-07-08T20:52:13+00:00"}],"members":[{"kind":"issue","number":60955,"title":"hermes-fallback-bug-report","url":null,"state":"closed"},{"kind":"pr","number":60992,"title":"fix(agent): reset fallback chain index after failed early activation (#60955)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#60955","number":60955},{"kind":"pr","value":"#35314","number":35314},{"kind":"pr","value":"#17446","number":17446},{"kind":"pr","value":"#61119","number":61119},{"kind":"pr","value":"#60987","number":60987},{"kind":"pr","value":"#20465","number":20465}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":61119,"title":"fix(gateway): reload fallback_providers for live messaging sessions (#60987 salvage)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_fallback_chain_reload.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -5008,6 +5008,74 @@ def _load_fallback_model() -> list | None:\n             pass\n         return None\n \n+    def _refresh_fallback_model(self) -> list | None:\n+        \"\"\"Re-read fallback_providers from disk for the next agent create/reuse.\n+\n+        Cron already does this per job via ``get_fallback_chain``; the gateway\n+        previously froze ``self._fallback_model`` at process start, so a chain\n+        configured (or changed) after ``hermes gateway`` was running never\n+        reached messaging sessions even though the same process's cron jobs\n+        fell back correctly. Fixes #60955.\n+\n+        A TRANSIENT read/parse failure (user mid-edit of config.yaml with a\n+        non-atomic write) keeps the last known-good chain instead of wiping a\n+        cached agent's working fallback for that turn.  Only a successful read\n+        that genuinely lacks the key clears the chain.\n+        \"\"\"\n+        try:\n+            import yaml as _y\n+            cfg_path = _hermes_home / \"config.yaml\"\n+            if not cfg_path.exists():\n+                self._fallback_model = None\n+                return self._fallback_model\n+            with open(cfg_path, encoding=\"utf-8\") as _f:\n+                cfg = _y.safe_load(_f) or {}\n+        except Exception:\n+            # Transient failure — keep last known-good chain.\n+            logger.debug(\n+                \"fallback_providers refresh: config.yaml read failed; \"\n+                \"keeping last known-good chain\", exc_info=True,\n+            )\n+            return self._fallback_model\n+        self._fallback_model = get_fallback_chain(cfg) or None\n+        return self._fallback_model\n+\n+    @staticmethod\n+    def _apply_fallback_chain_to_agent(agent: Any, chain: list | None) -> None:\n+        \"\"\"Keep a cached agent's fallback chain aligned with current config.\n+\n+        Skips rewrite while a cooldown is holding the agent on an already-\n+        activated fallback provider — ``restore_primary_runtime`` owns that\n+        turn-scoped lifecycle. When primary is active (or cooldown expired),\n+        replace the chain so mid-uptime ``fallback_providers`` edits take\n+        effect without requiring a gateway restart (#60955).\n+        \"\"\"\n+        if agent is None:\n+            return\n+        new_chain = list(chain or [])\n+        rate_limited_until = getattr(agent, \"_rate_limited_until\", 0) or 0\n+        if (\n+            getattr(agent, \"_fallback_activated\", False)\n+            and rate_limited_until > time.monotonic()\n+        ):\n+            return\n+        old_chain = list(getattr(agent, \"_fallback_chain\", []) or [])\n+        agent._fallback_chain = new_chain\n+        agent._fallback_model = new_chain[0] if new_chain else None\n+        if not getattr(agent, \"_fallback_activated\", False):\n+            agent._fallback_index = 0\n+        # A config edit signals the user changed something — drop the\n+        # session-scoped unavailability memo so re-configured entries\n+        # (e.g. credentials added mid-uptime for a previously-failing\n+        # provider) get retried instead of staying suppressed for the\n+        # cached agent's lifetime.  Only on actual content change, so\n+        # the per-message no-op refresh keeps the memo's rate-limiting\n+        # benefit (#60955).\n+        if new_chain != old_chain:\n+            unavailable = getattr(agent, \"_unavailable_fallback_keys\", None)\n+            if unavailable:\n+                unavailable.clear()\n+\n     def _snapshot_running_agents(self) -> Dict[str, Any]:\n         return {\n             session_key: agent","@@ -13244,7 +13312,8 @@ def run_sync():\n                     chat_type=source.chat_type,\n                     thread_id=source.thread_id,\n                     session_db=getattr(self._session_db, \"_db\", self._session_db),\n-                    fallback_model=self._fallback_model,\n+                    # Reload from disk — do not reuse the startup snapshot (#60955).\n+                    fallback_model=self._refresh_fallback_model(),\n                 )\n                 try:\n                     return agent.run_conversation(","@@ -18065,6 +18134,18 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:\n                             logger.debug(\"Reusing cached agent for session %s\", session_key)\n                             reused_cached_agent = True\n \n+            # Lock released — refresh the fallback chain from disk for the\n+            # reused agent OUTSIDE the cache lock (config.yaml read is disk\n+            # I/O; the idle-sweep watcher contends on this lock and stalls\n+            # Discord heartbeats — same reasoning as #52197).  A chain\n+            # configured after this agent was cached (or after gateway start)\n+            # must reach the next turn (#60955).  Per-session turn\n+            # serialization (_running_agents) keeps this safe post-lock.\n+            if reused_cached_agent and agent is not None:\n+                self._apply_fallback_chain_to_agent(\n+                    agent, self._refresh_fallback_model(),\n+                )\n+\n             # Lock released — now schedule cleanup of any cross-process-evicted\n             # agent on a daemon thread so memory-provider shutdown / socket\n             # teardown never blocks the gateway event loop or the cache lock","@@ -18117,7 +18198,8 @@ def _interim_assistant_cb(text: str, *, already_streamed: bool = False) -> None:\n                     thread_id=source.thread_id,\n                     gateway_session_key=session_key,\n                     session_db=getattr(self._session_db, \"_db\", self._session_db),\n-                    fallback_model=self._fallback_model,\n+                    # Reload from disk — do not reuse the startup snapshot (#60955).\n+                    fallback_model=self._refresh_fallback_model(),\n                 )\n                 if _cache_lock and _cache is not None:\n                     with _cache_lock:"]},{"file":"tests/gateway/test_fallback_chain_reload.py","hunks":["@@ -0,0 +1,239 @@\n+\"\"\"Regression tests for #60955: gateway must not freeze fallback_providers.\n+\n+Cron reloads ``fallback_providers`` from disk on every job. The gateway used to\n+freeze ``self._fallback_model`` at process start, so a chain configured (or\n+edited) after ``hermes gateway`` was already running never reached messaging\n+sessions — even though cron in the same process fell back correctly.\n+\n+These tests pin the reload + cached-agent apply helpers without driving the\n+full Feishu session path.\n+\"\"\"\n+\n+from __future__ import annotations\n+\n+import time\n+from types import SimpleNamespace\n+\n+\n+def test_refresh_fallback_model_rereads_config(tmp_path, monkeypatch):\n+    from gateway.run import GatewayRunner\n+\n+    monkeypatch.setattr(\"gateway.run._hermes_home\", tmp_path)\n+    cfg = tmp_path / \"config.yaml\"\n+    cfg.write_text(\n+        \"fallback_providers:\\n\"\n+        \"  - provider: deepseek\\n\"\n+        \"    model: deepseek-v4-flash\\n\"\n+    )\n+\n+    runner = SimpleNamespace(\n+        _fallback_model=None,\n+    )\n+    runner._load_fallback_model = GatewayRunner._load_fallback_model\n+    bound = GatewayRunner._refresh_fallback_model.__get__(runner)\n+    chain = bound()\n+\n+    assert chain == [{\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"}]\n+    assert runner._fallback_model == chain\n+\n+    cfg.write_text(\n+        \"fallback_providers:\\n\"\n+        \"  - provider: openrouter\\n\"\n+        \"    model: anthropic/claude-sonnet-4.6\\n\"\n+    )\n+    updated = bound()\n+    assert updated == [\n+        {\"provider\": \"openrouter\", \"model\": \"anthropic/claude-sonnet-4.6\"}\n+    ]\n+    assert runner._fallback_model == updated\n+\n+\n+def test_refresh_fallback_model_clears_when_config_removed(tmp_path, monkeypatch):\n+    from gateway.run import GatewayRunner\n+\n+    monkeypatch.setattr(\"gateway.run._hermes_home\", tmp_path)\n+    cfg = tmp_path / \"config.yaml\"\n+    cfg.write_text(\n+        \"fallback_providers:\\n\"\n+        \"  - provider: deepseek\\n\"\n+        \"    model: deepseek-v4-flash\\n\"\n+    )\n+\n+    runner = SimpleNamespace(\n+        _fallback_model=[{\"provider\": \"stale\", \"model\": \"x\"}],\n+    )\n+    runner._load_fallback_model = GatewayRunner._load_fallback_model\n+    bound = GatewayRunner._refresh_fallback_model.__get__(runner)\n+    assert bound() is not None\n+\n+    cfg.write_text(\"model:\\n  provider: nvidia\\n\")\n+    assert bound() is None\n+    assert runner._fallback_model is None\n+\n+\n+def test_refresh_fallback_model_keeps_last_known_good_on_read_failure(\n+    tmp_path, monkeypatch,\n+):\n+    \"\"\"A transient config.yaml read/parse failure (user mid-edit, non-atomic\n+    write) must NOT wipe the last known-good chain — only a successful read\n+    that genuinely lacks the key clears it.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    monkeypatch.setattr(\"gateway.run._hermes_home\", tmp_path)\n+    cfg = tmp_path / \"config.yaml\"\n+    cfg.write_text(\n+        \"fallback_providers:\\n\"\n+        \"  - provider: deepseek\\n\"\n+        \"    model: deepseek-v4-flash\\n\"\n+    )\n+\n+    runner = SimpleNamespace(_fallback_model=None)\n+    runner._load_fallback_model = GatewayRunner._load_fallback_model\n+    bound = GatewayRunner._refresh_fallback_model.__get__(runner)\n+    good = bound()\n+    assert good == [{\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"}]\n+\n+    # Simulate a mid-edit torn write: invalid YAML.\n+    cfg.write_text(\"fallback_providers:\\n  - provider: [unclosed\\n\")\n+    assert bound() == good\n+    assert runner._fallback_model == good\n+\n+\n+def test_apply_fallback_chain_updates_primary_agent():\n+    from gateway.run import GatewayRunner\n+\n+    agent = SimpleNamespace(\n+        _fallback_chain=[],\n+        _fallback_model=None,\n+        _fallback_index=0,\n+        _fallback_activated=False,\n+        _rate_limited_until=0,\n+    )\n+    chain = [{\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"}]\n+    GatewayRunner._apply_fallback_chain_to_agent(agent, chain)\n+\n+    assert agent._fallback_chain == chain\n+    assert agent._fallback_model == chain[0]\n+    assert agent._fallback_index == 0\n+\n+\n+def test_apply_fallback_chain_skips_while_cooldown_holds_fallback():\n+    \"\"\"Do not clobber a live fallback activation during its cooldown window.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    live = [{\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"}]\n+    agent = SimpleNamespace(\n+        _fallback_chain=live,\n+        _fallback_model=live[0],\n+        _fallback_index=1,\n+        _fallback_activated=True,\n+        _rate_limited_until=time.monotonic() + 30,\n+    )\n+    GatewayRunner._apply_fallback_chain_to_agent(\n+        agent,\n+        [{\"provider\": \"openrouter\", \"model\": \"anthropic/claude-sonnet-4.6\"}],\n+    )\n+\n+    assert agent._fallback_chain == live\n+    assert agent._fallback_index == 1\n+    assert agent._fallback_activated is True\n+\n+\n+def test_apply_fallback_chain_updates_after_cooldown_expires():\n+    from gateway.run import GatewayRunner\n+\n+    agent = SimpleNamespace(\n+        _fallback_chain=[{\"provider\": \"deepseek\", \"model\": \"old\"}],\n+        _fallback_model={\"provider\": \"deepseek\", \"model\": \"old\"},\n+        _fallback_index=1,\n+        _fallback_activated=True,\n+        _rate_limited_until=time.monotonic() - 1,\n+    )\n+    new_chain = [{\"provider\": \"openrouter\", \"model\": \"anthropic/claude-sonnet-4.6\"}]\n+    GatewayRunner._apply_fallback_chain_to_agent(agent, new_chain)\n+\n+    assert agent._fallback_chain == new_chain\n+    assert agent._fallback_model == new_chain[0]\n+    # Activated agents keep their index; restore_primary_runtime owns reset.\n+    assert agent._fallback_index == 1\n+\n+\n+def test_apply_fallback_chain_clears_unavailable_memo_on_content_change():\n+    \"\"\"A config edit must drop the session-scoped unavailability memo so a\n+    re-configured entry (credentials added mid-uptime) is retried instead of\n+    staying suppressed for the cached agent's lifetime.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    agent = SimpleNamespace(\n+        _fallback_chain=[{\"provider\": \"deepseek\", \"model\": \"old\"}],\n+        _fallback_model={\"provider\": \"deepseek\", \"model\": \"old\"},\n+        _fallback_index=0,\n+        _fallback_activated=False,\n+        _rate_limited_until=0,\n+        _unavailable_fallback_keys={(\"deepseek\", \"old\", \"\")},\n+    )\n+    new_chain = [{\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"}]\n+    GatewayRunner._apply_fallback_chain_to_agent(agent, new_chain)\n+\n+    assert agent._fallback_chain == new_chain\n+    assert agent._unavailable_fallback_keys == set()\n+\n+\n+def test_apply_fallback_chain_keeps_unavailable_memo_when_unchanged():\n+    \"\"\"The per-message no-op refresh must NOT clear the memo — it exists to\n+    rate-limit repeated activation attempts against dead entries.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    chain = [{\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"}]\n+    memo = {(\"deepseek\", \"deepseek-v4-flash\", \"\")}\n+    agent = SimpleNamespace(\n+        _fallback_chain=list(chain),\n+        _fallback_model=chain[0],\n+        _fallback_index=0,\n+        _fallback_activated=False,\n+        _rate_limited_until=0,\n+        _unavailable_fallback_keys=set(memo),\n+    )\n+    GatewayRunner._apply_fallback_chain_to_agent(agent, list(chain))\n+\n+    assert agent._unavailable_fallback_keys == memo\n+\n+\n+def test_background_and_main_agent_paths_call_refresh():\n+    \"\"\"Both AIAgent construction sites must pass a refreshed chain, not the\n+    startup snapshot, and the cached-agent reuse path must apply the refreshed\n+    chain. Source-level invariant for call sites that resist unit testing.\n+    \"\"\"\n+    from pathlib import Path\n+\n+    source = (\n+        Path(__file__).resolve().parent.parent.parent / \"gateway\" / \"run.py\"\n+    ).read_text(encoding=\"utf-8\")\n+    assert \"fallback_model=self._refresh_fallback_model()\" in source\n+    assert source.count(\"fallback_model=self._refresh_fallback_model()\") >= 2\n+    # The cached-agent reuse path (the load-bearing fix for a long-lived\n+    # session in a running gateway) must apply the refreshed chain.\n+    assert \"self._apply_fallback_chain_to_agent(\" in source\n+    # The stale startup-snapshot form must not remain at create sites.\n+    assert \"fallback_model=self._fallback_model,\" not in source\n+\n+\n+def test_load_fallback_model_static_unchanged_contract(tmp_path, monkeypatch):\n+    \"\"\"_load_fallback_model remains a pure static reader used by refresh.\"\"\"\n+    from gateway.run import GatewayRunner\n+\n+    monkeypatch.setattr(\"gateway.run._hermes_home\", tmp_path)\n+    (tmp_path / \"config.yaml\").write_text(\n+        \"fallback_providers:\\n\"\n+        \"  - provider: deepseek\\n\"\n+        \"    model: deepseek-v4-flash\\n\"\n+        \"fallback_model:\\n\"\n+        \"  provider: nous\\n\"\n+        \"  model: Hermes-4\\n\"\n+    )\n+\n+    chain = GatewayRunner._load_fallback_model()\n+    assert chain == [\n+        {\"provider\": \"deepseek\", \"model\": \"deepseek-v4-flash\"},\n+        {\"provider\": \"nous\", \"model\": \"Hermes-4\"},\n+    ]\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}