{"repo":"NousResearch/hermes-agent","public_id":"12607","display_label":"Gateway platform reconnect retry reliability fixes","summary":"These issues and PRs address problems where the gateway's reconnect watcher would permanently stop retrying platform connections after a limited number of failures or prolonged network loss. The work focuses on ensuring retryable platforms remain queued and reconnect attempts continue indefinitely through transient network and DNS failures.","subsystem":"gateway","counts":{"members":7,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":2,"merged_prs":2,"attempt_prs":3},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":35387},"resolution":{"state":"partially_resolved","by":"pr-26600"},"freshness":{"state":"verified","last_verified_at":"2026-05-30T14:33:34+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.25523809523809526},"fix_edges":[{"issue_number":12607,"pr_number":35387,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false},{"issue_number":35284,"pr_number":35387,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true},{"issue_number":35936,"pr_number":35387,"tier":"verified","basis":"superseded_ref","confidence":0.9,"recoverable_from_timeline":false}],"members":[{"kind":"issue","number":12607,"title":"fix(gateway): retryable reconnects stop after prolonged network loss","url":null,"state":"closed"},{"kind":"issue","number":17063,"title":"Gateway reconnect watcher permanently stops retryable platforms after 20 failed attempts","url":null,"state":"closed"},{"kind":"pr","number":13197,"title":"fix(gateway): keep retryable platform reconnects queued","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"45465b0d5","number":0},{"kind":"pr","value":"#35387","number":35387}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":17216,"title":"fix: gateway reconnect watcher retries indefinitely instead of giving up after 20 attempts","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"45465b0d5d8c7b2db7df6d9e466589cdef9136c0","number":0},{"kind":"pr","value":"#35387","number":35387}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":17219,"title":"fix(gateway): don't give up on retryable platform reconnect (#17063)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#17063","number":17063},{"kind":"pr","value":"#26600","number":26600}],"category":"superseded_by_pr","implication":"superseded"},"merged":false},{"kind":"pr","number":26600,"title":"fix(gateway): keep running when platforms fail; add per-platform circuit breaker + /platform","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"suggested","fix_detail_withheld":true},{"kind":"pr","number":35387,"title":"fix(gateway): never auto-pause platforms on transient network/DNS failures","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/run.py","tests/gateway/test_platform_reconnect.py"],"fix_hunks":[{"file":"gateway/run.py","hunks":["@@ -1734,9 +1734,9 @@ def __init__(self, config: Optional[GatewayConfig] = None):\n         # as a fallback when a fresh config read transiently returns an empty\n         # model (e.g. an mtime-keyed config-cache miss during a post-interrupt\n         # recovery turn). Without this, the agent is built with model=\"\" and\n-        # every API call fails HTTP 400 \"No models provided\" — the session\n-        # goes silent until the user manually re-sends. See #35314. The \"*\"\n-        # key holds a process-wide last-known-good for first-seen sessions.\n+        # every API call fails HTTP 400 \"No models provided\" — the session goes\n+        # silent until the user manually re-sends. See #35314. ``\"*\"`` holds a\n+        # process-wide last-known-good for sessions seen for the first time.\n         self._last_resolved_model: Dict[str, str] = {}\n         # Overflow buffer for explicit /queue commands.  The adapter-level\n         # _pending_messages dict is a single slot per session (designed for","@@ -2502,8 +2502,8 @@ def _resolve_session_agent_runtime(\n         # successfully resolved for this session (or, failing that, the most\n         # recent one resolved process-wide). Building an agent with model=\"\"\n         # makes every API call fail HTTP 400 \"No models provided\" and the\n-        # session goes silent until the user manually re-sends. getattr guards\n-        # against bare test runners built via object.__new__.\n+        # session goes silent until the user manually re-sends. ``getattr``\n+        # guards against bare test runners built via ``object.__new__``.\n         _last_good = getattr(self, \"_last_resolved_model\", None)\n         if _last_good is not None:\n             if not model:","@@ -2818,10 +2818,12 @@ def _pause_failed_platform(self, platform, *, reason: str = \"\") -> None:\n         \"\"\"Mark a queued platform as paused — keep it in ``_failed_platforms``\n         but stop the reconnect watcher from hammering it.\n \n-        Used by the circuit breaker after ``_PAUSE_AFTER_FAILURES`` consecutive\n-        retryable failures, and by ``/platform pause <name>`` for manual\n-        intervention.  Paused platforms are surfaced in ``/platform list``\n-        and resumed with ``/platform resume <name>``.\n+        Used by ``/platform pause <name>`` for manual operator intervention.\n+        Paused platforms are surfaced in ``/platform list`` and resumed with\n+        ``/platform resume <name>``.  Note: the reconnect watcher does NOT\n+        auto-pause — retryable (network/DNS) failures keep retrying at the\n+        backoff cap indefinitely so a transient outage self-heals without\n+        manual intervention.\n         \"\"\"\n         info = getattr(self, \"_failed_platforms\", {}).get(platform)\n         if info is None:","@@ -5899,15 +5901,17 @@ async def _platform_reconnect_watcher(self) -> None:\n         \"\"\"Background task that periodically retries connecting failed platforms.\n \n         Uses exponential backoff: 30s → 60s → 120s → 240s → 300s (cap).\n-        Retryable failures keep retrying at the backoff cap indefinitely\n-        — but if a platform fails ``_PAUSE_AFTER_FAILURES`` times in a row\n-        without ever succeeding, it is *paused*: kept in the retry queue\n-        but no longer hammered.  The user surfaces it with ``/platform list``\n-        and resumes it with ``/platform resume <name>``.  Non-retryable\n-        failures (bad auth, etc.) still drop out of the queue immediately.\n+        Retryable failures (network/DNS blips) keep retrying at the backoff\n+        cap indefinitely — they self-heal once connectivity returns, so a\n+        transient outage never requires manual intervention. Non-retryable\n+        failures (bad auth, etc.) drop out of the queue immediately. The\n+        circuit breaker (``_pause_failed_platform`` / ``/platform pause``)\n+        remains available for manual operator control via ``/platform list``\n+        and ``/platform resume <name>``, but is no longer triggered\n+        automatically — auto-pausing a recovered platform was the cause of\n+        bots silently staying dead after a transient DNS failure.\n         \"\"\"\n         _BACKOFF_CAP = 300  # 5 minutes max between retries\n-        _PAUSE_AFTER_FAILURES = 10  # circuit-breaker threshold\n \n         await asyncio.sleep(10)  # initial delay — let startup finish\n         while self._running:","@@ -6002,14 +6006,14 @@ async def _platform_reconnect_watcher(self) -> None:\n                             \"Reconnect %s failed, next retry in %ds\",\n                             platform.value, backoff,\n                         )\n-                        if attempt >= _PAUSE_AFTER_FAILURES:\n-                            self._pause_failed_platform(\n-                                platform,\n-                                reason=(\n-                                    adapter.fatal_error_message\n-                                    or \"failed to reconnect\"\n-                                ),\n-                            )\n+                        # Retryable failures (network/DNS blips) keep retrying\n+                        # at the backoff cap indefinitely — they self-heal once\n+                        # connectivity returns. We do NOT auto-pause them: a\n+                        # transient outage must never require manual `/platform\n+                        # resume` to recover. Non-retryable failures (bad auth,\n+                        # etc.) already drop out of the queue via the\n+                        # `not fatal_error_retryable` branch above, so anything\n+                        # reaching here is by definition retryable.\n                 except Exception as e:\n                     self._update_platform_runtime_status(\n                         platform.value,","@@ -6024,8 +6028,9 @@ async def _platform_reconnect_watcher(self) -> None:\n                         \"Reconnect %s error: %s, next retry in %ds\",\n                         platform.value, e, backoff,\n                     )\n-                    if attempt >= _PAUSE_AFTER_FAILURES:\n-                        self._pause_failed_platform(platform, reason=str(e))\n+                    # A raised exception during reconnect (connect timeout, DNS\n+                    # resolution failure, etc.) is inherently transient — keep\n+                    # retrying at the backoff cap rather than auto-pausing.\n \n             # Check every 10 seconds for platforms that need reconnection\n             for _ in range(10):"]},{"file":"tests/gateway/test_platform_reconnect.py","hunks":["@@ -294,19 +294,20 @@ async def fake_sleep(n):\n         assert runner._failed_platforms[Platform.TELEGRAM][\"attempts\"] == 2\n \n     @pytest.mark.asyncio\n-    async def test_reconnect_pauses_after_circuit_breaker_threshold(self):\n-        \"\"\"After enough consecutive retryable failures, the watcher should\n-        *pause* the platform (keep it in the queue but stop hammering it),\n-        not drop it. The user resumes via /platform resume.\n+    async def test_reconnect_never_auto_pauses_retryable_failures(self):\n+        \"\"\"Retryable failures (network/DNS) must keep retrying indefinitely —\n+        the watcher must NOT auto-pause them. Auto-pausing a transiently-failed\n+        platform left bots silently dead after a DNS blip (#35284). The pause\n+        circuit breaker remains available for manual /platform pause only.\n         \"\"\"\n         runner = _make_runner()\n \n         platform_config = PlatformConfig(enabled=True, token=\"test\")\n-        # 9 prior attempts — the next failure will be the 10th and should\n-        # trip the circuit breaker.\n+        # Far past the old circuit-breaker threshold (10): even after many\n+        # consecutive retryable failures the platform must stay unpaused.\n         runner._failed_platforms[Platform.TELEGRAM] = {\n             \"config\": platform_config,\n-            \"attempts\": 9,\n+            \"attempts\": 25,\n             \"next_retry\": time.monotonic() - 1,\n         }\n ","@@ -332,12 +333,15 @@ async def fake_sleep(n):\n \n             await run_one_iteration()\n \n-        # Platform stays in queue — paused, not dropped\n+        # Platform stays in queue and keeps retrying — never auto-paused.\n         assert Platform.TELEGRAM in runner._failed_platforms\n         info = runner._failed_platforms[Platform.TELEGRAM]\n-        assert info[\"paused\"] is True\n-        assert info[\"attempts\"] == 10\n-        assert \"pause_reason\" in info\n+        assert info.get(\"paused\") is not True\n+        assert \"pause_reason\" not in info\n+        assert info[\"attempts\"] == 26\n+        # next_retry is pushed out by the backoff (capped at 300s), not inf.\n+        assert info[\"next_retry\"] != float(\"inf\")\n+        assert info[\"next_retry\"] > time.monotonic()\n \n     @pytest.mark.asyncio\n     async def test_reconnect_skips_paused_platforms(self):\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}