{"repo":"NousResearch/hermes-agent","public_id":"7677","display_label":"Gateway connectivity reliability and silent failure fixes","summary":"Multiple issues and fixes address gateways (Slack, Discord) appearing healthy or running while actually disconnected, with related work on reconnection, startup failure handling, and surfacing errors that were previously silent.","subsystem":"gateway","counts":{"members":6,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":3,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":8016},"resolution":{"state":"partially_resolved","by":"pr-8016"},"freshness":{"state":"stale","last_verified_at":"2026-04-12T02:22:14+00:00","is_stale":true},"grouping":{"backbone":"STRUCTURAL_PARTIAL","link_density":0.24799999999999997},"fix_edges":[{"issue_number":7677,"pr_number":8016,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true}],"members":[{"kind":"issue","number":7677,"title":"Discord gateway can look healthy/running while not actually connected, causing silent message loss","url":null,"state":"closed"},{"kind":"issue","number":14326,"title":"[Bug]: Slack gateway can remain 'running' while Socket Mode is dead, requiring manual restart","url":null,"state":"closed"},{"kind":"issue","number":25476,"title":"[Bug]: Slack adapter Socket Mode silently drops without auto-reconnect; no log at default WARNING level","url":null,"state":"closed"},{"kind":"pr","number":7979,"title":"fix(discord): decouple readiness from slash sync","url":null,"state":"closed","close_reason":{"refs":[{"kind":"pr","value":"#8016","number":8016}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":8016,"title":"fix(discord): decouple readiness from slash sync","url":null,"state":"merged","close_reason":{"category":"merged","implication":"superseded"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/platforms/discord.py","gateway/run.py","gateway/status.py","tests/gateway/test_discord_connect.py","tests/gateway/test_runner_startup_failures.py","tests/gateway/test_status.py"],"fix_hunks":[{"file":"gateway/platforms/discord.py","hunks":["@@ -456,6 +456,7 @@ def __init__(self, config: PlatformConfig):\n         # show the standard typing gateway event for bots)\n         self._typing_tasks: Dict[str, asyncio.Task] = {}\n         self._bot_task: Optional[asyncio.Task] = None\n+        self._post_connect_task: Optional[asyncio.Task] = None\n         # Dedup cache: prevents duplicate bot responses when Discord\n         # RESUME replays events after reconnects.\n         self._dedup = MessageDeduplicator()","@@ -545,15 +546,14 @@ async def on_ready():\n \n                 # Resolve any usernames in the allowed list to numeric IDs\n                 await adapter_self._resolve_allowed_usernames()\n-\n-                # Sync slash commands with Discord\n-                try:\n-                    synced = await adapter_self._client.tree.sync()\n-                    logger.info(\"[%s] Synced %d slash command(s)\", adapter_self.name, len(synced))\n-                except Exception as e:  # pragma: no cover - defensive logging\n-                    logger.warning(\"[%s] Slash command sync failed: %s\", adapter_self.name, e, exc_info=True)\n                 adapter_self._ready_event.set()\n \n+                if adapter_self._post_connect_task and not adapter_self._post_connect_task.done():\n+                    adapter_self._post_connect_task.cancel()\n+                adapter_self._post_connect_task = asyncio.create_task(\n+                    adapter_self._run_post_connect_initialization()\n+                )\n+\n             @self._client.event\n             async def on_message(message: DiscordMessage):\n                 # Dedup: Discord RESUME replays events after reconnects (#4777)","@@ -686,14 +686,36 @@ async def disconnect(self) -> None:\n             except Exception as e:  # pragma: no cover - defensive logging\n                 logger.warning(\"[%s] Error during disconnect: %s\", self.name, e, exc_info=True)\n \n+        if self._post_connect_task and not self._post_connect_task.done():\n+            self._post_connect_task.cancel()\n+            try:\n+                await self._post_connect_task\n+            except asyncio.CancelledError:\n+                pass\n+\n         self._running = False\n         self._client = None\n         self._ready_event.clear()\n+        self._post_connect_task = None\n \n         self._release_platform_lock()\n \n         logger.info(\"[%s] Disconnected\", self.name)\n \n+    async def _run_post_connect_initialization(self) -> None:\n+        \"\"\"Finish non-critical startup work after Discord is connected.\"\"\"\n+        if not self._client:\n+            return\n+        try:\n+            synced = await asyncio.wait_for(self._client.tree.sync(), timeout=30)\n+            logger.info(\"[%s] Synced %d slash command(s)\", self.name, len(synced))\n+        except asyncio.TimeoutError:\n+            logger.warning(\"[%s] Slash command sync timed out after 30s\", self.name)\n+        except asyncio.CancelledError:\n+            raise\n+        except Exception as e:  # pragma: no cover - defensive logging\n+            logger.warning(\"[%s] Slash command sync failed: %s\", self.name, e, exc_info=True)\n+\n     async def _add_reaction(self, message: Any, emoji: str) -> bool:\n         \"\"\"Add an emoji reaction to a Discord message.\"\"\"\n         if not message or not hasattr(message, \"add_reaction\"):"]},{"file":"gateway/run.py","hunks":["@@ -916,6 +916,12 @@ async def _handle_adapter_fatal_error(self, adapter: BasePlatformAdapter) -> Non\n             adapter.fatal_error_code or \"unknown\",\n             adapter.fatal_error_message or \"unknown error\",\n         )\n+        self._update_platform_runtime_status(\n+            adapter.platform.value,\n+            platform_state=\"retrying\" if adapter.fatal_error_retryable else \"fatal\",\n+            error_code=adapter.fatal_error_code,\n+            error_message=adapter.fatal_error_message,\n+        )\n \n         existing = self.adapters.get(adapter.platform)\n         if existing is adapter:","@@ -993,6 +999,25 @@ def _update_runtime_status(self, gateway_state: Optional[str] = None, exit_reaso\n             )\n         except Exception:\n             pass\n+\n+    def _update_platform_runtime_status(\n+        self,\n+        platform: str,\n+        *,\n+        platform_state: Optional[str] = None,\n+        error_code: Optional[str] = None,\n+        error_message: Optional[str] = None,\n+    ) -> None:\n+        try:\n+            from gateway.status import write_runtime_status\n+            write_runtime_status(\n+                platform=platform,\n+                platform_state=platform_state,\n+                error_code=error_code,\n+                error_message=error_message,\n+            )\n+        except Exception:\n+            pass\n     \n     @staticmethod\n     def _load_prefill_messages() -> List[Dict[str, Any]]:","@@ -1498,16 +1523,34 @@ async def start(self) -> bool:\n             \n             # Try to connect\n             logger.info(\"Connecting to %s...\", platform.value)\n+            self._update_platform_runtime_status(\n+                platform.value,\n+                platform_state=\"connecting\",\n+                error_code=None,\n+                error_message=None,\n+            )\n             try:\n                 success = await adapter.connect()\n                 if success:\n                     self.adapters[platform] = adapter\n                     self._sync_voice_mode_state_to_adapter(adapter)\n                     connected_count += 1\n+                    self._update_platform_runtime_status(\n+                        platform.value,\n+                        platform_state=\"connected\",\n+                        error_code=None,\n+                        error_message=None,\n+                    )\n                     logger.info(\"✓ %s connected\", platform.value)\n                 else:\n                     logger.warning(\"✗ %s failed to connect\", platform.value)\n                     if adapter.has_fatal_error:\n+                        self._update_platform_runtime_status(\n+                            platform.value,\n+                            platform_state=\"retrying\" if adapter.fatal_error_retryable else \"fatal\",\n+                            error_code=adapter.fatal_error_code,\n+                            error_message=adapter.fatal_error_message,\n+                        )\n                         target = (\n                             startup_retryable_errors\n                             if adapter.fatal_error_retryable","@@ -1524,6 +1567,12 @@ async def start(self) -> bool:\n                                 \"next_retry\": time.monotonic() + 30,\n                             }\n                     else:\n+                        self._update_platform_runtime_status(\n+                            platform.value,\n+                            platform_state=\"retrying\",\n+                            error_code=None,\n+                            error_message=\"failed to connect\",\n+                        )\n                         startup_retryable_errors.append(\n                             f\"{platform.value}: failed to connect\"\n                         )","@@ -1535,6 +1584,12 @@ async def start(self) -> bool:\n                         }\n             except Exception as e:\n                 logger.error(\"✗ %s error: %s\", platform.value, e)\n+                self._update_platform_runtime_status(\n+                    platform.value,\n+                    platform_state=\"retrying\",\n+                    error_code=None,\n+                    error_message=str(e),\n+                )\n                 startup_retryable_errors.append(f\"{platform.value}: {e}\")\n                 # Unexpected exceptions are typically transient — queue for retry\n                 self._failed_platforms[platform] = {","@@ -1813,6 +1868,12 @@ async def _platform_reconnect_watcher(self) -> None:\n                         self._sync_voice_mode_state_to_adapter(adapter)\n                         self.delivery_router.adapters = self.adapters\n                         del self._failed_platforms[platform]\n+                        self._update_platform_runtime_status(\n+                            platform.value,\n+                            platform_state=\"connected\",\n+                            error_code=None,\n+                            error_message=None,\n+                        )\n                         logger.info(\"✓ %s reconnected successfully\", platform.value)\n \n                         # Rebuild channel directory with the new adapter","@@ -1824,12 +1885,24 @@ async def _platform_reconnect_watcher(self) -> None:\n                     else:\n                         # Check if the failure is non-retryable\n                         if adapter.has_fatal_error and not adapter.fatal_error_retryable:\n+                            self._update_platform_runtime_status(\n+                                platform.value,\n+                                platform_state=\"fatal\",\n+                                error_code=adapter.fatal_error_code,\n+                                error_message=adapter.fatal_error_message,\n+                            )\n                             logger.warning(\n                                 \"Reconnect %s: non-retryable error (%s), removing from retry queue\",\n                                 platform.value, adapter.fatal_error_message,\n                             )\n                             del self._failed_platforms[platform]\n                         else:\n+                            self._update_platform_runtime_status(\n+                                platform.value,\n+                                platform_state=\"retrying\",\n+                                error_code=adapter.fatal_error_code,\n+                                error_message=adapter.fatal_error_message or \"failed to reconnect\",\n+                            )\n                             backoff = min(30 * (2 ** (attempt - 1)), _BACKOFF_CAP)\n                             info[\"attempts\"] = attempt\n                             info[\"next_retry\"] = time.monotonic() + backoff","@@ -1838,6 +1911,12 @@ async def _platform_reconnect_watcher(self) -> None:\n                                 platform.value, backoff,\n                             )\n                 except Exception as e:\n+                    self._update_platform_runtime_status(\n+                        platform.value,\n+                        platform_state=\"retrying\",\n+                        error_code=None,\n+                        error_message=str(e),\n+                    )\n                     backoff = min(30 * (2 ** (attempt - 1)), _BACKOFF_CAP)\n                     info[\"attempts\"] = attempt\n                     info[\"next_retry\"] = time.monotonic() + backoff"]},{"file":"gateway/status.py","hunks":["@@ -26,6 +26,7 @@\n _RUNTIME_STATUS_FILE = \"gateway_state.json\"\n _LOCKS_DIRNAME = \"gateway-locks\"\n _IS_WINDOWS = sys.platform == \"win32\"\n+_UNSET = object()\n \n \n def _get_pid_path() -> Path:","@@ -218,14 +219,14 @@ def write_pid_file() -> None:\n \n def write_runtime_status(\n     *,\n-    gateway_state: Optional[str] = None,\n-    exit_reason: Optional[str] = None,\n-    restart_requested: Optional[bool] = None,\n-    active_agents: Optional[int] = None,\n-    platform: Optional[str] = None,\n-    platform_state: Optional[str] = None,\n-    error_code: Optional[str] = None,\n-    error_message: Optional[str] = None,\n+    gateway_state: Any = _UNSET,\n+    exit_reason: Any = _UNSET,\n+    restart_requested: Any = _UNSET,\n+    active_agents: Any = _UNSET,\n+    platform: Any = _UNSET,\n+    platform_state: Any = _UNSET,\n+    error_code: Any = _UNSET,\n+    error_message: Any = _UNSET,\n ) -> None:\n     \"\"\"Persist gateway runtime health information for diagnostics/status.\"\"\"\n     path = _get_runtime_status_path()","@@ -236,22 +237,22 @@ def write_runtime_status(\n     payload[\"start_time\"] = _get_process_start_time(os.getpid())\n     payload[\"updated_at\"] = _utc_now_iso()\n \n-    if gateway_state is not None:\n+    if gateway_state is not _UNSET:\n         payload[\"gateway_state\"] = gateway_state\n-    if exit_reason is not None:\n+    if exit_reason is not _UNSET:\n         payload[\"exit_reason\"] = exit_reason\n-    if restart_requested is not None:\n+    if restart_requested is not _UNSET:\n         payload[\"restart_requested\"] = bool(restart_requested)\n-    if active_agents is not None:\n+    if active_agents is not _UNSET:\n         payload[\"active_agents\"] = max(0, int(active_agents))\n \n-    if platform is not None:\n+    if platform is not _UNSET:\n         platform_payload = payload[\"platforms\"].get(platform, {})\n-        if platform_state is not None:\n+        if platform_state is not _UNSET:\n             platform_payload[\"state\"] = platform_state\n-        if error_code is not None:\n+        if error_code is not _UNSET:\n             platform_payload[\"error_code\"] = error_code\n-        if error_message is not None:\n+        if error_message is not _UNSET:\n             platform_payload[\"error_message\"] = error_message\n         platform_payload[\"updated_at\"] = _utc_now_iso()\n         payload[\"platforms\"][platform] = platform_payload"]},{"file":"tests/gateway/test_discord_connect.py","hunks":["@@ -74,6 +74,26 @@ async def close(self):\n         return None\n \n \n+class SlowSyncTree(FakeTree):\n+    def __init__(self):\n+        super().__init__()\n+        self.started = asyncio.Event()\n+        self.allow_finish = asyncio.Event()\n+\n+        async def _slow_sync():\n+            self.started.set()\n+            await self.allow_finish.wait()\n+            return []\n+\n+        self.sync = AsyncMock(side_effect=_slow_sync)\n+\n+\n+class SlowSyncBot(FakeBot):\n+    def __init__(self, *, intents, proxy=None):\n+        super().__init__(intents=intents, proxy=proxy)\n+        self.tree = SlowSyncTree()\n+\n+\n @pytest.mark.asyncio\n @pytest.mark.parametrize(\n     (\"allowed_users\", \"expected_members_intent\"),","@@ -138,3 +158,36 @@ async def fake_wait_for(awaitable, timeout):\n     assert ok is False\n     assert released == [(\"discord-bot-token\", \"test-token\")]\n     assert adapter._platform_lock_identity is None\n+\n+\n[redacted:email]\n+async def test_connect_does_not_wait_for_slash_sync(monkeypatch):\n+    adapter = DiscordAdapter(PlatformConfig(enabled=True, token=\"[redacted:secret-assignment]\"))\n+\n+    monkeypatch.setattr(\"gateway.status.acquire_scoped_lock\", lambda scope, identity, metadata=None: (True, None))\n+    monkeypatch.setattr(\"gateway.status.release_scoped_lock\", lambda scope, identity: None)\n+\n+    intents = SimpleNamespace(message_content=False, dm_messages=False, guild_messages=False, members=False, voice_states=False)\n+    monkeypatch.setattr(discord_platform.Intents, \"default\", lambda: intents)\n+\n+    created = {}\n+\n+    def fake_bot_factory(*, command_prefix, intents, proxy=None):\n+        bot = SlowSyncBot(intents=intents, proxy=proxy)\n+        created[\"bot\"] = bot\n+        return bot\n+\n+    monkeypatch.setattr(discord_platform.commands, \"Bot\", fake_bot_factory)\n+    monkeypatch.setattr(adapter, \"_resolve_allowed_usernames\", AsyncMock())\n+\n+    ok = await asyncio.wait_for(adapter.connect(), timeout=1.0)\n+\n+    assert ok is True\n+    assert adapter._ready_event.is_set()\n+\n+    await asyncio.wait_for(created[\"bot\"].tree.started.wait(), timeout=1.0)\n+    assert created[\"bot\"].tree.sync.await_count == 1\n+\n+    created[\"bot\"].tree.allow_finish.set()\n+    await asyncio.sleep(0)\n+    await adapter.disconnect()"]},{"file":"tests/gateway/test_runner_startup_failures.py","hunks":["@@ -1,4 +1,5 @@\n import pytest\n+from unittest.mock import AsyncMock\n \n from gateway.config import GatewayConfig, Platform, PlatformConfig\n from gateway.platforms.base import BasePlatformAdapter","@@ -45,6 +46,23 @@ async def get_chat_info(self, chat_id):\n         return {\"id\": chat_id}\n \n \n+class _SuccessfulAdapter(BasePlatformAdapter):\n+    def __init__(self):\n+        super().__init__(PlatformConfig(enabled=True, token=\"***\"), Platform.DISCORD)\n+\n+    async def connect(self) -> bool:\n+        return True\n+\n+    async def disconnect(self) -> None:\n+        self._mark_disconnected()\n+\n+    async def send(self, chat_id, content, reply_to=None, metadata=None):\n+        raise NotImplementedError\n+\n+    async def get_chat_info(self, chat_id):\n+        return {\"id\": chat_id}\n+\n+\n @pytest.mark.asyncio\n async def test_runner_returns_failure_for_retryable_startup_errors(monkeypatch, tmp_path):\n     monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))","@@ -65,7 +83,7 @@ async def test_runner_returns_failure_for_retryable_startup_errors(monkeypatch,\n     state = read_runtime_status()\n     assert state[\"gateway_state\"] == \"startup_failed\"\n     assert \"temporary DNS resolution failure\" in state[\"exit_reason\"]\n-    assert state[\"platforms\"][\"telegram\"][\"state\"] == \"fatal\"\n+    assert state[\"platforms\"][\"telegram\"][\"state\"] == \"retrying\"\n     assert state[\"platforms\"][\"telegram\"][\"error_code\"] == \"telegram_connect_error\"\n \n ","@@ -89,6 +107,31 @@ async def test_runner_allows_cron_only_mode_when_no_platforms_are_enabled(monkey\n     assert state[\"gateway_state\"] == \"running\"\n \n \n[redacted:email]\n+async def test_runner_records_connected_platform_state_on_success(monkeypatch, tmp_path):\n+    monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+    config = GatewayConfig(\n+        platforms={\n+            Platform.DISCORD: PlatformConfig(enabled=True, token=\"***\")\n+        },\n+        sessions_dir=tmp_path / \"sessions\",\n+    )\n+    runner = GatewayRunner(config)\n+\n+    monkeypatch.setattr(runner, \"_create_adapter\", lambda platform, platform_config: _SuccessfulAdapter())\n+    monkeypatch.setattr(runner.hooks, \"discover_and_load\", lambda: None)\n+    monkeypatch.setattr(runner.hooks, \"emit\", AsyncMock())\n+\n+    ok = await runner.start()\n+\n+    assert ok is True\n+    state = read_runtime_status()\n+    assert state[\"gateway_state\"] == \"running\"\n+    assert state[\"platforms\"][\"discord\"][\"state\"] == \"connected\"\n+    assert state[\"platforms\"][\"discord\"][\"error_code\"] is None\n+    assert state[\"platforms\"][\"discord\"][\"error_message\"] is None\n+\n+\n @pytest.mark.asyncio\n async def test_start_gateway_replace_force_uses_terminate_pid(monkeypatch, tmp_path):\n     monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))"]},{"file":"tests/gateway/test_status.py","hunks":["@@ -104,6 +104,34 @@ def test_write_runtime_status_records_platform_failure(self, tmp_path, monkeypat\n         assert payload[\"platforms\"][\"telegram\"][\"error_code\"] == \"telegram_polling_conflict\"\n         assert payload[\"platforms\"][\"telegram\"][\"error_message\"] == \"another poller is active\"\n \n+    def test_write_runtime_status_explicit_none_clears_stale_fields(self, tmp_path, monkeypatch):\n+        monkeypatch.setenv(\"HERMES_HOME\", str(tmp_path))\n+\n+        status.write_runtime_status(\n+            gateway_state=\"startup_failed\",\n+            exit_reason=\"stale error\",\n+            platform=\"discord\",\n+            platform_state=\"fatal\",\n+            error_code=\"discord_timeout\",\n+            error_message=\"stale platform error\",\n+        )\n+\n+        status.write_runtime_status(\n+            gateway_state=\"running\",\n+            exit_reason=None,\n+            platform=\"discord\",\n+            platform_state=\"connected\",\n+            error_code=None,\n+            error_message=None,\n+        )\n+\n+        payload = status.read_runtime_status()\n+        assert payload[\"gateway_state\"] == \"running\"\n+        assert payload[\"exit_reason\"] is None\n+        assert payload[\"platforms\"][\"discord\"][\"state\"] == \"connected\"\n+        assert payload[\"platforms\"][\"discord\"][\"error_code\"] is None\n+        assert payload[\"platforms\"][\"discord\"][\"error_message\"] is None\n+\n \n class TestTerminatePid:\n     def test_force_uses_taskkill_on_windows(self, monkeypatch):\n"]}]},{"kind":"pr","number":14377,"title":"fix(slack): surface Socket Mode task failures","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"7f8b0dd1e06fb21a0d5b69dd6611c8df1c588b9d","number":0},{"kind":"pr","value":"#28873","number":28873}],"category":"uncategorized","implication":"needs_llm"},"merged":false}],"trust":{"default_tier":"verified","includes_suggested":false}}