{"repo":"NousResearch/hermes-agent","public_id":"26828","display_label":"WeChat rate-limit circuit breaker fix","summary":"A reported issue and corresponding fix attempts address a retry storm triggered by WeChat rate-limiting that causes gateway out-of-memory crashes, with the proposed solution being a circuit breaker mechanism.","subsystem":"gateway","counts":{"members":3,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":1},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":41718,"merged_at":"2026-06-08T05:10:17+00:00"},"resolution":{"state":"resolved","by":"pr-41718"},"freshness":{"state":"verified","last_verified_at":"2026-06-08T05:10:17+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.6666666666666666},"fix_edges":[{"issue_number":26828,"pr_number":41718,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-08T05:10:17+00:00"}],"members":[{"kind":"issue","number":26828,"title":"WeChat (iLink) rate-limit retry storm causes gateway OOM and SIGKILL — no circuit breaker","url":null,"state":"closed"},{"kind":"pr","number":26904,"title":"fix(weixin): add rate-limit circuit breaker","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"b8469a81e3e3f0793615d9e4f71589652ae9bc9e","number":0},{"kind":"pr","value":"#41718","number":41718}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":41718,"title":"fix(weixin): add rate-limit circuit breaker","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["gateway/platforms/weixin.py","tests/gateway/test_weixin.py","tools/send_message_tool.py"],"fix_hunks":[{"file":"gateway/platforms/weixin.py","hunks":["@@ -1174,6 +1174,24 @@ def __init__(self, config: PlatformConfig):\n             extra.get(\"send_chunk_retry_delay_seconds\")\n             or os.getenv(\"WEIXIN_SEND_CHUNK_RETRY_DELAY_SECONDS\", \"1.0\")\n         )\n+        self._send_text_gate = asyncio.Lock()\n+        self._rate_limit_circuit_threshold = max(\n+            1,\n+            int(\n+                extra.get(\"rate_limit_circuit_threshold\")\n+                or os.getenv(\"WEIXIN_RATE_LIMIT_CIRCUIT_THRESHOLD\", \"1\")\n+            ),\n+        )\n+        self._rate_limit_circuit_window_seconds = float(\n+            extra.get(\"rate_limit_circuit_window_seconds\")\n+            or os.getenv(\"WEIXIN_RATE_LIMIT_CIRCUIT_WINDOW_SECONDS\", \"30.0\")\n+        )\n+        self._rate_limit_circuit_open_seconds = float(\n+            extra.get(\"rate_limit_circuit_open_seconds\")\n+            or os.getenv(\"WEIXIN_RATE_LIMIT_CIRCUIT_OPEN_SECONDS\", \"30.0\")\n+        )\n+        self._rate_limit_circuit_until = 0.0\n+        self._rate_limit_events: List[float] = []\n         self._dm_policy = str(extra.get(\"dm_policy\") or os.getenv(\"WEIXIN_DM_POLICY\", \"open\")).strip().lower()\n         self._group_policy = str(extra.get(\"group_policy\") or os.getenv(\"WEIXIN_GROUP_POLICY\", \"disabled\")).strip().lower()\n         allow_from = extra.get(\"allow_from\")","@@ -1647,6 +1665,37 @@ def _split_text(self, content: str) -> List[str]:\n             content, self.MAX_MESSAGE_LENGTH, self._split_multiline_messages,\n         )\n \n+    def _rate_limit_cooldown_remaining(self) -> float:\n+        return max(0.0, self._rate_limit_circuit_until - time.monotonic())\n+\n+    def _rate_limit_error(self) -> RuntimeError:\n+        return RuntimeError(\n+            f\"iLink sendmessage rate limited; cooldown active for {self._rate_limit_cooldown_remaining():.1f}s\"\n+        )\n+\n+    def _open_rate_limit_circuit(self) -> None:\n+        if self._rate_limit_circuit_open_seconds <= 0:\n+            return\n+        self._rate_limit_circuit_until = max(\n+            self._rate_limit_circuit_until,\n+            time.monotonic() + self._rate_limit_circuit_open_seconds,\n+        )\n+\n+    def _record_rate_limit_event(self) -> bool:\n+        \"\"\"Record a genuine iLink rate limit and return True if breaker opened.\"\"\"\n+        now = time.monotonic()\n+        window_start = now - self._rate_limit_circuit_window_seconds\n+        self._rate_limit_events = [ts for ts in self._rate_limit_events if ts >= window_start]\n+        self._rate_limit_events.append(now)\n+        if len(self._rate_limit_events) >= self._rate_limit_circuit_threshold:\n+            self._open_rate_limit_circuit()\n+            return self._rate_limit_cooldown_remaining() > 0\n+        return False\n+\n+    def _reset_rate_limit_circuit(self) -> None:\n+        self._rate_limit_events.clear()\n+        self._rate_limit_circuit_until = 0.0\n+\n     async def _send_text_chunk(\n         self,\n         *,","@@ -1662,9 +1711,28 @@ async def _send_text_chunk(\n         degraded fallback, which keeps cron-initiated push messages working\n         even when no user message has refreshed the session recently.\n         \"\"\"\n+        async with self._send_text_gate:\n+            await self._send_text_chunk_locked(\n+                chat_id=chat_id,\n+                chunk=chunk,\n+                context_token=[redacted:secret-assignment]\n+                client_id=client_id,\n+            )\n+\n+    async def _send_text_chunk_locked(\n+        self,\n+        *,\n+        chat_id: str,\n+        chunk: str,\n+        context_token: [redacted:secret-assignment]\n+        client_id: str,\n+    ) -> None:\n+        \"\"\"Send a text chunk while holding the adapter-wide outbound text gate.\"\"\"\n         last_error: Optional[Exception] = None\n         retried_without_token = False\n         for attempt in range(self._send_chunk_retries + 1):\n+            if self._rate_limit_cooldown_remaining() > 0:\n+                raise self._rate_limit_error()\n             try:\n                 resp = await _send_message(\n                     self._send_session,","@@ -1710,6 +1778,9 @@ async def _send_text_chunk(\n                             last_error = RuntimeError(\n                                 f\"iLink sendmessage rate limited: ret={ret} errcode={errcode} errmsg={errmsg}\"\n                             )\n+                            if self._record_rate_limit_event():\n+                                last_error = self._rate_limit_error()\n+                                break\n                             if attempt >= self._send_chunk_retries:\n                                 break\n                             wait = self._send_chunk_retry_delay_seconds * 3  # 3x backoff for rate limit","@@ -1723,6 +1794,7 @@ async def _send_text_chunk(\n                         raise RuntimeError(\n                             f\"iLink sendmessage error: ret={ret} errcode={errcode} errmsg={errmsg}\"\n                         )\n+                self._reset_rate_limit_circuit()\n                 return\n             except Exception as exc:\n                 last_error = exc"]},{"file":"tests/gateway/test_weixin.py","hunks":["@@ -411,6 +411,98 @@ async def flaky_send(*args, **kwargs):\n         assert first_try[\"text\"] == retry[\"text\"]\n         assert first_try[\"client_id\"] == retry[\"client_id\"]\n \n+    @patch(\"gateway.platforms.weixin.asyncio.sleep\", new_callable=AsyncMock)\n+    @patch(\"gateway.platforms.weixin._send_message\", new_callable=AsyncMock)\n+    def test_repeated_rate_limits_open_circuit_for_followup_sends(self, send_message_mock, sleep_mock):\n+        adapter = self._connected_adapter()\n+        adapter._send_chunk_retries = 3\n+        adapter._send_chunk_retry_delay_seconds = 0\n+        adapter._rate_limit_circuit_threshold = 2\n+        adapter._rate_limit_circuit_window_seconds = 60\n+        adapter._rate_limit_circuit_open_seconds = 60\n+\n+        send_message_mock.return_value = {\n+            \"ret\": weixin.RATE_LIMIT_ERRCODE,\n+            \"errcode\": weixin.RATE_LIMIT_ERRCODE,\n+            \"errmsg\": \"frequency limit\",\n+        }\n+\n+        first = asyncio.run(adapter.send(\"wxid_test123\", \"first\"))\n+        second = asyncio.run(adapter.send(\"wxid_test123\", \"second\"))\n+\n+        assert first.success is False\n+        assert \"cooldown\" in (first.error or \"\")\n+        assert second.success is False\n+        assert \"cooldown\" in (second.error or \"\")\n+        # The first rate-limit response is retried once. The second response\n+        # crosses the sliding-window threshold, opens the breaker, and both the\n+        # rest of the current chunk and follow-up sends fail fast.\n+        assert send_message_mock.await_count == 2\n+        assert sleep_mock.await_count == 1\n+\n+    @patch(\"gateway.platforms.weixin._send_message\", new_callable=AsyncMock)\n+    def test_open_rate_limit_circuit_fails_fast_without_sendmessage(self, send_message_mock):\n+        adapter = self._connected_adapter()\n+        adapter._rate_limit_circuit_open_seconds = 60\n+        adapter._open_rate_limit_circuit()\n+\n+        result = asyncio.run(adapter.send(\"wxid_test123\", \"blocked\"))\n+\n+        assert result.success is False\n+        assert \"cooldown\" in (result.error or \"\")\n+        send_message_mock.assert_not_awaited()\n+\n+    @patch(\"gateway.platforms.weixin._send_message\", new_callable=AsyncMock)\n+    def test_successful_send_after_cooldown_resets_rate_limit_state(self, send_message_mock):\n+        adapter = self._connected_adapter()\n+        adapter._rate_limit_circuit_until = weixin.time.monotonic() - 1\n+        adapter._rate_limit_events = [weixin.time.monotonic()]\n+        send_message_mock.return_value = {\"errcode\": 0}\n+\n+        result = asyncio.run(adapter.send(\"wxid_test123\", \"after cooldown\"))\n+\n+        assert result.success is True\n+        assert adapter._rate_limit_events == []\n+        assert adapter._rate_limit_circuit_until == 0.0\n+        send_message_mock.assert_awaited_once()\n+\n+    def test_concurrent_rate_limited_sends_are_serialized_by_gate(self):\n+        adapter = self._connected_adapter()\n+        adapter._send_chunk_retries = 3\n+        adapter._send_chunk_retry_delay_seconds = 0\n+        adapter._rate_limit_circuit_threshold = 1\n+        adapter._rate_limit_circuit_open_seconds = 60\n+        active = 0\n+        peak_active = 0\n+\n+        async def rate_limited_send(*args, **kwargs):\n+            nonlocal active, peak_active\n+            active += 1\n+            peak_active = max(peak_active, active)\n+            await asyncio.sleep(0)\n+            active -= 1\n+            return {\n+                \"ret\": weixin.RATE_LIMIT_ERRCODE,\n+                \"errcode\": weixin.RATE_LIMIT_ERRCODE,\n+                \"errmsg\": \"frequency limit\",\n+            }\n+\n+        async def run_burst():\n+            with patch(\"gateway.platforms.weixin._send_message\", side_effect=rate_limited_send) as send_message_mock:\n+                results = await asyncio.gather(\n+                    *(adapter.send(\"wxid_test123\", f\"message {idx}\") for idx in range(20))\n+                )\n+                return results, send_message_mock\n+\n+        results, send_message_mock = asyncio.run(run_burst())\n+\n+        assert all(not result.success for result in results)\n+        assert peak_active == 1\n+        # Once the first send observes iLink's rate limit, the breaker opens;\n+        # queued concurrent sends acquire the gate later and fail before making\n+        # their own iLink calls.\n+        assert send_message_mock.await_count == 1\n+\n \n class TestWeixinOutboundMedia:\n     def test_send_image_file_accepts_keyword_image_path(self):"]},{"file":"tools/send_message_tool.py","hunks":["@@ -588,6 +588,16 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,\n     (preserves code-block boundaries, adds part indicators).\n     \"\"\"\n     from gateway.config import Platform\n+\n+    media_files = media_files or []\n+\n+    # Weixin handles text/media delivery inside its native helper and does not\n+    # need the optional platform adapter imports below. Keep this branch early\n+    # so a Weixin send is not blocked by unrelated optional dependencies (for\n+    # example lark-oapi's heavy Feishu import path).\n+    if platform == Platform.WEIXIN:\n+        return await _send_weixin(pconfig, chat_id, message, media_files=media_files)\n+\n     from gateway.platforms.base import BasePlatformAdapter, utf16_len\n     from gateway.platforms.slack import SlackAdapter\n ","@@ -605,8 +615,6 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,\n     except ImportError:\n         _feishu_available = False\n \n-    media_files = media_files or []\n-\n     if platform == Platform.SLACK and message:\n         try:\n             slack_adapter = SlackAdapter.__new__(SlackAdapter)","@@ -663,10 +671,6 @@ async def _send_to_platform(platform, pconfig, chat_id, message, thread_id=None,\n             last_result = result\n         return last_result\n \n-    # --- Weixin: use the native one-shot adapter helper for text + media ---\n-    if platform == Platform.WEIXIN:\n-        return await _send_weixin(pconfig, chat_id, message, media_files=media_files)\n-\n     # --- Discord: chunked delivery via the registry's standalone_sender_fn.\n     # The plugin's ``_standalone_send`` (registered in\n     # plugins/platforms/discord/adapter.py) handles forum channels, threads,\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}