{"repo":"NousResearch/hermes-agent","public_id":"50185","display_label":"Photon/Envoy overflow errors retry and resilience","summary":"These items address handling of upstream overflow errors in the Photon adapter, specifically classifying them as retryable and adding typing cooldowns to improve resilience against transient Envoy/Spectrum overflow conditions. A related issue notes that such overflow errors can crash the entire gateway without a circuit breaker.","subsystem":"gateway","counts":{"members":4,"open_issues":0,"open_issues_distinct":0,"open_prs":0,"closed_issues":1,"merged_prs":1,"attempt_prs":2},"landed_fix":{"status":"merged_to_main","basis":"structural","pr":50339,"merged_at":"2026-06-21T19:15:45+00:00"},"resolution":{"state":"resolved","by":"pr-50339"},"freshness":{"state":"verified","last_verified_at":"2026-06-21T19:15:45+00:00","is_stale":false},"grouping":{"backbone":"STRUCTURAL_SPANNING","link_density":0.5},"fix_edges":[{"issue_number":50185,"pr_number":50339,"tier":"verified","basis":"closing_ref","confidence":0.95,"recoverable_from_timeline":true,"pr_merged_at":"2026-06-21T19:15:45+00:00"}],"members":[{"kind":"issue","number":50185,"title":"Photon adapter: no circuit breaker on sidecar trap-restart loop — a transient \"reset reason: overflow\" crashes the entire gateway","url":null,"state":"closed"},{"kind":"pr","number":50256,"title":"fix(photon): classify Envoy overflow errors as retryable; add typing cooldown","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"2a4542333","number":0},{"kind":"commit","value":"9578e5279","number":0},{"kind":"pr","value":"#50339","number":50339}],"category":"merged_via","implication":"superseded"},"merged":false},{"kind":"pr","number":50301,"title":"fix(photon): add overflow resilience for Spectrum upstream errors (#50185)","url":null,"state":"closed","close_reason":{"refs":[{"kind":"commit","value":"2a4542333ee107bbb5b9e434574347334f239258","number":0},{"kind":"commit","value":"9578e52795e35f8373fb43e9b5457beb8f279f71","number":0},{"kind":"pr","value":"#50185","number":50185},{"kind":"pr","value":"#50339","number":50339}],"category":"implemented_elsewhere","implication":"superseded"},"merged":false},{"kind":"pr","number":50339,"title":"fix(photon): ride through Spectrum/Envoy upstream overflow (retry + typing cooldown + crash detection)","url":null,"state":"merged","close_reason":{"category":"merged"},"merged":true,"is_fix_pr":true,"fix_tier":"verified","files_changed":["plugins/platforms/photon/adapter.py","tests/plugins/platforms/photon/test_overflow_recovery.py"],"fix_hunks":[{"file":"plugins/platforms/photon/adapter.py","hunks":["@@ -85,6 +85,20 @@\n \n _SIDECAR_DIR = Path(__file__).parent / \"sidecar\"\n \n+# Photon / Envoy / spectrum-ts error substrings that indicate a transient\n+# upstream overload rather than a permanent failure.  These are not in the\n+# core _RETRYABLE_ERROR_PATTERNS because they are specific to this adapter.\n+_PHOTON_RETRYABLE_PATTERNS = (\n+    \"internal sidecar error\",\n+    \"upstream connect error\",\n+    \"reset reason: overflow\",\n+)\n+\n+# Minimum seconds between typing-indicator calls for the same chat.\n+# iMessage is a personal channel — suppressing rapid repeats reduces\n+# upstream gRPC pressure during Photon overflow events.\n+_TYPING_COOLDOWN_SECONDS = 5.0\n+\n # Group-chat mention wake words. When ``require_mention`` is enabled, group\n # messages are ignored unless they match one of these patterns — same\n # behavior and defaults as the BlueBubbles iMessage channel so the two","@@ -234,6 +248,8 @@ def __init__(self, config: PlatformConfig):\n         # react action default to \"the message that triggered me\" without\n         # requiring the model to thread message ids through tool calls.\n         self._last_inbound_by_chat: Dict[str, str] = {}\n+        # Last time we sent a typing indicator per chat, for cooldown gating.\n+        self._typing_last_sent: Dict[str, float] = {}\n \n         # Group-chat mention gating (parity with BlueBubbles). When enabled,\n         # group messages are ignored unless they match a wake word; DMs are","@@ -839,6 +855,21 @@ async def _supervise_sidecar(self, proc: subprocess.Popen) -> None:\n                 logger.info(\"[photon-sidecar] %s\", line.decode(\"utf-8\", \"replace\").rstrip())\n         except Exception as e:  # pragma: no cover - defensive\n             logger.warning(\"[photon-sidecar] supervisor exited: %s\", e)\n+        if self._inbound_running:\n+            exit_code = proc.poll()\n+            logger.error(\n+                \"[photon] sidecar exited unexpectedly (code %s) — triggering reconnect\",\n+                exit_code,\n+            )\n+            self._set_fatal_error(\n+                \"SIDECAR_CRASHED\",\n+                f\"Photon sidecar exited unexpectedly (code {exit_code})\",\n+                retryable=True,\n+            )\n+            try:\n+                await self._notify_fatal_error()\n+            except Exception as exc:  # pragma: no cover - defensive\n+                logger.warning(\"[photon] fatal-error notification failed: %s\", exc)\n \n     async def _stop_sidecar(self) -> None:\n         proc = self._sidecar_proc","@@ -988,6 +1019,10 @@ async def send_animation(\n         )\n \n     async def send_typing(self, chat_id: str, metadata=None) -> None:\n+        now = time.time()\n+        if now - self._typing_last_sent.get(chat_id, 0.0) < _TYPING_COOLDOWN_SECONDS:\n+            return\n+        self._typing_last_sent[chat_id] = now\n         try:\n             await self._sidecar_call(\n                 \"/typing\", {\"spaceId\": chat_id, \"state\": \"start\"}","@@ -996,6 +1031,7 @@ async def send_typing(self, chat_id: str, metadata=None) -> None:\n             logger.debug(\"[photon] send_typing failed: %s\", e)\n \n     async def stop_typing(self, chat_id: str) -> None:\n+        self._typing_last_sent.pop(chat_id, None)\n         try:\n             await self._sidecar_call(\n                 \"/typing\", {\"spaceId\": chat_id, \"state\": \"stop\"}","@@ -1189,13 +1225,22 @@ def format_message(self, content: str) -> str:\n             return content\n         return strip_markdown(content)\n \n+    @staticmethod\n+    def _is_retryable_error(error: Optional[str]) -> bool:\n+        if BasePlatformAdapter._is_retryable_error(error):\n+            return True\n+        if not error:\n+            return False\n+        lowered = error.lower()\n+        return any(pat in lowered for pat in _PHOTON_RETRYABLE_PATTERNS)\n+\n     async def _send_with_retry(\n         self,\n         chat_id: str,\n         content: str,\n         reply_to: Optional[str] = None,\n         metadata: Any = None,\n-        max_retries: int = 2,\n+        max_retries: int = 1,\n         base_delay: float = 2.0,\n     ) -> SendResult:\n         \"\"\"Retry sends without the generic Markdown banner."]},{"file":"tests/plugins/platforms/photon/test_overflow_recovery.py","hunks":["@@ -0,0 +1,197 @@\n+\"\"\"Photon adapter resilience to transient Spectrum/Envoy upstream overflow.\n+\n+Covers the three behaviors that let the adapter ride through a Photon\n+\"reset reason: overflow\" event instead of degrading delivery and silently\n+dying (issue #50185):\n+\n+  1. ``_is_retryable_error`` classifies the Envoy/sidecar overflow strings as\n+     retryable so ``_send_with_retry`` actually engages its backoff loop.\n+  2. ``send_typing`` is rate-gated per chat, and ``stop_typing`` resets the\n+     gate so the next turn's typing indicator fires immediately.\n+  3. ``_supervise_sidecar`` detects an unexpected sidecar exit and raises a\n+     ``retryable=True`` fatal so the gateway reconnect watcher revives the\n+     platform — instead of returning silently and leaving ``_inbound_loop``\n+     spinning against a dead port.\n+\n+No Node sidecar is spawned and no ports are bound.\n+\"\"\"\n+from __future__ import annotations\n+\n+from typing import Any, Dict\n+\n+import pytest\n+\n+from gateway.config import PlatformConfig\n+from plugins.platforms.photon.adapter import PhotonAdapter\n+\n+\n+def _make_adapter(monkeypatch: pytest.MonkeyPatch) -> PhotonAdapter:\n+    monkeypatch.setenv(\"PHOTON_PROJECT_ID\", \"test-project-id\")\n+    monkeypatch.setenv(\"PHOTON_PROJECT_SECRET\", \"test-project-secret\")\n+    cfg = PlatformConfig(enabled=True, token=\"\", extra={})\n+    return PhotonAdapter(cfg)\n+\n+\n+# -- Gap 1: retryable classification of overflow errors ---------------------\n+\n[redacted:email](\n+    \"error\",\n+    [\n+        \"UNAVAILABLE: internal sidecar error\",\n+        \"upstream connect error or disconnect/reset before headers\",\n+        \"reset reason: overflow\",\n+        # Case-insensitive: real strings arrive with mixed case.\n+        \"Internal Sidecar Error\",\n+    ],\n+)\n+def test_overflow_strings_classified_retryable(error: str) -> None:\n+    assert PhotonAdapter._is_retryable_error(error) is True\n+\n+\n+def test_unrelated_error_not_retryable() -> None:\n+    # A genuine permanent failure must NOT be retried.\n+    assert PhotonAdapter._is_retryable_error(\"400 bad request: invalid spaceId\") is False\n+    assert PhotonAdapter._is_retryable_error(None) is False\n+\n+\n+def test_base_network_patterns_still_match() -> None:\n+    # The override delegates to the base classifier first, so generic\n+    # network strings keep working.\n+    assert PhotonAdapter._is_retryable_error(\"ConnectError: connection refused\") is True\n+\n+\n+# -- Gap 2: typing-indicator cooldown ---------------------------------------\n+\n[redacted:email]\n+async def test_typing_cooldown_suppresses_rapid_repeats(\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    adapter = _make_adapter(monkeypatch)\n+    calls: list[Dict[str, Any]] = []\n+\n+    async def _fake_call(path: str, payload: Dict[str, Any]) -> Any:\n+        calls.append(payload)\n+        return {\"ok\": True}\n+\n+    monkeypatch.setattr(adapter, \"_sidecar_call\", _fake_call)\n+\n+    # First call fires; immediate repeats are suppressed by the cooldown.\n+    await adapter.send_typing(\"chat-1\")\n+    await adapter.send_typing(\"chat-1\")\n+    await adapter.send_typing(\"chat-1\")\n+\n+    assert len(calls) == 1\n+\n+\n[redacted:email]\n+async def test_typing_cooldown_is_per_chat(\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    adapter = _make_adapter(monkeypatch)\n+    calls: list[str] = []\n+\n+    async def _fake_call(path: str, payload: Dict[str, Any]) -> Any:\n+        calls.append(payload[\"spaceId\"])\n+        return {\"ok\": True}\n+\n+    monkeypatch.setattr(adapter, \"_sidecar_call\", _fake_call)\n+\n+    # Different chats have independent cooldowns.\n+    await adapter.send_typing(\"chat-1\")\n+    await adapter.send_typing(\"chat-2\")\n+\n+    assert calls == [\"chat-1\", \"chat-2\"]\n+\n+\n[redacted:email]\n+async def test_stop_typing_resets_cooldown(\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    adapter = _make_adapter(monkeypatch)\n+    starts = 0\n+\n+    async def _fake_call(path: str, payload: Dict[str, Any]) -> Any:\n+        nonlocal starts\n+        if payload.get(\"state\") == \"start\":\n+            starts += 1\n+        return {\"ok\": True}\n+\n+    monkeypatch.setattr(adapter, \"_sidecar_call\", _fake_call)\n+\n+    # A start, then a stop (end of turn), then a start for the next turn must\n+    # fire immediately — the cooldown only suppresses rapid consecutive starts\n+    # without an intervening stop.\n+    await adapter.send_typing(\"chat-1\")\n+    await adapter.stop_typing(\"chat-1\")\n+    await adapter.send_typing(\"chat-1\")\n+\n+    assert starts == 2\n+\n+\n+# -- Gap 3: sidecar crash detection -----------------------------------------\n+\n+class _EofStdout:\n+    \"\"\"A proc.stdout whose readline() reports immediate EOF (dead sidecar).\"\"\"\n+\n+    def readline(self) -> bytes:\n+        return b\"\"\n+\n+\n+class _DeadProc:\n+    \"\"\"Minimal subprocess.Popen stand-in for a sidecar that has exited.\"\"\"\n+\n+    def __init__(self, exit_code: int = 1) -> None:\n+        self.stdout = _EofStdout()\n+        self.stdin = None\n+        self._exit_code = exit_code\n+\n+    def poll(self) -> int:\n+        return self._exit_code\n+\n+\n[redacted:email]\n+async def test_unexpected_sidecar_exit_raises_retryable_fatal(\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    adapter = _make_adapter(monkeypatch)\n+    # Simulate a live session whose sidecar then dies underneath it.\n+    adapter._inbound_running = True\n+\n+    notified: list[bool] = []\n+\n+    async def _fake_notify() -> None:\n+        notified.append(True)\n+\n+    monkeypatch.setattr(adapter, \"_notify_fatal_error\", _fake_notify)\n+\n+    await adapter._supervise_sidecar(_DeadProc(exit_code=137))  # type: ignore[arg-type]\n+\n+    assert adapter.has_fatal_error is True\n+    assert adapter.fatal_error_code == \"SIDECAR_CRASHED\"\n+    # retryable=True routes the platform into the reconnect watcher rather\n+    # than crashing the whole gateway.\n+    assert adapter.fatal_error_retryable is True\n+    assert adapter._running is False\n+    assert notified == [True]\n+\n+\n[redacted:email]\n+async def test_clean_shutdown_does_not_raise_fatal(\n+    monkeypatch: pytest.MonkeyPatch,\n+) -> None:\n+    adapter = _make_adapter(monkeypatch)\n+    # disconnect() sets _inbound_running = False before stopping the sidecar,\n+    # so the detection block must NOT fire on a clean shutdown.\n+    adapter._inbound_running = False\n+\n+    notified: list[bool] = []\n+\n+    async def _fake_notify() -> None:\n+        notified.append(True)\n+\n+    monkeypatch.setattr(adapter, \"_notify_fatal_error\", _fake_notify)\n+\n+    await adapter._supervise_sidecar(_DeadProc(exit_code=0))  # type: ignore[arg-type]\n+\n+    assert adapter.has_fatal_error is False\n+    assert notified == []\n"]}]}],"trust":{"default_tier":"verified","includes_suggested":false}}